libeuropa/io: Get JSF STREAMS extraction working

As it turns out the thing I marked as "header size" is more likely just a discarded revision.

Anyways, to detail this: pak version 5 adds fields to the header for sector alignment. The reader now handles this and reads the sectoralignment TOC entry, which now allows STREAMS (and any other packages with this set?) to extract properly.

Woohoo.

hexpat/pak.hexpat

@@ -17,13 +17,16 @@ namespace europa {
 		char magic[16]; // "Europa Packfile\0"
 		
 		// Doesn't include magic
-		u16 headerSize;
+		u8 revision;
+
+		// This seems to be the start of the actual header
+		u8 pad;
 
 		// 0x3 - PMDL
 		// 0x4 - Starfighter
 		// 0x5 - Jedi Starfighter
 		u16 Version;
-		u8 pad;
+		u8 pad2;
 
 		u32 tocOffset;
 
@@ -36,6 +39,13 @@ namespace europa {
 
 		// Set to 0 in basically every file
 		u32 reserved;
+
+		// Version 5 has additional fields to support sector-alignment
+		if(Version == 5) {
+			u32 sectorSize;
+			u8 sectorAligned;
+			u8 pad3;
+		}
 	};
 
 	// "Pascal" string used
@@ -54,6 +64,13 @@ namespace europa {
 		u32 offset;
 		u32 size;
 
+		if(parent.header.Version == 5) {
+			if(parent.header.sectorAligned == 0x1) {
+				// Start LBA of the file
+				u32 startLBA;
+			}
+		}
+
 		// Seems to be the same as he header.
 		u32 creationTime;
 	};

include/europa/io/PakFile.hpp

@@ -13,6 +13,7 @@
 #include <europa/structs/Pak.hpp>
 #include <europa/util/Overloaded.hpp>
 #include <filesystem>
+#include <optional>
 #include <stdexcept>
 #include <variant>
 #include <vector>

include/europa/structs/Pak.hpp

@@ -9,14 +9,11 @@
 #ifndef EUROPA_STRUCTS_PAK_H
 #define EUROPA_STRUCTS_PAK_H
 
-#include <cstdint>
+#include <cstdio>
 #include <cstring>
 #include <europa/structs/ImHexAdapter.hpp>
-#include <optional>
 #include <variant>
 
-#include <cstdio>
-
 namespace europa::structs {
 
 	constexpr static const char VALID_MAGIC[16] = "Europa Packfile";
@@ -34,7 +31,8 @@ namespace europa::structs {
 		/**
 		 * Header size. Doesn't include the magic.
 		 */
-		u16 headerSize;
+		u8 revision;
+		u8 padding;
 
 		PakVersion version;
 
@@ -47,13 +45,6 @@ namespace europa::structs {
 	struct [[gnu::packed]] PakHeader_Impl : PakHeader_Common {
 		constexpr static auto VERSION = Version;
 
-		/**
-		 * Get the real header size (including the magic).
-		 */
-		[[nodiscard]] constexpr std::size_t RealHeaderSize() const {
-			return sizeof(magic) + static_cast<std::size_t>(headerSize);
-		}
-
 		constexpr static u16 HeaderSize() {
 			return sizeof(Impl) - (sizeof(VALID_MAGIC) - 1);
 		}
@@ -64,15 +55,15 @@ namespace europa::structs {
 
 			version = Version;
 
-			// Copy important things & set proper header size.
+			// Copy important things & set proper revision. I guess
 			std::memcpy(&magic[0], &VALID_MAGIC[0], sizeof(VALID_MAGIC));
-			headerSize = HeaderSize();
+			revision = 0x1a;
 		}
 
 		explicit PakHeader_Impl(const PakHeader_Common& header) {
 			memcpy(&magic[0], &header.magic[0], sizeof(header.magic));
 			version = header.version;
-			headerSize = header.headerSize;
+			revision = header.revision;
 		}
 
 		[[nodiscard]] bool Valid() const noexcept {
@@ -80,10 +71,6 @@ namespace europa::structs {
 			if(!reinterpret_cast<const PakHeader_Common*>(this)->Valid())
 				return false;
 
-			// Check header size.
-			if(headerSize != HeaderSize() && headerSize != HeaderSize() + 1)
-				return false;
-
 			return version == Version;
 		}
 	};
@@ -109,7 +96,7 @@ namespace europa::structs {
 		u32 creationUnixTime;
 
 		// Zeroes.
-		u32 reservedPad{};
+		u32 reservedPad {};
 	};
 
 	struct [[gnu::packed]] PakHeader_V4 : public PakHeader_Impl<PakHeader_V4, PakVersion::Ver4> {
@@ -144,6 +131,14 @@ namespace europa::structs {
 			u32 creationUnixTime;
 		};
 
+		struct [[gnu::packed]] TocEntry_SectorAligned {
+			u32 offset;
+			u32 size;
+			// Start in LBA (offset / kCDSectorSize)
+			u32 startLBA;
+			u32 creationUnixTime;
+		};
+
 		u8 pad;
 
 		u32 tocOffset;
@@ -156,6 +151,12 @@ namespace europa::structs {
 
 		// Zeroes.
 		u32 reservedPad;
+
+		//
+		u32 sectorAlignment;
+
+		u8 sectorAlignedFlag;
+		u8 pad2;
 	};
 
 	using PakHeaderVariant = std::variant<
@@ -166,20 +167,20 @@ namespace europa::structs {
 	using PakTocEntryVariant = std::variant<
 	structs::PakHeader_V3::TocEntry,
 	structs::PakHeader_V4::TocEntry,
-	structs::PakHeader_V5::TocEntry>;
+	// PAK V5 has two kinds of toc entries:
+	// - normal
+	// - sector aligned
+	structs::PakHeader_V5::TocEntry,
+	structs::PakHeader_V5::TocEntry_SectorAligned>;
 
 	static_assert(sizeof(PakHeader_V3) == 0x28, "PakHeader_V3 wrong size");
-	// TODO: their format really seems to be wrong, 0x19 is proper, but some v3 archives have 0x1a header size
-	// ??? very weird
-	//static_assert(sizeof(PakHeader_V3) - (sizeof(VALID_MAGIC) - 1) == 0x1a, "PakHeader_V3::headerSize will be invalid when writing archives.");
 	static_assert(sizeof(PakHeader_V4) == 0x29, "PakHeader_V4 wrong size!!");
-	static_assert(sizeof(PakHeader_V4) - (sizeof(VALID_MAGIC) - 1) == 0x1a, "PakHeader_V4::headerSize will be invalid when writing archives.");
-	static_assert(sizeof(PakHeader_V5) == 0x29, "PakHeader_V5 wrong size!!");
-	static_assert(sizeof(PakHeader_V5) - (sizeof(VALID_MAGIC) - 1) == 0x1a, "PakHeader_V5::headerSize will be invalid when writing archives.");
+	static_assert(sizeof(PakHeader_V5) == 0x2f, "PakHeader_V5 wrong size!!");
 
 	static_assert(sizeof(PakHeader_V3::TocEntry) == 0xe, "V3 TocEntry wrong size!");
 	static_assert(sizeof(PakHeader_V4::TocEntry) == 0xc, "V4 PakTocEntry wrong size!");
 	static_assert(sizeof(PakHeader_V5::TocEntry) == 0xc, "V5 PakTocEntry wrong size!");
+	static_assert(sizeof(PakHeader_V5::TocEntry_SectorAligned) == 0x10, "V5 TocEntry_SectorAligned wrong size!");
 
 } // namespace europa::structs
 

src/libeuropa/io/PakReader.cpp

@@ -30,11 +30,6 @@ namespace europa::io {
 			return;
 		}
 		
-		//		bool isStreams { false };
-
-		//		if(header_type.tocOffset > 0x17000000)
-		//			isStreams = true;
-
 		// Read the archive TOC
 		stream.seekg(header_type.tocOffset, std::istream::beg);
 		for(std::uint32_t i = 0; i < header_type.fileCount; ++i) {
@@ -43,13 +38,18 @@ namespace europa::io {
 			//
 			// Read this in first.
 			auto filename = impl::ReadPString(stream);
-			files[filename].InitWithExistingTocEntry(impl::ReadStreamType<typename T::TocEntry>(stream));
-
-			// Don't think this is needed
-			// if(isStreams)
-			//	files[filename].Visit([&](auto& tocEntry) {
-			//		tocEntry.creationUnixTime = impl::ReadStreamType<structs::u32>(stream);
-			//	});
+			if constexpr(std::is_same_v<T, structs::PakHeader_V5>) {
+				// Version 5 supports sector aligned packages which have an additional field in them
+				// so we need to handle it here
+				// (not feeling quite as hot about all the crazy template magic here anymore)
+				if(header_type.sectorAlignedFlag) {
+					files[filename].InitWithExistingTocEntry(impl::ReadStreamType<typename T::TocEntry_SectorAligned>(stream));
+				} else {
+					files[filename].InitWithExistingTocEntry(impl::ReadStreamType<typename T::TocEntry>(stream));
+				}
+			} else {
+				files[filename].InitWithExistingTocEntry(impl::ReadStreamType<typename T::TocEntry>(stream));
+			}
 		}
 
 		header = header_type;

src/tools/eupak/tasks/InfoTask.cpp

@@ -12,6 +12,7 @@
 #include <iostream>
 #include <tasks/InfoTask.hpp>
 #include <Utils.hpp>
+#include "europa/structs/Pak.hpp"
 
 namespace eupak::tasks {
 
@@ -60,6 +61,10 @@ namespace eupak::tasks {
 				file.VisitTocEntry([&](auto& tocEntry) {
 					std::cout << "    Created: " << FormatUnixTimestamp(tocEntry.creationUnixTime, DATE_FORMAT) << '\n';
 					std::cout << "    Size: " << FormatUnit(tocEntry.size) << '\n';
+
+					if constexpr(std::is_same_v<std::decay_t<decltype(tocEntry)>, estructs::PakHeader_V5::TocEntry_SectorAligned>) {
+						std::cout << "    Start LBA (CD-ROM Sector): " << tocEntry.startLBA << '\n';
+					}
 				});
 			}
 		}