Merge pull request #12208 from liamwhite/romfs

romfs: optimize parsing and building
This commit is contained in:
liamwhite 2023-12-08 12:25:58 -05:00 committed by GitHub
commit 13131e602f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 136 additions and 112 deletions

View file

@ -2,6 +2,7 @@
// SPDX-License-Identifier: GPL-2.0-or-later // SPDX-License-Identifier: GPL-2.0-or-later
#include <cstring> #include <cstring>
#include <span>
#include <string_view> #include <string_view>
#include "common/alignment.h" #include "common/alignment.h"
#include "common/assert.h" #include "common/assert.h"
@ -134,7 +135,7 @@ void RomFSBuildContext::VisitDirectory(VirtualDir romfs_dir, VirtualDir ext_dir,
child->size = child->source->GetSize(); child->size = child->source->GetSize();
AddFile(parent, child); AddFile(parent, std::move(child));
} }
for (auto& child_romfs_dir : romfs_dir->GetSubdirectories()) { for (auto& child_romfs_dir : romfs_dir->GetSubdirectories()) {
@ -163,36 +164,24 @@ void RomFSBuildContext::VisitDirectory(VirtualDir romfs_dir, VirtualDir ext_dir,
bool RomFSBuildContext::AddDirectory(std::shared_ptr<RomFSBuildDirectoryContext> parent_dir_ctx, bool RomFSBuildContext::AddDirectory(std::shared_ptr<RomFSBuildDirectoryContext> parent_dir_ctx,
std::shared_ptr<RomFSBuildDirectoryContext> dir_ctx) { std::shared_ptr<RomFSBuildDirectoryContext> dir_ctx) {
// Check whether it's already in the known directories.
const auto [it, is_new] = directories.emplace(dir_ctx->path, nullptr);
if (!is_new) {
return false;
}
// Add a new directory. // Add a new directory.
num_dirs++; num_dirs++;
dir_table_size += dir_table_size +=
sizeof(RomFSDirectoryEntry) + Common::AlignUp(dir_ctx->path_len - dir_ctx->cur_path_ofs, 4); sizeof(RomFSDirectoryEntry) + Common::AlignUp(dir_ctx->path_len - dir_ctx->cur_path_ofs, 4);
dir_ctx->parent = parent_dir_ctx; dir_ctx->parent = std::move(parent_dir_ctx);
it->second = dir_ctx; directories.emplace_back(std::move(dir_ctx));
return true; return true;
} }
bool RomFSBuildContext::AddFile(std::shared_ptr<RomFSBuildDirectoryContext> parent_dir_ctx, bool RomFSBuildContext::AddFile(std::shared_ptr<RomFSBuildDirectoryContext> parent_dir_ctx,
std::shared_ptr<RomFSBuildFileContext> file_ctx) { std::shared_ptr<RomFSBuildFileContext> file_ctx) {
// Check whether it's already in the known files.
const auto [it, is_new] = files.emplace(file_ctx->path, nullptr);
if (!is_new) {
return false;
}
// Add a new file. // Add a new file.
num_files++; num_files++;
file_table_size += file_table_size +=
sizeof(RomFSFileEntry) + Common::AlignUp(file_ctx->path_len - file_ctx->cur_path_ofs, 4); sizeof(RomFSFileEntry) + Common::AlignUp(file_ctx->path_len - file_ctx->cur_path_ofs, 4);
file_ctx->parent = parent_dir_ctx; file_ctx->parent = std::move(parent_dir_ctx);
it->second = file_ctx; files.emplace_back(std::move(file_ctx));
return true; return true;
} }
@ -201,7 +190,7 @@ RomFSBuildContext::RomFSBuildContext(VirtualDir base_, VirtualDir ext_)
: base(std::move(base_)), ext(std::move(ext_)) { : base(std::move(base_)), ext(std::move(ext_)) {
root = std::make_shared<RomFSBuildDirectoryContext>(); root = std::make_shared<RomFSBuildDirectoryContext>();
root->path = "\0"; root->path = "\0";
directories.emplace(root->path, root); directories.emplace_back(root);
num_dirs = 1; num_dirs = 1;
dir_table_size = 0x18; dir_table_size = 0x18;
@ -210,28 +199,43 @@ RomFSBuildContext::RomFSBuildContext(VirtualDir base_, VirtualDir ext_)
RomFSBuildContext::~RomFSBuildContext() = default; RomFSBuildContext::~RomFSBuildContext() = default;
std::multimap<u64, VirtualFile> RomFSBuildContext::Build() { std::vector<std::pair<u64, VirtualFile>> RomFSBuildContext::Build() {
const u64 dir_hash_table_entry_count = romfs_get_hash_table_count(num_dirs); const u64 dir_hash_table_entry_count = romfs_get_hash_table_count(num_dirs);
const u64 file_hash_table_entry_count = romfs_get_hash_table_count(num_files); const u64 file_hash_table_entry_count = romfs_get_hash_table_count(num_files);
dir_hash_table_size = 4 * dir_hash_table_entry_count; dir_hash_table_size = 4 * dir_hash_table_entry_count;
file_hash_table_size = 4 * file_hash_table_entry_count; file_hash_table_size = 4 * file_hash_table_entry_count;
// Assign metadata pointers // Assign metadata pointers.
RomFSHeader header{}; RomFSHeader header{};
std::vector<u32> dir_hash_table(dir_hash_table_entry_count, ROMFS_ENTRY_EMPTY); std::vector<u8> metadata(file_hash_table_size + file_table_size + dir_hash_table_size +
std::vector<u32> file_hash_table(file_hash_table_entry_count, ROMFS_ENTRY_EMPTY); dir_table_size);
u32* const dir_hash_table_pointer = reinterpret_cast<u32*>(metadata.data());
u8* const dir_table_pointer = metadata.data() + dir_hash_table_size;
u32* const file_hash_table_pointer =
reinterpret_cast<u32*>(metadata.data() + dir_hash_table_size + dir_table_size);
u8* const file_table_pointer =
metadata.data() + dir_hash_table_size + dir_table_size + file_hash_table_size;
std::vector<u8> dir_table(dir_table_size); std::span<u32> dir_hash_table(dir_hash_table_pointer, dir_hash_table_entry_count);
std::vector<u8> file_table(file_table_size); std::span<u32> file_hash_table(file_hash_table_pointer, file_hash_table_entry_count);
std::span<u8> dir_table(dir_table_pointer, dir_table_size);
std::span<u8> file_table(file_table_pointer, file_table_size);
std::shared_ptr<RomFSBuildFileContext> cur_file; // Initialize hash tables.
std::memset(dir_hash_table.data(), 0xFF, dir_hash_table.size_bytes());
std::memset(file_hash_table.data(), 0xFF, file_hash_table.size_bytes());
// Sort tables by name.
std::sort(files.begin(), files.end(),
[](const auto& a, const auto& b) { return a->path < b->path; });
std::sort(directories.begin(), directories.end(),
[](const auto& a, const auto& b) { return a->path < b->path; });
// Determine file offsets. // Determine file offsets.
u32 entry_offset = 0; u32 entry_offset = 0;
std::shared_ptr<RomFSBuildFileContext> prev_file = nullptr; std::shared_ptr<RomFSBuildFileContext> prev_file = nullptr;
for (const auto& it : files) { for (const auto& cur_file : files) {
cur_file = it.second;
file_partition_size = Common::AlignUp(file_partition_size, 16); file_partition_size = Common::AlignUp(file_partition_size, 16);
cur_file->offset = file_partition_size; cur_file->offset = file_partition_size;
file_partition_size += cur_file->size; file_partition_size += cur_file->size;
@ -243,34 +247,48 @@ std::multimap<u64, VirtualFile> RomFSBuildContext::Build() {
} }
// Assign deferred parent/sibling ownership. // Assign deferred parent/sibling ownership.
for (auto it = files.rbegin(); it != files.rend(); ++it) { for (auto it = files.rbegin(); it != files.rend(); ++it) {
cur_file = it->second; auto& cur_file = *it;
cur_file->sibling = cur_file->parent->file; cur_file->sibling = cur_file->parent->file;
cur_file->parent->file = cur_file; cur_file->parent->file = cur_file;
} }
std::shared_ptr<RomFSBuildDirectoryContext> cur_dir;
// Determine directory offsets. // Determine directory offsets.
entry_offset = 0; entry_offset = 0;
for (const auto& it : directories) { for (const auto& cur_dir : directories) {
cur_dir = it.second;
cur_dir->entry_offset = entry_offset; cur_dir->entry_offset = entry_offset;
entry_offset += entry_offset +=
static_cast<u32>(sizeof(RomFSDirectoryEntry) + static_cast<u32>(sizeof(RomFSDirectoryEntry) +
Common::AlignUp(cur_dir->path_len - cur_dir->cur_path_ofs, 4)); Common::AlignUp(cur_dir->path_len - cur_dir->cur_path_ofs, 4));
} }
// Assign deferred parent/sibling ownership. // Assign deferred parent/sibling ownership.
for (auto it = directories.rbegin(); it->second != root; ++it) { for (auto it = directories.rbegin(); (*it) != root; ++it) {
cur_dir = it->second; auto& cur_dir = *it;
cur_dir->sibling = cur_dir->parent->child; cur_dir->sibling = cur_dir->parent->child;
cur_dir->parent->child = cur_dir; cur_dir->parent->child = cur_dir;
} }
std::multimap<u64, VirtualFile> out; // Create output map.
std::vector<std::pair<u64, VirtualFile>> out;
out.reserve(num_files + 2);
// Set header fields.
header.header_size = sizeof(RomFSHeader);
header.file_hash_table_size = file_hash_table_size;
header.file_table_size = file_table_size;
header.dir_hash_table_size = dir_hash_table_size;
header.dir_table_size = dir_table_size;
header.file_partition_ofs = ROMFS_FILEPARTITION_OFS;
header.dir_hash_table_ofs = Common::AlignUp(header.file_partition_ofs + file_partition_size, 4);
header.dir_table_ofs = header.dir_hash_table_ofs + header.dir_hash_table_size;
header.file_hash_table_ofs = header.dir_table_ofs + header.dir_table_size;
header.file_table_ofs = header.file_hash_table_ofs + header.file_hash_table_size;
std::vector<u8> header_data(sizeof(RomFSHeader));
std::memcpy(header_data.data(), &header, header_data.size());
out.emplace_back(0, std::make_shared<VectorVfsFile>(std::move(header_data)));
// Populate file tables. // Populate file tables.
for (const auto& it : files) { for (const auto& cur_file : files) {
cur_file = it.second;
RomFSFileEntry cur_entry{}; RomFSFileEntry cur_entry{};
cur_entry.parent = cur_file->parent->entry_offset; cur_entry.parent = cur_file->parent->entry_offset;
@ -287,7 +305,7 @@ std::multimap<u64, VirtualFile> RomFSBuildContext::Build() {
cur_entry.name_size = name_size; cur_entry.name_size = name_size;
out.emplace(cur_file->offset + ROMFS_FILEPARTITION_OFS, std::move(cur_file->source)); out.emplace_back(cur_file->offset + ROMFS_FILEPARTITION_OFS, std::move(cur_file->source));
std::memcpy(file_table.data() + cur_file->entry_offset, &cur_entry, sizeof(RomFSFileEntry)); std::memcpy(file_table.data() + cur_file->entry_offset, &cur_entry, sizeof(RomFSFileEntry));
std::memset(file_table.data() + cur_file->entry_offset + sizeof(RomFSFileEntry), 0, std::memset(file_table.data() + cur_file->entry_offset + sizeof(RomFSFileEntry), 0,
Common::AlignUp(cur_entry.name_size, 4)); Common::AlignUp(cur_entry.name_size, 4));
@ -296,8 +314,7 @@ std::multimap<u64, VirtualFile> RomFSBuildContext::Build() {
} }
// Populate dir tables. // Populate dir tables.
for (const auto& it : directories) { for (const auto& cur_dir : directories) {
cur_dir = it.second;
RomFSDirectoryEntry cur_entry{}; RomFSDirectoryEntry cur_entry{};
cur_entry.parent = cur_dir == root ? 0 : cur_dir->parent->entry_offset; cur_entry.parent = cur_dir == root ? 0 : cur_dir->parent->entry_offset;
@ -323,34 +340,13 @@ std::multimap<u64, VirtualFile> RomFSBuildContext::Build() {
cur_dir->path.data() + cur_dir->cur_path_ofs, name_size); cur_dir->path.data() + cur_dir->cur_path_ofs, name_size);
} }
// Set header fields. // Write metadata.
header.header_size = sizeof(RomFSHeader); out.emplace_back(header.dir_hash_table_ofs,
header.file_hash_table_size = file_hash_table_size; std::make_shared<VectorVfsFile>(std::move(metadata)));
header.file_table_size = file_table_size;
header.dir_hash_table_size = dir_hash_table_size;
header.dir_table_size = dir_table_size;
header.file_partition_ofs = ROMFS_FILEPARTITION_OFS;
header.dir_hash_table_ofs = Common::AlignUp(header.file_partition_ofs + file_partition_size, 4);
header.dir_table_ofs = header.dir_hash_table_ofs + header.dir_hash_table_size;
header.file_hash_table_ofs = header.dir_table_ofs + header.dir_table_size;
header.file_table_ofs = header.file_hash_table_ofs + header.file_hash_table_size;
std::vector<u8> header_data(sizeof(RomFSHeader)); // Sort the output.
std::memcpy(header_data.data(), &header, header_data.size()); std::sort(out.begin(), out.end(),
out.emplace(0, std::make_shared<VectorVfsFile>(std::move(header_data))); [](const auto& a, const auto& b) { return a.first < b.first; });
std::vector<u8> metadata(file_hash_table_size + file_table_size + dir_hash_table_size +
dir_table_size);
std::size_t index = 0;
std::memcpy(metadata.data(), dir_hash_table.data(), dir_hash_table.size() * sizeof(u32));
index += dir_hash_table.size() * sizeof(u32);
std::memcpy(metadata.data() + index, dir_table.data(), dir_table.size());
index += dir_table.size();
std::memcpy(metadata.data() + index, file_hash_table.data(),
file_hash_table.size() * sizeof(u32));
index += file_hash_table.size() * sizeof(u32);
std::memcpy(metadata.data() + index, file_table.data(), file_table.size());
out.emplace(header.dir_hash_table_ofs, std::make_shared<VectorVfsFile>(std::move(metadata)));
return out; return out;
} }

View file

@ -22,14 +22,14 @@ public:
~RomFSBuildContext(); ~RomFSBuildContext();
// This finalizes the context. // This finalizes the context.
std::multimap<u64, VirtualFile> Build(); std::vector<std::pair<u64, VirtualFile>> Build();
private: private:
VirtualDir base; VirtualDir base;
VirtualDir ext; VirtualDir ext;
std::shared_ptr<RomFSBuildDirectoryContext> root; std::shared_ptr<RomFSBuildDirectoryContext> root;
std::map<std::string, std::shared_ptr<RomFSBuildDirectoryContext>, std::less<>> directories; std::vector<std::shared_ptr<RomFSBuildDirectoryContext>> directories;
std::map<std::string, std::shared_ptr<RomFSBuildFileContext>, std::less<>> files; std::vector<std::shared_ptr<RomFSBuildFileContext>> files;
u64 num_dirs = 0; u64 num_dirs = 0;
u64 num_files = 0; u64 num_files = 0;
u64 dir_table_size = 0; u64 dir_table_size = 0;

View file

@ -55,44 +55,68 @@ struct FileEntry {
}; };
static_assert(sizeof(FileEntry) == 0x20, "FileEntry has incorrect size."); static_assert(sizeof(FileEntry) == 0x20, "FileEntry has incorrect size.");
template <typename Entry> struct RomFSTraversalContext {
std::pair<Entry, std::string> GetEntry(const VirtualFile& file, std::size_t offset) { RomFSHeader header;
Entry entry{}; VirtualFile file;
if (file->ReadObject(&entry, offset) != sizeof(Entry)) std::vector<u8> directory_meta;
std::vector<u8> file_meta;
};
template <typename EntryType, auto Member>
std::pair<EntryType, std::string> GetEntry(const RomFSTraversalContext& ctx, size_t offset) {
const size_t entry_end = offset + sizeof(EntryType);
const std::vector<u8>& vec = ctx.*Member;
const size_t size = vec.size();
const u8* data = vec.data();
EntryType entry{};
if (entry_end > size) {
return {}; return {};
std::string string(entry.name_length, '\0'); }
if (file->ReadArray(&string[0], string.size(), offset + sizeof(Entry)) != string.size()) std::memcpy(&entry, data + offset, sizeof(EntryType));
return {};
return {entry, string}; const size_t name_length = std::min(entry_end + entry.name_length, size) - entry_end;
std::string name(reinterpret_cast<const char*>(data + entry_end), name_length);
return {entry, std::move(name)};
} }
void ProcessFile(const VirtualFile& file, std::size_t file_offset, std::size_t data_offset, std::pair<DirectoryEntry, std::string> GetDirectoryEntry(const RomFSTraversalContext& ctx,
u32 this_file_offset, std::shared_ptr<VectorVfsDirectory>& parent) { size_t directory_offset) {
while (this_file_offset != ROMFS_ENTRY_EMPTY) { return GetEntry<DirectoryEntry, &RomFSTraversalContext::directory_meta>(ctx, directory_offset);
auto entry = GetEntry<FileEntry>(file, file_offset + this_file_offset); }
parent->AddFile(std::make_shared<OffsetVfsFile>( std::pair<FileEntry, std::string> GetFileEntry(const RomFSTraversalContext& ctx,
file, entry.first.size, entry.first.offset + data_offset, entry.second)); size_t file_offset) {
return GetEntry<FileEntry, &RomFSTraversalContext::file_meta>(ctx, file_offset);
}
void ProcessFile(const RomFSTraversalContext& ctx, u32 this_file_offset,
std::shared_ptr<VectorVfsDirectory>& parent) {
while (this_file_offset != ROMFS_ENTRY_EMPTY) {
auto entry = GetFileEntry(ctx, this_file_offset);
parent->AddFile(std::make_shared<OffsetVfsFile>(ctx.file, entry.first.size,
entry.first.offset + ctx.header.data_offset,
std::move(entry.second)));
this_file_offset = entry.first.sibling; this_file_offset = entry.first.sibling;
} }
} }
void ProcessDirectory(const VirtualFile& file, std::size_t dir_offset, std::size_t file_offset, void ProcessDirectory(const RomFSTraversalContext& ctx, u32 this_dir_offset,
std::size_t data_offset, u32 this_dir_offset,
std::shared_ptr<VectorVfsDirectory>& parent) { std::shared_ptr<VectorVfsDirectory>& parent) {
while (this_dir_offset != ROMFS_ENTRY_EMPTY) { while (this_dir_offset != ROMFS_ENTRY_EMPTY) {
auto entry = GetEntry<DirectoryEntry>(file, dir_offset + this_dir_offset); auto entry = GetDirectoryEntry(ctx, this_dir_offset);
auto current = std::make_shared<VectorVfsDirectory>( auto current = std::make_shared<VectorVfsDirectory>(
std::vector<VirtualFile>{}, std::vector<VirtualDir>{}, entry.second); std::vector<VirtualFile>{}, std::vector<VirtualDir>{}, entry.second);
if (entry.first.child_file != ROMFS_ENTRY_EMPTY) { if (entry.first.child_file != ROMFS_ENTRY_EMPTY) {
ProcessFile(file, file_offset, data_offset, entry.first.child_file, current); ProcessFile(ctx, entry.first.child_file, current);
} }
if (entry.first.child_dir != ROMFS_ENTRY_EMPTY) { if (entry.first.child_dir != ROMFS_ENTRY_EMPTY) {
ProcessDirectory(file, dir_offset, file_offset, data_offset, entry.first.child_dir, ProcessDirectory(ctx, entry.first.child_dir, current);
current);
} }
parent->AddDirectory(current); parent->AddDirectory(current);
@ -107,22 +131,25 @@ VirtualDir ExtractRomFS(VirtualFile file) {
return root_container; return root_container;
} }
RomFSHeader header{}; RomFSTraversalContext ctx{};
if (file->ReadObject(&header) != sizeof(RomFSHeader)) {
return root_container; if (file->ReadObject(&ctx.header) != sizeof(RomFSHeader)) {
return nullptr;
} }
if (header.header_size != sizeof(RomFSHeader)) { if (ctx.header.header_size != sizeof(RomFSHeader)) {
return root_container; return nullptr;
} }
const u64 file_offset = header.file_meta.offset; ctx.file = file;
const u64 dir_offset = header.directory_meta.offset; ctx.directory_meta =
file->ReadBytes(ctx.header.directory_meta.size, ctx.header.directory_meta.offset);
ctx.file_meta = file->ReadBytes(ctx.header.file_meta.size, ctx.header.file_meta.offset);
ProcessDirectory(file, dir_offset, file_offset, header.data_offset, 0, root_container); ProcessDirectory(ctx, 0, root_container);
if (auto root = root_container->GetSubdirectory(""); root) { if (auto root = root_container->GetSubdirectory(""); root) {
return std::make_shared<CachedVfsDirectory>(std::move(root)); return root;
} }
ASSERT(false); ASSERT(false);

View file

@ -59,8 +59,8 @@ VirtualFile ConcatenatedVfsFile::MakeConcatenatedFile(std::string&& name,
return VirtualFile(new ConcatenatedVfsFile(std::move(name), std::move(concatenation_map))); return VirtualFile(new ConcatenatedVfsFile(std::move(name), std::move(concatenation_map)));
} }
VirtualFile ConcatenatedVfsFile::MakeConcatenatedFile(u8 filler_byte, std::string&& name, VirtualFile ConcatenatedVfsFile::MakeConcatenatedFile(
std::multimap<u64, VirtualFile>&& files) { u8 filler_byte, std::string&& name, std::vector<std::pair<u64, VirtualFile>>&& files) {
// Fold trivial cases. // Fold trivial cases.
if (files.empty()) { if (files.empty()) {
return nullptr; return nullptr;

View file

@ -37,7 +37,7 @@ public:
/// Convenience function that turns a map of offsets to files into a concatenated file, filling /// Convenience function that turns a map of offsets to files into a concatenated file, filling
/// gaps with a given filler byte. /// gaps with a given filler byte.
static VirtualFile MakeConcatenatedFile(u8 filler_byte, std::string&& name, static VirtualFile MakeConcatenatedFile(u8 filler_byte, std::string&& name,
std::multimap<u64, VirtualFile>&& files); std::vector<std::pair<u64, VirtualFile>>&& files);
std::string GetName() const override; std::string GetName() const override;
std::size_t GetSize() const override; std::size_t GetSize() const override;

View file

@ -3,6 +3,7 @@
#include <algorithm> #include <algorithm>
#include <set> #include <set>
#include <unordered_set>
#include <utility> #include <utility>
#include "core/file_sys/vfs_layered.h" #include "core/file_sys/vfs_layered.h"
@ -59,13 +60,12 @@ std::string LayeredVfsDirectory::GetFullPath() const {
std::vector<VirtualFile> LayeredVfsDirectory::GetFiles() const { std::vector<VirtualFile> LayeredVfsDirectory::GetFiles() const {
std::vector<VirtualFile> out; std::vector<VirtualFile> out;
std::set<std::string, std::less<>> out_names; std::unordered_set<std::string> out_names;
for (const auto& layer : dirs) { for (const auto& layer : dirs) {
for (auto& file : layer->GetFiles()) { for (auto& file : layer->GetFiles()) {
auto file_name = file->GetName(); const auto [it, is_new] = out_names.emplace(file->GetName());
if (!out_names.contains(file_name)) { if (is_new) {
out_names.emplace(std::move(file_name));
out.emplace_back(std::move(file)); out.emplace_back(std::move(file));
} }
} }
@ -75,18 +75,19 @@ std::vector<VirtualFile> LayeredVfsDirectory::GetFiles() const {
} }
std::vector<VirtualDir> LayeredVfsDirectory::GetSubdirectories() const { std::vector<VirtualDir> LayeredVfsDirectory::GetSubdirectories() const {
std::vector<std::string> names; std::vector<VirtualDir> out;
std::unordered_set<std::string> out_names;
for (const auto& layer : dirs) { for (const auto& layer : dirs) {
for (const auto& sd : layer->GetSubdirectories()) { for (const auto& sd : layer->GetSubdirectories()) {
if (std::find(names.begin(), names.end(), sd->GetName()) == names.end()) out_names.emplace(sd->GetName());
names.push_back(sd->GetName());
} }
} }
std::vector<VirtualDir> out; out.reserve(out_names.size());
out.reserve(names.size()); for (const auto& subdir : out_names) {
for (const auto& subdir : names)
out.emplace_back(GetSubdirectory(subdir)); out.emplace_back(GetSubdirectory(subdir));
}
return out; return out;
} }