From 672bcbae012164ec59ef26ff79995c4585bd0e97 Mon Sep 17 00:00:00 2001 From: lizzie Date: Sun, 30 Aug 2026 08:10:51 +0200 Subject: [PATCH] [file_sys] robust-er IPSwitch compiler (#3911) this PR reworks IPS parser to be less stupid what do i mean by this? well generally give it a bit of love so it doesn't do a lot of unsound allocations also simplify the logic greatly (and use memcmp() string idiom instead of trusting the compiler so blindly...) no this doesn't mean to uber optimize IPS to handle 999 gb/s it's more so it doesn't outright crash with edge cases as the previous codebase was quite spaghetty also the major overhead is obviously the vector shenanigans and the I/O -- but thats out of scope Test that IPSwitch mods still properly work WITH ANY GAME IF THERE IS ANY REGRESSION IN SOME GAME/MOD THEN TELL ME Signed-off-by: lizzie Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/3911 Reviewed-by: crueter Reviewed-by: CamilleLaVey --- src/core/file_sys/ips_layer.cpp | 389 +++++++++++----------------- src/core/file_sys/ips_layer.h | 14 +- src/core/file_sys/patch_manager.cpp | 11 +- src/qt_common/util/mod.cpp | 5 +- 4 files changed, 168 insertions(+), 251 deletions(-) diff --git a/src/core/file_sys/ips_layer.cpp b/src/core/file_sys/ips_layer.cpp index 9981c1865d..2022c9c2e7 100644 --- a/src/core/file_sys/ips_layer.cpp +++ b/src/core/file_sys/ips_layer.cpp @@ -3,10 +3,12 @@ #include #include -#include #include #include #include +#include +#include +#include #include "common/hex_util.h" #include "common/logging.h" @@ -22,60 +24,29 @@ enum class IPSFileType { Error, }; -constexpr std::array, 11> ESCAPE_CHARACTER_MAP{{ - {"\\a", "\a"}, - {"\\b", "\b"}, - {"\\f", "\f"}, - {"\\n", "\n"}, - {"\\r", "\r"}, - {"\\t", "\t"}, - {"\\v", "\v"}, - {"\\\\", "\\"}, - {"\\\'", "\'"}, - {"\\\"", "\""}, - {"\\\?", "\?"}, -}}; - -static IPSFileType IdentifyMagic(const std::vector& magic) { - if (magic.size() != 5) { - return IPSFileType::Error; - } - - static constexpr std::array patch_magic{{'P', 'A', 'T', 'C', 'H'}}; - if (std::equal(magic.begin(), magic.end(), patch_magic.begin())) { - return IPSFileType::IPS; - } - - static constexpr std::array ips32_magic{{'I', 'P', 'S', '3', '2'}}; - if (std::equal(magic.begin(), magic.end(), ips32_magic.begin())) { - return IPSFileType::IPS32; +static IPSFileType IdentifyMagic(std::span magic) { + if (magic.size() >= 5) { + if (std::memcmp(magic.data(), "PATCH", 5) == 0) + return IPSFileType::IPS; + if (std::memcmp(magic.data(), "IPS32", 5) == 0) + return IPSFileType::IPS32; } - return IPSFileType::Error; } -static bool IsEOF(IPSFileType type, const std::vector& data) { - static constexpr std::array eof{{'E', 'O', 'F'}}; - if (type == IPSFileType::IPS && std::equal(data.begin(), data.end(), eof.begin())) { - return true; - } - - static constexpr std::array eeof{{'E', 'E', 'O', 'F'}}; - return type == IPSFileType::IPS32 && std::equal(data.begin(), data.end(), eeof.begin()); +static bool IsEOF(IPSFileType type, std::span magic) { + return (type == IPSFileType::IPS && magic.size() > 3 && std::memcmp(magic.data(), "EOF", 3) == 0) + || (type == IPSFileType::IPS32 && magic.size() > 4 && std::memcmp(magic.data(), "EEOF", 4) == 0); } VirtualFile PatchIPS(const VirtualFile& in, const VirtualFile& ips) { if (in == nullptr || ips == nullptr) return nullptr; - const auto type = IdentifyMagic(ips->ReadBytes(0x5)); - if (type == IPSFileType::Error) - return nullptr; - auto in_data = in->ReadAllBytes(); - if (in_data.size() == 0) { + auto const type = IdentifyMagic(in_data); + if (type == IPSFileType::Error) return nullptr; - } std::vector temp(type == IPSFileType::IPS ? 3 : 4); u64 offset = 5; // After header @@ -85,12 +56,9 @@ VirtualFile PatchIPS(const VirtualFile& in, const VirtualFile& ips) { break; } - u32 real_offset{}; - if (type == IPSFileType::IPS32) - real_offset = (temp[0] << 24) | (temp[1] << 16) | (temp[2] << 8) | temp[3]; - else - real_offset = (temp[0] << 16) | (temp[1] << 8) | temp[2]; - + u32 real_offset = (type == IPSFileType::IPS32) + ? ((temp[0] << 24) | (temp[1] << 16) | (temp[2] << 8) | temp[3]) + : ((temp[0] << 16) | (temp[1] << 8) | temp[2]); if (real_offset > in_data.size()) { return nullptr; } @@ -113,34 +81,35 @@ VirtualFile PatchIPS(const VirtualFile& in, const VirtualFile& ips) { return nullptr; if (real_offset + rle_size > in_data.size()) - rle_size = static_cast(in_data.size() - real_offset); + rle_size = u16(in_data.size() - real_offset); std::memset(in_data.data() + real_offset, *data, rle_size); } else { // Standard Patch auto read = data_size; if (real_offset + read > in_data.size()) - read = static_cast(in_data.size() - real_offset); + read = u16(in_data.size() - real_offset); if (ips->Read(in_data.data() + real_offset, read, offset) != data_size) return nullptr; offset += data_size; } } - - if (!IsEOF(type, temp)) { - return nullptr; + if (IsEOF(type, temp)) { + return std::make_shared(std::move(in_data), in->GetName(), in->GetContainingDirectory()); } - - return std::make_shared(std::move(in_data), in->GetName(), - in->GetContainingDirectory()); + return nullptr; } + +struct IPSwitchRecord { + std::array data; + size_t count; +}; struct IPSwitchCompiler::IPSwitchPatch { - std::string name; + ankerl::unordered_dense::map records; bool enabled; - std::map> records; }; IPSwitchCompiler::IPSwitchCompiler(VirtualFile patch_text_) : patch_text(std::move(patch_text_)) { - Parse(); + Parse(patch_text->ReadAllBytes()); } IPSwitchCompiler::~IPSwitchCompiler() = default; @@ -149,201 +118,159 @@ std::array IPSwitchCompiler::GetBuildID() const { return nso_build_id; } -bool IPSwitchCompiler::IsValid() const { - return valid; -} - -static bool StartsWith(std::string_view base, std::string_view check) { - return base.size() >= check.size() && base.substr(0, check.size()) == check; -} - -static std::string EscapeStringSequences(std::string in) { - for (const auto& seq : ESCAPE_CHARACTER_MAP) { - for (auto index = in.find(seq.first); index != std::string::npos; - index = in.find(seq.first, index)) { - in.replace(index, std::strlen(seq.first), seq.second); - index += std::strlen(seq.second); +static IPSwitchRecord EscapeStringSequences(std::string_view sv) { + IPSwitchRecord r{}; + for (auto it = sv.cbegin(); it != sv.cend(); ) { + if (*it == '\\' && it + 1 < sv.cend()) { + switch (it[1]) { + case 'n': r.data[r.count] = '\n'; break; + case 't': r.data[r.count] = '\t'; break; + case 'b': r.data[r.count] = '\b'; break; + case 'r': r.data[r.count] = '\r'; break; + case 'e': r.data[r.count] = '\e'; break; + case 'v': r.data[r.count] = '\v'; break; + case '?': r.data[r.count] = '\?'; break; + default: r.data[r.count] = it[1]; break; + } + ++r.count; + it += 2; + } else { + ++r.count; + ++it; } } - - return in; + return r; } -void IPSwitchCompiler::ParseFlag(const std::string& line) { - if (StartsWith(line, "@flag offset_shift ")) { - // Offset Shift Flag - offset_shift = std::strtoll(line.substr(19).c_str(), nullptr, 0); - } else if (StartsWith(line, "@little-endian")) { - // Set values to read as little endian - is_little_endian = true; - } else if (StartsWith(line, "@big-endian")) { - // Set values to read as big endian - is_little_endian = false; - } else if (StartsWith(line, "@flag print_values")) { - // Force printing of applied values - print_values = true; - } +[[nodiscard]] static inline std::array ReadNSOBuildId(std::string_view const s) { + std::array r{}; + for (std::size_t i = 0; i < s.size(); ++i) + r[i / 2] |= u8(u8(Common::ToHexNibble(s[i])) << u8((i % 2) * 4)); + return r; } -void IPSwitchCompiler::Parse() { - const auto bytes = patch_text->ReadAllBytes(); - std::stringstream s; - s.write(reinterpret_cast(bytes.data()), bytes.size()); - - std::vector lines; - std::string stream_line; - while (std::getline(s, stream_line)) { - // Remove a trailing \r - if (!stream_line.empty() && stream_line.back() == '\r') - stream_line.pop_back(); - lines.push_back(std::move(stream_line)); - } - - for (std::size_t i = 0; i < lines.size(); ++i) { - auto line = lines[i]; - - // Remove midline comments - std::size_t comment_index = std::string::npos; - bool within_string = false; - for (std::size_t k = 0; k < line.size(); ++k) { - if (line[k] == '\"' && (k > 0 && line[k - 1] != '\\')) { - within_string = !within_string; - } else if (line[k] == '\\' && (k < line.size() - 1 && line[k + 1] == '\\')) { - comment_index = k; - break; - } - } - - if (!StartsWith(line, "//") && comment_index != std::string::npos) { - last_comment = line.substr(comment_index + 2); - line = line.substr(0, comment_index); - } - - if (StartsWith(line, "@stop")) { - // Force stop - break; - } else if (StartsWith(line, "@nsobid-")) { - // NSO Build ID Specifier - const auto raw_build_id = fmt::format("{:0<64}", line.substr(8)); - nso_build_id = Common::HexStringToArray<0x20>(raw_build_id); - } else if (StartsWith(line, "#")) { - // Mandatory Comment - LOG_INFO(Loader, "[IPSwitchCompiler ('{}')] Forced output comment: {}", - patch_text->GetName(), line.substr(1)); - } else if (StartsWith(line, "//")) { - // Normal Comment - last_comment = line.substr(2); - if (last_comment.find_first_not_of(' ') == std::string::npos) - continue; - if (last_comment.find_first_not_of(' ') != 0) - last_comment = last_comment.substr(last_comment.find_first_not_of(' ')); - } else if (StartsWith(line, "@enabled") || StartsWith(line, "@disabled")) { - // Start of patch - const auto enabled = StartsWith(line, "@enabled"); - if (i == 0) - return; - LOG_INFO(Loader, "[IPSwitchCompiler ('{}')] Parsing patch '{}' ({})", - patch_text->GetName(), last_comment, line.substr(1)); - - IPSwitchPatch patch{last_comment, enabled, {}}; - - // Read rest of patch - while (true) { - if (i + 1 >= lines.size()) { - break; - } - - const auto& patch_line = lines[++i]; - - // Patch line may contain comments - if (StartsWith(patch_line, "//") || StartsWith(patch_line, "#")) { - continue; - } - - // Start of new patch - if (StartsWith(patch_line, "@enabled") || StartsWith(patch_line, "@disabled")) { - --i; - break; +void IPSwitchCompiler::Parse(std::span bytes) { + LOG_INFO(Loader, "IPSwitchCompiler: '{}'", patch_text->GetName()); + bool is_little_endian = false; + s64 offset_shift = 0; + //bool print_values = false; + + auto const parse_line = [&](std::string_view const line) { + // Keep in mind lines have trimmed spaces (at the end & start)! + LOG_INFO(Loader, "<{}>", line); + if (line.starts_with("@stop")) { + return false; // Force stop + } else if (line.starts_with("@nsobid-")) { // NSO Build ID Specifier + nso_build_id = ReadNSOBuildId(line.substr(8)); + } else if (line.starts_with("@enabled")) { + patches.push_back({{}, true}); //enabled patch + } else if (line.starts_with("@disabled")) { + patches.push_back({{}, false}); //disabled patch + } else if (line.starts_with("@flag offset_shift ")) { + offset_shift = std::strtoll(line.data() + 19, nullptr, 0); // Offset Shift Flag + } else if (line.starts_with("@little-endian")) { + is_little_endian = true; // Set values to read as little endian + } else if (line.starts_with("@big-endian")) { + is_little_endian = false; // Set values to read as big endian + } else if (line.starts_with("@flag print_values")) { + //print_values = true; // Force printing of applied values + } else if (line.starts_with("@")) { + LOG_WARNING(Loader, "Unknown flag {}", line); + } else { + size_t offset = size_t(std::strtoul(line.data(), nullptr, 16)); + offset += size_t(offset_shift); + if (auto const first_quote = line.find_first_of("\"\'"); first_quote != std::string::npos) { + // string replacement + char quote = line[first_quote]; + auto const start = line.cbegin() + first_quote + 1; + auto end = start; + for (; end < line.cend() && *end != quote; ) + end += (*end == '\\') ? 2 : 1; + if (start <= line.cend() && end <= line.cend()) { + LOG_INFO(Loader, "[S] value @ {:#08X} ", offset); + patches.back().records.insert_or_assign(u32(offset), EscapeStringSequences({start, end})); + } else { + LOG_WARNING(Loader, "invalid string"); } - - // Check for a flag - if (StartsWith(patch_line, "@")) { - ParseFlag(patch_line); - continue; + } else if (auto const first_space = line.find_last_of(" /\t\r\n"); first_space != std::string::npos) { + IPSwitchRecord r{}; // hex replacement + auto const start = line.cbegin() + first_space + 1; + auto const end = line.cend(); + if (start <= line.cend() && end <= line.cend()) { + auto const hs = Common::HexStringToVector({start, end}, is_little_endian); + std::memcpy(r.data.data(), hs.data(), hs.size()); + r.count = hs.size(); + LOG_INFO(Loader, "[H] value @ {:#08X}", offset); + patches.back().records.insert_or_assign(u32(offset), std::move(r)); + } else { + LOG_WARNING(Loader, "invalid line"); } - - // 11 - 8 hex digit offset + space + minimum two digit overwrite val - if (patch_line.length() < 11) + } else { + LOG_WARNING(Loader, "unhandled line!"); + } + } + return true; //continue + }; + + for (auto it = bytes.begin(); it < bytes.end(); ) { + auto const start = it; + auto end = start; + for (; end < bytes.end() && *end != '\n' && *end != '\r'; ++end) + ; + it = end + 1; //prepare for next line + std::string_view const sline{ + reinterpret_cast(bytes.data() + std::distance(bytes.begin(), start)), + size_t(std::distance(start, end)) + }; + if (sline.size() > 0) { + auto p = sline.cbegin(); + // skip space off line + for (; p < sline.cend() && std::isspace(*p); ++p) + ; + // now make a nominal preprocessed line: remove comments + char quote = '\0'; + auto const sline_start = p; + for (; p < sline.cend(); ) { + if ((!quote && p + 1 < sline.cend() && p[0] == '/' && p[1] == '/') + || (!quote && p[0] == '#')) { break; - auto offset = std::strtoul(patch_line.substr(0, 8).c_str(), nullptr, 16); - offset += static_cast(offset_shift); - - std::vector replace; - // 9 - first char of replacement val - if (patch_line[9] == '\"') { - // string replacement - auto end_index = patch_line.find('\"', 10); - if (end_index == std::string::npos || end_index < 10) - return; - while (patch_line[end_index - 1] == '\\') { - end_index = patch_line.find('\"', end_index + 1); - if (end_index == std::string::npos || end_index < 10) - return; - } - - auto value = patch_line.substr(10, end_index - 10); - value = EscapeStringSequences(value); - replace.reserve(value.size()); - std::copy(value.begin(), value.end(), std::back_inserter(replace)); + } else if (p[0] == '\"' || p[0] == '\'') { + quote = (p[0] == quote) ? '\0' : p[0]; + ++p; + } else if (p + 1 < sline.cend() && p[0] == '\\') { + p += 2; } else { - // hex replacement - const auto value = - patch_line.substr(9, patch_line.find_first_of(" /\r\n", 9) - 9); - replace = Common::HexStringToVector(value, is_little_endian); - } - - if (print_values) { - LOG_INFO(Loader, - "[IPSwitchCompiler ('{}')] - Patching value at offset {:#08x} " - "with byte string '{}'", - patch_text->GetName(), offset, Common::HexToString(replace)); + ++p; } - - patch.records.insert_or_assign(static_cast(offset), std::move(replace)); } - - patches.push_back(std::move(patch)); - } else if (StartsWith(line, "@")) { - ParseFlag(line); + // now we have the preprocessed string ;) + std::string_view pp_str(sline_start, p); + if (pp_str.size() > 0 && !parse_line(pp_str)) { + break; + } } } - - valid = true; } VirtualFile IPSwitchCompiler::Apply(const VirtualFile& in) const { - if (in == nullptr || !valid) + if (in == nullptr) return nullptr; auto in_data = in->ReadAllBytes(); for (const auto& patch : patches) { - if (!patch.enabled) - continue; - - for (const auto& record : patch.records) { - if (record.first >= in_data.size()) - continue; - auto replace_size = record.second.size(); - if (record.first + replace_size > in_data.size()) - replace_size = in_data.size() - record.first; - for (std::size_t i = 0; i < replace_size; ++i) - in_data[i + record.first] = record.second[i]; + if (patch.enabled) { + for (const auto& record : patch.records) { + if (record.first < in_data.size()) { + auto replace_size = record.second.count; + if (record.first + replace_size > in_data.size()) + replace_size = in_data.size() - record.first; + std::memcpy(in_data.data() + record.first, record.second.data.data(), replace_size); + } + } } } - - return std::make_shared(std::move(in_data), in->GetName(), - in->GetContainingDirectory()); + return std::make_shared(std::move(in_data), in->GetName(), in->GetContainingDirectory()); } } // namespace FileSys diff --git a/src/core/file_sys/ips_layer.h b/src/core/file_sys/ips_layer.h index d81378e8ad..e3c61f161b 100644 --- a/src/core/file_sys/ips_layer.h +++ b/src/core/file_sys/ips_layer.h @@ -1,11 +1,14 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later #pragma once #include -#include #include +#include #include "common/common_types.h" #include "core/file_sys/vfs/vfs.h" @@ -20,24 +23,17 @@ public: ~IPSwitchCompiler(); std::array GetBuildID() const; - bool IsValid() const; VirtualFile Apply(const VirtualFile& in) const; private: struct IPSwitchPatch; void ParseFlag(const std::string& flag); - void Parse(); - - bool valid = false; + void Parse(std::span bytes); VirtualFile patch_text; std::vector patches; std::array nso_build_id{}; - bool is_little_endian = false; - s64 offset_shift = 0; - bool print_values = false; - std::string last_comment = ""; }; } // namespace FileSys diff --git a/src/core/file_sys/patch_manager.cpp b/src/core/file_sys/patch_manager.cpp index 6ae4679054..ada8c44e78 100644 --- a/src/core/file_sys/patch_manager.cpp +++ b/src/core/file_sys/patch_manager.cpp @@ -345,8 +345,7 @@ VirtualDir PatchManager::PatchExeFS(VirtualDir exefs) const { return exefs; } -std::vector PatchManager::CollectPatches(const std::vector& patch_dirs, - const std::string& build_id) const { +std::vector PatchManager::CollectPatches(const std::vector& patch_dirs, const std::string& build_id) const { const auto& disabled = Settings::values.disabled_addons[title_id]; const auto nso_build_id = fmt::format("{:0<64}", build_id); @@ -361,16 +360,11 @@ std::vector PatchManager::CollectPatches(const std::vectorGetFiles()) { if (file->GetExtension() == "ips") { auto name = file->GetName(); - - const auto this_build_id = - fmt::format("{:0<64}", name.substr(0, name.find('.'))); + const auto this_build_id = fmt::format("{:0<64}", name.substr(0, name.find('.'))); if (nso_build_id == this_build_id) out.push_back(file); } else if (file->GetExtension() == "pchtxt") { IPSwitchCompiler compiler{file}; - if (!compiler.IsValid()) - continue; - const auto this_build_id = Common::HexToString(compiler.GetBuildID()); if (nso_build_id == this_build_id) out.push_back(file); @@ -378,7 +372,6 @@ std::vector PatchManager::CollectPatches(const std::vector +#include #include #include "frontend_common/mod_manager.h" #include "mod.h" @@ -124,8 +125,8 @@ const QString ExtractMod(const QString& path) { fs::remove_all(tmp, ec); if (!fs::create_directories(tmp, ec)) { QtCommon::Frontend::Critical(tr("Mod Extract Failed"), - tr("Failed to create temporary directory %1") - .arg(QString::fromStdString(tmp.string()))); + tr("Failed to create temporary directory %1") + .arg(QString::fromStdString(tmp.string()))); return QString(); }