Merge pull request #2093 from FreddyFunk/disk-cache-better-compression

Better LZ4 compression utilization for the disk based shader cache and the yuzu build system

src/common/CMakeLists.txt

@@ -91,6 +91,8 @@ add_library(common STATIC
     logging/log.h
     logging/text_formatter.cpp
     logging/text_formatter.h
+    lz4_compression.cpp
+    lz4_compression.h
     math_util.h
     memory_hook.cpp
     memory_hook.h
@@ -136,3 +138,4 @@ endif()
 create_target_directory_groups(common)
 
 target_link_libraries(common PUBLIC Boost::boost fmt microprofile)
+target_link_libraries(common PRIVATE lz4_static)

src/common/lz4_compression.cpp

@@ -0,0 +1,78 @@
+// Copyright 2019 yuzu Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#pragma once
+
+#include <algorithm>
+#include <lz4hc.h>
+
+#include "common/assert.h"
+#include "common/lz4_compression.h"
+
+namespace Common::Compression {
+
+std::vector<u8> CompressDataLZ4(const u8* source, std::size_t source_size) {
+    ASSERT_MSG(source_size <= LZ4_MAX_INPUT_SIZE, "Source size exceeds LZ4 maximum input size");
+
+    const auto source_size_int = static_cast<int>(source_size);
+    const int max_compressed_size = LZ4_compressBound(source_size_int);
+    std::vector<u8> compressed(max_compressed_size);
+
+    const int compressed_size = LZ4_compress_default(reinterpret_cast<const char*>(source),
+                                                     reinterpret_cast<char*>(compressed.data()),
+                                                     source_size_int, max_compressed_size);
+
+    if (compressed_size <= 0) {
+        // Compression failed
+        return {};
+    }
+
+    compressed.resize(compressed_size);
+
+    return compressed;
+}
+
+std::vector<u8> CompressDataLZ4HC(const u8* source, std::size_t source_size,
+                                  s32 compression_level) {
+    ASSERT_MSG(source_size <= LZ4_MAX_INPUT_SIZE, "Source size exceeds LZ4 maximum input size");
+
+    compression_level = std::clamp(compression_level, LZ4HC_CLEVEL_MIN, LZ4HC_CLEVEL_MAX);
+
+    const auto source_size_int = static_cast<int>(source_size);
+    const int max_compressed_size = LZ4_compressBound(source_size_int);
+    std::vector<u8> compressed(max_compressed_size);
+
+    const int compressed_size = LZ4_compress_HC(
+        reinterpret_cast<const char*>(source), reinterpret_cast<char*>(compressed.data()),
+        source_size_int, max_compressed_size, compression_level);
+
+    if (compressed_size <= 0) {
+        // Compression failed
+        return {};
+    }
+
+    compressed.resize(compressed_size);
+
+    return compressed;
+}
+
+std::vector<u8> CompressDataLZ4HCMax(const u8* source, std::size_t source_size) {
+    return CompressDataLZ4HC(source, source_size, LZ4HC_CLEVEL_MAX);
+}
+
+std::vector<u8> DecompressDataLZ4(const std::vector<u8>& compressed,
+                                  std::size_t uncompressed_size) {
+    std::vector<u8> uncompressed(uncompressed_size);
+    const int size_check = LZ4_decompress_safe(reinterpret_cast<const char*>(compressed.data()),
+                                               reinterpret_cast<char*>(uncompressed.data()),
+                                               static_cast<int>(compressed.size()),
+                                               static_cast<int>(uncompressed.size()));
+    if (static_cast<int>(uncompressed_size) != size_check) {
+        // Decompression failed
+        return {};
+    }
+    return uncompressed;
+}
+
+} // namespace Common::Compression

src/common/lz4_compression.h

@@ -0,0 +1,55 @@
+// Copyright 2019 yuzu Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#include <vector>
+
+#include "common/common_types.h"
+
+namespace Common::Compression {
+
+/**
+ * Compresses a source memory region with LZ4 and returns the compressed data in a vector.
+ *
+ * @param source the uncompressed source memory region.
+ * @param source_size the size in bytes of the uncompressed source memory region.
+ *
+ * @return the compressed data.
+ */
+std::vector<u8> CompressDataLZ4(const u8* source, std::size_t source_size);
+
+/**
+ * Utilizes the LZ4 subalgorithm LZ4HC with the specified compression level. Higher compression
+ * levels result in a smaller compressed size, but require more CPU time for compression. The
+ * compression level has almost no impact on decompression speed. Data compressed with LZ4HC can
+ * also be decompressed with the default LZ4 decompression.
+ *
+ * @param source the uncompressed source memory region.
+ * @param source_size the size in bytes of the uncompressed source memory region.
+ * @param compression_level the used compression level. Should be between 3 and 12.
+ *
+ * @return the compressed data.
+ */
+std::vector<u8> CompressDataLZ4HC(const u8* source, std::size_t source_size, s32 compression_level);
+
+/**
+ * Utilizes the LZ4 subalgorithm LZ4HC with the highest possible compression level.
+ *
+ * @param source the uncompressed source memory region.
+ * @param source_size the size in bytes of the uncompressed source memory region.
+ *
+ * @return the compressed data.
+ */
+std::vector<u8> CompressDataLZ4HCMax(const u8* source, std::size_t source_size);
+
+/**
+ * Decompresses a source memory region with LZ4 and returns the uncompressed data in a vector.
+ *
+ * @param compressed the compressed source memory region.
+ * @param uncompressed_size the size in bytes of the uncompressed data.
+ *
+ * @return the decompressed data.
+ */
+std::vector<u8> DecompressDataLZ4(const std::vector<u8>& compressed, std::size_t uncompressed_size);
+
+} // namespace Common::Compression

src/core/CMakeLists.txt

@@ -458,7 +458,7 @@ add_library(core STATIC
 create_target_directory_groups(core)
 
 target_link_libraries(core PUBLIC common PRIVATE audio_core video_core)
-target_link_libraries(core PUBLIC Boost::boost PRIVATE fmt lz4_static mbedtls opus unicorn open_source_archives)
+target_link_libraries(core PUBLIC Boost::boost PRIVATE fmt mbedtls opus unicorn open_source_archives)
 if (ENABLE_WEB_SERVICE)
     target_compile_definitions(core PRIVATE -DENABLE_WEB_SERVICE)
     target_link_libraries(core PRIVATE web_service)

src/core/loader/nso.cpp

@@ -4,11 +4,12 @@
 
 #include <cinttypes>
 #include <vector>
-#include <lz4.h>
+
 #include "common/common_funcs.h"
 #include "common/file_util.h"
 #include "common/hex_util.h"
 #include "common/logging/log.h"
+#include "common/lz4_compression.h"
 #include "common/swap.h"
 #include "core/core.h"
 #include "core/file_sys/patch_manager.h"
@@ -35,15 +36,11 @@ static_assert(sizeof(MODHeader) == 0x1c, "MODHeader has incorrect size.");
 
 std::vector<u8> DecompressSegment(const std::vector<u8>& compressed_data,
                                   const NSOSegmentHeader& header) {
-    std::vector<u8> uncompressed_data(header.size);
-    const int bytes_uncompressed =
-        LZ4_decompress_safe(reinterpret_cast<const char*>(compressed_data.data()),
-                            reinterpret_cast<char*>(uncompressed_data.data()),
-                            static_cast<int>(compressed_data.size()), header.size);
-
-    ASSERT_MSG(bytes_uncompressed == static_cast<int>(header.size) &&
-                   bytes_uncompressed == static_cast<int>(uncompressed_data.size()),
-               "{} != {} != {}", bytes_uncompressed, header.size, uncompressed_data.size());
+    const std::vector<u8> uncompressed_data =
+        Common::Compression::DecompressDataLZ4(compressed_data, header.size);
+
+    ASSERT_MSG(uncompressed_data.size() == static_cast<int>(header.size), "{} != {}", header.size,
+               uncompressed_data.size());
 
     return uncompressed_data;
 }

src/video_core/CMakeLists.txt

@@ -139,4 +139,4 @@ endif()
 create_target_directory_groups(video_core)
 
 target_link_libraries(video_core PUBLIC common core)
-target_link_libraries(video_core PRIVATE glad lz4_static)
+target_link_libraries(video_core PRIVATE glad)

src/video_core/renderer_opengl/gl_shader_disk_cache.cpp

@@ -4,13 +4,13 @@
 
 #include <cstring>
 #include <fmt/format.h>
-#include <lz4.h>
 
 #include "common/assert.h"
 #include "common/common_paths.h"
 #include "common/common_types.h"
 #include "common/file_util.h"
 #include "common/logging/log.h"
+#include "common/lz4_compression.h"
 #include "common/scm_rev.h"
 
 #include "core/core.h"
@@ -49,39 +49,6 @@ ShaderCacheVersionHash GetShaderCacheVersionHash() {
     return hash;
 }
 
-template <typename T>
-std::vector<u8> CompressData(const T* source, std::size_t source_size) {
-    if (source_size > LZ4_MAX_INPUT_SIZE) {
-        // Source size exceeds LZ4 maximum input size
-        return {};
-    }
-    const auto source_size_int = static_cast<int>(source_size);
-    const int max_compressed_size = LZ4_compressBound(source_size_int);
-    std::vector<u8> compressed(max_compressed_size);
-    const int compressed_size = LZ4_compress_default(reinterpret_cast<const char*>(source),
-                                                     reinterpret_cast<char*>(compressed.data()),
-                                                     source_size_int, max_compressed_size);
-    if (compressed_size <= 0) {
-        // Compression failed
-        return {};
-    }
-    compressed.resize(compressed_size);
-    return compressed;
-}
-
-std::vector<u8> DecompressData(const std::vector<u8>& compressed, std::size_t uncompressed_size) {
-    std::vector<u8> uncompressed(uncompressed_size);
-    const int size_check = LZ4_decompress_safe(reinterpret_cast<const char*>(compressed.data()),
-                                               reinterpret_cast<char*>(uncompressed.data()),
-                                               static_cast<int>(compressed.size()),
-                                               static_cast<int>(uncompressed.size()));
-    if (static_cast<int>(uncompressed_size) != size_check) {
-        // Decompression failed
-        return {};
-    }
-    return uncompressed;
-}
-
 } // namespace
 
 ShaderDiskCacheRaw::ShaderDiskCacheRaw(u64 unique_identifier, Maxwell::ShaderProgram program_type,
@@ -292,7 +259,7 @@ ShaderDiskCacheOpenGL::LoadPrecompiledFile(FileUtil::IOFile& file) {
                 return {};
             }
 
-            dump.binary = DecompressData(compressed_binary, binary_length);
+            dump.binary = Common::Compression::DecompressDataLZ4(compressed_binary, binary_length);
             if (dump.binary.empty()) {
                 return {};
             }
@@ -321,7 +288,7 @@ std::optional<ShaderDiskCacheDecompiled> ShaderDiskCacheOpenGL::LoadDecompiledEn
         return {};
     }
 
-    const std::vector<u8> code = DecompressData(compressed_code, code_size);
+    const std::vector<u8> code = Common::Compression::DecompressDataLZ4(compressed_code, code_size);
     if (code.empty()) {
         return {};
     }
@@ -507,7 +474,8 @@ void ShaderDiskCacheOpenGL::SaveDecompiled(u64 unique_identifier, const std::str
     if (!IsUsable())
         return;
 
-    const std::vector<u8> compressed_code{CompressData(code.data(), code.size())};
+    const std::vector<u8> compressed_code{Common::Compression::CompressDataLZ4HC(
+        reinterpret_cast<const u8*>(code.data()), code.size(), 9)};
     if (compressed_code.empty()) {
         LOG_ERROR(Render_OpenGL, "Failed to compress GLSL code - skipping shader {:016x}",
                   unique_identifier);
@@ -537,7 +505,9 @@ void ShaderDiskCacheOpenGL::SaveDump(const ShaderDiskCacheUsage& usage, GLuint p
     std::vector<u8> binary(binary_length);
     glGetProgramBinary(program, binary_length, nullptr, &binary_format, binary.data());
 
-    const std::vector<u8> compressed_binary = CompressData(binary.data(), binary.size());
+    const std::vector<u8> compressed_binary =
+        Common::Compression::CompressDataLZ4HC(binary.data(), binary.size(), 9);
+
     if (compressed_binary.empty()) {
         LOG_ERROR(Render_OpenGL, "Failed to compress binary program in shader={:016x}",
                   usage.unique_identifier);