[TEST] Adjustments on pipeline hash collisions

src/video_core/renderer_vulkan/vk_render_pass_cache.h

@@ -9,6 +9,7 @@
 #include <mutex>
 #include <ankerl/unordered_dense.h>
 
+#include "common/container_hash.h"
 #include "video_core/surface.h"
 #include "video_core/vulkan_common/vulkan_wrapper.h"
 
@@ -31,17 +32,26 @@ struct RenderPassKey {
 namespace std {
 template <>
 struct hash<Vulkan::RenderPassKey> {
+    static_assert(std::tuple_size_v<decltype(Vulkan::RenderPassKey::color_formats)> <= 8);
+    static_assert(static_cast<u32>(VideoCore::Surface::PixelFormat::Invalid) <= 0xFF);
+    static_assert(static_cast<u32>(VideoCore::Surface::PixelFormat::Max) <= 0xFF);
+    static_assert(VK_SAMPLE_COUNT_64_BIT <= 0xFF);
+
     [[nodiscard]] size_t operator()(const Vulkan::RenderPassKey& key) const noexcept {
-        size_t value = static_cast<size_t>(key.depth_format) << 48;
-        value ^= static_cast<size_t>(key.samples) << 52;
-        value ^= static_cast<size_t>(key.resolve_color) << 63;
-        value ^= static_cast<size_t>(key.color_clear_mask) << 54;
-        value ^= static_cast<size_t>(key.depth_stencil_clear) << 62;
-        value ^= static_cast<size_t>(key.color_discard_mask) << 24;
-        for (size_t i = 0; i < key.color_formats.size(); ++i) {
-            value ^= static_cast<size_t>(key.color_formats[i]) << (i * 6);
+        u64 formats = 0;
+        for (size_t index = 0; index < key.color_formats.size(); ++index) {
+            formats |= static_cast<u64>(key.color_formats[index]) << (index * 8);
         }
-        return value;
+        const u64 state = static_cast<u64>(key.depth_format) |
+                          (static_cast<u64>(key.samples) << 8) |
+                          (static_cast<u64>(key.color_clear_mask) << 16) |
+                          (static_cast<u64>(key.color_discard_mask) << 24) |
+                          (static_cast<u64>(key.resolve_color) << 32) |
+                          (static_cast<u64>(key.depth_stencil_clear) << 33);
+        size_t seed = 0;
+        Common::HashCombine(seed, formats);
+        Common::HashCombine(seed, state);
+        return seed;
     }
 };
 } // namespace std

src/video_core/renderer_vulkan/vk_texture_cache.cpp

@@ -2832,11 +2832,25 @@ VkRenderPass Framebuffer::RenderPassVariant(u32 color_clear_mask, bool depth_ste
     if (color_clear_mask == 0 && !depth_stencil_clear && color_discard_mask == 0) {
         return renderpass;
     }
+    static_assert(NUM_RT <= 8);
+    const u32 variant_key = color_clear_mask | (color_discard_mask << 8) |
+                            (static_cast<u32>(depth_stencil_clear) << 16);
+    for (u32 index = 0; index < num_memoized_variants; ++index) {
+        if (variant_keys[index] == variant_key) {
+            return variant_render_passes[index];
+        }
+    }
     RenderPassKey key = render_pass_key;
     key.color_clear_mask = color_clear_mask;
     key.depth_stencil_clear = depth_stencil_clear;
     key.color_discard_mask = color_discard_mask;
-    return render_pass_cache->Get(key);
+    const VkRenderPass variant = render_pass_cache->Get(key);
+    if (num_memoized_variants < variant_keys.size()) {
+        variant_keys[num_memoized_variants] = variant_key;
+        variant_render_passes[num_memoized_variants] = variant;
+        ++num_memoized_variants;
+    }
+    return variant;
 }
 
 void TextureCacheRuntime::AccelerateImageUpload(

src/video_core/renderer_vulkan/vk_texture_cache.h

@@ -248,6 +248,8 @@ public:
     }
 
 private:
+    static constexpr size_t NUM_MEMOIZED_RENDER_PASS_VARIANTS = 8;
+
     vk::Framebuffer framebuffer;
     VkRenderPass renderpass{};
     VkExtent2D render_area{};
@@ -265,6 +267,9 @@ private:
     RenderPassKey render_pass_key{};
     RenderPassCache* render_pass_cache{nullptr};
     bool discard_msaa_color{};
+    mutable std::array<u32, NUM_MEMOIZED_RENDER_PASS_VARIANTS> variant_keys{};
+    mutable std::array<VkRenderPass, NUM_MEMOIZED_RENDER_PASS_VARIANTS> variant_render_passes{};
+    mutable u32 num_memoized_variants{};
 };
 
 class Image : public VideoCommon::ImageBase {

src/video_core/textures/decoders.cpp

@@ -36,8 +36,7 @@ constexpr u32 SWIZZLE_RUN_SHIFT = 4;
 constexpr u32 SWIZZLE_RUN_MASK = SWIZZLE_RUN_BYTES - 1;
 constexpr u32 SWIZZLE_RUN_INDEX_MASK = GOB_SIZE_X / SWIZZLE_RUN_BYTES - 1;
 
-static_assert((SWIZZLE_X_BITS & SWIZZLE_RUN_MASK) == SWIZZLE_RUN_MASK,
-              "A swizzled run is only contiguous while the low bits of X map to themselves");
+static_assert((SWIZZLE_X_BITS & SWIZZLE_RUN_MASK) == SWIZZLE_RUN_MASK);
 
 constexpr std::array<u32, GOB_SIZE_X / SWIZZLE_RUN_BYTES> SWIZZLE_X_RUN_TABLE = [] {
     std::array<u32, GOB_SIZE_X / SWIZZLE_RUN_BYTES> table{};