[texture_cache] Enforce conservative 3D texture aliasing. Consistently merge/lookup images. (#196)

Previously, the merging strategy is permissive but the lookup strategy is strict. Together they create a infinite merge loop because lookup always fails and merges are always executed.
The last attempt to fix this issue made the lookup strategy more permissive which created unexpected aliased textures.
This commit implements an alternative: make the merging strategy strict.

Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/196
Reviewed-by: Shinmegumi <shinmegumi@eden-emu.dev>
Co-authored-by: weakboson <weakboson@quantum-field.net>
Co-committed-by: weakboson <weakboson@quantum-field.net>

src/video_core/texture_cache/texture_cache.h

@@ -1426,7 +1426,6 @@ ImageId TextureCache<P>::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, DA
         if (solution) {
             gpu_addr = solution->gpu_addr;
             cpu_addr = solution->cpu_addr;
-            // TODO: properly update new_info.size.depth.
             new_info.resources = solution->resources;
             join_overlap_ids.push_back(overlap_id);
             join_copies_to_do.emplace_back(JoinCopy{false, overlap_id});

src/video_core/texture_cache/util.cpp

@@ -346,7 +346,7 @@ template <u32 GOB_EXTENT>
     if (!IsBlockLinearSizeCompatible(new_info, info, base.level, 0, strict_size)) {
         return std::nullopt;
     }
-    const u32 mip_depth = std::max(1U, new_info.size.depth << base.level);
+    const u32 mip_depth = AdjustMipSize(new_info.size.depth, base.level);
     if (mip_depth < info.size.depth + base.layer) {
         return std::nullopt;
     }
@@ -431,8 +431,14 @@ template <u32 GOB_EXTENT>
         return std::nullopt;
     }
     const SubresourceExtent resources = new_info.resources;
-    s32 layers = 1;
-    if (info.type != ImageType::e3D) {
+    s32 layers;
+    if (info.type == ImageType::e3D) {
+        const u32 mip_depth = AdjustMipSize(info.size.depth, base->level);
+        if (mip_depth < new_info.size.depth + base->layer) {
+            return std::nullopt;
+        }
+        layers = 1;
+    } else {
         layers = std::max(resources.layers, info.resources.layers + base->layer);
     }
     return OverlapResult{
@@ -1219,10 +1225,10 @@ std::optional<SubresourceBase> FindSubresource(const ImageInfo& candidate, const
         return std::nullopt;
     }
     if (existing.type == ImageType::e3D) {
-        // const u32 mip_depth = std::max(1U, existing.size.depth << base->level);
-        // if (mip_depth < candidate.size.depth + base->layer) {
-        //     return std::nullopt;
-        // }
+        const u32 mip_depth = AdjustMipSize(existing.size.depth, base->level);
+        if (mip_depth < candidate.size.depth + base->layer) {
+            return std::nullopt;
+        }
     } else if (existing.resources.layers < candidate.resources.layers + base->layer) {
         return std::nullopt;
     }
@@ -1250,7 +1256,7 @@ bool IsSubCopy(const ImageInfo& candidate, const ImageBase& image, GPUVAddr cand
         return false;
     }
     if (existing.type == ImageType::e3D) {
-        const u32 mip_depth = std::max(1U, existing.size.depth << base->level);
+        const u32 mip_depth = AdjustMipSize(existing.size.depth, base->level);
         if (mip_depth < candidate.size.depth + base->layer) {
             return false;
         }