[Vk] Improve Stencil Handling and Fix Read-After-Write Hazard (#235)

1. Improves stencil handling:
   - Adds surface type detection to distinguish between color, depth, stencil, and depth-stencil formats
   - Only enables stencil load/store operations for surfaces that actually contain stencil data
   - Avoids unnecessary stencil operations for non-stencil formats (DONT_CARE)

2. Fixes read-after-write (RAW) synchronization hazards:
   - Adds a subpass self-dependency (subpass 0 → subpass 0)
   - Synchronizes color/depth writes with subsequent shader reads
   - Uses VK_DEPENDENCY_BY_REGION_BIT for efficient synchronization
   - Covers all possible relevant stages,
     • src: Color output + Early/Late fragment tests
     • dst: Fragment shader
     • Access: Write → Read transitions
here is what hazard looks like [1147.550616] Render.Vulkan <Critical> video_core/vulkan_common/vulkan_debug_callback.cpp:DebugUtilCallback:55: Validation Error: [ SYNC-HAZARD-READ-AFTER-WRITE ] Object 0: handle = 0x7409630000000192, type = VK_OBJECT_TYPE_IMAGE_VIEW; | MessageID = 0xe4d96472 | vkCmdDrawIndexed: Hazard READ_AFTER_WRITE for VkImageView 0x7409630000000192[], in VkCommandBuffer 0xb400007cb003ea70[], and VkPipeline 0x44d3470000000213[], VkDescriptorSet 0x0[], type: VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, imageLayout: VK_IMAGE_LAYOUT_GENERAL, binding #2, index 0. Access info (usage: SYNC_FRAGMENT_SHADER_SHADER_SAMPLED_READ, prior_usage: SYNC_IMAGE_LAYOUT_TRANSITION, write_barriers: SYNC_FRAGMENT_SHADER_COLOR_ATTACHMENT_READ|SYNC_FRAGMENT_SHADER_DEPTH_STENCIL_ATTACHMENT_READ|SYNC_FRAGMENT_SHADER_INPUT_ATTACHMENT_READ|SYNC_EARLY_FRAGMENT_TESTS_DEPTH_STENCIL_ATTACHMENT_READ|SYNC_EARLY_FRAGMENT_TESTS_DEPTH_STENCIL_ATTACHMENT_WRITE|SYNC_LATE_FRAGMENT_TESTS_DEPTH_STENCIL_ATTACHMENT_READ|SYNC_LATE_FRAGMENT_TESTS_DEPTH_STENCIL_ATTACHMENT_WRITE|SYNC_COLOR_ATTACHMENT_OUTPUT_COLOR_ATTACHMENT_READ|SYNC_COLOR_ATTACHMENT_OUTPUT_COLOR_ATTACHMENT_WRITE|SYNC_SUBPASS_SHADER_HUAWEI_INPUT_ATTACHMENT_READ, command: vkCmdPipelineBarrier, seq_no: 45, reset_no: 129).

Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/235
Reviewed-by: crueter <crueter@eden-emu.dev>
Reviewed-by: Shinmegumi <shinmegumi@eden-emu.dev>
Co-authored-by: wildcard <nubieluv@gmail.com>
Co-committed-by: wildcard <nubieluv@gmail.com>

src/video_core/renderer_vulkan/vk_render_pass_cache.cpp

@@ -1,3 +1,6 @@
+// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
@@ -14,23 +17,55 @@
 namespace Vulkan {
 namespace {
 using VideoCore::Surface::PixelFormat;
+using VideoCore::Surface::SurfaceType;
 
-VkAttachmentDescription AttachmentDescription(const Device& device, PixelFormat format,
-                                              VkSampleCountFlagBits samples) {
-    using MaxwellToVK::SurfaceFormat;
-    return {
-        .flags = {},
-        .format = SurfaceFormat(device, FormatType::Optimal, true, format).format,
-        .samples = samples,
-        .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
-        .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
-        .stencilLoadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
-        .stencilStoreOp = VK_ATTACHMENT_STORE_OP_STORE,
-        .initialLayout = VK_IMAGE_LAYOUT_GENERAL,
-        .finalLayout = VK_IMAGE_LAYOUT_GENERAL,
-    };
-}
-} // Anonymous namespace
+        constexpr SurfaceType GetSurfaceType(PixelFormat format) {
+            switch (format) {
+                // Depth formats
+                case PixelFormat::D16_UNORM:
+                case PixelFormat::D32_FLOAT:
+                case PixelFormat::X8_D24_UNORM:
+                    return SurfaceType::Depth;
+
+                    // Stencil formats
+                case PixelFormat::S8_UINT:
+                    return SurfaceType::Stencil;
+
+                    // Depth+Stencil formats
+                case PixelFormat::D24_UNORM_S8_UINT:
+                case PixelFormat::S8_UINT_D24_UNORM:
+                case PixelFormat::D32_FLOAT_S8_UINT:
+                    return SurfaceType::DepthStencil;
+
+                    // Everything else is a color texture
+                default:
+                    return SurfaceType::ColorTexture;
+            }
+        }
+
+        VkAttachmentDescription AttachmentDescription(const Device& device, PixelFormat format,
+                                                      VkSampleCountFlagBits samples) {
+            using MaxwellToVK::SurfaceFormat;
+
+            const SurfaceType surface_type = GetSurfaceType(format);
+            const bool has_stencil = surface_type == SurfaceType::DepthStencil ||
+                                     surface_type == SurfaceType::Stencil;
+
+            return {
+                .flags = {},
+                .format = SurfaceFormat(device, FormatType::Optimal, true, format).format,
+                .samples = samples,
+                .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
+                .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
+                .stencilLoadOp = has_stencil ? VK_ATTACHMENT_LOAD_OP_LOAD
+                                                 : VK_ATTACHMENT_LOAD_OP_DONT_CARE,
+                .stencilStoreOp = has_stencil ? VK_ATTACHMENT_STORE_OP_STORE
+                                                  : VK_ATTACHMENT_STORE_OP_DONT_CARE,
+                .initialLayout = VK_IMAGE_LAYOUT_GENERAL,
+                .finalLayout = VK_IMAGE_LAYOUT_GENERAL,
+            };
+        }
+    } // Anonymous namespace
 
 RenderPassCache::RenderPassCache(const Device& device_) : device{&device_} {}
 
@@ -78,6 +113,18 @@ VkRenderPass RenderPassCache::Get(const RenderPassKey& key) {
         .preserveAttachmentCount = 0,
         .pPreserveAttachments = nullptr,
     };
+    const VkSubpassDependency dependency{
+            .srcSubpass = 0,  // Current subpass
+            .dstSubpass = 0,  // Same subpass (self-dependency)
+            .srcStageMask = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT |
+                            VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT |
+                            VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT,
+            .dstStageMask = VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT,
+            .srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT |
+                             VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT,
+            .dstAccessMask = VK_ACCESS_SHADER_READ_BIT,
+            .dependencyFlags = VK_DEPENDENCY_BY_REGION_BIT
+    };
     pair->second = device->GetLogical().CreateRenderPass({
         .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO,
         .pNext = nullptr,
@@ -86,8 +133,8 @@ VkRenderPass RenderPassCache::Get(const RenderPassKey& key) {
         .pAttachments = descriptions.empty() ? nullptr : descriptions.data(),
         .subpassCount = 1,
         .pSubpasses = &subpass,
-        .dependencyCount = 0,
-        .pDependencies = nullptr,
+        .dependencyCount = 1,
+        .pDependencies = &dependency,
     });
     return *pair->second;
 }