[moltenVK] MacOS SpirV to msl format fix

Geometry issues present, but more games launch on MacOS now

* Fixed runtime_info.h to properly guard VkFormat usage with #ifdef APPLE
* Fixed MoltenVK subgroup scope errors in SPIR-V emission
* Various pipeline cache updates for integer render target support

src/shader_recompiler/backend/spirv/spirv_emit_context.cpp

@@ -1677,7 +1677,40 @@ void EmitContext::DefineOutputs(const IR::Program& program) {
             if (!info.stores_frag_color[index] && !profile.need_declared_frag_colors) {
                 continue;
             }
-            frag_color[index] = DefineOutput(*this, F32[4], std::nullopt);
+            // MoltenVK/Metal: Use integer output type for integer render targets
+            // Metal requires fragment shader outputs to match render target formats.
+            // On other platforms, always use F32[4] (float vec4) as usual.
+            Id output_type = F32[4];  // Default to float
+            if (runtime_info.is_moltenvk && index < runtime_info.color_formats.size()) {
+                const auto& format = runtime_info.color_formats[index];
+                // Check if the render target format is an integer format
+                const bool is_integer_format = (
+                    format == VK_FORMAT_R8_UINT ||
+                    format == VK_FORMAT_R8_SINT ||
+                    format == VK_FORMAT_R16_UINT ||
+                    format == VK_FORMAT_R16_SINT ||
+                    format == VK_FORMAT_R32_UINT ||
+                    format == VK_FORMAT_R32_SINT ||
+                    format == VK_FORMAT_R8G8_UINT ||
+                    format == VK_FORMAT_R8G8_SINT ||
+                    format == VK_FORMAT_R16G16_UINT ||
+                    format == VK_FORMAT_R16G16_SINT ||
+                    format == VK_FORMAT_R32G32_UINT ||
+                    format == VK_FORMAT_R32G32_SINT ||
+                    format == VK_FORMAT_R8G8B8A8_UINT ||
+                    format == VK_FORMAT_R8G8B8A8_SINT ||
+                    format == VK_FORMAT_R16G16B16A16_UINT ||
+                    format == VK_FORMAT_R16G16B16A16_SINT ||
+                    format == VK_FORMAT_R32G32B32A32_UINT ||
+                    format == VK_FORMAT_R32G32B32A32_SINT ||
+                    format == VK_FORMAT_A8B8G8R8_UINT_PACK32 ||
+                    format == VK_FORMAT_A8B8G8R8_SINT_PACK32
+                );
+                if (is_integer_format) {
+                    output_type = U32[4];  // Use unsigned int vec4 for integer formats
+                }
+            }
+            frag_color[index] = DefineOutput(*this, output_type, std::nullopt);
             Decorate(frag_color[index], spv::Decoration::Location, index);
             Name(frag_color[index], fmt::format("frag_color{}", index));
         }

src/shader_recompiler/runtime_info.h

@@ -8,6 +8,10 @@
 #include <optional>
 #include <vector>
 
+#ifdef __APPLE__
+#include <vulkan/vulkan_core.h>
+#endif
+
 #include "common/common_types.h"
 #include "shader_recompiler/varying_state.h"
 
@@ -104,6 +108,11 @@ struct RuntimeInfo {
     /// Transform feedback state for each varying
     std::array<TransformFeedbackVarying, 256> xfb_varyings{};
     u32 xfb_count{0};
+
+#ifdef __APPLE__
+    bool is_moltenvk{};
+    std::array<VkFormat, 8>  color_formats{}; //for color format change
+#endif
 };
 
 } // namespace Shader

src/video_core/renderer_vulkan/vk_pipeline_cache.cpp

@@ -271,6 +271,10 @@ Shader::RuntimeInfo MakeRuntimeInfo(std::span<const Shader::IR::Program> program
     }
     info.force_early_z = key.state.early_z != 0;
     info.y_negate = key.state.y_negate != 0;
+#ifdef __APPLE__
+    // MoltenVK: Check for integer color attachments
+    info.is_moltenvk = true;
+#endif
     return info;
 }
 
@@ -702,7 +706,22 @@ std::unique_ptr<GraphicsPipeline> PipelineCache::CreateGraphicsPipeline(
         const size_t stage_index{index - 1};
         infos[stage_index] = &program.info;
 
-        const auto runtime_info{MakeRuntimeInfo(programs, key, program, previous_stage, device)};
+        auto runtime_info{MakeRuntimeInfo(programs, key, program, previous_stage, device)};
+#ifdef __APPLE__
+        // MoltenVK: Populate color attachment formats for integer RT handling
+        if (program.stage == Shader::Stage::Fragment) {
+            for (size_t i = 0; i < 8; ++i) {
+                if (key.state.color_formats[i] != 0) {
+                    const auto rt_format = static_cast<Tegra::RenderTargetFormat>(key.state.color_formats[i]);
+                    const auto pixel_format = VideoCore::Surface::PixelFormatFromRenderTargetFormat(rt_format);
+                    if (pixel_format != VideoCore::Surface::PixelFormat::Invalid) {
+                        const auto format_info = MaxwellToVK::SurfaceFormat(device, FormatType::Optimal, false, pixel_format);
+                        runtime_info.color_formats[i] = format_info.format;
+                    }
+                }
+            }
+        }
+#endif
         ConvertLegacyToGeneric(program, runtime_info);
         const std::vector<u32> code{EmitSPIRV(profile, runtime_info, program, binding, this->optimize_spirv_output)};
         device.SaveShader(code);