[gl, vk, spv] Added component type handling for texture buffers and resolve pixel format variants

src/shader_recompiler/ir_opt/texture_pass.cpp

@@ -672,6 +672,7 @@ void TexturePass(Environment& env, IR::Program& program, const HostTranslateInfo
                     .secondary_shift_left = cbuf.secondary_shift_left,
                     .count = cbuf.count,
                     .size_shift = DESCRIPTOR_SIZE_SHIFT,
+                    .component_type = ReadTextureComponentType(env, cbuf),
                 });
             } else {
                 index = descriptors.Add(TextureDescriptor{

src/shader_recompiler/shader_info.h

@@ -194,6 +194,7 @@ struct TextureBufferDescriptor {
     u32 secondary_shift_left;
     u32 count;
     u32 size_shift;
+    SamplerComponentType component_type;
 
     auto operator<=>(const TextureBufferDescriptor&) const = default;
 };

src/video_core/renderer_opengl/gl_compute_pipeline.cpp

@@ -7,13 +7,50 @@
 #include <cstring>
 #include <bit>
 #include <numeric>
+#include <optional>
 #include "common/cityhash.h"
 #include "common/settings.h" // for enum class Settings::ShaderBackend
 #include "video_core/renderer_opengl/gl_compute_pipeline.h"
 #include "video_core/renderer_opengl/gl_shader_manager.h"
 #include "video_core/renderer_opengl/gl_shader_util.h"
+#include "video_core/surface/surface.h"
 
 namespace OpenGL {
+namespace {
+
+std::optional<VideoCore::Surface::PixelFormatNumeric>
+NumericFromComponentType(Shader::SamplerComponentType component_type) {
+    using VideoCore::Surface::PixelFormatNumeric;
+    switch (component_type) {
+    case Shader::SamplerComponentType::Float:
+        return PixelFormatNumeric::Float;
+    case Shader::SamplerComponentType::Sint:
+        return PixelFormatNumeric::Sint;
+    case Shader::SamplerComponentType::Uint:
+        return PixelFormatNumeric::Uint;
+    default:
+        return std::nullopt;
+    }
+}
+
+VideoCore::Surface::PixelFormat ResolveTexelBufferFormat(
+    VideoCore::Surface::PixelFormat format, Shader::SamplerComponentType component_type) {
+    const auto desired_numeric = NumericFromComponentType(component_type);
+    if (!desired_numeric) {
+        return format;
+    }
+    const auto current_numeric = VideoCore::Surface::GetPixelFormatNumericType(format);
+    if (*desired_numeric == current_numeric) {
+        return format;
+    }
+    if (const auto variant =
+            VideoCore::Surface::FindPixelFormatVariant(format, *desired_numeric)) {
+        return *variant;
+    }
+    return format;
+}
+
+} // Anonymous namespace
 
 using Shader::ImageBufferDescriptor;
 using Tegra::Texture::TexturePair;
@@ -174,8 +211,12 @@ void ComputePipeline::Configure() {
                 is_written = desc.is_written;
             }
             ImageView& image_view{texture_cache.GetImageView(views[texbuf_index].id)};
+            auto buffer_format = image_view.format;
+            if constexpr (!is_image) {
+                buffer_format = ResolveTexelBufferFormat(buffer_format, desc.component_type);
+            }
             buffer_cache.BindComputeTextureBuffer(texbuf_index, image_view.GpuAddr(),
-                                                  image_view.BufferSize(), image_view.format,
+                                                  image_view.BufferSize(), buffer_format,
                                                   is_written, is_image);
             ++texbuf_index;
         }

src/video_core/renderer_opengl/gl_graphics_pipeline.cpp

@@ -6,6 +6,7 @@
 
 #include <algorithm>
 #include <array>
+#include <optional>
 #include <string>
 #include <vector>
 #include <bit>
@@ -18,6 +19,7 @@
 #include "video_core/renderer_opengl/gl_shader_util.h"
 #include "video_core/renderer_opengl/gl_state_tracker.h"
 #include "video_core/shader_notify.h"
+#include "video_core/surface/surface.h"
 #include "video_core/texture_cache/texture_cache.h"
 
 #if defined(_MSC_VER) && defined(NDEBUG)
@@ -39,6 +41,38 @@ using VideoCommon::ImageId;
 constexpr u32 MAX_TEXTURES = 64;
 constexpr u32 MAX_IMAGES = 8;
 
+std::optional<VideoCore::Surface::PixelFormatNumeric>
+NumericFromComponentType(Shader::SamplerComponentType component_type) {
+    using VideoCore::Surface::PixelFormatNumeric;
+    switch (component_type) {
+    case Shader::SamplerComponentType::Float:
+        return PixelFormatNumeric::Float;
+    case Shader::SamplerComponentType::Sint:
+        return PixelFormatNumeric::Sint;
+    case Shader::SamplerComponentType::Uint:
+        return PixelFormatNumeric::Uint;
+    default:
+        return std::nullopt;
+    }
+}
+
+VideoCore::Surface::PixelFormat ResolveTexelBufferFormat(
+    VideoCore::Surface::PixelFormat format, Shader::SamplerComponentType component_type) {
+    const auto desired_numeric = NumericFromComponentType(component_type);
+    if (!desired_numeric) {
+        return format;
+    }
+    const auto current_numeric = VideoCore::Surface::GetPixelFormatNumericType(format);
+    if (*desired_numeric == current_numeric) {
+        return format;
+    }
+    if (const auto variant =
+            VideoCore::Surface::FindPixelFormatVariant(format, *desired_numeric)) {
+        return *variant;
+    }
+    return format;
+}
+
 GLenum Stage(size_t stage_index) {
     switch (stage_index) {
     case 0:
@@ -397,8 +431,12 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) {
                     is_written = desc.is_written;
                 }
                 ImageView& image_view{texture_cache.GetImageView(texture_buffer_it->id)};
+                auto buffer_format = image_view.format;
+                if constexpr (!is_image) {
+                    buffer_format = ResolveTexelBufferFormat(buffer_format, desc.component_type);
+                }
                 buffer_cache.BindGraphicsTextureBuffer(stage, index, image_view.GpuAddr(),
-                                                       image_view.BufferSize(), image_view.format,
+                                                       image_view.BufferSize(), buffer_format,
                                                        is_written, is_image);
                 ++index;
                 ++texture_buffer_it;

src/video_core/renderer_vulkan/vk_compute_pipeline.cpp

@@ -20,6 +20,7 @@
 #include "video_core/shader_notify.h"
 #include "video_core/vulkan_common/vulkan_device.h"
 #include "video_core/vulkan_common/vulkan_wrapper.h"
+#include <optional>
 
 namespace Vulkan {
 
@@ -40,6 +41,38 @@ ComputePipeline::ComputePipeline(const Device& device_, vk::PipelineCache& pipel
     if (shader_notify) {
         shader_notify->MarkShaderBuilding();
     }
+    std::optional<VideoCore::Surface::PixelFormatNumeric>
+    NumericFromComponentType(Shader::SamplerComponentType component_type) {
+        using VideoCore::Surface::PixelFormatNumeric;
+        switch (component_type) {
+        case Shader::SamplerComponentType::Float:
+            return PixelFormatNumeric::Float;
+        case Shader::SamplerComponentType::Sint:
+            return PixelFormatNumeric::Sint;
+        case Shader::SamplerComponentType::Uint:
+            return PixelFormatNumeric::Uint;
+        default:
+            return std::nullopt;
+        }
+    }
+
+    VideoCore::Surface::PixelFormat ResolveTexelBufferFormat(
+        VideoCore::Surface::PixelFormat format, Shader::SamplerComponentType component_type) {
+        const auto desired_numeric = NumericFromComponentType(component_type);
+        if (!desired_numeric) {
+            return format;
+        }
+        const auto current_numeric = VideoCore::Surface::GetPixelFormatNumericType(format);
+        if (*desired_numeric == current_numeric) {
+            return format;
+        }
+        if (const auto variant =
+                VideoCore::Surface::FindPixelFormatVariant(format, *desired_numeric)) {
+            return *variant;
+        }
+        return format;
+    }
+
     std::copy_n(info.constant_buffer_used_sizes.begin(), uniform_buffer_sizes.size(),
                 uniform_buffer_sizes.begin());
 
@@ -182,8 +215,12 @@ void ComputePipeline::Configure(Tegra::Engines::KeplerCompute& kepler_compute,
                 is_written = desc.is_written;
             }
             ImageView& image_view = texture_cache.GetImageView(views[index].id);
+            VideoCore::Surface::PixelFormat buffer_format = image_view.format;
+            if constexpr (!is_image) {
+                buffer_format = ResolveTexelBufferFormat(buffer_format, desc.component_type);
+            }
             buffer_cache.BindComputeTextureBuffer(index, image_view.GpuAddr(),
-                                                  image_view.BufferSize(), image_view.format,
+                                                  image_view.BufferSize(), buffer_format,
                                                   is_written, is_image);
             ++index;
         }

src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp

@@ -6,6 +6,7 @@
 
 #include <algorithm>
 #include <array>
+#include <optional>
 #include <iostream>
 #include <span>
 #include <string_view>
@@ -64,6 +65,37 @@ DescriptorLayoutBuilder MakeBuilder(const Device& device, std::span<const Shader
         };
         builder.Add(infos[index], stages.at(index));
     }
+    std::optional<VideoCore::Surface::PixelFormatNumeric>
+    NumericFromComponentType(Shader::SamplerComponentType component_type) {
+        using VideoCore::Surface::PixelFormatNumeric;
+        switch (component_type) {
+        case Shader::SamplerComponentType::Float:
+            return PixelFormatNumeric::Float;
+        case Shader::SamplerComponentType::Sint:
+            return PixelFormatNumeric::Sint;
+        case Shader::SamplerComponentType::Uint:
+            return PixelFormatNumeric::Uint;
+        default:
+            return std::nullopt;
+        }
+    }
+
+    VideoCore::Surface::PixelFormat ResolveTexelBufferFormat(
+        VideoCore::Surface::PixelFormat format, Shader::SamplerComponentType component_type) {
+        const auto desired_numeric = NumericFromComponentType(component_type);
+        if (!desired_numeric) {
+            return format;
+        }
+        const auto current_numeric = VideoCore::Surface::GetPixelFormatNumericType(format);
+        if (*desired_numeric == current_numeric) {
+            return format;
+        }
+        if (const auto variant =
+                VideoCore::Surface::FindPixelFormatVariant(format, *desired_numeric)) {
+            return *variant;
+        }
+        return format;
+    }
     return builder;
 }
 
@@ -421,8 +453,12 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) {
                     is_written = desc.is_written;
                 }
                 ImageView& image_view{texture_cache.GetImageView(texture_buffer_it->id)};
+                VideoCore::Surface::PixelFormat buffer_format = image_view.format;
+                if constexpr (!is_image) {
+                    buffer_format = ResolveTexelBufferFormat(buffer_format, desc.component_type);
+                }
                 buffer_cache.BindGraphicsTextureBuffer(stage, index, image_view.GpuAddr(),
-                                                       image_view.BufferSize(), image_view.format,
+                                                       image_view.BufferSize(), buffer_format,
                                                        is_written, is_image);
                 ++index;
                 ++texture_buffer_it;

src/video_core/renderer_vulkan/vk_texture_cache.cpp

@@ -48,6 +48,7 @@ using VideoCore::Surface::HasAlpha;
 using VideoCore::Surface::IsPixelFormatASTC;
 using VideoCore::Surface::IsPixelFormatInteger;
 using VideoCore::Surface::SurfaceType;
+using VideoCore::Surface::PixelFormatNumeric;
 
 namespace {
 constexpr VkBorderColor ConvertBorderColor(const std::array<float, 4>& color) {
@@ -2315,6 +2316,19 @@ std::optional<u32> ImageView::LayerCountOverride(Shader::TextureType texture_typ
     default:
         return std::nullopt;
     }
+
+    std::optional<PixelFormatNumeric> ComponentNumericType(Shader::SamplerComponentType component) {
+        switch (component) {
+        case Shader::SamplerComponentType::Float:
+            return PixelFormatNumeric::Float;
+        case Shader::SamplerComponentType::Sint:
+            return PixelFormatNumeric::Sint;
+        case Shader::SamplerComponentType::Uint:
+            return PixelFormatNumeric::Uint;
+        default:
+            return std::nullopt;
+        }
+    }
 }
 
 VkImageView ImageView::DepthView() {
@@ -2382,8 +2396,30 @@ VkImageView ImageView::SampledView(Shader::TextureType texture_type,
     default:
         break;
     }
+    const auto desired_numeric = ComponentNumericType(component_type);
+    if (!desired_numeric) {
         return Handle(texture_type);
     }
+    const PixelFormatNumeric current_numeric =
+        VideoCore::Surface::GetPixelFormatNumericType(format);
+    if (*desired_numeric == current_numeric) {
+        return Handle(texture_type);
+    }
+    const auto remapped_format =
+        VideoCore::Surface::FindPixelFormatVariant(format, *desired_numeric);
+    if (!remapped_format) {
+        return Handle(texture_type);
+    }
+    auto& cached_view = sampled_component_views[static_cast<size_t>(*desired_numeric)]
+                                            [static_cast<size_t>(texture_type)];
+    if (cached_view) {
+        return *cached_view;
+    }
+    const auto& info =
+        MaxwellToVK::SurfaceFormat(*device, FormatType::Optimal, true, *remapped_format);
+    cached_view = MakeView(info.format, VK_IMAGE_ASPECT_COLOR_BIT, texture_type);
+    return *cached_view;
+}
 
 VkImageView ImageView::StorageView(Shader::TextureType texture_type,
                                    Shader::ImageFormat image_format) {

src/video_core/renderer_vulkan/vk_texture_cache.h

@@ -277,6 +277,8 @@ private:
         std::array<vk::ImageView, Shader::NUM_TEXTURE_TYPES> typeless;
     };
 
+    static constexpr size_t NUMERIC_VIEW_TYPES = 3;
+
     [[nodiscard]] Shader::TextureType BaseTextureType() const noexcept;
     [[nodiscard]] std::optional<u32> LayerCountOverride(Shader::TextureType texture_type) const noexcept;
     [[nodiscard]] VkImageView DepthView(Shader::TextureType texture_type);
@@ -293,6 +295,8 @@ private:
     std::unique_ptr<StorageViews> storage_views;
     std::array<vk::ImageView, Shader::NUM_TEXTURE_TYPES> depth_views;
     std::array<vk::ImageView, Shader::NUM_TEXTURE_TYPES> stencil_views;
+    std::array<std::array<vk::ImageView, Shader::NUM_TEXTURE_TYPES>, NUMERIC_VIEW_TYPES>
+        sampled_component_views;
     vk::ImageView color_view;
     vk::Image null_image;
     VkImage image_handle = VK_NULL_HANDLE;

src/video_core/shader_environment.cpp

@@ -70,6 +70,46 @@ static Shader::TexturePixelFormat ConvertTexturePixelFormat(const Tegra::Texture
                                    entry.a_type, entry.srgb_conversion));
 }
 
+namespace {
+
+[[nodiscard]] bool UsesSwizzleSource(const Tegra::Texture::TICEntry& entry,
+                                     Tegra::Texture::SwizzleSource source) {
+    const std::array swizzles{entry.x_source.Value(), entry.y_source.Value(),
+                              entry.z_source.Value(), entry.w_source.Value()};
+    return std::ranges::any_of(swizzles, [source](Tegra::Texture::SwizzleSource current) {
+        return current == source;
+    });
+}
+
+[[nodiscard]] std::optional<Shader::SamplerComponentType> DepthStencilComponentFromSwizzle(
+    const Tegra::Texture::TICEntry& entry, VideoCore::Surface::PixelFormat pixel_format) {
+    using Tegra::Texture::SwizzleSource;
+    const bool uses_r = UsesSwizzleSource(entry, SwizzleSource::R);
+    const bool uses_g = UsesSwizzleSource(entry, SwizzleSource::G);
+
+    switch (pixel_format) {
+    case VideoCore::Surface::PixelFormat::D24_UNORM_S8_UINT:
+    case VideoCore::Surface::PixelFormat::D32_FLOAT_S8_UINT:
+        if (uses_r != uses_g) {
+            return uses_r ? Shader::SamplerComponentType::Depth
+                          : Shader::SamplerComponentType::Stencil;
+        }
+        break;
+    case VideoCore::Surface::PixelFormat::S8_UINT_D24_UNORM:
+        if (uses_r != uses_g) {
+            return uses_r ? Shader::SamplerComponentType::Stencil
+                          : Shader::SamplerComponentType::Depth;
+        }
+        break;
+    default:
+        break;
+    }
+
+    return std::nullopt;
+}
+
+} // Anonymous namespace
+
 static Shader::SamplerComponentType ConvertSamplerComponentType(
     const Tegra::Texture::TICEntry& entry) {
     const auto pixel_format = PixelFormatFromTextureInfo(entry.format, entry.r_type, entry.g_type,
@@ -83,6 +123,9 @@ static Shader::SamplerComponentType ConvertSamplerComponentType(
         return Shader::SamplerComponentType::Stencil;
     }
     if (surface_type == VideoCore::Surface::SurfaceType::DepthStencil) {
+        if (const auto inferred = DepthStencilComponentFromSwizzle(entry, pixel_format)) {
+            return *inferred;
+        }
         return entry.depth_texture != 0 ? Shader::SamplerComponentType::Depth
                                         : Shader::SamplerComponentType::Stencil;
     }

src/video_core/surface.cpp

@@ -4,6 +4,8 @@
 // SPDX-FileCopyrightText: 2014 Citra Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
+#include <optional>
+
 #include "common/common_types.h"
 #include "common/math_util.h"
 #include "common/settings.h"
@@ -408,6 +410,126 @@ bool IsPixelFormatSignedInteger(PixelFormat format) {
     }
 }
 
+namespace {
+
+struct NumericVariantSet {
+    PixelFormat float_format = PixelFormat::Invalid;
+    PixelFormat uint_format = PixelFormat::Invalid;
+    PixelFormat sint_format = PixelFormat::Invalid;
+
+    [[nodiscard]] std::optional<PixelFormat> Select(PixelFormatNumeric numeric) const {
+        PixelFormat candidate = PixelFormat::Invalid;
+        switch (numeric) {
+        case PixelFormatNumeric::Float:
+            candidate = float_format;
+            break;
+        case PixelFormatNumeric::Uint:
+            candidate = uint_format;
+            break;
+        case PixelFormatNumeric::Sint:
+            candidate = sint_format;
+            break;
+        }
+        if (candidate == PixelFormat::Invalid) {
+            return std::nullopt;
+        }
+        return candidate;
+    }
+};
+
+constexpr NumericVariantSet MakeVariant(PixelFormat float_format, PixelFormat uint_format,
+                                        PixelFormat sint_format) {
+    return NumericVariantSet{
+        .float_format = float_format,
+        .uint_format = uint_format,
+        .sint_format = sint_format,
+    };
+}
+
+std::optional<NumericVariantSet> LookupNumericVariantSet(PixelFormat format) {
+    switch (format) {
+    case PixelFormat::R8_UNORM:
+    case PixelFormat::R8_SNORM:
+    case PixelFormat::R8_UINT:
+    case PixelFormat::R8_SINT:
+        return MakeVariant(PixelFormat::R8_UNORM, PixelFormat::R8_UINT, PixelFormat::R8_SINT);
+    case PixelFormat::R16_FLOAT:
+    case PixelFormat::R16_UNORM:
+    case PixelFormat::R16_SNORM:
+    case PixelFormat::R16_UINT:
+    case PixelFormat::R16_SINT:
+        return MakeVariant(PixelFormat::R16_FLOAT, PixelFormat::R16_UINT, PixelFormat::R16_SINT);
+    case PixelFormat::R32_FLOAT:
+    case PixelFormat::R32_UINT:
+    case PixelFormat::R32_SINT:
+        return MakeVariant(PixelFormat::R32_FLOAT, PixelFormat::R32_UINT, PixelFormat::R32_SINT);
+    case PixelFormat::R8G8_UNORM:
+    case PixelFormat::R8G8_SNORM:
+    case PixelFormat::R8G8_UINT:
+    case PixelFormat::R8G8_SINT:
+        return MakeVariant(PixelFormat::R8G8_UNORM, PixelFormat::R8G8_UINT, PixelFormat::R8G8_SINT);
+    case PixelFormat::R16G16_FLOAT:
+    case PixelFormat::R16G16_UNORM:
+    case PixelFormat::R16G16_SNORM:
+    case PixelFormat::R16G16_UINT:
+    case PixelFormat::R16G16_SINT:
+        return MakeVariant(PixelFormat::R16G16_FLOAT, PixelFormat::R16G16_UINT,
+                           PixelFormat::R16G16_SINT);
+    case PixelFormat::R32G32_FLOAT:
+    case PixelFormat::R32G32_UINT:
+    case PixelFormat::R32G32_SINT:
+        return MakeVariant(PixelFormat::R32G32_FLOAT, PixelFormat::R32G32_UINT,
+                           PixelFormat::R32G32_SINT);
+    case PixelFormat::R16G16B16A16_FLOAT:
+    case PixelFormat::R16G16B16A16_UNORM:
+    case PixelFormat::R16G16B16A16_SNORM:
+    case PixelFormat::R16G16B16A16_UINT:
+    case PixelFormat::R16G16B16A16_SINT:
+        return MakeVariant(PixelFormat::R16G16B16A16_FLOAT, PixelFormat::R16G16B16A16_UINT,
+                           PixelFormat::R16G16B16A16_SINT);
+    case PixelFormat::R32G32B32A32_FLOAT:
+    case PixelFormat::R32G32B32A32_UINT:
+    case PixelFormat::R32G32B32A32_SINT:
+        return MakeVariant(PixelFormat::R32G32B32A32_FLOAT, PixelFormat::R32G32B32A32_UINT,
+                           PixelFormat::R32G32B32A32_SINT);
+    case PixelFormat::A8B8G8R8_UNORM:
+    case PixelFormat::A8B8G8R8_SNORM:
+    case PixelFormat::A8B8G8R8_SRGB:
+    case PixelFormat::A8B8G8R8_UINT:
+    case PixelFormat::A8B8G8R8_SINT:
+        return MakeVariant(PixelFormat::A8B8G8R8_UNORM, PixelFormat::A8B8G8R8_UINT,
+                           PixelFormat::A8B8G8R8_SINT);
+    case PixelFormat::A2B10G10R10_UNORM:
+    case PixelFormat::A2B10G10R10_UINT:
+        return MakeVariant(PixelFormat::A2B10G10R10_UNORM, PixelFormat::A2B10G10R10_UINT,
+                           PixelFormat::Invalid);
+    default:
+        return std::nullopt;
+    }
+}
+
+} // Anonymous namespace
+
+PixelFormatNumeric GetPixelFormatNumericType(PixelFormat format) {
+    if (IsPixelFormatInteger(format)) {
+        return IsPixelFormatSignedInteger(format) ? PixelFormatNumeric::Sint
+                                                  : PixelFormatNumeric::Uint;
+    }
+    return PixelFormatNumeric::Float;
+}
+
+std::optional<PixelFormat> FindPixelFormatVariant(PixelFormat format,
+                                                  PixelFormatNumeric target_numeric) {
+    const auto variants = LookupNumericVariantSet(format);
+    if (!variants) {
+        return std::nullopt;
+    }
+    if (const auto candidate = variants->Select(target_numeric)) {
+        return candidate;
+    }
+    return std::nullopt;
+}
+
 size_t PixelComponentSizeBitsInteger(PixelFormat format) {
     switch (format) {
     case PixelFormat::A8B8G8R8_SINT:

src/video_core/surface.h

@@ -4,6 +4,7 @@
 #pragma once
 
 #include <climits>
+#include <optional>
 #include <utility>
 #include "common/assert.h"
 #include "common/common_types.h"
@@ -517,6 +518,16 @@ bool IsPixelFormatSignedInteger(PixelFormat format);
 
 size_t PixelComponentSizeBitsInteger(PixelFormat format);
 
+enum class PixelFormatNumeric {
+    Float,
+    Uint,
+    Sint,
+};
+
+PixelFormatNumeric GetPixelFormatNumericType(PixelFormat format);
+std::optional<PixelFormat> FindPixelFormatVariant(PixelFormat format,
+                                                  PixelFormatNumeric target_numeric);
+
 std::pair<u32, u32> GetASTCBlockSize(PixelFormat format);
 
 u64 TranscodedAstcSize(u64 base_size, PixelFormat format);