[vulkan] Re-adjusting ASTC resolve (#4250)

This is an adjustment made due to problems caused by my ASTC changes some pr's earlier; added sanity to decoded blocks, removes the inconditional storage view for ASTC formats (verified against MaxwellToVk logic), added support to VK_EXT_astc_decode_mode which will ensure a better way to resolve RGFA16/RGFA32 on tilers, simplified ASTC HDR decoding into LDR valid formats.

This fixes some issues with SteamDeck, Fire Emblem: Engage washed colors and other problems related to ASTC.

_Special Thanks_

1.- Meowly The Smol (@Gidoly)
2.- @simply0001 for the actual idea on the redundant storage views on the transcoded ASTC path.

Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/4250

src/video_core/host_shaders/astc_decoder.comp

@@ -77,14 +77,8 @@ uvec4 local_buff;
 uvec4 color_endpoint_data;
 int color_bitsread = 0;
 
-// Global "vector" to be pushed into when decoding
-// At most will require BLOCK_WIDTH x BLOCK_HEIGHT in single plane mode
-// At most will require BLOCK_WIDTH x BLOCK_HEIGHT x 2 in dual plane mode
-// So the maximum would be 144 (12 x 12) elements, x 2 for two planes
-#define DIVCEIL(number, divisor) (number + divisor - 1) / divisor
-#define ARRAY_NUM_ELEMENTS 144
-#define VECTOR_ARRAY_SIZE DIVCEIL(ARRAY_NUM_ELEMENTS * 2, 4)
-uint result_vector[ARRAY_NUM_ELEMENTS * 2];
+#define MAX_WEIGHT_VALUES 64
+uint result_vector[MAX_WEIGHT_VALUES];
 
 int result_index = 0;
 uint result_vector_max_index;
@@ -492,7 +486,7 @@ void DecodeColorValues(uvec4 modes, uint num_partitions, uint color_data_bits, o
         A = ReplicateBitTo9((bitval & 1));
         switch (encoding) {
         case JUST_BITS:
-            color_values[++out_index] = FastReplicateTo8(bitval, bitlen);
+            color_values[out_index++] = FastReplicateTo8(bitval, bitlen);
             break;
         case TRIT: {
             D = QuintTritValue(val);
@@ -571,7 +565,7 @@ void DecodeColorValues(uvec4 modes, uint num_partitions, uint color_data_bits, o
             uint T = (D * C) + B;
             T ^= A;
             T = (A & 0x80) | (T >> 2);
-            color_values[++out_index] = T;
+            color_values[out_index++] = T;
         }
     }
 }
@@ -753,12 +747,12 @@ void ComputeEndpoints(out uvec4 ep1, out uvec4 ep2, uint color_endpoint_mode, ui
 #define READ_UINT_VALUES(N)                                                                        \
     uvec4 V[2];                                                                                    \
     for (uint i = 0; i < N; i++) {                                                                 \
-        V[i / 4][i % 4] = color_values[++colvals_index];                      \
+        V[i / 4][i % 4] = color_values[colvals_index++];                      \
     }
 #define READ_INT_VALUES(N)                                                                         \
     ivec4 V[2];                                                                                    \
     for (uint i = 0; i < N; i++) {                                                                 \
-        V[i / 4][i % 4] = int(color_values[++colvals_index]);                      \
+        V[i / 4][i % 4] = int(color_values[colvals_index++]);                      \
     }
 
     switch (color_endpoint_mode) {
@@ -1225,6 +1219,10 @@ void DecompressBlock(ivec3 coord) {
         FillError(coord);
         return;
     }
+    if (GetNumWeightValues(size_params, dual_plane) > MAX_WEIGHT_VALUES) {
+        FillError(coord);
+        return;
+    }
     uint partition_index = 1;
     uvec4 color_endpoint_mode = uvec4(0);
     uint ced_pointer = 0;
@@ -1239,6 +1237,10 @@ void DecompressBlock(ivec3 coord) {
     const uint base_mode = base_cem & 3;
     const uint max_weight = DecodeMaxWeight(mode);
     const uint weight_bits = GetPackedBitSize(size_params, dual_plane, max_weight);
+    if (weight_bits < 24 || weight_bits > 96) {
+        FillError(coord);
+        return;
+    }
     uint remaining_bits = 128 - weight_bits - total_bitsread;
     uint extra_cem_bits = 0;
     if (base_mode > 0) {
@@ -1253,6 +1255,7 @@ void DecompressBlock(ivec3 coord) {
             extra_cem_bits += 8;
             break;
         default:
+            FillError(coord);
             return;
         }
     }
@@ -1262,6 +1265,7 @@ void DecompressBlock(ivec3 coord) {
     if (remaining_bits > 128) {
         // Bad data, more remaining bits than 4 bytes
         // return early
+        FillError(coord);
         return;
     }
     // Read color data...
@@ -1384,11 +1388,7 @@ void DecompressBlock(ivec3 coord) {
                 p = Cf / 65535.0f;
             }
 
-#ifdef VULKAN
-            imageStore(dest_image, coord + ivec3(i, j, 0), p.gbar);
-#else
             imageStore(dest_image, coord + ivec3(i, j, 0), clamp(p, 0.0f, 1.0f).gbar);
-#endif
         }
     }
 }

src/video_core/renderer_vulkan/vk_texture_cache.cpp

@@ -144,11 +144,6 @@ constexpr VkBorderColor ConvertBorderColor(const std::array<float, 4>& color) {
           info.size.depth == 1;
 }
 
-[[nodiscard]] bool WillUseWidenedAstcFormat(const Device& device, const ImageInfo& info) {
-    return WillUseAcceleratedAstcDecode(device, info) &&
-           !VideoCore::Surface::IsPixelFormatSRGB(info.format);
-}
-
 [[nodiscard]] VkImageCreateInfo MakeImageCreateInfo(const Device& device, const ImageInfo& info,
                                                     std::optional<VkFormat> format_override = {}) {
     auto format_info =
@@ -269,6 +264,10 @@ constexpr VkBorderColor ConvertBorderColor(const std::array<float, 4>& color) {
     }
 }
 
+[[nodiscard]] bool IsLdrAstcFormat(VkFormat format) {
+    return format >= VK_FORMAT_ASTC_4x4_UNORM_BLOCK && format <= VK_FORMAT_ASTC_12x12_SRGB_BLOCK;
+}
+
 [[nodiscard]] VkImageAspectFlags ImageViewAspectMask(const VideoCommon::ImageViewInfo& info) {
     if (info.IsRenderTarget()) {
         return ImageAspectMask(info.format);
@@ -1780,12 +1779,7 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu
     : VideoCommon::ImageBase(info_, gpu_addr_, cpu_addr_), scheduler{&runtime_.scheduler},
       runtime{&runtime_},
       original_image(MakeImage(runtime_.device, runtime_.memory_allocator, info,
-                               WillUseWidenedAstcFormat(runtime_.device, info)
-                                   ? std::span<const VkFormat>{}
-                                   : runtime->ViewFormats(info.format),
-                               WillUseWidenedAstcFormat(runtime_.device, info)
-                                   ? std::make_optional(VK_FORMAT_R32G32B32A32_SFLOAT)
-                                   : std::nullopt)),
+                               runtime->ViewFormats(info.format))),
       aspect_mask(ImageAspectMask(info.format)) {
     if (IsPixelFormatASTC(info.format) && !runtime->device.IsOptimalAstcSupported()) {
         switch (Settings::values.accelerate_astc.GetValue()) {
@@ -1812,13 +1806,9 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu
     }
     current_image = &Image::original_image;
     storage_image_views.resize(info.resources.levels);
-    if (IsPixelFormatASTC(info.format) && !runtime->device.IsOptimalAstcSupported() &&
-        Settings::values.astc_recompression.GetValue() ==
-            Settings::AstcRecompression::Uncompressed) {
+    if (WillUseAcceleratedAstcDecode(runtime->device, info)) {
         const auto& device = runtime->device.GetLogical();
-        const VkFormat storage_format = WillUseWidenedAstcFormat(runtime->device, info)
-                                            ? VK_FORMAT_R32G32B32A32_SFLOAT
-                                            : VK_FORMAT_A8B8G8R8_UNORM_PACK32;
+        const VkFormat storage_format = VK_FORMAT_A8B8G8R8_UNORM_PACK32;
         for (s32 level = 0; level < info.resources.levels; ++level) {
             storage_image_views[level] =
                 MakeStorageView(device, level, *original_image, storage_format);
@@ -2204,9 +2194,7 @@ VkImageView Image::StorageImageView(s32 level) noexcept {
         auto format_info =
             MaxwellToVK::SurfaceFormat(runtime->device, FormatType::Optimal, true, info.format);
         if (WillUseAcceleratedAstcDecode(runtime->device, info)) {
-            format_info.format = WillUseWidenedAstcFormat(runtime->device, info)
-                                     ? VK_FORMAT_R32G32B32A32_SFLOAT
-                                     : VK_FORMAT_A8B8G8R8_UNORM_PACK32;
+            format_info.format = VK_FORMAT_A8B8G8R8_UNORM_PACK32;
         }
         view = MakeStorageView(runtime->device.GetLogical(), level, *(this->*current_image),
                                format_info.format);
@@ -2382,11 +2370,7 @@ ImageView::ImageView(TextureCacheRuntime& runtime, const VideoCommon::ImageViewI
             SanitizeDepthStencilSwizzle(swizzle, device->SupportsDepthStencilSwizzleOne());
         }
     }
-    uses_widened_astc_format = WillUseWidenedAstcFormat(*device, image.info);
     auto format_info = MaxwellToVK::SurfaceFormat(*device, FormatType::Optimal, true, format);
-    if (uses_widened_astc_format) {
-        format_info.format = VK_FORMAT_R32G32B32A32_SFLOAT;
-    }
     if (device->ApiVersion() >= VK_API_VERSION_1_3) {
         const VkFormatProperties3 properties3 =
             device->GetPhysical().GetFormatProperties3(format_info.format);
@@ -2404,9 +2388,18 @@ ImageView::ImageView(TextureCacheRuntime& runtime, const VideoCommon::ImageViewI
         .pNext = nullptr,
         .usage = clamped_view_usage,
     };
+    const VkImageViewASTCDecodeModeEXT astc_decode_mode{
+        .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_ASTC_DECODE_MODE_EXT,
+        .pNext = &image_view_usage,
+        .decodeMode = VK_FORMAT_R8G8B8A8_UNORM,
+    };
+    const void* view_next = &image_view_usage;
+    if (device->IsExtAstcDecodeModeSupported() && IsLdrAstcFormat(format_info.format)) {
+        view_next = &astc_decode_mode;
+    }
     const VkImageViewCreateInfo create_info{
         .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
-        .pNext = &image_view_usage,
+        .pNext = view_next,
         .flags = 0,
         .image = image.Handle(),
         .viewType = VkImageViewType{},
@@ -2531,9 +2524,6 @@ VkImageView ImageView::StorageView(Shader::TextureType texture_type,
         if (image_format == Shader::ImageFormat::Typeless) {
             if (!typeless_storage_view) {
                 auto info = MaxwellToVK::SurfaceFormat(*device, FormatType::Optimal, true, format);
-                if (uses_widened_astc_format) {
-                    info.format = VK_FORMAT_R32G32B32A32_SFLOAT;
-                }
                 typeless_storage_view = MakeView(info.format, VK_IMAGE_ASPECT_COLOR_BIT, texture_type);
             }
             return *typeless_storage_view;

src/video_core/renderer_vulkan/vk_texture_cache.h

@@ -436,7 +436,6 @@ private:
     VkSampleCountFlagBits samples = VK_SAMPLE_COUNT_1_BIT;
     u32 buffer_size = 0;
 
-    bool uses_widened_astc_format = false;
     bool supports_depth_comparison = false;
 };
 

src/video_core/textures/astc.cpp

@@ -1769,6 +1769,12 @@ static void DecompressBlock(std::span<const u8, 16> inBuf, const u32 blockWidth,
         return;
     }
 
+    if (weightParams.GetNumWeightValues() > 64) {
+        assert(false && "Too many weights in the weight grid");
+        FillError(outBuf, blockWidth, blockHeight);
+        return;
+    }
+
     // Read num partitions
     u32 nPartitions = strm.ReadBits<2>() + 1;
     assert(nPartitions <= 4);
@@ -1805,6 +1811,11 @@ static void DecompressBlock(std::span<const u8, 16> inBuf, const u32 blockWidth,
 
     // Remaining bits are color endpoint data...
     u32 nWeightBits = weightParams.GetPackedBitSize();
+    if (nWeightBits < 24 || nWeightBits > 96) {
+        assert(false && "Invalid weight bit count");
+        FillError(outBuf, blockWidth, blockHeight);
+        return;
+    }
     s32 remainingBits = 128 - nWeightBits - static_cast<int>(strm.GetBitsRead());
 
     // Consider extra bits prior to texel data...

src/video_core/vulkan_common/vulkan_device.cpp

@@ -820,14 +820,13 @@ bool Device::ComputeIsOptimalAstcSupported() const {
     if (!features.features.textureCompressionASTC_LDR) {
         return false;
     }
-    const auto format_feature_usage{VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT |
-                                    VK_FORMAT_FEATURE_BLIT_SRC_BIT |
-                                    VK_FORMAT_FEATURE_BLIT_DST_BIT |
-                                    VK_FORMAT_FEATURE_TRANSFER_SRC_BIT |
-                                    VK_FORMAT_FEATURE_TRANSFER_DST_BIT};
+    const VkFormatFeatureFlags format_feature_usage{
+        VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT | VK_FORMAT_FEATURE_SAMPLED_IMAGE_FILTER_LINEAR_BIT |
+        VK_FORMAT_FEATURE_TRANSFER_SRC_BIT | VK_FORMAT_FEATURE_TRANSFER_DST_BIT};
     for (const auto format : astc_formats) {
         const auto physical_format_properties{physical.GetFormatProperties(format)};
-        if ((physical_format_properties.optimalTilingFeatures & format_feature_usage) == 0) {
+        if ((physical_format_properties.optimalTilingFeatures & format_feature_usage) !=
+            format_feature_usage) {
             return false;
         }
     }

src/video_core/vulkan_common/vulkan_device.h

@@ -72,13 +72,12 @@ VK_DEFINE_HANDLE(VmaAllocator)
     FEATURE(KHR, PipelineExecutableProperties, PIPELINE_EXECUTABLE_PROPERTIES,                     \
             pipeline_executable_properties)                                                        \
     FEATURE(KHR, WorkgroupMemoryExplicitLayout, WORKGROUP_MEMORY_EXPLICIT_LAYOUT,                  \
-            workgroup_memory_explicit_layout)                                                      \
-    FEATURE(EXT, TextureCompressionASTCHDR, TEXTURE_COMPRESSION_ASTC_HDR,                          \
-            texture_compression_astc_hdr)
+            workgroup_memory_explicit_layout)
 
 
 // Define miscellaneous extensions which may be used by the implementation here.
 #define FOR_EACH_VK_EXTENSION(EXTENSION)                                                           \
+    EXTENSION(EXT, ASTC_DECODE_MODE, astc_decode_mode)                                             \
     EXTENSION(EXT, CONDITIONAL_RENDERING, conditional_rendering)                                   \
     EXTENSION(EXT, CONSERVATIVE_RASTERIZATION, conservative_rasterization)                         \
     EXTENSION(EXT, DEPTH_RANGE_UNRESTRICTED, depth_range_unrestricted)                             \
@@ -369,8 +368,7 @@ FN_MAX_LIMIT_LIST
     }
 
     bool IsOptimalAstcSupported() const {
-        return features.features.textureCompressionASTC_LDR &&
-               features.texture_compression_astc_hdr.textureCompressionASTC_HDR;
+        return is_optimal_astc_supported;
     }
 
     /// Returns true if BCn is natively supported.
@@ -816,6 +814,10 @@ FN_MAX_LIMIT_LIST
         return extensions.conditional_rendering;
     }
 
+    bool IsExtAstcDecodeModeSupported() const {
+        return extensions.astc_decode_mode;
+    }
+
     bool HasTimelineSemaphore() const;
 
     /// Returns true if the device supports VK_KHR_synchronization2.