[vk] Configuring VK_EXT_Custom_Border_Color of Hell.

Signed-off-by: Caio Oliveira <caiooliveirafarias0@gmail.com>

src/video_core/renderer_vulkan/vk_texture_cache.cpp

@@ -2214,15 +2214,27 @@ Sampler::Sampler(TextureCacheRuntime& runtime, const Tegra::Texture::TSCEntry& t
     const bool arbitrary_borders = runtime.device.IsExtCustomBorderColorSupported();
     const auto color = tsc.BorderColor();
 
+    // VK_EXT_custom_border_color has two features:
+    // - customBorderColors: Enables VK_BORDER_COLOR_*_CUSTOM_EXT, requires format OR customBorderColorWithoutFormat
+    // - customBorderColorWithoutFormat: Allows VK_FORMAT_UNDEFINED (format-agnostic custom borders)
+    //
+    // Configuration logic:
+    // 1. If BOTH features available: Use VK_BORDER_COLOR_FLOAT_CUSTOM_EXT + VK_FORMAT_UNDEFINED (optimal)
+    // 2. If only customBorderColors: Use VK_BORDER_COLOR_FLOAT_CUSTOM_EXT + specific format (spec compliant)
+    // 3. If only customBorderColorWithoutFormat: Shouldn't happen per spec, but handle as case 2
+    // 4. If neither: Use standard border colors (fallback)
+    const bool has_custom_colors = device.HasCustomBorderColorFeature();
+    const bool has_without_format = device.HasCustomBorderColorWithoutFormatFeature();
+    const bool use_custom_border = arbitrary_borders && has_custom_colors;
+
     const VkSamplerCustomBorderColorCreateInfoEXT border_ci{
         .sType = VK_STRUCTURE_TYPE_SAMPLER_CUSTOM_BORDER_COLOR_CREATE_INFO_EXT,
         .pNext = nullptr,
-        // TODO: Make use of std::bit_cast once libc++ supports it.
         .customBorderColor = std::bit_cast<VkClearColorValue>(color),
-        .format = VK_FORMAT_UNDEFINED,
+        .format = has_without_format ? VK_FORMAT_UNDEFINED : VK_FORMAT_R8G8B8A8_UNORM,
     };
     const void* pnext = nullptr;
-    if (arbitrary_borders) {
+    if (use_custom_border) {
         pnext = &border_ci;
     }
     const VkSamplerReductionModeCreateInfoEXT reduction_ci{
@@ -2257,7 +2269,7 @@ Sampler::Sampler(TextureCacheRuntime& runtime, const Tegra::Texture::TSCEntry& t
             .minLod = tsc.mipmap_filter == TextureMipmapFilter::None ? 0.0f : tsc.MinLod(),
             .maxLod = tsc.mipmap_filter == TextureMipmapFilter::None ? 0.25f : tsc.MaxLod(),
             .borderColor =
-                arbitrary_borders ? VK_BORDER_COLOR_FLOAT_CUSTOM_EXT : ConvertBorderColor(color),
+                use_custom_border ? VK_BORDER_COLOR_FLOAT_CUSTOM_EXT : ConvertBorderColor(color),
             .unnormalizedCoordinates = VK_FALSE,
         });
     };

src/video_core/vulkan_common/vulkan_device.cpp

@@ -1320,8 +1320,11 @@ bool Device::GetSuitability(bool requires_swapchain) {
 
 void Device::RemoveUnsuitableExtensions() {
     // VK_EXT_custom_border_color
-    extensions.custom_border_color = features.custom_border_color.customBorderColors &&
-                                     features.custom_border_color.customBorderColorWithoutFormat;
+    // Enable if at least one feature is available (customBorderColors OR customBorderColorWithoutFormat)
+    const bool has_any_custom_border_color =
+        features.custom_border_color.customBorderColors ||
+        features.custom_border_color.customBorderColorWithoutFormat;
+    extensions.custom_border_color = has_any_custom_border_color;
     RemoveExtensionFeatureIfUnsuitable(extensions.custom_border_color, features.custom_border_color,
                                        VK_EXT_CUSTOM_BORDER_COLOR_EXTENSION_NAME);
 

src/video_core/vulkan_common/vulkan_device.h

@@ -537,6 +537,16 @@ public:
         return extensions.custom_border_color;
     }
 
+    /// Returns true if customBorderColors feature is enabled (allows VK_BORDER_COLOR_*_CUSTOM_EXT).
+    bool HasCustomBorderColorFeature() const {
+        return features.custom_border_color.customBorderColors;
+    }
+
+    /// Returns true if customBorderColorWithoutFormat feature is enabled (allows VK_FORMAT_UNDEFINED).
+    bool HasCustomBorderColorWithoutFormatFeature() const {
+        return features.custom_border_color.customBorderColorWithoutFormat;
+    }
+
     /// Base Vulkan Dynamic State support checks.
     /// These provide granular control over each base dynamic state, allowing individual states
     /// to be disabled if broken driver implementations are detected at device initialization.