From 7a98e77d57f1360c64376a76b664fa0e4fcfd48f Mon Sep 17 00:00:00 2001
From: CamilleLaVey <camillelavey99@gmail.com>
Date: Sun, 23 Nov 2025 21:40:07 -0400
Subject: [PATCH] [vk] Adjusting Custom Border Color

---
 .../renderer_vulkan/vk_texture_cache.cpp      | 20 +++++++++++++------
 .../vulkan_common/vulkan_device.cpp           | 14 +++++++++++--
 src/video_core/vulkan_common/vulkan_device.h  | 10 ++++++++++
 3 files changed, 36 insertions(+), 8 deletions(-)

diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp
index f821c71b7b..d1d39d5ef3 100644
--- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp
+++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp
@@ -2222,18 +2222,26 @@ vk::ImageView ImageView::MakeView(VkFormat vk_format, VkImageAspectFlags aspect_
 
 Sampler::Sampler(TextureCacheRuntime& runtime, const Tegra::Texture::TSCEntry& tsc) {
     const auto& device = runtime.device;
-    const bool arbitrary_borders = runtime.device.IsExtCustomBorderColorSupported();
+    // Check if custom border colors are supported
+    const bool has_custom_border_colors = runtime.device.IsCustomBorderColorsSupported();
+    const bool has_format_undefined = runtime.device.IsCustomBorderColorWithoutFormatSupported();
     const auto color = tsc.BorderColor();
 
+    // Determine border format based on available features:
+    // - If customBorderColorWithoutFormat is available: use VK_FORMAT_UNDEFINED (most flexible)
+    // - If only customBorderColors is available: use concrete format (R8G8B8A8_UNORM)
+    // - If neither is available: use standard border colors (handled by ConvertBorderColor)
+    const VkFormat border_format = has_format_undefined ? VK_FORMAT_UNDEFINED
+                                                        : VK_FORMAT_R8G8B8A8_UNORM;
+
     const VkSamplerCustomBorderColorCreateInfoEXT border_ci{
         .sType = VK_STRUCTURE_TYPE_SAMPLER_CUSTOM_BORDER_COLOR_CREATE_INFO_EXT,
         .pNext = nullptr,
-        // TODO: Make use of std::bit_cast once libc++ supports it.
         .customBorderColor = std::bit_cast<VkClearColorValue>(color),
-        .format = VK_FORMAT_UNDEFINED,
+        .format = border_format,
     };
     const void* pnext = nullptr;
-    if (arbitrary_borders) {
+    if (has_custom_border_colors) {
         pnext = &border_ci;
     }
     const VkSamplerReductionModeCreateInfoEXT reduction_ci{
@@ -2267,8 +2275,8 @@ Sampler::Sampler(TextureCacheRuntime& runtime, const Tegra::Texture::TSCEntry& t
             .compareOp = MaxwellToVK::Sampler::DepthCompareFunction(tsc.depth_compare_func),
             .minLod = tsc.mipmap_filter == TextureMipmapFilter::None ? 0.0f : tsc.MinLod(),
             .maxLod = tsc.mipmap_filter == TextureMipmapFilter::None ? 0.25f : tsc.MaxLod(),
-            .borderColor =
-                arbitrary_borders ? VK_BORDER_COLOR_FLOAT_CUSTOM_EXT : ConvertBorderColor(color),
+            .borderColor = has_custom_border_colors ? VK_BORDER_COLOR_FLOAT_CUSTOM_EXT
+                                                    : ConvertBorderColor(color),
             .unnormalizedCoordinates = VK_FALSE,
         });
     };
diff --git a/src/video_core/vulkan_common/vulkan_device.cpp b/src/video_core/vulkan_common/vulkan_device.cpp
index 95283b746f..5eccf690cf 100644
--- a/src/video_core/vulkan_common/vulkan_device.cpp
+++ b/src/video_core/vulkan_common/vulkan_device.cpp
@@ -1170,8 +1170,18 @@ bool Device::GetSuitability(bool requires_swapchain) {
 
 void Device::RemoveUnsuitableExtensions() {
     // VK_EXT_custom_border_color
-    extensions.custom_border_color = features.custom_border_color.customBorderColors &&
-                                     features.custom_border_color.customBorderColorWithoutFormat;
+    // Enable extension if driver supports it, then check individual features
+    // - customBorderColors: Required to use VK_BORDER_COLOR_FLOAT_CUSTOM_EXT
+    // - customBorderColorWithoutFormat: Optional, allows VK_FORMAT_UNDEFINED
+    // If only customBorderColors is available, we must provide a specific format
+    if (extensions.custom_border_color) {
+        // Verify that at least customBorderColors is available
+        if (!features.custom_border_color.customBorderColors) {
+            LOG_WARNING(Render_Vulkan,
+                        "VK_EXT_custom_border_color reported but customBorderColors feature not available, disabling");
+            extensions.custom_border_color = false;
+        }
+    }
     RemoveExtensionFeatureIfUnsuitable(extensions.custom_border_color, features.custom_border_color,
                                        VK_EXT_CUSTOM_BORDER_COLOR_EXTENSION_NAME);
 
diff --git a/src/video_core/vulkan_common/vulkan_device.h b/src/video_core/vulkan_common/vulkan_device.h
index 26136a6380..a8f40653bd 100644
--- a/src/video_core/vulkan_common/vulkan_device.h
+++ b/src/video_core/vulkan_common/vulkan_device.h
@@ -535,6 +535,16 @@ public:
         return extensions.custom_border_color;
     }
 
+    /// Returns true if customBorderColors feature is available (required for custom colors).
+    bool IsCustomBorderColorsSupported() const {
+        return features.custom_border_color.customBorderColors;
+    }
+
+    /// Returns true if customBorderColorWithoutFormat feature is available (allows VK_FORMAT_UNDEFINED).
+    bool IsCustomBorderColorWithoutFormatSupported() const {
+        return features.custom_border_color.customBorderColorWithoutFormat;
+    }
+
     /// Returns true if the device supports VK_EXT_extended_dynamic_state.
     bool IsExtExtendedDynamicStateSupported() const {
         return extensions.extended_dynamic_state;