Another intent on write/read on the fly

src/video_core/buffer_cache/buffer_cache.h

@@ -1085,11 +1085,25 @@ void BufferCache<P>::BindHostGraphicsStorageBuffers(size_t stage) {
         Buffer& buffer = slot_buffers[binding.buffer_id];
         TouchBuffer(buffer, binding.buffer_id);
         const u32 size = binding.size;
+        const bool is_written = ((channel_state->written_storage_buffers[stage] >> index) & 1) != 0;
+
+        if constexpr (USE_UNIFIED_MEMORY) {
+            const auto window = TryResolveUnifiedRange(binding.device_addr, size);
+            if (window && runtime.IsUnifiedStorageRange(size, window->offset)) {
+                if (is_written) {
+                    memory_tracker.MarkRegionAsCpuModified(binding.device_addr, size);
+                }
+                runtime.BindStorageBuffer(runtime.UnifiedWindowBuffer(window->window),
+                                          runtime.UnifiedWindowAddress(window->window),
+                                          static_cast<u32>(window->offset), size, is_written);
+                return;
+            }
+        }
+
         SynchronizeBuffer(buffer, binding.device_addr, size);
 
         const u32 offset = buffer.Offset(binding.device_addr);
         buffer.MarkUsage(offset, size);
-        const bool is_written = ((channel_state->written_storage_buffers[stage] >> index) & 1) != 0;
 
         if (is_written) {
             MarkWrittenBuffer(binding.buffer_id, binding.device_addr, size);
@@ -1842,6 +1856,57 @@ bool BufferCache<P>::ResolveUnifiedWindows(
     }
 }
 
+template <class P>
+std::optional<typename BufferCache<P>::UnifiedWindowRange>
+BufferCache<P>::TryResolveUnifiedRange([[maybe_unused]] DAddr device_addr,
+                                       [[maybe_unused]] u64 size) {
+    if constexpr (USE_UNIFIED_MEMORY) {
+        if (size == 0 || !runtime.IsUnifiedMemoryBindable()) {
+            return std::nullopt;
+        }
+        const u64 window_size = runtime.UnifiedMemoryWindowSize();
+        if (window_size == 0) {
+            return std::nullopt;
+        }
+        const u8* const first = device_memory.GetPointer<u8>(device_addr);
+        if (first == nullptr) {
+            return std::nullopt;
+        }
+        const u64 phys_offset = static_cast<u64>(first - device_memory.GetPhysicalBase());
+        const u64 unified_base = runtime.UnifiedMemoryBase();
+        if (phys_offset < unified_base) {
+            return std::nullopt;
+        }
+        const u64 relative = phys_offset - unified_base;
+        const u64 unified_size = runtime.UnifiedMemorySize();
+        if (relative >= unified_size || unified_size - relative < size) {
+            return std::nullopt;
+        }
+        const u64 local_offset = relative % window_size;
+        if (window_size - local_offset < size) {
+            return std::nullopt;
+        }
+        if (memory_tracker.IsRegionGpuModified(device_addr, size) ||
+            IsRegionGpuModified(device_addr, size)) {
+            return std::nullopt;
+        }
+        u64 walked = Core::DEVICE_PAGESIZE - (device_addr & Core::DEVICE_PAGEMASK);
+        while (walked < size) {
+            const u8* const next = device_memory.GetPointer<u8>(device_addr + walked);
+            if (next != first + walked) {
+                return std::nullopt;
+            }
+            walked += Core::DEVICE_PAGESIZE;
+        }
+        return UnifiedWindowRange{
+            .window = static_cast<size_t>(relative / window_size),
+            .offset = local_offset,
+        };
+    } else {
+        return std::nullopt;
+    }
+}
+
 template <class P>
 bool BufferCache<P>::TryUnifiedDownloadMemory([[maybe_unused]] Buffer& buffer,
                                               [[maybe_unused]] std::span<BufferCopy> copies) {

src/video_core/buffer_cache/buffer_cache_base.h

@@ -12,6 +12,7 @@
 #include <functional>
 #include <memory>
 #include <mutex>
+#include <optional>
 #include <numeric>
 #include <span>
 #include <vector>
@@ -446,6 +447,13 @@ private:
 
     bool TryUnifiedDownloadMemory(Buffer& buffer, std::span<BufferCopy> copies);
 
+    struct UnifiedWindowRange {
+        size_t window;
+        u64 offset;
+    };
+
+    std::optional<UnifiedWindowRange> TryResolveUnifiedRange(DAddr device_addr, u64 size);
+
     using UnifiedWindowGroups =
         boost::container::small_vector<boost::container::small_vector<BufferCopy, 16>, 4>;
 

src/video_core/renderer_vulkan/vk_buffer_cache.h

@@ -195,6 +195,29 @@ public:
         BindBuffer(buffer, offset, size);
     }
 
+    void BindStorageBuffer(VkBuffer buffer, VkDeviceAddress address, u32 offset, u32 size,
+                           [[maybe_unused]] bool is_written) {
+        guest_descriptor_queue.AddBuffer(buffer, address, offset, size);
+    }
+
+    [[nodiscard]] bool IsUnifiedMemoryBindable() const noexcept {
+        return unified_memory != nullptr && unified_memory->IsValid() &&
+               unified_memory->IsBindable();
+    }
+
+    [[nodiscard]] VkBuffer UnifiedWindowBuffer(size_t index) const noexcept {
+        return unified_memory->GetWindowBuffer(index);
+    }
+
+    [[nodiscard]] VkDeviceAddress UnifiedWindowAddress(size_t index) const noexcept {
+        return unified_memory->GetWindowAddress(index);
+    }
+
+    [[nodiscard]] bool IsUnifiedStorageRange(u32 size, u64 offset) const {
+        return size <= device.GetMaxStorageBufferRange() &&
+               (offset % device.GetStorageBufferAlignment()) == 0;
+    }
+
     void BindTextureBuffer(Buffer& buffer, u32 offset, u32 size,
                            VideoCore::Surface::PixelFormat format) {
         guest_descriptor_queue.AddTexelBuffer(buffer.View(offset, size, format),

src/video_core/vulkan_common/vulkan_device.h

@@ -341,7 +341,11 @@ public:
         return properties.properties.limits.minUniformBufferOffsetAlignment;
     }
 
-    /// Returns storage alignment requirement.
+    /// Returns max storage buffer range requirement.
+    u32 GetMaxStorageBufferRange() const {
+        return properties.properties.limits.maxStorageBufferRange;
+    }
+
     VkDeviceSize GetStorageBufferAlignment() const {
         return properties.properties.limits.minStorageBufferOffsetAlignment;
     }

src/video_core/vulkan_common/vulkan_memory_allocator.cpp

@@ -379,88 +379,125 @@ namespace Vulkan {
         const auto memory_props = device.GetPhysical().GetMemoryProperties().memoryProperties;
         window_size = hardware_buffer_window;
         base_offset = hardware_buffer_base;
-        for (size_t i = 0; i < hardware_buffers.size(); ++i) {
-            const size_t offset = hardware_buffer_base + i * hardware_buffer_window;
-            if (offset >= size) {
-                break;
-            }
-            const VkDeviceSize window_len = (std::min)(
-                    static_cast<VkDeviceSize>(size - offset),
-                    static_cast<VkDeviceSize>(hardware_buffer_window));
-            VkAndroidHardwareBufferPropertiesANDROID ahb_props{
-                    .sType = VK_STRUCTURE_TYPE_ANDROID_HARDWARE_BUFFER_PROPERTIES_ANDROID,
-                    .pNext = nullptr,
-                    .allocationSize = 0,
-                    .memoryTypeBits = 0,
-            };
-            if (logical.GetAndroidHardwareBufferPropertiesANDROID(hardware_buffers[i],
-                                                                  &ahb_props) != VK_SUCCESS ||
-                ahb_props.memoryTypeBits == 0 || ahb_props.allocationSize < window_len) {
-                break;
-            }
-            const VkExternalMemoryBufferCreateInfo external_info{
-                    .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO,
-                    .pNext = nullptr,
-                    .handleTypes =
-                            VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID,
-            };
-            const VkBufferCreateInfo buffer_ci{
-                    .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
-                    .pNext = &external_info,
-                    .flags = 0,
-                    .size = window_len,
-                    .usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT,
-                    .sharingMode = VK_SHARING_MODE_EXCLUSIVE,
-                    .queueFamilyIndexCount = 0,
-                    .pQueueFamilyIndices = nullptr,
-            };
-            VkBuffer new_buffer{};
-            if (logical.CreateBufferRaw(buffer_ci, &new_buffer) != VK_SUCCESS) {
-                break;
-            }
-            const VkMemoryRequirements requirements =
-                    logical.GetBufferMemoryRequirements(new_buffer);
-            const u32 type_mask = requirements.memoryTypeBits & ahb_props.memoryTypeBits;
-            if (type_mask == 0 || requirements.size > ahb_props.allocationSize) {
-                logical.DestroyBufferRaw(new_buffer);
-                break;
-            }
-            const auto type_index = FindImportMemoryType(memory_props, type_mask);
-            if (!type_index) {
-                logical.DestroyBufferRaw(new_buffer);
-                break;
-            }
-            const VkImportAndroidHardwareBufferInfoANDROID import_info{
-                    .sType = VK_STRUCTURE_TYPE_IMPORT_ANDROID_HARDWARE_BUFFER_INFO_ANDROID,
-                    .pNext = nullptr,
-                    .buffer = hardware_buffers[i],
-            };
-            const VkMemoryDedicatedAllocateInfo dedicated_info{
-                    .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO,
-                    .pNext = &import_info,
-                    .image = VK_NULL_HANDLE,
-                    .buffer = new_buffer,
-            };
-            const VkMemoryAllocateInfo alloc_info{
-                    .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
-                    .pNext = &dedicated_info,
-                    .allocationSize = ahb_props.allocationSize,
-                    .memoryTypeIndex = *type_index,
-            };
-            vk::DeviceMemory memory = logical.TryAllocateMemory(alloc_info);
-            if (!memory) {
-                logical.DestroyBufferRaw(new_buffer);
-                break;
-            }
-            if (logical.BindBufferMemory(new_buffer, *memory, 0) != VK_SUCCESS) {
-                logical.DestroyBufferRaw(new_buffer);
-                break;
+
+        const auto import_all = [&](VkBufferUsageFlags usage, bool want_address) {
+            for (size_t i = 0; i < hardware_buffers.size(); ++i) {
+                const size_t offset = hardware_buffer_base + i * hardware_buffer_window;
+                if (offset >= size) {
+                    break;
+                }
+                const VkDeviceSize window_len = (std::min)(
+                        static_cast<VkDeviceSize>(size - offset),
+                        static_cast<VkDeviceSize>(hardware_buffer_window));
+                VkAndroidHardwareBufferPropertiesANDROID ahb_props{
+                        .sType = VK_STRUCTURE_TYPE_ANDROID_HARDWARE_BUFFER_PROPERTIES_ANDROID,
+                        .pNext = nullptr,
+                        .allocationSize = 0,
+                        .memoryTypeBits = 0,
+                };
+                if (logical.GetAndroidHardwareBufferPropertiesANDROID(hardware_buffers[i],
+                                                                      &ahb_props) != VK_SUCCESS ||
+                    ahb_props.memoryTypeBits == 0 || ahb_props.allocationSize < window_len) {
+                    break;
+                }
+                const VkExternalMemoryBufferCreateInfo external_info{
+                        .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO,
+                        .pNext = nullptr,
+                        .handleTypes =
+                                VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID,
+                };
+                const VkBufferCreateInfo buffer_ci{
+                        .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
+                        .pNext = &external_info,
+                        .flags = 0,
+                        .size = window_len,
+                        .usage = usage,
+                        .sharingMode = VK_SHARING_MODE_EXCLUSIVE,
+                        .queueFamilyIndexCount = 0,
+                        .pQueueFamilyIndices = nullptr,
+                };
+                VkBuffer new_buffer{};
+                if (logical.CreateBufferRaw(buffer_ci, &new_buffer) != VK_SUCCESS) {
+                    break;
+                }
+                const VkMemoryRequirements requirements =
+                        logical.GetBufferMemoryRequirements(new_buffer);
+                const u32 type_mask = requirements.memoryTypeBits & ahb_props.memoryTypeBits;
+                if (type_mask == 0 || requirements.size > ahb_props.allocationSize) {
+                    logical.DestroyBufferRaw(new_buffer);
+                    break;
+                }
+                const auto type_index = FindImportMemoryType(memory_props, type_mask);
+                if (!type_index) {
+                    logical.DestroyBufferRaw(new_buffer);
+                    break;
+                }
+                const VkImportAndroidHardwareBufferInfoANDROID import_info{
+                        .sType = VK_STRUCTURE_TYPE_IMPORT_ANDROID_HARDWARE_BUFFER_INFO_ANDROID,
+                        .pNext = nullptr,
+                        .buffer = hardware_buffers[i],
+                };
+                const VkMemoryDedicatedAllocateInfo dedicated_info{
+                        .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO,
+                        .pNext = &import_info,
+                        .image = VK_NULL_HANDLE,
+                        .buffer = new_buffer,
+                };
+                const VkMemoryAllocateFlagsInfo flags_info{
+                        .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO,
+                        .pNext = &dedicated_info,
+                        .flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT,
+                        .deviceMask = 0,
+                };
+                const void *alloc_next = &dedicated_info;
+                if (want_address) {
+                    alloc_next = &flags_info;
+                }
+                const VkMemoryAllocateInfo alloc_info{
+                        .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
+                        .pNext = alloc_next,
+                        .allocationSize = ahb_props.allocationSize,
+                        .memoryTypeIndex = *type_index,
+                };
+                vk::DeviceMemory memory = logical.TryAllocateMemory(alloc_info);
+                if (!memory) {
+                    logical.DestroyBufferRaw(new_buffer);
+                    break;
+                }
+                if (logical.BindBufferMemory(new_buffer, *memory, 0) != VK_SUCCESS) {
+                    logical.DestroyBufferRaw(new_buffer);
+                    break;
+                }
+                VkDeviceAddress address = 0;
+                if (want_address) {
+                    address = logical.GetBufferDeviceAddress(new_buffer);
+                }
+                windows.push_back(Window{
+                        .memory = std::move(memory),
+                        .buffer = new_buffer,
+                        .address = address,
+                });
+                imported_size += static_cast<size_t>(window_len);
             }
-            windows.push_back(Window{
-                    .memory = std::move(memory),
-                    .buffer = new_buffer,
-            });
-            imported_size += static_cast<size_t>(window_len);
+            return !windows.empty();
+        };
+
+        constexpr VkBufferUsageFlags TransferUsage =
+                VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
+        VkBufferUsageFlags shader_usage = TransferUsage |
+                VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT |
+                VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT | VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT |
+                VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT |
+                VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT;
+        const bool want_address = device.IsBufferDeviceAddressSupported();
+        if (want_address) {
+            shader_usage |= VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT;
+        }
+
+        bindable = import_all(shader_usage, want_address);
+        if (!bindable) {
+            imported_size = 0;
+            import_all(TransferUsage, false);
         }
         if (windows.empty()) {
             window_size = 0;

src/video_core/vulkan_common/vulkan_memory_allocator.h

@@ -122,14 +122,23 @@ namespace Vulkan {
             return windows[index].buffer;
         }
 
+        [[nodiscard]] VkDeviceAddress GetWindowAddress(size_t index) const noexcept {
+            return windows[index].address;
+        }
+
         [[nodiscard]] size_t GetWindowCount() const noexcept {
             return windows.size();
         }
 
+        [[nodiscard]] bool IsBindable() const noexcept {
+            return bindable;
+        }
+
     private:
         struct Window {
             vk::DeviceMemory memory;
             VkBuffer buffer{};
+            VkDeviceAddress address{};
         };
 
         bool ImportHostPointer(void *base, size_t size);
@@ -144,6 +153,7 @@ namespace Vulkan {
         size_t imported_size{};
         size_t base_offset{};
         bool foreign_ownership{};
+        bool bindable{};
     };
 
 /// Memory allocator container.