diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index a3949c5ac9..75cfea3644 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -1070,7 +1070,6 @@ void BufferCache

::BindHostGraphicsUniformBuffers(size_t stage) { template void BufferCache

::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 binding_index, bool needs_bind) { - ++channel_state->uniform_cache_shots[0]; const Binding& binding = channel_state->uniform_buffers[stage][index]; const DAddr device_addr = binding.device_addr; const u32 size = (std::min)(binding.size, (*channel_state->uniform_buffer_sizes)[stage][index]); @@ -1089,6 +1088,19 @@ void BufferCache

::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 return alignment > 1 && (offset % alignment) != 0; } }(); + if constexpr (USE_UNIFIED_DIRECT_BINDING && !NEEDS_BIND_UNIFORM_INDEX) { + u64 window_index = 0; + u64 window_offset = 0; + if (runtime.SupportsUnifiedDescriptorBinding() && + size <= runtime.UnifiedMaxUniformBufferRange() && + ResolveUnifiedDirectBinding(device_addr, size, runtime.UnifiedUniformBufferAlignment(), + window_index, window_offset)) { + runtime.BindBufferFromWindow(window_index, static_cast(window_offset), size); + channel_state->fast_bound_uniform_buffers[stage] &= ~(1u << binding_index); + return; + } + } + ++channel_state->uniform_cache_shots[0]; const bool use_fast_buffer = needs_alignment_stream || (has_host_buffer && size <= channel_state->uniform_buffer_skip_cache_size && !memory_tracker.IsRegionGpuModified(device_addr, size)); @@ -1168,8 +1180,7 @@ void BufferCache

::BindHostGraphicsStorageBuffers(size_t stage) { if (is_written) { MarkWrittenBufferInPlace(binding.device_addr, size); } - runtime.BindStorageBufferFromWindow(window_index, static_cast(window_offset), - size); + runtime.BindBufferFromWindow(window_index, static_cast(window_offset), size); return; } } @@ -1198,9 +1209,27 @@ void BufferCache

::BindHostGraphicsTextureBuffers(size_t stage) { const TextureBufferBinding& binding = channel_state->texture_buffers[stage][index]; Buffer& buffer = slot_buffers[binding.buffer_id]; const u32 size = binding.size; + const bool is_written = ((channel_state->written_texture_buffers[stage] >> index) & 1) != 0; + + if constexpr (USE_UNIFIED_DIRECT_BINDING && !SEPARATE_IMAGE_BUFFERS_BINDINGS) { + u64 window_index = 0; + u64 window_offset = 0; + if (runtime.SupportsUnifiedDescriptorBinding() && + ResolveUnifiedDirectBinding(binding.device_addr, size, + runtime.UnifiedTexelBufferAlignment(), window_index, + window_offset) && + runtime.BindTextureBufferFromWindow(window_index, + static_cast(window_offset), size, + binding.format)) { + if (is_written) { + MarkWrittenBufferInPlace(binding.device_addr, size); + } + return; + } + } + SynchronizeBuffer(buffer, binding.device_addr, size); - const bool is_written = ((channel_state->written_texture_buffers[stage] >> index) & 1) != 0; if (is_written) { MarkWrittenBuffer(binding.buffer_id, binding.device_addr, size); } @@ -1323,8 +1352,7 @@ void BufferCache

::BindHostComputeStorageBuffers() { if (is_written) { MarkWrittenBufferInPlace(binding.device_addr, size); } - runtime.BindStorageBufferFromWindow(window_index, static_cast(window_offset), - size); + runtime.BindBufferFromWindow(window_index, static_cast(window_offset), size); return; } } @@ -1353,10 +1381,28 @@ void BufferCache

::BindHostComputeTextureBuffers() { const TextureBufferBinding& binding = channel_state->compute_texture_buffers[index]; Buffer& buffer = slot_buffers[binding.buffer_id]; const u32 size = binding.size; - SynchronizeBuffer(buffer, binding.device_addr, size); - const bool is_written = ((channel_state->written_compute_texture_buffers >> index) & 1) != 0; + + if constexpr (USE_UNIFIED_DIRECT_BINDING && !SEPARATE_IMAGE_BUFFERS_BINDINGS) { + u64 window_index = 0; + u64 window_offset = 0; + if (runtime.SupportsUnifiedDescriptorBinding() && + ResolveUnifiedDirectBinding(binding.device_addr, size, + runtime.UnifiedTexelBufferAlignment(), window_index, + window_offset) && + runtime.BindTextureBufferFromWindow(window_index, + static_cast(window_offset), size, + binding.format)) { + if (is_written) { + MarkWrittenBufferInPlace(binding.device_addr, size); + } + return; + } + } + + SynchronizeBuffer(buffer, binding.device_addr, size); + if (is_written) { MarkWrittenBuffer(binding.buffer_id, binding.device_addr, size); } diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp index f713840d76..ff86648c66 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp @@ -676,6 +676,55 @@ void BufferCacheRuntime::ClearBuffer(VkBuffer dest_buffer, u32 offset, size_t si }); } +VkBufferView BufferCacheRuntime::UnifiedWindowTexelView(size_t window_index, u32 offset, u32 size, + VideoCore::Surface::PixelFormat format) { + const auto it{std::ranges::find_if(unified_window_views, [&](const UnifiedWindowView& view) { + return view.window_index == window_index && view.offset == offset && view.size == size && + view.format == format; + })}; + if (it != unified_window_views.end()) { + return *it->handle; + } + if (unified_window_views.size() >= MAX_UNIFIED_WINDOW_VIEWS) { + return VK_NULL_HANDLE; + } + const VkBuffer window_buffer = UnifiedWindowBuffer(window_index); + if (window_buffer == VK_NULL_HANDLE) { + return VK_NULL_HANDLE; + } + unified_window_views.push_back({ + .window_index = window_index, + .offset = offset, + .size = size, + .format = format, + .handle = device.GetLogical().CreateBufferView({ + .sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .buffer = window_buffer, + .format = TexelBufferFormat(format), + .offset = offset, + .range = size, + }), + }); + return *unified_window_views.back().handle; +} + +bool BufferCacheRuntime::BindTextureBufferFromWindow(size_t window_index, u32 offset, u32 size, + VideoCore::Surface::PixelFormat format) { + if (guest_descriptor_queue.UsesDescriptorBuffer()) { + guest_descriptor_queue.AddTexelBuffer(VK_NULL_HANDLE, UnifiedWindowAddress(window_index), + offset, size, TexelBufferFormat(format)); + return true; + } + const VkBufferView view = UnifiedWindowTexelView(window_index, offset, size, format); + if (view == VK_NULL_HANDLE) { + return false; + } + guest_descriptor_queue.AddTexelBuffer(view); + return true; +} + bool BufferCacheRuntime::CanBindIndexBufferDirectly(PrimitiveTopology topology, IndexFormat index_format) const { if (topology == PrimitiveTopology::Quads || topology == PrimitiveTopology::QuadStrip) { diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index bba84bd951..2790ff1369 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -9,6 +9,7 @@ #include #include #include +#include #include "video_core/buffer_cache/buffer_cache_base.h" #include "video_core/buffer_cache/memory_tracker_base.h" @@ -133,8 +134,20 @@ public: [[nodiscard]] bool CanBindIndexBufferDirectly(PrimitiveTopology topology, IndexFormat index_format) const; + [[nodiscard]] VkDeviceAddress UnifiedWindowAddress(size_t window_index) const noexcept { + if (!unified_memory || !unified_memory->SupportsDirectAddresses() || + window_index >= unified_memory->GetWindowCount()) { + return 0; + } + return unified_memory->GetWindowAddress(window_index); + } + [[nodiscard]] bool SupportsUnifiedDescriptorBinding() const noexcept { - return SupportsUnifiedDirectBinding() && !guest_descriptor_queue.UsesDescriptorBuffer(); + if (!SupportsUnifiedDirectBinding()) { + return false; + } + return !guest_descriptor_queue.UsesDescriptorBuffer() || + unified_memory->SupportsDirectAddresses(); } [[nodiscard]] u64 UnifiedStorageBufferAlignment() const noexcept { @@ -145,10 +158,26 @@ public: return device.GetMaxStorageBufferRange(); } - void BindStorageBufferFromWindow(size_t window_index, u32 offset, u32 size) { - guest_descriptor_queue.AddBuffer(UnifiedWindowBuffer(window_index), offset, size); + [[nodiscard]] u64 UnifiedUniformBufferAlignment() const noexcept { + return device.GetUniformBufferAlignment(); + } + + [[nodiscard]] u64 UnifiedMaxUniformBufferRange() const noexcept { + return device.GetMaxUniformBufferRange(); + } + + [[nodiscard]] u64 UnifiedTexelBufferAlignment() const noexcept { + return device.GetTexelBufferAlignment(); + } + + void BindBufferFromWindow(size_t window_index, u32 offset, u32 size) { + guest_descriptor_queue.AddBuffer(UnifiedWindowBuffer(window_index), + UnifiedWindowAddress(window_index), offset, size); } + bool BindTextureBufferFromWindow(size_t window_index, u32 offset, u32 size, + VideoCore::Surface::PixelFormat format); + void CopyToUnifiedMemory(size_t window_index, VkBuffer src_buffer, std::span copies); @@ -257,6 +286,19 @@ private: void AcquireUnifiedWindowsFromForeign(); + static constexpr size_t MAX_UNIFIED_WINDOW_VIEWS = 256; + + struct UnifiedWindowView { + size_t window_index; + u32 offset; + u32 size; + VideoCore::Surface::PixelFormat format; + vk::BufferView handle; + }; + + [[nodiscard]] VkBufferView UnifiedWindowTexelView(size_t window_index, u32 offset, u32 size, + VideoCore::Surface::PixelFormat format); + void ReserveNullBuffer(); vk::Buffer CreateNullBuffer(); @@ -271,6 +313,7 @@ private: vk::Buffer null_buffer; std::unique_ptr unified_memory; + std::vector unified_window_views; std::unique_ptr uint8_pass; QuadIndexedPass quad_index_pass; diff --git a/src/video_core/vulkan_common/vulkan_device.h b/src/video_core/vulkan_common/vulkan_device.h index fc93dada44..e16b006468 100644 --- a/src/video_core/vulkan_common/vulkan_device.h +++ b/src/video_core/vulkan_common/vulkan_device.h @@ -347,6 +347,16 @@ public: return properties.properties.limits.maxStorageBufferRange; } + /// Returns the maximum range for uniform buffers. + VkDeviceSize GetMaxUniformBufferRange() const { + return properties.properties.limits.maxUniformBufferRange; + } + + /// Returns texel buffer alignment requirement. + VkDeviceSize GetTexelBufferAlignment() const { + return properties.properties.limits.minTexelBufferOffsetAlignment; + } + std::array GetMaxComputeWorkGroupCount() const { const auto& count = properties.properties.limits.maxComputeWorkGroupCount; return {count[0], count[1], count[2]}; diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp index 91430cd8ff..56fdcf5653 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp @@ -72,6 +72,11 @@ namespace Vulkan { return flags; } + [[nodiscard]] VkBufferUsageFlags ImportedDescriptorAddressUsage(const Device &device) { + return ImportedDescriptorUsage(device) | + VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT; + } + // Helpers translating MemoryUsage to flags/usage [[maybe_unused]] VkMemoryPropertyFlags MemoryUsagePropertyFlags(MemoryUsage usage) { @@ -297,7 +302,16 @@ namespace Vulkan { bool HostMemoryImport::CreateDescriptorCapableWindowBuffer( const void *external_info, VkDeviceSize length, u32 external_type_bits, VkDeviceSize capacity, VkBuffer *out_buffer, u32 *out_type_mask, - bool *out_descriptor_capable) const { + bool *out_descriptor_capable, bool *out_address_capable, bool allow_address) const { + if (allow_address && device.IsBufferDeviceAddressSupported() && + TryCreateWindowBuffer(external_info, length, external_type_bits, capacity, + ImportedDescriptorAddressUsage(device), out_buffer, + out_type_mask)) { + *out_descriptor_capable = true; + *out_address_capable = true; + return true; + } + *out_address_capable = false; if (TryCreateWindowBuffer(external_info, length, external_type_bits, capacity, ImportedDescriptorUsage(device), out_buffer, out_type_mask)) { *out_descriptor_capable = true; @@ -443,6 +457,7 @@ namespace Vulkan { window_size = hardware_buffer_window; base_offset = hardware_buffer_base; bool every_window_descriptor_capable = true; + bool every_window_address_capable = true; for (size_t i = 0; i < hardware_buffers.size(); ++i) { const size_t offset = hardware_buffer_base + i * hardware_buffer_window; if (offset >= size) { @@ -468,54 +483,81 @@ namespace Vulkan { .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID, }; - VkBuffer new_buffer{}; - u32 type_mask = 0; + Window window{}; bool descriptor_capable = false; - if (!CreateDescriptorCapableWindowBuffer(&external_info, window_len, - ahb_props.memoryTypeBits, - ahb_props.allocationSize, &new_buffer, - &type_mask, &descriptor_capable)) { - break; - } - const auto type_index = FindImportMemoryType(memory_props, type_mask); - if (!type_index) { - logical.DestroyBufferRaw(new_buffer); - break; - } - const VkImportAndroidHardwareBufferInfoANDROID import_info{ - .sType = VK_STRUCTURE_TYPE_IMPORT_ANDROID_HARDWARE_BUFFER_INFO_ANDROID, - .pNext = nullptr, - .buffer = hardware_buffers[i], - }; - const VkMemoryDedicatedAllocateInfo dedicated_info{ - .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO, - .pNext = &import_info, - .image = VK_NULL_HANDLE, - .buffer = new_buffer, - }; - const VkMemoryAllocateInfo alloc_info{ - .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, - .pNext = &dedicated_info, - .allocationSize = ahb_props.allocationSize, - .memoryTypeIndex = *type_index, + bool address_capable = false; + const auto build_window = [&](bool allow_address) { + VkBuffer new_buffer{}; + u32 type_mask = 0; + if (!CreateDescriptorCapableWindowBuffer(&external_info, window_len, + ahb_props.memoryTypeBits, + ahb_props.allocationSize, &new_buffer, + &type_mask, &descriptor_capable, + &address_capable, allow_address)) { + return false; + } + const auto type_index = FindImportMemoryType(memory_props, type_mask); + if (!type_index) { + logical.DestroyBufferRaw(new_buffer); + return false; + } + const VkImportAndroidHardwareBufferInfoANDROID import_info{ + .sType = VK_STRUCTURE_TYPE_IMPORT_ANDROID_HARDWARE_BUFFER_INFO_ANDROID, + .pNext = nullptr, + .buffer = hardware_buffers[i], + }; + const VkMemoryDedicatedAllocateInfo dedicated_info{ + .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO, + .pNext = &import_info, + .image = VK_NULL_HANDLE, + .buffer = new_buffer, + }; + const VkMemoryAllocateFlagsInfo flags_info{ + .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO, + .pNext = &dedicated_info, + .flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT, + .deviceMask = 0, + }; + const VkMemoryAllocateInfo alloc_info{ + .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, + .pNext = address_capable ? static_cast(&flags_info) + : static_cast(&dedicated_info), + .allocationSize = ahb_props.allocationSize, + .memoryTypeIndex = *type_index, + }; + vk::DeviceMemory memory = logical.TryAllocateMemory(alloc_info); + if (!memory) { + logical.DestroyBufferRaw(new_buffer); + return false; + } + if (logical.BindBufferMemory(new_buffer, *memory, 0) != VK_SUCCESS) { + logical.DestroyBufferRaw(new_buffer); + return false; + } + VkDeviceAddress address = 0; + if (address_capable) { + address = logical.GetBufferDeviceAddress(new_buffer); + if (address == 0) { + address_capable = false; + } + } + window = Window{ + .memory = std::move(memory), + .buffer = new_buffer, + .address = address, + }; + return true; }; - vk::DeviceMemory memory = logical.TryAllocateMemory(alloc_info); - if (!memory) { - logical.DestroyBufferRaw(new_buffer); - break; - } - if (logical.BindBufferMemory(new_buffer, *memory, 0) != VK_SUCCESS) { - logical.DestroyBufferRaw(new_buffer); + if (!build_window(true) && !build_window(false)) { break; } - windows.push_back(Window{ - .memory = std::move(memory), - .buffer = new_buffer, - }); + windows.push_back(std::move(window)); every_window_descriptor_capable &= descriptor_capable; + every_window_address_capable &= address_capable; imported_size += static_cast(window_len); } direct_descriptors = !windows.empty() && every_window_descriptor_capable; + direct_addresses = direct_descriptors && every_window_address_capable; if (windows.empty()) { window_size = 0; base_offset = 0; diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.h b/src/video_core/vulkan_common/vulkan_memory_allocator.h index e7a59a04a3..020531c359 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.h +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.h @@ -130,10 +130,19 @@ namespace Vulkan { return direct_descriptors; } + [[nodiscard]] bool SupportsDirectAddresses() const noexcept { + return direct_addresses; + } + + [[nodiscard]] VkDeviceAddress GetWindowAddress(size_t index) const noexcept { + return windows[index].address; + } + private: struct Window { vk::DeviceMemory memory; VkBuffer buffer{}; + VkDeviceAddress address{}; }; bool ImportHostPointer(void *base, size_t size); @@ -150,7 +159,9 @@ namespace Vulkan { bool CreateDescriptorCapableWindowBuffer(const void *external_info, VkDeviceSize length, u32 external_type_bits, VkDeviceSize capacity, VkBuffer *out_buffer, u32 *out_type_mask, - bool *out_descriptor_capable) const; + bool *out_descriptor_capable, + bool *out_address_capable, + bool allow_address) const; const Device &device; std::vector windows; @@ -159,6 +170,7 @@ namespace Vulkan { size_t base_offset{}; bool foreign_ownership{}; bool direct_descriptors{}; + bool direct_addresses{}; }; /// Memory allocator container.