diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 0db44953fe..32c2066363 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -820,6 +820,22 @@ void BufferCache

::BindHostIndexBuffer() { const u32 size = channel_state->index_buffer.size; const auto& draw_state = maxwell3d->draw_manager.draw_state; if (draw_state.inline_index_draw_indexes.empty()) { + if constexpr (USE_UNIFIED_DIRECT_BINDING && !HAS_FULL_INDEX_AND_PRIMITIVE_SUPPORT) { + u64 window_index = 0; + u64 window_offset = 0; + if (runtime.CanBindIndexBufferDirectly(draw_state.topology, + draw_state.index_buffer.format) && + ResolveUnifiedDirectBinding(channel_state->index_buffer.device_addr, size, + draw_state.index_buffer.FormatSizeInBytes(), + window_index, window_offset)) { + runtime.BindIndexBuffer(draw_state.topology, draw_state.index_buffer.format, + draw_state.index_buffer.first, + draw_state.index_buffer.count, + runtime.UnifiedWindowBuffer(window_index), + static_cast(window_offset), size); + return; + } + } SynchronizeBuffer(buffer, channel_state->index_buffer.device_addr, size); } else { if constexpr (USE_MEMORY_MAPS_FOR_UPLOADS) { @@ -905,13 +921,30 @@ void BufferCache

::BindHostVertexBuffers() { const Binding& binding = VertexBufferSlot(index); Buffer& buffer = slot_buffers[binding.buffer_id]; TouchBuffer(buffer, binding.buffer_id); - SynchronizeBuffer(buffer, binding.device_addr, binding.size); + u64 window_index = 0; + u64 window_offset = 0; + const bool bind_from_window = + ResolveUnifiedDirectBinding(binding.device_addr, binding.size, + UNIFIED_VERTEX_BINDING_ALIGNMENT, window_index, + window_offset); + if (!bind_from_window) { + SynchronizeBuffer(buffer, binding.device_addr, binding.size); + } if (!flags[Dirty::VertexBuffer0 + index]) { flush_bindings(); continue; } flags[Dirty::VertexBuffer0 + index] = false; const u32 stride = maxwell3d->regs.vertex_streams[index].stride; + if constexpr (USE_UNIFIED_DIRECT_BINDING) { + if (bind_from_window) { + flush_bindings(); + runtime.BindVertexBuffer(index, runtime.UnifiedWindowBuffer(window_index), + static_cast(window_offset), binding.size, + stride); + continue; + } + } const u32 offset = buffer.Offset(binding.device_addr); buffer.MarkUsage(offset, binding.size); if (!bindings.buffers.empty() && index != last_index + 1) { @@ -1084,11 +1117,26 @@ void BufferCache

::BindHostGraphicsStorageBuffers(size_t stage) { Buffer& buffer = slot_buffers[binding.buffer_id]; TouchBuffer(buffer, binding.buffer_id); const u32 size = binding.size; + const bool is_written = ((channel_state->written_storage_buffers[stage] >> index) & 1) != 0; + + if constexpr (USE_UNIFIED_DIRECT_BINDING && !NEEDS_BIND_STORAGE_INDEX) { + u64 window_index = 0; + u64 window_offset = 0; + if (!is_written && runtime.SupportsUnifiedDescriptorBinding() && + size <= runtime.UnifiedMaxStorageBufferRange() && + ResolveUnifiedDirectBinding(binding.device_addr, size, + runtime.UnifiedStorageBufferAlignment(), window_index, + window_offset)) { + runtime.BindStorageBufferFromWindow(window_index, static_cast(window_offset), + size); + return; + } + } + SynchronizeBuffer(buffer, binding.device_addr, size); const u32 offset = buffer.Offset(binding.device_addr); buffer.MarkUsage(offset, size); - const bool is_written = ((channel_state->written_storage_buffers[stage] >> index) & 1) != 0; if (is_written) { MarkWrittenBuffer(binding.buffer_id, binding.device_addr, size); @@ -1220,12 +1268,27 @@ void BufferCache

::BindHostComputeStorageBuffers() { Buffer& buffer = slot_buffers[binding.buffer_id]; TouchBuffer(buffer, binding.buffer_id); const u32 size = binding.size; + const bool is_written = + ((channel_state->written_compute_storage_buffers >> index) & 1) != 0; + + if constexpr (USE_UNIFIED_DIRECT_BINDING && !NEEDS_BIND_STORAGE_INDEX) { + u64 window_index = 0; + u64 window_offset = 0; + if (!is_written && runtime.SupportsUnifiedDescriptorBinding() && + size <= runtime.UnifiedMaxStorageBufferRange() && + ResolveUnifiedDirectBinding(binding.device_addr, size, + runtime.UnifiedStorageBufferAlignment(), window_index, + window_offset)) { + runtime.BindStorageBufferFromWindow(window_index, static_cast(window_offset), + size); + return; + } + } + SynchronizeBuffer(buffer, binding.device_addr, size); const u32 offset = buffer.Offset(binding.device_addr); buffer.MarkUsage(offset, size); - const bool is_written = - ((channel_state->written_compute_storage_buffers >> index) & 1) != 0; if (is_written) { MarkWrittenBuffer(binding.buffer_id, binding.device_addr, size); @@ -1780,6 +1843,62 @@ void BufferCache

::ImmediateUploadMemory([[maybe_unused]] Buffer& buffer, } } +template +bool BufferCache

::ResolveUnifiedDirectBinding([[maybe_unused]] DAddr device_addr, + [[maybe_unused]] u64 size, + [[maybe_unused]] u64 alignment, + [[maybe_unused]] u64& window_index, + [[maybe_unused]] u64& window_offset) { + if constexpr (USE_UNIFIED_DIRECT_BINDING) { + if (size == 0 || !runtime.SupportsUnifiedDirectBinding()) { + return false; + } + const u64 window_size = runtime.UnifiedMemoryWindowSize(); + if (window_size == 0) { + return false; + } + const u8* const physical_base = device_memory.GetPhysicalBase(); + const u8* const host_ptr = device_memory.GetPointer(device_addr); + if (host_ptr == nullptr) { + return false; + } + const u64 phys_offset = static_cast(host_ptr - physical_base); + const u64 unified_base = runtime.UnifiedMemoryBase(); + if (phys_offset < unified_base) { + return false; + } + const u64 relative = phys_offset - unified_base; + if (relative + size > runtime.UnifiedMemorySize()) { + return false; + } + const u64 local_offset = relative % window_size; + if (local_offset + size > window_size) { + return false; + } + if (alignment != 0 && (local_offset & (alignment - 1)) != 0) { + return false; + } + u64 contiguous = (std::min)(size, static_cast(Core::DEVICE_PAGESIZE) - + (device_addr & Core::DEVICE_PAGEMASK)); + while (contiguous < size) { + const u8* const next = device_memory.GetPointer(device_addr + contiguous); + if (next == nullptr || + static_cast(next - physical_base) != phys_offset + contiguous) { + return false; + } + contiguous += Core::DEVICE_PAGESIZE; + } + if (IsRegionGpuModified(device_addr, size)) { + return false; + } + window_index = relative / window_size; + window_offset = local_offset; + return true; + } else { + return false; + } +} + template bool BufferCache

::ResolveUnifiedWindows( [[maybe_unused]] DAddr device_addr, [[maybe_unused]] u64 buffer_offset, diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index bb407694f5..e30f768bb5 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -182,6 +182,8 @@ class BufferCache : public VideoCommon::ChannelSetupCaches& window_ids, UnifiedWindowGroups& groups); + bool ResolveUnifiedDirectBinding(DAddr device_addr, u64 size, u64 alignment, + u64& window_index, u64& window_offset); + void DownloadBufferMemory(Buffer& buffer_id); void DownloadBufferMemory(Buffer& buffer_id, DAddr device_addr, u64 size); diff --git a/src/video_core/renderer_opengl/gl_buffer_cache.h b/src/video_core/renderer_opengl/gl_buffer_cache.h index bd1902ca63..8532a67f76 100644 --- a/src/video_core/renderer_opengl/gl_buffer_cache.h +++ b/src/video_core/renderer_opengl/gl_buffer_cache.h @@ -263,6 +263,7 @@ struct BufferCacheParams { static constexpr bool USE_MEMORY_MAPS_FOR_UPLOADS = false; static constexpr bool USE_UNIFIED_MEMORY = false; static constexpr bool USE_UNIFIED_UPLOADS = false; + static constexpr bool USE_UNIFIED_DIRECT_BINDING = false; }; using BufferCache = VideoCommon::BufferCache; diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp index a2d7f80aef..5c54267857 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp @@ -707,6 +707,15 @@ void BufferCacheRuntime::ClearBuffer(VkBuffer dest_buffer, u32 offset, size_t si }); } +bool BufferCacheRuntime::CanBindIndexBufferDirectly(PrimitiveTopology topology, + IndexFormat index_format) const { + if (topology == PrimitiveTopology::Quads || topology == PrimitiveTopology::QuadStrip) { + return false; + } + return MaxwellToVK::IndexFormat(index_format) != VK_INDEX_TYPE_UINT8_EXT || + device.IsExtIndexTypeUint8Supported(); +} + void BufferCacheRuntime::BindIndexBuffer(PrimitiveTopology topology, IndexFormat index_format, u32 base_vertex, u32 num_indices, VkBuffer buffer, u32 offset, [[maybe_unused]] u32 size) { diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index a4dd0362ee..e1e4039bbb 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -119,6 +119,36 @@ public: return unified_memory ? unified_memory->GetWindowSize() : 0; } + [[nodiscard]] bool SupportsUnifiedDirectBinding() const noexcept { + return unified_memory != nullptr && unified_memory->SupportsDirectDescriptors(); + } + + [[nodiscard]] VkBuffer UnifiedWindowBuffer(size_t window_index) const noexcept { + if (!unified_memory || window_index >= unified_memory->GetWindowCount()) { + return VK_NULL_HANDLE; + } + return unified_memory->GetWindowBuffer(window_index); + } + + [[nodiscard]] bool CanBindIndexBufferDirectly(PrimitiveTopology topology, + IndexFormat index_format) const; + + [[nodiscard]] bool SupportsUnifiedDescriptorBinding() const noexcept { + return SupportsUnifiedDirectBinding() && !guest_descriptor_queue.UsesDescriptorBuffer(); + } + + [[nodiscard]] u64 UnifiedStorageBufferAlignment() const noexcept { + return device.GetStorageBufferAlignment(); + } + + [[nodiscard]] u64 UnifiedMaxStorageBufferRange() const noexcept { + return device.GetMaxStorageBufferRange(); + } + + void BindStorageBufferFromWindow(size_t window_index, u32 offset, u32 size) { + guest_descriptor_queue.AddBuffer(UnifiedWindowBuffer(window_index), offset, size); + } + void CopyToUnifiedMemory(size_t window_index, VkBuffer src_buffer, std::span copies); @@ -261,6 +291,7 @@ struct BufferCacheParams { static constexpr bool USE_MEMORY_MAPS_FOR_UPLOADS = true; static constexpr bool USE_UNIFIED_MEMORY = true; static constexpr bool USE_UNIFIED_UPLOADS = true; + static constexpr bool USE_UNIFIED_DIRECT_BINDING = true; }; using BufferCache = VideoCommon::BufferCache; diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp index 4f205938c9..91430cd8ff 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp @@ -53,6 +53,25 @@ namespace Vulkan { return type_index; } + constexpr VkBufferUsageFlags ImportedTransferUsage = + VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + + [[nodiscard]] VkBufferUsageFlags ImportedDescriptorUsage(const Device &device) { + VkBufferUsageFlags flags = + ImportedTransferUsage | VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT | + VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT | + VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | + VK_BUFFER_USAGE_INDEX_BUFFER_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | + VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT; + if (device.IsExtTransformFeedbackSupported()) { + flags |= VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_BUFFER_BIT_EXT; + } + if (device.IsExtConditionalRendering()) { + flags |= VK_BUFFER_USAGE_CONDITIONAL_RENDERING_BIT_EXT; + } + return flags; + } + // Helpers translating MemoryUsage to flags/usage [[maybe_unused]] VkMemoryPropertyFlags MemoryUsagePropertyFlags(MemoryUsage usage) { @@ -245,6 +264,50 @@ namespace Vulkan { LOG_INFO(Render_Vulkan, "Unified memory disabled, no host memory import path"); } + bool HostMemoryImport::TryCreateWindowBuffer(const void *external_info, VkDeviceSize length, + u32 external_type_bits, VkDeviceSize capacity, + VkBufferUsageFlags usage, VkBuffer *out_buffer, + u32 *out_type_mask) const { + const VkBufferCreateInfo buffer_ci{ + .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, + .pNext = external_info, + .flags = 0, + .size = length, + .usage = usage, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, + .queueFamilyIndexCount = 0, + .pQueueFamilyIndices = nullptr, + }; + const auto &logical = device.GetLogical(); + VkBuffer new_buffer{}; + if (logical.CreateBufferRaw(buffer_ci, &new_buffer) != VK_SUCCESS) { + return false; + } + const VkMemoryRequirements requirements = logical.GetBufferMemoryRequirements(new_buffer); + const u32 type_mask = requirements.memoryTypeBits & external_type_bits; + if (type_mask == 0 || requirements.size > capacity) { + logical.DestroyBufferRaw(new_buffer); + return false; + } + *out_buffer = new_buffer; + *out_type_mask = type_mask; + return true; + } + + bool HostMemoryImport::CreateDescriptorCapableWindowBuffer( + const void *external_info, VkDeviceSize length, u32 external_type_bits, + VkDeviceSize capacity, VkBuffer *out_buffer, u32 *out_type_mask, + bool *out_descriptor_capable) const { + if (TryCreateWindowBuffer(external_info, length, external_type_bits, capacity, + ImportedDescriptorUsage(device), out_buffer, out_type_mask)) { + *out_descriptor_capable = true; + return true; + } + *out_descriptor_capable = false; + return TryCreateWindowBuffer(external_info, length, external_type_bits, capacity, + ImportedTransferUsage, out_buffer, out_type_mask); + } + bool HostMemoryImport::ImportHostPointer(void *base, size_t size) { if (!device.IsExtExternalMemoryHostSupported()) { return false; @@ -299,7 +362,7 @@ namespace Vulkan { .pNext = &external_info, .flags = 0, .size = window_len, - .usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT, + .usage = ImportedTransferUsage, .sharingMode = VK_SHARING_MODE_EXCLUSIVE, .queueFamilyIndexCount = 0, .pQueueFamilyIndices = nullptr, @@ -379,6 +442,7 @@ namespace Vulkan { const auto memory_props = device.GetPhysical().GetMemoryProperties().memoryProperties; window_size = hardware_buffer_window; base_offset = hardware_buffer_base; + bool every_window_descriptor_capable = true; for (size_t i = 0; i < hardware_buffers.size(); ++i) { const size_t offset = hardware_buffer_base + i * hardware_buffer_window; if (offset >= size) { @@ -404,25 +468,13 @@ namespace Vulkan { .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID, }; - const VkBufferCreateInfo buffer_ci{ - .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, - .pNext = &external_info, - .flags = 0, - .size = window_len, - .usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT, - .sharingMode = VK_SHARING_MODE_EXCLUSIVE, - .queueFamilyIndexCount = 0, - .pQueueFamilyIndices = nullptr, - }; VkBuffer new_buffer{}; - if (logical.CreateBufferRaw(buffer_ci, &new_buffer) != VK_SUCCESS) { - break; - } - const VkMemoryRequirements requirements = - logical.GetBufferMemoryRequirements(new_buffer); - const u32 type_mask = requirements.memoryTypeBits & ahb_props.memoryTypeBits; - if (type_mask == 0 || requirements.size > ahb_props.allocationSize) { - logical.DestroyBufferRaw(new_buffer); + u32 type_mask = 0; + bool descriptor_capable = false; + if (!CreateDescriptorCapableWindowBuffer(&external_info, window_len, + ahb_props.memoryTypeBits, + ahb_props.allocationSize, &new_buffer, + &type_mask, &descriptor_capable)) { break; } const auto type_index = FindImportMemoryType(memory_props, type_mask); @@ -460,8 +512,10 @@ namespace Vulkan { .memory = std::move(memory), .buffer = new_buffer, }); + every_window_descriptor_capable &= descriptor_capable; imported_size += static_cast(window_len); } + direct_descriptors = !windows.empty() && every_window_descriptor_capable; if (windows.empty()) { window_size = 0; base_offset = 0; diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.h b/src/video_core/vulkan_common/vulkan_memory_allocator.h index 13d11e6d0d..e7a59a04a3 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.h +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.h @@ -126,6 +126,10 @@ namespace Vulkan { return windows.size(); } + [[nodiscard]] bool SupportsDirectDescriptors() const noexcept { + return direct_descriptors; + } + private: struct Window { vk::DeviceMemory memory; @@ -138,12 +142,23 @@ namespace Vulkan { size_t hardware_buffer_window, size_t hardware_buffer_base, size_t size); + bool TryCreateWindowBuffer(const void *external_info, VkDeviceSize length, + u32 external_type_bits, VkDeviceSize capacity, + VkBufferUsageFlags usage, VkBuffer *out_buffer, + u32 *out_type_mask) const; + + bool CreateDescriptorCapableWindowBuffer(const void *external_info, VkDeviceSize length, + u32 external_type_bits, VkDeviceSize capacity, + VkBuffer *out_buffer, u32 *out_type_mask, + bool *out_descriptor_capable) const; + const Device &device; std::vector windows; VkDeviceSize window_size{}; size_t imported_size{}; size_t base_offset{}; bool foreign_ownership{}; + bool direct_descriptors{}; }; /// Memory allocator container.