From b69a998b1a79543c686461c1750169ab90a29900 Mon Sep 17 00:00:00 2001 From: CamilleLaVey Date: Tue, 11 Aug 2026 17:37:46 -0400 Subject: [PATCH] another take --- src/video_core/buffer_cache/buffer_cache.h | 48 ++++ .../buffer_cache/buffer_cache_base.h | 2 + .../renderer_vulkan/vk_buffer_cache.cpp | 220 ++++++++++++++---- .../renderer_vulkan/vk_buffer_cache.h | 18 ++ 4 files changed, 241 insertions(+), 47 deletions(-) diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 9f446ec32b..e05f7e55bb 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -714,6 +714,7 @@ void BufferCache

::CommitAsyncFlushesHigh() { runtime.CopyToUnifiedMemory(queued.window, slot_buffers[queued.buffer_id], group_span); } if (!unified_copy_queue.empty()) { + runtime.FlushUnifiedMemoryCopies(); runtime.UnifiedMemoryHostBarrier(); } } @@ -1861,6 +1862,7 @@ bool BufferCache

::TryUnifiedDownloadMemory([[maybe_unused]] Buffer& buffer, const std::span group_span(groups[i].data(), groups[i].size()); runtime.CopyToUnifiedMemory(window_ids[i], buffer, group_span); } + runtime.FlushUnifiedMemoryCopies(); runtime.UnifiedMemoryHostBarrier(); runtime.Finish(); return true; @@ -1869,11 +1871,57 @@ bool BufferCache

::TryUnifiedDownloadMemory([[maybe_unused]] Buffer& buffer, } } +template +bool BufferCache

::TryUnifiedUploadMemory([[maybe_unused]] Buffer& buffer, + [[maybe_unused]] std::span copies) { + if constexpr (USE_UNIFIED_MEMORY) { + boost::container::small_vector window_ids; + UnifiedWindowGroups groups; + for (const BufferCopy& copy : copies) { + if (!ResolveUnifiedWindows(buffer.CpuAddr() + copy.dst_offset, copy.dst_offset, + copy.size, window_ids, groups)) { + return false; + } + } + for (const BufferCopy& copy : copies) { + buffer.MarkUsage(copy.dst_offset, copy.size); + } + runtime.PreCopyBarrier(); + boost::container::small_vector window_source_copies; + for (size_t i = 0; i < window_ids.size(); ++i) { + window_source_copies.clear(); + window_source_copies.reserve(groups[i].size()); + for (const BufferCopy& copy : groups[i]) { + window_source_copies.push_back(BufferCopy{ + .src_offset = copy.dst_offset, + .dst_offset = copy.src_offset, + .size = copy.size, + }); + } + const std::span group_span(window_source_copies.data(), + window_source_copies.size()); + runtime.CopyFromUnifiedMemory(window_ids[i], buffer, group_span); + } + runtime.FlushUnifiedMemoryCopies(); + runtime.PostCopyBarrier(); + return true; + } else { + return false; + } +} + template void BufferCache

::MappedUploadMemory([[maybe_unused]] Buffer& buffer, [[maybe_unused]] u64 total_size_bytes, [[maybe_unused]] std::span copies) { if constexpr (USE_MEMORY_MAPS) { + if constexpr (USE_UNIFIED_MEMORY) { + if (runtime.HasUnifiedMemory() && + !Settings::values.enable_gpu_buffer_readback.GetValue() && + TryUnifiedUploadMemory(buffer, copies)) { + return; + } + } auto upload_staging = runtime.UploadStagingBuffer(total_size_bytes); const std::span staging_pointer = upload_staging.mapped_span; for (BufferCopy& copy : copies) { diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 9d66fe9b40..041d88c9f2 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -446,6 +446,8 @@ private: bool TryUnifiedDownloadMemory(Buffer& buffer, std::span copies); + bool TryUnifiedUploadMemory(Buffer& buffer, std::span copies); + using UnifiedWindowGroups = boost::container::small_vector, 4>; diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp index 5bd4d1f2a0..d7564fb383 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include "video_core/buffer_cache/buffer_cache_base.h" @@ -32,6 +33,32 @@ VkBufferCopy MakeBufferCopy(const VideoCommon::BufferCopy& copy) { }; } +constexpr size_t MAX_WINDOW_BARRIER_RANGES = 8; + +using WindowRange = std::pair; +using WindowRanges = boost::container::small_vector; + +void CoalesceWindowRanges(WindowRanges& ranges) { + if (ranges.size() < 2) { + return; + } + std::sort(ranges.begin(), ranges.end()); + size_t merged = 0; + for (size_t index = 1; index < ranges.size(); ++index) { + if (ranges[index].first <= ranges[merged].second) { + ranges[merged].second = (std::max)(ranges[merged].second, ranges[index].second); + } else { + ranges[++merged] = ranges[index]; + } + } + ranges.resize(merged + 1); + if (ranges.size() > MAX_WINDOW_BARRIER_RANGES) { + const WindowRange bounding{ranges.front().first, ranges.back().second}; + ranges.clear(); + ranges.push_back(bounding); + } +} + VkIndexType IndexTypeFromNumElements(const Device& device, u32 num_elements) { if (num_elements <= 0xff && device.IsExtIndexTypeUint8Supported()) { return VK_INDEX_TYPE_UINT8_EXT; @@ -376,66 +403,164 @@ void BufferCacheRuntime::TryEnableUnifiedMemory(void* base, size_t size, } } +void BufferCacheRuntime::QueueUnifiedCopy(size_t window_index, VkBuffer buffer, + std::span copies, + bool reads_window) { + if (!unified_memory || buffer == VK_NULL_HANDLE || copies.empty() || + window_index >= unified_memory->GetWindowCount() || + unified_memory->GetWindowBuffer(window_index) == VK_NULL_HANDLE) { + return; + } + PendingUnifiedCopy& pending = pending_unified_copies.emplace_back(); + pending.window = window_index; + pending.buffer = buffer; + pending.reads_window = reads_window; + pending.copies.resize(copies.size()); + std::ranges::transform(copies, pending.copies.begin(), MakeBufferCopy); +} + void BufferCacheRuntime::CopyToUnifiedMemory( size_t window_index, VkBuffer src_buffer, std::span copies) { - if (!unified_memory || src_buffer == VK_NULL_HANDLE || copies.empty() || - window_index >= unified_memory->GetWindowCount()) { - return; - } - const VkBuffer dst_buffer = unified_memory->GetWindowBuffer(window_index); - if (dst_buffer == VK_NULL_HANDLE) { + QueueUnifiedCopy(window_index, src_buffer, copies, false); +} + +void BufferCacheRuntime::CopyFromUnifiedMemory( + size_t window_index, VkBuffer dst_buffer, + std::span copies) { + QueueUnifiedCopy(window_index, dst_buffer, copies, true); +} + +void BufferCacheRuntime::FlushUnifiedMemoryCopies() { + if (pending_unified_copies.empty()) { return; } - VkDeviceSize covered_begin = std::numeric_limits::max(); - VkDeviceSize covered_end = 0; - for (const VideoCommon::BufferCopy& copy : copies) { - covered_begin = (std::min)(covered_begin, static_cast(copy.dst_offset)); - covered_end = (std::max)(covered_end, - static_cast(copy.dst_offset + copy.size)); - } + struct UnifiedCopyCommand { + VkBuffer buffer; + bool reads_window; + boost::container::small_vector copies; + }; + static constexpr VkMemoryBarrier WINDOW_TRANSFER_BARRIER{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, + }; - boost::container::small_vector vk_copies(copies.size()); - std::ranges::transform(copies, vk_copies.begin(), MakeBufferCopy); + std::stable_sort(pending_unified_copies.begin(), pending_unified_copies.end(), + [](const PendingUnifiedCopy& lhs, const PendingUnifiedCopy& rhs) { + if (lhs.window != rhs.window) { + return lhs.window < rhs.window; + } + return lhs.reads_window && !rhs.reads_window; + }); const bool foreign = unified_memory->NeedsForeignOwnershipTransfer(); const u32 queue_family = device.GetGraphicsFamily(); - scheduler.RequestOutsideRenderPassOperationContext(); - scheduler.Record([src_buffer, dst_buffer, vk_copies, foreign, queue_family, covered_begin, - covered_end](vk::CommandBuffer cmdbuf) { - if (foreign) { - const VkBufferMemoryBarrier acquire{ - .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = 0, - .dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, - .srcQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT, - .dstQueueFamilyIndex = queue_family, - .buffer = dst_buffer, - .offset = covered_begin, - .size = covered_end - covered_begin, - }; - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, - VK_PIPELINE_STAGE_TRANSFER_BIT, 0, acquire); + size_t group_begin = 0; + while (group_begin < pending_unified_copies.size()) { + const size_t window = pending_unified_copies[group_begin].window; + size_t group_end = group_begin; + while (group_end < pending_unified_copies.size() && + pending_unified_copies[group_end].window == window) { + ++group_end; } - cmdbuf.CopyBuffer(src_buffer, dst_buffer, VideoCommon::FixSmallVectorADL(vk_copies)); + const VkBuffer window_buffer = unified_memory->GetWindowBuffer(window); + + WindowRanges ranges; + VkAccessFlags window_access = 0; + size_t reads_in_group = 0; + for (size_t index = group_begin; index < group_end; ++index) { + const PendingUnifiedCopy& pending = pending_unified_copies[index]; + if (pending.reads_window) { + window_access |= VK_ACCESS_TRANSFER_READ_BIT; + ++reads_in_group; + } else { + window_access |= VK_ACCESS_TRANSFER_WRITE_BIT; + } + for (const VkBufferCopy& copy : pending.copies) { + const VkDeviceSize offset = + pending.reads_window ? copy.srcOffset : copy.dstOffset; + ranges.emplace_back(offset, offset + copy.size); + } + } + CoalesceWindowRanges(ranges); + + boost::container::small_vector acquire; + boost::container::small_vector release; if (foreign) { - const VkBufferMemoryBarrier release{ - .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = 0, - .srcQueueFamilyIndex = queue_family, - .dstQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT, - .buffer = dst_buffer, - .offset = covered_begin, - .size = covered_end - covered_begin, - }; - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, - VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0, release); + for (const WindowRange& range : ranges) { + acquire.push_back(VkBufferMemoryBarrier{ + .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = 0, + .dstAccessMask = window_access, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT, + .dstQueueFamilyIndex = queue_family, + .buffer = window_buffer, + .offset = range.first, + .size = range.second - range.first, + }); + release.push_back(VkBufferMemoryBarrier{ + .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = window_access, + .dstAccessMask = 0, + .srcQueueFamilyIndex = queue_family, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT, + .buffer = window_buffer, + .offset = range.first, + .size = range.second - range.first, + }); + } } - }); + + boost::container::small_vector commands; + commands.reserve(group_end - group_begin); + for (size_t index = group_begin; index < group_end; ++index) { + PendingUnifiedCopy& pending = pending_unified_copies[index]; + commands.push_back(UnifiedCopyCommand{pending.buffer, pending.reads_window, + std::move(pending.copies)}); + } + const size_t barrier_before_writes = (reads_in_group > 0 && reads_in_group < commands.size()) + ? reads_in_group + : commands.size(); + + scheduler.RequestOutsideRenderPassOperationContext(); + scheduler.Record([window_buffer, barrier_before_writes, acquire = std::move(acquire), + release = std::move(release), + commands = std::move(commands)](vk::CommandBuffer cmdbuf) { + if (!acquire.empty()) { + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, {}, + VideoCommon::FixSmallVectorADL(acquire), {}); + } + for (size_t index = 0; index < commands.size(); ++index) { + if (index == barrier_before_writes) { + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, + WINDOW_TRANSFER_BARRIER); + } + const UnifiedCopyCommand& command = commands[index]; + if (command.reads_window) { + cmdbuf.CopyBuffer(window_buffer, command.buffer, + VideoCommon::FixSmallVectorADL(command.copies)); + } else { + cmdbuf.CopyBuffer(command.buffer, window_buffer, + VideoCommon::FixSmallVectorADL(command.copies)); + } + } + if (!release.empty()) { + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0, {}, + VideoCommon::FixSmallVectorADL(release), {}); + } + }); + + group_begin = group_end; + } + pending_unified_copies.clear(); } void BufferCacheRuntime::UnifiedMemoryHostBarrier() { @@ -489,6 +614,7 @@ u32 BufferCacheRuntime::GetStorageBufferAlignment() const { } void BufferCacheRuntime::TickFrame(Common::SlotVector& slot_buffers) noexcept { + FlushUnifiedMemoryCopies(); for (auto it = slot_buffers.begin(); it != slot_buffers.end(); it++) { if (scheduler.IsFree(it->LastUsageTick())) { it->ResetUsageTracking(); diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index c7870505eb..40ba394b4e 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -10,6 +10,8 @@ #include #include +#include + #include "video_core/buffer_cache/buffer_cache_base.h" #include "video_core/buffer_cache/memory_tracker_base.h" #include "video_core/buffer_cache/usage_tracker.h" @@ -122,6 +124,11 @@ public: void CopyToUnifiedMemory(size_t window_index, VkBuffer src_buffer, std::span copies); + void CopyFromUnifiedMemory(size_t window_index, VkBuffer dst_buffer, + std::span copies); + + void FlushUnifiedMemoryCopies(); + void UnifiedMemoryHostBarrier(); u64 CurrentTick(); @@ -207,6 +214,16 @@ public: } private: + struct PendingUnifiedCopy { + size_t window; + VkBuffer buffer; + bool reads_window; + boost::container::small_vector copies; + }; + + void QueueUnifiedCopy(size_t window_index, VkBuffer buffer, + std::span copies, bool reads_window); + void BindBuffer(const Buffer& buffer, u32 offset, u32 size) { const VkBuffer handle = buffer.Handle(); if (handle == VK_NULL_HANDLE) { @@ -232,6 +249,7 @@ private: vk::Buffer null_buffer; std::unique_ptr unified_memory; + boost::container::small_vector pending_unified_copies; std::unique_ptr uint8_pass; QuadIndexedPass quad_index_pass;