diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 5394cadbc5..603f5f4932 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -670,7 +670,11 @@ void BufferCache

::AccumulateFlushes() { template bool BufferCache

::ShouldWaitAsyncFlushes() const noexcept { - return (!async_buffers.empty() && async_buffers.front().has_value()); + if (async_buffers.empty()) { + return false; + } + return async_buffers.front().has_value() || + !pending_downloads.front().unified_copies.empty(); } template @@ -678,6 +682,7 @@ void BufferCache

::CommitAsyncFlushesHigh() { AccumulateFlushes(); if (committed_gpu_modified_ranges.empty()) { + pending_downloads.emplace_back(); async_buffers.emplace_back(std::optional{}); return; } @@ -737,27 +742,83 @@ void BufferCache

::CommitAsyncFlushesHigh() { } committed_gpu_modified_ranges.clear(); if (downloads.empty()) { + pending_downloads.emplace_back(); async_buffers.emplace_back(std::optional{}); return; } - auto download_staging = runtime.DownloadStagingBuffer(total_size_bytes, true); - boost::container::small_vector normalized_copies; - runtime.PreCopyBarrier(); + + struct QueuedUnifiedCopy { + u64 window; + BufferId buffer_id; + boost::container::small_vector copies; + }; + + AsyncDownloadBatch batch; + boost::container::small_vector, 16> staging_downloads; + boost::container::small_vector unified_copy_queue; + boost::container::small_vector window_ids; + UnifiedWindowGroups groups; + u64 staging_size_bytes = 0; for (auto& [copy, buffer_id] : downloads) { - copy.dst_offset += download_staging.offset; + Buffer& buffer = slot_buffers[buffer_id]; + const DAddr orig_device_addr = buffer.CpuAddr() + copy.src_offset; + bool unified = false; + if constexpr (USE_UNIFIED_MEMORY) { + if (runtime.HasUnifiedMemory()) { + window_ids.clear(); + groups.clear(); + unified = ResolveUnifiedWindows(orig_device_addr, copy.src_offset, copy.size, + window_ids, groups); + } + } + BufferCopy record{copy}; + record.src_offset = static_cast(orig_device_addr); + if (unified) { + async_downloads.Add(orig_device_addr, copy.size); + buffer.MarkUsage(copy.src_offset, copy.size); + for (size_t i = 0; i < window_ids.size(); ++i) { + unified_copy_queue.push_back( + QueuedUnifiedCopy{window_ids[i], buffer_id, std::move(groups[i])}); + } + batch.unified_copies.push_back(record); + continue; + } + copy.dst_offset = staging_size_bytes; + constexpr u64 align = 64ULL; + staging_size_bytes += (copy.size + align - 1) & ~(align - 1ULL); + staging_downloads.push_back({copy, buffer_id}); + } + + std::optional download_staging; + if (!staging_downloads.empty()) { + download_staging = runtime.DownloadStagingBuffer(staging_size_bytes, true); + } + runtime.PreCopyBarrier(); + for (auto& [copy, buffer_id] : staging_downloads) { + copy.dst_offset += download_staging->offset; const std::array copies{copy}; - BufferCopy second_copy{copy}; Buffer& buffer = slot_buffers[buffer_id]; - second_copy.src_offset = static_cast(buffer.CpuAddr()) + copy.src_offset; - const DAddr orig_device_addr = static_cast(second_copy.src_offset); + BufferCopy record{copy}; + record.src_offset = static_cast(buffer.CpuAddr()) + copy.src_offset; + const DAddr orig_device_addr = static_cast(record.src_offset); async_downloads.Add(orig_device_addr, copy.size); buffer.MarkUsage(copy.src_offset, copy.size); - runtime.CopyBuffer(download_staging.buffer, buffer, copies, false); - normalized_copies.push_back(second_copy); + runtime.CopyBuffer(download_staging->buffer, buffer, copies, false); + batch.staging_copies.push_back(record); + } + if constexpr (USE_UNIFIED_MEMORY) { + for (const auto& queued : unified_copy_queue) { + const std::span group_span(queued.copies.data(), + queued.copies.size()); + runtime.CopyToUnifiedMemory(queued.window, slot_buffers[queued.buffer_id], group_span); + } + if (!unified_copy_queue.empty()) { + runtime.UnifiedMemoryHostBarrier(); + } } runtime.PostCopyBarrier(); - pending_downloads.emplace_back(std::move(normalized_copies)); - async_buffers.emplace_back(download_staging); + pending_downloads.emplace_back(std::move(batch)); + async_buffers.emplace_back(std::move(download_staging)); } template @@ -783,27 +844,32 @@ void BufferCache

::PopAsyncBuffers() { if (async_buffers.empty()) { return; } - if (!async_buffers.front().has_value()) { - async_buffers.pop_front(); - return; - } - auto& downloads = pending_downloads.front(); + auto& batch = pending_downloads.front(); auto& async_buffer = async_buffers.front(); - const u8* base = async_buffer->mapped_span.data(); - const size_t base_offset = async_buffer->offset; - for (const auto& copy : downloads) { + if (async_buffer.has_value()) { + const u8* base = async_buffer->mapped_span.data(); + const size_t base_offset = async_buffer->offset; + for (const auto& copy : batch.staging_copies) { + const DAddr device_addr = static_cast(copy.src_offset); + const u64 dst_offset = copy.dst_offset - base_offset; + const u8* read_mapped_memory = base + dst_offset; + async_downloads.ForEachInRange( + device_addr, copy.size, [&](DAddr start, DAddr end, s32) { + writebacks.push_back( + {start, &read_mapped_memory[start - device_addr], end - start}); + }); + async_downloads.Subtract(device_addr, copy.size, [&](DAddr start, DAddr end) { + gpu_modified_ranges.Subtract(start, end - start); + }); + } + async_buffers_death_ring.emplace_back(*async_buffer); + } + for (const auto& copy : batch.unified_copies) { const DAddr device_addr = static_cast(copy.src_offset); - const u64 dst_offset = copy.dst_offset - base_offset; - const u8* read_mapped_memory = base + dst_offset; - async_downloads.ForEachInRange(device_addr, copy.size, [&](DAddr start, DAddr end, s32) { - writebacks.push_back( - {start, &read_mapped_memory[start - device_addr], end - start}); - }); async_downloads.Subtract(device_addr, copy.size, [&](DAddr start, DAddr end) { gpu_modified_ranges.Subtract(start, end - start); }); } - async_buffers_death_ring.emplace_back(*async_buffer); async_buffers.pop_front(); pending_downloads.pop_front(); } @@ -1815,8 +1881,10 @@ void BufferCache

::ImmediateUploadMemory([[maybe_unused]] Buffer& buffer, } template -bool BufferCache

::TryUnifiedDownloadMemory([[maybe_unused]] Buffer& buffer, - [[maybe_unused]] std::span copies) { +bool BufferCache

::ResolveUnifiedWindows( + [[maybe_unused]] DAddr device_addr, [[maybe_unused]] u64 buffer_offset, + [[maybe_unused]] u64 size, [[maybe_unused]] boost::container::small_vector& window_ids, + [[maybe_unused]] UnifiedWindowGroups& groups) { if constexpr (USE_UNIFIED_MEMORY) { const u8* const physical_base = device_memory.GetPhysicalBase(); const u64 unified_base = runtime.UnifiedMemoryBase(); @@ -1825,8 +1893,6 @@ bool BufferCache

::TryUnifiedDownloadMemory([[maybe_unused]] Buffer& buffer, if (window_size == 0) { return false; } - boost::container::small_vector window_ids; - boost::container::small_vector, 4> groups; const auto group_for = [&](u64 window) -> boost::container::small_vector& { for (size_t i = 0; i < window_ids.size(); ++i) { if (window_ids[i] == window) { @@ -1837,52 +1903,68 @@ bool BufferCache

::TryUnifiedDownloadMemory([[maybe_unused]] Buffer& buffer, groups.emplace_back(); return groups.back(); }; - for (const BufferCopy& copy : copies) { - const DAddr device_addr = buffer.CpuAddr() + copy.src_offset; - u64 downloaded = 0; - while (downloaded < copy.size) { - const DAddr page_addr = device_addr + downloaded; - const u8* const ptr = device_memory.GetPointer(page_addr); - if (ptr == nullptr) { - return false; - } - const u64 page_offset = page_addr & Core::DEVICE_PAGEMASK; - u64 chunk = (std::min)(copy.size - downloaded, - static_cast(Core::DEVICE_PAGESIZE) - page_offset); - const u64 phys_offset = static_cast(ptr - physical_base); - if (phys_offset < unified_base || - phys_offset - unified_base + chunk > unified_size) { - return false; - } - const u64 relative = phys_offset - unified_base; - const u64 window = relative / window_size; - const u64 local_offset = relative % window_size; - chunk = (std::min)(chunk, window_size - local_offset); - auto& group = group_for(window); - if (!group.empty()) { - BufferCopy& last = group.back(); - if (last.src_offset + last.size == copy.src_offset + downloaded && - last.dst_offset + last.size == local_offset) { - last.size += chunk; - downloaded += chunk; - continue; - } + u64 downloaded = 0; + while (downloaded < size) { + const DAddr page_addr = device_addr + downloaded; + const u8* const ptr = device_memory.GetPointer(page_addr); + if (ptr == nullptr) { + return false; + } + const u64 page_offset = page_addr & Core::DEVICE_PAGEMASK; + u64 chunk = (std::min)(size - downloaded, + static_cast(Core::DEVICE_PAGESIZE) - page_offset); + const u64 phys_offset = static_cast(ptr - physical_base); + if (phys_offset < unified_base || phys_offset - unified_base + chunk > unified_size) { + return false; + } + const u64 relative = phys_offset - unified_base; + const u64 window = relative / window_size; + const u64 local_offset = relative % window_size; + chunk = (std::min)(chunk, window_size - local_offset); + auto& group = group_for(window); + if (!group.empty()) { + BufferCopy& last = group.back(); + if (last.src_offset + last.size == buffer_offset + downloaded && + last.dst_offset + last.size == local_offset) { + last.size += chunk; + downloaded += chunk; + continue; } - group.push_back(BufferCopy{ - .src_offset = copy.src_offset + downloaded, - .dst_offset = local_offset, - .size = chunk, - }); - downloaded += chunk; + } + group.push_back(BufferCopy{ + .src_offset = buffer_offset + downloaded, + .dst_offset = local_offset, + .size = chunk, + }); + downloaded += chunk; + } + return true; + } else { + return false; + } +} + +template +bool BufferCache

::TryUnifiedDownloadMemory([[maybe_unused]] Buffer& buffer, + [[maybe_unused]] std::span copies) { + if constexpr (USE_UNIFIED_MEMORY) { + boost::container::small_vector window_ids; + UnifiedWindowGroups groups; + for (const BufferCopy& copy : copies) { + if (!ResolveUnifiedWindows(buffer.CpuAddr() + copy.src_offset, copy.src_offset, + copy.size, window_ids, groups)) { + return false; } } for (const BufferCopy& copy : copies) { buffer.MarkUsage(copy.src_offset, copy.size); } + runtime.PreCopyBarrier(); for (size_t i = 0; i < window_ids.size(); ++i) { const std::span group_span(groups[i].data(), groups[i].size()); runtime.CopyToUnifiedMemory(window_ids[i], buffer, group_span); } + runtime.UnifiedMemoryHostBarrier(); runtime.Finish(); return true; } else { diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index c3f39d39cb..0b4896fbe8 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -453,6 +453,13 @@ private: bool TryUnifiedDownloadMemory(Buffer& buffer, std::span copies); + using UnifiedWindowGroups = + boost::container::small_vector, 4>; + + bool ResolveUnifiedWindows(DAddr device_addr, u64 buffer_offset, u64 size, + boost::container::small_vector& window_ids, + UnifiedWindowGroups& groups); + void DownloadBufferMemory(Buffer& buffer_id); void DownloadBufferMemory(Buffer& buffer_id, DAddr device_addr, u64 size); @@ -503,9 +510,14 @@ private: std::deque> committed_gpu_modified_ranges; // Async Buffers + struct AsyncDownloadBatch { + boost::container::small_vector staging_copies; + boost::container::small_vector unified_copies; + }; + Common::OverlapRangeSet async_downloads; std::deque> async_buffers; - std::deque> pending_downloads; + std::deque pending_downloads; std::optional current_buffer; std::deque async_buffers_death_ring; diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp index 0d33df9d51..e67a4ada3b 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp @@ -403,14 +403,6 @@ void BufferCacheRuntime::CopyToUnifiedMemory( scheduler.RequestOutsideRenderPassOperationContext(); scheduler.Record([src_buffer, dst_buffer, vk_copies, foreign, queue_family, covered_begin, covered_end](vk::CommandBuffer cmdbuf) { - static constexpr VkMemoryBarrier READ_BARRIER{ - .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT, - .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, - }; - cmdbuf.PipelineBarrier(vk::PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER, - VK_PIPELINE_STAGE_TRANSFER_BIT, 0, READ_BARRIER); if (foreign) { const VkBufferMemoryBarrier acquire{ .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, @@ -442,12 +434,18 @@ void BufferCacheRuntime::CopyToUnifiedMemory( cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0, release); } - static constexpr VkMemoryBarrier HOST_BARRIER{ - .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_HOST_READ_BIT, - }; + }); +} + +void BufferCacheRuntime::UnifiedMemoryHostBarrier() { + static constexpr VkMemoryBarrier HOST_BARRIER{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_HOST_READ_BIT, + }; + scheduler.RequestOutsideRenderPassOperationContext(); + scheduler.Record([](vk::CommandBuffer cmdbuf) { cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, 0, HOST_BARRIER); }); diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index e14ce55b4a..4fc316850f 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -122,6 +122,8 @@ public: void CopyToUnifiedMemory(size_t window_index, VkBuffer src_buffer, std::span copies); + void UnifiedMemoryHostBarrier(); + u64 CurrentTick(); u64 KnownGpuTick();