diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 6ed3daf5f9..3f6b2b182e 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -115,7 +115,8 @@ void BufferCache

::TickFrame() { template void BufferCache

::WriteMemory(DAddr device_addr, u64 size) { - if (memory_tracker.IsRegionGpuModified(device_addr, size)) { + if (memory_tracker.IsRegionGpuModified(device_addr, size) || + IntersectsInPlaceWrites(device_addr, size)) { ClearDownload(device_addr, size); ForgetGpuModifiedRange(device_addr, size); } @@ -605,16 +606,24 @@ void BufferCache

::CommitAsyncFlushesHigh() { struct QueuedDownload { BufferCopy copy; BufferId buffer_id; - bool written_in_place; }; boost::container::small_vector downloads; + boost::container::small_vector in_place_downloads; u64 total_size_bytes = 0; u64 largest_copy = 0; for (const Common::RangeSet& range_set : committed_gpu_modified_ranges) { range_set.ForEach([&](DAddr interval_lower, DAddr interval_upper) { const std::size_t size = interval_upper - interval_lower; const DAddr device_addr = interval_lower; + in_place_gpu_written_ranges.ForEachInRange( + device_addr, size, [&](DAddr start, DAddr end) { + in_place_downloads.push_back(BufferCopy{ + .src_offset = static_cast(start), + .dst_offset = 0, + .size = end - start, + }); + }); ForEachBufferInRange(device_addr, size, [&](BufferId buffer_id, Buffer& buffer) { const DAddr buffer_start = buffer.CpuAddr(); const DAddr buffer_end = buffer_start + buffer.SizeBytes(); @@ -624,8 +633,7 @@ void BufferCache

::CommitAsyncFlushesHigh() { new_start, new_end - new_start, false, [&](u64 device_addr_out, u64 range_size) { const DAddr buffer_addr = buffer.CpuAddr(); - const auto add_download = [&](DAddr start, DAddr end, - bool written_in_place) { + const auto add_download = [&](DAddr start, DAddr end) { const u64 new_offset = start - buffer_addr; const u64 new_size = end - start; downloads.push_back({ @@ -635,7 +643,6 @@ void BufferCache

::CommitAsyncFlushesHigh() { .size = new_size, }, buffer_id, - written_in_place, }); // Align up to avoid cache conflicts constexpr u64 align = 64ULL; @@ -643,29 +650,28 @@ void BufferCache

::CommitAsyncFlushesHigh() { total_size_bytes += (new_size + align - 1) & mask; largest_copy = (std::max)(largest_copy, new_size); }; - const auto split_by_in_place = [&](DAddr start, DAddr end) { + const auto skip_in_place = [&](DAddr start, DAddr end) { DAddr cursor = start; in_place_gpu_written_ranges.ForEachInRange( start, end - start, [&](DAddr in_start, DAddr in_end) { if (cursor < in_start) { - add_download(cursor, in_start, false); + add_download(cursor, in_start); } - add_download(in_start, in_end, true); cursor = in_end; }); if (cursor < end) { - add_download(cursor, end, false); + add_download(cursor, end); } }; gpu_modified_ranges.ForEachInRange(device_addr_out, range_size, - split_by_in_place); + skip_in_place); }); }); }); } committed_gpu_modified_ranges.clear(); - if (downloads.empty()) { + if (downloads.empty() && in_place_downloads.empty()) { pending_downloads.emplace_back(); async_buffers.emplace_back(std::optional{}); return; @@ -683,18 +689,13 @@ void BufferCache

::CommitAsyncFlushesHigh() { boost::container::small_vector window_ids; UnifiedWindowGroups groups; u64 staging_size_bytes = 0; - bool has_in_place_writes = false; - for (auto& [copy, buffer_id, written_in_place] : downloads) { + for (const BufferCopy& copy : in_place_downloads) { + async_downloads.Add(static_cast(copy.src_offset), copy.size); + batch.unified_copies.push_back(copy); + } + for (auto& [copy, buffer_id] : downloads) { Buffer& buffer = slot_buffers[buffer_id]; const DAddr orig_device_addr = buffer.CpuAddr() + copy.src_offset; - if (written_in_place) { - BufferCopy record{copy}; - record.src_offset = static_cast(orig_device_addr); - async_downloads.Add(orig_device_addr, copy.size); - batch.unified_copies.push_back(record); - has_in_place_writes = true; - continue; - } bool unified = false; if constexpr (USE_UNIFIED_MEMORY) { if (runtime.HasUnifiedMemory()) { @@ -750,7 +751,7 @@ void BufferCache

::CommitAsyncFlushesHigh() { } } if constexpr (USE_UNIFIED_DIRECT_BINDING) { - if (has_in_place_writes) { + if (!in_place_downloads.empty()) { runtime.UnifiedMemoryShaderWriteBarrier(); } } @@ -1608,6 +1609,12 @@ void BufferCache

::UpdateComputeTextureBuffers() { template void BufferCache

::MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u32 size) { + if constexpr (USE_UNIFIED_DIRECT_BINDING) { + if (!in_place_gpu_written_ranges.Empty() && IntersectsInPlaceWrites(device_addr, size)) { + SynchronizeBuffer(slot_buffers[buffer_id], device_addr, size); + in_place_gpu_written_ranges.Subtract(device_addr, size); + } + } if constexpr (!IS_OPENGL) { Buffer& buffer = slot_buffers[buffer_id]; buffer.setWriteTick(runtime.CurrentTick()); @@ -1619,10 +1626,13 @@ void BufferCache

::MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u3 template void BufferCache

::MarkWrittenBufferInPlace(DAddr device_addr, u32 size) { - memory_tracker.MarkRegionAsGpuModified(device_addr, size); + memory_tracker.MarkRegionAsCpuModified(device_addr, size); gpu_modified_ranges.Add(device_addr, size); uncommitted_gpu_modified_ranges.Add(device_addr, size); in_place_gpu_written_ranges.Add(device_addr, size); + if constexpr (!IS_OPENGL) { + in_place_write_tick = runtime.CurrentTick(); + } } template @@ -1631,6 +1641,31 @@ void BufferCache

::ForgetGpuModifiedRange(DAddr device_addr, u64 size) { in_place_gpu_written_ranges.Subtract(device_addr, size); } +template +bool BufferCache

::IsRegionWrittenInPlace(DAddr device_addr, u64 size) { + u64 covered = 0; + in_place_gpu_written_ranges.ForEachInRange( + device_addr, size, [&](DAddr start, DAddr end) { covered += end - start; }); + return covered == size; +} + +template +bool BufferCache

::IntersectsInPlaceWrites(DAddr device_addr, u64 size) { + bool intersects = false; + in_place_gpu_written_ranges.ForEachInRange(device_addr, size, + [&](DAddr, DAddr) { intersects = true; }); + return intersects; +} + +template +void BufferCache

::WaitForInPlaceWrites() { + if constexpr (!IS_OPENGL) { + if (in_place_write_tick > runtime.KnownGpuTick()) { + runtime.Wait(in_place_write_tick); + } + } +} + template BufferId BufferCache

::FindBuffer(DAddr device_addr, u32 size) { if (device_addr == 0) { @@ -1858,6 +1893,17 @@ void BufferCache

::UploadMemory(Buffer& buffer, u64 total_size_bytes, u64 larg if (TryUnifiedUploadMemory(buffer, copies)) { return; } + if constexpr (USE_UNIFIED_DIRECT_BINDING) { + if (!in_place_gpu_written_ranges.Empty()) { + const DAddr buffer_start = buffer.CpuAddr(); + for (const BufferCopy& copy : copies) { + if (IntersectsInPlaceWrites(buffer_start + copy.dst_offset, copy.size)) { + WaitForInPlaceWrites(); + break; + } + } + } + } if constexpr (USE_MEMORY_MAPS_FOR_UPLOADS) { MappedUploadMemory(buffer, total_size_bytes, copies); } else { @@ -1939,7 +1985,8 @@ bool BufferCache

::ResolveUnifiedDirectBinding([[maybe_unused]] DAddr device_a } contiguous += Core::DEVICE_PAGESIZE; } - if (IsRegionGpuModified(device_addr, size)) { + if (IsRegionGpuModified(device_addr, size) && + !IsRegionWrittenInPlace(device_addr, size)) { return false; } window_index = relative / window_size; diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 49cc1ba08f..97bcc65f92 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -422,6 +422,12 @@ private: void ForgetGpuModifiedRange(DAddr device_addr, u64 size); + [[nodiscard]] bool IsRegionWrittenInPlace(DAddr device_addr, u64 size); + + [[nodiscard]] bool IntersectsInPlaceWrites(DAddr device_addr, u64 size); + + void WaitForInPlaceWrites(); + [[nodiscard]] BufferId FindBuffer(DAddr device_addr, u32 size); void WaitForGpuFenceIfNeeded(Buffer& buffer); @@ -518,6 +524,7 @@ private: Common::RangeSet uncommitted_gpu_modified_ranges; Common::RangeSet gpu_modified_ranges; Common::RangeSet in_place_gpu_written_ranges; + u64 in_place_write_tick = 0; std::deque> committed_gpu_modified_ranges; // Async Buffers