diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 86851ed67f..a33d0bb1db 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -204,6 +204,17 @@ bool BufferCache

::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am return false; } + std::optional src_window; + std::optional dst_window; + if constexpr (USE_UNIFIED_DIRECT_BINDING) { + src_window = TryObtainWindowBuffer(*cpu_src_address, static_cast(amount), + UNIFIED_COPY_BINDING_ALIGNMENT, + ObtainBufferOperation::DoNothing); + dst_window = TryObtainWindowBuffer(*cpu_dest_address, static_cast(amount), + UNIFIED_COPY_BINDING_ALIGNMENT, + ObtainBufferOperation::DoNothing); + } + ClearDownload(*cpu_dest_address, amount); BufferId buffer_a; @@ -215,11 +226,19 @@ bool BufferCache

::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am } while (channel_state->has_deleted_buffers); auto& src_buffer = slot_buffers[buffer_a]; auto& dest_buffer = slot_buffers[buffer_b]; - SynchronizeBuffer(src_buffer, *cpu_src_address, static_cast(amount)); - SynchronizeBuffer(dest_buffer, *cpu_dest_address, static_cast(amount)); + if (!src_window) { + SynchronizeBuffer(src_buffer, *cpu_src_address, static_cast(amount)); + } + if (!dst_window) { + SynchronizeBuffer(dest_buffer, *cpu_dest_address, static_cast(amount)); + } + const u64 src_copy_offset = + src_window ? u64{src_window->offset} : u64{src_buffer.Offset(*cpu_src_address)}; + const u64 dst_copy_offset = + dst_window ? u64{dst_window->offset} : u64{dest_buffer.Offset(*cpu_dest_address)}; std::array copies{BufferCopy{ - .src_offset = src_buffer.Offset(*cpu_src_address), - .dst_offset = dest_buffer.Offset(*cpu_dest_address), + .src_offset = src_copy_offset, + .dst_offset = dst_copy_offset, .size = amount, }}; @@ -231,7 +250,9 @@ bool BufferCache

::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am tmp_intervals.push_back({new_base_address, size}); uncommitted_gpu_modified_ranges.Add(new_base_address, size); }; - gpu_modified_ranges.ForEachInRange(*cpu_src_address, amount, mirror); + if (!dst_window) { + gpu_modified_ranges.ForEachInRange(*cpu_src_address, amount, mirror); + } // This subtraction in this order is important for overlapping copies. ForgetGpuModifiedRange(*cpu_dest_address, amount); const bool has_new_downloads = tmp_intervals.size() != 0; @@ -239,10 +260,25 @@ bool BufferCache

::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am gpu_modified_ranges.Add(pair.first, pair.second); } const auto& copy = copies[0]; - src_buffer.MarkUsage(copy.src_offset, copy.size); - dest_buffer.MarkUsage(copy.dst_offset, copy.size); - runtime.CopyBuffer(dest_buffer, src_buffer, copies, true); - if (has_new_downloads) { + if (!src_window) { + src_buffer.MarkUsage(copy.src_offset, copy.size); + } + if (!dst_window) { + dest_buffer.MarkUsage(copy.dst_offset, copy.size); + } + if constexpr (USE_UNIFIED_DIRECT_BINDING) { + runtime.CopyBuffer( + runtime.ResolveWindowHandle(dest_buffer, dst_window.has_value(), + dst_window ? dst_window->window_index : 0), + runtime.ResolveWindowHandle(src_buffer, src_window.has_value(), + src_window ? src_window->window_index : 0), + copies, true); + } else { + runtime.CopyBuffer(dest_buffer, src_buffer, copies, true); + } + if (dst_window) { + MarkWrittenBufferInPlace(*cpu_dest_address, static_cast(amount)); + } else if (has_new_downloads) { memory_tracker.MarkRegionAsGpuModified(*cpu_dest_address, amount); } @@ -275,6 +311,54 @@ bool BufferCache

::DMAClear(GPUVAddr dst_address, u64 amount, u32 value) { return true; } +template +auto BufferCache

::TryObtainWindowBuffer([[maybe_unused]] DAddr device_addr, + [[maybe_unused]] u32 size, + [[maybe_unused]] u64 alignment, + [[maybe_unused]] ObtainBufferOperation post_op) + -> std::optional { + if constexpr (USE_UNIFIED_DIRECT_BINDING) { + if (!runtime.SupportsUnifiedDirectBinding()) { + return std::nullopt; + } + u64 window_index = 0; + u64 window_offset = 0; + if (!ResolveUnifiedDirectBinding(device_addr, size, alignment, window_index, + window_offset)) { + return std::nullopt; + } + switch (post_op) { + case ObtainBufferOperation::MarkAsWritten: + MarkWrittenBufferInPlace(device_addr, size); + break; + case ObtainBufferOperation::DiscardWrite: { + const DAddr aligned_start = Common::AlignDown(device_addr, 64); + const DAddr aligned_end = Common::AlignUp(device_addr + size, 64); + const size_t aligned_size = aligned_end - aligned_start; + ClearDownload(aligned_start, aligned_size); + ForgetGpuModifiedRange(aligned_start, aligned_size); + break; + } + default: + break; + } + return WindowBufferRef{window_index, static_cast(window_offset)}; + } else { + return std::nullopt; + } +} + +template +auto BufferCache

::TryObtainWindowBufferGpu(GPUVAddr gpu_addr, u32 size, u64 alignment, + ObtainBufferOperation post_op) + -> std::optional { + const std::optional device_addr = gpu_memory->GpuToCpuAddress(gpu_addr); + if (!device_addr) { + return std::nullopt; + } + return TryObtainWindowBuffer(*device_addr, size, alignment, post_op); +} + template std::pair BufferCache

::ObtainBuffer(GPUVAddr gpu_addr, u32 size, ObtainBufferSynchronize sync_info, @@ -752,7 +836,7 @@ void BufferCache

::CommitAsyncFlushesHigh() { } if constexpr (USE_UNIFIED_DIRECT_BINDING) { if (!in_place_downloads.empty()) { - runtime.UnifiedMemoryShaderWriteBarrier(); + runtime.UnifiedMemoryWriteBarrier(); } } runtime.PostCopyBarrier(); @@ -1041,15 +1125,25 @@ void BufferCache

::BindHostVertexBuffers() { template void BufferCache

::BindHostDrawIndirectBuffers() { - const auto bind_buffer = [this](const Binding& binding) { + const auto bind_buffer = [this](const Binding& binding) -> std::optional { Buffer& buffer = slot_buffers[binding.buffer_id]; TouchBuffer(buffer, binding.buffer_id); + if constexpr (USE_UNIFIED_DIRECT_BINDING) { + if (auto window = TryObtainWindowBuffer(binding.device_addr, binding.size, + UNIFIED_INDIRECT_BINDING_ALIGNMENT, + ObtainBufferOperation::DoNothing)) { + return window; + } + } SynchronizeBuffer(buffer, binding.device_addr, binding.size); + return std::nullopt; }; + draw_indirect_count_window.reset(); + draw_indirect_window.reset(); if (current_draw_indirect->include_count) { - bind_buffer(channel_state->count_buffer_binding); + draw_indirect_count_window = bind_buffer(channel_state->count_buffer_binding); } - bind_buffer(channel_state->indirect_buffer_binding); + draw_indirect_window = bind_buffer(channel_state->indirect_buffer_binding); } template diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 8368986b9e..5225bfea56 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -271,6 +272,20 @@ public: void BindComputeTextureBuffer(size_t tbo_index, GPUVAddr gpu_addr, u32 size, PixelFormat format, bool is_written, bool is_image); + struct WindowBufferRef { + u64 window_index; + u32 offset; + }; + + static constexpr u64 UNIFIED_INDIRECT_BINDING_ALIGNMENT = 4; + static constexpr u64 UNIFIED_COPY_BINDING_ALIGNMENT = 1; + + [[nodiscard]] std::optional TryObtainWindowBuffer( + DAddr device_addr, u32 size, u64 alignment, ObtainBufferOperation post_op); + + [[nodiscard]] std::optional TryObtainWindowBufferGpu( + GPUVAddr gpu_addr, u32 size, u64 alignment, ObtainBufferOperation post_op); + [[nodiscard]] std::pair ObtainBuffer(GPUVAddr gpu_addr, u32 size, ObtainBufferSynchronize sync_info, ObtainBufferOperation post_op); @@ -318,6 +333,15 @@ public: [[nodiscard]] std::pair GetDrawIndirectBuffer(); + [[nodiscard]] const std::optional& GetDrawIndirectWindow() const noexcept { + return draw_indirect_window; + } + + [[nodiscard]] const std::optional& GetDrawIndirectCountWindow() + const noexcept { + return draw_indirect_count_window; + } + template void BufferOperations(Func&& func) { do { @@ -513,6 +537,9 @@ private: const Tegra::Engines::Maxwell3D::DrawManager::IndirectParams* current_draw_indirect{}; + std::optional draw_indirect_window; + std::optional draw_indirect_count_window; + u32 last_index_count = 0; u32 enabled_vertex_buffers_mask = 0; diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp index 13ff4a99b4..30ae53ee45 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp @@ -491,9 +491,9 @@ void BufferCacheRuntime::UnifiedMemoryHostBarrier() { }); } -void BufferCacheRuntime::UnifiedMemoryShaderWriteBarrier() { - VkAccessFlags src_access = VK_ACCESS_SHADER_WRITE_BIT; - VkPipelineStageFlags src_stages = vk::PIPELINE_STAGE_GRAPHICS_COMPUTE; +void BufferCacheRuntime::UnifiedMemoryWriteBarrier() { + VkAccessFlags src_access = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; + VkPipelineStageFlags src_stages = vk::PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER; if (device.IsExtTransformFeedbackSupported()) { src_access |= VK_ACCESS_TRANSFORM_FEEDBACK_WRITE_BIT_EXT; src_stages |= VK_PIPELINE_STAGE_TRANSFORM_FEEDBACK_BIT_EXT; diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index b95cbfaa98..15658380bc 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -187,6 +187,11 @@ public: BindTransformFeedbackBuffer(index, VK_NULL_HANDLE, 0, 0); } + [[nodiscard]] VkBuffer ResolveWindowHandle(const Buffer& buffer, bool from_window, + size_t window_index) const noexcept { + return from_window ? UnifiedWindowBuffer(window_index) : buffer.Handle(); + } + void BindBufferFromWindow(size_t window_index, u32 offset, u32 size) { guest_descriptor_queue.AddBuffer(UnifiedWindowBuffer(window_index), UnifiedWindowAddress(window_index), offset, size); @@ -205,7 +210,7 @@ public: void UnifiedMemoryUploadBarrier(); - void UnifiedMemoryShaderWriteBarrier(); + void UnifiedMemoryWriteBarrier(); u64 CurrentTick(); diff --git a/src/video_core/renderer_vulkan/vk_query_cache.cpp b/src/video_core/renderer_vulkan/vk_query_cache.cpp index a12dc00771..9c52d25a0f 100644 --- a/src/video_core/renderer_vulkan/vk_query_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_query_cache.cpp @@ -1412,10 +1412,16 @@ void QueryCacheRuntime::HostConditionalRenderingCompareValueImpl(VideoCommon::Lo std::scoped_lock lk(impl->buffer_cache.mutex); static constexpr auto sync_info = VideoCommon::ObtainBufferSynchronize::FullSynchronize; const auto post_op = VideoCommon::ObtainBufferOperation::DoNothing; - const auto [buffer, offset] = - impl->buffer_cache.ObtainCPUBuffer(object.address, 8, sync_info, post_op); - impl->hcr_buffer = buffer->Handle(); - impl->hcr_offset = offset; + if (const auto window = impl->buffer_cache.TryObtainWindowBuffer( + object.address, 8, Vulkan::BufferCache::UNIFIED_INDIRECT_BINDING_ALIGNMENT, post_op)) { + impl->hcr_buffer = impl->buffer_cache.runtime.UnifiedWindowBuffer(window->window_index); + impl->hcr_offset = window->offset; + } else { + const auto [buffer, offset] = + impl->buffer_cache.ObtainCPUBuffer(object.address, 8, sync_info, post_op); + impl->hcr_buffer = buffer->Handle(); + impl->hcr_offset = offset; + } } if (impl->hcr_is_set) { if (impl->hcr_setup.buffer == impl->hcr_buffer && @@ -1446,10 +1452,17 @@ void QueryCacheRuntime::HostConditionalRenderingCompareBCImpl(DAddr address, boo std::scoped_lock lk(impl->buffer_cache.mutex); const auto sync_info = VideoCommon::ObtainBufferSynchronize::FullSynchronize; const auto post_op = VideoCommon::ObtainBufferOperation::DoNothing; - const auto [buffer, offset] = - impl->buffer_cache.ObtainCPUBuffer(address, resolve_size, sync_info, post_op); - to_resolve = buffer->Handle(); - to_resolve_offset = static_cast(offset); + if (const auto window = impl->buffer_cache.TryObtainWindowBuffer( + address, resolve_size, + impl->buffer_cache.runtime.UnifiedStorageBufferAlignment(), post_op)) { + to_resolve = impl->buffer_cache.runtime.UnifiedWindowBuffer(window->window_index); + to_resolve_offset = window->offset; + } else { + const auto [buffer, offset] = + impl->buffer_cache.ObtainCPUBuffer(address, resolve_size, sync_info, post_op); + to_resolve = buffer->Handle(); + to_resolve_offset = static_cast(offset); + } } bool was_running = impl->is_hcr_running; if (was_running) { diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index 1bfdf66e0b..3648482424 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -303,11 +303,18 @@ void RasterizerVulkan::DrawIndirect() { const auto& params = maxwell3d->draw_manager.indirect_state; buffer_cache.SetDrawIndirect(¶ms); PrepareDraw(params.is_indexed, [this, ¶ms] { - const auto indirect_buffer = buffer_cache.GetDrawIndirectBuffer(); - const auto& buffer = indirect_buffer.first; - const auto& offset = indirect_buffer.second; + VkBuffer buffer_handle; + u32 offset; + if (const auto& window = buffer_cache.GetDrawIndirectWindow()) { + buffer_handle = buffer_cache.runtime.UnifiedWindowBuffer(window->window_index); + offset = window->offset; + } else { + const auto indirect_buffer = buffer_cache.GetDrawIndirectBuffer(); + buffer_handle = indirect_buffer.first->Handle(); + offset = indirect_buffer.second; + } if (params.is_byte_count) { - scheduler.Record([buffer_obj = buffer->Handle(), offset, + scheduler.Record([buffer_obj = buffer_handle, offset, stride = params.stride](vk::CommandBuffer cmdbuf) { cmdbuf.DrawIndirectByteCountEXT(1, 0, buffer_obj, offset, 0, static_cast(stride)); @@ -315,11 +322,19 @@ void RasterizerVulkan::DrawIndirect() { return; } if (params.include_count) { - const auto count = buffer_cache.GetDrawIndirectCount(); - const auto& draw_buffer = count.first; - const auto& offset_base = count.second; - scheduler.Record([draw_buffer_obj = draw_buffer->Handle(), - buffer_obj = buffer->Handle(), offset_base, offset, + VkBuffer draw_buffer_handle; + u32 offset_base; + if (const auto& count_window = buffer_cache.GetDrawIndirectCountWindow()) { + draw_buffer_handle = + buffer_cache.runtime.UnifiedWindowBuffer(count_window->window_index); + offset_base = count_window->offset; + } else { + const auto count = buffer_cache.GetDrawIndirectCount(); + draw_buffer_handle = count.first->Handle(); + offset_base = count.second; + } + scheduler.Record([draw_buffer_obj = draw_buffer_handle, + buffer_obj = buffer_handle, offset_base, offset, params](vk::CommandBuffer cmdbuf) { if (params.is_indexed) { cmdbuf.DrawIndexedIndirectCount( @@ -333,7 +348,7 @@ void RasterizerVulkan::DrawIndirect() { }); return; } - scheduler.Record([buffer_obj = buffer->Handle(), offset, params](vk::CommandBuffer cmdbuf) { + scheduler.Record([buffer_obj = buffer_handle, offset, params](vk::CommandBuffer cmdbuf) { if (params.is_indexed) { cmdbuf.DrawIndexedIndirect(buffer_obj, offset, static_cast(params.max_draw_counts), @@ -605,11 +620,21 @@ void RasterizerVulkan::DispatchCompute() { // DispatchIndirect static constexpr auto sync_info = VideoCommon::ObtainBufferSynchronize::FullSynchronize; const auto post_op = VideoCommon::ObtainBufferOperation::DiscardWrite; - const auto [buffer, offset] = - buffer_cache.ObtainBuffer(*indirect_address, 12, sync_info, post_op); + VkBuffer indirect_handle; + u32 indirect_base; + if (const auto window = buffer_cache.TryObtainWindowBufferGpu( + *indirect_address, 12, BufferCache::UNIFIED_INDIRECT_BINDING_ALIGNMENT, post_op)) { + indirect_handle = buffer_cache.runtime.UnifiedWindowBuffer(window->window_index); + indirect_base = window->offset; + } else { + const auto [buffer, offset] = + buffer_cache.ObtainBuffer(*indirect_address, 12, sync_info, post_op); + indirect_handle = buffer->Handle(); + indirect_base = offset; + } scheduler.RequestOutsideRenderPassOperationContext(); - scheduler.Record([pipeline, indirect_buffer = buffer->Handle(), - indirect_offset = offset](vk::CommandBuffer cmdbuf) { + scheduler.Record([pipeline, indirect_buffer = indirect_handle, + indirect_offset = indirect_base](vk::CommandBuffer cmdbuf) { if (!pipeline->IsBound()) { return; } @@ -1030,8 +1055,19 @@ bool AccelerateDMA::DmaBufferImageCopy(const Tegra::DMA::ImageCopy& copy_info, static constexpr auto sync_info = VideoCommon::ObtainBufferSynchronize::FullSynchronize; const auto post_op = IS_IMAGE_UPLOAD ? VideoCommon::ObtainBufferOperation::DoNothing : VideoCommon::ObtainBufferOperation::MarkAsWritten; - const auto [buffer, offset] = - buffer_cache.ObtainBuffer(buffer_operand.address, buffer_size, sync_info, post_op); + VkBuffer buffer_handle; + u32 offset; + if (const auto window = buffer_cache.TryObtainWindowBufferGpu( + buffer_operand.address, buffer_size, + BufferCache::UNIFIED_INDIRECT_BINDING_ALIGNMENT, post_op)) { + buffer_handle = buffer_cache.runtime.UnifiedWindowBuffer(window->window_index); + offset = window->offset; + } else { + const auto [buffer, buffer_offset] = + buffer_cache.ObtainBuffer(buffer_operand.address, buffer_size, sync_info, post_op); + buffer_handle = buffer->Handle(); + offset = buffer_offset; + } const auto [image, copy] = texture_cache.DmaBufferImageCopy( copy_info, buffer_operand, image_operand, image_id, IS_IMAGE_UPLOAD); @@ -1039,12 +1075,12 @@ bool AccelerateDMA::DmaBufferImageCopy(const Tegra::DMA::ImageCopy& copy_info, if constexpr (IS_IMAGE_UPLOAD) { texture_cache.PrepareImage(image_id, true, false); - image->UploadMemory(buffer->Handle(), offset, copy_span); + image->UploadMemory(buffer_handle, offset, copy_span); } else { if (offset % BytesPerBlock(image->info.format)) { return false; } - texture_cache.DownloadImageIntoBuffer(image, buffer->Handle(), offset, copy_span, + texture_cache.DownloadImageIntoBuffer(image, buffer_handle, offset, copy_span, buffer_operand.address, buffer_size); } return true;