From 87ebbfb9b1cd99b2a644c189087f0bfd990a96f2 Mon Sep 17 00:00:00 2001 From: CamilleLaVey Date: Fri, 31 Jul 2026 04:11:40 -0400 Subject: [PATCH] Coalescing memory entries on MLP --- src/video_core/memory_manager.cpp | 66 ++++++- .../renderer_vulkan/vk_compute_pass.cpp | 177 ++++++++---------- 2 files changed, 136 insertions(+), 107 deletions(-) diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index c583fbdf7d..b124058928 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -370,8 +370,32 @@ inline void MemoryManager::MemoryOperation(GPUVAddr gpu_src_addr, std::size_t si template void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size, [[maybe_unused]] VideoCommon::CacheType which) const { + const u8* run_src{nullptr}; + u8* run_dst{nullptr}; + std::size_t run_size{0}; + auto flush_run = [&] { + if (run_size == 0) { + return; + } + std::memcpy(run_dst, run_src, run_size); + run_src = nullptr; + run_dst = nullptr; + run_size = 0; + }; + auto append_run = [&](const u8* physical, std::size_t copy_amount) { + if (run_size != 0 && run_src + run_size == physical && + run_dst + run_size == static_cast(dest_buffer)) { + run_size += copy_amount; + return; + } + flush_run(); + run_src = physical; + run_dst = static_cast(dest_buffer); + run_size = copy_amount; + }; auto set_to_zero = [&]([[maybe_unused]] std::size_t page_index, [[maybe_unused]] std::size_t offset, std::size_t copy_amount) { + flush_run(); std::memset(dest_buffer, 0, copy_amount); dest_buffer = static_cast(dest_buffer) + copy_amount; }; @@ -381,8 +405,7 @@ void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std: if constexpr (is_safe) { rasterizer->FlushRegion(dev_addr_base, copy_amount, which); } - u8* physical = memory.GetPointer(dev_addr_base); - std::memcpy(dest_buffer, physical, copy_amount); + append_run(memory.GetPointer(dev_addr_base), copy_amount); dest_buffer = static_cast(dest_buffer) + copy_amount; }; auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { @@ -392,10 +415,10 @@ void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std: rasterizer->FlushRegion(dev_addr_base, copy_amount, which); } if (!IsBigPageContinuous(page_index)) [[unlikely]] { + flush_run(); memory.ReadBlockUnsafe(dev_addr_base, dest_buffer, copy_amount); } else { - u8* physical = memory.GetPointer(dev_addr_base); - std::memcpy(dest_buffer, physical, copy_amount); + append_run(memory.GetPointer(dev_addr_base), copy_amount); } dest_buffer = static_cast(dest_buffer) + copy_amount; }; @@ -405,6 +428,7 @@ void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std: MemoryOperation(base, copy_amount, mapped_normal, set_to_zero, set_to_zero); }; MemoryOperation(gpu_src_addr, size, mapped_big, set_to_zero, read_short_pages); + flush_run(); } void MemoryManager::ReadBlock(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size, @@ -420,8 +444,32 @@ void MemoryManager::ReadBlockUnsafe(GPUVAddr gpu_src_addr, void* dest_buffer, template void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size, [[maybe_unused]] VideoCommon::CacheType which) { + const u8* run_src{nullptr}; + u8* run_dst{nullptr}; + std::size_t run_size{0}; + auto flush_run = [&] { + if (run_size == 0) { + return; + } + std::memcpy(run_dst, run_src, run_size); + run_src = nullptr; + run_dst = nullptr; + run_size = 0; + }; + auto append_run = [&](u8* physical, std::size_t copy_amount) { + if (run_size != 0 && run_dst + run_size == physical && + run_src + run_size == static_cast(src_buffer)) { + run_size += copy_amount; + return; + } + flush_run(); + run_src = static_cast(src_buffer); + run_dst = physical; + run_size = copy_amount; + }; auto just_advance = [&]([[maybe_unused]] std::size_t page_index, [[maybe_unused]] std::size_t offset, std::size_t copy_amount) { + flush_run(); src_buffer = static_cast(src_buffer) + copy_amount; }; auto mapped_normal = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { @@ -430,8 +478,7 @@ void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffe if constexpr (is_safe) { rasterizer->InvalidateRegion(dev_addr_base, copy_amount, which); } - u8* physical = memory.GetPointer(dev_addr_base); - std::memcpy(physical, src_buffer, copy_amount); + append_run(memory.GetPointer(dev_addr_base), copy_amount); src_buffer = static_cast(src_buffer) + copy_amount; }; auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { @@ -441,10 +488,10 @@ void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffe rasterizer->InvalidateRegion(dev_addr_base, copy_amount, which); } if (!IsBigPageContinuous(page_index)) [[unlikely]] { + flush_run(); memory.WriteBlockUnsafe(dev_addr_base, src_buffer, copy_amount); } else { - u8* physical = memory.GetPointer(dev_addr_base); - std::memcpy(physical, src_buffer, copy_amount); + append_run(memory.GetPointer(dev_addr_base), copy_amount); } src_buffer = static_cast(src_buffer) + copy_amount; }; @@ -454,6 +501,7 @@ void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffe MemoryOperation(base, copy_amount, mapped_normal, just_advance, just_advance); }; MemoryOperation(gpu_dest_addr, size, mapped_big, just_advance, write_short_pages); + flush_run(); } void MemoryManager::WriteBlock(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size, @@ -605,7 +653,7 @@ bool MemoryManager::IsGranularRange(GPUVAddr gpu_addr, std::size_t size) const { if (GetEntry(gpu_addr) == EntryType::Mapped) [[likely]] { size_t page_index = gpu_addr >> big_page_bits; if (IsBigPageContinuous(page_index)) [[likely]] { - const std::size_t page{(page_index & big_page_mask) + size}; + const std::size_t page{(gpu_addr & big_page_mask) + size}; return page <= big_page_size; } const std::size_t page{(gpu_addr & Core::DEVICE_PAGEMASK) + size}; diff --git a/src/video_core/renderer_vulkan/vk_compute_pass.cpp b/src/video_core/renderer_vulkan/vk_compute_pass.cpp index babaf13644..7dab36e81c 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pass.cpp +++ b/src/video_core/renderer_vulkan/vk_compute_pass.cpp @@ -1248,11 +1248,6 @@ constexpr DescriptorBankInfo BL3DB_BANK_INFO{ }; constexpr bool BL3DB_VERIFY_AGAINST_CPU = false; - -constexpr u32 MipSize(u32 size, u32 level) { - const u32 shifted = size >> level; - return shifted != 0 ? shifted : 1u; -} } // Anonymous namespace BlockLinearUnswizzle3DBufferPass::BlockLinearUnswizzle3DBufferPass( @@ -1273,7 +1268,7 @@ bool BlockLinearUnswizzle3DBufferPass::IsSupported(const Device& device, if (info.type != VideoCommon::ImageType::e3D) { return false; } - if (info.resources.layers != 1) { + if (info.resources.levels != 1 || info.resources.layers != 1) { return false; } if (info.num_samples > 1) { @@ -1299,107 +1294,78 @@ void BlockLinearUnswizzle3DBufferPass::Unswizzle( return; } - scheduler.RequestOutsideRenderPassOperationContext(); + const VideoCommon::SwizzleParameters& sw = swizzles.front(); + const auto params = VideoCommon::Accelerated::MakeBlockLinearSwizzle3DParams(sw, image.info); - const bool is_initialized = image.ExchangeInitialization(); - const VkImage dst_image = image.Handle(); - const VkImageAspectFlags aspect = image.AspectMask(); + const u32 blocks_x = sw.num_tiles.width; + const u32 blocks_y = sw.num_tiles.height; + const u32 blocks_z = sw.num_tiles.depth; + const u32 bytes_per_block = 1u << params.bytes_per_block_log2; + const VkDeviceSize output_size = + static_cast(blocks_x) * blocks_y * blocks_z * bytes_per_block; - struct LevelCopy { - VkBuffer buffer; - VkBufferImageCopy copy; - }; - std::vector level_copies; - level_copies.reserve(swizzles.size()); - - for (const VideoCommon::SwizzleParameters& sw : swizzles) { - const auto params = - VideoCommon::Accelerated::MakeBlockLinearSwizzle3DParams(sw, image.info); - - const u32 blocks_x = sw.num_tiles.width; - const u32 blocks_y = sw.num_tiles.height; - const u32 blocks_z = sw.num_tiles.depth; - const u32 bytes_per_block = 1u << params.bytes_per_block_log2; - const VkDeviceSize output_size = - static_cast(blocks_x) * blocks_y * blocks_z * bytes_per_block; - if (output_size == 0) { - continue; - } + const StagingBufferRef output = + staging_buffer_pool.Request(static_cast(output_size), MemoryUsage::DeviceLocal); - const StagingBufferRef output = staging_buffer_pool.Request( - static_cast(output_size), MemoryUsage::DeviceLocal); - - BlockLinearUnswizzle3DBufferPushConstants pc{}; - pc.dim = {blocks_x, blocks_y, blocks_z}; - pc.bytes_per_block_log2 = params.bytes_per_block_log2; - pc.origin = params.origin; - pc.slice_size = params.slice_size; - pc.block_size = params.block_size; - pc.x_shift = params.x_shift; - pc.block_height = params.block_height; - pc.block_height_mask = params.block_height_mask; - pc.block_depth = params.block_depth; - pc.block_depth_mask = params.block_depth_mask; + BlockLinearUnswizzle3DBufferPushConstants pc{}; + pc.dim = {blocks_x, blocks_y, blocks_z}; + pc.bytes_per_block_log2 = params.bytes_per_block_log2; + pc.origin = params.origin; + pc.slice_size = params.slice_size; + pc.block_size = params.block_size; + pc.x_shift = params.x_shift; + pc.block_height = params.block_height; + pc.block_height_mask = params.block_height_mask; + pc.block_depth = params.block_depth; + pc.block_depth_mask = params.block_depth_mask; - compute_pass_descriptor_queue.Acquire(scheduler, 2); - compute_pass_descriptor_queue.AddBuffer(swizzled.buffer, sw.buffer_offset + swizzled.offset, - image.guest_size_bytes - sw.buffer_offset); - compute_pass_descriptor_queue.AddBuffer(output.buffer, output.offset, output_size); + scheduler.RequestOutsideRenderPassOperationContext(); - const void* descriptor_data = compute_pass_descriptor_queue.UpdateData(); - const VkDescriptorSet set = descriptor_allocator.Commit(); + compute_pass_descriptor_queue.Acquire(scheduler, 2); + compute_pass_descriptor_queue.AddBuffer(swizzled.buffer, sw.buffer_offset + swizzled.offset, + image.guest_size_bytes - sw.buffer_offset); + compute_pass_descriptor_queue.AddBuffer(output.buffer, output.offset, output_size); - const u32 gx = Common::DivCeil(blocks_x, 8u); - const u32 gy = Common::DivCeil(blocks_y, 8u); - const u32 gz = Common::DivCeil(blocks_z, 4u); + const void* descriptor_data = compute_pass_descriptor_queue.UpdateData(); + const VkDescriptorSet set = descriptor_allocator.Commit(); - scheduler.Record([this, set, descriptor_data, pc, gx, gy, gz](vk::CommandBuffer cmdbuf) { - device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data); - cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, *pipeline); - cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, *layout, 0, set, {}); - cmdbuf.PushConstants(*layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc); - cmdbuf.Dispatch(gx, gy, gz); - }); + const u32 gx = Common::DivCeil(blocks_x, 8u); + const u32 gy = Common::DivCeil(blocks_y, 8u); + const u32 gz = Common::DivCeil(blocks_z, 4u); + const bool is_initialized = image.ExchangeInitialization(); - const u32 level = static_cast(sw.level); - level_copies.push_back(LevelCopy{ - .buffer = output.buffer, - .copy{ - .bufferOffset = output.offset, - .bufferRowLength = 0, - .bufferImageHeight = 0, - .imageSubresource{ - .aspectMask = aspect, - .mipLevel = level, - .baseArrayLayer = 0, - .layerCount = 1, - }, - .imageOffset = {0, 0, 0}, - .imageExtent{ - .width = MipSize(image.info.size.width, level), - .height = MipSize(image.info.size.height, level), - .depth = MipSize(image.info.size.depth, level), - }, - }, - }); + const VkBuffer out_buffer = output.buffer; + const VkDeviceSize out_offset = output.offset; + const VkImage dst_image = image.Handle(); + const VkImageAspectFlags aspect = image.AspectMask(); + const VkExtent3D extent{ + .width = image.info.size.width, + .height = image.info.size.height, + .depth = image.info.size.depth, + }; - if constexpr (BL3DB_VERIFY_AGAINST_CPU) { - VerifyAgainstCpu(swizzled, sw, image.info, output, output_size, blocks_x, blocks_y, - blocks_z, bytes_per_block); + scheduler.Record([this, set, descriptor_data, pc, gx, gy, gz, output_size, out_buffer, + out_offset, dst_image, aspect, extent, + is_initialized](vk::CommandBuffer cmdbuf) { + if (dst_image == VK_NULL_HANDLE || out_buffer == VK_NULL_HANDLE) { + return; } - } - - if (level_copies.empty() || dst_image == VK_NULL_HANDLE) { - return; - } + device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data); + cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, *pipeline); + cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, *layout, 0, set, {}); + cmdbuf.PushConstants(*layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc); + cmdbuf.Dispatch(gx, gy, gz); - scheduler.Record([copies = std::move(level_copies), dst_image, aspect, - is_initialized](vk::CommandBuffer cmdbuf) { - const VkMemoryBarrier memory_barrier{ - .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + const VkBufferMemoryBarrier buffer_barrier{ + .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, .pNext = nullptr, .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT, .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .buffer = out_buffer, + .offset = out_offset, + .size = output_size, }; const VkImageMemoryBarrier pre_copy{ .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, @@ -1423,12 +1389,22 @@ void BlockLinearUnswizzle3DBufferPass::Unswizzle( cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | (is_initialized ? VK_PIPELINE_STAGE_ALL_COMMANDS_BIT : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT), - VK_PIPELINE_STAGE_TRANSFER_BIT, 0, memory_barrier, {}, pre_copy); + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, {}, buffer_barrier, pre_copy); - for (const LevelCopy& level_copy : copies) { - cmdbuf.CopyBufferToImage(level_copy.buffer, dst_image, - VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, level_copy.copy); - } + const VkBufferImageCopy copy{ + .bufferOffset = out_offset, + .bufferRowLength = 0, + .bufferImageHeight = 0, + .imageSubresource{ + .aspectMask = aspect, + .mipLevel = 0, + .baseArrayLayer = 0, + .layerCount = 1, + }, + .imageOffset = {0, 0, 0}, + .imageExtent = extent, + }; + cmdbuf.CopyBufferToImage(out_buffer, dst_image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, copy); const VkImageMemoryBarrier post_copy{ .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, @@ -1451,6 +1427,11 @@ void BlockLinearUnswizzle3DBufferPass::Unswizzle( cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, {}, {}, post_copy); }); + + if constexpr (BL3DB_VERIFY_AGAINST_CPU) { + VerifyAgainstCpu(swizzled, sw, image.info, output, output_size, blocks_x, blocks_y, + blocks_z, bytes_per_block); + } } void BlockLinearUnswizzle3DBufferPass::VerifyAgainstCpu(