eden-emu
/
eden
mirror of https://git.eden-emu.dev/eden-emu/eden.git
1
0
Fork 0
Code Releases Wiki Activity
Browse Source

Coalescing memory entries on MLP

temporary-branch
CamilleLaVey 3 days ago
parent
471778c850
commit
87ebbfb9b1
2 changed files with 136 additions and 107 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 66
      src/video_core/memory_manager.cpp
  2. 177
      src/video_core/renderer_vulkan/vk_compute_pass.cpp

66
src/video_core/memory_manager.cpp
View File

@ -370,8 +370,32 @@ inline void MemoryManager::MemoryOperation(GPUVAddr gpu_src_addr, std::size_t si
template <bool is_safe> template <bool is_safe>
void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size, void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size,
[[maybe_unused]] VideoCommon::CacheType which) const { [[maybe_unused]] VideoCommon::CacheType which) const {
const u8* run_src{nullptr};
u8* run_dst{nullptr};
std::size_t run_size{0};
auto flush_run = [&] {
if (run_size == 0) {
return;
}
std::memcpy(run_dst, run_src, run_size);
run_src = nullptr;
run_dst = nullptr;
run_size = 0;
};
auto append_run = [&](const u8* physical, std::size_t copy_amount) {
if (run_size != 0 && run_src + run_size == physical &&
run_dst + run_size == static_cast<u8*>(dest_buffer)) {
run_size += copy_amount;
return;
}
flush_run();
run_src = physical;
run_dst = static_cast<u8*>(dest_buffer);
run_size = copy_amount;
};
auto set_to_zero = [&]([[maybe_unused]] std::size_t page_index, auto set_to_zero = [&]([[maybe_unused]] std::size_t page_index,
[[maybe_unused]] std::size_t offset, std::size_t copy_amount) { [[maybe_unused]] std::size_t offset, std::size_t copy_amount) {
flush_run();
std::memset(dest_buffer, 0, copy_amount); std::memset(dest_buffer, 0, copy_amount);
dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount; dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount;
}; };
@ -381,8 +405,7 @@ void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std:
if constexpr (is_safe) { if constexpr (is_safe) {
rasterizer->FlushRegion(dev_addr_base, copy_amount, which); rasterizer->FlushRegion(dev_addr_base, copy_amount, which);
} }
u8* physical = memory.GetPointer<u8>(dev_addr_base);
std::memcpy(dest_buffer, physical, copy_amount);
append_run(memory.GetPointer<u8>(dev_addr_base), copy_amount);
dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount; dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount;
}; };
auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
@ -392,10 +415,10 @@ void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std:
rasterizer->FlushRegion(dev_addr_base, copy_amount, which); rasterizer->FlushRegion(dev_addr_base, copy_amount, which);
} }
if (!IsBigPageContinuous(page_index)) [[unlikely]] { if (!IsBigPageContinuous(page_index)) [[unlikely]] {
flush_run();
memory.ReadBlockUnsafe(dev_addr_base, dest_buffer, copy_amount); memory.ReadBlockUnsafe(dev_addr_base, dest_buffer, copy_amount);
} else { } else {
u8* physical = memory.GetPointer<u8>(dev_addr_base);
std::memcpy(dest_buffer, physical, copy_amount);
append_run(memory.GetPointer<u8>(dev_addr_base), copy_amount);
} }
dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount; dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount;
}; };
@ -405,6 +428,7 @@ void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std:
MemoryOperation<false>(base, copy_amount, mapped_normal, set_to_zero, set_to_zero); MemoryOperation<false>(base, copy_amount, mapped_normal, set_to_zero, set_to_zero);
}; };
MemoryOperation<true>(gpu_src_addr, size, mapped_big, set_to_zero, read_short_pages); MemoryOperation<true>(gpu_src_addr, size, mapped_big, set_to_zero, read_short_pages);
flush_run();
} }
void MemoryManager::ReadBlock(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size, void MemoryManager::ReadBlock(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size,
@ -420,8 +444,32 @@ void MemoryManager::ReadBlockUnsafe(GPUVAddr gpu_src_addr, void* dest_buffer,
template <bool is_safe> template <bool is_safe>
void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size, void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size,
[[maybe_unused]] VideoCommon::CacheType which) { [[maybe_unused]] VideoCommon::CacheType which) {
const u8* run_src{nullptr};
u8* run_dst{nullptr};
std::size_t run_size{0};
auto flush_run = [&] {
if (run_size == 0) {
return;
}
std::memcpy(run_dst, run_src, run_size);
run_src = nullptr;
run_dst = nullptr;
run_size = 0;
};
auto append_run = [&](u8* physical, std::size_t copy_amount) {
if (run_size != 0 && run_dst + run_size == physical &&
run_src + run_size == static_cast<const u8*>(src_buffer)) {
run_size += copy_amount;
return;
}
flush_run();
run_src = static_cast<const u8*>(src_buffer);
run_dst = physical;
run_size = copy_amount;
};
auto just_advance = [&]([[maybe_unused]] std::size_t page_index, auto just_advance = [&]([[maybe_unused]] std::size_t page_index,
[[maybe_unused]] std::size_t offset, std::size_t copy_amount) { [[maybe_unused]] std::size_t offset, std::size_t copy_amount) {
flush_run();
src_buffer = static_cast<const u8*>(src_buffer) + copy_amount; src_buffer = static_cast<const u8*>(src_buffer) + copy_amount;
}; };
auto mapped_normal = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { auto mapped_normal = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
@ -430,8 +478,7 @@ void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffe
if constexpr (is_safe) { if constexpr (is_safe) {
rasterizer->InvalidateRegion(dev_addr_base, copy_amount, which); rasterizer->InvalidateRegion(dev_addr_base, copy_amount, which);
} }
u8* physical = memory.GetPointer<u8>(dev_addr_base);
std::memcpy(physical, src_buffer, copy_amount);
append_run(memory.GetPointer<u8>(dev_addr_base), copy_amount);
src_buffer = static_cast<const u8*>(src_buffer) + copy_amount; src_buffer = static_cast<const u8*>(src_buffer) + copy_amount;
}; };
auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
@ -441,10 +488,10 @@ void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffe
rasterizer->InvalidateRegion(dev_addr_base, copy_amount, which); rasterizer->InvalidateRegion(dev_addr_base, copy_amount, which);
} }
if (!IsBigPageContinuous(page_index)) [[unlikely]] { if (!IsBigPageContinuous(page_index)) [[unlikely]] {
flush_run();
memory.WriteBlockUnsafe(dev_addr_base, src_buffer, copy_amount); memory.WriteBlockUnsafe(dev_addr_base, src_buffer, copy_amount);
} else { } else {
u8* physical = memory.GetPointer<u8>(dev_addr_base);
std::memcpy(physical, src_buffer, copy_amount);
append_run(memory.GetPointer<u8>(dev_addr_base), copy_amount);
} }
src_buffer = static_cast<const u8*>(src_buffer) + copy_amount; src_buffer = static_cast<const u8*>(src_buffer) + copy_amount;
}; };
@ -454,6 +501,7 @@ void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffe
MemoryOperation<false>(base, copy_amount, mapped_normal, just_advance, just_advance); MemoryOperation<false>(base, copy_amount, mapped_normal, just_advance, just_advance);
}; };
MemoryOperation<true>(gpu_dest_addr, size, mapped_big, just_advance, write_short_pages); MemoryOperation<true>(gpu_dest_addr, size, mapped_big, just_advance, write_short_pages);
flush_run();
} }
void MemoryManager::WriteBlock(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size, void MemoryManager::WriteBlock(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size,
@ -605,7 +653,7 @@ bool MemoryManager::IsGranularRange(GPUVAddr gpu_addr, std::size_t size) const {
if (GetEntry<true>(gpu_addr) == EntryType::Mapped) [[likely]] { if (GetEntry<true>(gpu_addr) == EntryType::Mapped) [[likely]] {
size_t page_index = gpu_addr >> big_page_bits; size_t page_index = gpu_addr >> big_page_bits;
if (IsBigPageContinuous(page_index)) [[likely]] { if (IsBigPageContinuous(page_index)) [[likely]] {
const std::size_t page{(page_index & big_page_mask) + size};
const std::size_t page{(gpu_addr & big_page_mask) + size};
return page <= big_page_size; return page <= big_page_size;
} }
const std::size_t page{(gpu_addr & Core::DEVICE_PAGEMASK) + size}; const std::size_t page{(gpu_addr & Core::DEVICE_PAGEMASK) + size};

177
src/video_core/renderer_vulkan/vk_compute_pass.cpp
View File

@ -1248,11 +1248,6 @@ constexpr DescriptorBankInfo BL3DB_BANK_INFO{
}; };
constexpr bool BL3DB_VERIFY_AGAINST_CPU = false; constexpr bool BL3DB_VERIFY_AGAINST_CPU = false;
constexpr u32 MipSize(u32 size, u32 level) {
const u32 shifted = size >> level;
return shifted != 0 ? shifted : 1u;
}
} // Anonymous namespace } // Anonymous namespace
BlockLinearUnswizzle3DBufferPass::BlockLinearUnswizzle3DBufferPass( BlockLinearUnswizzle3DBufferPass::BlockLinearUnswizzle3DBufferPass(
@ -1273,7 +1268,7 @@ bool BlockLinearUnswizzle3DBufferPass::IsSupported(const Device& device,
if (info.type != VideoCommon::ImageType::e3D) { if (info.type != VideoCommon::ImageType::e3D) {
return false; return false;
} }
if (info.resources.layers != 1) {
if (info.resources.levels != 1 || info.resources.layers != 1) {
return false; return false;
} }
if (info.num_samples > 1) { if (info.num_samples > 1) {
@ -1299,107 +1294,78 @@ void BlockLinearUnswizzle3DBufferPass::Unswizzle(
return; return;
} }
scheduler.RequestOutsideRenderPassOperationContext();
const VideoCommon::SwizzleParameters& sw = swizzles.front();
const auto params = VideoCommon::Accelerated::MakeBlockLinearSwizzle3DParams(sw, image.info);
const bool is_initialized = image.ExchangeInitialization();
const VkImage dst_image = image.Handle();
const VkImageAspectFlags aspect = image.AspectMask();
const u32 blocks_x = sw.num_tiles.width;
const u32 blocks_y = sw.num_tiles.height;
const u32 blocks_z = sw.num_tiles.depth;
const u32 bytes_per_block = 1u << params.bytes_per_block_log2;
const VkDeviceSize output_size =
static_cast<VkDeviceSize>(blocks_x) * blocks_y * blocks_z * bytes_per_block;
struct LevelCopy {
VkBuffer buffer;
VkBufferImageCopy copy;
};
std::vector<LevelCopy> level_copies;
level_copies.reserve(swizzles.size());
for (const VideoCommon::SwizzleParameters& sw : swizzles) {
const auto params =
VideoCommon::Accelerated::MakeBlockLinearSwizzle3DParams(sw, image.info);
const u32 blocks_x = sw.num_tiles.width;
const u32 blocks_y = sw.num_tiles.height;
const u32 blocks_z = sw.num_tiles.depth;
const u32 bytes_per_block = 1u << params.bytes_per_block_log2;
const VkDeviceSize output_size =
static_cast<VkDeviceSize>(blocks_x) * blocks_y * blocks_z * bytes_per_block;
if (output_size == 0) {
continue;
}
const StagingBufferRef output =
staging_buffer_pool.Request(static_cast<size_t>(output_size), MemoryUsage::DeviceLocal);
const StagingBufferRef output = staging_buffer_pool.Request(
static_cast<size_t>(output_size), MemoryUsage::DeviceLocal);
BlockLinearUnswizzle3DBufferPushConstants pc{};
pc.dim = {blocks_x, blocks_y, blocks_z};
pc.bytes_per_block_log2 = params.bytes_per_block_log2;
pc.origin = params.origin;
pc.slice_size = params.slice_size;
pc.block_size = params.block_size;
pc.x_shift = params.x_shift;
pc.block_height = params.block_height;
pc.block_height_mask = params.block_height_mask;
pc.block_depth = params.block_depth;
pc.block_depth_mask = params.block_depth_mask;
BlockLinearUnswizzle3DBufferPushConstants pc{};
pc.dim = {blocks_x, blocks_y, blocks_z};
pc.bytes_per_block_log2 = params.bytes_per_block_log2;
pc.origin = params.origin;
pc.slice_size = params.slice_size;
pc.block_size = params.block_size;
pc.x_shift = params.x_shift;
pc.block_height = params.block_height;
pc.block_height_mask = params.block_height_mask;
pc.block_depth = params.block_depth;
pc.block_depth_mask = params.block_depth_mask;
compute_pass_descriptor_queue.Acquire(scheduler, 2);
compute_pass_descriptor_queue.AddBuffer(swizzled.buffer, sw.buffer_offset + swizzled.offset,
image.guest_size_bytes - sw.buffer_offset);
compute_pass_descriptor_queue.AddBuffer(output.buffer, output.offset, output_size);
scheduler.RequestOutsideRenderPassOperationContext();
const void* descriptor_data = compute_pass_descriptor_queue.UpdateData();
const VkDescriptorSet set = descriptor_allocator.Commit();
compute_pass_descriptor_queue.Acquire(scheduler, 2);
compute_pass_descriptor_queue.AddBuffer(swizzled.buffer, sw.buffer_offset + swizzled.offset,
image.guest_size_bytes - sw.buffer_offset);
compute_pass_descriptor_queue.AddBuffer(output.buffer, output.offset, output_size);
const u32 gx = Common::DivCeil(blocks_x, 8u);
const u32 gy = Common::DivCeil(blocks_y, 8u);
const u32 gz = Common::DivCeil(blocks_z, 4u);
const void* descriptor_data = compute_pass_descriptor_queue.UpdateData();
const VkDescriptorSet set = descriptor_allocator.Commit();
scheduler.Record([this, set, descriptor_data, pc, gx, gy, gz](vk::CommandBuffer cmdbuf) {
device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data);
cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, *pipeline);
cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, *layout, 0, set, {});
cmdbuf.PushConstants(*layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);
cmdbuf.Dispatch(gx, gy, gz);
});
const u32 gx = Common::DivCeil(blocks_x, 8u);
const u32 gy = Common::DivCeil(blocks_y, 8u);
const u32 gz = Common::DivCeil(blocks_z, 4u);
const bool is_initialized = image.ExchangeInitialization();
const u32 level = static_cast<u32>(sw.level);
level_copies.push_back(LevelCopy{
.buffer = output.buffer,
.copy{
.bufferOffset = output.offset,
.bufferRowLength = 0,
.bufferImageHeight = 0,
.imageSubresource{
.aspectMask = aspect,
.mipLevel = level,
.baseArrayLayer = 0,
.layerCount = 1,
},
.imageOffset = {0, 0, 0},
.imageExtent{
.width = MipSize(image.info.size.width, level),
.height = MipSize(image.info.size.height, level),
.depth = MipSize(image.info.size.depth, level),
},
},
});
const VkBuffer out_buffer = output.buffer;
const VkDeviceSize out_offset = output.offset;
const VkImage dst_image = image.Handle();
const VkImageAspectFlags aspect = image.AspectMask();
const VkExtent3D extent{
.width = image.info.size.width,
.height = image.info.size.height,
.depth = image.info.size.depth,
};
if constexpr (BL3DB_VERIFY_AGAINST_CPU) {
VerifyAgainstCpu(swizzled, sw, image.info, output, output_size, blocks_x, blocks_y,
blocks_z, bytes_per_block);
scheduler.Record([this, set, descriptor_data, pc, gx, gy, gz, output_size, out_buffer,
out_offset, dst_image, aspect, extent,
is_initialized](vk::CommandBuffer cmdbuf) {
if (dst_image == VK_NULL_HANDLE || out_buffer == VK_NULL_HANDLE) {
return;
} }
}
if (level_copies.empty() || dst_image == VK_NULL_HANDLE) {
return;
}
device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data);
cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, *pipeline);
cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, *layout, 0, set, {});
cmdbuf.PushConstants(*layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);
cmdbuf.Dispatch(gx, gy, gz);
scheduler.Record([copies = std::move(level_copies), dst_image, aspect,
is_initialized](vk::CommandBuffer cmdbuf) {
const VkMemoryBarrier memory_barrier{
.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
const VkBufferMemoryBarrier buffer_barrier{
.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,
.pNext = nullptr, .pNext = nullptr,
.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT, .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT,
.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
.buffer = out_buffer,
.offset = out_offset,
.size = output_size,
}; };
const VkImageMemoryBarrier pre_copy{ const VkImageMemoryBarrier pre_copy{
.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
@ -1423,12 +1389,22 @@ void BlockLinearUnswizzle3DBufferPass::Unswizzle(
cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT |
(is_initialized ? VK_PIPELINE_STAGE_ALL_COMMANDS_BIT (is_initialized ? VK_PIPELINE_STAGE_ALL_COMMANDS_BIT
: VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT), : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT),
VK_PIPELINE_STAGE_TRANSFER_BIT, 0, memory_barrier, {}, pre_copy);
VK_PIPELINE_STAGE_TRANSFER_BIT, 0, {}, buffer_barrier, pre_copy);
for (const LevelCopy& level_copy : copies) {
cmdbuf.CopyBufferToImage(level_copy.buffer, dst_image,
VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, level_copy.copy);
}
const VkBufferImageCopy copy{
.bufferOffset = out_offset,
.bufferRowLength = 0,
.bufferImageHeight = 0,
.imageSubresource{
.aspectMask = aspect,
.mipLevel = 0,
.baseArrayLayer = 0,
.layerCount = 1,
},
.imageOffset = {0, 0, 0},
.imageExtent = extent,
};
cmdbuf.CopyBufferToImage(out_buffer, dst_image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, copy);
const VkImageMemoryBarrier post_copy{ const VkImageMemoryBarrier post_copy{
.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
@ -1451,6 +1427,11 @@ void BlockLinearUnswizzle3DBufferPass::Unswizzle(
cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
0, {}, {}, post_copy); 0, {}, {}, post_copy);
}); });
if constexpr (BL3DB_VERIFY_AGAINST_CPU) {
VerifyAgainstCpu(swizzled, sw, image.info, output, output_size, blocks_x, blocks_y,
blocks_z, bytes_per_block);
}
} }
void BlockLinearUnswizzle3DBufferPass::VerifyAgainstCpu( void BlockLinearUnswizzle3DBufferPass::VerifyAgainstCpu(

Write Preview
Loading…
Cancel
Save