diff --git a/src/common/settings.cpp b/src/common/settings.cpp index db318613dd..d39e57f732 100644 --- a/src/common/settings.cpp +++ b/src/common/settings.cpp @@ -47,6 +47,7 @@ SETTING(u16, false); SWITCHABLE(AnisotropyMode, true); SWITCHABLE(AntiAliasing, false); SWITCHABLE(AspectRatio, true); +SWITCHABLE(TexUnswizzleMode, true); SWITCHABLE(AstcDecodeMode, true); SWITCHABLE(AstcRecompression, true); SWITCHABLE(AudioMode, true); diff --git a/src/common/settings.h b/src/common/settings.h index 0f35045622..6db1cefea9 100644 --- a/src/common/settings.h +++ b/src/common/settings.h @@ -64,6 +64,7 @@ SETTING(u16, false); SWITCHABLE(AnisotropyMode, true); SWITCHABLE(AntiAliasing, false); SWITCHABLE(AspectRatio, true); +SWITCHABLE(TexUnswizzleMode, true); SWITCHABLE(AstcDecodeMode, true); SWITCHABLE(AstcRecompression, true); SWITCHABLE(AudioMode, true); @@ -464,6 +465,10 @@ struct Values { #endif "max_anisotropy", Category::RendererAdvanced}; + SwitchableSetting accelerate_unswizzle{linkage, + TexUnswizzleMode::Cpu, + "accelerate_unswizzle", + Category::RendererAdvanced}; SwitchableSetting accelerate_astc{linkage, AstcDecodeMode::Gpu, "accelerate_astc", diff --git a/src/common/settings_enums.h b/src/common/settings_enums.h index df71972b94..c4895d1b57 100644 --- a/src/common/settings_enums.h +++ b/src/common/settings_enums.h @@ -129,6 +129,7 @@ ENUM(TimeZone, Auto, Default, Cet, Cst6Cdt, Cuba, Eet, Egypt, Eire, Est, Est5Edt Japan, Kwajalein, Libya, Met, Mst, Mst7Mdt, Navajo, Nz, NzChat, Poland, Portugal, Prc, Pst8Pdt, Roc, Rok, Singapore, Turkey, Uct, Universal, Utc, WSu, Wet, Zulu); ENUM(AnisotropyMode, Automatic, Default, X2, X4, X8, X16, X32, X64, None); +ENUM(TexUnswizzleMode, Cpu, Gpu); ENUM(AstcDecodeMode, Cpu, Gpu, CpuAsynchronous); ENUM(AstcRecompression, Uncompressed, Bc1, Bc3); ENUM(FramePacingMode, Target_Auto, Target_30, Target_60, Target_90, Target_120); diff --git a/src/qt_common/config/shared_translation.cpp b/src/qt_common/config/shared_translation.cpp index bb4d27c198..f086e72a7d 100644 --- a/src/qt_common/config/shared_translation.cpp +++ b/src/qt_common/config/shared_translation.cpp @@ -169,6 +169,10 @@ std::unique_ptr InitializeTranslations(QObject* parent) { tr("Specifies how videos should be decoded.\nIt can either use the CPU or the GPU for " "decoding, or perform no decoding at all (black screen on videos).\n" "In most cases, GPU decoding provides the best performance.")); + INSERT(Settings, accelerate_unswizzle, tr("Texture Unsiwzzle Method:"), + tr("This option controls how generic textures should be unswizzled.\n" + "CPU: Use the CPU for unswizzling.\n" + "GPU: Use the GPU's compute shaders to unswizzling generic textures (recommended).")); INSERT(Settings, accelerate_astc, tr("ASTC Decoding Method:"), tr("This option controls how ASTC textures should be decoded.\n" "CPU: Use the CPU for decoding.\n" @@ -230,7 +234,7 @@ std::unique_ptr InitializeTranslations(QObject* parent) { INSERT(Settings, gpu_clock, tr("GPU Clocks"), tr("Makes the game believe GPU work finishes faster than it does, so it stops lowering " "resolution and render distance to fit the Switch's clocks.")); - INSERT(Settings, gpu_unswizzle_enabled, tr("GPU Unswizzle"), + INSERT(Settings, gpu_unswizzle_enabled, tr("Chunked GPU Unswizzle"), tr("Accelerates BCn 3D texture decoding using GPU compute.\n" "Disable if experiencing crashes or graphical glitches.")); INSERT(Settings, gpu_unswizzle_texture_size, tr("GPU Unswizzle Max Texture Size"), @@ -386,6 +390,11 @@ std::unique_ptr ComboboxEnumeration(QObject* parent) { PAIR(SpirvOptimizeMode, OnLoad, tr("On Load")), PAIR(SpirvOptimizeMode, Always, tr("Always")), }}); + translations->insert({Settings::EnumMetadata::Index(), + { + PAIR(TexUnswizzleMode, Cpu, tr("CPU")), + PAIR(TexUnswizzleMode, Gpu, tr("GPU")), + }}); translations->insert({Settings::EnumMetadata::Index(), { PAIR(AstcDecodeMode, Cpu, tr("CPU")), diff --git a/src/video_core/host_shaders/CMakeLists.txt b/src/video_core/host_shaders/CMakeLists.txt index 5b8f47a7a6..3d6b49807f 100644 --- a/src/video_core/host_shaders/CMakeLists.txt +++ b/src/video_core/host_shaders/CMakeLists.txt @@ -21,6 +21,7 @@ set(SHADER_FILES ${CMAKE_CURRENT_SOURCE_DIR}/blit_depth_msaa.frag ${CMAKE_CURRENT_SOURCE_DIR}/blit_depth_stencil_msaa.frag ${CMAKE_CURRENT_SOURCE_DIR}/block_linear_unswizzle_3d.comp + ${CMAKE_CURRENT_SOURCE_DIR}/block_linear_unswizzle_2d_bcn.comp ${CMAKE_CURRENT_SOURCE_DIR}/block_linear_unswizzle_3d_bcn.comp ${CMAKE_CURRENT_SOURCE_DIR}/convert_abgr8_to_d24s8.frag ${CMAKE_CURRENT_SOURCE_DIR}/convert_abgr8_to_d32f.frag diff --git a/src/video_core/host_shaders/block_linear_unswizzle_2d.comp b/src/video_core/host_shaders/block_linear_unswizzle_2d.comp index 2a3446ca66..80625f1d09 100644 --- a/src/video_core/host_shaders/block_linear_unswizzle_2d.comp +++ b/src/video_core/host_shaders/block_linear_unswizzle_2d.comp @@ -39,6 +39,7 @@ UNIFORM(4) uint block_size; UNIFORM(5) uint x_shift; UNIFORM(6) uint block_height; UNIFORM(7) uint block_height_mask; +UNIFORM(8) uvec3 valid_extent; END_PUSH_CONSTANTS #if HAS_EXTENDED_TYPES @@ -104,6 +105,10 @@ uvec4 ReadTexel(uint offset) { } void main() { + if (any(greaterThanEqual(gl_GlobalInvocationID, valid_extent))) { + return; + } + uvec3 pos = gl_GlobalInvocationID + origin; pos.x <<= bytes_per_block_log2; diff --git a/src/video_core/host_shaders/block_linear_unswizzle_2d_bcn.comp b/src/video_core/host_shaders/block_linear_unswizzle_2d_bcn.comp new file mode 100644 index 0000000000..39b1ef3851 --- /dev/null +++ b/src/video_core/host_shaders/block_linear_unswizzle_2d_bcn.comp @@ -0,0 +1,176 @@ +// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#version 430 + +#ifdef VULKAN + +#extension GL_EXT_shader_16bit_storage : require +#extension GL_EXT_shader_8bit_storage : require +#define HAS_EXTENDED_TYPES 1 +#define BEGIN_PUSH_CONSTANTS layout(push_constant) uniform PushConstants { +#define END_PUSH_CONSTANTS }; +#define UNIFORM(n) +#define BINDING_INPUT_BUFFER 0 +#define BINDING_OUTPUT_BUFFER 1 + +#else // ^^^ Vulkan ^^^ // vvv OpenGL vvv + +#extension GL_NV_gpu_shader5 : enable +#ifdef GL_NV_gpu_shader5 +#define HAS_EXTENDED_TYPES 1 +#else +#define HAS_EXTENDED_TYPES 0 +#endif +#define BEGIN_PUSH_CONSTANTS +#define END_PUSH_CONSTANTS +#define UNIFORM(n) layout (location = n) uniform +#define BINDING_INPUT_BUFFER 0 +#define BINDING_OUTPUT_BUFFER 0 + +#endif + +BEGIN_PUSH_CONSTANTS +UNIFORM(0) uvec3 origin; +UNIFORM(1) ivec3 destination; +UNIFORM(2) uint bytes_per_block_log2; +UNIFORM(3) uint layer_stride; +UNIFORM(4) uint block_size; +UNIFORM(5) uint x_shift; +UNIFORM(6) uint block_height; +UNIFORM(7) uint block_height_mask; +UNIFORM(8) uvec3 valid_extent; +UNIFORM(9) uint out_pitch; +UNIFORM(10) uint out_layer_stride; +END_PUSH_CONSTANTS + +#if HAS_EXTENDED_TYPES +layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU8 { uint8_t u8data[]; }; +layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU16 { uint16_t u16data[]; }; +#endif +layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU32 { uint u32data[]; }; +layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU64 { uvec2 u64data[]; }; +layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU128 { uvec4 u128data[]; }; + +#if HAS_EXTENDED_TYPES +layout(binding = BINDING_OUTPUT_BUFFER, std430) writeonly buffer OutputBufferU8 { uint8_t out8data[]; }; +layout(binding = BINDING_OUTPUT_BUFFER, std430) writeonly buffer OutputBufferU16 { uint16_t out16data[]; }; +#endif +layout(binding = BINDING_OUTPUT_BUFFER, std430) buffer OutputBufferU32 { uint out32data[]; }; + +layout(binding = BINDING_OUTPUT_BUFFER, std430) writeonly buffer OutputBufferU64 { uvec2 out64data[]; }; +layout(binding = BINDING_OUTPUT_BUFFER, std430) writeonly buffer OutputBufferU128 { uvec4 out128data[]; }; + +layout(local_size_x = 32, local_size_y = 32, local_size_z = 1) in; + +const uint GOB_SIZE_X = 64; +const uint GOB_SIZE_Y = 8; +const uint GOB_SIZE_Z = 1; +const uint GOB_SIZE = GOB_SIZE_X * GOB_SIZE_Y * GOB_SIZE_Z; + +const uint GOB_SIZE_X_SHIFT = 6; +const uint GOB_SIZE_Y_SHIFT = 3; +const uint GOB_SIZE_Z_SHIFT = 0; +const uint GOB_SIZE_SHIFT = GOB_SIZE_X_SHIFT + GOB_SIZE_Y_SHIFT + GOB_SIZE_Z_SHIFT; + +const uvec2 SWIZZLE_MASK = uvec2(GOB_SIZE_X - 1, GOB_SIZE_Y - 1); + +uint SwizzleTable(uint pos) { + const uint t[8] = uint[]( + 0x12100200, 0x13110301, 0x16140604, 0x17150705, + 0x1a180a08, 0x1b190b09, 0x1e1c0e0c, 0x1f1d0f0d + ); + const uint i = pos >> 4; + const uint h = (t[i / 4] >> ((i % 4) * 8)) & 0xff; + return (h << 4) | (pos & 0xf); +} + +uint SwizzleOffset(uvec2 pos) { + pos = pos & SWIZZLE_MASK; + return SwizzleTable(pos.y * 64 + pos.x); +} + +uvec4 ReadTexel(uint offset) { + switch (bytes_per_block_log2) { + #if HAS_EXTENDED_TYPES + case 0: + return uvec4(u8data[offset], 0, 0, 0); + case 1: + return uvec4(u16data[offset / 2], 0, 0, 0); + #else + case 0: + return uvec4(bitfieldExtract(u32data[offset / 4], int((offset * 8) & 24), 8), 0, 0, 0); + case 1: + return uvec4(bitfieldExtract(u32data[offset / 4], int((offset * 8) & 16), 16), 0, 0, 0); + #endif + case 2: + return uvec4(u32data[offset / 4], 0, 0, 0); + case 3: + return uvec4(u64data[offset / 8], 0, 0); + case 4: + return u128data[offset / 16]; + } + return uvec4(0); +} + +void WriteTexel(uint offset, uvec4 texel) { + switch (bytes_per_block_log2) { + #if HAS_EXTENDED_TYPES + case 0: + out8data[offset] = uint8_t(texel.x); + break; + case 1: + out16data[offset / 2] = uint16_t(texel.x); + break; + #else + case 0: { + const uint shift = (offset & 3) * 8; + atomicAnd(out32data[offset / 4], ~(0xFFu << shift)); + atomicOr(out32data[offset / 4], texel.x << shift); + break; + } + case 1: { + const uint shift = (offset & 2) * 8; + atomicAnd(out32data[offset / 4], ~(0xFFFFu << shift)); + atomicOr(out32data[offset / 4], texel.x << shift); + break; + } + #endif + case 2: + out32data[offset / 4] = texel.x; + break; + case 3: + out64data[offset / 8] = texel.xy; + break; + case 4: + out128data[offset / 16] = texel; + break; + } +} + +void main() { + if (any(greaterThanEqual(gl_GlobalInvocationID, valid_extent))) { + return; + } + + uvec3 pos = gl_GlobalInvocationID + origin; + pos.x <<= bytes_per_block_log2; + + const uint swizzle = SwizzleOffset(pos.xy); + const uint block_y = pos.y >> GOB_SIZE_Y_SHIFT; + + uint offset = 0; + offset += pos.z * layer_stride; + const uint width_in_gobs = (out_pitch + 63) / 64; + const uint row_stride = width_in_gobs * block_size; + offset += (block_y >> block_height) * row_stride; + offset += (block_y & block_height_mask) << GOB_SIZE_SHIFT; + offset += (pos.x >> GOB_SIZE_X_SHIFT) << x_shift; + offset += swizzle; + + const uvec4 texel = ReadTexel(offset); + const ivec3 coord = ivec3(gl_GlobalInvocationID) + destination; + const uint out_offset = (coord.z * out_layer_stride) + (coord.y * out_pitch) + (coord.x << bytes_per_block_log2); + + WriteTexel(out_offset, texel); +} \ No newline at end of file diff --git a/src/video_core/host_shaders/block_linear_unswizzle_3d.comp b/src/video_core/host_shaders/block_linear_unswizzle_3d.comp index 34e4f93307..185fdff818 100644 --- a/src/video_core/host_shaders/block_linear_unswizzle_3d.comp +++ b/src/video_core/host_shaders/block_linear_unswizzle_3d.comp @@ -41,6 +41,7 @@ UNIFORM(6) uint block_height; UNIFORM(7) uint block_height_mask; UNIFORM(8) uint block_depth; UNIFORM(9) uint block_depth_mask; +UNIFORM(10) uvec3 valid_extent; END_PUSH_CONSTANTS #if HAS_EXTENDED_TYPES @@ -106,6 +107,10 @@ uvec4 ReadTexel(uint offset) { } void main() { + if (any(greaterThanEqual(gl_GlobalInvocationID, valid_extent))) { + return; + } + uvec3 pos = gl_GlobalInvocationID + origin; pos.x <<= bytes_per_block_log2; diff --git a/src/video_core/renderer_opengl/gl_texture_cache.cpp b/src/video_core/renderer_opengl/gl_texture_cache.cpp index 22676cd8ee..8ce267c16a 100644 --- a/src/video_core/renderer_opengl/gl_texture_cache.cpp +++ b/src/video_core/renderer_opengl/gl_texture_cache.cpp @@ -552,6 +552,10 @@ TextureCacheRuntime::TextureCacheRuntime(const Device& device_, ProgramManager& TextureCacheRuntime::~TextureCacheRuntime() = default; +bool TextureCacheRuntime::IsUnswizzleStorageFormatSupported(PixelFormat format) { + return false; +} + void TextureCacheRuntime::Finish() { glFinish(); } @@ -652,9 +656,7 @@ void TextureCacheRuntime::BlitFramebuffer(Framebuffer* dst, Framebuffer* src, void TextureCacheRuntime::AccelerateImageUpload(Image &image, const StagingBufferMap &map, std::span swizzles, - u32 z_src_start, u32 z_image_start, u32 z_count, - [[maybe_unused]] std::span slice_has_data, - std::span slice_bounds, [[maybe_unused]] bool image_already_uploaded) { + u32 z_src_start, u32 z_image_start) { switch (image.info.type) { case ImageType::e2D: if (IsPixelFormatASTC(image.info.format)) { diff --git a/src/video_core/renderer_opengl/gl_texture_cache.h b/src/video_core/renderer_opengl/gl_texture_cache.h index f3dfcab14d..21a23381f4 100644 --- a/src/video_core/renderer_opengl/gl_texture_cache.h +++ b/src/video_core/renderer_opengl/gl_texture_cache.h @@ -41,6 +41,8 @@ using VideoCommon::NUM_RT; using VideoCommon::Region2D; using VideoCommon::RenderTargets; +using VideoCore::Surface::PixelFormat; + struct FormatProperties { GLenum compatibility_class; bool compatibility_by_size; @@ -73,6 +75,8 @@ public: StagingBufferPool& staging_buffer_pool); ~TextureCacheRuntime(); + bool IsUnswizzleStorageFormatSupported(PixelFormat format); + void Finish(); StagingBufferMap UploadStagingBuffer(size_t size, bool deferred = false); @@ -100,6 +104,10 @@ public: return true; } + bool CanAccelerateUnswizzle() const noexcept { + return false; + } + void CopyImage(Image& dst, Image& src, std::span copies); void CopyImageMSAA(Image& dst, Image& src, std::span copies); @@ -118,12 +126,9 @@ public: const Region2D& src_region, Tegra::Engines::Fermi2D::Filter filter, Tegra::Engines::Fermi2D::Operation operation); - void AccelerateImageUpload(Image&, const StagingBufferMap&, - std::span, - u32 z_src_start, u32 z_image_start, u32 z_count, - std::span slice_has_data = {}, - std::span slice_bounds = {}, - bool image_already_uploaded = false); + void AccelerateImageUpload(Image &, const StagingBufferMap &, + std::span, + u32 z_src_start, u32 z_image_start); void InsertUploadMemoryBarrier(); diff --git a/src/video_core/renderer_opengl/util_shaders.cpp b/src/video_core/renderer_opengl/util_shaders.cpp index 7ced8c77b4..c431bd0cd1 100644 --- a/src/video_core/renderer_opengl/util_shaders.cpp +++ b/src/video_core/renderer_opengl/util_shaders.cpp @@ -140,6 +140,7 @@ void UtilShaders::BlockLinearUpload2D(Image& image, const StagingBufferMap& map, glUniform1ui(5, params.x_shift); glUniform1ui(6, params.block_height); glUniform1ui(7, params.block_height_mask); + glUniform3ui(8, num_tiles.width, num_tiles.height, image.info.resources.layers); glBindBufferRange(GL_SHADER_STORAGE_BUFFER, BINDING_INPUT_BUFFER, map.buffer, input_offset, image.guest_size_bytes - swizzle.buffer_offset); glBindImageTexture(BINDING_OUTPUT_IMAGE, image.StorageHandle(), swizzle.level, GL_TRUE, 0, @@ -178,6 +179,7 @@ void UtilShaders::BlockLinearUpload3D(Image& image, const StagingBufferMap& map, glUniform1ui(7, params.block_height_mask); glUniform1ui(8, params.block_depth); glUniform1ui(9, params.block_depth_mask); + glUniform3ui(10, num_tiles.width, num_tiles.height, num_tiles.depth); glBindBufferRange(GL_SHADER_STORAGE_BUFFER, BINDING_INPUT_BUFFER, map.buffer, input_offset, image.guest_size_bytes - swizzle.buffer_offset); glBindImageTexture(BINDING_OUTPUT_IMAGE, image.StorageHandle(), swizzle.level, GL_TRUE, 0, diff --git a/src/video_core/renderer_vulkan/vk_compute_pass.cpp b/src/video_core/renderer_vulkan/vk_compute_pass.cpp index 46418aecc3..345f83e53c 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pass.cpp +++ b/src/video_core/renderer_vulkan/vk_compute_pass.cpp @@ -22,7 +22,11 @@ #include "video_core/host_shaders/resolve_conditional_render_comp_spv.h" #include "video_core/host_shaders/vulkan_quad_indexed_comp_spv.h" #include "video_core/host_shaders/vulkan_uint8_comp_spv.h" +#include "video_core/host_shaders/block_linear_unswizzle_2d_bcn_comp_spv.h" #include "video_core/host_shaders/block_linear_unswizzle_3d_bcn_comp_spv.h" +#include "video_core/host_shaders/block_linear_unswizzle_2d_comp_spv.h" +#include "video_core/host_shaders/block_linear_unswizzle_3d_comp_spv.h" +#include "video_core/host_shaders/pitch_unswizzle_comp_spv.h" #include "video_core/renderer_vulkan/vk_compute_pass.h" #include "video_core/renderer_vulkan/vk_descriptor_pool.h" #include "video_core/renderer_vulkan/vk_scheduler.h" @@ -30,6 +34,7 @@ #include "video_core/renderer_vulkan/vk_update_descriptor.h" #include "video_core/texture_cache/accelerated_swizzle.h" #include "video_core/texture_cache/types.h" +#include "video_core/texture_cache/util.h" #include "video_core/textures/decoders.h" #include "video_core/vulkan_common/vulkan_device.h" #include "video_core/vulkan_common/vulkan_wrapper.h" @@ -615,6 +620,390 @@ void ASTCDecoderPass::Assemble(Image& image, const StagingBufferRef& map, scheduler.Finish(); } +struct BlockLinearUnswizzle2DImagePushConstants { + u32 origin[3]; + u32 padding1 = 0; + + s32 destination[3]; + u32 bytes_per_block_log2; + + u32 layer_stride; + u32 block_size; + u32 x_shift; + u32 block_height; + u32 block_height_mask; + u32 padding2[3] = {0}; + + u32 valid_extent[3]; + u32 padding3 = 0; +}; +static_assert(sizeof(BlockLinearUnswizzle2DImagePushConstants) == 80, "Push constants size is incorrect"); + +struct BlockLinearUnswizzle3DImagePushConstants { + u32 origin[3]; + u32 padding1 = 0; + + s32 destination[3]; + u32 bytes_per_block_log2; + + u32 slice_size; + u32 block_size; + u32 x_shift; + u32 block_height; + u32 block_height_mask; + u32 block_depth; + u32 block_depth_mask; + u32 padding2 = 0; + + u32 valid_extent[3]; + u32 padding3 = 0; +}; +static_assert(sizeof(BlockLinearUnswizzle3DImagePushConstants) == 80, "Push constants size is incorrect"); + +BlockLinearUnswizzle2DImagePass::BlockLinearUnswizzle2DImagePass( + const Device& device_, Scheduler& scheduler_, DescriptorPool& descriptor_pool_, + StagingBufferPool& staging_buffer_pool_, + ComputePassDescriptorQueue& compute_pass_descriptor_queue_) + : ComputePass(device_, scheduler_, descriptor_pool_, ASTC_DESCRIPTOR_SET_BINDINGS, + ASTC_PASS_DESCRIPTOR_UPDATE_TEMPLATE_ENTRY, ASTC_BANK_INFO, + COMPUTE_PUSH_CONSTANT_RANGE, + BLOCK_LINEAR_UNSWIZZLE_2D_COMP_SPV), + scheduler{scheduler_}, staging_buffer_pool{staging_buffer_pool_}, + compute_pass_descriptor_queue{compute_pass_descriptor_queue_} {} + +BlockLinearUnswizzle2DImagePass::~BlockLinearUnswizzle2DImagePass() = default; + +void BlockLinearUnswizzle2DImagePass::Unswizzle( + Image& image, const StagingBufferRef& map, + std::span swizzles) { + + using namespace VideoCommon::Accelerated; + + scheduler.RequestOutsideRenderPassOperationContext(); + const VkPipeline vk_pipeline = *pipeline; + const VkImageAspectFlags aspect_mask = image.AspectMask(); + const VkImage vk_image = image.Handle(); + const bool is_initialized = image.ExchangeInitialization(); + scheduler.Record([vk_pipeline, vk_image, aspect_mask, + is_initialized](vk::CommandBuffer cmdbuf) { + const VkImageMemoryBarrier image_barrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = static_cast(is_initialized ? VK_ACCESS_SHADER_WRITE_BIT + : VK_ACCESS_NONE), + .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT, + .oldLayout = is_initialized ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_UNDEFINED, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = vk_image, + .subresourceRange{ + .aspectMask = aspect_mask, + .baseMipLevel = 0, + .levelCount = VK_REMAINING_MIP_LEVELS, + .baseArrayLayer = 0, + .layerCount = VK_REMAINING_ARRAY_LAYERS, + }, + }; + cmdbuf.PipelineBarrier(is_initialized ? vk::PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER + : VkPipelineStageFlags(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT), + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, image_barrier); + cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, vk_pipeline); + }); + for (const VideoCommon::SwizzleParameters& swizzle : swizzles) { + const size_t input_offset = swizzle.buffer_offset + map.offset; + const u32 num_dispatches_x = Common::DivCeil(swizzle.num_tiles.width, 16U); + const u32 num_dispatches_y = Common::DivCeil(swizzle.num_tiles.height, 8U); + const u32 num_dispatches_z = Common::DivCeil(swizzle.num_tiles.depth, 8U); + + compute_pass_descriptor_queue.Acquire(scheduler, 2); + compute_pass_descriptor_queue.AddBuffer(map.buffer, input_offset, + image.guest_size_bytes - swizzle.buffer_offset); + compute_pass_descriptor_queue.AddImage(image.StorageImageView(swizzle.level)); + const void* const descriptor_data{compute_pass_descriptor_queue.UpdateData()}; + + const auto params = MakeBlockLinearSwizzle3DParams(swizzle, image.info); + const std::array valid_extent{swizzle.num_tiles.width, swizzle.num_tiles.height, + swizzle.num_tiles.depth}; + scheduler.Record([this, num_dispatches_x, num_dispatches_y, num_dispatches_z, + params, valid_extent, descriptor_data](vk::CommandBuffer cmdbuf) { + const BlockLinearUnswizzle3DImagePushConstants uniforms{ + .origin = {params.origin[0], params.origin[1], params.origin[2]}, + .destination = {params.destination[0], params.destination[1], + params.destination[2]}, + .bytes_per_block_log2 = params.bytes_per_block_log2, + .slice_size = params.slice_size, + .block_size = params.block_size, + .x_shift = params.x_shift, + .block_height = params.block_height, + .block_height_mask = params.block_height_mask, + .block_depth = params.block_depth, + .block_depth_mask = params.block_depth_mask, + .valid_extent = {valid_extent[0], valid_extent[1], valid_extent[2]}, + }; + const VkDescriptorSet set = descriptor_allocator.Commit(); + device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data); + cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, *layout, 0, set, {}); + cmdbuf.PushConstants(*layout, VK_SHADER_STAGE_COMPUTE_BIT, uniforms); + cmdbuf.Dispatch(num_dispatches_x, num_dispatches_y, num_dispatches_z); + }); + } + scheduler.Record([vk_image, aspect_mask](vk::CommandBuffer cmdbuf) { + const VkImageMemoryBarrier image_barrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT, + .oldLayout = VK_IMAGE_LAYOUT_GENERAL, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = vk_image, + .subresourceRange{ + .aspectMask = aspect_mask, + .baseMipLevel = 0, + .levelCount = VK_REMAINING_MIP_LEVELS, + .baseArrayLayer = 0, + .layerCount = VK_REMAINING_ARRAY_LAYERS, + }, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, 0, image_barrier); + }); + //Causes device loss! + //scheduler.Finish(); +} + +BlockLinearUnswizzle3DImagePass::BlockLinearUnswizzle3DImagePass( + const Device& device_, Scheduler& scheduler_, DescriptorPool& descriptor_pool_, + StagingBufferPool& staging_buffer_pool_, + ComputePassDescriptorQueue& compute_pass_descriptor_queue_) + : ComputePass(device_, scheduler_, descriptor_pool_, ASTC_DESCRIPTOR_SET_BINDINGS, + ASTC_PASS_DESCRIPTOR_UPDATE_TEMPLATE_ENTRY, ASTC_BANK_INFO, + COMPUTE_PUSH_CONSTANT_RANGE, + BLOCK_LINEAR_UNSWIZZLE_3D_COMP_SPV), + scheduler{scheduler_}, staging_buffer_pool{staging_buffer_pool_}, + compute_pass_descriptor_queue{compute_pass_descriptor_queue_} {} + +BlockLinearUnswizzle3DImagePass::~BlockLinearUnswizzle3DImagePass() = default; + +void BlockLinearUnswizzle3DImagePass::Unswizzle( + Image& image, const StagingBufferRef& map, + std::span swizzles, + u32 z_src, u32 z_image_start) { + + using namespace VideoCommon::Accelerated; + + scheduler.RequestOutsideRenderPassOperationContext(); + const VkPipeline vk_pipeline = *pipeline; + const VkImageAspectFlags aspect_mask = image.AspectMask(); + const VkImage vk_image = image.Handle(); + const bool is_initialized = image.ExchangeInitialization(); + scheduler.Record([vk_pipeline, vk_image, aspect_mask, + is_initialized](vk::CommandBuffer cmdbuf) { + const VkImageMemoryBarrier image_barrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = static_cast(is_initialized ? VK_ACCESS_SHADER_WRITE_BIT + : VK_ACCESS_NONE), + .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT, + .oldLayout = is_initialized ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_UNDEFINED, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = vk_image, + .subresourceRange{ + .aspectMask = aspect_mask, + .baseMipLevel = 0, + .levelCount = VK_REMAINING_MIP_LEVELS, + .baseArrayLayer = 0, + .layerCount = VK_REMAINING_ARRAY_LAYERS, + }, + }; + cmdbuf.PipelineBarrier(is_initialized ? vk::PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER + : VkPipelineStageFlags(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT), + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, image_barrier); + cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, vk_pipeline); + }); + for (const VideoCommon::SwizzleParameters& swizzle : swizzles) { + const size_t input_offset = swizzle.buffer_offset + map.offset; + const u32 num_dispatches_x = Common::DivCeil(swizzle.num_tiles.width, 16U); + const u32 num_dispatches_y = Common::DivCeil(swizzle.num_tiles.height, 8U); + const u32 num_dispatches_z = Common::DivCeil(swizzle.num_tiles.depth, 8U); + + compute_pass_descriptor_queue.Acquire(scheduler, 2); + compute_pass_descriptor_queue.AddBuffer(map.buffer, input_offset, + image.guest_size_bytes - swizzle.buffer_offset); + compute_pass_descriptor_queue.AddImage(image.StorageImageView(swizzle.level)); + const void* const descriptor_data{compute_pass_descriptor_queue.UpdateData()}; + + const auto params_base = MakeBlockLinearSwizzle3DParams(swizzle, image.info); + auto params = params_base; + params.origin[2] += z_src; + params.destination[2] += static_cast(z_image_start); + + const std::array valid_extent{swizzle.num_tiles.width, swizzle.num_tiles.height, + swizzle.num_tiles.depth}; + scheduler.Record([this, num_dispatches_x, num_dispatches_y, num_dispatches_z, + params, valid_extent, descriptor_data](vk::CommandBuffer cmdbuf) { + const BlockLinearUnswizzle3DImagePushConstants uniforms{ + .origin = {params.origin[0], params.origin[1], params.origin[2]}, + .destination = {params.destination[0], params.destination[1], + params.destination[2]}, + .bytes_per_block_log2 = params.bytes_per_block_log2, + .slice_size = params.slice_size, + .block_size = params.block_size, + .x_shift = params.x_shift, + .block_height = params.block_height, + .block_height_mask = params.block_height_mask, + .block_depth = params.block_depth, + .block_depth_mask = params.block_depth_mask, + .valid_extent = {valid_extent[0], valid_extent[1], valid_extent[2]}, + }; + const VkDescriptorSet set = descriptor_allocator.Commit(); + device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data); + cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, *layout, 0, set, {}); + cmdbuf.PushConstants(*layout, VK_SHADER_STAGE_COMPUTE_BIT, uniforms); + cmdbuf.Dispatch(num_dispatches_x, num_dispatches_y, num_dispatches_z); + }); + } + scheduler.Record([vk_image, aspect_mask](vk::CommandBuffer cmdbuf) { + const VkImageMemoryBarrier image_barrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT, + .oldLayout = VK_IMAGE_LAYOUT_GENERAL, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = vk_image, + .subresourceRange{ + .aspectMask = aspect_mask, + .baseMipLevel = 0, + .levelCount = VK_REMAINING_MIP_LEVELS, + .baseArrayLayer = 0, + .layerCount = VK_REMAINING_ARRAY_LAYERS, + }, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, 0, image_barrier); + }); + //Causes device loss! + //scheduler.Finish(); +} + +struct BlockLinearUnswizzleLinearImagePushConstants { + std::array origin; + std::array destination; + u32 bytes_per_block; + u32 pitch; + std::array valid_extent; +}; + +BlockLinearUnswizzleLinearImagePass::BlockLinearUnswizzleLinearImagePass(const Device& device_, Scheduler& scheduler_, + DescriptorPool& descriptor_pool_, + StagingBufferPool& staging_buffer_pool_, + ComputePassDescriptorQueue& compute_pass_descriptor_queue_) + : ComputePass(device_, scheduler_, descriptor_pool_, ASTC_DESCRIPTOR_SET_BINDINGS, + ASTC_PASS_DESCRIPTOR_UPDATE_TEMPLATE_ENTRY, ASTC_BANK_INFO, + COMPUTE_PUSH_CONSTANT_RANGE, + PITCH_UNSWIZZLE_COMP_SPV), + scheduler{scheduler_}, staging_buffer_pool{staging_buffer_pool_}, + compute_pass_descriptor_queue{compute_pass_descriptor_queue_} {} + +BlockLinearUnswizzleLinearImagePass::~BlockLinearUnswizzleLinearImagePass() = default; + +void BlockLinearUnswizzleLinearImagePass::Unswizzle(Image& image, const StagingBufferRef& map, + std::span swizzles) { + + scheduler.RequestOutsideRenderPassOperationContext(); + const VkPipeline vk_pipeline = *pipeline; + const VkImageAspectFlags aspect_mask = image.AspectMask(); + const VkImage vk_image = image.Handle(); + const bool is_initialized = image.ExchangeInitialization(); + scheduler.Record([vk_pipeline, vk_image, aspect_mask, + is_initialized](vk::CommandBuffer cmdbuf) { + const VkImageMemoryBarrier image_barrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = static_cast(is_initialized ? VK_ACCESS_SHADER_WRITE_BIT + : VK_ACCESS_NONE), + .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT, + .oldLayout = is_initialized ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_UNDEFINED, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = vk_image, + .subresourceRange{ + .aspectMask = aspect_mask, + .baseMipLevel = 0, + .levelCount = VK_REMAINING_MIP_LEVELS, + .baseArrayLayer = 0, + .layerCount = VK_REMAINING_ARRAY_LAYERS, + }, + }; + cmdbuf.PipelineBarrier(is_initialized ? vk::PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER + : VkPipelineStageFlags(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT), + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, image_barrier); + cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, vk_pipeline); + }); + const u32 bytes_per_block = VideoCore::Surface::BytesPerBlock(image.info.format); + for (const VideoCommon::SwizzleParameters& swizzle : swizzles) { + const size_t input_offset = swizzle.buffer_offset + map.offset; + const u32 num_dispatches_x = Common::DivCeil(swizzle.num_tiles.width, 32u); + const u32 num_dispatches_y = Common::DivCeil(swizzle.num_tiles.height, 32u); + + compute_pass_descriptor_queue.Acquire(scheduler, 2); + compute_pass_descriptor_queue.AddBuffer(map.buffer, input_offset, + image.guest_size_bytes - swizzle.buffer_offset); + compute_pass_descriptor_queue.AddImage(image.StorageImageView(swizzle.level)); + const void* const descriptor_data{compute_pass_descriptor_queue.UpdateData()}; + + const s32 image_layers = image.info.resources.layers; + + const BlockLinearUnswizzleLinearImagePushConstants params{ + .origin = {0, 0}, + .destination = {0, 0}, + .bytes_per_block = bytes_per_block, + .pitch = image.info.pitch, + .valid_extent = {swizzle.num_tiles.width, swizzle.num_tiles.height} + }; + scheduler.Record([this, num_dispatches_x, num_dispatches_y, image_layers, params, + descriptor_data](vk::CommandBuffer cmdbuf) { + const VkDescriptorSet set = descriptor_allocator.Commit(); + device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data); + cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, *layout, 0, set, {}); + cmdbuf.PushConstants(*layout, VK_SHADER_STAGE_COMPUTE_BIT, params); + cmdbuf.Dispatch(num_dispatches_x, num_dispatches_y, image_layers); + }); + } + scheduler.Record([vk_image, aspect_mask](vk::CommandBuffer cmdbuf) { + const VkImageMemoryBarrier image_barrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT, + .oldLayout = VK_IMAGE_LAYOUT_GENERAL, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = vk_image, + .subresourceRange{ + .aspectMask = aspect_mask, + .baseMipLevel = 0, + .levelCount = VK_REMAINING_MIP_LEVELS, + .baseArrayLayer = 0, + .layerCount = VK_REMAINING_ARRAY_LAYERS, + }, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, 0, image_barrier); + }); + //Causes device loss! + //scheduler.Finish(); +} + constexpr u32 BL3D_BINDING_INPUT_BUFFER = 0; constexpr u32 BL3D_BINDING_OUTPUT_BUFFER = 1; @@ -708,7 +1097,7 @@ void BlockLinearUnswizzle3DPass::Unswizzle( { using namespace VideoCommon::Accelerated; - const u32 MAX_BATCH_SLICES = (std::min)(z_count, image.info.size.depth); + const u32 MAX_BATCH_SLICES = (std::clamp)(z_count, 1u, image.info.size.depth); static constexpr u32 MAX_WINDOW_SLICES = 4; static constexpr float AREA_GROWTH_LIMIT = 3.0f; @@ -720,8 +1109,8 @@ void BlockLinearUnswizzle3DPass::Unswizzle( const auto& sw = swizzles[0]; const auto params = MakeBlockLinearSwizzle3DParams(sw, image.info); - const u32 blocks_x = (image.info.size.width + 3) / 4; - const u32 blocks_y = (image.info.size.height + 3) / 4; + const u32 blocks_x = sw.num_tiles.width; + const u32 blocks_y = sw.num_tiles.height; const VkImageLayout initial_prior_layout = image_already_uploaded ? VK_IMAGE_LAYOUT_GENERAL @@ -729,7 +1118,7 @@ void BlockLinearUnswizzle3DPass::Unswizzle( scheduler.RequestOutsideRenderPassOperationContext(); - scheduler.Record([dst_image = image.Handle(), aspect = image.AspectMask(), initial_prior_layout](vk::CommandBuffer cmdbuf) { + scheduler.Record([dst_image = image.Handle(), aspect = image.AspectMask(), initial_prior_layout, level = sw.level](vk::CommandBuffer cmdbuf) { if (dst_image == VK_NULL_HANDLE) { return; } @@ -749,7 +1138,7 @@ void BlockLinearUnswizzle3DPass::Unswizzle( .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = dst_image, - .subresourceRange = {aspect, 0, 1, 0, 1}, + .subresourceRange = {aspect, static_cast(level), 1, 0, 1}, }; cmdbuf.PipelineBarrier(src_stage, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, pre_barrier); @@ -868,7 +1257,7 @@ void BlockLinearUnswizzle3DPass::Unswizzle( } } - scheduler.Record([dst_image = image.Handle(), aspect = image.AspectMask()](vk::CommandBuffer cmdbuf) { + scheduler.Record([dst_image = image.Handle(), aspect = image.AspectMask(), level = sw.level](vk::CommandBuffer cmdbuf) { if (dst_image == VK_NULL_HANDLE) return; const VkImageMemoryBarrier post_barrier{ @@ -881,7 +1270,7 @@ void BlockLinearUnswizzle3DPass::Unswizzle( .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = dst_image, - .subresourceRange = {aspect, 0, 1, 0, 1}, + .subresourceRange = {aspect, static_cast(level), 1, 0, 1}, }; cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, @@ -946,20 +1335,39 @@ void BlockLinearUnswizzle3DPass::UnswizzleChunk( const VkBuffer out_buffer = *image.compute_unswizzle_buffer; const VkImage dst_image = image.Handle(); const VkImageAspectFlags aspect = image.AspectMask(); + const u32 level = sw.level; const s32 dst_x = static_cast(origin_x * 4u); const s32 dst_y = static_cast(origin_y * 4u); - const u32 copy_width = extent_x * 4u; - const u32 copy_height = extent_y * 4u; + + const VideoCommon::Extent3D level_size = VideoCommon::MipSize(image.info.size, level); + const u32 level_width = (std::max)(level_size.width, 1u); + const u32 level_height = (std::max)(level_size.height, 1u); + const u32 copy_width = (std::min)(extent_x * 4u, level_width - (std::min)(level_width, static_cast(dst_x))); + const u32 copy_height = (std::min)(extent_y * 4u, level_height - (std::min)(level_height, static_cast(dst_y))); scheduler.Record([this, set, descriptor_data, pc, gx, gy, gz, z_dst, z_count, barrier_size, - out_buffer, dst_image, aspect, + out_buffer, dst_image, aspect, level, dst_x, dst_y, copy_width, copy_height](vk::CommandBuffer cmdbuf) { if (dst_image == VK_NULL_HANDLE || out_buffer == VK_NULL_HANDLE) { return; } + const VkBufferMemoryBarrier war_barrier{ + .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT, + .dstAccessMask = VK_ACCESS_SHADER_WRITE_BIT, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .buffer = out_buffer, + .offset = 0, + .size = VK_WHOLE_SIZE, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + 0, war_barrier); + device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data); cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, *pipeline); cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, *layout, 0, set, {}); @@ -985,7 +1393,7 @@ void BlockLinearUnswizzle3DPass::UnswizzleChunk( .bufferOffset = 0, // Read from start of staging buffer .bufferRowLength = 0, .bufferImageHeight = 0, - .imageSubresource = {aspect, 0, 0, 1}, + .imageSubresource = {aspect, level, 0, 1}, .imageOffset = {dst_x, dst_y, static_cast(z_dst)}, .imageExtent = {copy_width, copy_height, z_count}, }; @@ -1154,6 +1562,274 @@ void BlockLinearUnswizzle3DPass::UnswizzleZeroChunk( }); } +struct alignas(16) BlockLinearUnswizzle2DPushConstants { + u32 origin[3]; + u32 padding1; + + s32 destination[3]; + u32 bytes_per_block_log2; + + u32 layer_stride; + u32 block_size; + u32 x_shift; + u32 block_height; + + u32 block_height_mask; + u32 padding2[3]; + + u32 valid_extent[3]; + u32 out_pitch; + + u32 out_layer_stride; + u32 padding3[3]; +}; + +BlockLinearUnswizzle2DPass::BlockLinearUnswizzle2DPass( + const Device& device_, Scheduler& scheduler_, + DescriptorPool& descriptor_pool_, + StagingBufferPool& staging_buffer_pool_, + ComputePassDescriptorQueue& compute_pass_descriptor_queue_) + : ComputePass(device_, scheduler_, descriptor_pool_, + std::array{{ + { + .binding = BL3D_BINDING_INPUT_BUFFER, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .descriptorCount = 1, + .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, + .pImmutableSamplers = nullptr, + }, + { + .binding = BL3D_BINDING_OUTPUT_BUFFER, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .descriptorCount = 1, + .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, + .pImmutableSamplers = nullptr, + }, + }}, + std::array{{ + { + .dstBinding = BL3D_BINDING_INPUT_BUFFER, + .dstArrayElement = 0, + .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .offset = BL3D_BINDING_INPUT_BUFFER * sizeof(DescriptorUpdateEntry), + .stride = sizeof(DescriptorUpdateEntry), + }, + { + .dstBinding = BL3D_BINDING_OUTPUT_BUFFER, + .dstArrayElement = 0, + .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .offset = BL3D_BINDING_OUTPUT_BUFFER * sizeof(DescriptorUpdateEntry), + .stride = sizeof(DescriptorUpdateEntry), + } + }}, + DescriptorBankInfo{ + .uniform_buffers = 0, + .storage_buffers = 2, + .texture_buffers = 0, + .image_buffers = 0, + .textures = 0, + .images = 0, + .score = 2, + }, + COMPUTE_PUSH_CONSTANT_RANGE, + BLOCK_LINEAR_UNSWIZZLE_2D_BCN_COMP_SPV), + scheduler{scheduler_}, + staging_buffer_pool{staging_buffer_pool_}, + compute_pass_descriptor_queue{compute_pass_descriptor_queue_} {} + +BlockLinearUnswizzle2DPass::~BlockLinearUnswizzle2DPass() = default; + +void BlockLinearUnswizzle2DPass::Unswizzle( + Image& image, const StagingBufferRef& map, + std::span swizzles) { + + scheduler.RequestOutsideRenderPassOperationContext(); + const VkPipeline vk_pipeline = *pipeline; + const VkImageAspectFlags aspect_mask = image.AspectMask(); + const VkImage vk_image = image.Handle(); + const bool is_initialized = image.ExchangeInitialization(); + + scheduler.Record([vk_image, aspect_mask, is_initialized, vk_pipeline](vk::CommandBuffer cmdbuf) { + if (vk_image == VK_NULL_HANDLE) { + return; + } + const VkAccessFlags src_access = + is_initialized ? (VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT) : 0u; + const VkPipelineStageFlags src_stage = is_initialized + ? (VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT) + : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT; + const VkImageMemoryBarrier pre_barrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = src_access, + .dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .oldLayout = is_initialized ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_UNDEFINED, + .newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = vk_image, + .subresourceRange = {aspect_mask, 0, VK_REMAINING_MIP_LEVELS, 0, + VK_REMAINING_ARRAY_LAYERS}, + }; + cmdbuf.PipelineBarrier(src_stage, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, pre_barrier); + cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, vk_pipeline); + }); + + for (const VideoCommon::SwizzleParameters& sw : swizzles) { + const u32 bytes_per_block = VideoCore::Surface::BytesPerBlock(image.info.format); + const u32 out_pitch = sw.num_tiles.width * bytes_per_block; + const u32 out_layer_stride = sw.num_tiles.height * out_pitch; + const u32 out_tile_height = sw.num_tiles.height; + + const VkDeviceSize required_size = static_cast(out_layer_stride) * image.info.resources.layers; + + if (!image.has_compute_unswizzle_buffer || required_size > image.compute_unswizzle_buffer_size) { + image.compute_unswizzle_buffer_size = required_size; + + VkBufferCreateInfo ci{ + .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .size = image.compute_unswizzle_buffer_size, + .usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | + VK_BUFFER_USAGE_TRANSFER_SRC_BIT, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, + .queueFamilyIndexCount = 0, + .pQueueFamilyIndices = nullptr, + }; + + image.compute_unswizzle_buffer = + image.runtime->memory_allocator.CreateBuffer(ci, MemoryUsage::DeviceLocal); + + image.has_compute_unswizzle_buffer = true; + image.compute_unswizzle_buffer_is_zero = false; + } + + const size_t input_offset = sw.buffer_offset + map.offset; + const u32 num_dispatches_x = Common::DivCeil(sw.num_tiles.width, 32u); + const u32 num_dispatches_y = Common::DivCeil(sw.num_tiles.height, 32u); + + compute_pass_descriptor_queue.Acquire(scheduler, 2); + compute_pass_descriptor_queue.AddBuffer(map.buffer, input_offset, + image.guest_size_bytes - sw.buffer_offset); + compute_pass_descriptor_queue.AddBuffer(*image.compute_unswizzle_buffer, 0, + image.compute_unswizzle_buffer_size); + const void* const descriptor_data{compute_pass_descriptor_queue.UpdateData()}; + + const auto params = VideoCommon::Accelerated::MakeBlockLinearSwizzle2DParams(sw, image.info); + const std::array valid_extent{sw.num_tiles.width, sw.num_tiles.height, 1u}; + const VkBuffer out_buffer = *image.compute_unswizzle_buffer; + const VkImage dst_image = vk_image; + const u32 level = sw.level; + const s32 image_layers = image.info.resources.layers; + + scheduler.Record([out_buffer](vk::CommandBuffer cmdbuf) { + const VkBufferMemoryBarrier war_barrier{ + .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT, + .dstAccessMask = VK_ACCESS_SHADER_WRITE_BIT, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .buffer = out_buffer, + .offset = 0, + .size = VK_WHOLE_SIZE, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, war_barrier); + }); + + scheduler.Record([this, num_dispatches_x, num_dispatches_y, image_layers, params, valid_extent, + out_pitch, out_layer_stride, descriptor_data](vk::CommandBuffer cmdbuf) { + const BlockLinearUnswizzle2DPushConstants uniforms{ + .origin = {params.origin[0], params.origin[1], params.origin[2]}, + .destination = {params.destination[0], params.destination[1], + params.destination[2]}, + .bytes_per_block_log2 = params.bytes_per_block_log2, + .layer_stride = params.layer_stride, + .block_size = params.block_size, + .x_shift = params.x_shift, + .block_height = params.block_height, + .block_height_mask = params.block_height_mask, + .valid_extent = {valid_extent[0], valid_extent[1], valid_extent[2]}, + .out_pitch = out_pitch, + .out_layer_stride = out_layer_stride, + }; + const VkDescriptorSet set = descriptor_allocator.Commit(); + device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data); + cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, *layout, 0, set, {}); + cmdbuf.PushConstants(*layout, VK_SHADER_STAGE_COMPUTE_BIT, uniforms); + cmdbuf.Dispatch(num_dispatches_x, num_dispatches_y, image_layers); + }); + + const VideoCommon::Extent3D level_size = VideoCommon::MipSize(image.info.size, level); + const u32 level_width = (std::max)(level_size.width, 1u); + const u32 level_height = (std::max)(level_size.height, 1u); + const PixelFormat image_format = image.info.format; + + scheduler.Record([out_buffer, dst_image, aspect_mask, level, image_layers, out_tile_height, out_pitch, bytes_per_block, + level_width, level_height, image_format](vk::CommandBuffer cmdbuf) { + if (dst_image == VK_NULL_HANDLE) { + return; + } + const VkBufferMemoryBarrier buffer_barrier{ + .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .buffer = out_buffer, + .offset = 0, + .size = VK_WHOLE_SIZE, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, buffer_barrier); + + const u32 block_width = VideoCore::Surface::DefaultBlockWidth(image_format); + const u32 block_height = VideoCore::Surface::DefaultBlockHeight(image_format); + + const VkBufferImageCopy copy{ + .bufferOffset = 0, + .bufferRowLength = (out_pitch / bytes_per_block) * block_width, + .bufferImageHeight = out_tile_height * block_height, + .imageSubresource = {aspect_mask, level, 0, static_cast(image_layers)}, + .imageOffset = {0, 0, 0}, + .imageExtent = {level_width, level_height, 1}, + }; + cmdbuf.CopyBufferToImage(out_buffer, dst_image, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, copy); + }); + } + + scheduler.Record([vk_image, aspect_mask](vk::CommandBuffer cmdbuf) { + if (vk_image == VK_NULL_HANDLE) { + return; + } + const VkImageMemoryBarrier post_barrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT, + .oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = vk_image, + .subresourceRange = {aspect_mask, 0, VK_REMAINING_MIP_LEVELS, 0, + VK_REMAINING_ARRAY_LAYERS}, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + 0, post_barrier); + }); + //Causes device loss! + //scheduler.Finish(); +} + MSAACopyPass::MSAACopyPass(const Device& device_, Scheduler& scheduler_, DescriptorPool& descriptor_pool_, StagingBufferPool& staging_buffer_pool_, diff --git a/src/video_core/renderer_vulkan/vk_compute_pass.h b/src/video_core/renderer_vulkan/vk_compute_pass.h index b92f03c86e..5a2253da21 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pass.h +++ b/src/video_core/renderer_vulkan/vk_compute_pass.h @@ -172,4 +172,110 @@ private: ComputePassDescriptorQueue& compute_pass_descriptor_queue; }; + +class MSAACopyPass final : public ComputePass { +public: + explicit MSAACopyPass(const Device& device_, Scheduler& scheduler_, + DescriptorPool& descriptor_pool_, StagingBufferPool& staging_buffer_pool_, + ComputePassDescriptorQueue& compute_pass_descriptor_queue_); + ~MSAACopyPass(); + + void CopyImage(Image& dst_image, Image& src_image, + std::span copies, bool msaa_to_non_msaa); + +private: + Scheduler& scheduler; + StagingBufferPool& staging_buffer_pool; + ComputePassDescriptorQueue& compute_pass_descriptor_queue; + std::array modules; + std::array pipelines; +}; + +class BlockLinearUnswizzle2DPass final : public ComputePass { +public: + explicit BlockLinearUnswizzle2DPass(const Device& device_, Scheduler& scheduler_, + DescriptorPool& descriptor_pool_, + StagingBufferPool& staging_buffer_pool_, + ComputePassDescriptorQueue& compute_pass_descriptor_queue_); + ~BlockLinearUnswizzle2DPass(); + + void Unswizzle(Image& image, const StagingBufferRef& map, + std::span swizzles); + +private: + Scheduler& scheduler; + StagingBufferPool& staging_buffer_pool; + ComputePassDescriptorQueue& compute_pass_descriptor_queue; +}; + +class BlockLinearUnswizzle2DImagePass final : public ComputePass { +public: + explicit BlockLinearUnswizzle2DImagePass(const Device& device_, Scheduler& scheduler_, + DescriptorPool& descriptor_pool_, + StagingBufferPool& staging_buffer_pool_, + ComputePassDescriptorQueue& compute_pass_descriptor_queue_); + ~BlockLinearUnswizzle2DImagePass(); + + void Unswizzle(Image& image, const StagingBufferRef& map, + std::span swizzles); + +private: + Scheduler& scheduler; + StagingBufferPool& staging_buffer_pool; + ComputePassDescriptorQueue& compute_pass_descriptor_queue; +}; + +class BlockLinearUnswizzle3DImagePass final : public ComputePass { +public: + explicit BlockLinearUnswizzle3DImagePass(const Device& device_, Scheduler& scheduler_, + DescriptorPool& descriptor_pool_, + StagingBufferPool& staging_buffer_pool_, + ComputePassDescriptorQueue& compute_pass_descriptor_queue_); + ~BlockLinearUnswizzle3DImagePass(); + + void Unswizzle(Image& image, const StagingBufferRef& map, + std::span swizzles, + u32 z_src = 0, u32 z_image_start = 0); + +private: + Scheduler& scheduler; + StagingBufferPool& staging_buffer_pool; + ComputePassDescriptorQueue& compute_pass_descriptor_queue; +}; + +class BlockLinearUnswizzleLinearImagePass final : public ComputePass { +public: + explicit BlockLinearUnswizzleLinearImagePass(const Device& device_, Scheduler& scheduler_, + DescriptorPool& descriptor_pool_, + StagingBufferPool& staging_buffer_pool_, + ComputePassDescriptorQueue& compute_pass_descriptor_queue_); + ~BlockLinearUnswizzleLinearImagePass(); + + void Unswizzle(Image& image, const StagingBufferRef& map, + std::span swizzles); + +private: + Scheduler& scheduler; + StagingBufferPool& staging_buffer_pool; + ComputePassDescriptorQueue& compute_pass_descriptor_queue; +}; + +class MSAACopyPass final : public ComputePass { +public: + explicit MSAACopyPass(const Device& device_, Scheduler& scheduler_, + DescriptorPool& descriptor_pool_, StagingBufferPool& staging_buffer_pool_, + ComputePassDescriptorQueue& compute_pass_descriptor_queue_); + ~MSAACopyPass(); + + void CopyImage(Image& dst_image, Image& src_image, + std::span copies, bool msaa_to_non_msaa); + +private: + Scheduler& scheduler; + StagingBufferPool& staging_buffer_pool; + ComputePassDescriptorQueue& compute_pass_descriptor_queue; + std::array modules; + std::array pipelines; +}; + } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp index 3397282e0a..6529437b73 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp @@ -185,6 +185,20 @@ constexpr VkBorderColor ConvertBorderColor(const std::array& color) { }; } +[[nodiscard]] std::optional BlockTexelViewFormat(PixelFormat format) { + if (!VideoCore::Surface::IsPixelFormatBCn(format)) { + return std::nullopt; + } + switch (VideoCore::Surface::BytesPerBlock(format)) { + case 8: + return VK_FORMAT_R32G32_UINT; + case 16: + return VK_FORMAT_R32G32B32A32_UINT; + default: + return std::nullopt; + } +} + [[nodiscard]] vk::Image MakeImage(const Device& device, const MemoryAllocator& allocator, const ImageInfo& info, std::span view_formats, std::optional format_override = {}) { @@ -202,12 +216,18 @@ constexpr VkBorderColor ConvertBorderColor(const std::array& color) { image_ci.flags |= VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT | VK_IMAGE_CREATE_EXTENDED_USAGE_BIT; + bool is_block_format = false; + if (BlockTexelViewFormat(info.format)) { + image_ci.flags |= VK_IMAGE_CREATE_BLOCK_TEXEL_VIEW_COMPATIBLE_BIT; + is_block_format = true; + } + const bool has_storage_compatible_view = std::any_of(view_formats.begin(), view_formats.end(), [&device](VkFormat view_format) { return device.IsFormatSupported(view_format, VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT, FormatType::Optimal); }); - if (has_storage_compatible_view) { + if (has_storage_compatible_view && !is_block_format) { image_ci.usage |= VK_IMAGE_USAGE_STORAGE_BIT; } @@ -218,6 +238,24 @@ constexpr VkBorderColor ConvertBorderColor(const std::array& color) { return allocator.CreateImage(image_ci); } +[[nodiscard]] VkFormat UnswizzleStorageFormat(u32 bytes_per_block) { + switch (bytes_per_block) { + case 1: + return VK_FORMAT_R8_UINT; + case 2: + return VK_FORMAT_R16_UINT; + case 4: + return VK_FORMAT_R32_UINT; + case 8: + return VK_FORMAT_R32G32_UINT; + case 16: + return VK_FORMAT_R32G32B32A32_UINT; + default: + ASSERT_MSG(false, "Invalid bytes_per_block={} for accelerated unswizzle", bytes_per_block); + return VK_FORMAT_R32_UINT; + } +} + [[nodiscard]] vk::ImageView MakeStorageView(const vk::Device& device, u32 level, VkImage image, VkFormat format) { static constexpr VkImageViewUsageCreateInfo storage_image_view_usage_create_info{ @@ -918,6 +956,11 @@ TextureCacheRuntime::TextureCacheRuntime(const Device& device_, Scheduler& sched const auto image_format = static_cast(index_a); if (IsPixelFormatASTC(image_format) && !device.IsOptimalAstcSupported()) { view_formats[index_a].push_back(VK_FORMAT_A8B8G8R8_UNORM_PACK32); + } else if (!IsPixelFormatASTC(image_format) && !IsPixelFormatBCn(image_format)) { + if (IsUnswizzleStorageFormatSupported(image_format)) { + const u32 bpp = VideoCore::Surface::BytesPerBlock(image_format); + view_formats[index_a].push_back(UnswizzleStorageFormat(bpp)); + } } for (size_t index_b = 0; index_b < VideoCore::Surface::MaxPixelFormat; index_b++) { const auto view_format = static_cast(index_b); @@ -930,11 +973,62 @@ TextureCacheRuntime::TextureCacheRuntime(const Device& device_, Scheduler& sched } if (Settings::values.gpu_unswizzle_enabled.GetValue()) { + bl2d_unswizzle_pass.emplace(device, scheduler, descriptor_pool, + staging_buffer_pool, compute_pass_descriptor_queue); bl3d_unswizzle_pass.emplace(device, scheduler, descriptor_pool, staging_buffer_pool, compute_pass_descriptor_queue); + generic_2d_unswizzle_pass.emplace(device, scheduler, descriptor_pool, + staging_buffer_pool, compute_pass_descriptor_queue); + generic_3d_unswizzle_pass.emplace(device, scheduler, descriptor_pool, + staging_buffer_pool, compute_pass_descriptor_queue); + generic_linear_unswizzle_pass.emplace(device, scheduler, descriptor_pool, + staging_buffer_pool, compute_pass_descriptor_queue); } } +[nodicard] bool ByteWidthSupported(u32 bpp, const Device& device) { + switch (bpp) { + case 1: + return device.IsStorageBuffer8BitAccessSupported(); + case 2: + return device.IsStorageBuffer16BitAccessSupported(); + case 4: + case 8: + case 16: + return true; + default: + return false; + } +} + +bool TextureCacheRuntime::IsUnswizzleStorageFormatSupported(PixelFormat format) const { + const u32 bytes_per_block = VideoCore::Surface::BytesPerBlock(format); + if (!ByteWidthSupported(bytes_per_block, device)) { + return false; + } + PixelFormat reference_format; + switch (bytes_per_block) { + case 1: + reference_format = PixelFormat::R8_UINT; + break; + case 2: + reference_format = PixelFormat::R16_UINT; + break; + case 4: + reference_format = PixelFormat::R32_UINT; + break; + case 8: + reference_format = PixelFormat::R32G32_UINT; + break; + case 16: + reference_format = PixelFormat::R32G32B32A32_UINT; + break; + default: + return false; + } + return VideoCore::Surface::IsViewCompatible(format, reference_format, false, true); +} + void TextureCacheRuntime::Finish() { scheduler.Finish(); } @@ -1772,6 +1866,8 @@ void TextureCacheRuntime::TickFrame() { sentenced_unswizzle_buffers.Tick(); } +// Is this even needed? I wasn't sure if the buffer was cleared when done with, +// someone who knows how these systems works better than me could probably explain void TextureCacheRuntime::ReleaseSparseUnswizzleBuffer(Image& image) { if (image.has_compute_unswizzle_buffer) { sentenced_unswizzle_buffers.Push(std::move(image.compute_unswizzle_buffer)); @@ -1784,11 +1880,36 @@ void TextureCacheRuntime::ReleaseSparseUnswizzleBuffer(Image& image) { }); } +[[nodiscard]] boost::container::small_vector BuildViewFormats( + const ImageInfo& info, std::span base_view_formats) { + boost::container::small_vector formats(base_view_formats.begin(), + base_view_formats.end()); + if (Settings::values.gpu_unswizzle_enabled.GetValue()) { + if (const auto block_view_format = BlockTexelViewFormat(info.format)) { + formats.push_back(*block_view_format); + } + } + return formats; +} + Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu_addr_, VAddr cpu_addr_) : VideoCommon::ImageBase(info_, gpu_addr_, cpu_addr_), scheduler{&runtime_.scheduler}, runtime{&runtime_}, original_image(MakeImage(runtime_.device, runtime_.memory_allocator, info, + WillUseWidenedAstcFormat(runtime_.device, info) + ? std::span{} + : runtime->ViewFormats(info.format), + WillUseWidenedAstcFormat(runtime_.device, info) + ? std::make_optional(VK_FORMAT_R32G32B32A32_SFLOAT) + : std::nullopt)), + WillUseWidenedAstcFormat(runtime_.device, info) + ? std::span{} + : std::span(BuildViewFormats( + info, runtime->ViewFormats(info.format))), + WillUseWidenedAstcFormat(runtime_.device, info) + ? std::make_optional(VK_FORMAT_R32G32B32A32_SFLOAT) + : std::nullopt)), runtime->ViewFormats(info.format))), aspect_mask(ImageAspectMask(info.format)) { if (IsPixelFormatASTC(info.format) && !runtime->device.IsOptimalAstcSupported()) { @@ -1811,6 +1932,10 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu flags |= VideoCommon::ImageFlagBits::Converted; flags |= VideoCommon::ImageFlagBits::CostlyLoad; } + if (runtime->IsUnswizzleStorageFormatSupported(info.format) && + (info.type == ImageType::e2D || info.type == ImageType::e3D || info.type == ImageType::Linear)) { + flags |= VideoCommon::ImageFlagBits::AcceleratedUpload; + } if (runtime->device.HasDebuggingToolAttached()) { original_image.SetObjectNameEXT(VideoCommon::Name(*this).c_str()); } @@ -1823,6 +1948,14 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu storage_image_views[level] = MakeStorageView(device, level, *original_image, storage_format); } + } else if (True(flags & VideoCommon::ImageFlagBits::AcceleratedUpload) && runtime->IsUnswizzleStorageFormatSupported(info.format)) { + const auto& device = runtime->device.GetLogical(); + const VkFormat storage_format = + UnswizzleStorageFormat(VideoCore::Surface::BytesPerBlock(info.format)); + for (s32 level = 0; level < info.resources.levels; ++level) { + storage_image_views[level] = + MakeStorageView(device, level, *original_image, storage_format); + } } } @@ -1843,8 +1976,8 @@ void Image::AllocateComputeUnswizzleBuffer(u32 max_slices) { using VideoCore::Surface::BytesPerBlock; const u32 block_bytes = BytesPerBlock(info.format); // 8 for BC1, 16 for BC6H - const u32 block_width = 4; - const u32 block_height = 4; + const u32 block_width = VideoCore::Surface::DefaultBlockWidth(info.format); + const u32 block_height = VideoCore::Surface::DefaultBlockHeight(info.format); // BCn is 4x4x1 blocks const u32 blocks_x = (info.size.width + block_width - 1) / block_width; @@ -2205,6 +2338,14 @@ VkImageView Image::StorageImageView(s32 level) noexcept { auto format_info = MaxwellToVK::SurfaceFormat(runtime->device, FormatType::Optimal, true, info.format); if (WillUseAcceleratedAstcDecode(runtime->device, info)) { + format_info.format = WillUseWidenedAstcFormat(runtime->device, info) + ? VK_FORMAT_R32G32B32A32_SFLOAT + : VK_FORMAT_A8B8G8R8_UNORM_PACK32; + format_info.format = WillUseWidenedAstcFormat(runtime->device, info) + ? VK_FORMAT_R32G32B32A32_SFLOAT + : VK_FORMAT_A8B8G8R8_UNORM_PACK32; + } else if (const auto block_view_format = BlockTexelViewFormat(info.format)) { + format_info.format = *block_view_format; format_info.format = VK_FORMAT_A8B8G8R8_UNORM_PACK32; } view = MakeStorageView(runtime->device.GetLogical(), level, *(this->*current_image), @@ -2848,33 +2989,26 @@ VkRenderPass Framebuffer::RenderPassVariant(u32 color_clear_mask, bool depth_ste void TextureCacheRuntime::AccelerateImageUpload( Image& image, const StagingBufferRef& map, std::span swizzles, - u32 z_src_start, u32 z_image_start, u32 z_count, - std::span slice_has_data, - std::span slice_bounds, - bool image_already_uploaded) { + u32 z_src_start, u32 z_image_start) { if (IsPixelFormatASTC(image.info.format)) { return astc_decoder_pass->Assemble(image, map, swizzles); } - if (!Settings::values.gpu_unswizzle_enabled.GetValue() || !bl3d_unswizzle_pass) { - if (IsPixelFormatBCn(image.info.format) && image.info.type == ImageType::e3D) { - ASSERT(false && "GPU unswizzle is disabled for BCn 3D texture"); - } - ASSERT(false); + if (!Settings::values.gpu_unswizzle_enabled.GetValue() || (!generic_2d_unswizzle_pass && !generic_3d_unswizzle_pass)) { + ASSERT(false && "GPU unswizzle is disabled for this texture"); return; } - if (bl3d_unswizzle_pass && - IsPixelFormatBCn(image.info.format) && - image.info.type == ImageType::e3D && - image.info.resources.levels == 1 && - image.info.resources.layers == 1) { - - return bl3d_unswizzle_pass->Unswizzle(image, map, swizzles, - z_src_start, z_image_start, z_count, - slice_has_data, slice_bounds, - image_already_uploaded); + if (image.info.type == ImageType::e2D && generic_2d_unswizzle_pass) { + return generic_2d_unswizzle_pass->Unswizzle(image, map, swizzles); + } + if (image.info.type == ImageType::e3D && generic_3d_unswizzle_pass) { + return generic_3d_unswizzle_pass->Unswizzle(image, map, swizzles, + z_src_start, z_image_start); + } + if (image.info.type == ImageType::Linear && generic_linear_unswizzle_pass) { + return generic_linear_unswizzle_pass->Unswizzle(image, map, swizzles); } ASSERT(false); diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h index b6ca70da58..475ef83c90 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.h +++ b/src/video_core/renderer_vulkan/vk_texture_cache.h @@ -51,6 +51,8 @@ public: DescriptorPool& descriptor_pool, ComputePassDescriptorQueue& compute_pass_descriptor_queue); + bool IsUnswizzleStorageFormatSupported(PixelFormat format) const; + void Finish(); StagingBufferRef UploadStagingBuffer(size_t size, bool deferred = false); @@ -94,12 +96,14 @@ public: return true; } - void AccelerateImageUpload(Image&, const StagingBufferRef&, - std::span, - u32 z_src_start, u32 z_image_start, u32 z_count, - std::span slice_has_data = {}, - std::span slice_bounds = {}, - bool image_already_uploaded = false); + bool CanAccelerateUnswizzle() const noexcept { + return Settings::values.accelerate_unswizzle.GetValue() == + Settings::TexUnswizzleMode::Gpu; + } + + void AccelerateImageUpload(Image &, const StagingBufferRef &, + std::span, + u32 z_src_start, u32 z_image_start); void InsertUploadMemoryBarrier() {} @@ -155,6 +159,12 @@ public: std::optional astc_decoder_pass; std::optional bl3d_unswizzle_pass; + std::optional msaa_copy_pass; + std::optional bl2d_unswizzle_pass; + std::optional generic_2d_unswizzle_pass; + std::optional generic_3d_unswizzle_pass; + std::optional generic_linear_unswizzle_pass; + std::optional msaa_copy_pass; const Settings::ResolutionScalingInfo& resolution; std::array, VideoCore::Surface::MaxPixelFormat> view_formats; @@ -332,6 +342,7 @@ public: u64 allocation_tick; friend class BlockLinearUnswizzle3DPass; + friend class BlockLinearUnswizzle2DPass; friend class TextureCacheRuntime; private: diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 685810337f..62a1edb990 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -1119,9 +1119,7 @@ void TextureCache

::RefreshContents(Image& image, ImageId image_id) { return; } - const bool gpu_unswizzle_enabled = Settings::values.gpu_unswizzle_enabled.GetValue(); - - if (gpu_unswizzle_enabled && + if (Settings::values.gpu_unswizzle_enabled.GetValue() && IsPixelFormatBCn(image.info.format) && image.info.type == ImageType::e3D && image.info.resources.levels == 1 && @@ -1144,11 +1142,13 @@ void TextureCache

::UploadImageContents(Image& image, StagingBuffer& staging) const GPUVAddr gpu_addr = image.gpu_addr; if (True(image.flags & ImageFlagBits::AcceleratedUpload)) { - gpu_memory->ReadBlock(gpu_addr, mapped_span.data(), mapped_span.size_bytes(), - VideoCommon::CacheType::NoTextureCache); - const auto uploads = FullUploadSwizzles(image.info); - runtime.AccelerateImageUpload(image, staging, FixSmallVectorADL(uploads), 0, 0, 0); - return; + if (IsPixelFormatASTC(image.info.format) || runtime.CanAccelerateUnswizzle()) { + const auto uploads = FullUploadSwizzles(image.info); + gpu_memory->ReadBlock(gpu_addr, mapped_span.data(), mapped_span.size_bytes(), + VideoCommon::CacheType::NoTextureCache); + runtime.AccelerateImageUpload(image, staging, FixSmallVectorADL(uploads), 0, 0); + return; + } } Tegra::Memory::GpuGuestMemory swizzle_data( @@ -1427,65 +1427,31 @@ void TextureCache

::TickAsyncUnswizzle() { gpu_memory->GetSubmappedRange(image.gpu_addr, image.guest_size_bytes); task.sparse_segments.assign(segs.begin(), segs.end()); - std::sort(task.sparse_segments.begin(), task.sparse_segments.end(), - [](const auto& a, const auto& b) { return a.first < b.first; }); - task.segment_scan_cursor = 0; - task.slice_has_data.assign(image.info.size.depth, 0u); - task.slice_bounds.assign(image.info.size.depth, {}); if (image.info.size.depth > 1) { const auto uploads = FullUploadSwizzles(task.info); const auto sp = VideoCommon::Accelerated::MakeBlockLinearSwizzle3DParams( uploads[0], task.info); - const u64 swizzled_slice_size = sp.slice_size; - task.swizzled_slice_size = swizzled_slice_size; - task.swizzle_block_depth = sp.block_depth; - - const u32 blocks_x = Common::DivCeil(task.info.size.width, 4u); - const u32 blocks_y = Common::DivCeil(task.info.size.height, 4u); - const u32 zz_count = 1u << sp.block_depth; - const u32 total_groups = Common::DivCeil( - static_cast(image.info.size.depth), zz_count); - u32 group_watermark = 0; - - for (const auto& [seg_gpu_addr, seg_size] : task.sparse_segments) { - const u64 seg_start = seg_gpu_addr - image.gpu_addr; - const u64 seg_end = seg_start + seg_size; - - while (group_watermark < total_groups && - static_cast(group_watermark) * swizzled_slice_size + swizzled_slice_size <= seg_start) { - ++group_watermark; - } - - for (u32 g = group_watermark; g < total_groups; ++g) { - const u64 group_base = static_cast(g) * swizzled_slice_size; - if (group_base >= seg_end) break; - - const u64 local_start = (std::max)(seg_start, group_base) - group_base; - const u64 local_end = (std::min)(seg_end, group_base + swizzled_slice_size) - group_base; - - VideoCommon::Accelerated::ForEachZInGroupOverlap( - local_start, local_end, - sp.block_size, sp.x_shift, sp.block_height, sp.block_height_mask, - sp.block_depth, sp.block_depth_mask, bytes_per_block, blocks_x, blocks_y, - [&](u32 zz, const VideoCommon::Accelerated::SliceBBox& box) { - const u32 z = g * zz_count + zz; - if (z >= static_cast(image.info.size.depth)) return; - - task.slice_has_data[z] = 1u; - auto& acc = task.slice_bounds[z]; - if (acc.x1 <= acc.x0 || acc.y1 <= acc.y0) { - acc = box; - } else { - acc.x0 = (std::min)(acc.x0, box.x0); - acc.y0 = (std::min)(acc.y0, box.y0); - acc.x1 = (std::max)(acc.x1, box.x1); - acc.y1 = (std::max)(acc.y1, box.y1); - } - }); + task.swizzle_group_size = sp.slice_size; + task.slices_per_group = 1u << sp.block_depth; + + const u32 num_groups = Common::DivCeil( + static_cast(image.info.size.depth), task.slices_per_group); + task.slice_has_data.assign(num_groups, 0u); + + if (task.swizzle_group_size > 0) { + for (const auto& [seg_gpu_addr, seg_size] : task.sparse_segments) { + if (seg_gpu_addr < image.gpu_addr) continue; + const u64 seg_start = seg_gpu_addr - image.gpu_addr; + const u64 seg_end = seg_start + seg_size; + const u32 g_first = static_cast(seg_start / task.swizzle_group_size); + const u32 g_last = static_cast((seg_end - 1) / task.swizzle_group_size); + for (u32 g = g_first; g <= g_last && g < num_groups; ++g) { + task.slice_has_data[g] = 1u; + } } } } else { @@ -1495,10 +1461,6 @@ void TextureCache

::TickAsyncUnswizzle() { } task.initialized = true; - task.was_rescaled = image.IsRescaled(); - if (task.was_rescaled) { - image.ScaleDown(true); - } } // Read data @@ -1526,8 +1488,8 @@ void TextureCache

::TickAsyncUnswizzle() { size_t cursor = read_start; const bool can_smart_skip = - task.swizzle_block_depth == 0 && - task.swizzled_slice_size > 0 && + task.swizzle_group_size > 0 && + task.slices_per_group > 0 && !task.slice_has_data.empty(); auto fill_gap = [&](size_t gap_start_rel, size_t gap_end_rel) { @@ -1539,14 +1501,14 @@ void TextureCache

::TickAsyncUnswizzle() { size_t pos = gap_start_rel; while (pos < gap_end_rel) { const u64 abs_pos = pos + base_off; - const u32 z_abs = static_cast(abs_pos / task.swizzled_slice_size); - const u32 z_in_bm = z_abs - task.incremental_z_start; - if (z_in_bm >= static_cast(task.slice_has_data.size())) break; - const size_t slice_abs_end = - (static_cast(z_abs) + 1) * task.swizzled_slice_size; - const size_t end_rel = (std::min)(gap_end_rel, - slice_abs_end - base_off); - if (task.slice_has_data[z_in_bm]) + const u32 z_group = static_cast(abs_pos / task.swizzle_group_size); + const u32 z_group_local = z_group - + (task.incremental_z_start / task.slices_per_group); + if (z_group_local >= static_cast(task.slice_has_data.size())) break; + const size_t group_abs_end = + (static_cast(z_group) + 1) * task.swizzle_group_size; + const size_t end_rel = (std::min)(gap_end_rel, group_abs_end - base_off); + if (task.slice_has_data[z_group_local]) std::memset(staging_base + pos, 0, end_rel - pos); pos = end_rel; } @@ -1589,12 +1551,12 @@ void TextureCache

::TickAsyncUnswizzle() { } const bool is_final_batch = task.current_offset >= task.total_size; - const size_t bytes_ready = task.current_offset - task.last_submitted_offset; - const u32 complete_slices = static_cast(bytes_ready / task.bytes_per_slice); + const bool is_sparse_3d = task.is_sparse && task.swizzle_group_size > 0 && task.slices_per_group > 0; + const size_t effective_group_size = is_sparse_3d ? task.swizzle_group_size : task.bytes_per_slice; + const u32 slices_per_group_val = is_sparse_3d ? task.slices_per_group : 1u; - const std::span sparse_hint = - task.is_sparse ? std::span(task.slice_has_data) - : std::span{}; + const size_t bytes_ready = task.current_offset - task.last_submitted_offset; + const u32 complete_slices = static_cast(bytes_ready / effective_group_size) * slices_per_group_val; const u32 total_slices = task.is_incremental ? task.incremental_z_count @@ -1603,37 +1565,34 @@ void TextureCache

::TickAsyncUnswizzle() { ? task.incremental_z_count : (swizzle_slices_per_batch == 0 ? image.info.size.depth : swizzle_slices_per_batch); - if (complete_slices >= batch || (is_final_batch && complete_slices > 0)) { - const u32 z_src = static_cast(task.last_submitted_offset / task.bytes_per_slice); - const u32 z_image = task.incremental_z_start + z_src; // + 0 for full tasks + if (complete_slices >= slices_per_group_val || (is_final_batch && complete_slices > 0)) { + const u32 z_src = static_cast(task.last_submitted_offset / effective_group_size) * slices_per_group_val; + const u32 z_image = task.incremental_z_start + z_src; const u32 z_count = (std::min)({complete_slices, batch, total_slices - z_src}); if (z_count > 0) { auto uploads = FullUploadSwizzles(task.info); if (task.is_incremental) { uploads[0].num_tiles.depth = task.incremental_z_count; + } else if (task.is_sparse) { + uploads[0].num_tiles.depth = z_count; } runtime.AccelerateImageUpload(image, task.staging_buffer, FixSmallVectorADL(uploads), - z_src, z_image, z_count, - sparse_hint, - std::span(task.slice_bounds), - task.is_incremental); - task.last_submitted_offset += static_cast(z_count) * task.bytes_per_slice; + z_src, z_image); + const u32 groups_dispatched = z_count / slices_per_group_val; + task.last_submitted_offset += static_cast(groups_dispatched) * effective_group_size; } } // Check if complete - const u32 slices_submitted = static_cast(task.last_submitted_offset / task.bytes_per_slice); + const u32 slices_submitted = static_cast(task.last_submitted_offset / effective_group_size) * slices_per_group_val; const bool all_submitted = slices_submitted >= total_slices || - (is_final_batch && bytes_ready < task.bytes_per_slice); + (is_final_batch && bytes_ready < effective_group_size); if (is_final_batch && all_submitted) { runtime.FreeDeferredStagingBuffer(task.staging_buffer); runtime.ReleaseSparseUnswizzleBuffer(image); image.flags &= ~ImageFlagBits::IsDecoding; - if (task.was_rescaled) { - image.ScaleUp(); - } unswizzle_queue.pop_front(); } } diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index 69c494254b..03b11298ba 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -144,11 +144,10 @@ class TextureCache : public VideoCommon::ChannelSetupCaches slice_has_data; - std::vector slice_bounds; std::vector> sparse_segments; size_t segment_scan_cursor = 0; - u64 swizzled_slice_size = 0; - u32 swizzle_block_depth = 0; + size_t swizzle_group_size = 0; + u32 slices_per_group = 0; bool is_incremental = false; size_t staging_base_byte_offset = 0; u32 incremental_z_start = 0; @@ -440,6 +439,10 @@ private: void QueueAsyncUnswizzle(Image& image, ImageId image_id); void TickAsyncUnswizzle(); + bool IsUnswizzleStorageFormatSupported(PixelFormat format) { + return runtime.IsUnswizzleStorageFormatSupported(format); + } + Runtime& runtime; Tegra::MaxwellDeviceMemoryManager& device_memory;