Browse Source

Step back + keeping fixes from unified download + shared mem refactor

unified-memory-access
CamilleLaVey 2 days ago
parent
commit
4ae14993ae
  1. 4
      src/shader_recompiler/backend/spirv/emit_spirv_atomic.cpp
  2. 12
      src/shader_recompiler/backend/spirv/emit_spirv_shared_memory.cpp
  3. 21
      src/shader_recompiler/backend/spirv/spirv_emit_context.cpp
  4. 2
      src/shader_recompiler/backend/spirv/spirv_emit_context.h
  5. 3
      src/shader_recompiler/profile.h
  6. 46
      src/video_core/buffer_cache/buffer_cache.h
  7. 2
      src/video_core/buffer_cache/buffer_cache_base.h
  8. 1
      src/video_core/renderer_opengl/gl_shader_cache.cpp
  9. 80
      src/video_core/renderer_vulkan/vk_buffer_cache.cpp
  10. 7
      src/video_core/renderer_vulkan/vk_buffer_cache.h
  11. 14
      src/video_core/renderer_vulkan/vk_pipeline_cache.cpp

4
src/shader_recompiler/backend/spirv/emit_spirv_atomic.cpp

@ -13,6 +13,7 @@
namespace Shader::Backend::SPIRV { namespace Shader::Backend::SPIRV {
namespace { namespace {
Id SharedPointer(EmitContext& ctx, Id offset, u32 index_offset = 0) { Id SharedPointer(EmitContext& ctx, Id offset, u32 index_offset = 0) {
offset = ctx.BoundSharedOffset(offset, 4 + index_offset * 4);
const Id shift_id{ctx.Const(2U)}; const Id shift_id{ctx.Const(2U)};
Id index{ctx.OpShiftRightArithmetic(ctx.U32[1], offset, shift_id)}; Id index{ctx.OpShiftRightArithmetic(ctx.U32[1], offset, shift_id)};
if (index_offset > 0) { if (index_offset > 0) {
@ -160,7 +161,8 @@ Id EmitSharedAtomicExchange32(EmitContext& ctx, Id offset, Id value) {
Id EmitSharedAtomicExchange64(EmitContext& ctx, Id offset, Id value) { Id EmitSharedAtomicExchange64(EmitContext& ctx, Id offset, Id value) {
if (ctx.profile.support_shared_int64_atomics && ctx.uses_explicit_workgroup_layout) { if (ctx.profile.support_shared_int64_atomics && ctx.uses_explicit_workgroup_layout) {
const Id shift_id{ctx.Const(3U)}; const Id shift_id{ctx.Const(3U)};
const Id index{ctx.OpShiftRightArithmetic(ctx.U32[1], offset, shift_id)};
const Id index{
ctx.OpShiftRightArithmetic(ctx.U32[1], ctx.BoundSharedOffset(offset, 8), shift_id)};
const Id pointer{ const Id pointer{
ctx.OpAccessChain(ctx.shared_u64, ctx.shared_memory_u64, ctx.u32_zero_value, index)}; ctx.OpAccessChain(ctx.shared_u64, ctx.shared_memory_u64, ctx.u32_zero_value, index)};
const auto [scope, semantics]{AtomicArgs(ctx)}; const auto [scope, semantics]{AtomicArgs(ctx)};

12
src/shader_recompiler/backend/spirv/emit_spirv_shared_memory.cpp

@ -31,6 +31,7 @@ std::pair<Id, Id> ExtractArgs(EmitContext& ctx, Id offset, u32 mask, u32 count)
} // Anonymous namespace } // Anonymous namespace
Id EmitLoadSharedU8(EmitContext& ctx, Id offset) { Id EmitLoadSharedU8(EmitContext& ctx, Id offset) {
offset = ctx.BoundSharedOffset(offset, 1);
if (ctx.uses_explicit_workgroup_layout) { if (ctx.uses_explicit_workgroup_layout) {
const Id pointer{ const Id pointer{
ctx.OpAccessChain(ctx.shared_u8, ctx.shared_memory_u8, ctx.u32_zero_value, offset)}; ctx.OpAccessChain(ctx.shared_u8, ctx.shared_memory_u8, ctx.u32_zero_value, offset)};
@ -42,6 +43,7 @@ Id EmitLoadSharedU8(EmitContext& ctx, Id offset) {
} }
Id EmitLoadSharedS8(EmitContext& ctx, Id offset) { Id EmitLoadSharedS8(EmitContext& ctx, Id offset) {
offset = ctx.BoundSharedOffset(offset, 1);
if (ctx.uses_explicit_workgroup_layout) { if (ctx.uses_explicit_workgroup_layout) {
const Id pointer{ const Id pointer{
ctx.OpAccessChain(ctx.shared_u8, ctx.shared_memory_u8, ctx.u32_zero_value, offset)}; ctx.OpAccessChain(ctx.shared_u8, ctx.shared_memory_u8, ctx.u32_zero_value, offset)};
@ -53,6 +55,7 @@ Id EmitLoadSharedS8(EmitContext& ctx, Id offset) {
} }
Id EmitLoadSharedU16(EmitContext& ctx, Id offset) { Id EmitLoadSharedU16(EmitContext& ctx, Id offset) {
offset = ctx.BoundSharedOffset(offset, 2);
if (ctx.uses_explicit_workgroup_layout) { if (ctx.uses_explicit_workgroup_layout) {
const Id pointer{Pointer(ctx, ctx.shared_u16, ctx.shared_memory_u16, offset, 1)}; const Id pointer{Pointer(ctx, ctx.shared_u16, ctx.shared_memory_u16, offset, 1)};
return ctx.OpUConvert(ctx.U32[1], ctx.OpLoad(ctx.U16, pointer)); return ctx.OpUConvert(ctx.U32[1], ctx.OpLoad(ctx.U16, pointer));
@ -63,6 +66,7 @@ Id EmitLoadSharedU16(EmitContext& ctx, Id offset) {
} }
Id EmitLoadSharedS16(EmitContext& ctx, Id offset) { Id EmitLoadSharedS16(EmitContext& ctx, Id offset) {
offset = ctx.BoundSharedOffset(offset, 2);
if (ctx.uses_explicit_workgroup_layout) { if (ctx.uses_explicit_workgroup_layout) {
const Id pointer{Pointer(ctx, ctx.shared_u16, ctx.shared_memory_u16, offset, 1)}; const Id pointer{Pointer(ctx, ctx.shared_u16, ctx.shared_memory_u16, offset, 1)};
return ctx.OpSConvert(ctx.U32[1], ctx.OpLoad(ctx.U16, pointer)); return ctx.OpSConvert(ctx.U32[1], ctx.OpLoad(ctx.U16, pointer));
@ -73,6 +77,7 @@ Id EmitLoadSharedS16(EmitContext& ctx, Id offset) {
} }
Id EmitLoadSharedU32(EmitContext& ctx, Id offset) { Id EmitLoadSharedU32(EmitContext& ctx, Id offset) {
offset = ctx.BoundSharedOffset(offset, 4);
if (ctx.uses_explicit_workgroup_layout) { if (ctx.uses_explicit_workgroup_layout) {
const Id pointer{Pointer(ctx, ctx.shared_u32, ctx.shared_memory_u32, offset, 2)}; const Id pointer{Pointer(ctx, ctx.shared_u32, ctx.shared_memory_u32, offset, 2)};
return ctx.OpLoad(ctx.U32[1], pointer); return ctx.OpLoad(ctx.U32[1], pointer);
@ -82,6 +87,7 @@ Id EmitLoadSharedU32(EmitContext& ctx, Id offset) {
} }
Id EmitLoadSharedU64(EmitContext& ctx, Id offset) { Id EmitLoadSharedU64(EmitContext& ctx, Id offset) {
offset = ctx.BoundSharedOffset(offset, 8);
if (ctx.uses_explicit_workgroup_layout) { if (ctx.uses_explicit_workgroup_layout) {
const Id pointer{Pointer(ctx, ctx.shared_u32x2, ctx.shared_memory_u32x2, offset, 3)}; const Id pointer{Pointer(ctx, ctx.shared_u32x2, ctx.shared_memory_u32x2, offset, 3)};
return ctx.OpLoad(ctx.U32[2], pointer); return ctx.OpLoad(ctx.U32[2], pointer);
@ -97,6 +103,7 @@ Id EmitLoadSharedU64(EmitContext& ctx, Id offset) {
} }
Id EmitLoadSharedU128(EmitContext& ctx, Id offset) { Id EmitLoadSharedU128(EmitContext& ctx, Id offset) {
offset = ctx.BoundSharedOffset(offset, 16);
if (ctx.uses_explicit_workgroup_layout) { if (ctx.uses_explicit_workgroup_layout) {
const Id pointer{Pointer(ctx, ctx.shared_u32x4, ctx.shared_memory_u32x4, offset, 4)}; const Id pointer{Pointer(ctx, ctx.shared_u32x4, ctx.shared_memory_u32x4, offset, 4)};
return ctx.OpLoad(ctx.U32[4], pointer); return ctx.OpLoad(ctx.U32[4], pointer);
@ -113,6 +120,7 @@ Id EmitLoadSharedU128(EmitContext& ctx, Id offset) {
} }
void EmitWriteSharedU8(EmitContext& ctx, Id offset, Id value) { void EmitWriteSharedU8(EmitContext& ctx, Id offset, Id value) {
offset = ctx.BoundSharedOffset(offset, 1);
if (ctx.uses_explicit_workgroup_layout) { if (ctx.uses_explicit_workgroup_layout) {
const Id pointer{ const Id pointer{
ctx.OpAccessChain(ctx.shared_u8, ctx.shared_memory_u8, ctx.u32_zero_value, offset)}; ctx.OpAccessChain(ctx.shared_u8, ctx.shared_memory_u8, ctx.u32_zero_value, offset)};
@ -123,6 +131,7 @@ void EmitWriteSharedU8(EmitContext& ctx, Id offset, Id value) {
} }
void EmitWriteSharedU16(EmitContext& ctx, Id offset, Id value) { void EmitWriteSharedU16(EmitContext& ctx, Id offset, Id value) {
offset = ctx.BoundSharedOffset(offset, 2);
if (ctx.uses_explicit_workgroup_layout) { if (ctx.uses_explicit_workgroup_layout) {
const Id pointer{Pointer(ctx, ctx.shared_u16, ctx.shared_memory_u16, offset, 1)}; const Id pointer{Pointer(ctx, ctx.shared_u16, ctx.shared_memory_u16, offset, 1)};
ctx.OpStore(pointer, ctx.OpUConvert(ctx.U16, value)); ctx.OpStore(pointer, ctx.OpUConvert(ctx.U16, value));
@ -132,6 +141,7 @@ void EmitWriteSharedU16(EmitContext& ctx, Id offset, Id value) {
} }
void EmitWriteSharedU32(EmitContext& ctx, Id offset, Id value) { void EmitWriteSharedU32(EmitContext& ctx, Id offset, Id value) {
offset = ctx.BoundSharedOffset(offset, 4);
Id pointer{}; Id pointer{};
if (ctx.uses_explicit_workgroup_layout) { if (ctx.uses_explicit_workgroup_layout) {
pointer = Pointer(ctx, ctx.shared_u32, ctx.shared_memory_u32, offset, 2); pointer = Pointer(ctx, ctx.shared_u32, ctx.shared_memory_u32, offset, 2);
@ -144,6 +154,7 @@ void EmitWriteSharedU32(EmitContext& ctx, Id offset, Id value) {
} }
void EmitWriteSharedU64(EmitContext& ctx, Id offset, Id value) { void EmitWriteSharedU64(EmitContext& ctx, Id offset, Id value) {
offset = ctx.BoundSharedOffset(offset, 8);
if (ctx.uses_explicit_workgroup_layout) { if (ctx.uses_explicit_workgroup_layout) {
const Id pointer{Pointer(ctx, ctx.shared_u32x2, ctx.shared_memory_u32x2, offset, 3)}; const Id pointer{Pointer(ctx, ctx.shared_u32x2, ctx.shared_memory_u32x2, offset, 3)};
ctx.OpStore(pointer, value); ctx.OpStore(pointer, value);
@ -159,6 +170,7 @@ void EmitWriteSharedU64(EmitContext& ctx, Id offset, Id value) {
} }
void EmitWriteSharedU128(EmitContext& ctx, Id offset, Id value) { void EmitWriteSharedU128(EmitContext& ctx, Id offset, Id value) {
offset = ctx.BoundSharedOffset(offset, 16);
if (ctx.uses_explicit_workgroup_layout) { if (ctx.uses_explicit_workgroup_layout) {
const Id pointer{Pointer(ctx, ctx.shared_u32x4, ctx.shared_memory_u32x4, offset, 4)}; const Id pointer{Pointer(ctx, ctx.shared_u32x4, ctx.shared_memory_u32x4, offset, 4)};
ctx.OpStore(pointer, value); ctx.OpStore(pointer, value);

21
src/shader_recompiler/backend/spirv/spirv_emit_context.cpp

@ -600,6 +600,16 @@ void EmitContext::DefineLocalMemory(const IR::Program& program) {
} }
} }
Id EmitContext::BoundSharedOffset(Id offset, u32 access_bytes) {
if (shared_memory_declared_bytes == 0) {
return offset;
}
const u32 last_valid{shared_memory_declared_bytes > access_bytes
? shared_memory_declared_bytes - access_bytes
: 0U};
return OpUMin(U32[1], offset, Const(last_valid));
}
void EmitContext::DefineSharedMemory(const IR::Program& program) { void EmitContext::DefineSharedMemory(const IR::Program& program) {
uses_explicit_workgroup_layout = uses_explicit_workgroup_layout =
profile.support_explicit_workgroup_layout && profile.support_explicit_workgroup_layout &&
@ -608,8 +618,15 @@ void EmitContext::DefineSharedMemory(const IR::Program& program) {
if (program.shared_memory_size == 0) { if (program.shared_memory_size == 0) {
return; return;
} }
const u32 device_limit{profile.max_shared_memory_size};
const u32 shared_memory_size{device_limit != 0 && program.shared_memory_size > device_limit
? device_limit
: program.shared_memory_size};
if (shared_memory_size != program.shared_memory_size) {
shared_memory_declared_bytes = shared_memory_size;
}
const auto make{[&](Id element_type, u32 element_size) { const auto make{[&](Id element_type, u32 element_size) {
const u32 num_elements{Common::DivCeil(program.shared_memory_size, element_size)};
const u32 num_elements{Common::DivCeil(shared_memory_size, element_size)};
const Id array_type{TypeArray(element_type, Const(num_elements))}; const Id array_type{TypeArray(element_type, Const(num_elements))};
Decorate(array_type, spv::Decoration::ArrayStride, element_size); Decorate(array_type, spv::Decoration::ArrayStride, element_size);
@ -644,7 +661,7 @@ void EmitContext::DefineSharedMemory(const IR::Program& program) {
std::tie(shared_memory_u32x4, shared_u32x4, std::ignore) = make(U32[4], 16); std::tie(shared_memory_u32x4, shared_u32x4, std::ignore) = make(U32[4], 16);
return; return;
} }
const u32 num_elements{Common::DivCeil(program.shared_memory_size, 4U)};
const u32 num_elements{Common::DivCeil(shared_memory_size, 4U)};
const Id type{TypeArray(U32[1], Const(num_elements))}; const Id type{TypeArray(U32[1], Const(num_elements))};
shared_memory_u32_type = TypePointer(spv::StorageClass::Workgroup, type); shared_memory_u32_type = TypePointer(spv::StorageClass::Workgroup, type);

2
src/shader_recompiler/backend/spirv/spirv_emit_context.h

@ -312,6 +312,8 @@ public:
Id local_memory{}; Id local_memory{};
bool uses_explicit_workgroup_layout{}; bool uses_explicit_workgroup_layout{};
u32 shared_memory_declared_bytes{};
[[nodiscard]] Id BoundSharedOffset(Id offset, u32 access_bytes);
Id shared_memory_u8{}; Id shared_memory_u8{};
Id shared_memory_u16{}; Id shared_memory_u16{};
Id shared_memory_u32{}; Id shared_memory_u32{};

3
src/shader_recompiler/profile.h

@ -101,6 +101,9 @@ struct Profile {
u32 gl_max_compute_smem_size{}; u32 gl_max_compute_smem_size{};
/// Largest workgroup shared memory allocation the device accepts, 0 when unconstrained
u32 max_shared_memory_size{};
/// Maxwell and earlier nVidia architectures have broken robust support /// Maxwell and earlier nVidia architectures have broken robust support
bool has_broken_robust{}; bool has_broken_robust{};

46
src/video_core/buffer_cache/buffer_cache.h

@ -1871,57 +1871,11 @@ bool BufferCache<P>::TryUnifiedDownloadMemory([[maybe_unused]] Buffer& buffer,
} }
} }
template <class P>
bool BufferCache<P>::TryUnifiedUploadMemory([[maybe_unused]] Buffer& buffer,
[[maybe_unused]] std::span<const BufferCopy> copies) {
if constexpr (USE_UNIFIED_MEMORY) {
boost::container::small_vector<u64, 4> window_ids;
UnifiedWindowGroups groups;
for (const BufferCopy& copy : copies) {
if (!ResolveUnifiedWindows(buffer.CpuAddr() + copy.dst_offset, copy.dst_offset,
copy.size, window_ids, groups)) {
return false;
}
}
for (const BufferCopy& copy : copies) {
buffer.MarkUsage(copy.dst_offset, copy.size);
}
runtime.PreCopyBarrier();
boost::container::small_vector<BufferCopy, 16> window_source_copies;
for (size_t i = 0; i < window_ids.size(); ++i) {
window_source_copies.clear();
window_source_copies.reserve(groups[i].size());
for (const BufferCopy& copy : groups[i]) {
window_source_copies.push_back(BufferCopy{
.src_offset = copy.dst_offset,
.dst_offset = copy.src_offset,
.size = copy.size,
});
}
const std::span<const BufferCopy> group_span(window_source_copies.data(),
window_source_copies.size());
runtime.CopyFromUnifiedMemory(window_ids[i], buffer, group_span);
}
runtime.FlushUnifiedMemoryCopies();
runtime.PostCopyBarrier();
return true;
} else {
return false;
}
}
template <class P> template <class P>
void BufferCache<P>::MappedUploadMemory([[maybe_unused]] Buffer& buffer, void BufferCache<P>::MappedUploadMemory([[maybe_unused]] Buffer& buffer,
[[maybe_unused]] u64 total_size_bytes, [[maybe_unused]] u64 total_size_bytes,
[[maybe_unused]] std::span<BufferCopy> copies) { [[maybe_unused]] std::span<BufferCopy> copies) {
if constexpr (USE_MEMORY_MAPS) { if constexpr (USE_MEMORY_MAPS) {
if constexpr (USE_UNIFIED_MEMORY) {
if (runtime.HasUnifiedMemory() &&
!Settings::values.enable_gpu_buffer_readback.GetValue() &&
TryUnifiedUploadMemory(buffer, copies)) {
return;
}
}
auto upload_staging = runtime.UploadStagingBuffer(total_size_bytes); auto upload_staging = runtime.UploadStagingBuffer(total_size_bytes);
const std::span<u8> staging_pointer = upload_staging.mapped_span; const std::span<u8> staging_pointer = upload_staging.mapped_span;
for (BufferCopy& copy : copies) { for (BufferCopy& copy : copies) {

2
src/video_core/buffer_cache/buffer_cache_base.h

@ -446,8 +446,6 @@ private:
bool TryUnifiedDownloadMemory(Buffer& buffer, std::span<BufferCopy> copies); bool TryUnifiedDownloadMemory(Buffer& buffer, std::span<BufferCopy> copies);
bool TryUnifiedUploadMemory(Buffer& buffer, std::span<const BufferCopy> copies);
using UnifiedWindowGroups = using UnifiedWindowGroups =
boost::container::small_vector<boost::container::small_vector<BufferCopy, 16>, 4>; boost::container::small_vector<boost::container::small_vector<BufferCopy, 16>, 4>;

1
src/video_core/renderer_opengl/gl_shader_cache.cpp

@ -237,6 +237,7 @@ ShaderCache::ShaderCache(Tegra::MaxwellDeviceMemoryManager& device_memory_,
.has_gl_bool_ref_bug = device.HasBoolRefBug(), .has_gl_bool_ref_bug = device.HasBoolRefBug(),
.ignore_nan_fp_comparisons = true, .ignore_nan_fp_comparisons = true,
.gl_max_compute_smem_size = device.GetMaxComputeSharedMemorySize(), .gl_max_compute_smem_size = device.GetMaxComputeSharedMemorySize(),
.max_shared_memory_size = device.GetMaxComputeSharedMemorySize(),
.min_ssbo_alignment = device.GetShaderStorageBufferAlignment(), .min_ssbo_alignment = device.GetShaderStorageBufferAlignment(),
// Use the host limit, but never more than the guest can produce. Maxwell exposes 8 clip // Use the host limit, but never more than the guest can produce. Maxwell exposes 8 clip
// distances and the SPIR-V output array is sized for at most 8, so clamping here keeps a // distances and the SPIR-V output array is sized for at most 8, so clamping here keeps a

80
src/video_core/renderer_vulkan/vk_buffer_cache.cpp

@ -403,56 +403,33 @@ void BufferCacheRuntime::TryEnableUnifiedMemory(void* base, size_t size,
} }
} }
void BufferCacheRuntime::QueueUnifiedCopy(size_t window_index, VkBuffer buffer,
std::span<const VideoCommon::BufferCopy> copies,
bool reads_window) {
if (!unified_memory || buffer == VK_NULL_HANDLE || copies.empty() ||
void BufferCacheRuntime::CopyToUnifiedMemory(
size_t window_index, VkBuffer src_buffer,
std::span<const VideoCommon::BufferCopy> copies) {
if (!unified_memory || src_buffer == VK_NULL_HANDLE || copies.empty() ||
window_index >= unified_memory->GetWindowCount() || window_index >= unified_memory->GetWindowCount() ||
unified_memory->GetWindowBuffer(window_index) == VK_NULL_HANDLE) { unified_memory->GetWindowBuffer(window_index) == VK_NULL_HANDLE) {
return; return;
} }
PendingUnifiedCopy& pending = pending_unified_copies.emplace_back(); PendingUnifiedCopy& pending = pending_unified_copies.emplace_back();
pending.window = window_index; pending.window = window_index;
pending.buffer = buffer;
pending.reads_window = reads_window;
pending.buffer = src_buffer;
pending.copies.resize(copies.size()); pending.copies.resize(copies.size());
std::ranges::transform(copies, pending.copies.begin(), MakeBufferCopy); std::ranges::transform(copies, pending.copies.begin(), MakeBufferCopy);
} }
void BufferCacheRuntime::CopyToUnifiedMemory(
size_t window_index, VkBuffer src_buffer,
std::span<const VideoCommon::BufferCopy> copies) {
QueueUnifiedCopy(window_index, src_buffer, copies, false);
}
void BufferCacheRuntime::CopyFromUnifiedMemory(
size_t window_index, VkBuffer dst_buffer,
std::span<const VideoCommon::BufferCopy> copies) {
QueueUnifiedCopy(window_index, dst_buffer, copies, true);
}
void BufferCacheRuntime::FlushUnifiedMemoryCopies() { void BufferCacheRuntime::FlushUnifiedMemoryCopies() {
if (pending_unified_copies.empty()) { if (pending_unified_copies.empty()) {
return; return;
} }
struct UnifiedCopyCommand { struct UnifiedCopyCommand {
VkBuffer buffer; VkBuffer buffer;
bool reads_window;
boost::container::small_vector<VkBufferCopy, 8> copies; boost::container::small_vector<VkBufferCopy, 8> copies;
}; };
static constexpr VkMemoryBarrier WINDOW_TRANSFER_BARRIER{
.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
.pNext = nullptr,
.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT,
.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT,
};
std::stable_sort(pending_unified_copies.begin(), pending_unified_copies.end(), std::stable_sort(pending_unified_copies.begin(), pending_unified_copies.end(),
[](const PendingUnifiedCopy& lhs, const PendingUnifiedCopy& rhs) { [](const PendingUnifiedCopy& lhs, const PendingUnifiedCopy& rhs) {
if (lhs.window != rhs.window) {
return lhs.window < rhs.window;
}
return lhs.reads_window && !rhs.reads_window;
return lhs.window < rhs.window;
}); });
const bool foreign = unified_memory->NeedsForeignOwnershipTransfer(); const bool foreign = unified_memory->NeedsForeignOwnershipTransfer();
@ -469,20 +446,9 @@ void BufferCacheRuntime::FlushUnifiedMemoryCopies() {
const VkBuffer window_buffer = unified_memory->GetWindowBuffer(window); const VkBuffer window_buffer = unified_memory->GetWindowBuffer(window);
WindowRanges ranges; WindowRanges ranges;
VkAccessFlags window_access = 0;
size_t reads_in_group = 0;
for (size_t index = group_begin; index < group_end; ++index) { for (size_t index = group_begin; index < group_end; ++index) {
const PendingUnifiedCopy& pending = pending_unified_copies[index];
if (pending.reads_window) {
window_access |= VK_ACCESS_TRANSFER_READ_BIT;
++reads_in_group;
} else {
window_access |= VK_ACCESS_TRANSFER_WRITE_BIT;
}
for (const VkBufferCopy& copy : pending.copies) {
const VkDeviceSize offset =
pending.reads_window ? copy.srcOffset : copy.dstOffset;
ranges.emplace_back(offset, offset + copy.size);
for (const VkBufferCopy& copy : pending_unified_copies[index].copies) {
ranges.emplace_back(copy.dstOffset, copy.dstOffset + copy.size);
} }
} }
CoalesceWindowRanges(ranges); CoalesceWindowRanges(ranges);
@ -495,7 +461,7 @@ void BufferCacheRuntime::FlushUnifiedMemoryCopies() {
.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,
.pNext = nullptr, .pNext = nullptr,
.srcAccessMask = 0, .srcAccessMask = 0,
.dstAccessMask = window_access,
.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT,
.dstQueueFamilyIndex = queue_family, .dstQueueFamilyIndex = queue_family,
.buffer = window_buffer, .buffer = window_buffer,
@ -505,7 +471,7 @@ void BufferCacheRuntime::FlushUnifiedMemoryCopies() {
release.push_back(VkBufferMemoryBarrier{ release.push_back(VkBufferMemoryBarrier{
.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,
.pNext = nullptr, .pNext = nullptr,
.srcAccessMask = window_access,
.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
.dstAccessMask = 0, .dstAccessMask = 0,
.srcQueueFamilyIndex = queue_family, .srcQueueFamilyIndex = queue_family,
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT,
@ -520,36 +486,20 @@ void BufferCacheRuntime::FlushUnifiedMemoryCopies() {
commands.reserve(group_end - group_begin); commands.reserve(group_end - group_begin);
for (size_t index = group_begin; index < group_end; ++index) { for (size_t index = group_begin; index < group_end; ++index) {
PendingUnifiedCopy& pending = pending_unified_copies[index]; PendingUnifiedCopy& pending = pending_unified_copies[index];
commands.push_back(UnifiedCopyCommand{pending.buffer, pending.reads_window,
std::move(pending.copies)});
commands.push_back(UnifiedCopyCommand{pending.buffer, std::move(pending.copies)});
} }
const size_t barrier_before_writes = (reads_in_group > 0 && reads_in_group < commands.size())
? reads_in_group
: commands.size();
scheduler.RequestOutsideRenderPassOperationContext(); scheduler.RequestOutsideRenderPassOperationContext();
scheduler.Record([window_buffer, barrier_before_writes, acquire = std::move(acquire),
release = std::move(release),
scheduler.Record([window_buffer, acquire = std::move(acquire), release = std::move(release),
commands = std::move(commands)](vk::CommandBuffer cmdbuf) { commands = std::move(commands)](vk::CommandBuffer cmdbuf) {
if (!acquire.empty()) { if (!acquire.empty()) {
cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
VK_PIPELINE_STAGE_TRANSFER_BIT, 0, {}, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, {},
VideoCommon::FixSmallVectorADL(acquire), {}); VideoCommon::FixSmallVectorADL(acquire), {});
} }
for (size_t index = 0; index < commands.size(); ++index) {
if (index == barrier_before_writes) {
cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT,
VK_PIPELINE_STAGE_TRANSFER_BIT, 0,
WINDOW_TRANSFER_BARRIER);
}
const UnifiedCopyCommand& command = commands[index];
if (command.reads_window) {
cmdbuf.CopyBuffer(window_buffer, command.buffer,
VideoCommon::FixSmallVectorADL(command.copies));
} else {
cmdbuf.CopyBuffer(command.buffer, window_buffer,
VideoCommon::FixSmallVectorADL(command.copies));
}
for (const UnifiedCopyCommand& command : commands) {
cmdbuf.CopyBuffer(command.buffer, window_buffer,
VideoCommon::FixSmallVectorADL(command.copies));
} }
if (!release.empty()) { if (!release.empty()) {
cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT,

7
src/video_core/renderer_vulkan/vk_buffer_cache.h

@ -124,9 +124,6 @@ public:
void CopyToUnifiedMemory(size_t window_index, VkBuffer src_buffer, void CopyToUnifiedMemory(size_t window_index, VkBuffer src_buffer,
std::span<const VideoCommon::BufferCopy> copies); std::span<const VideoCommon::BufferCopy> copies);
void CopyFromUnifiedMemory(size_t window_index, VkBuffer dst_buffer,
std::span<const VideoCommon::BufferCopy> copies);
void FlushUnifiedMemoryCopies(); void FlushUnifiedMemoryCopies();
void UnifiedMemoryHostBarrier(); void UnifiedMemoryHostBarrier();
@ -217,13 +214,9 @@ private:
struct PendingUnifiedCopy { struct PendingUnifiedCopy {
size_t window; size_t window;
VkBuffer buffer; VkBuffer buffer;
bool reads_window;
boost::container::small_vector<VkBufferCopy, 8> copies; boost::container::small_vector<VkBufferCopy, 8> copies;
}; };
void QueueUnifiedCopy(size_t window_index, VkBuffer buffer,
std::span<const VideoCommon::BufferCopy> copies, bool reads_window);
void BindBuffer(const Buffer& buffer, u32 offset, u32 size) { void BindBuffer(const Buffer& buffer, u32 offset, u32 size) {
const VkBuffer handle = buffer.Handle(); const VkBuffer handle = buffer.Handle();
if (handle == VK_NULL_HANDLE) { if (handle == VK_NULL_HANDLE) {

14
src/video_core/renderer_vulkan/vk_pipeline_cache.cpp

@ -446,6 +446,7 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_,
.has_broken_fp16_float_controls = driver_id == VK_DRIVER_ID_NVIDIA_PROPRIETARY, .has_broken_fp16_float_controls = driver_id == VK_DRIVER_ID_NVIDIA_PROPRIETARY,
.ignore_nan_fp_comparisons = false, .ignore_nan_fp_comparisons = false,
.has_broken_spirv_subgroup_mask_vector_extract_dynamic = false, .has_broken_spirv_subgroup_mask_vector_extract_dynamic = false,
.max_shared_memory_size = device.GetMaxComputeSharedMemorySize(),
.has_broken_robust = .has_broken_robust =
device.IsNvidia() && device.GetNvidiaArch() <= NvidiaArchitecture::Arch_Pascal, device.IsNvidia() && device.GetNvidiaArch() <= NvidiaArchitecture::Arch_Pascal,
.min_ssbo_alignment = device.GetStorageBufferAlignment(), .min_ssbo_alignment = device.GetStorageBufferAlignment(),
@ -921,19 +922,6 @@ std::unique_ptr<ComputePipeline> PipelineCache::CreateComputePipeline(
} }
auto program{TranslateProgram(pools.inst, pools.block, env, cfg, host_info)}; auto program{TranslateProgram(pools.inst, pools.block, env, cfg, host_info)};
const VkDriverIdKHR driver_id = device.GetDriverID();
const bool needs_shared_mem_clamp =
driver_id == VK_DRIVER_ID_QUALCOMM_PROPRIETARY ||
driver_id == VK_DRIVER_ID_ARM_PROPRIETARY;
const u32 max_shared_memory = device.GetMaxComputeSharedMemorySize();
if (needs_shared_mem_clamp && program.shared_memory_size > max_shared_memory) {
LOG_WARNING(Render_Vulkan,
"Compute shader {:#016x} requests {}KB shared memory but device max is {}KB - clamping",
key.unique_hash,
program.shared_memory_size / 1024,
max_shared_memory / 1024);
program.shared_memory_size = max_shared_memory;
}
const std::vector<u32> code{EmitSPIRV(profile, program)}; const std::vector<u32> code{EmitSPIRV(profile, program)};
device.SaveShader(code); device.SaveShader(code);
vk::ShaderModule spv_module{BuildShader(device, code)}; vk::ShaderModule spv_module{BuildShader(device, code)};

Loading…
Cancel
Save