Browse Source

[TEST] Refactor on memory collections

vk-experiments9
CamilleLaVey 6 days ago
parent
commit
43b28d3731
  1. 2
      src/android/app/build.gradle.kts
  2. 1
      src/video_core/CMakeLists.txt
  3. 91
      src/video_core/buffer_cache/buffer_cache.h
  4. 31
      src/video_core/buffer_cache/buffer_cache_base.h
  5. 51
      src/video_core/deferred_destruction_queue.h
  6. 20
      src/video_core/renderer_opengl/gl_buffer_cache.h
  7. 22
      src/video_core/renderer_opengl/gl_texture_cache.h
  8. 14
      src/video_core/renderer_vulkan/vk_buffer_cache.cpp
  9. 6
      src/video_core/renderer_vulkan/vk_buffer_cache.h
  10. 3
      src/video_core/renderer_vulkan/vk_present_manager.cpp
  11. 8
      src/video_core/renderer_vulkan/vk_rasterizer.cpp
  12. 19
      src/video_core/renderer_vulkan/vk_texture_cache.cpp
  13. 9
      src/video_core/renderer_vulkan/vk_texture_cache.h
  14. 210
      src/video_core/texture_cache/texture_cache.h
  15. 57
      src/video_core/texture_cache/texture_cache_base.h
  16. 29
      src/video_core/vulkan_common/vulkan_device.cpp
  17. 12
      src/video_core/vulkan_common/vulkan_device.h
  18. 65
      src/video_core/vulkan_common/vulkan_memory_allocator.cpp
  19. 15
      src/video_core/vulkan_common/vulkan_memory_allocator.h

2
src/android/app/build.gradle.kts

@ -65,7 +65,7 @@ android {
defaultConfig {
applicationId = "dev.eden.eden_emulator"
minSdk = 24
minSdk = 33
targetSdk = 36
versionName = getGitVersion()
versionCode = autoVersion

1
src/video_core/CMakeLists.txt

@ -33,6 +33,7 @@ add_library(video_core STATIC
control/channel_state_cache.h
control/scheduler.cpp
control/scheduler.h
deferred_destruction_queue.h
delayed_destruction_ring.h
dirty_flags.cpp
dirty_flags.h

91
src/video_core/buffer_cache/buffer_cache.h

@ -31,44 +31,74 @@ BufferCache<P>::BufferCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, R
immediately_free = (Settings::values.vram_usage_mode.GetValue() == Settings::VramUsageMode::Aggressive);
#endif
if (!runtime.CanReportMemoryUsage()) {
minimum_memory = DEFAULT_EXPECTED_MEMORY;
critical_memory = DEFAULT_CRITICAL_MEMORY;
memory_budget = FALLBACK_MEMORY_BUDGET;
return;
}
const s64 device_local_memory = static_cast<s64>(runtime.GetDeviceLocalMemory());
const s64 min_spacing_expected = device_local_memory - 1_GiB;
const s64 min_spacing_critical = device_local_memory - 512_MiB;
const s64 mem_threshold = (std::min)(device_local_memory, TARGET_THRESHOLD);
const s64 min_vacancy_expected = (6 * mem_threshold) / 10;
const s64 min_vacancy_critical = (2 * mem_threshold) / 10;
minimum_memory = static_cast<u64>(
(std::max)((std::min)(device_local_memory - min_vacancy_expected, min_spacing_expected),
DEFAULT_EXPECTED_MEMORY));
critical_memory = static_cast<u64>(
(std::max)((std::min)(device_local_memory - min_vacancy_critical, min_spacing_critical),
DEFAULT_CRITICAL_MEMORY));
memory_budget = runtime.GetDeviceLocalMemory();
}
template <class P>
BufferCache<P>::~BufferCache() = default;
template <class P>
void BufferCache<P>::RunGarbageCollector() {
const bool aggressive_gc = total_used_memory >= critical_memory;
const u64 ticks_to_destroy = aggressive_gc ? 60 : 120;
int num_iterations = aggressive_gc ? 64 : 32;
const auto clean_up = [this, &num_iterations](BufferId buffer_id) {
if (num_iterations == 0) {
u64 BufferCache<P>::DeviceUsage(bool force_refresh) {
if (!runtime.CanReportMemoryUsage()) {
return total_used_memory;
}
if (force_refresh || usage_refresh_countdown == 0) {
cached_device_usage = runtime.GetDeviceAllocationUsage();
usage_refresh_countdown = USAGE_REFRESH_INTERVAL;
} else {
--usage_refresh_countdown;
}
return cached_device_usage;
}
template <class P>
u64 BufferCache<P>::ReclaimMemory(u64 target_bytes, bool allow_download) {
if (target_bytes == 0 || in_reclaim) {
return 0;
}
in_reclaim = true;
u64 freed = 0;
const auto clean_up = [&](BufferId buffer_id) {
if (freed >= target_bytes) {
return true;
}
--num_iterations;
auto& buffer = slot_buffers[buffer_id];
if (!allow_download && IsRegionGpuModified(buffer.CpuAddr(), buffer.SizeBytes())) {
return false;
}
const u64 buffer_bytes = Common::AlignUp(buffer.SizeBytes(), 1024);
DownloadBufferMemory(buffer);
DeleteBuffer(buffer_id);
freed += buffer_bytes;
return false;
};
lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, clean_up);
const u64 cold_tick =
frame_tick > RECLAIM_GUARD_FRAMES ? frame_tick - RECLAIM_GUARD_FRAMES : 0;
lru_cache.ForEachItemBelow(cold_tick, clean_up);
if (freed < target_bytes) {
lru_cache.ForEachItemBelow(frame_tick > 0 ? frame_tick - 1 : 0, clean_up);
}
in_reclaim = false;
usage_refresh_countdown = 0;
reclaim_stalled = freed == 0;
return freed;
}
template <class P>
void BufferCache<P>::EnsureHeadroom(bool allow_download) {
if (reclaim_stalled) {
return;
}
const u64 limit = memory_budget > RECLAIM_HEADROOM ? memory_budget - RECLAIM_HEADROOM : 0;
const u64 usage = DeviceUsage(false);
if (usage <= limit) {
return;
}
ReclaimMemory(usage - limit, allow_download);
}
template <class P>
@ -96,15 +126,11 @@ void BufferCache<P>::TickFrame() {
const bool skip_preferred = hits * 256 < shots * 251;
channel_state->uniform_buffer_skip_cache_size = skip_preferred ? DEFAULT_SKIP_CACHE_SIZE : 0;
// If we can obtain the memory info, use it instead of the estimate.
if (runtime.CanReportMemoryUsage()) {
total_used_memory = runtime.GetDeviceMemoryUsage();
}
if (total_used_memory >= minimum_memory) {
RunGarbageCollector();
}
usage_refresh_countdown = 0;
reclaim_stalled = false;
EnsureHeadroom(true);
++frame_tick;
delayed_destruction_ring.Tick();
sentenced_buffers.Reclaim(runtime.CompletedSyncPoint());
for (auto& buffer : async_buffers_death_ring) {
runtime.FreeDeferredStagingBuffer(buffer);
@ -1576,6 +1602,7 @@ void BufferCache<P>::JoinOverlap(BufferId new_buffer_id, BufferId overlap_id,
template <class P>
BufferId BufferCache<P>::CreateBuffer(DAddr device_addr, u32 wanted_size) {
EnsureHeadroom(false);
DAddr device_addr_end = Common::AlignUp(device_addr + wanted_size, CACHING_PAGESIZE);
device_addr = Common::AlignDown(device_addr, CACHING_PAGESIZE);
wanted_size = static_cast<u32>(device_addr_end - device_addr);
@ -1613,7 +1640,7 @@ void BufferCache<P>::ChangeRegister(BufferId buffer_id) {
total_used_memory += Common::AlignUp(size, 1024);
buffer.setLRUID(lru_cache.Insert(buffer_id, frame_tick));
} else {
total_used_memory -= Common::AlignUp(size, 1024);
total_used_memory -= (std::min)(total_used_memory, Common::AlignUp(size, 1024));
lru_cache.Free(buffer.getLRUID());
}
const DAddr device_addr_begin = buffer.CpuAddr();
@ -1872,7 +1899,7 @@ void BufferCache<P>::DeleteBuffer(BufferId buffer_id, bool do_not_mark) {
#ifdef YUZU_LEGACY
if (!do_not_mark || !immediately_free)
#endif
delayed_destruction_ring.Push(std::move(slot_buffers[buffer_id]));
sentenced_buffers.Push(std::move(slot_buffers[buffer_id]), runtime.CurrentSyncPoint());
slot_buffers.erase(buffer_id);

31
src/video_core/buffer_cache/buffer_cache_base.h

@ -30,7 +30,7 @@
#include "common/slot_vector.h"
#include "video_core/buffer_cache/buffer_base.h"
#include "video_core/control/channel_state_cache.h"
#include "video_core/delayed_destruction_ring.h"
#include "video_core/deferred_destruction_queue.h"
#include "video_core/dirty_flags.h"
#include "video_core/engines/maxwell_3d.h"
#include "video_core/engines/kepler_compute.h"
@ -182,13 +182,14 @@ class BufferCache : public VideoCommon::ChannelSetupCaches<BufferCacheChannelInf
static constexpr bool USE_MEMORY_MAPS_FOR_UPLOADS = P::USE_MEMORY_MAPS_FOR_UPLOADS;
#ifdef YUZU_LEGACY
static constexpr s64 TARGET_THRESHOLD = 3_GiB;
static constexpr u64 RECLAIM_HEADROOM = 384_MiB;
#else
static constexpr s64 TARGET_THRESHOLD = 4_GiB;
static constexpr u64 RECLAIM_HEADROOM = 512_MiB;
#endif
static constexpr s64 DEFAULT_EXPECTED_MEMORY = 512_MiB;
static constexpr s64 DEFAULT_CRITICAL_MEMORY = 1_GiB;
static constexpr u64 FALLBACK_MEMORY_BUDGET = 2_GiB;
static constexpr u32 USAGE_REFRESH_INTERVAL = 16;
static constexpr u64 RECLAIM_GUARD_FRAMES = 8;
// Debug Flags.
@ -215,6 +216,8 @@ public:
void TickFrame();
u64 ReclaimMemory(u64 target_bytes, bool allow_download);
void WriteMemory(DAddr device_addr, u64 size);
void CachedWriteMemory(DAddr device_addr, u64 size);
@ -358,7 +361,9 @@ private:
((device_addr + size) & ~Core::DEVICE_PAGEMASK);
}
void RunGarbageCollector();
u64 DeviceUsage(bool force_refresh);
void EnsureHeadroom(bool allow_download);
void BindHostIndexBuffer();
@ -475,12 +480,7 @@ private:
Tegra::MaxwellDeviceMemoryManager& device_memory;
Common::SlotVector<Buffer> slot_buffers;
#ifdef YUZU_LEGACY
static constexpr size_t TICKS_TO_DESTROY = 6;
#else
static constexpr size_t TICKS_TO_DESTROY = 8;
#endif
DelayedDestructionRing<Buffer, TICKS_TO_DESTROY> delayed_destruction_ring;
DeferredDestructionQueue<Buffer> sentenced_buffers;
const Tegra::Engines::Maxwell3D::DrawManager::IndirectParams* current_draw_indirect{};
@ -515,8 +515,11 @@ private:
Common::LeastRecentlyUsedCache<LRUItemParams> lru_cache;
u64 frame_tick = 0;
u64 total_used_memory = 0;
u64 minimum_memory = 0;
u64 critical_memory = 0;
u64 memory_budget = 0;
u64 cached_device_usage = 0;
u32 usage_refresh_countdown = 0;
bool in_reclaim = false;
bool reclaim_stalled = false;
BufferId inline_buffer_id;
#ifdef YUZU_LEGACY
bool immediately_free = false;

51
src/video_core/deferred_destruction_queue.h

@ -0,0 +1,51 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
#pragma once
#include <cstddef>
#include <deque>
#include <utility>
#include "common/common_types.h"
namespace VideoCommon {
template <typename T>
class DeferredDestructionQueue {
public:
void Push(T&& object, u64 sync_point) {
entries.emplace_back(std::move(object), sync_point);
}
void Reclaim(u64 completed_sync_point) {
while (!entries.empty() && entries.front().sync_point <= completed_sync_point) {
entries.pop_front();
}
}
void Clear() {
entries.clear();
}
[[nodiscard]] size_t Size() const noexcept {
return entries.size();
}
[[nodiscard]] bool Empty() const noexcept {
return entries.empty();
}
private:
struct Entry {
Entry(T&& object_, u64 sync_point_) noexcept
: object{std::move(object_)}, sync_point{sync_point_} {}
T object;
u64 sync_point;
};
std::deque<Entry> entries;
};
} // namespace VideoCommon

20
src/video_core/renderer_opengl/gl_buffer_cache.h

@ -93,7 +93,17 @@ public:
void PostCopyBarrier();
void Finish();
void TickFrame(Common::SlotVector<Buffer>&) noexcept {}
void TickFrame(Common::SlotVector<Buffer>&) noexcept {
++sync_point;
}
u64 CurrentSyncPoint() const noexcept {
return sync_point;
}
u64 CompletedSyncPoint() const noexcept {
return sync_point > SYNC_POINT_DELAY ? sync_point - SYNC_POINT_DELAY : 0;
}
void ClearBuffer(Buffer& dest_buffer, u32 offset, size_t size, u32 value);
@ -128,6 +138,10 @@ public:
u64 GetDeviceMemoryUsage() const;
u64 GetDeviceAllocationUsage() const {
return GetDeviceMemoryUsage();
}
void BindFastUniformBuffer(size_t stage, u32 binding_index, u32 size) {
const GLuint handle = fast_uniforms[stage][binding_index].handle;
const GLsizeiptr gl_size = static_cast<GLsizeiptr>(size);
@ -213,9 +227,13 @@ private:
GL_FRAGMENT_PROGRAM_PARAMETER_BUFFER_NV,
};
static constexpr u64 SYNC_POINT_DELAY = 8;
const Device& device;
StagingBufferPool& staging_buffer_pool;
u64 sync_point = 1;
bool has_fast_buffer_sub_data = false;
bool use_assembly_shaders = false;
bool has_unified_vertex_buffers = false;

22
src/video_core/renderer_opengl/gl_texture_cache.h

@ -87,6 +87,10 @@ public:
u64 GetDeviceMemoryUsage() const;
u64 GetDeviceAllocationUsage() const {
return GetDeviceMemoryUsage();
}
bool CanReportMemoryUsage() const {
return device.CanReportMemoryUsage();
}
@ -139,7 +143,19 @@ public:
bool HasNativeASTC() const noexcept;
void TickFrame() {}
void TickFrame() {
++sync_point;
}
u64 CurrentSyncPoint() const noexcept {
return sync_point;
}
u64 CompletedSyncPoint() const noexcept {
return sync_point > SYNC_POINT_DELAY ? sync_point - SYNC_POINT_DELAY : 0;
}
void WaitSyncPoint(u64) {}
StateTracker& GetStateTracker() {
return state_tracker;
@ -174,6 +190,9 @@ private:
std::array<OGLFramebuffer, 4> rescale_read_fbos;
const Settings::ResolutionScalingInfo& resolution;
u64 device_access_memory;
static constexpr u64 SYNC_POINT_DELAY = 8;
u64 sync_point = 1;
};
class Image : public VideoCommon::ImageBase {
@ -370,6 +389,7 @@ struct TextureCacheParams {
static constexpr bool HAS_EMULATED_COPIES = true;
static constexpr bool HAS_DEVICE_MEMORY_INFO = true;
static constexpr bool IMPLEMENTS_ASYNC_DOWNLOADS = true;
static constexpr bool HAS_TIMELINE_SYNC_POINTS = false;
using Runtime = OpenGL::TextureCacheRuntime;
using Image = OpenGL::Image;

14
src/video_core/renderer_vulkan/vk_buffer_cache.cpp

@ -376,6 +376,10 @@ u64 BufferCacheRuntime::GetDeviceMemoryUsage() const {
return device.GetDeviceMemoryUsage();
}
u64 BufferCacheRuntime::GetDeviceAllocationUsage() const {
return device.GetMemoryBudgetInfo().allocation_bytes;
}
bool BufferCacheRuntime::CanReportMemoryUsage() const {
return device.CanReportMemoryUsage();
}
@ -404,6 +408,16 @@ u64 BufferCacheRuntime::KnownGpuTick() {
return scheduler.GetMasterSemaphore().KnownGpuTick();
}
u64 BufferCacheRuntime::CurrentSyncPoint() const noexcept {
return scheduler.GetMasterSemaphore().CurrentTick();
}
u64 BufferCacheRuntime::CompletedSyncPoint() const {
auto& master_semaphore = scheduler.GetMasterSemaphore();
master_semaphore.Refresh();
return master_semaphore.KnownGpuTick();
}
void BufferCacheRuntime::Wait(u64 buffer_tick) {
scheduler.Wait(buffer_tick);
}

6
src/video_core/renderer_vulkan/vk_buffer_cache.h

@ -100,10 +100,16 @@ public:
void Finish();
u64 CurrentSyncPoint() const noexcept;
u64 CompletedSyncPoint() const;
u64 GetDeviceLocalMemory() const;
u64 GetDeviceMemoryUsage() const;
u64 GetDeviceAllocationUsage() const;
bool CanReportMemoryUsage() const;
u32 GetUniformBufferAlignment() const;

3
src/video_core/renderer_vulkan/vk_present_manager.cpp

@ -296,9 +296,6 @@ void PresentManager::RecreateSwapchain(Frame* frame) {
}
void PresentManager::SetImageCount() {
// We cannot have more than 7 images in flight at any given time.
// FRAMES_IN_FLIGHT is 8, and the cache TICKS_TO_DESTROY is 8.
// Mali drivers will give us 6.
image_count = std::min<size_t>(swapchain.GetImageCount(), 7);
}

8
src/video_core/renderer_vulkan/vk_rasterizer.cpp

@ -221,9 +221,17 @@ RasterizerVulkan::RasterizerVulkan(Core::Frontend::EmuWindow& emu_window_, Tegra
fence_manager(*this, gpu, texture_cache, buffer_cache, query_cache, device, scheduler),
wfi_event(device.GetLogical().CreateEvent()) {
scheduler.SetQueryCache(query_cache);
memory_allocator.SetReclaimCallback([this](u64 bytes) -> u64 {
u64 freed = texture_cache.ReclaimMemory(bytes, false);
if (freed < bytes) {
freed += buffer_cache.ReclaimMemory(bytes - freed, false);
}
return freed;
});
}
RasterizerVulkan::~RasterizerVulkan() {
memory_allocator.SetReclaimCallback(nullptr);
scheduler.WaitWorker();
scheduler.Finish();
}

19
src/video_core/renderer_vulkan/vk_texture_cache.cpp

@ -1753,6 +1753,20 @@ void TextureCacheRuntime::CopyImageMSAA(Image& dst, Image& src,
src.info.format, num_samples, copies, msaa_to_non_msaa);
}
u64 TextureCacheRuntime::CurrentSyncPoint() const noexcept {
return scheduler.CurrentTick();
}
u64 TextureCacheRuntime::CompletedSyncPoint() const {
auto& master_semaphore = scheduler.GetMasterSemaphore();
master_semaphore.Refresh();
return master_semaphore.KnownGpuTick();
}
void TextureCacheRuntime::WaitSyncPoint(u64 sync_point) {
scheduler.Wait(sync_point);
}
u64 TextureCacheRuntime::GetDeviceLocalMemory() const {
return device.GetDeviceLocalMemory();
}
@ -1761,6 +1775,10 @@ u64 TextureCacheRuntime::GetDeviceMemoryUsage() const {
return device.GetDeviceMemoryUsage();
}
u64 TextureCacheRuntime::GetDeviceAllocationUsage() const {
return device.GetMemoryBudgetInfo().allocation_bytes;
}
bool TextureCacheRuntime::CanReportMemoryUsage() const {
return device.CanReportMemoryUsage();
}
@ -1770,6 +1788,7 @@ std::optional<size_t> TextureCacheRuntime::GetSamplerHeapBudget() const {
}
void TextureCacheRuntime::TickFrame() {
device.TickAllocatorFrame();
std::erase_if(pending_msaa_images, [this](const auto& pending) {
return scheduler.IsFree(pending.first);
});

9
src/video_core/renderer_vulkan/vk_texture_cache.h

@ -60,10 +60,18 @@ public:
void TickFrame();
u64 CurrentSyncPoint() const noexcept;
u64 CompletedSyncPoint() const;
void WaitSyncPoint(u64 sync_point);
u64 GetDeviceLocalMemory() const;
u64 GetDeviceMemoryUsage() const;
u64 GetDeviceAllocationUsage() const;
bool CanReportMemoryUsage() const;
std::optional<size_t> GetSamplerHeapBudget() const;
@ -487,6 +495,7 @@ struct TextureCacheParams {
static constexpr bool HAS_EMULATED_COPIES = false;
static constexpr bool HAS_DEVICE_MEMORY_INFO = true;
static constexpr bool IMPLEMENTS_ASYNC_DOWNLOADS = true;
static constexpr bool HAS_TIMELINE_SYNC_POINTS = true;
using Runtime = Vulkan::TextureCacheRuntime;
using Image = Vulkan::Image;

210
src/video_core/texture_cache/texture_cache.h

@ -58,23 +58,9 @@ TextureCache<P>::TextureCache(Runtime& runtime_, Tegra::MaxwellDeviceMemoryManag
void(slot_samplers.insert(runtime, sampler_descriptor));
if constexpr (HAS_DEVICE_MEMORY_INFO) {
const s64 device_local_memory = static_cast<s64>(runtime.GetDeviceLocalMemory());
const s64 min_spacing_expected = device_local_memory - 1_GiB;
const s64 min_spacing_critical = device_local_memory - 512_MiB;
const s64 mem_threshold = (std::min)(device_local_memory, TARGET_THRESHOLD);
const s64 min_vacancy_expected = (6 * mem_threshold) / 10;
const s64 min_vacancy_critical = (2 * mem_threshold) / 10;
expected_memory = static_cast<u64>(
(std::max)((std::min)(device_local_memory - min_vacancy_expected, min_spacing_expected),
DEFAULT_EXPECTED_MEMORY));
critical_memory = static_cast<u64>(
(std::max)((std::min)(device_local_memory - min_vacancy_critical, min_spacing_critical),
DEFAULT_CRITICAL_MEMORY));
minimum_memory = static_cast<u64>((device_local_memory - mem_threshold) / 2);
memory_budget = runtime.GetDeviceLocalMemory();
} else {
expected_memory = DEFAULT_EXPECTED_MEMORY + 512_MiB;
critical_memory = DEFAULT_CRITICAL_MEMORY + 1_GiB;
minimum_memory = 0;
memory_budget = FALLBACK_MEMORY_BUDGET;
}
const bool gpu_unswizzle_enabled = Settings::values.gpu_unswizzle_enabled.GetValue();
@ -114,71 +100,140 @@ TextureCache<P>::TextureCache(Runtime& runtime_, Tegra::MaxwellDeviceMemoryManag
}
template <class P>
void TextureCache<P>::RunGarbageCollector() {
bool high_priority_mode = false;
bool aggressive_mode = false;
u64 ticks_to_destroy = 0;
size_t num_iterations = 0;
const auto Configure = [&](bool allow_aggressive) {
high_priority_mode = total_used_memory >= expected_memory;
aggressive_mode = allow_aggressive && total_used_memory >= critical_memory;
ticks_to_destroy = aggressive_mode ? 10ULL : high_priority_mode ? 25ULL : 50ULL;
num_iterations = aggressive_mode ? 40 : (high_priority_mode ? 20 : 10);
};
const auto Cleanup = [this, &num_iterations, &high_priority_mode, &aggressive_mode](ImageId image_id) {
if (num_iterations == 0) {
void TextureCache<P>::QueueEvictionDownload(Image& image) {
auto copies = FullDownloadCopies(image.info);
auto staging = runtime.DownloadStagingBuffer(image.unswizzled_size_bytes, true);
image.DownloadMemory(staging, FixSmallVectorADL(copies));
pending_eviction_downloads.push_back(PendingEvictionDownload{
.staging = staging,
.gpu_memory = gpu_memory,
.gpu_addr = image.gpu_addr,
.info = image.info,
.copies = std::move(copies),
.sync_point = runtime.CurrentSyncPoint(),
});
}
template <class P>
void TextureCache<P>::TickEvictionDownloads(u64 completed_sync_point) {
while (!pending_eviction_downloads.empty() &&
pending_eviction_downloads.front().sync_point <= completed_sync_point) {
auto& entry = pending_eviction_downloads.front();
SwizzleImage(*entry.gpu_memory, entry.gpu_addr, entry.info, FixSmallVectorADL(entry.copies),
entry.staging.mapped_span.subspan(entry.staging.offset), swizzle_data_buffer);
runtime.FreeDeferredStagingBuffer(entry.staging);
pending_eviction_downloads.pop_front();
}
}
template <class P>
void TextureCache<P>::FlushEvictionDownloads() {
if (pending_eviction_downloads.empty()) {
return;
}
const u64 last_sync_point = pending_eviction_downloads.back().sync_point;
runtime.WaitSyncPoint(last_sync_point);
TickEvictionDownloads(last_sync_point);
}
template <class P>
u64 TextureCache<P>::ImageSizeBytes(const ImageBase& image) {
u64 tentative_size = (std::max)(image.guest_size_bytes, image.unswizzled_size_bytes);
if ((IsPixelFormatASTC(image.info.format) &&
True(image.flags & ImageFlagBits::AcceleratedUpload)) ||
True(image.flags & ImageFlagBits::Converted)) {
tentative_size = TranscodedAstcSize(tentative_size, image.info.format);
}
u64 size = Common::AlignUp(tentative_size, 1024);
if (image.HasScaled()) {
size += GetScaledImageSizeBytes(image);
}
return size;
}
template <class P>
u64 TextureCache<P>::DeviceUsage(bool force_refresh) {
if (!runtime.CanReportMemoryUsage()) {
return total_used_memory;
}
if (force_refresh || usage_refresh_countdown == 0) {
cached_device_usage = runtime.GetDeviceAllocationUsage();
usage_refresh_countdown = USAGE_REFRESH_INTERVAL;
} else {
--usage_refresh_countdown;
}
return cached_device_usage;
}
template <class P>
u64 TextureCache<P>::ReclaimMemory(u64 target_bytes, bool allow_download) {
if (target_bytes == 0 || in_reclaim) {
return 0;
}
in_reclaim = true;
u64 freed = 0;
const auto evict = [&](ImageId image_id) {
if (freed >= target_bytes) {
return true;
}
--num_iterations;
auto& image = slot_images[image_id];
if (True(image.flags & ImageFlagBits::IsDecoding)) {
return false;
}
const bool must_download = image.IsSafeDownload() && False(image.flags & ImageFlagBits::BadOverlap);
if ((!aggressive_mode && True(image.flags & ImageFlagBits::CostlyLoad)) || (!high_priority_mode && must_download)) {
return false;
}
const bool must_download =
image.IsSafeDownload() && False(image.flags & ImageFlagBits::BadOverlap);
bool queued_download = false;
if (must_download) {
auto map = runtime.DownloadStagingBuffer(image.unswizzled_size_bytes);
const auto copies = FixSmallVectorADL(FullDownloadCopies(image.info));
image.DownloadMemory(map, copies);
runtime.Finish();
SwizzleImage(*gpu_memory, image.gpu_addr, image.info, copies, map.mapped_span, swizzle_data_buffer);
if (!allow_download || !HAS_TIMELINE_SYNC_POINTS) {
return false;
}
QueueEvictionDownload(image);
queued_download = true;
}
const u64 image_bytes = ImageSizeBytes(image);
if (True(image.flags & ImageFlagBits::Tracked)) {
UntrackImage(image, image_id);
}
UnregisterImage(image_id);
DeleteImage(image_id, image.scale_tick > frame_tick + 5);
if (aggressive_mode && total_used_memory < critical_memory) {
num_iterations >>= 2;
aggressive_mode = false;
} else if (high_priority_mode && total_used_memory < expected_memory) {
num_iterations >>= 1;
high_priority_mode = false;
}
DeleteImage(image_id, !queued_download && image.scale_tick > frame_tick + 5);
freed += image_bytes;
return false;
};
Configure(false);
lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup);
if (total_used_memory >= critical_memory) {
Configure(true);
lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup);
const u64 cold_tick =
frame_tick > RECLAIM_GUARD_FRAMES ? frame_tick - RECLAIM_GUARD_FRAMES : 0;
lru_cache.ForEachItemBelow(cold_tick, evict);
if (freed < target_bytes) {
lru_cache.ForEachItemBelow(frame_tick > 0 ? frame_tick - 1 : 0, evict);
}
in_reclaim = false;
usage_refresh_countdown = 0;
reclaim_stalled = freed == 0;
return freed;
}
template <class P>
void TextureCache<P>::TickFrame() {
// If we can obtain the memory info, use it instead of the estimate.
if (runtime.CanReportMemoryUsage()) {
total_used_memory = runtime.GetDeviceMemoryUsage();
void TextureCache<P>::EnsureHeadroom(bool allow_download) {
if (reclaim_stalled) {
return;
}
if (total_used_memory > minimum_memory) {
RunGarbageCollector();
const u64 limit = memory_budget > RECLAIM_HEADROOM ? memory_budget - RECLAIM_HEADROOM : 0;
const u64 usage = DeviceUsage(false);
if (usage <= limit) {
return;
}
sentenced_images.Tick();
sentenced_framebuffers.Tick();
sentenced_image_view.Tick();
ReclaimMemory(usage - limit, allow_download);
}
template <class P>
void TextureCache<P>::TickFrame() {
usage_refresh_countdown = 0;
reclaim_stalled = false;
EnsureHeadroom(true);
const u64 completed_sync_point = runtime.CompletedSyncPoint();
TickEvictionDownloads(completed_sync_point);
sentenced_images.Reclaim(completed_sync_point);
sentenced_framebuffers.Reclaim(completed_sync_point);
sentenced_image_view.Reclaim(completed_sync_point);
TickAsyncDecode();
TickAsyncUnswizzle();
@ -596,6 +651,7 @@ void TextureCache<P>::WriteMemory(DAddr cpu_addr, size_t size) {
template <class P>
void TextureCache<P>::DownloadMemory(DAddr cpu_addr, size_t size) {
FlushEvictionDownloads();
boost::container::small_vector<ImageId, 16> images;
ForEachImageInRegion(cpu_addr, size, [&images](ImageId image_id, ImageBase& image) {
if (!image.IsSafeDownload()) {
@ -894,6 +950,7 @@ void TextureCache<P>::CommitAsyncFlushes() {
template <class P>
void TextureCache<P>::PopAsyncFlushes() {
FlushEvictionDownloads();
if (committed_downloads.empty()) {
return;
}
@ -1294,8 +1351,9 @@ void TextureCache<P>::InvalidateScale(Image& image) {
}
RemoveImageViewReferences(image_view_ids);
RemoveFramebuffers(image_view_ids);
const u64 sync_point = runtime.CurrentSyncPoint();
for (const ImageViewId image_view_id : image_view_ids) {
sentenced_image_view.Push(std::move(slot_image_views[image_view_id]));
sentenced_image_view.Push(std::move(slot_image_views[image_view_id]), sync_point);
slot_image_views.erase(image_view_id);
}
image.image_view_ids.clear();
@ -1523,6 +1581,7 @@ ImageId TextureCache<P>::InsertImage(const ImageInfo& info, GPUVAddr gpu_addr,
template <class P>
ImageId TextureCache<P>::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, DAddr cpu_addr) {
EnsureHeadroom(false);
ImageInfo new_info = info;
const size_t size_bytes = CalculateGuestSizeInBytes(new_info);
const bool broken_views = runtime.HasBrokenTextureViewFormats();
@ -2185,13 +2244,7 @@ void TextureCache<P>::RegisterImage(ImageId image_id) {
ASSERT_MSG(False(image.flags & ImageFlagBits::Registered),
"Trying to register an already registered image");
image.flags |= ImageFlagBits::Registered;
u64 tentative_size = (std::max)(image.guest_size_bytes, image.unswizzled_size_bytes);
if ((IsPixelFormatASTC(image.info.format) &&
True(image.flags & ImageFlagBits::AcceleratedUpload)) ||
True(image.flags & ImageFlagBits::Converted)) {
tentative_size = TranscodedAstcSize(tentative_size, image.info.format);
}
total_used_memory += Common::AlignUp(tentative_size, 1024);
total_used_memory += ImageSizeBytes(image);
image.lru_index = lru_cache.Insert(image_id, frame_tick);
ForEachGPUPage(image.gpu_addr, image.guest_size_bytes, [this, image_id](u64 page) {
@ -2354,16 +2407,7 @@ void TextureCache<P>::UntrackImage(ImageBase& image, ImageId image_id) {
template <class P>
void TextureCache<P>::DeleteImage(ImageId image_id, bool immediate_delete) {
ImageBase& image = slot_images[image_id];
if (image.HasScaled()) {
total_used_memory -= GetScaledImageSizeBytes(image);
}
u64 tentative_size = (std::max)(image.guest_size_bytes, image.unswizzled_size_bytes);
if ((IsPixelFormatASTC(image.info.format) &&
True(image.flags & ImageFlagBits::AcceleratedUpload)) ||
True(image.flags & ImageFlagBits::Converted)) {
tentative_size = TranscodedAstcSize(tentative_size, image.info.format);
}
total_used_memory -= Common::AlignUp(tentative_size, 1024);
total_used_memory -= (std::min)(total_used_memory, ImageSizeBytes(image));
const GPUVAddr gpu_addr = image.gpu_addr;
const auto alloc_it = image_allocs_table.find(gpu_addr);
if (alloc_it == image_allocs_table.end()) {
@ -2417,14 +2461,15 @@ void TextureCache<P>::DeleteImage(ImageId image_id, bool immediate_delete) {
ASSERT_MSG(num_removed_overlaps == 1, "Invalid number of removed overlapps: {}",
num_removed_overlaps);
}
const u64 sync_point = runtime.CurrentSyncPoint();
for (const ImageViewId image_view_id : image_view_ids) {
if (!immediate_delete) {
sentenced_image_view.Push(std::move(slot_image_views[image_view_id]));
sentenced_image_view.Push(std::move(slot_image_views[image_view_id]), sync_point);
}
slot_image_views.erase(image_view_id);
}
if (!immediate_delete) {
sentenced_images.Push(std::move(slot_images[image_id]));
sentenced_images.Push(std::move(slot_images[image_id]), sync_point);
}
slot_images.erase(image_id);
@ -2470,7 +2515,8 @@ void TextureCache<P>::RemoveFramebuffers(std::span<const ImageViewId> removed_vi
last_framebuffer_id = {};
last_framebuffer_serial = 0;
}
sentenced_framebuffers.Push(std::move(slot_framebuffers[framebuffer_id]));
sentenced_framebuffers.Push(std::move(slot_framebuffers[framebuffer_id]),
runtime.CurrentSyncPoint());
it = framebuffers.erase(it);
} else {
++it;

57
src/video_core/texture_cache/texture_cache_base.h

@ -30,7 +30,7 @@
#include "common/thread_worker.h"
#include "video_core/compatible_formats.h"
#include "video_core/control/channel_state_cache.h"
#include "video_core/delayed_destruction_ring.h"
#include "video_core/deferred_destruction_queue.h"
#include "video_core/engines/fermi_2d.h"
#include "video_core/surface.h"
#include "video_core/texture_cache/descriptor_table.h"
@ -108,18 +108,19 @@ class TextureCache : public VideoCommon::ChannelSetupCaches<TextureCacheChannelI
static constexpr bool HAS_DEVICE_MEMORY_INFO = P::HAS_DEVICE_MEMORY_INFO;
/// True when the API can do asynchronous texture downloads.
static constexpr bool IMPLEMENTS_ASYNC_DOWNLOADS = P::IMPLEMENTS_ASYNC_DOWNLOADS;
static constexpr bool HAS_TIMELINE_SYNC_POINTS = P::HAS_TIMELINE_SYNC_POINTS;
static constexpr size_t UNSET_CHANNEL{(std::numeric_limits<size_t>::max)()};
#ifdef YUZU_LEGACY
static constexpr s64 TARGET_THRESHOLD = 3_GiB;
static constexpr u64 RECLAIM_HEADROOM = 384_MiB;
#else
static constexpr s64 TARGET_THRESHOLD = 4_GiB;
static constexpr u64 RECLAIM_HEADROOM = 512_MiB;
#endif
static constexpr s64 DEFAULT_EXPECTED_MEMORY = 1_GiB + 125_MiB;
static constexpr s64 DEFAULT_CRITICAL_MEMORY = 1_GiB + 625_MiB;
static constexpr size_t GC_EMERGENCY_COUNTS = 2;
static constexpr u64 FALLBACK_MEMORY_BUDGET = 2_GiB;
static constexpr u32 USAGE_REFRESH_INTERVAL = 16;
static constexpr u64 RECLAIM_GUARD_FRAMES = 8;
using Runtime = typename P::Runtime;
using Image = typename P::Image;
@ -154,6 +155,8 @@ public:
/// Notify the cache that a new frame has been queued
void TickFrame();
u64 ReclaimMemory(u64 target_bytes, bool allow_download);
/// Return a constant reference to the given image view id
[[nodiscard]] const ImageView& GetImageView(ImageViewId id) const noexcept;
@ -293,8 +296,17 @@ private:
void OnGPUASRegister(size_t map_id) final override;
/// Runs the Garbage Collector.
void RunGarbageCollector();
u64 ImageSizeBytes(const ImageBase& image);
u64 DeviceUsage(bool force_refresh);
void EnsureHeadroom(bool allow_download);
void QueueEvictionDownload(Image& image);
void TickEvictionDownloads(u64 completed_sync_point);
void FlushEvictionDownloads();
/// Find or create an image view in the guest descriptor table
ImageViewId VisitImageView(u32 index, bool compute);
@ -451,9 +463,11 @@ private:
bool has_deleted_images = false;
bool is_rescaling = false;
u64 total_used_memory = 0;
u64 minimum_memory;
u64 expected_memory;
u64 critical_memory;
u64 memory_budget = 0;
u64 cached_device_usage = 0;
u32 usage_refresh_countdown = 0;
bool in_reclaim = false;
bool reclaim_stalled = false;
size_t gpu_unswizzle_maxsize = 0;
size_t swizzle_chunk_size = 0;
u32 swizzle_slices_per_batch = 0;
@ -491,14 +505,19 @@ private:
};
Common::LeastRecentlyUsedCache<LRUItemParams> lru_cache;
#ifdef YUZU_LEGACY
static constexpr size_t TICKS_TO_DESTROY = 6;
#else
static constexpr size_t TICKS_TO_DESTROY = 8;
#endif
DelayedDestructionRing<Image, TICKS_TO_DESTROY> sentenced_images;
DelayedDestructionRing<ImageView, TICKS_TO_DESTROY> sentenced_image_view;
DelayedDestructionRing<Framebuffer, TICKS_TO_DESTROY> sentenced_framebuffers;
DeferredDestructionQueue<Image> sentenced_images;
DeferredDestructionQueue<ImageView> sentenced_image_view;
DeferredDestructionQueue<Framebuffer> sentenced_framebuffers;
struct PendingEvictionDownload {
AsyncBuffer staging;
Tegra::MemoryManager* gpu_memory;
GPUVAddr gpu_addr;
VideoCommon::ImageInfo info;
boost::container::small_vector<VideoCommon::BufferImageCopy, 16> copies;
u64 sync_point;
};
std::deque<PendingEvictionDownload> pending_eviction_downloads;
ankerl::unordered_dense::map<GPUVAddr, ImageAllocId> image_allocs_table;

29
src/video_core/vulkan_common/vulkan_device.cpp

@ -5,6 +5,7 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include <algorithm>
#include <array>
#include <bitset>
#include <chrono>
#include <optional>
@ -732,7 +733,7 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR
.device = *logical,
.preferredLargeHeapBlockSize = is_integrated
? (64u * 1024u * 1024u)
: (256u * 1024u * 1024u),
: (128u * 1024u * 1024u),
.pAllocationCallbacks = nullptr,
.pDeviceMemoryCallbacks = nullptr,
.pHeapSizeLimit = nullptr,
@ -1444,6 +1445,23 @@ std::optional<size_t> Device::GetSamplerHeapBudget() const {
return sampler_heap_budget;
}
Device::MemoryBudgetInfo Device::GetMemoryBudgetInfo() const {
std::array<VmaBudget, VK_MAX_MEMORY_HEAPS> budgets{};
vmaGetHeapBudgets(allocator, budgets.data());
MemoryBudgetInfo info{};
for (const size_t heap : valid_heap_memory) {
info.usage += budgets[heap].usage;
info.budget += budgets[heap].budget;
info.block_bytes += budgets[heap].statistics.blockBytes;
info.allocation_bytes += budgets[heap].statistics.allocationBytes;
}
return info;
}
void Device::TickAllocatorFrame() const {
vmaSetCurrentFrameIndex(allocator, ++allocator_frame_index);
}
u64 Device::GetDeviceMemoryUsage() const {
VkPhysicalDeviceMemoryBudgetPropertiesEXT budget;
budget.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT;
@ -1493,9 +1511,12 @@ void Device::CollectPhysicalMemoryInfo() {
device_access_memory -= reserve_memory;
if (Settings::values.vram_usage_mode.GetValue() != Settings::VramUsageMode::Aggressive) {
// Account for resolution scaling in memory limits
const size_t normal_memory = 6_GiB;
const size_t scaler_memory = 1_GiB * Settings::values.resolution_info.ScaleUp(1);
device_access_memory = std::min<u64>(device_access_memory, normal_memory + scaler_memory);
const u64 normal_memory = 6_GiB;
const u64 scaler_memory = 1_GiB * Settings::values.resolution_info.ScaleUp(1);
const u64 baseline = normal_memory + scaler_memory;
const u64 proportional = (device_access_memory / 4) * 3;
device_access_memory =
std::min<u64>(device_access_memory, std::max<u64>(baseline, proportional));
}
}
}

12
src/video_core/vulkan_common/vulkan_device.h

@ -255,6 +255,17 @@ public:
return allocator;
}
struct MemoryBudgetInfo {
u64 usage;
u64 budget;
u64 block_bytes;
u64 allocation_bytes;
};
MemoryBudgetInfo GetMemoryBudgetInfo() const;
void TickAllocatorFrame() const;
/// Returns the logical device.
const vk::Device& GetLogical() const {
return logical;
@ -1060,6 +1071,7 @@ private:
private:
VkInstance instance; ///< Vulkan instance.
VmaAllocator allocator; ///< VMA allocator.
mutable u32 allocator_frame_index{};
vk::DeviceDispatch dld; ///< Device function pointers.
vk::PhysicalDevice physical; ///< Physical device.
vk::Device logical; ///< Logical device.

65
src/video_core/vulkan_common/vulkan_memory_allocator.cpp

@ -224,6 +224,27 @@ namespace Vulkan {
MemoryAllocator::~MemoryAllocator() = default;
void MemoryAllocator::SetReclaimCallback(ReclaimCallback callback) {
reclaim_callback = std::move(callback);
owner_thread = std::this_thread::get_id();
}
bool MemoryAllocator::ReclaimAtLeast(u64 hint_bytes) const {
if (!reclaim_callback || in_reclaim) {
return false;
}
in_reclaim = true;
const u64 freed = reclaim_callback(hint_bytes);
in_reclaim = false;
return freed > 0;
}
void MemoryAllocator::AssertOwnerThread() const {
DEBUG_ASSERT_MSG(owner_thread == std::thread::id{} ||
owner_thread == std::this_thread::get_id(),
"VmaAllocator is externally synchronized but was used off-thread");
}
vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci) const
{
const VmaAllocationCreateInfo alloc_ci = {
@ -240,7 +261,26 @@ namespace Vulkan {
VkImage handle{};
VmaAllocation allocation{};
VmaAllocationInfo alloc_info{};
vk::Check(vmaCreateImage(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info));
AssertOwnerThread();
VkResult res = vmaCreateImage(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info);
if (res != VK_SUCCESS && ReclaimAtLeast(IMAGE_RECLAIM_HINT)) {
res = vmaCreateImage(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info);
}
if (res != VK_SUCCESS) {
auto relaxed_ci = alloc_ci;
relaxed_ci.flags &= ~VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT;
res = vmaCreateImage(allocator, &ci, &relaxed_ci, &handle, &allocation, &alloc_info);
if (res != VK_SUCCESS) {
relaxed_ci.preferredFlags &= ~VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
res = vmaCreateImage(allocator, &ci, &relaxed_ci, &handle, &allocation, &alloc_info);
}
}
vk::Check(res);
// Log GPU memory allocation for images
if (GPU::Logging::IsActive() &&
@ -277,7 +317,28 @@ namespace Vulkan {
VmaAllocation allocation{};
VkMemoryPropertyFlags property_flags{};
vk::Check(vmaCreateBuffer(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info));
AssertOwnerThread();
VkResult res = vmaCreateBuffer(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info);
if (res != VK_SUCCESS && ReclaimAtLeast(ci.size)) {
res = vmaCreateBuffer(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info);
}
if (res != VK_SUCCESS) {
auto relaxed_ci = alloc_ci;
relaxed_ci.flags &= ~VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT;
res = vmaCreateBuffer(allocator, &ci, &relaxed_ci, &handle, &allocation, &alloc_info);
if (res != VK_SUCCESS &&
(relaxed_ci.preferredFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)) {
relaxed_ci.preferredFlags &= ~VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
res = vmaCreateBuffer(allocator, &ci, &relaxed_ci, &handle, &allocation,
&alloc_info);
}
}
vk::Check(res);
vmaGetAllocationMemoryProperties(allocator, allocation, &property_flags);
// Log GPU memory allocation for buffers

15
src/video_core/vulkan_common/vulkan_memory_allocator.h

@ -6,8 +6,10 @@
#pragma once
#include <functional>
#include <memory>
#include <span>
#include <thread>
#include <vector>
#include "common/common_types.h"
@ -120,7 +122,17 @@ namespace Vulkan {
/// Commits memory required by the buffer and binds it (for buffers created outside VMA).
MemoryCommit Commit(const vk::Buffer &buffer, MemoryUsage usage);
using ReclaimCallback = std::function<u64(u64)>;
void SetReclaimCallback(ReclaimCallback callback);
private:
bool ReclaimAtLeast(u64 hint_bytes) const;
void AssertOwnerThread() const;
static constexpr u64 IMAGE_RECLAIM_HINT = 64ULL * 1024 * 1024;
static bool IsAutoUsage(VmaMemoryUsage u) noexcept {
switch (u) {
case VMA_MEMORY_USAGE_AUTO:
@ -137,6 +149,9 @@ namespace Vulkan {
const VkPhysicalDeviceMemoryProperties properties; ///< Physical device memory properties.
VkDeviceSize buffer_image_granularity; ///< Adjacent buffer/image granularity
u32 valid_memory_types{~0u};
ReclaimCallback reclaim_callback;
mutable bool in_reclaim{false};
std::thread::id owner_thread;
};
} // namespace Vulkan
Loading…
Cancel
Save