diff --git a/src/android/app/build.gradle.kts b/src/android/app/build.gradle.kts index a4076978a1..fd7c40e415 100644 --- a/src/android/app/build.gradle.kts +++ b/src/android/app/build.gradle.kts @@ -65,7 +65,7 @@ android { defaultConfig { applicationId = "dev.eden.eden_emulator" - minSdk = 24 + minSdk = 33 targetSdk = 36 versionName = getGitVersion() versionCode = autoVersion diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt index 5c02b5ed90..6c593eb3e0 100644 --- a/src/video_core/CMakeLists.txt +++ b/src/video_core/CMakeLists.txt @@ -33,6 +33,7 @@ add_library(video_core STATIC control/channel_state_cache.h control/scheduler.cpp control/scheduler.h + deferred_destruction_queue.h delayed_destruction_ring.h dirty_flags.cpp dirty_flags.h diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index e40aad1fb5..1dc0cc8357 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -31,44 +31,74 @@ BufferCache

::BufferCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, R immediately_free = (Settings::values.vram_usage_mode.GetValue() == Settings::VramUsageMode::Aggressive); #endif if (!runtime.CanReportMemoryUsage()) { - minimum_memory = DEFAULT_EXPECTED_MEMORY; - critical_memory = DEFAULT_CRITICAL_MEMORY; + memory_budget = FALLBACK_MEMORY_BUDGET; return; } - const s64 device_local_memory = static_cast(runtime.GetDeviceLocalMemory()); - const s64 min_spacing_expected = device_local_memory - 1_GiB; - const s64 min_spacing_critical = device_local_memory - 512_MiB; - const s64 mem_threshold = (std::min)(device_local_memory, TARGET_THRESHOLD); - const s64 min_vacancy_expected = (6 * mem_threshold) / 10; - const s64 min_vacancy_critical = (2 * mem_threshold) / 10; - minimum_memory = static_cast( - (std::max)((std::min)(device_local_memory - min_vacancy_expected, min_spacing_expected), - DEFAULT_EXPECTED_MEMORY)); - critical_memory = static_cast( - (std::max)((std::min)(device_local_memory - min_vacancy_critical, min_spacing_critical), - DEFAULT_CRITICAL_MEMORY)); + memory_budget = runtime.GetDeviceLocalMemory(); } template BufferCache

::~BufferCache() = default; template -void BufferCache

::RunGarbageCollector() { - const bool aggressive_gc = total_used_memory >= critical_memory; - const u64 ticks_to_destroy = aggressive_gc ? 60 : 120; - int num_iterations = aggressive_gc ? 64 : 32; - const auto clean_up = [this, &num_iterations](BufferId buffer_id) { - if (num_iterations == 0) { +u64 BufferCache

::DeviceUsage(bool force_refresh) { + if (!runtime.CanReportMemoryUsage()) { + return total_used_memory; + } + if (force_refresh || usage_refresh_countdown == 0) { + cached_device_usage = runtime.GetDeviceAllocationUsage(); + usage_refresh_countdown = USAGE_REFRESH_INTERVAL; + } else { + --usage_refresh_countdown; + } + return cached_device_usage; +} + +template +u64 BufferCache

::ReclaimMemory(u64 target_bytes, bool allow_download) { + if (target_bytes == 0 || in_reclaim) { + return 0; + } + in_reclaim = true; + u64 freed = 0; + const auto clean_up = [&](BufferId buffer_id) { + if (freed >= target_bytes) { return true; } - --num_iterations; auto& buffer = slot_buffers[buffer_id]; + if (!allow_download && IsRegionGpuModified(buffer.CpuAddr(), buffer.SizeBytes())) { + return false; + } + const u64 buffer_bytes = Common::AlignUp(buffer.SizeBytes(), 1024); DownloadBufferMemory(buffer); DeleteBuffer(buffer_id); + freed += buffer_bytes; return false; }; - lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, clean_up); + const u64 cold_tick = + frame_tick > RECLAIM_GUARD_FRAMES ? frame_tick - RECLAIM_GUARD_FRAMES : 0; + lru_cache.ForEachItemBelow(cold_tick, clean_up); + if (freed < target_bytes) { + lru_cache.ForEachItemBelow(frame_tick > 0 ? frame_tick - 1 : 0, clean_up); + } + in_reclaim = false; + usage_refresh_countdown = 0; + reclaim_stalled = freed == 0; + return freed; +} + +template +void BufferCache

::EnsureHeadroom(bool allow_download) { + if (reclaim_stalled) { + return; + } + const u64 limit = memory_budget > RECLAIM_HEADROOM ? memory_budget - RECLAIM_HEADROOM : 0; + const u64 usage = DeviceUsage(false); + if (usage <= limit) { + return; + } + ReclaimMemory(usage - limit, allow_download); } template @@ -96,15 +126,11 @@ void BufferCache

::TickFrame() { const bool skip_preferred = hits * 256 < shots * 251; channel_state->uniform_buffer_skip_cache_size = skip_preferred ? DEFAULT_SKIP_CACHE_SIZE : 0; - // If we can obtain the memory info, use it instead of the estimate. - if (runtime.CanReportMemoryUsage()) { - total_used_memory = runtime.GetDeviceMemoryUsage(); - } - if (total_used_memory >= minimum_memory) { - RunGarbageCollector(); - } + usage_refresh_countdown = 0; + reclaim_stalled = false; + EnsureHeadroom(true); ++frame_tick; - delayed_destruction_ring.Tick(); + sentenced_buffers.Reclaim(runtime.CompletedSyncPoint()); for (auto& buffer : async_buffers_death_ring) { runtime.FreeDeferredStagingBuffer(buffer); @@ -1576,6 +1602,7 @@ void BufferCache

::JoinOverlap(BufferId new_buffer_id, BufferId overlap_id, template BufferId BufferCache

::CreateBuffer(DAddr device_addr, u32 wanted_size) { + EnsureHeadroom(false); DAddr device_addr_end = Common::AlignUp(device_addr + wanted_size, CACHING_PAGESIZE); device_addr = Common::AlignDown(device_addr, CACHING_PAGESIZE); wanted_size = static_cast(device_addr_end - device_addr); @@ -1613,7 +1640,7 @@ void BufferCache

::ChangeRegister(BufferId buffer_id) { total_used_memory += Common::AlignUp(size, 1024); buffer.setLRUID(lru_cache.Insert(buffer_id, frame_tick)); } else { - total_used_memory -= Common::AlignUp(size, 1024); + total_used_memory -= (std::min)(total_used_memory, Common::AlignUp(size, 1024)); lru_cache.Free(buffer.getLRUID()); } const DAddr device_addr_begin = buffer.CpuAddr(); @@ -1872,7 +1899,7 @@ void BufferCache

::DeleteBuffer(BufferId buffer_id, bool do_not_mark) { #ifdef YUZU_LEGACY if (!do_not_mark || !immediately_free) #endif - delayed_destruction_ring.Push(std::move(slot_buffers[buffer_id])); + sentenced_buffers.Push(std::move(slot_buffers[buffer_id]), runtime.CurrentSyncPoint()); slot_buffers.erase(buffer_id); diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 14ab3e6ebc..7d618e51b1 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -30,7 +30,7 @@ #include "common/slot_vector.h" #include "video_core/buffer_cache/buffer_base.h" #include "video_core/control/channel_state_cache.h" -#include "video_core/delayed_destruction_ring.h" +#include "video_core/deferred_destruction_queue.h" #include "video_core/dirty_flags.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/engines/kepler_compute.h" @@ -182,13 +182,14 @@ class BufferCache : public VideoCommon::ChannelSetupCaches slot_buffers; -#ifdef YUZU_LEGACY - static constexpr size_t TICKS_TO_DESTROY = 6; -#else - static constexpr size_t TICKS_TO_DESTROY = 8; -#endif - DelayedDestructionRing delayed_destruction_ring; + DeferredDestructionQueue sentenced_buffers; const Tegra::Engines::Maxwell3D::DrawManager::IndirectParams* current_draw_indirect{}; @@ -515,8 +515,11 @@ private: Common::LeastRecentlyUsedCache lru_cache; u64 frame_tick = 0; u64 total_used_memory = 0; - u64 minimum_memory = 0; - u64 critical_memory = 0; + u64 memory_budget = 0; + u64 cached_device_usage = 0; + u32 usage_refresh_countdown = 0; + bool in_reclaim = false; + bool reclaim_stalled = false; BufferId inline_buffer_id; #ifdef YUZU_LEGACY bool immediately_free = false; diff --git a/src/video_core/deferred_destruction_queue.h b/src/video_core/deferred_destruction_queue.h new file mode 100644 index 0000000000..adc276f4fe --- /dev/null +++ b/src/video_core/deferred_destruction_queue.h @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include +#include +#include + +#include "common/common_types.h" + +namespace VideoCommon { + +template +class DeferredDestructionQueue { +public: + void Push(T&& object, u64 sync_point) { + entries.emplace_back(std::move(object), sync_point); + } + + void Reclaim(u64 completed_sync_point) { + while (!entries.empty() && entries.front().sync_point <= completed_sync_point) { + entries.pop_front(); + } + } + + void Clear() { + entries.clear(); + } + + [[nodiscard]] size_t Size() const noexcept { + return entries.size(); + } + + [[nodiscard]] bool Empty() const noexcept { + return entries.empty(); + } + +private: + struct Entry { + Entry(T&& object_, u64 sync_point_) noexcept + : object{std::move(object_)}, sync_point{sync_point_} {} + + T object; + u64 sync_point; + }; + + std::deque entries; +}; + +} // namespace VideoCommon diff --git a/src/video_core/renderer_opengl/gl_buffer_cache.h b/src/video_core/renderer_opengl/gl_buffer_cache.h index a0acfc48c1..7ee8464c48 100644 --- a/src/video_core/renderer_opengl/gl_buffer_cache.h +++ b/src/video_core/renderer_opengl/gl_buffer_cache.h @@ -93,7 +93,17 @@ public: void PostCopyBarrier(); void Finish(); - void TickFrame(Common::SlotVector&) noexcept {} + void TickFrame(Common::SlotVector&) noexcept { + ++sync_point; + } + + u64 CurrentSyncPoint() const noexcept { + return sync_point; + } + + u64 CompletedSyncPoint() const noexcept { + return sync_point > SYNC_POINT_DELAY ? sync_point - SYNC_POINT_DELAY : 0; + } void ClearBuffer(Buffer& dest_buffer, u32 offset, size_t size, u32 value); @@ -128,6 +138,10 @@ public: u64 GetDeviceMemoryUsage() const; + u64 GetDeviceAllocationUsage() const { + return GetDeviceMemoryUsage(); + } + void BindFastUniformBuffer(size_t stage, u32 binding_index, u32 size) { const GLuint handle = fast_uniforms[stage][binding_index].handle; const GLsizeiptr gl_size = static_cast(size); @@ -213,9 +227,13 @@ private: GL_FRAGMENT_PROGRAM_PARAMETER_BUFFER_NV, }; + static constexpr u64 SYNC_POINT_DELAY = 8; + const Device& device; StagingBufferPool& staging_buffer_pool; + u64 sync_point = 1; + bool has_fast_buffer_sub_data = false; bool use_assembly_shaders = false; bool has_unified_vertex_buffers = false; diff --git a/src/video_core/renderer_opengl/gl_texture_cache.h b/src/video_core/renderer_opengl/gl_texture_cache.h index dfcef4b0b6..942ffdd514 100644 --- a/src/video_core/renderer_opengl/gl_texture_cache.h +++ b/src/video_core/renderer_opengl/gl_texture_cache.h @@ -87,6 +87,10 @@ public: u64 GetDeviceMemoryUsage() const; + u64 GetDeviceAllocationUsage() const { + return GetDeviceMemoryUsage(); + } + bool CanReportMemoryUsage() const { return device.CanReportMemoryUsage(); } @@ -139,7 +143,19 @@ public: bool HasNativeASTC() const noexcept; - void TickFrame() {} + void TickFrame() { + ++sync_point; + } + + u64 CurrentSyncPoint() const noexcept { + return sync_point; + } + + u64 CompletedSyncPoint() const noexcept { + return sync_point > SYNC_POINT_DELAY ? sync_point - SYNC_POINT_DELAY : 0; + } + + void WaitSyncPoint(u64) {} StateTracker& GetStateTracker() { return state_tracker; @@ -174,6 +190,9 @@ private: std::array rescale_read_fbos; const Settings::ResolutionScalingInfo& resolution; u64 device_access_memory; + + static constexpr u64 SYNC_POINT_DELAY = 8; + u64 sync_point = 1; }; class Image : public VideoCommon::ImageBase { @@ -370,6 +389,7 @@ struct TextureCacheParams { static constexpr bool HAS_EMULATED_COPIES = true; static constexpr bool HAS_DEVICE_MEMORY_INFO = true; static constexpr bool IMPLEMENTS_ASYNC_DOWNLOADS = true; + static constexpr bool HAS_TIMELINE_SYNC_POINTS = false; using Runtime = OpenGL::TextureCacheRuntime; using Image = OpenGL::Image; diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp index eb37f922a1..d181e53af3 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp @@ -376,6 +376,10 @@ u64 BufferCacheRuntime::GetDeviceMemoryUsage() const { return device.GetDeviceMemoryUsage(); } +u64 BufferCacheRuntime::GetDeviceAllocationUsage() const { + return device.GetMemoryBudgetInfo().allocation_bytes; +} + bool BufferCacheRuntime::CanReportMemoryUsage() const { return device.CanReportMemoryUsage(); } @@ -404,6 +408,16 @@ u64 BufferCacheRuntime::KnownGpuTick() { return scheduler.GetMasterSemaphore().KnownGpuTick(); } +u64 BufferCacheRuntime::CurrentSyncPoint() const noexcept { + return scheduler.GetMasterSemaphore().CurrentTick(); +} + +u64 BufferCacheRuntime::CompletedSyncPoint() const { + auto& master_semaphore = scheduler.GetMasterSemaphore(); + master_semaphore.Refresh(); + return master_semaphore.KnownGpuTick(); +} + void BufferCacheRuntime::Wait(u64 buffer_tick) { scheduler.Wait(buffer_tick); } diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index d4ad156073..ed45aa7125 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -100,10 +100,16 @@ public: void Finish(); + u64 CurrentSyncPoint() const noexcept; + + u64 CompletedSyncPoint() const; + u64 GetDeviceLocalMemory() const; u64 GetDeviceMemoryUsage() const; + u64 GetDeviceAllocationUsage() const; + bool CanReportMemoryUsage() const; u32 GetUniformBufferAlignment() const; diff --git a/src/video_core/renderer_vulkan/vk_present_manager.cpp b/src/video_core/renderer_vulkan/vk_present_manager.cpp index 53c3f6759f..13838794de 100644 --- a/src/video_core/renderer_vulkan/vk_present_manager.cpp +++ b/src/video_core/renderer_vulkan/vk_present_manager.cpp @@ -296,9 +296,6 @@ void PresentManager::RecreateSwapchain(Frame* frame) { } void PresentManager::SetImageCount() { - // We cannot have more than 7 images in flight at any given time. - // FRAMES_IN_FLIGHT is 8, and the cache TICKS_TO_DESTROY is 8. - // Mali drivers will give us 6. image_count = std::min(swapchain.GetImageCount(), 7); } diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index 720f3b868c..c63d1d1053 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -221,9 +221,17 @@ RasterizerVulkan::RasterizerVulkan(Core::Frontend::EmuWindow& emu_window_, Tegra fence_manager(*this, gpu, texture_cache, buffer_cache, query_cache, device, scheduler), wfi_event(device.GetLogical().CreateEvent()) { scheduler.SetQueryCache(query_cache); + memory_allocator.SetReclaimCallback([this](u64 bytes) -> u64 { + u64 freed = texture_cache.ReclaimMemory(bytes, false); + if (freed < bytes) { + freed += buffer_cache.ReclaimMemory(bytes - freed, false); + } + return freed; + }); } RasterizerVulkan::~RasterizerVulkan() { + memory_allocator.SetReclaimCallback(nullptr); scheduler.WaitWorker(); scheduler.Finish(); } diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp index 44a4227a19..b481c03013 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp @@ -1753,6 +1753,20 @@ void TextureCacheRuntime::CopyImageMSAA(Image& dst, Image& src, src.info.format, num_samples, copies, msaa_to_non_msaa); } +u64 TextureCacheRuntime::CurrentSyncPoint() const noexcept { + return scheduler.CurrentTick(); +} + +u64 TextureCacheRuntime::CompletedSyncPoint() const { + auto& master_semaphore = scheduler.GetMasterSemaphore(); + master_semaphore.Refresh(); + return master_semaphore.KnownGpuTick(); +} + +void TextureCacheRuntime::WaitSyncPoint(u64 sync_point) { + scheduler.Wait(sync_point); +} + u64 TextureCacheRuntime::GetDeviceLocalMemory() const { return device.GetDeviceLocalMemory(); } @@ -1761,6 +1775,10 @@ u64 TextureCacheRuntime::GetDeviceMemoryUsage() const { return device.GetDeviceMemoryUsage(); } +u64 TextureCacheRuntime::GetDeviceAllocationUsage() const { + return device.GetMemoryBudgetInfo().allocation_bytes; +} + bool TextureCacheRuntime::CanReportMemoryUsage() const { return device.CanReportMemoryUsage(); } @@ -1770,6 +1788,7 @@ std::optional TextureCacheRuntime::GetSamplerHeapBudget() const { } void TextureCacheRuntime::TickFrame() { + device.TickAllocatorFrame(); std::erase_if(pending_msaa_images, [this](const auto& pending) { return scheduler.IsFree(pending.first); }); diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h index 99f2978cca..1ef3639853 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.h +++ b/src/video_core/renderer_vulkan/vk_texture_cache.h @@ -60,10 +60,18 @@ public: void TickFrame(); + u64 CurrentSyncPoint() const noexcept; + + u64 CompletedSyncPoint() const; + + void WaitSyncPoint(u64 sync_point); + u64 GetDeviceLocalMemory() const; u64 GetDeviceMemoryUsage() const; + u64 GetDeviceAllocationUsage() const; + bool CanReportMemoryUsage() const; std::optional GetSamplerHeapBudget() const; @@ -487,6 +495,7 @@ struct TextureCacheParams { static constexpr bool HAS_EMULATED_COPIES = false; static constexpr bool HAS_DEVICE_MEMORY_INFO = true; static constexpr bool IMPLEMENTS_ASYNC_DOWNLOADS = true; + static constexpr bool HAS_TIMELINE_SYNC_POINTS = true; using Runtime = Vulkan::TextureCacheRuntime; using Image = Vulkan::Image; diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 98a5e053a7..4baf27cb5a 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -58,23 +58,9 @@ TextureCache

::TextureCache(Runtime& runtime_, Tegra::MaxwellDeviceMemoryManag void(slot_samplers.insert(runtime, sampler_descriptor)); if constexpr (HAS_DEVICE_MEMORY_INFO) { - const s64 device_local_memory = static_cast(runtime.GetDeviceLocalMemory()); - const s64 min_spacing_expected = device_local_memory - 1_GiB; - const s64 min_spacing_critical = device_local_memory - 512_MiB; - const s64 mem_threshold = (std::min)(device_local_memory, TARGET_THRESHOLD); - const s64 min_vacancy_expected = (6 * mem_threshold) / 10; - const s64 min_vacancy_critical = (2 * mem_threshold) / 10; - expected_memory = static_cast( - (std::max)((std::min)(device_local_memory - min_vacancy_expected, min_spacing_expected), - DEFAULT_EXPECTED_MEMORY)); - critical_memory = static_cast( - (std::max)((std::min)(device_local_memory - min_vacancy_critical, min_spacing_critical), - DEFAULT_CRITICAL_MEMORY)); - minimum_memory = static_cast((device_local_memory - mem_threshold) / 2); + memory_budget = runtime.GetDeviceLocalMemory(); } else { - expected_memory = DEFAULT_EXPECTED_MEMORY + 512_MiB; - critical_memory = DEFAULT_CRITICAL_MEMORY + 1_GiB; - minimum_memory = 0; + memory_budget = FALLBACK_MEMORY_BUDGET; } const bool gpu_unswizzle_enabled = Settings::values.gpu_unswizzle_enabled.GetValue(); @@ -114,71 +100,140 @@ TextureCache

::TextureCache(Runtime& runtime_, Tegra::MaxwellDeviceMemoryManag } template -void TextureCache

::RunGarbageCollector() { - bool high_priority_mode = false; - bool aggressive_mode = false; - u64 ticks_to_destroy = 0; - size_t num_iterations = 0; - const auto Configure = [&](bool allow_aggressive) { - high_priority_mode = total_used_memory >= expected_memory; - aggressive_mode = allow_aggressive && total_used_memory >= critical_memory; - ticks_to_destroy = aggressive_mode ? 10ULL : high_priority_mode ? 25ULL : 50ULL; - num_iterations = aggressive_mode ? 40 : (high_priority_mode ? 20 : 10); - }; - const auto Cleanup = [this, &num_iterations, &high_priority_mode, &aggressive_mode](ImageId image_id) { - if (num_iterations == 0) { +void TextureCache

::QueueEvictionDownload(Image& image) { + auto copies = FullDownloadCopies(image.info); + auto staging = runtime.DownloadStagingBuffer(image.unswizzled_size_bytes, true); + image.DownloadMemory(staging, FixSmallVectorADL(copies)); + pending_eviction_downloads.push_back(PendingEvictionDownload{ + .staging = staging, + .gpu_memory = gpu_memory, + .gpu_addr = image.gpu_addr, + .info = image.info, + .copies = std::move(copies), + .sync_point = runtime.CurrentSyncPoint(), + }); +} + +template +void TextureCache

::TickEvictionDownloads(u64 completed_sync_point) { + while (!pending_eviction_downloads.empty() && + pending_eviction_downloads.front().sync_point <= completed_sync_point) { + auto& entry = pending_eviction_downloads.front(); + SwizzleImage(*entry.gpu_memory, entry.gpu_addr, entry.info, FixSmallVectorADL(entry.copies), + entry.staging.mapped_span.subspan(entry.staging.offset), swizzle_data_buffer); + runtime.FreeDeferredStagingBuffer(entry.staging); + pending_eviction_downloads.pop_front(); + } +} + +template +void TextureCache

::FlushEvictionDownloads() { + if (pending_eviction_downloads.empty()) { + return; + } + const u64 last_sync_point = pending_eviction_downloads.back().sync_point; + runtime.WaitSyncPoint(last_sync_point); + TickEvictionDownloads(last_sync_point); +} + +template +u64 TextureCache

::ImageSizeBytes(const ImageBase& image) { + u64 tentative_size = (std::max)(image.guest_size_bytes, image.unswizzled_size_bytes); + if ((IsPixelFormatASTC(image.info.format) && + True(image.flags & ImageFlagBits::AcceleratedUpload)) || + True(image.flags & ImageFlagBits::Converted)) { + tentative_size = TranscodedAstcSize(tentative_size, image.info.format); + } + u64 size = Common::AlignUp(tentative_size, 1024); + if (image.HasScaled()) { + size += GetScaledImageSizeBytes(image); + } + return size; +} + +template +u64 TextureCache

::DeviceUsage(bool force_refresh) { + if (!runtime.CanReportMemoryUsage()) { + return total_used_memory; + } + if (force_refresh || usage_refresh_countdown == 0) { + cached_device_usage = runtime.GetDeviceAllocationUsage(); + usage_refresh_countdown = USAGE_REFRESH_INTERVAL; + } else { + --usage_refresh_countdown; + } + return cached_device_usage; +} + +template +u64 TextureCache

::ReclaimMemory(u64 target_bytes, bool allow_download) { + if (target_bytes == 0 || in_reclaim) { + return 0; + } + in_reclaim = true; + u64 freed = 0; + const auto evict = [&](ImageId image_id) { + if (freed >= target_bytes) { return true; } - --num_iterations; auto& image = slot_images[image_id]; if (True(image.flags & ImageFlagBits::IsDecoding)) { return false; } - const bool must_download = image.IsSafeDownload() && False(image.flags & ImageFlagBits::BadOverlap); - if ((!aggressive_mode && True(image.flags & ImageFlagBits::CostlyLoad)) || (!high_priority_mode && must_download)) { - return false; - } + const bool must_download = + image.IsSafeDownload() && False(image.flags & ImageFlagBits::BadOverlap); + bool queued_download = false; if (must_download) { - auto map = runtime.DownloadStagingBuffer(image.unswizzled_size_bytes); - const auto copies = FixSmallVectorADL(FullDownloadCopies(image.info)); - image.DownloadMemory(map, copies); - runtime.Finish(); - SwizzleImage(*gpu_memory, image.gpu_addr, image.info, copies, map.mapped_span, swizzle_data_buffer); + if (!allow_download || !HAS_TIMELINE_SYNC_POINTS) { + return false; + } + QueueEvictionDownload(image); + queued_download = true; } + const u64 image_bytes = ImageSizeBytes(image); if (True(image.flags & ImageFlagBits::Tracked)) { UntrackImage(image, image_id); } UnregisterImage(image_id); - DeleteImage(image_id, image.scale_tick > frame_tick + 5); - if (aggressive_mode && total_used_memory < critical_memory) { - num_iterations >>= 2; - aggressive_mode = false; - } else if (high_priority_mode && total_used_memory < expected_memory) { - num_iterations >>= 1; - high_priority_mode = false; - } + DeleteImage(image_id, !queued_download && image.scale_tick > frame_tick + 5); + freed += image_bytes; return false; }; - Configure(false); - lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup); - if (total_used_memory >= critical_memory) { - Configure(true); - lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup); + const u64 cold_tick = + frame_tick > RECLAIM_GUARD_FRAMES ? frame_tick - RECLAIM_GUARD_FRAMES : 0; + lru_cache.ForEachItemBelow(cold_tick, evict); + if (freed < target_bytes) { + lru_cache.ForEachItemBelow(frame_tick > 0 ? frame_tick - 1 : 0, evict); } + in_reclaim = false; + usage_refresh_countdown = 0; + reclaim_stalled = freed == 0; + return freed; } template -void TextureCache

::TickFrame() { - // If we can obtain the memory info, use it instead of the estimate. - if (runtime.CanReportMemoryUsage()) { - total_used_memory = runtime.GetDeviceMemoryUsage(); +void TextureCache

::EnsureHeadroom(bool allow_download) { + if (reclaim_stalled) { + return; } - if (total_used_memory > minimum_memory) { - RunGarbageCollector(); + const u64 limit = memory_budget > RECLAIM_HEADROOM ? memory_budget - RECLAIM_HEADROOM : 0; + const u64 usage = DeviceUsage(false); + if (usage <= limit) { + return; } - sentenced_images.Tick(); - sentenced_framebuffers.Tick(); - sentenced_image_view.Tick(); + ReclaimMemory(usage - limit, allow_download); +} + +template +void TextureCache

::TickFrame() { + usage_refresh_countdown = 0; + reclaim_stalled = false; + EnsureHeadroom(true); + const u64 completed_sync_point = runtime.CompletedSyncPoint(); + TickEvictionDownloads(completed_sync_point); + sentenced_images.Reclaim(completed_sync_point); + sentenced_framebuffers.Reclaim(completed_sync_point); + sentenced_image_view.Reclaim(completed_sync_point); TickAsyncDecode(); TickAsyncUnswizzle(); @@ -596,6 +651,7 @@ void TextureCache

::WriteMemory(DAddr cpu_addr, size_t size) { template void TextureCache

::DownloadMemory(DAddr cpu_addr, size_t size) { + FlushEvictionDownloads(); boost::container::small_vector images; ForEachImageInRegion(cpu_addr, size, [&images](ImageId image_id, ImageBase& image) { if (!image.IsSafeDownload()) { @@ -894,6 +950,7 @@ void TextureCache

::CommitAsyncFlushes() { template void TextureCache

::PopAsyncFlushes() { + FlushEvictionDownloads(); if (committed_downloads.empty()) { return; } @@ -1294,8 +1351,9 @@ void TextureCache

::InvalidateScale(Image& image) { } RemoveImageViewReferences(image_view_ids); RemoveFramebuffers(image_view_ids); + const u64 sync_point = runtime.CurrentSyncPoint(); for (const ImageViewId image_view_id : image_view_ids) { - sentenced_image_view.Push(std::move(slot_image_views[image_view_id])); + sentenced_image_view.Push(std::move(slot_image_views[image_view_id]), sync_point); slot_image_views.erase(image_view_id); } image.image_view_ids.clear(); @@ -1523,6 +1581,7 @@ ImageId TextureCache

::InsertImage(const ImageInfo& info, GPUVAddr gpu_addr, template ImageId TextureCache

::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, DAddr cpu_addr) { + EnsureHeadroom(false); ImageInfo new_info = info; const size_t size_bytes = CalculateGuestSizeInBytes(new_info); const bool broken_views = runtime.HasBrokenTextureViewFormats(); @@ -2185,13 +2244,7 @@ void TextureCache

::RegisterImage(ImageId image_id) { ASSERT_MSG(False(image.flags & ImageFlagBits::Registered), "Trying to register an already registered image"); image.flags |= ImageFlagBits::Registered; - u64 tentative_size = (std::max)(image.guest_size_bytes, image.unswizzled_size_bytes); - if ((IsPixelFormatASTC(image.info.format) && - True(image.flags & ImageFlagBits::AcceleratedUpload)) || - True(image.flags & ImageFlagBits::Converted)) { - tentative_size = TranscodedAstcSize(tentative_size, image.info.format); - } - total_used_memory += Common::AlignUp(tentative_size, 1024); + total_used_memory += ImageSizeBytes(image); image.lru_index = lru_cache.Insert(image_id, frame_tick); ForEachGPUPage(image.gpu_addr, image.guest_size_bytes, [this, image_id](u64 page) { @@ -2354,16 +2407,7 @@ void TextureCache

::UntrackImage(ImageBase& image, ImageId image_id) { template void TextureCache

::DeleteImage(ImageId image_id, bool immediate_delete) { ImageBase& image = slot_images[image_id]; - if (image.HasScaled()) { - total_used_memory -= GetScaledImageSizeBytes(image); - } - u64 tentative_size = (std::max)(image.guest_size_bytes, image.unswizzled_size_bytes); - if ((IsPixelFormatASTC(image.info.format) && - True(image.flags & ImageFlagBits::AcceleratedUpload)) || - True(image.flags & ImageFlagBits::Converted)) { - tentative_size = TranscodedAstcSize(tentative_size, image.info.format); - } - total_used_memory -= Common::AlignUp(tentative_size, 1024); + total_used_memory -= (std::min)(total_used_memory, ImageSizeBytes(image)); const GPUVAddr gpu_addr = image.gpu_addr; const auto alloc_it = image_allocs_table.find(gpu_addr); if (alloc_it == image_allocs_table.end()) { @@ -2417,14 +2461,15 @@ void TextureCache

::DeleteImage(ImageId image_id, bool immediate_delete) { ASSERT_MSG(num_removed_overlaps == 1, "Invalid number of removed overlapps: {}", num_removed_overlaps); } + const u64 sync_point = runtime.CurrentSyncPoint(); for (const ImageViewId image_view_id : image_view_ids) { if (!immediate_delete) { - sentenced_image_view.Push(std::move(slot_image_views[image_view_id])); + sentenced_image_view.Push(std::move(slot_image_views[image_view_id]), sync_point); } slot_image_views.erase(image_view_id); } if (!immediate_delete) { - sentenced_images.Push(std::move(slot_images[image_id])); + sentenced_images.Push(std::move(slot_images[image_id]), sync_point); } slot_images.erase(image_id); @@ -2470,7 +2515,8 @@ void TextureCache

::RemoveFramebuffers(std::span removed_vi last_framebuffer_id = {}; last_framebuffer_serial = 0; } - sentenced_framebuffers.Push(std::move(slot_framebuffers[framebuffer_id])); + sentenced_framebuffers.Push(std::move(slot_framebuffers[framebuffer_id]), + runtime.CurrentSyncPoint()); it = framebuffers.erase(it); } else { ++it; diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index fbc2bb4cf7..25129144f9 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -30,7 +30,7 @@ #include "common/thread_worker.h" #include "video_core/compatible_formats.h" #include "video_core/control/channel_state_cache.h" -#include "video_core/delayed_destruction_ring.h" +#include "video_core/deferred_destruction_queue.h" #include "video_core/engines/fermi_2d.h" #include "video_core/surface.h" #include "video_core/texture_cache/descriptor_table.h" @@ -108,18 +108,19 @@ class TextureCache : public VideoCommon::ChannelSetupCaches::max)()}; #ifdef YUZU_LEGACY - static constexpr s64 TARGET_THRESHOLD = 3_GiB; + static constexpr u64 RECLAIM_HEADROOM = 384_MiB; #else - static constexpr s64 TARGET_THRESHOLD = 4_GiB; + static constexpr u64 RECLAIM_HEADROOM = 512_MiB; #endif - static constexpr s64 DEFAULT_EXPECTED_MEMORY = 1_GiB + 125_MiB; - static constexpr s64 DEFAULT_CRITICAL_MEMORY = 1_GiB + 625_MiB; - static constexpr size_t GC_EMERGENCY_COUNTS = 2; + static constexpr u64 FALLBACK_MEMORY_BUDGET = 2_GiB; + static constexpr u32 USAGE_REFRESH_INTERVAL = 16; + static constexpr u64 RECLAIM_GUARD_FRAMES = 8; using Runtime = typename P::Runtime; using Image = typename P::Image; @@ -154,6 +155,8 @@ public: /// Notify the cache that a new frame has been queued void TickFrame(); + u64 ReclaimMemory(u64 target_bytes, bool allow_download); + /// Return a constant reference to the given image view id [[nodiscard]] const ImageView& GetImageView(ImageViewId id) const noexcept; @@ -293,8 +296,17 @@ private: void OnGPUASRegister(size_t map_id) final override; - /// Runs the Garbage Collector. - void RunGarbageCollector(); + u64 ImageSizeBytes(const ImageBase& image); + + u64 DeviceUsage(bool force_refresh); + + void EnsureHeadroom(bool allow_download); + + void QueueEvictionDownload(Image& image); + + void TickEvictionDownloads(u64 completed_sync_point); + + void FlushEvictionDownloads(); /// Find or create an image view in the guest descriptor table ImageViewId VisitImageView(u32 index, bool compute); @@ -451,9 +463,11 @@ private: bool has_deleted_images = false; bool is_rescaling = false; u64 total_used_memory = 0; - u64 minimum_memory; - u64 expected_memory; - u64 critical_memory; + u64 memory_budget = 0; + u64 cached_device_usage = 0; + u32 usage_refresh_countdown = 0; + bool in_reclaim = false; + bool reclaim_stalled = false; size_t gpu_unswizzle_maxsize = 0; size_t swizzle_chunk_size = 0; u32 swizzle_slices_per_batch = 0; @@ -491,14 +505,19 @@ private: }; Common::LeastRecentlyUsedCache lru_cache; - #ifdef YUZU_LEGACY - static constexpr size_t TICKS_TO_DESTROY = 6; - #else - static constexpr size_t TICKS_TO_DESTROY = 8; -#endif - DelayedDestructionRing sentenced_images; - DelayedDestructionRing sentenced_image_view; - DelayedDestructionRing sentenced_framebuffers; + DeferredDestructionQueue sentenced_images; + DeferredDestructionQueue sentenced_image_view; + DeferredDestructionQueue sentenced_framebuffers; + + struct PendingEvictionDownload { + AsyncBuffer staging; + Tegra::MemoryManager* gpu_memory; + GPUVAddr gpu_addr; + VideoCommon::ImageInfo info; + boost::container::small_vector copies; + u64 sync_point; + }; + std::deque pending_eviction_downloads; ankerl::unordered_dense::map image_allocs_table; diff --git a/src/video_core/vulkan_common/vulkan_device.cpp b/src/video_core/vulkan_common/vulkan_device.cpp index 47b47d580f..ef6093494d 100644 --- a/src/video_core/vulkan_common/vulkan_device.cpp +++ b/src/video_core/vulkan_common/vulkan_device.cpp @@ -5,6 +5,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include +#include #include #include #include @@ -732,7 +733,7 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR .device = *logical, .preferredLargeHeapBlockSize = is_integrated ? (64u * 1024u * 1024u) - : (256u * 1024u * 1024u), + : (128u * 1024u * 1024u), .pAllocationCallbacks = nullptr, .pDeviceMemoryCallbacks = nullptr, .pHeapSizeLimit = nullptr, @@ -1444,6 +1445,23 @@ std::optional Device::GetSamplerHeapBudget() const { return sampler_heap_budget; } +Device::MemoryBudgetInfo Device::GetMemoryBudgetInfo() const { + std::array budgets{}; + vmaGetHeapBudgets(allocator, budgets.data()); + MemoryBudgetInfo info{}; + for (const size_t heap : valid_heap_memory) { + info.usage += budgets[heap].usage; + info.budget += budgets[heap].budget; + info.block_bytes += budgets[heap].statistics.blockBytes; + info.allocation_bytes += budgets[heap].statistics.allocationBytes; + } + return info; +} + +void Device::TickAllocatorFrame() const { + vmaSetCurrentFrameIndex(allocator, ++allocator_frame_index); +} + u64 Device::GetDeviceMemoryUsage() const { VkPhysicalDeviceMemoryBudgetPropertiesEXT budget; budget.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT; @@ -1493,9 +1511,12 @@ void Device::CollectPhysicalMemoryInfo() { device_access_memory -= reserve_memory; if (Settings::values.vram_usage_mode.GetValue() != Settings::VramUsageMode::Aggressive) { // Account for resolution scaling in memory limits - const size_t normal_memory = 6_GiB; - const size_t scaler_memory = 1_GiB * Settings::values.resolution_info.ScaleUp(1); - device_access_memory = std::min(device_access_memory, normal_memory + scaler_memory); + const u64 normal_memory = 6_GiB; + const u64 scaler_memory = 1_GiB * Settings::values.resolution_info.ScaleUp(1); + const u64 baseline = normal_memory + scaler_memory; + const u64 proportional = (device_access_memory / 4) * 3; + device_access_memory = + std::min(device_access_memory, std::max(baseline, proportional)); } } } diff --git a/src/video_core/vulkan_common/vulkan_device.h b/src/video_core/vulkan_common/vulkan_device.h index c449f60324..9ac91fc3d0 100644 --- a/src/video_core/vulkan_common/vulkan_device.h +++ b/src/video_core/vulkan_common/vulkan_device.h @@ -255,6 +255,17 @@ public: return allocator; } + struct MemoryBudgetInfo { + u64 usage; + u64 budget; + u64 block_bytes; + u64 allocation_bytes; + }; + + MemoryBudgetInfo GetMemoryBudgetInfo() const; + + void TickAllocatorFrame() const; + /// Returns the logical device. const vk::Device& GetLogical() const { return logical; @@ -1060,6 +1071,7 @@ private: private: VkInstance instance; ///< Vulkan instance. VmaAllocator allocator; ///< VMA allocator. + mutable u32 allocator_frame_index{}; vk::DeviceDispatch dld; ///< Device function pointers. vk::PhysicalDevice physical; ///< Physical device. vk::Device logical; ///< Logical device. diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp index e57864ede8..9e7393858c 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp @@ -224,6 +224,27 @@ namespace Vulkan { MemoryAllocator::~MemoryAllocator() = default; + void MemoryAllocator::SetReclaimCallback(ReclaimCallback callback) { + reclaim_callback = std::move(callback); + owner_thread = std::this_thread::get_id(); + } + + bool MemoryAllocator::ReclaimAtLeast(u64 hint_bytes) const { + if (!reclaim_callback || in_reclaim) { + return false; + } + in_reclaim = true; + const u64 freed = reclaim_callback(hint_bytes); + in_reclaim = false; + return freed > 0; + } + + void MemoryAllocator::AssertOwnerThread() const { + DEBUG_ASSERT_MSG(owner_thread == std::thread::id{} || + owner_thread == std::this_thread::get_id(), + "VmaAllocator is externally synchronized but was used off-thread"); + } + vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci) const { const VmaAllocationCreateInfo alloc_ci = { @@ -240,7 +261,26 @@ namespace Vulkan { VkImage handle{}; VmaAllocation allocation{}; VmaAllocationInfo alloc_info{}; - vk::Check(vmaCreateImage(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info)); + AssertOwnerThread(); + + VkResult res = vmaCreateImage(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info); + + if (res != VK_SUCCESS && ReclaimAtLeast(IMAGE_RECLAIM_HINT)) { + res = vmaCreateImage(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info); + } + + if (res != VK_SUCCESS) { + auto relaxed_ci = alloc_ci; + relaxed_ci.flags &= ~VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT; + res = vmaCreateImage(allocator, &ci, &relaxed_ci, &handle, &allocation, &alloc_info); + + if (res != VK_SUCCESS) { + relaxed_ci.preferredFlags &= ~VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + res = vmaCreateImage(allocator, &ci, &relaxed_ci, &handle, &allocation, &alloc_info); + } + } + + vk::Check(res); // Log GPU memory allocation for images if (GPU::Logging::IsActive() && @@ -277,7 +317,28 @@ namespace Vulkan { VmaAllocation allocation{}; VkMemoryPropertyFlags property_flags{}; - vk::Check(vmaCreateBuffer(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info)); + AssertOwnerThread(); + + VkResult res = vmaCreateBuffer(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info); + + if (res != VK_SUCCESS && ReclaimAtLeast(ci.size)) { + res = vmaCreateBuffer(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info); + } + + if (res != VK_SUCCESS) { + auto relaxed_ci = alloc_ci; + relaxed_ci.flags &= ~VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT; + res = vmaCreateBuffer(allocator, &ci, &relaxed_ci, &handle, &allocation, &alloc_info); + + if (res != VK_SUCCESS && + (relaxed_ci.preferredFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)) { + relaxed_ci.preferredFlags &= ~VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + res = vmaCreateBuffer(allocator, &ci, &relaxed_ci, &handle, &allocation, + &alloc_info); + } + } + + vk::Check(res); vmaGetAllocationMemoryProperties(allocator, allocation, &property_flags); // Log GPU memory allocation for buffers diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.h b/src/video_core/vulkan_common/vulkan_memory_allocator.h index 581f2e66d2..10b18df43c 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.h +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.h @@ -6,8 +6,10 @@ #pragma once +#include #include #include +#include #include #include "common/common_types.h" @@ -120,7 +122,17 @@ namespace Vulkan { /// Commits memory required by the buffer and binds it (for buffers created outside VMA). MemoryCommit Commit(const vk::Buffer &buffer, MemoryUsage usage); + using ReclaimCallback = std::function; + + void SetReclaimCallback(ReclaimCallback callback); + private: + bool ReclaimAtLeast(u64 hint_bytes) const; + + void AssertOwnerThread() const; + + static constexpr u64 IMAGE_RECLAIM_HINT = 64ULL * 1024 * 1024; + static bool IsAutoUsage(VmaMemoryUsage u) noexcept { switch (u) { case VMA_MEMORY_USAGE_AUTO: @@ -137,6 +149,9 @@ namespace Vulkan { const VkPhysicalDeviceMemoryProperties properties; ///< Physical device memory properties. VkDeviceSize buffer_image_granularity; ///< Adjacent buffer/image granularity u32 valid_memory_types{~0u}; + ReclaimCallback reclaim_callback; + mutable bool in_reclaim{false}; + std::thread::id owner_thread; }; } // namespace Vulkan