From 2a15ff7b69d302017173f1d160341424ff892025 Mon Sep 17 00:00:00 2001 From: CamilleLaVey Date: Fri, 7 Aug 2026 00:38:36 -0400 Subject: [PATCH] [memory, vulkan] Initial implementation for Unified Memory --- .../features/settings/model/BooleanSetting.kt | 1 + .../settings/model/view/SettingsItem.kt | 7 + .../settings/ui/SettingsFragmentPresenter.kt | 1 + .../app/src/main/res/values/strings.xml | 2 + src/common/heap_tracker.cpp | 48 +- src/common/heap_tracker.h | 6 + src/common/host_memory.cpp | 409 +++++++++++++++++- src/common/host_memory.h | 21 +- src/common/memory_detect.cpp | 55 +++ src/common/memory_detect.h | 4 + src/common/settings.h | 3 + src/core/core.cpp | 6 +- src/core/device_memory.cpp | 11 +- src/core/device_memory_manager.h | 35 ++ src/core/device_memory_manager.inc | 5 + src/qt_common/config/shared_translation.cpp | 2 + src/video_core/buffer_cache/buffer_cache.h | 246 +++++++++-- .../buffer_cache/buffer_cache_base.h | 17 +- .../renderer_opengl/gl_buffer_cache.h | 1 + .../renderer_vulkan/vk_buffer_cache.cpp | 87 ++++ .../renderer_vulkan/vk_buffer_cache.h | 29 ++ .../renderer_vulkan/vk_rasterizer.cpp | 7 + .../vulkan_common/vulkan_device.cpp | 50 +++ src/video_core/vulkan_common/vulkan_device.h | 37 ++ .../vulkan_common/vulkan_memory_allocator.cpp | 310 +++++++++++++ .../vulkan_common/vulkan_memory_allocator.h | 62 +++ .../vulkan_common/vulkan_wrapper.cpp | 4 + src/video_core/vulkan_common/vulkan_wrapper.h | 32 ++ 28 files changed, 1439 insertions(+), 59 deletions(-) diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/BooleanSetting.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/BooleanSetting.kt index 7f6e793199..ff19577766 100644 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/BooleanSetting.kt +++ b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/BooleanSetting.kt @@ -27,6 +27,7 @@ enum class BooleanSetting(override val key: String) : AbstractBooleanSetting { RENDERER_ASYNCHRONOUS_GPU_EMULATION("use_asynchronous_gpu_emulation"), RENDERER_ASYNC_PRESENTATION("async_presentation"), RENDERER_ASYNCHRONOUS_SHADERS("use_asynchronous_shaders"), + RENDERER_UNIFIED_MEMORY("use_unified_memory"), RENDERER_REACTIVE_FLUSHING("use_reactive_flushing"), ENABLE_BUFFER_HISTORY("enable_buffer_history"), USE_OPTIMIZED_VERTEX_BUFFERS("use_optimized_vertex_buffers"), diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/view/SettingsItem.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/view/SettingsItem.kt index 63ac7b0630..5e103e1977 100644 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/view/SettingsItem.kt +++ b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/view/SettingsItem.kt @@ -685,6 +685,13 @@ abstract class SettingsItem( descriptionId = R.string.renderer_asynchronous_shaders_description ) ) + put( + SwitchSetting( + BooleanSetting.RENDERER_UNIFIED_MEMORY, + titleId = R.string.renderer_unified_memory, + descriptionId = R.string.renderer_unified_memory_description + ) + ) put( SingleChoiceSetting( IntSetting.FAST_GPU_TIME, diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/SettingsFragmentPresenter.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/SettingsFragmentPresenter.kt index 594099f159..93a02b727d 100644 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/SettingsFragmentPresenter.kt +++ b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/SettingsFragmentPresenter.kt @@ -304,6 +304,7 @@ class SettingsFragmentPresenter( add(BooleanSetting.EMULATE_BGR565.key) add(BooleanSetting.RESCALE_HACK.key) add(BooleanSetting.RENDERER_ASYNCHRONOUS_SHADERS.key) + add(BooleanSetting.RENDERER_UNIFIED_MEMORY.key) add(IntSetting.ANDROID_PIPELINE_WORKERS.key) add(BooleanSetting.RENDERER_ASYNCHRONOUS_GPU_EMULATION.key) add(BooleanSetting.RENDERER_ASYNC_PRESENTATION.key) diff --git a/src/android/app/src/main/res/values/strings.xml b/src/android/app/src/main/res/values/strings.xml index bb1d85db82..df9091a3b0 100644 --- a/src/android/app/src/main/res/values/strings.xml +++ b/src/android/app/src/main/res/values/strings.xml @@ -524,6 +524,8 @@ Enables a legacy handling for the rescale configuration pass for games by using a quick rescale path Use asynchronous shaders Compiles shaders asynchronously. This may reduce stutters but may also introduce glitches. + Unified memory access + Allows GPU write buffer readbacks directly into guest memory, skipping the CPU staging copy. GPU Unswizzle Settings Configure GPU-based texture unswizzling parameters or disable it entirely. Adjust these settings to balance performance and texture loading quality. Enable GPU Unswizzle diff --git a/src/common/heap_tracker.cpp b/src/common/heap_tracker.cpp index b72abb16d8..0ca8683a34 100644 --- a/src/common/heap_tracker.cpp +++ b/src/common/heap_tracker.cpp @@ -4,34 +4,43 @@ // SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later -#include +#include #include "common/heap_tracker.h" #include "common/logging.h" #include "common/assert.h" +#include "common/memory_detect.h" namespace Common { namespace { -s64 GetMaxPermissibleResidentMapCount() { +s64 GetMaxPermissibleResidentMapCount(const Common::HostMemory& buffer) { // Default value. - s64 value = 65530; + constexpr s64 DefaultMaxMapCount = 65530; // Try to read how many mappings we can make. - std::ifstream s("/proc/sys/vm/max_map_count"); - s >> value; + const u64 reported = Common::GetMaxMapCount(); + const s64 value = reported != 0 ? static_cast(reported) : DefaultMaxMapCount; // Print, for debug. LOG_INFO(HW_Memory, "Current maximum map count: {}", value); // Allow 20000 maps for other code and to account for split inaccuracy. - return std::max(value - 20000, 0); + constexpr s64 ForeignMapReservation = 20000; + + const size_t hardware_buffer_windows = buffer.BackingHardwareBuffers().size(); + const s64 unmergeable_window_boundaries = + hardware_buffer_windows != 0 ? static_cast(hardware_buffer_windows) + 1 : 0; + + return std::max(value - ForeignMapReservation - unmergeable_window_boundaries, 0); } } // namespace HeapTracker::HeapTracker(Common::HostMemory& buffer) - : m_buffer(buffer), m_max_resident_map_count(GetMaxPermissibleResidentMapCount()) {} + : m_buffer(buffer), + m_has_hardware_buffer_backing(!buffer.BackingHardwareBuffers().empty()), + m_max_resident_map_count(GetMaxPermissibleResidentMapCount(buffer)) {} HeapTracker::~HeapTracker() = default; void HeapTracker::Map(size_t virtual_offset, size_t host_offset, size_t length, @@ -85,7 +94,8 @@ void HeapTracker::Unmap(size_t virtual_offset, size_t size, bool is_separate_hea // If resident, erase from resident map. if (item->is_resident) { - ASSERT(--m_resident_map_count >= 0); + m_resident_map_count -= this->HostMapCount(item->paddr, item->size); + ASSERT(m_resident_map_count >= 0); m_resident_mappings.erase(m_resident_mappings.iterator_to(*item)); } @@ -191,7 +201,7 @@ bool HeapTracker::DeferredMapSeparateHeap(size_t virtual_offset) { // This map is now resident. it->is_resident = true; - m_resident_map_count++; + m_resident_map_count += this->HostMapCount(it->paddr, it->size); m_resident_mappings.insert(*it); } @@ -213,17 +223,17 @@ void HeapTracker::RebuildSeparateHeapAddressSpace() { // Despite being worse in theory, this has proven to be better in practice than more // regularly dumping a smaller amount, because it significantly reduces average case // lock contention. - std::size_t const desired_count = (std::min)(m_resident_map_count, m_max_resident_map_count) / 2; - std::size_t const evict_count = m_resident_map_count - desired_count; + s64 const desired_count = (std::min)(m_resident_map_count, m_max_resident_map_count) / 2; auto it = m_resident_mappings.begin(); - for (size_t i = 0; i < evict_count && it != m_resident_mappings.end(); i++) { + while (m_resident_map_count > desired_count && it != m_resident_mappings.end()) { // Unmark and unmap. it->is_resident = false; m_buffer.Unmap(it->vaddr, it->size, false); // Advance. - ASSERT(--m_resident_map_count >= 0); + m_resident_map_count -= this->HostMapCount(it->paddr, it->size); + ASSERT(m_resident_map_count >= 0); it = m_resident_mappings.erase(it); } } @@ -245,6 +255,7 @@ void HeapTracker::SplitHeapMapLocked(VAddr offset) { // Cache the original values. auto* const left = std::addressof(*it); const size_t orig_size = left->size; + const s64 orig_host_map_count = this->HostMapCount(left->paddr, orig_size); // Adjust the left map. const size_t left_size = offset - left->vaddr; @@ -266,11 +277,20 @@ void HeapTracker::SplitHeapMapLocked(VAddr offset) { // If resident, also insert into resident map. if (right->is_resident) { - m_resident_map_count++; + m_resident_map_count += this->HostMapCount(left->paddr, left->size) + + this->HostMapCount(right->paddr, right->size) - + orig_host_map_count; m_resident_mappings.insert(*right); } } +s64 HeapTracker::HostMapCount(PAddr paddr, size_t size) const { + if (!m_has_hardware_buffer_backing) { + return size != 0 ? 1 : 0; + } + return static_cast(m_buffer.BackingMapCount(paddr, size)); +} + HeapTracker::AddrTree::iterator HeapTracker::GetNearestHeapMapLocked(VAddr offset) { const SeparateHeapMap key{ .vaddr = offset, diff --git a/src/common/heap_tracker.h b/src/common/heap_tracker.h index ee5b0bf43a..8be66a1d4d 100644 --- a/src/common/heap_tracker.h +++ b/src/common/heap_tracker.h @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + // SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later @@ -82,10 +85,13 @@ private: AddrTree::iterator GetNearestHeapMapLocked(VAddr offset); + s64 HostMapCount(PAddr paddr, size_t size) const; + void RebuildSeparateHeapAddressSpace(); private: Common::HostMemory& m_buffer; + const bool m_has_hardware_buffer_backing; const s64 m_max_resident_map_count; std::shared_mutex m_rebuild_lock{}; diff --git a/src/common/host_memory.cpp b/src/common/host_memory.cpp index 445d34c444..88b71b32d3 100644 --- a/src/common/host_memory.cpp +++ b/src/common/host_memory.cpp @@ -51,14 +51,45 @@ #endif // ^^^ POSIX ^^^ +#include #include #include +#include #include "common/alignment.h" #include "common/assert.h" #include "common/free_region_manager.h" #include "common/host_memory.h" #include "common/logging.h" +#include "common/memory_detect.h" +#include "common/settings.h" + +#ifdef __ANDROID__ +#include +#include + +namespace { + +struct NativeHandle { + int version; + int numFds; + int numInts; + int data[1]; +}; + +using PFN_AHardwareBuffer_getNativeHandle = const NativeHandle* (*)(const AHardwareBuffer*); + +PFN_AHardwareBuffer_getNativeHandle ResolveGetNativeHandle() { + void* const lib = dlopen("libnativewindow.so", RTLD_NOW); + if (lib == nullptr) { + return nullptr; + } + return reinterpret_cast( + dlsym(lib, "AHardwareBuffer_getNativeHandle")); +} + +} // namespace +#endif #if defined(__ANDROID__) && __ANDROID_API__ < 30 #include @@ -75,6 +106,12 @@ namespace Common { [[maybe_unused]] constexpr size_t PageAlignment = 0x1000; [[maybe_unused]] constexpr size_t HugePageSize = 0x200000; +static std::atomic committed_backing_size{}; + +u64 GetCommittedBackingSize() noexcept { + return committed_backing_size.load(std::memory_order_relaxed); +} + #ifdef _WIN32 // Manually imported for MinGW compatibility @@ -123,7 +160,7 @@ static void GetFuncAddress(Common::DynamicLibrary& dll, const char* name, T& pfn class HostMemory::Impl { public: - explicit Impl(size_t backing_size_, size_t virtual_size_) + explicit Impl(size_t backing_size_, size_t virtual_size_, size_t) : backing_size{backing_size_} , virtual_size{virtual_size_} , process{GetCurrentProcess()} @@ -229,6 +266,10 @@ public: UNREACHABLE(); } + bool IsBackingShared() const noexcept { + return true; + } + const size_t backing_size; ///< Size of the backing memory in bytes const size_t virtual_size; ///< Size of the virtual address placeholder in bytes @@ -501,9 +542,10 @@ static int shm_open_anon(int flags, mode_t mode) { class HostMemory::Impl { public: - explicit Impl(size_t backing_size_, size_t virtual_size_) + explicit Impl(size_t backing_size_, size_t virtual_size_, size_t preferred_offset_) : backing_size{backing_size_} , virtual_size{virtual_size_} + , preferred_offset{preferred_offset_} {} bool Init() { @@ -543,10 +585,15 @@ public: LOG_WARNING(Common_Memory, "Using private mappings instead of shared ones"); backing_base = static_cast(mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0)); if (fd > 0) { - fd = -1; close(fd); } + fd = -1; } else { +#ifdef __ANDROID__ + if (InitAhbBacking()) { + return InitVirtual(); + } +#endif backing_base = static_cast(mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0)); } if (backing_base == MAP_FAILED) { @@ -554,7 +601,10 @@ public: return false; } - // Virtual memory initialization + return InitVirtual(); + } + + bool InitVirtual() { virtual_base = virtual_map_base = static_cast(ChooseVirtualBase(virtual_size)); if (virtual_base == MAP_FAILED) { LOG_CRITICAL(HW_Memory, "mmap failed: {}", strerror(errno)); @@ -567,6 +617,277 @@ public: return true; } +#ifdef __ANDROID__ + static AHardwareBuffer_Desc MakeBlobDesc(size_t len) { + return AHardwareBuffer_Desc{ + .width = static_cast(len), + .height = 1, + .layers = 1, + .format = AHARDWAREBUFFER_FORMAT_BLOB, + .usage = AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN | + AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN | + AHARDWAREBUFFER_USAGE_GPU_DATA_BUFFER, + .stride = 0, + .rfu0 = 0, + .rfu1 = 0, + }; + } + + static bool ProbeAhbBacking(PFN_AHardwareBuffer_getNativeHandle get_native_handle) { + const AHardwareBuffer_Desc desc = MakeBlobDesc(PageAlignment * 2); + AHardwareBuffer* buffer{}; + if (AHardwareBuffer_allocate(&desc, &buffer) != 0 || buffer == nullptr) { + LOG_WARNING(HW_Memory, "Hardware buffer probe allocation failed"); + return false; + } + const NativeHandle* const handle = get_native_handle(buffer); + if (handle == nullptr || handle->numFds < 1) { + LOG_WARNING(HW_Memory, "Hardware buffer has no mappable file descriptor"); + AHardwareBuffer_release(buffer); + return false; + } + const int probe_fd = handle->data[0]; + bool ok = true; + const auto try_map = [&](int prot, off_t offset, const char* what) { + if (!ok) { + return; + } + void* const ptr = mmap(nullptr, PageAlignment, prot, MAP_SHARED, probe_fd, offset); + if (ptr == MAP_FAILED) { + LOG_WARNING(HW_Memory, "Hardware buffer backing rejects {}: {}", what, + strerror(errno)); + ok = false; + return; + } + munmap(ptr, PageAlignment); + }; + try_map(PROT_READ | PROT_WRITE, 0, "shared mappings"); + try_map(PROT_READ | PROT_WRITE, static_cast(PageAlignment), "mappings at an offset"); +#ifdef ARCHITECTURE_arm64 + try_map(PROT_READ | PROT_EXEC, 0, "executable mappings"); +#endif + AHardwareBuffer_release(buffer); + return ok; + } + + size_t ComputeAhbWindowCount(size_t window_size) const { + const u64 total_physical = Common::GetMemInfo().TotalPhysicalMemory; + if (total_physical == 0) { + LOG_WARNING(HW_Memory, "Host memory size is unknown, not committing hardware buffers"); + return 0; + } + constexpr u64 MinimumTotalPhysical = 5632ULL << 20; + if (total_physical < MinimumTotalPhysical) { + LOG_INFO(HW_Memory, + "Skipping hardware buffer backing, {} MiB of RAM is below the {} MiB minimum", + total_physical >> 20, MinimumTotalPhysical >> 20); + return 0; + } + constexpr u64 MaxWindows = 2; + u64 windows = MaxWindows; + windows = (std::min)(windows, (total_physical / 6) / window_size); + const u64 available = Common::GetAvailablePhysicalMemory(); + if (available != 0) { + constexpr u64 Headroom = 2ULL << 30; + const u64 spare = available > Headroom ? available - Headroom : 0; + windows = (std::min)(windows, spare / window_size); + } + windows = (std::min)(windows, static_cast(backing_size) / window_size); + if (windows == 0) { + LOG_INFO(HW_Memory, + "Skipping hardware buffer backing, no {} MiB window fits on a {} MiB system " + "with {} MiB available", + window_size >> 20, total_physical >> 20, available >> 20); + return 0; + } + return static_cast(windows); + } + + bool InitAhbBacking() { + if (!Settings::values.use_unified_memory.GetValue()) { + return false; + } + static const PFN_AHardwareBuffer_getNativeHandle get_native_handle = + ResolveGetNativeHandle(); + if (get_native_handle == nullptr) { + LOG_WARNING(HW_Memory, "AHardwareBuffer_getNativeHandle is not available"); + return false; + } + constexpr size_t WindowCandidates[] = { + 512ULL << 20, + 256ULL << 20, + 128ULL << 20, + 64ULL << 20, + }; + static_assert(WindowCandidates[0] <= 0xFFFFFFFFull, + "AHARDWAREBUFFER_FORMAT_BLOB encodes its size in a u32 width"); + size_t window_size = 0; + for (const size_t candidate : WindowCandidates) { + const AHardwareBuffer_Desc candidate_desc = MakeBlobDesc(candidate); + if (AHardwareBuffer_isSupported(&candidate_desc) != 0) { + window_size = candidate; + break; + } + LOG_INFO(HW_Memory, "Allocator rejects {} MiB hardware buffer windows", candidate >> 20); + } + if (window_size == 0) { + LOG_WARNING(HW_Memory, "No hardware buffer window size is supported"); + return false; + } + const size_t num_windows = ComputeAhbWindowCount(window_size); + if (num_windows == 0) { + return false; + } + if (!ProbeAhbBacking(get_native_handle)) { + return false; + } + const size_t aligned_backing = Common::AlignDown(backing_size, window_size); + const size_t region_size = num_windows * window_size; + const size_t region_base = Common::AlignDown( + (std::min)(preferred_offset, aligned_backing - region_size), window_size); + + std::vector buffers; + std::vector buffer_fds; + const auto cleanup = [&] { + for (AHardwareBuffer* buffer : buffers) { + AHardwareBuffer_release(buffer); + } + buffers.clear(); + buffer_fds.clear(); + }; + for (size_t i = 0; i < num_windows; ++i) { + const AHardwareBuffer_Desc desc = MakeBlobDesc(window_size); + AHardwareBuffer* buffer{}; + if (AHardwareBuffer_allocate(&desc, &buffer) != 0 || buffer == nullptr) { + LOG_WARNING(HW_Memory, "Hardware buffer allocation failed for window {} of {}", i, + num_windows); + cleanup(); + return false; + } + buffers.push_back(buffer); + const NativeHandle* const handle = get_native_handle(buffer); + if (handle == nullptr || handle->numFds < 1) { + LOG_WARNING(HW_Memory, "Hardware buffer has no mappable file descriptor"); + cleanup(); + return false; + } + const int buffer_fd = handle->data[0]; + const off_t buffer_len = lseek(buffer_fd, 0, SEEK_END); + if (buffer_len < static_cast(window_size)) { + LOG_WARNING(HW_Memory, "Hardware buffer descriptor smaller than requested"); + cleanup(); + return false; + } + buffer_fds.push_back(buffer_fd); + } + u8* const base = static_cast(mmap(nullptr, backing_size, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0)); + if (base == MAP_FAILED) { + LOG_WARNING(HW_Memory, "Failed to reserve backing address space: {}", strerror(errno)); + cleanup(); + return false; + } + const auto map_over_reservation = [&](size_t offset, size_t len, int map_fd, + off_t map_offset) { + if (len == 0) { + return true; + } + if (mmap(base + offset, len, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, map_fd, + map_offset) == MAP_FAILED) { + LOG_WARNING(HW_Memory, "Backing mmap failed: {}", strerror(errno)); + munmap(base, backing_size); + cleanup(); + return false; + } + return true; + }; + if (!map_over_reservation(0, region_base, fd, 0)) { + return false; + } + for (size_t i = 0; i < num_windows; ++i) { + if (!map_over_reservation(region_base + i * window_size, window_size, buffer_fds[i], + 0)) { + return false; + } + } + const size_t tail_offset = region_base + region_size; + if (!map_over_reservation(tail_offset, backing_size - tail_offset, fd, + static_cast(tail_offset))) { + return false; + } + backing_base = base; + ahb_windows = std::move(buffers); + ahb_fds = std::move(buffer_fds); + ahb_window_size = window_size; + ahb_base = region_base; + ahb_bytes = region_size; + committed_backing_size.store(region_size, std::memory_order_relaxed); + LOG_INFO(HW_Memory, + "Guest memory {:#x}-{:#x} backed by {} hardware buffer windows, {} MiB committed", + region_base, region_base + region_size, ahb_windows.size(), region_size >> 20); + return true; + } + + void MapBackingRange(size_t virtual_offset, size_t host_offset, size_t length, int prot_flags) { + while (length > 0) { + int map_fd = fd; + off_t map_offset = static_cast(host_offset); + size_t chunk = length; + if (host_offset < ahb_base) { + chunk = (std::min)(chunk, ahb_base - host_offset); + } else if (host_offset < ahb_base + ahb_bytes) { + const size_t relative = host_offset - ahb_base; + const size_t window = relative / ahb_window_size; + const size_t local = relative % ahb_window_size; + map_fd = ahb_fds[window]; + map_offset = static_cast(local); + chunk = (std::min)(chunk, ahb_window_size - local); + } + void* const ret = mmap(virtual_base + virtual_offset, chunk, prot_flags, + MAP_SHARED | MAP_FIXED, map_fd, map_offset); + ASSERT_MSG(ret != MAP_FAILED, "mmap: {}", strerror(errno)); + virtual_offset += chunk; + host_offset += chunk; + length -= chunk; + } + } + + size_t BackingMapCount(size_t host_offset, size_t length) const noexcept { + if (length == 0) { + return 0; + } + if (ahb_bytes == 0) { + return 1; + } + size_t count = 0; + while (length > 0) { + size_t chunk = length; + if (host_offset < ahb_base) { + chunk = (std::min)(chunk, ahb_base - host_offset); + } else if (host_offset < ahb_base + ahb_bytes) { + const size_t local = (host_offset - ahb_base) % ahb_window_size; + chunk = (std::min)(chunk, ahb_window_size - local); + } + host_offset += chunk; + length -= chunk; + ++count; + } + return count; + } + + std::span AhbWindows() const noexcept { + return ahb_windows; + } + + size_t AhbWindowSize() const noexcept { + return ahb_bytes != 0 ? ahb_window_size : 0; + } + + size_t AhbBase() const noexcept { + return ahb_base; + } +#endif + ~Impl() { Release(); } @@ -587,6 +908,12 @@ public: #ifdef ARCHITECTURE_arm64 if (True(perms & MemoryPermission::Execute)) prot_flags |= PROT_EXEC; +#endif +#ifdef __ANDROID__ + if (ahb_bytes != 0) { + MapBackingRange(virtual_offset, host_offset, length, prot_flags); + return; + } #endif int flags = (fd >= 0 ? MAP_SHARED : MAP_PRIVATE) | MAP_FIXED; void* ret = mmap(virtual_base + virtual_offset, length, prot_flags, flags, fd, host_offset); @@ -632,8 +959,18 @@ public: virtual_base = nullptr; } + bool IsBackingShared() const noexcept { +#ifdef __ANDROID__ + if (ahb_bytes != 0) { + return true; + } +#endif + return fd >= 0; + } + const size_t backing_size; ///< Size of the backing memory in bytes const size_t virtual_size; ///< Size of the virtual address placeholder in bytes + const size_t preferred_offset; u8* backing_base{reinterpret_cast(MAP_FAILED)}; u8* virtual_base{reinterpret_cast(MAP_FAILED)}; @@ -656,6 +993,18 @@ private: int ret = close(fd); ASSERT_MSG(ret == 0, "close failed: {}", strerror(errno)); } + +#ifdef __ANDROID__ + for (AHardwareBuffer* buffer : ahb_windows) { + AHardwareBuffer_release(buffer); + } + ahb_windows.clear(); + ahb_fds.clear(); + if (ahb_bytes != 0) { + committed_backing_size.store(0, std::memory_order_relaxed); + ahb_bytes = 0; + } +#endif } void AdjustMap(size_t* virtual_offset, size_t* length) { @@ -681,11 +1030,19 @@ private: int fd{-1}; // memfd file descriptor, -1 is the error value of memfd_create FreeRegionManager free_manager{}; + +#ifdef __ANDROID__ + std::vector ahb_windows; + std::vector ahb_fds; + size_t ahb_window_size{}; + size_t ahb_base{}; + size_t ahb_bytes{}; +#endif }; #endif // ^^^ POSIX ^^^ -HostMemory::HostMemory(size_t backing_size_, size_t virtual_size_) +HostMemory::HostMemory(size_t backing_size_, size_t virtual_size_, size_t preferred_offset_) : backing_size(backing_size_) , virtual_size(virtual_size_) { @@ -697,7 +1054,7 @@ HostMemory::HostMemory(size_t backing_size_, size_t virtual_size_) #else // Try to allocate a fastmem arena. // The implementation will fail with std::bad_alloc on errors. - impl = std::make_unique(AlignUp(backing_size, PageAlignment), AlignUp(virtual_size, PageAlignment) + HugePageSize); + impl = std::make_unique(AlignUp(backing_size, PageAlignment), AlignUp(virtual_size, PageAlignment) + HugePageSize, preferred_offset_); if (impl->Init()) { backing_base = impl->backing_base; virtual_base = impl->virtual_base; @@ -767,6 +1124,46 @@ void HostMemory::ClearBackingRegion(size_t physical_offset, size_t length, u32 f std::memset(backing_base + physical_offset, fill_value, length); } +std::span HostMemory::BackingHardwareBuffers() const noexcept { +#ifdef __ANDROID__ + return impl ? impl->AhbWindows() : std::span{}; +#else + return {}; +#endif +} + +size_t HostMemory::BackingMapCount(size_t host_offset, size_t length) const noexcept { +#ifdef __ANDROID__ + return impl ? impl->BackingMapCount(host_offset, length) : (length != 0 ? 1 : 0); +#else + return length != 0 ? 1 : 0; +#endif +} + +size_t HostMemory::BackingHardwareBufferWindowSize() const noexcept { +#ifdef __ANDROID__ + return impl ? impl->AhbWindowSize() : 0; +#else + return 0; +#endif +} + +bool HostMemory::IsBackingShared() const noexcept { +#if defined(__OPENORBIS__) || defined(__managarm__) + return false; +#else + return impl && impl->IsBackingShared(); +#endif +} + +size_t HostMemory::BackingHardwareBufferBase() const noexcept { +#ifdef __ANDROID__ + return impl ? impl->AhbBase() : 0; +#else + return 0; +#endif +} + void HostMemory::EnableDirectMappedAddress() { #if !(defined(__OPENORBIS__) || defined(__managarm__)) if (impl) { diff --git a/src/common/host_memory.h b/src/common/host_memory.h index 24f4670732..a70ceebc20 100644 --- a/src/common/host_memory.h +++ b/src/common/host_memory.h @@ -8,12 +8,17 @@ #include #include +#include #include "common/common_funcs.h" #include "common/common_types.h" #include "common/virtual_buffer.h" +struct AHardwareBuffer; + namespace Common { +[[nodiscard]] u64 GetCommittedBackingSize() noexcept; + enum class MemoryPermission : u32 { Read = 1 << 0, Write = 1 << 1, @@ -28,7 +33,7 @@ DECLARE_ENUM_FLAG_OPERATORS(MemoryPermission) */ class HostMemory { public: - explicit HostMemory(size_t backing_size_, size_t virtual_size_); + explicit HostMemory(size_t backing_size_, size_t virtual_size_, size_t preferred_offset_ = 0); ~HostMemory(); /** @@ -62,6 +67,20 @@ public: return backing_base; } + [[nodiscard]] size_t BackingSize() const noexcept { + return backing_size; + } + + [[nodiscard]] size_t BackingMapCount(size_t host_offset, size_t length) const noexcept; + + [[nodiscard]] std::span BackingHardwareBuffers() const noexcept; + + [[nodiscard]] size_t BackingHardwareBufferWindowSize() const noexcept; + + [[nodiscard]] size_t BackingHardwareBufferBase() const noexcept; + + [[nodiscard]] bool IsBackingShared() const noexcept; + [[nodiscard]] u8* VirtualBasePointer() noexcept { return virtual_base; } diff --git a/src/common/memory_detect.cpp b/src/common/memory_detect.cpp index 86a3abcc6d..d0f9518bf5 100644 --- a/src/common/memory_detect.cpp +++ b/src/common/memory_detect.cpp @@ -17,6 +17,10 @@ #endif #endif +#include +#include +#include + #include "common/memory_detect.h" namespace Common { @@ -69,4 +73,55 @@ const MemoryInfo& GetMemInfo() { return mem_info; } +u64 GetAvailablePhysicalMemory() { +#ifdef _WIN32 + MEMORYSTATUSEX memorystatus; + memorystatus.dwLength = sizeof(memorystatus); + if (GlobalMemoryStatusEx(&memorystatus)) { + return memorystatus.ullAvailPhys; + } + return 0; +#elif defined(__linux__) + if (std::FILE* const file = std::fopen("/proc/meminfo", "re")) { + char line[256]; + u64 available = 0; + while (std::fgets(line, sizeof(line), file) != nullptr) { + if (std::strncmp(line, "MemAvailable:", 13) == 0) { + available = std::strtoull(line + 13, nullptr, 10) * 1024ULL; + break; + } + } + std::fclose(file); + if (available != 0) { + return available; + } + } + struct sysinfo info; + if (sysinfo(&info) == 0) { + const u64 unit = info.mem_unit != 0 ? info.mem_unit : 1ULL; + return (static_cast(info.freeram) + static_cast(info.bufferram)) * unit; + } + return 0; +#else + return 0; +#endif +} + +u64 GetMaxMapCount() { +#ifdef __linux__ + if (std::FILE* const file = std::fopen("/proc/sys/vm/max_map_count", "re")) { + char line[32]; + u64 count = 0; + if (std::fgets(line, sizeof(line), file) != nullptr) { + count = std::strtoull(line, nullptr, 10); + } + std::fclose(file); + return count; + } + return 0; +#else + return 0; +#endif +} + } // namespace Common diff --git a/src/common/memory_detect.h b/src/common/memory_detect.h index c8f239aed3..3bf013b79a 100644 --- a/src/common/memory_detect.h +++ b/src/common/memory_detect.h @@ -18,4 +18,8 @@ struct MemoryInfo { */ [[nodiscard]] const MemoryInfo& GetMemInfo(); +[[nodiscard]] u64 GetAvailablePhysicalMemory(); + +[[nodiscard]] u64 GetMaxMapCount(); + } // namespace Common diff --git a/src/common/settings.h b/src/common/settings.h index d25d5f690b..bc972301ed 100644 --- a/src/common/settings.h +++ b/src/common/settings.h @@ -587,6 +587,9 @@ struct Values { SwitchableSetting use_asynchronous_shaders{linkage, false, "use_asynchronous_shaders", Category::RendererHacks}; + SwitchableSetting use_unified_memory{linkage, false, "use_unified_memory", + Category::RendererHacks}; + SwitchableSetting gpu_unswizzle_texture_size{linkage, GpuUnswizzleSize::Large, "gpu_unswizzle_texture_size", diff --git a/src/core/core.cpp b/src/core/core.cpp index d110aa5f8b..94a7870de8 100644 --- a/src/core/core.cpp +++ b/src/core/core.cpp @@ -119,6 +119,7 @@ struct System::Impl { is_multicore = Settings::values.use_multi_core.GetValue(); extended_memory_layout = Settings::values.memory_layout_mode.GetValue() != Settings::MemoryLayout::Memory_4Gb; + unified_memory = Settings::values.use_unified_memory.GetValue(); core_timing.SetMulticore(is_multicore); core_timing.Initialize([&system]() { system.RegisterHostThread(); }); @@ -146,7 +147,8 @@ struct System::Impl { !device_memory.has_value() || is_multicore != Settings::values.use_multi_core.GetValue() || extended_memory_layout != (Settings::values.memory_layout_mode.GetValue() != - Settings::MemoryLayout::Memory_4Gb); + Settings::MemoryLayout::Memory_4Gb) || + unified_memory != Settings::values.use_unified_memory.GetValue(); if (!must_reinitialize) { return; @@ -157,6 +159,7 @@ struct System::Impl { is_multicore = Settings::values.use_multi_core.GetValue(); extended_memory_layout = Settings::values.memory_layout_mode.GetValue() != Settings::MemoryLayout::Memory_4Gb; + unified_memory = Settings::values.use_unified_memory.GetValue(); Initialize(system); } @@ -503,6 +506,7 @@ struct System::Impl { std::atomic_bool is_powered_on{}; bool is_multicore : 1 = false; bool extended_memory_layout : 1 = false; + bool unified_memory : 1 = false; bool exit_locked : 1 = false; bool exit_requested : 1 = false; bool nvdec_active : 1 = false; diff --git a/src/core/device_memory.cpp b/src/core/device_memory.cpp index 1aea56a991..e36ed42e31 100644 --- a/src/core/device_memory.cpp +++ b/src/core/device_memory.cpp @@ -12,9 +12,18 @@ constexpr size_t VirtualReserveSize = 1ULL << 38; constexpr size_t VirtualReserveSize = 1ULL << 39; #endif +namespace { +size_t ApplicationPoolOffset() { + using Init = Kernel::Board::Nintendo::Nx::KSystemControl::Init; + const size_t dram_size = Init::GetIntendedMemorySize(); + const size_t application_pool_size = Init::GetApplicationPoolSize(); + return dram_size > application_pool_size ? dram_size - application_pool_size : 0; +} +} + DeviceMemory::DeviceMemory() : buffer{Kernel::Board::Nintendo::Nx::KSystemControl::Init::GetIntendedMemorySize(), - VirtualReserveSize} {} + VirtualReserveSize, ApplicationPoolOffset()} {} DeviceMemory::~DeviceMemory() = default; diff --git a/src/core/device_memory_manager.h b/src/core/device_memory_manager.h index 3d97fdcc5c..3b8bf038a5 100644 --- a/src/core/device_memory_manager.h +++ b/src/core/device_memory_manager.h @@ -20,6 +20,8 @@ #include "common/scratch_buffer.h" #include "common/virtual_buffer.h" +struct AHardwareBuffer; + namespace Core { constexpr size_t DEVICE_PAGEBITS = 12ULL; @@ -95,6 +97,34 @@ public: ApplyOpOnPAddr(address, buffer, operation); } + u8* GetPhysicalBase() noexcept { + return reinterpret_cast(physical_base); + } + + const u8* GetPhysicalBase() const noexcept { + return reinterpret_cast(physical_base); + } + + size_t GetPhysicalSize() const noexcept { + return physical_size; + } + + std::span GetBackingHardwareBuffers() const noexcept { + return ahb_windows; + } + + size_t GetBackingHardwareBufferWindowSize() const noexcept { + return ahb_window_size; + } + + size_t GetBackingHardwareBufferBase() const noexcept { + return ahb_base; + } + + bool IsBackingShared() const noexcept { + return backing_is_shared; + } + PAddr GetPhysicalRawAddressFromDAddr(DAddr address) const { PAddr subbits = PAddr(address & page_mask); auto paddr = tracked_entries[(address >> page_bits)].compressed_physical_ptr; @@ -171,6 +201,11 @@ private: std::unique_ptr> impl; const uintptr_t physical_base; + const size_t physical_size; + const std::span ahb_windows; + const size_t ahb_window_size; + const size_t ahb_base; + const bool backing_is_shared; DeviceInterface* device_inter; struct TrackedEntry { diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index 9866f3a4f7..678b2e00e9 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -171,6 +171,11 @@ struct DeviceMemoryManagerAllocator { template DeviceMemoryManager::DeviceMemoryManager(const DeviceMemory& device_memory_) : physical_base{uintptr_t(device_memory_.buffer.BackingBasePointer())} + , physical_size{device_memory_.buffer.BackingSize()} + , ahb_windows{device_memory_.buffer.BackingHardwareBuffers()} + , ahb_window_size{device_memory_.buffer.BackingHardwareBufferWindowSize()} + , ahb_base{device_memory_.buffer.BackingHardwareBufferBase()} + , backing_is_shared{device_memory_.buffer.IsBackingShared()} , device_inter{nullptr} , compressed_device_addr(1ULL << ((Settings::values.memory_layout_mode.GetValue() == Settings::MemoryLayout::Memory_4Gb ? physical_min_bits : physical_max_bits) - Memory::YUZU_PAGEBITS)) , tracked_entries(device_as_size >> Memory::YUZU_PAGEBITS) diff --git a/src/qt_common/config/shared_translation.cpp b/src/qt_common/config/shared_translation.cpp index 26597d412c..5e97a9e58c 100644 --- a/src/qt_common/config/shared_translation.cpp +++ b/src/qt_common/config/shared_translation.cpp @@ -227,6 +227,8 @@ std::unique_ptr InitializeTranslations(QObject* parent) { tr("Preserves GPU-modified data by reading it back before uploading.\nSome games require this to render certain effects properly.")); INSERT(Settings, use_asynchronous_shaders, tr("Enable asynchronous shader compilation"), tr("May reduce shader stutter.")); + INSERT(Settings, use_unified_memory, tr("Enable unified memory access"), + tr("Lets the GPU write buffer readbacks directly into guest memory.")); INSERT(Settings, gpu_clock, tr("GPU Clocks"), tr("Makes the game believe GPU work finishes faster than it does, so it stops lowering " "resolution and render distance to fit the Switch's clocks.")); diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index e40aad1fb5..9f446ec32b 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -571,7 +571,11 @@ void BufferCache

::AccumulateFlushes() { template bool BufferCache

::ShouldWaitAsyncFlushes() const noexcept { - return (!async_buffers.empty() && async_buffers.front().has_value()); + if (async_buffers.empty()) { + return false; + } + return async_buffers.front().has_value() || + !pending_downloads.front().unified_copies.empty(); } template @@ -579,6 +583,7 @@ void BufferCache

::CommitAsyncFlushesHigh() { AccumulateFlushes(); if (committed_gpu_modified_ranges.empty()) { + pending_downloads.emplace_back(); async_buffers.emplace_back(std::optional{}); return; } @@ -638,27 +643,83 @@ void BufferCache

::CommitAsyncFlushesHigh() { } committed_gpu_modified_ranges.clear(); if (downloads.empty()) { + pending_downloads.emplace_back(); async_buffers.emplace_back(std::optional{}); return; } - auto download_staging = runtime.DownloadStagingBuffer(total_size_bytes, true); - boost::container::small_vector normalized_copies; - runtime.PreCopyBarrier(); + + struct QueuedUnifiedCopy { + u64 window; + BufferId buffer_id; + boost::container::small_vector copies; + }; + + AsyncDownloadBatch batch; + boost::container::small_vector, 16> staging_downloads; + boost::container::small_vector unified_copy_queue; + boost::container::small_vector window_ids; + UnifiedWindowGroups groups; + u64 staging_size_bytes = 0; for (auto& [copy, buffer_id] : downloads) { - copy.dst_offset += download_staging.offset; + Buffer& buffer = slot_buffers[buffer_id]; + const DAddr orig_device_addr = buffer.CpuAddr() + copy.src_offset; + bool unified = false; + if constexpr (USE_UNIFIED_MEMORY) { + if (runtime.HasUnifiedMemory()) { + window_ids.clear(); + groups.clear(); + unified = ResolveUnifiedWindows(orig_device_addr, copy.src_offset, copy.size, + window_ids, groups); + } + } + BufferCopy record{copy}; + record.src_offset = static_cast(orig_device_addr); + if (unified) { + async_downloads.Add(orig_device_addr, copy.size); + buffer.MarkUsage(copy.src_offset, copy.size); + for (size_t i = 0; i < window_ids.size(); ++i) { + unified_copy_queue.push_back( + QueuedUnifiedCopy{window_ids[i], buffer_id, std::move(groups[i])}); + } + batch.unified_copies.push_back(record); + continue; + } + copy.dst_offset = staging_size_bytes; + constexpr u64 align = 64ULL; + staging_size_bytes += (copy.size + align - 1) & ~(align - 1ULL); + staging_downloads.push_back({copy, buffer_id}); + } + + std::optional download_staging; + if (!staging_downloads.empty()) { + download_staging = runtime.DownloadStagingBuffer(staging_size_bytes, true); + } + runtime.PreCopyBarrier(); + for (auto& [copy, buffer_id] : staging_downloads) { + copy.dst_offset += download_staging->offset; const std::array copies{copy}; - BufferCopy second_copy{copy}; Buffer& buffer = slot_buffers[buffer_id]; - second_copy.src_offset = static_cast(buffer.CpuAddr()) + copy.src_offset; - const DAddr orig_device_addr = static_cast(second_copy.src_offset); + BufferCopy record{copy}; + record.src_offset = static_cast(buffer.CpuAddr()) + copy.src_offset; + const DAddr orig_device_addr = static_cast(record.src_offset); async_downloads.Add(orig_device_addr, copy.size); buffer.MarkUsage(copy.src_offset, copy.size); - runtime.CopyBuffer(download_staging.buffer, buffer, copies, false); - normalized_copies.push_back(second_copy); + runtime.CopyBuffer(download_staging->buffer, buffer, copies, false); + batch.staging_copies.push_back(record); + } + if constexpr (USE_UNIFIED_MEMORY) { + for (const auto& queued : unified_copy_queue) { + const std::span group_span(queued.copies.data(), + queued.copies.size()); + runtime.CopyToUnifiedMemory(queued.window, slot_buffers[queued.buffer_id], group_span); + } + if (!unified_copy_queue.empty()) { + runtime.UnifiedMemoryHostBarrier(); + } } runtime.PostCopyBarrier(); - pending_downloads.emplace_back(std::move(normalized_copies)); - async_buffers.emplace_back(download_staging); + pending_downloads.emplace_back(std::move(batch)); + async_buffers.emplace_back(std::move(download_staging)); } template @@ -673,32 +734,49 @@ void BufferCache

::PopAsyncFlushes() { template void BufferCache

::PopAsyncBuffers() { - if (async_buffers.empty()) { - return; - } - if (!async_buffers.front().has_value()) { + struct Writeback { + DAddr addr; + const u8* src; + u64 size; + }; + boost::container::small_vector writebacks; + { + std::scoped_lock lock{mutex}; + if (async_buffers.empty()) { + return; + } + auto& batch = pending_downloads.front(); + auto& async_buffer = async_buffers.front(); + if (async_buffer.has_value()) { + const u8* base = async_buffer->mapped_span.data(); + const size_t base_offset = async_buffer->offset; + for (const auto& copy : batch.staging_copies) { + const DAddr device_addr = static_cast(copy.src_offset); + const u64 dst_offset = copy.dst_offset - base_offset; + const u8* read_mapped_memory = base + dst_offset; + async_downloads.ForEachInRange( + device_addr, copy.size, [&](DAddr start, DAddr end, s32) { + writebacks.push_back( + {start, &read_mapped_memory[start - device_addr], end - start}); + }); + async_downloads.Subtract(device_addr, copy.size, [&](DAddr start, DAddr end) { + gpu_modified_ranges.Subtract(start, end - start); + }); + } + async_buffers_death_ring.emplace_back(*async_buffer); + } + for (const auto& copy : batch.unified_copies) { + const DAddr device_addr = static_cast(copy.src_offset); + async_downloads.Subtract(device_addr, copy.size, [&](DAddr start, DAddr end) { + gpu_modified_ranges.Subtract(start, end - start); + }); + } async_buffers.pop_front(); - return; + pending_downloads.pop_front(); } - auto& downloads = pending_downloads.front(); - auto& async_buffer = async_buffers.front(); - u8* base = async_buffer->mapped_span.data(); - const size_t base_offset = async_buffer->offset; - for (const auto& copy : downloads) { - const DAddr device_addr = static_cast(copy.src_offset); - const u64 dst_offset = copy.dst_offset - base_offset; - const u8* read_mapped_memory = base + dst_offset; - async_downloads.ForEachInRange(device_addr, copy.size, [&](DAddr start, DAddr end, s32) { - device_memory.WriteBlockUnsafe(start, &read_mapped_memory[start - device_addr], - end - start); - }); - async_downloads.Subtract(device_addr, copy.size, [&](DAddr start, DAddr end) { - gpu_modified_ranges.Subtract(start, end - start); - }); + for (const auto& wb : writebacks) { + device_memory.WriteBlockUnsafe(wb.addr, wb.src, wb.size); } - async_buffers_death_ring.emplace_back(*async_buffer); - async_buffers.pop_front(); - pending_downloads.pop_front(); } template @@ -1699,6 +1777,98 @@ void BufferCache

::ImmediateUploadMemory([[maybe_unused]] Buffer& buffer, } } +template +bool BufferCache

::ResolveUnifiedWindows( + [[maybe_unused]] DAddr device_addr, [[maybe_unused]] u64 buffer_offset, + [[maybe_unused]] u64 size, [[maybe_unused]] boost::container::small_vector& window_ids, + [[maybe_unused]] UnifiedWindowGroups& groups) { + if constexpr (USE_UNIFIED_MEMORY) { + const u8* const physical_base = device_memory.GetPhysicalBase(); + const u64 unified_base = runtime.UnifiedMemoryBase(); + const u64 unified_size = runtime.UnifiedMemorySize(); + const u64 window_size = runtime.UnifiedMemoryWindowSize(); + if (window_size == 0) { + return false; + } + const auto group_for = [&](u64 window) -> boost::container::small_vector& { + for (size_t i = 0; i < window_ids.size(); ++i) { + if (window_ids[i] == window) { + return groups[i]; + } + } + window_ids.push_back(window); + groups.emplace_back(); + return groups.back(); + }; + u64 downloaded = 0; + while (downloaded < size) { + const DAddr page_addr = device_addr + downloaded; + const u8* const ptr = device_memory.GetPointer(page_addr); + if (ptr == nullptr) { + return false; + } + const u64 page_offset = page_addr & Core::DEVICE_PAGEMASK; + u64 chunk = (std::min)(size - downloaded, + static_cast(Core::DEVICE_PAGESIZE) - page_offset); + const u64 phys_offset = static_cast(ptr - physical_base); + if (phys_offset < unified_base || phys_offset - unified_base + chunk > unified_size) { + return false; + } + const u64 relative = phys_offset - unified_base; + const u64 window = relative / window_size; + const u64 local_offset = relative % window_size; + chunk = (std::min)(chunk, window_size - local_offset); + auto& group = group_for(window); + if (!group.empty()) { + BufferCopy& last = group.back(); + if (last.src_offset + last.size == buffer_offset + downloaded && + last.dst_offset + last.size == local_offset) { + last.size += chunk; + downloaded += chunk; + continue; + } + } + group.push_back(BufferCopy{ + .src_offset = buffer_offset + downloaded, + .dst_offset = local_offset, + .size = chunk, + }); + downloaded += chunk; + } + return true; + } else { + return false; + } +} + +template +bool BufferCache

::TryUnifiedDownloadMemory([[maybe_unused]] Buffer& buffer, + [[maybe_unused]] std::span copies) { + if constexpr (USE_UNIFIED_MEMORY) { + boost::container::small_vector window_ids; + UnifiedWindowGroups groups; + for (const BufferCopy& copy : copies) { + if (!ResolveUnifiedWindows(buffer.CpuAddr() + copy.src_offset, copy.src_offset, + copy.size, window_ids, groups)) { + return false; + } + } + for (const BufferCopy& copy : copies) { + buffer.MarkUsage(copy.src_offset, copy.size); + } + runtime.PreCopyBarrier(); + for (size_t i = 0; i < window_ids.size(); ++i) { + const std::span group_span(groups[i].data(), groups[i].size()); + runtime.CopyToUnifiedMemory(window_ids[i], buffer, group_span); + } + runtime.UnifiedMemoryHostBarrier(); + runtime.Finish(); + return true; + } else { + return false; + } +} + template void BufferCache

::MappedUploadMemory([[maybe_unused]] Buffer& buffer, [[maybe_unused]] u64 total_size_bytes, @@ -1802,6 +1972,12 @@ void BufferCache

::DownloadBufferMemory(Buffer& buffer, DAddr device_addr, u64 } if constexpr (USE_MEMORY_MAPS) { + if constexpr (USE_UNIFIED_MEMORY) { + if (runtime.HasUnifiedMemory() && + TryUnifiedDownloadMemory(buffer, std::span(copies.data(), copies.size()))) { + return; + } + } auto download_staging = runtime.DownloadStagingBuffer(total_size_bytes); const u8* const mapped_memory = download_staging.mapped_span.data(); const std::span copies_span(copies.data(), copies.data() + copies.size()); diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 14ab3e6ebc..9d66fe9b40 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -180,6 +180,7 @@ class BufferCache : public VideoCommon::ChannelSetupCaches copies); + bool TryUnifiedDownloadMemory(Buffer& buffer, std::span copies); + + using UnifiedWindowGroups = + boost::container::small_vector, 4>; + + bool ResolveUnifiedWindows(DAddr device_addr, u64 buffer_offset, u64 size, + boost::container::small_vector& window_ids, + UnifiedWindowGroups& groups); + void DownloadBufferMemory(Buffer& buffer_id); void DownloadBufferMemory(Buffer& buffer_id, DAddr device_addr, u64 size); @@ -498,9 +508,14 @@ private: std::deque> committed_gpu_modified_ranges; // Async Buffers + struct AsyncDownloadBatch { + boost::container::small_vector staging_copies; + boost::container::small_vector unified_copies; + }; + Common::OverlapRangeSet async_downloads; std::deque> async_buffers; - std::deque> pending_downloads; + std::deque pending_downloads; std::optional current_buffer; std::deque async_buffers_death_ring; diff --git a/src/video_core/renderer_opengl/gl_buffer_cache.h b/src/video_core/renderer_opengl/gl_buffer_cache.h index a0acfc48c1..15c73ae1f4 100644 --- a/src/video_core/renderer_opengl/gl_buffer_cache.h +++ b/src/video_core/renderer_opengl/gl_buffer_cache.h @@ -261,6 +261,7 @@ struct BufferCacheParams { // TODO: Investigate why OpenGL seems to perform worse with persistently mapped buffer uploads static constexpr bool USE_MEMORY_MAPS_FOR_UPLOADS = false; + static constexpr bool USE_UNIFIED_MEMORY = false; }; using BufferCache = VideoCommon::BufferCache; diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp index a734bd049c..5bd4d1f2a0 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp @@ -365,6 +365,93 @@ BufferCacheRuntime::BufferCacheRuntime(const Device& device_, MemoryAllocator& m scheduler_, staging_pool_); } +void BufferCacheRuntime::TryEnableUnifiedMemory(void* base, size_t size, + std::span hardware_buffers, + size_t hardware_buffer_window, + size_t hardware_buffer_base) { + unified_memory = std::make_unique( + device, base, size, hardware_buffers, hardware_buffer_window, hardware_buffer_base); + if (!unified_memory->IsValid()) { + unified_memory.reset(); + } +} + +void BufferCacheRuntime::CopyToUnifiedMemory( + size_t window_index, VkBuffer src_buffer, + std::span copies) { + if (!unified_memory || src_buffer == VK_NULL_HANDLE || copies.empty() || + window_index >= unified_memory->GetWindowCount()) { + return; + } + const VkBuffer dst_buffer = unified_memory->GetWindowBuffer(window_index); + if (dst_buffer == VK_NULL_HANDLE) { + return; + } + VkDeviceSize covered_begin = std::numeric_limits::max(); + VkDeviceSize covered_end = 0; + for (const VideoCommon::BufferCopy& copy : copies) { + covered_begin = (std::min)(covered_begin, static_cast(copy.dst_offset)); + covered_end = (std::max)(covered_end, + static_cast(copy.dst_offset + copy.size)); + } + + boost::container::small_vector vk_copies(copies.size()); + std::ranges::transform(copies, vk_copies.begin(), MakeBufferCopy); + + const bool foreign = unified_memory->NeedsForeignOwnershipTransfer(); + const u32 queue_family = device.GetGraphicsFamily(); + + scheduler.RequestOutsideRenderPassOperationContext(); + scheduler.Record([src_buffer, dst_buffer, vk_copies, foreign, queue_family, covered_begin, + covered_end](vk::CommandBuffer cmdbuf) { + if (foreign) { + const VkBufferMemoryBarrier acquire{ + .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = 0, + .dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT, + .dstQueueFamilyIndex = queue_family, + .buffer = dst_buffer, + .offset = covered_begin, + .size = covered_end - covered_begin, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, acquire); + } + cmdbuf.CopyBuffer(src_buffer, dst_buffer, VideoCommon::FixSmallVectorADL(vk_copies)); + if (foreign) { + const VkBufferMemoryBarrier release{ + .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .dstAccessMask = 0, + .srcQueueFamilyIndex = queue_family, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT, + .buffer = dst_buffer, + .offset = covered_begin, + .size = covered_end - covered_begin, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0, release); + } + }); +} + +void BufferCacheRuntime::UnifiedMemoryHostBarrier() { + static constexpr VkMemoryBarrier HOST_BARRIER{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_HOST_READ_BIT, + }; + scheduler.RequestOutsideRenderPassOperationContext(); + scheduler.Record([](vk::CommandBuffer cmdbuf) { + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, 0, + HOST_BARRIER); + }); +} + StagingBufferRef BufferCacheRuntime::UploadStagingBuffer(size_t size) { return staging_pool.Request(size, MemoryUsage::Upload); } diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index 9b3dd50eaa..c7870505eb 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -7,6 +7,8 @@ #pragma once #include +#include +#include #include "video_core/buffer_cache/buffer_cache_base.h" #include "video_core/buffer_cache/memory_tracker_base.h" @@ -97,6 +99,31 @@ public: void TickFrame(Common::SlotVector& slot_buffers) noexcept; + void TryEnableUnifiedMemory(void* base, size_t size, + std::span hardware_buffers, + size_t hardware_buffer_window, size_t hardware_buffer_base); + + [[nodiscard]] bool HasUnifiedMemory() const noexcept { + return unified_memory != nullptr && unified_memory->IsValid(); + } + + [[nodiscard]] u64 UnifiedMemorySize() const noexcept { + return unified_memory ? unified_memory->GetSize() : 0; + } + + [[nodiscard]] u64 UnifiedMemoryBase() const noexcept { + return unified_memory ? unified_memory->GetBaseOffset() : 0; + } + + [[nodiscard]] u64 UnifiedMemoryWindowSize() const noexcept { + return unified_memory ? unified_memory->GetWindowSize() : 0; + } + + void CopyToUnifiedMemory(size_t window_index, VkBuffer src_buffer, + std::span copies); + + void UnifiedMemoryHostBarrier(); + u64 CurrentTick(); u64 KnownGpuTick(); @@ -204,6 +231,7 @@ private: std::shared_ptr quad_strip_index_buffer; vk::Buffer null_buffer; + std::unique_ptr unified_memory; std::unique_ptr uint8_pass; QuadIndexedPass quad_index_pass; @@ -226,6 +254,7 @@ struct BufferCacheParams { static constexpr bool USE_MEMORY_MAPS = true; static constexpr bool SEPARATE_IMAGE_BUFFER_BINDINGS = false; static constexpr bool USE_MEMORY_MAPS_FOR_UPLOADS = true; + static constexpr bool USE_UNIFIED_MEMORY = true; }; using BufferCache = VideoCommon::BufferCache; diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index a10d2e01e6..1bfdf66e0b 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -225,6 +225,13 @@ RasterizerVulkan::RasterizerVulkan(Core::Frontend::EmuWindow& emu_window_, Tegra fence_manager(*this, gpu, texture_cache, buffer_cache, query_cache, device, scheduler), wfi_event(device.GetLogical().CreateEvent()) { scheduler.SetQueryCache(query_cache); + if (Settings::values.use_unified_memory.GetValue() && device_memory.IsBackingShared()) { + buffer_cache_runtime.TryEnableUnifiedMemory( + device_memory.GetPhysicalBase(), device_memory.GetPhysicalSize(), + device_memory.GetBackingHardwareBuffers(), + device_memory.GetBackingHardwareBufferWindowSize(), + device_memory.GetBackingHardwareBufferBase()); + } } RasterizerVulkan::~RasterizerVulkan() { diff --git a/src/video_core/vulkan_common/vulkan_device.cpp b/src/video_core/vulkan_common/vulkan_device.cpp index 8c41a7eb67..1250934c02 100644 --- a/src/video_core/vulkan_common/vulkan_device.cpp +++ b/src/video_core/vulkan_common/vulkan_device.cpp @@ -16,6 +16,7 @@ #include #include "common/assert.h" +#include "common/host_memory.h" #include "common/literals.h" #include #include "common/settings.h" @@ -974,6 +975,7 @@ bool Device::GetSuitability(bool requires_swapchain) { FOR_EACH_VK_FEATURE_EXT(FEATURE_EXTENSION); FOR_EACH_VK_EXTENSION(EXTENSION); + FOR_EACH_VK_PLATFORM_EXTENSION(EXTENSION); if (supported_extensions.contains(VK_KHR_ROBUSTNESS_2_EXTENSION_NAME)) { loaded_extensions.erase(VK_EXT_ROBUSTNESS_2_EXTENSION_NAME); @@ -986,6 +988,17 @@ bool Device::GetSuitability(bool requires_swapchain) { extensions.robustness_2 = false; } +#ifdef __ANDROID__ + if (extensions.external_memory_ahb && !extensions.queue_family_foreign) { + LOG_INFO(Render_Vulkan, + "Not loading {} because its dependency {} is unavailable", + VK_ANDROID_EXTERNAL_MEMORY_ANDROID_HARDWARE_BUFFER_EXTENSION_NAME, + VK_EXT_QUEUE_FAMILY_FOREIGN_EXTENSION_NAME); + loaded_extensions.erase(VK_ANDROID_EXTERNAL_MEMORY_ANDROID_HARDWARE_BUFFER_EXTENSION_NAME); + extensions.external_memory_ahb = false; + } +#endif + #undef FEATURE_EXTENSION #undef EXTENSION @@ -1142,6 +1155,21 @@ bool Device::GetSuitability(bool requires_swapchain) { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_5_PROPERTIES_KHR; SetNext(next, properties.maintenance5); } + if (extensions.external_memory_host) { + properties.external_memory_host.sType = + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT; + SetNext(next, properties.external_memory_host); + } + if (extensions.maintenance3 || instance_version >= VK_API_VERSION_1_1) { + properties.maintenance3.sType = + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_3_PROPERTIES; + SetNext(next, properties.maintenance3); + } + if (extensions.maintenance4 || features.maintenance4.maintenance4) { + properties.maintenance4.sType = + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_4_PROPERTIES; + SetNext(next, properties.maintenance4); + } // Perform the property fetch. physical.GetProperties2(properties2); @@ -1516,12 +1544,27 @@ void Device::CollectPhysicalMemoryInfo() { device_access_memory = 0; u64 device_initial_usage = 0; u64 local_memory = 0; + const auto heap_has_usable_type = [&mem_properties](size_t heap) { + for (u32 index = 0; index < mem_properties.memoryTypeCount; ++index) { + if (mem_properties.memoryTypes[index].heapIndex != heap) { + continue; + } + if ((mem_properties.memoryTypes[index].propertyFlags & + VK_MEMORY_PROPERTY_PROTECTED_BIT) == 0) { + return true; + } + } + return false; + }; for (size_t element = 0; element < num_properties; ++element) { const bool is_heap_local = (mem_properties.memoryHeaps[element].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0; if (!is_integrated && !is_heap_local) { continue; } + if (!heap_has_usable_type(element)) { + continue; + } valid_heap_memory.push_back(element); if (is_heap_local) { local_memory += mem_properties.memoryHeaps[element].size; @@ -1533,6 +1576,13 @@ void Device::CollectPhysicalMemoryInfo() { } device_access_memory += mem_properties.memoryHeaps[element].size; } + const u64 committed_backing = Common::GetCommittedBackingSize(); + if (committed_backing != 0) { + LOG_INFO(Render_Vulkan, "Discounting {} MiB of guest memory committed by the host", + committed_backing >> 20); + local_memory -= std::min(local_memory, committed_backing); + device_access_memory -= std::min(device_access_memory, committed_backing); + } if (is_integrated) { const s64 available_memory = static_cast(device_access_memory - device_initial_usage); const u64 memory_size = Settings::values.vram_usage_mode.GetValue() == Settings::VramUsageMode::Aggressive ? 6_GiB : 4_GiB; diff --git a/src/video_core/vulkan_common/vulkan_device.h b/src/video_core/vulkan_common/vulkan_device.h index 2708281297..fc93dada44 100644 --- a/src/video_core/vulkan_common/vulkan_device.h +++ b/src/video_core/vulkan_common/vulkan_device.h @@ -83,6 +83,7 @@ VK_DEFINE_HANDLE(VmaAllocator) EXTENSION(EXT, CONDITIONAL_RENDERING, conditional_rendering) \ EXTENSION(EXT, CONSERVATIVE_RASTERIZATION, conservative_rasterization) \ EXTENSION(EXT, DEPTH_RANGE_UNRESTRICTED, depth_range_unrestricted) \ + EXTENSION(EXT, EXTERNAL_MEMORY_HOST, external_memory_host) \ EXTENSION(EXT, MEMORY_BUDGET, memory_budget) \ EXTENSION(EXT, ROBUSTNESS_2, robustness_2) \ EXTENSION(EXT, SAMPLER_FILTER_MINMAX, sampler_filter_minmax) \ @@ -112,6 +113,14 @@ VK_DEFINE_HANDLE(VmaAllocator) EXTENSION(IMG, FILTER_CUBIC, filter_cubic_img) \ EXTENSION(QCOM, FILTER_CUBIC_WEIGHTS, filter_cubic_weights) +#ifdef __ANDROID__ +#define FOR_EACH_VK_PLATFORM_EXTENSION(EXTENSION) \ + EXTENSION(EXT, QUEUE_FAMILY_FOREIGN, queue_family_foreign) \ + EXTENSION(ANDROID, EXTERNAL_MEMORY_ANDROID_HARDWARE_BUFFER, external_memory_ahb) +#else +#define FOR_EACH_VK_PLATFORM_EXTENSION(EXTENSION) +#endif + // Define extensions which must be supported. #define FOR_EACH_VK_MANDATORY_EXTENSION(EXTENSION_NAME) \ EXTENSION_NAME(VK_EXT_VERTEX_ATTRIBUTE_DIVISOR_EXTENSION_NAME) \ @@ -838,6 +847,30 @@ FN_MAX_LIMIT_LIST return extensions.conditional_rendering; } + bool IsExtExternalMemoryHostSupported() const { + return extensions.external_memory_host; + } + + bool IsExtExternalMemoryAhbSupported() const { +#ifdef __ANDROID__ + return extensions.external_memory_ahb && extensions.queue_family_foreign; +#else + return false; +#endif + } + + u64 GetMinImportedHostPointerAlignment() const { + return properties.external_memory_host.minImportedHostPointerAlignment; + } + + u64 GetMaxBufferSize() const { + return properties.maintenance4.maxBufferSize; + } + + u64 GetMaxMemoryAllocationSize() const { + return properties.maintenance3.maxMemoryAllocationSize; + } + bool IsExtAstcDecodeModeSupported() const { return extensions.astc_decode_mode; } @@ -1110,6 +1143,7 @@ private: FOR_EACH_VK_FEATURE_1_4(FEATURE); FOR_EACH_VK_FEATURE_EXT(FEATURE); FOR_EACH_VK_EXTENSION(EXTENSION); + FOR_EACH_VK_PLATFORM_EXTENSION(EXTENSION); #undef EXTENSION #undef FEATURE @@ -1141,7 +1175,10 @@ private: VkPhysicalDeviceDescriptorBufferPropertiesEXT descriptor_buffer{}; VkPhysicalDeviceSubgroupSizeControlProperties subgroup_size_control{}; VkPhysicalDeviceTransformFeedbackPropertiesEXT transform_feedback{}; + VkPhysicalDeviceMaintenance3Properties maintenance3{}; + VkPhysicalDeviceMaintenance4Properties maintenance4{}; VkPhysicalDeviceMaintenance5PropertiesKHR maintenance5{}; + VkPhysicalDeviceExternalMemoryHostPropertiesEXT external_memory_host{}; VkPhysicalDeviceProperties properties{}; }; diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp index e57864ede8..a8d04a6ed9 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp @@ -25,9 +25,34 @@ #include "video_core/gpu_logging/gpu_logging.h" #include "common/settings.h" +#ifdef __ANDROID__ +#include +#endif + namespace Vulkan { namespace { + [[nodiscard]] std::optional FindImportMemoryType( + const VkPhysicalDeviceMemoryProperties &props, u32 type_mask) { + const auto find = [&](VkMemoryPropertyFlags wanted) -> std::optional { + for (u32 i = 0; i < props.memoryTypeCount; ++i) { + if (((type_mask >> i) & 1u) != 0 && + (props.memoryTypes[i].propertyFlags & wanted) == wanted) { + return i; + } + } + return std::nullopt; + }; + auto type_index = find(VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | + VK_MEMORY_PROPERTY_HOST_CACHED_BIT); + if (!type_index) { + type_index = find(VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); + } + return type_index; + } + // Helpers translating MemoryUsage to flags/usage [[maybe_unused]] VkMemoryPropertyFlags MemoryUsagePropertyFlags(MemoryUsage usage) { @@ -200,6 +225,291 @@ namespace Vulkan { size = 0; } + HostMemoryImport::HostMemoryImport(const Device &device_, void *base, size_t size, + std::span hardware_buffers, + size_t hardware_buffer_window, size_t hardware_buffer_base) + : device{device_} { + if (ImportHardwareBuffers(hardware_buffers, hardware_buffer_window, hardware_buffer_base, + size)) { + return; + } + if (device.IsTiler()) { + LOG_INFO(Render_Vulkan, + "Unified memory disabled, hardware buffer import is the only path supported " + "by tiler drivers"); + return; + } + if (ImportHostPointer(base, size)) { + return; + } + LOG_INFO(Render_Vulkan, "Unified memory disabled, no host memory import path"); + } + + bool HostMemoryImport::ImportHostPointer(void *base, size_t size) { + if (!device.IsExtExternalMemoryHostSupported()) { + LOG_INFO(Render_Vulkan, + "Unified memory disabled, VK_EXT_external_memory_host is not supported"); + return false; + } + const u64 alignment = device.GetMinImportedHostPointerAlignment(); + if (alignment == 0 || !Common::IsAligned(reinterpret_cast(base), alignment) || + !Common::IsAligned(size, alignment)) { + LOG_INFO(Render_Vulkan, + "Unified memory disabled, host allocation does not satisfy alignment {}", + alignment); + return false; + } + using namespace Common::Literals; + constexpr VkDeviceSize DesktopWindowSize = 4_GiB; + VkDeviceSize candidate_window = DesktopWindowSize; + const u64 max_buffer_size = device.GetMaxBufferSize(); + if (max_buffer_size != 0 && max_buffer_size < candidate_window) { + candidate_window = max_buffer_size; + } + const u64 max_allocation_size = device.GetMaxMemoryAllocationSize(); + if (max_allocation_size != 0 && max_allocation_size < candidate_window) { + candidate_window = max_allocation_size; + } + candidate_window = Common::AlignDown(candidate_window, alignment); + if (candidate_window == 0) { + return false; + } + window_size = candidate_window; + + const auto &logical = device.GetLogical(); + const auto memory_props = device.GetPhysical().GetMemoryProperties().memoryProperties; + + for (size_t offset = 0; offset < size; offset += window_size) { + u8 *const window_base = static_cast(base) + offset; + const VkDeviceSize window_len = + (std::min)(static_cast(size - offset), window_size); + VkMemoryHostPointerPropertiesEXT host_props{ + .sType = VK_STRUCTURE_TYPE_MEMORY_HOST_POINTER_PROPERTIES_EXT, + .pNext = nullptr, + .memoryTypeBits = 0, + }; + if (logical.GetMemoryHostPointerPropertiesEXT( + VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, window_base, + &host_props) != VK_SUCCESS || + host_props.memoryTypeBits == 0) { + break; + } + const VkExternalMemoryBufferCreateInfo external_info{ + .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO, + .pNext = nullptr, + .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, + }; + const VkBufferCreateInfo buffer_ci{ + .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, + .pNext = &external_info, + .flags = 0, + .size = window_len, + .usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, + .queueFamilyIndexCount = 0, + .pQueueFamilyIndices = nullptr, + }; + VkBuffer new_buffer{}; + if (logical.CreateBufferRaw(buffer_ci, &new_buffer) != VK_SUCCESS) { + break; + } + const VkMemoryRequirements requirements = + logical.GetBufferMemoryRequirements(new_buffer); + const u32 type_mask = requirements.memoryTypeBits & host_props.memoryTypeBits; + if (type_mask == 0 || requirements.size > window_len) { + logical.DestroyBufferRaw(new_buffer); + break; + } + const auto type_index = FindImportMemoryType(memory_props, type_mask); + if (!type_index) { + logical.DestroyBufferRaw(new_buffer); + break; + } + constexpr VkDeviceSize MaxHeapFractionDenominator = 2; + const u32 heap_index = memory_props.memoryTypes[*type_index].heapIndex; + const VkDeviceSize heap_size = memory_props.memoryHeaps[heap_index].size; + const VkDeviceSize heap_import_limit = heap_size / MaxHeapFractionDenominator; + if (imported_size + window_len > heap_import_limit) { + LOG_INFO(Render_Vulkan, + "Stopping guest memory import at {} MiB to leave room on heap {} of {} MiB", + imported_size >> 20, heap_index, heap_size >> 20); + logical.DestroyBufferRaw(new_buffer); + break; + } + const VkImportMemoryHostPointerInfoEXT import_info{ + .sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT, + .pNext = nullptr, + .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, + .pHostPointer = window_base, + }; + const VkMemoryAllocateInfo alloc_info{ + .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, + .pNext = &import_info, + .allocationSize = window_len, + .memoryTypeIndex = *type_index, + }; + vk::DeviceMemory memory = logical.TryAllocateMemory(alloc_info); + if (!memory) { + logical.DestroyBufferRaw(new_buffer); + break; + } + if (logical.BindBufferMemory(new_buffer, *memory, 0) != VK_SUCCESS) { + logical.DestroyBufferRaw(new_buffer); + break; + } + windows.push_back(Window{ + .memory = std::move(memory), + .buffer = new_buffer, + }); + imported_size += static_cast(window_len); + } + if (windows.empty()) { + LOG_INFO(Render_Vulkan, "Host pointer import failed"); + return false; + } + LOG_INFO(Render_Vulkan, + "Imported {} MiB of guest memory for unified memory access in {} windows", + imported_size >> 20, windows.size()); + return true; + } + + bool HostMemoryImport::ImportHardwareBuffers( + [[maybe_unused]] std::span hardware_buffers, + [[maybe_unused]] size_t hardware_buffer_window, + [[maybe_unused]] size_t hardware_buffer_base, [[maybe_unused]] size_t size) { +#ifdef __ANDROID__ + if (hardware_buffers.empty() || hardware_buffer_window == 0 || + !device.IsExtExternalMemoryAhbSupported()) { + return false; + } + using namespace Common::Literals; + u64 max_allocation_size = device.GetMaxMemoryAllocationSize(); + if (device.IsTiler()) { + constexpr u64 TilerAllocationLimit = 1_GiB; + max_allocation_size = max_allocation_size != 0 + ? (std::min)(max_allocation_size, TilerAllocationLimit) + : TilerAllocationLimit; + } + if (max_allocation_size != 0 && hardware_buffer_window > max_allocation_size) { + LOG_WARNING(Render_Vulkan, + "Hardware buffer windows of {} MiB exceed the {} MiB allocation limit", + hardware_buffer_window >> 20, max_allocation_size >> 20); + return false; + } + if (hardware_buffer_base >= size) { + return false; + } + const auto &logical = device.GetLogical(); + const auto memory_props = device.GetPhysical().GetMemoryProperties().memoryProperties; + window_size = hardware_buffer_window; + base_offset = hardware_buffer_base; + for (size_t i = 0; i < hardware_buffers.size(); ++i) { + const size_t offset = hardware_buffer_base + i * hardware_buffer_window; + if (offset >= size) { + break; + } + const VkDeviceSize window_len = (std::min)( + static_cast(size - offset), + static_cast(hardware_buffer_window)); + VkAndroidHardwareBufferPropertiesANDROID ahb_props{ + .sType = VK_STRUCTURE_TYPE_ANDROID_HARDWARE_BUFFER_PROPERTIES_ANDROID, + .pNext = nullptr, + .allocationSize = 0, + .memoryTypeBits = 0, + }; + if (logical.GetAndroidHardwareBufferPropertiesANDROID(hardware_buffers[i], + &ahb_props) != VK_SUCCESS || + ahb_props.memoryTypeBits == 0 || ahb_props.allocationSize < window_len) { + break; + } + const VkExternalMemoryBufferCreateInfo external_info{ + .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO, + .pNext = nullptr, + .handleTypes = + VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID, + }; + const VkBufferCreateInfo buffer_ci{ + .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, + .pNext = &external_info, + .flags = 0, + .size = window_len, + .usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, + .queueFamilyIndexCount = 0, + .pQueueFamilyIndices = nullptr, + }; + VkBuffer new_buffer{}; + if (logical.CreateBufferRaw(buffer_ci, &new_buffer) != VK_SUCCESS) { + break; + } + const VkMemoryRequirements requirements = + logical.GetBufferMemoryRequirements(new_buffer); + const u32 type_mask = requirements.memoryTypeBits & ahb_props.memoryTypeBits; + if (type_mask == 0 || requirements.size > ahb_props.allocationSize) { + logical.DestroyBufferRaw(new_buffer); + break; + } + const auto type_index = FindImportMemoryType(memory_props, type_mask); + if (!type_index) { + logical.DestroyBufferRaw(new_buffer); + break; + } + const VkImportAndroidHardwareBufferInfoANDROID import_info{ + .sType = VK_STRUCTURE_TYPE_IMPORT_ANDROID_HARDWARE_BUFFER_INFO_ANDROID, + .pNext = nullptr, + .buffer = hardware_buffers[i], + }; + const VkMemoryDedicatedAllocateInfo dedicated_info{ + .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO, + .pNext = &import_info, + .image = VK_NULL_HANDLE, + .buffer = new_buffer, + }; + const VkMemoryAllocateInfo alloc_info{ + .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, + .pNext = &dedicated_info, + .allocationSize = ahb_props.allocationSize, + .memoryTypeIndex = *type_index, + }; + vk::DeviceMemory memory = logical.TryAllocateMemory(alloc_info); + if (!memory) { + logical.DestroyBufferRaw(new_buffer); + break; + } + if (logical.BindBufferMemory(new_buffer, *memory, 0) != VK_SUCCESS) { + logical.DestroyBufferRaw(new_buffer); + break; + } + windows.push_back(Window{ + .memory = std::move(memory), + .buffer = new_buffer, + }); + imported_size += static_cast(window_len); + } + if (windows.empty()) { + LOG_INFO(Render_Vulkan, "Hardware buffer import failed"); + window_size = 0; + base_offset = 0; + return false; + } + foreign_ownership = true; + LOG_INFO(Render_Vulkan, + "Imported {} MiB of guest memory at {:#x} via hardware buffers in {} windows", + imported_size >> 20, base_offset, windows.size()); + return true; +#else + return false; +#endif + } + + HostMemoryImport::~HostMemoryImport() { + for (Window &window : windows) { + if (window.buffer != VK_NULL_HANDLE) { + device.GetLogical().DestroyBufferRaw(window.buffer); + } + } + } + MemoryAllocator::MemoryAllocator(const Device &device_) : device{device_}, allocator{device.GetAllocator()}, properties{device_.GetPhysical().GetMemoryProperties().memoryProperties}, diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.h b/src/video_core/vulkan_common/vulkan_memory_allocator.h index 581f2e66d2..9ece2bbfe4 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.h +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.h @@ -15,6 +15,8 @@ #include "video_core/vulkan_common/vulkan_wrapper.h" #include "video_core/vulkan_common/vma.h" +struct AHardwareBuffer; + namespace Vulkan { class Device; @@ -84,6 +86,66 @@ namespace Vulkan { void *mapped_ptr{}; ///< Optional persistent mapped pointer }; + class HostMemoryImport { + public: + explicit HostMemoryImport(const Device &device_, void *base, size_t size, + std::span hardware_buffers, + size_t hardware_buffer_window, size_t hardware_buffer_base); + + ~HostMemoryImport(); + + HostMemoryImport(const HostMemoryImport &) = delete; + + HostMemoryImport &operator=(const HostMemoryImport &) = delete; + + [[nodiscard]] bool IsValid() const noexcept { + return !windows.empty(); + } + + [[nodiscard]] size_t GetSize() const noexcept { + return imported_size; + } + + [[nodiscard]] size_t GetBaseOffset() const noexcept { + return base_offset; + } + + [[nodiscard]] bool NeedsForeignOwnershipTransfer() const noexcept { + return foreign_ownership; + } + + [[nodiscard]] VkDeviceSize GetWindowSize() const noexcept { + return window_size; + } + + [[nodiscard]] VkBuffer GetWindowBuffer(size_t index) const noexcept { + return windows[index].buffer; + } + + [[nodiscard]] size_t GetWindowCount() const noexcept { + return windows.size(); + } + + private: + struct Window { + vk::DeviceMemory memory; + VkBuffer buffer{}; + }; + + bool ImportHostPointer(void *base, size_t size); + + bool ImportHardwareBuffers(std::span hardware_buffers, + size_t hardware_buffer_window, size_t hardware_buffer_base, + size_t size); + + const Device &device; + std::vector windows; + VkDeviceSize window_size{}; + size_t imported_size{}; + size_t base_offset{}; + bool foreign_ownership{}; + }; + /// Memory allocator container. /// Allocates and releases memory allocations on demand. class MemoryAllocator { diff --git a/src/video_core/vulkan_common/vulkan_wrapper.cpp b/src/video_core/vulkan_common/vulkan_wrapper.cpp index 24f9da0a9e..2357bb80f9 100644 --- a/src/video_core/vulkan_common/vulkan_wrapper.cpp +++ b/src/video_core/vulkan_common/vulkan_wrapper.cpp @@ -216,12 +216,16 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept { X(vkGetBufferMemoryRequirements2); X(vkGetDeviceQueue); X(vkGetEventStatus); + X(vkGetMemoryHostPointerPropertiesEXT); X(vkGetFenceStatus); X(vkGetImageMemoryRequirements); X(vkGetPipelineCacheData); X(vkGetMemoryFdKHR); #ifdef _WIN32 X(vkGetMemoryWin32HandleKHR); +#endif +#ifdef __ANDROID__ + X(vkGetAndroidHardwareBufferPropertiesANDROID); #endif X(vkGetQueryPoolResults); X(vkGetPipelineExecutablePropertiesKHR); diff --git a/src/video_core/vulkan_common/vulkan_wrapper.h b/src/video_core/vulkan_common/vulkan_wrapper.h index c2d867838d..121d647b00 100644 --- a/src/video_core/vulkan_common/vulkan_wrapper.h +++ b/src/video_core/vulkan_common/vulkan_wrapper.h @@ -332,12 +332,16 @@ struct DeviceDispatch : InstanceDispatch { PFN_vkGetBufferMemoryRequirements2 vkGetBufferMemoryRequirements2{}; PFN_vkGetDeviceQueue vkGetDeviceQueue{}; PFN_vkGetEventStatus vkGetEventStatus{}; + PFN_vkGetMemoryHostPointerPropertiesEXT vkGetMemoryHostPointerPropertiesEXT{}; PFN_vkGetFenceStatus vkGetFenceStatus{}; PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements{}; PFN_vkGetPipelineCacheData vkGetPipelineCacheData{}; PFN_vkGetMemoryFdKHR vkGetMemoryFdKHR{}; #ifdef _WIN32 PFN_vkGetMemoryWin32HandleKHR vkGetMemoryWin32HandleKHR{}; +#endif +#ifdef __ANDROID__ + PFN_vkGetAndroidHardwareBufferPropertiesANDROID vkGetAndroidHardwareBufferPropertiesANDROID{}; #endif PFN_vkGetPipelineExecutablePropertiesKHR vkGetPipelineExecutablePropertiesKHR{}; PFN_vkGetPipelineExecutableStatisticsKHR vkGetPipelineExecutableStatisticsKHR{}; @@ -1082,6 +1086,34 @@ public: VkMemoryRequirements GetBufferMemoryRequirements(VkBuffer buffer, void* pnext = nullptr) const noexcept; + VkResult GetMemoryHostPointerPropertiesEXT( + VkExternalMemoryHandleTypeFlagBits handle_type, const void* host_pointer, + VkMemoryHostPointerPropertiesEXT* out_properties) const noexcept { + return dld->vkGetMemoryHostPointerPropertiesEXT(handle, handle_type, host_pointer, + out_properties); + } + +#ifdef __ANDROID__ + VkResult GetAndroidHardwareBufferPropertiesANDROID( + const struct AHardwareBuffer* buffer, + VkAndroidHardwareBufferPropertiesANDROID* out_properties) const noexcept { + return dld->vkGetAndroidHardwareBufferPropertiesANDROID(handle, buffer, out_properties); + } +#endif + + VkResult CreateBufferRaw(const VkBufferCreateInfo& ci, VkBuffer* out_buffer) const noexcept { + return dld->vkCreateBuffer(handle, &ci, nullptr, out_buffer); + } + + void DestroyBufferRaw(VkBuffer buffer) const noexcept { + dld->vkDestroyBuffer(handle, buffer, nullptr); + } + + VkResult BindBufferMemory(VkBuffer buffer, VkDeviceMemory memory, + VkDeviceSize offset) const noexcept { + return dld->vkBindBufferMemory(handle, buffer, memory, offset); + } + VkMemoryRequirements GetImageMemoryRequirements(VkImage image) const noexcept; std::vector GetPipelineExecutablePropertiesKHR(