From a686b50333c8244547a2af790e3b45b6d18e1180 Mon Sep 17 00:00:00 2001 From: CamilleLaVey Date: Sat, 1 Aug 2026 22:04:31 -0400 Subject: [PATCH] 3rd step on UMA implementation --- src/common/host_memory.cpp | 230 ++++++++++++------ src/common/host_memory.h | 6 +- src/common/memory_detect.cpp | 38 +++ src/common/memory_detect.h | 2 + src/core/device_memory.cpp | 11 +- src/core/device_memory_manager.h | 10 + src/core/device_memory_manager.inc | 2 + src/video_core/buffer_cache/buffer_cache.h | 14 +- .../renderer_vulkan/vk_buffer_cache.cpp | 85 ++++++- .../renderer_vulkan/vk_buffer_cache.h | 11 +- .../renderer_vulkan/vk_rasterizer.cpp | 5 +- .../vulkan_common/vulkan_device.cpp | 5 + src/video_core/vulkan_common/vulkan_device.h | 5 + .../vulkan_common/vulkan_memory_allocator.cpp | 60 +++-- .../vulkan_common/vulkan_memory_allocator.h | 17 +- 15 files changed, 392 insertions(+), 109 deletions(-) diff --git a/src/common/host_memory.cpp b/src/common/host_memory.cpp index 5199dd88b1..bae706e6ca 100644 --- a/src/common/host_memory.cpp +++ b/src/common/host_memory.cpp @@ -78,15 +78,9 @@ struct NativeHandle { }; using PFN_AHardwareBuffer_getNativeHandle = const NativeHandle* (*)(const AHardwareBuffer*); -using PFN_AHardwareBuffer_isSupported = int (*)(const AHardwareBuffer_Desc*); - -void* NativeWindowLibrary() { - static void* const lib = dlopen("libnativewindow.so", RTLD_NOW); - return lib; -} PFN_AHardwareBuffer_getNativeHandle ResolveGetNativeHandle() { - void* const lib = NativeWindowLibrary(); + void* const lib = dlopen("libnativewindow.so", RTLD_NOW); if (lib == nullptr) { return nullptr; } @@ -94,15 +88,6 @@ PFN_AHardwareBuffer_getNativeHandle ResolveGetNativeHandle() { dlsym(lib, "AHardwareBuffer_getNativeHandle")); } -PFN_AHardwareBuffer_isSupported ResolveIsSupported() { - void* const lib = NativeWindowLibrary(); - if (lib == nullptr) { - return nullptr; - } - return reinterpret_cast( - dlsym(lib, "AHardwareBuffer_isSupported")); -} - } // namespace #endif @@ -175,7 +160,7 @@ static void GetFuncAddress(Common::DynamicLibrary& dll, const char* name, T& pfn class HostMemory::Impl { public: - explicit Impl(size_t backing_size_, size_t virtual_size_) + explicit Impl(size_t backing_size_, size_t virtual_size_, size_t) : backing_size{backing_size_} , virtual_size{virtual_size_} , process{GetCurrentProcess()} @@ -281,6 +266,10 @@ public: UNREACHABLE(); } + bool IsBackingShared() const noexcept { + return true; + } + const size_t backing_size; ///< Size of the backing memory in bytes const size_t virtual_size; ///< Size of the virtual address placeholder in bytes @@ -553,19 +542,15 @@ static int shm_open_anon(int flags, mode_t mode) { class HostMemory::Impl { public: - explicit Impl(size_t backing_size_, size_t virtual_size_) + explicit Impl(size_t backing_size_, size_t virtual_size_, size_t preferred_offset_) : backing_size{backing_size_} , virtual_size{virtual_size_} + , preferred_offset{preferred_offset_} {} bool Init() { long page_size = sysconf(_SC_PAGESIZE); ASSERT_MSG(page_size == 0x1000, "page size {:#x} is incompatible with 4K paging", page_size); -#ifdef __ANDROID__ - if (InitAhbBacking()) { - return InitVirtual(); - } -#endif // Backing memory initialization #if defined(__sun__) || defined(__HAIKU__) || defined(__NetBSD__) || defined(__DragonFly__) fd = shm_open_anon(O_RDWR | O_CREAT | O_EXCL | O_NOFOLLOW, 0600); @@ -600,10 +585,15 @@ public: LOG_WARNING(Common_Memory, "Using private mappings instead of shared ones"); backing_base = static_cast(mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0)); if (fd > 0) { - fd = -1; close(fd); } + fd = -1; } else { +#ifdef __ANDROID__ + if (InitAhbBacking()) { + return InitVirtual(); + } +#endif backing_base = static_cast(mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0)); } if (backing_base == MAP_FAILED) { @@ -680,6 +670,38 @@ public: return ok; } + size_t ComputeAhbBudget(size_t window_size) const { + const u64 total_physical = Common::GetMemInfo().TotalPhysicalMemory; + if (total_physical == 0) { + LOG_WARNING(HW_Memory, "Host memory size is unknown, not committing hardware buffers"); + return 0; + } + constexpr u64 MinimumTotalPhysical = 7ULL << 30; + if (total_physical < MinimumTotalPhysical) { + LOG_INFO(HW_Memory, + "Skipping hardware buffer backing, {} MiB of RAM is below the {} MiB minimum", + total_physical >> 20, MinimumTotalPhysical >> 20); + return 0; + } + u64 budget = total_physical / 6; + const u64 available = Common::GetAvailablePhysicalMemory(); + if (available != 0) { + constexpr u64 Headroom = 2ULL << 30; + budget = (std::min)(budget, available > Headroom ? available - Headroom : 0); + } + budget = (std::min)(budget, static_cast(backing_size)); + budget = Common::AlignDown(budget, window_size); + constexpr u64 MinimumBudget = 256ULL << 20; + if (budget < MinimumBudget) { + LOG_INFO(HW_Memory, + "Skipping hardware buffer backing, only {} MiB could be committed on a {} MiB " + "system with {} MiB available", + budget >> 20, total_physical >> 20, available >> 20); + return 0; + } + return static_cast(budget); + } + bool InitAhbBacking() { if (!Settings::values.use_unified_memory.GetValue()) { return false; @@ -690,30 +712,26 @@ public: LOG_WARNING(HW_Memory, "AHardwareBuffer_getNativeHandle is not available"); return false; } - static const PFN_AHardwareBuffer_isSupported is_supported = ResolveIsSupported(); - if (is_supported == nullptr) { - LOG_WARNING(HW_Memory, "AHardwareBuffer_isSupported is not available"); - return false; - } - const u64 total_physical = Common::GetMemInfo().TotalPhysicalMemory; - if (total_physical != 0 && backing_size > total_physical / 2) { - LOG_WARNING(HW_Memory, - "Hardware buffer backing would commit {} MiB on a {} MiB system, keeping " - "lazily committed memory", - backing_size >> 20, total_physical >> 20); - return false; - } - constexpr size_t window_size = 256ULL << 20; + constexpr size_t window_size = 64ULL << 20; const AHardwareBuffer_Desc window_desc = MakeBlobDesc(window_size); - if (is_supported(&window_desc) == 0) { + if (AHardwareBuffer_isSupported(&window_desc) == 0) { LOG_WARNING(HW_Memory, "Allocator rejects {} MiB hardware buffer windows", window_size >> 20); return false; } + const size_t budget = ComputeAhbBudget(window_size); + if (budget == 0) { + return false; + } if (!ProbeAhbBacking(get_native_handle)) { return false; } - const size_t num_windows = (backing_size + window_size - 1) / window_size; + const size_t aligned_backing = Common::AlignDown(backing_size, window_size); + const size_t region_size = (std::min)(budget, aligned_backing); + const size_t region_base = Common::AlignDown( + (std::min)(preferred_offset, aligned_backing - region_size), window_size); + const size_t num_windows = region_size / window_size; + std::vector buffers; std::vector buffer_fds; const auto cleanup = [&] { @@ -724,11 +742,11 @@ public: buffer_fds.clear(); }; for (size_t i = 0; i < num_windows; ++i) { - const size_t len = (std::min)(window_size, backing_size - i * window_size); - const AHardwareBuffer_Desc desc = MakeBlobDesc(len); + const AHardwareBuffer_Desc desc = MakeBlobDesc(window_size); AHardwareBuffer* buffer{}; if (AHardwareBuffer_allocate(&desc, &buffer) != 0 || buffer == nullptr) { - LOG_WARNING(HW_Memory, "Hardware buffer allocation failed for window {}", i); + LOG_WARNING(HW_Memory, "Hardware buffer allocation failed for window {} of {}", i, + num_windows); cleanup(); return false; } @@ -741,7 +759,7 @@ public: } const int buffer_fd = handle->data[0]; const off_t buffer_len = lseek(buffer_fd, 0, SEEK_END); - if (buffer_len < static_cast(len)) { + if (buffer_len < static_cast(window_size)) { LOG_WARNING(HW_Memory, "Hardware buffer descriptor smaller than requested"); cleanup(); return false; @@ -751,36 +769,85 @@ public: u8* const base = static_cast(mmap(nullptr, backing_size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0)); if (base == MAP_FAILED) { + LOG_WARNING(HW_Memory, "Failed to reserve backing address space: {}", strerror(errno)); cleanup(); return false; } - for (size_t i = 0; i < num_windows; ++i) { - const size_t len = (std::min)(window_size, backing_size - i * window_size); - if (mmap(base + i * window_size, len, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, - buffer_fds[i], 0) == MAP_FAILED) { - LOG_WARNING(HW_Memory, "Hardware buffer mmap failed: {}", strerror(errno)); + const auto map_over_reservation = [&](size_t offset, size_t len, int map_fd, + off_t map_offset) { + if (len == 0) { + return true; + } + if (mmap(base + offset, len, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, map_fd, + map_offset) == MAP_FAILED) { + LOG_WARNING(HW_Memory, "Backing mmap failed: {}", strerror(errno)); munmap(base, backing_size); cleanup(); return false; } + return true; + }; + if (!map_over_reservation(0, region_base, fd, 0)) { + return false; + } + for (size_t i = 0; i < num_windows; ++i) { + if (!map_over_reservation(region_base + i * window_size, window_size, buffer_fds[i], + 0)) { + return false; + } + } + const size_t tail_offset = region_base + region_size; + if (!map_over_reservation(tail_offset, backing_size - tail_offset, fd, + static_cast(tail_offset))) { + return false; } backing_base = base; ahb_windows = std::move(buffers); ahb_fds = std::move(buffer_fds); ahb_window_size = window_size; - ahb_backing = true; - committed_backing_size.store(backing_size, std::memory_order_relaxed); - LOG_INFO(HW_Memory, "Guest memory backed by {} hardware buffer windows, {} MiB committed", - ahb_windows.size(), backing_size >> 20); + ahb_base = region_base; + ahb_bytes = region_size; + committed_backing_size.store(region_size, std::memory_order_relaxed); + LOG_INFO(HW_Memory, + "Guest memory {:#x}-{:#x} backed by {} hardware buffer windows, {} MiB committed", + region_base, region_base + region_size, ahb_windows.size(), region_size >> 20); return true; } + void MapBackingRange(size_t virtual_offset, size_t host_offset, size_t length, int prot_flags) { + while (length > 0) { + int map_fd = fd; + off_t map_offset = static_cast(host_offset); + size_t chunk = length; + if (host_offset < ahb_base) { + chunk = (std::min)(chunk, ahb_base - host_offset); + } else if (host_offset < ahb_base + ahb_bytes) { + const size_t relative = host_offset - ahb_base; + const size_t window = relative / ahb_window_size; + const size_t local = relative % ahb_window_size; + map_fd = ahb_fds[window]; + map_offset = static_cast(local); + chunk = (std::min)(chunk, ahb_window_size - local); + } + void* const ret = mmap(virtual_base + virtual_offset, chunk, prot_flags, + MAP_SHARED | MAP_FIXED, map_fd, map_offset); + ASSERT_MSG(ret != MAP_FAILED, "mmap: {}", strerror(errno)); + virtual_offset += chunk; + host_offset += chunk; + length -= chunk; + } + } + std::span AhbWindows() const noexcept { return ahb_windows; } size_t AhbWindowSize() const noexcept { - return ahb_backing ? ahb_window_size : 0; + return ahb_bytes != 0 ? ahb_window_size : 0; + } + + size_t AhbBase() const noexcept { + return ahb_base; } #endif @@ -806,22 +873,8 @@ public: prot_flags |= PROT_EXEC; #endif #ifdef __ANDROID__ - if (ahb_backing) { - size_t voff = virtual_offset; - size_t hoff = host_offset; - size_t remaining = length; - while (remaining > 0) { - const size_t window = hoff / ahb_window_size; - const size_t local = hoff % ahb_window_size; - const size_t chunk = (std::min)(remaining, ahb_window_size - local); - void* const ret = - mmap(virtual_base + voff, chunk, prot_flags, MAP_SHARED | MAP_FIXED, - ahb_fds[window], static_cast(local)); - ASSERT_MSG(ret != MAP_FAILED, "mmap: {}", strerror(errno)); - voff += chunk; - hoff += chunk; - remaining -= chunk; - } + if (ahb_bytes != 0) { + MapBackingRange(virtual_offset, host_offset, length, prot_flags); return; } #endif @@ -869,8 +922,18 @@ public: virtual_base = nullptr; } + bool IsBackingShared() const noexcept { +#ifdef __ANDROID__ + if (ahb_bytes != 0) { + return true; + } +#endif + return fd >= 0; + } + const size_t backing_size; ///< Size of the backing memory in bytes const size_t virtual_size; ///< Size of the virtual address placeholder in bytes + const size_t preferred_offset; u8* backing_base{reinterpret_cast(MAP_FAILED)}; u8* virtual_base{reinterpret_cast(MAP_FAILED)}; @@ -900,9 +963,9 @@ private: } ahb_windows.clear(); ahb_fds.clear(); - if (ahb_backing) { + if (ahb_bytes != 0) { committed_backing_size.store(0, std::memory_order_relaxed); - ahb_backing = false; + ahb_bytes = 0; } #endif } @@ -932,16 +995,17 @@ private: FreeRegionManager free_manager{}; #ifdef __ANDROID__ - bool ahb_backing{}; std::vector ahb_windows; std::vector ahb_fds; size_t ahb_window_size{}; + size_t ahb_base{}; + size_t ahb_bytes{}; #endif }; #endif // ^^^ POSIX ^^^ -HostMemory::HostMemory(size_t backing_size_, size_t virtual_size_) +HostMemory::HostMemory(size_t backing_size_, size_t virtual_size_, size_t preferred_offset_) : backing_size(backing_size_) , virtual_size(virtual_size_) { @@ -953,7 +1017,7 @@ HostMemory::HostMemory(size_t backing_size_, size_t virtual_size_) #else // Try to allocate a fastmem arena. // The implementation will fail with std::bad_alloc on errors. - impl = std::make_unique(AlignUp(backing_size, PageAlignment), AlignUp(virtual_size, PageAlignment) + HugePageSize); + impl = std::make_unique(AlignUp(backing_size, PageAlignment), AlignUp(virtual_size, PageAlignment) + HugePageSize, preferred_offset_); if (impl->Init()) { backing_base = impl->backing_base; virtual_base = impl->virtual_base; @@ -1039,6 +1103,22 @@ size_t HostMemory::BackingHardwareBufferWindowSize() const noexcept { #endif } +bool HostMemory::IsBackingShared() const noexcept { +#if defined(__OPENORBIS__) || defined(__managarm__) + return false; +#else + return impl && impl->IsBackingShared(); +#endif +} + +size_t HostMemory::BackingHardwareBufferBase() const noexcept { +#ifdef __ANDROID__ + return impl ? impl->AhbBase() : 0; +#else + return 0; +#endif +} + void HostMemory::EnableDirectMappedAddress() { #if !(defined(__OPENORBIS__) || defined(__managarm__)) if (impl) { diff --git a/src/common/host_memory.h b/src/common/host_memory.h index 5d385247d1..13d4830028 100644 --- a/src/common/host_memory.h +++ b/src/common/host_memory.h @@ -33,7 +33,7 @@ DECLARE_ENUM_FLAG_OPERATORS(MemoryPermission) */ class HostMemory { public: - explicit HostMemory(size_t backing_size_, size_t virtual_size_); + explicit HostMemory(size_t backing_size_, size_t virtual_size_, size_t preferred_offset_ = 0); ~HostMemory(); /** @@ -75,6 +75,10 @@ public: [[nodiscard]] size_t BackingHardwareBufferWindowSize() const noexcept; + [[nodiscard]] size_t BackingHardwareBufferBase() const noexcept; + + [[nodiscard]] bool IsBackingShared() const noexcept; + [[nodiscard]] u8* VirtualBasePointer() noexcept { return virtual_base; } diff --git a/src/common/memory_detect.cpp b/src/common/memory_detect.cpp index 86a3abcc6d..d7d8a0741d 100644 --- a/src/common/memory_detect.cpp +++ b/src/common/memory_detect.cpp @@ -17,6 +17,10 @@ #endif #endif +#include +#include +#include + #include "common/memory_detect.h" namespace Common { @@ -69,4 +73,38 @@ const MemoryInfo& GetMemInfo() { return mem_info; } +u64 GetAvailablePhysicalMemory() { +#ifdef _WIN32 + MEMORYSTATUSEX memorystatus; + memorystatus.dwLength = sizeof(memorystatus); + if (GlobalMemoryStatusEx(&memorystatus)) { + return memorystatus.ullAvailPhys; + } + return 0; +#elif defined(__linux__) + if (std::FILE* const file = std::fopen("/proc/meminfo", "re")) { + char line[256]; + u64 available = 0; + while (std::fgets(line, sizeof(line), file) != nullptr) { + if (std::strncmp(line, "MemAvailable:", 13) == 0) { + available = std::strtoull(line + 13, nullptr, 10) * 1024ULL; + break; + } + } + std::fclose(file); + if (available != 0) { + return available; + } + } + struct sysinfo info; + if (sysinfo(&info) == 0) { + const u64 unit = info.mem_unit != 0 ? info.mem_unit : 1ULL; + return (static_cast(info.freeram) + static_cast(info.bufferram)) * unit; + } + return 0; +#else + return 0; +#endif +} + } // namespace Common diff --git a/src/common/memory_detect.h b/src/common/memory_detect.h index c8f239aed3..7141df1172 100644 --- a/src/common/memory_detect.h +++ b/src/common/memory_detect.h @@ -18,4 +18,6 @@ struct MemoryInfo { */ [[nodiscard]] const MemoryInfo& GetMemInfo(); +[[nodiscard]] u64 GetAvailablePhysicalMemory(); + } // namespace Common diff --git a/src/core/device_memory.cpp b/src/core/device_memory.cpp index 1aea56a991..e36ed42e31 100644 --- a/src/core/device_memory.cpp +++ b/src/core/device_memory.cpp @@ -12,9 +12,18 @@ constexpr size_t VirtualReserveSize = 1ULL << 38; constexpr size_t VirtualReserveSize = 1ULL << 39; #endif +namespace { +size_t ApplicationPoolOffset() { + using Init = Kernel::Board::Nintendo::Nx::KSystemControl::Init; + const size_t dram_size = Init::GetIntendedMemorySize(); + const size_t application_pool_size = Init::GetApplicationPoolSize(); + return dram_size > application_pool_size ? dram_size - application_pool_size : 0; +} +} + DeviceMemory::DeviceMemory() : buffer{Kernel::Board::Nintendo::Nx::KSystemControl::Init::GetIntendedMemorySize(), - VirtualReserveSize} {} + VirtualReserveSize, ApplicationPoolOffset()} {} DeviceMemory::~DeviceMemory() = default; diff --git a/src/core/device_memory_manager.h b/src/core/device_memory_manager.h index 98cc7babcd..2261a75c00 100644 --- a/src/core/device_memory_manager.h +++ b/src/core/device_memory_manager.h @@ -117,6 +117,14 @@ public: return ahb_window_size; } + size_t GetBackingHardwareBufferBase() const noexcept { + return ahb_base; + } + + bool IsBackingShared() const noexcept { + return backing_is_shared; + } + PAddr GetPhysicalRawAddressFromDAddr(DAddr address) const { PAddr subbits = PAddr(address & page_mask); auto paddr = tracked_entries[(address >> page_bits)].compressed_physical_ptr; @@ -200,6 +208,8 @@ private: const size_t physical_size; const std::span ahb_windows; const size_t ahb_window_size; + const size_t ahb_base; + const bool backing_is_shared; DeviceInterface* device_inter; struct TrackedEntry { diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index 8c247e228b..2d00a6e570 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -174,6 +174,8 @@ DeviceMemoryManager::DeviceMemoryManager(const DeviceMemory& device_memo , physical_size{device_memory_.buffer.BackingSize()} , ahb_windows{device_memory_.buffer.BackingHardwareBuffers()} , ahb_window_size{device_memory_.buffer.BackingHardwareBufferWindowSize()} + , ahb_base{device_memory_.buffer.BackingHardwareBufferBase()} + , backing_is_shared{device_memory_.buffer.IsBackingShared()} , device_inter{nullptr} , compressed_device_addr(1ULL << ((Settings::values.memory_layout_mode.GetValue() == Settings::MemoryLayout::Memory_4Gb ? physical_min_bits : physical_max_bits) - Memory::YUZU_PAGEBITS)) , tracked_entries(device_as_size >> Memory::YUZU_PAGEBITS) diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index ca6d65c2b0..5394cadbc5 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -1819,6 +1819,7 @@ bool BufferCache

::TryUnifiedDownloadMemory([[maybe_unused]] Buffer& buffer, [[maybe_unused]] std::span copies) { if constexpr (USE_UNIFIED_MEMORY) { const u8* const physical_base = device_memory.GetPhysicalBase(); + const u64 unified_base = runtime.UnifiedMemoryBase(); const u64 unified_size = runtime.UnifiedMemorySize(); const u64 window_size = runtime.UnifiedMemoryWindowSize(); if (window_size == 0) { @@ -1849,11 +1850,13 @@ bool BufferCache

::TryUnifiedDownloadMemory([[maybe_unused]] Buffer& buffer, u64 chunk = (std::min)(copy.size - downloaded, static_cast(Core::DEVICE_PAGESIZE) - page_offset); const u64 phys_offset = static_cast(ptr - physical_base); - if (phys_offset + chunk > unified_size) { + if (phys_offset < unified_base || + phys_offset - unified_base + chunk > unified_size) { return false; } - const u64 window = phys_offset / window_size; - const u64 local_offset = phys_offset % window_size; + const u64 relative = phys_offset - unified_base; + const u64 window = relative / window_size; + const u64 local_offset = relative % window_size; chunk = (std::min)(chunk, window_size - local_offset); auto& group = group_for(window); if (!group.empty()) { @@ -1877,9 +1880,8 @@ bool BufferCache

::TryUnifiedDownloadMemory([[maybe_unused]] Buffer& buffer, buffer.MarkUsage(copy.src_offset, copy.size); } for (size_t i = 0; i < window_ids.size(); ++i) { - const std::span group_span(groups[i].data(), groups[i].size()); - runtime.CopyBuffer(runtime.UnifiedMemoryWindowBuffer(window_ids[i]), buffer, - group_span, true); + const std::span group_span(groups[i].data(), groups[i].size()); + runtime.CopyToUnifiedMemory(window_ids[i], buffer, group_span); } runtime.Finish(); return true; diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp index 6c9ae97b03..0d33df9d51 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp @@ -366,14 +366,93 @@ BufferCacheRuntime::BufferCacheRuntime(const Device& device_, MemoryAllocator& m void BufferCacheRuntime::TryEnableUnifiedMemory(void* base, size_t size, std::span hardware_buffers, - size_t hardware_buffer_window) { - unified_memory = std::make_unique(device, base, size, hardware_buffers, - hardware_buffer_window); + size_t hardware_buffer_window, + size_t hardware_buffer_base) { + unified_memory = std::make_unique( + device, base, size, hardware_buffers, hardware_buffer_window, hardware_buffer_base); if (!unified_memory->IsValid()) { unified_memory.reset(); } } +void BufferCacheRuntime::CopyToUnifiedMemory( + size_t window_index, VkBuffer src_buffer, + std::span copies) { + if (!unified_memory || src_buffer == VK_NULL_HANDLE || copies.empty() || + window_index >= unified_memory->GetWindowCount()) { + return; + } + const VkBuffer dst_buffer = unified_memory->GetWindowBuffer(window_index); + if (dst_buffer == VK_NULL_HANDLE) { + return; + } + VkDeviceSize covered_begin = std::numeric_limits::max(); + VkDeviceSize covered_end = 0; + for (const VideoCommon::BufferCopy& copy : copies) { + covered_begin = (std::min)(covered_begin, static_cast(copy.dst_offset)); + covered_end = (std::max)(covered_end, + static_cast(copy.dst_offset + copy.size)); + } + + boost::container::small_vector vk_copies(copies.size()); + std::ranges::transform(copies, vk_copies.begin(), MakeBufferCopy); + + const bool foreign = unified_memory->NeedsForeignOwnershipTransfer(); + const u32 queue_family = device.GetGraphicsFamily(); + + scheduler.RequestOutsideRenderPassOperationContext(); + scheduler.Record([src_buffer, dst_buffer, vk_copies, foreign, queue_family, covered_begin, + covered_end](vk::CommandBuffer cmdbuf) { + static constexpr VkMemoryBarrier READ_BARRIER{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT, + .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, + }; + cmdbuf.PipelineBarrier(vk::PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, READ_BARRIER); + if (foreign) { + const VkBufferMemoryBarrier acquire{ + .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = 0, + .dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT, + .dstQueueFamilyIndex = queue_family, + .buffer = dst_buffer, + .offset = covered_begin, + .size = covered_end - covered_begin, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, acquire); + } + cmdbuf.CopyBuffer(src_buffer, dst_buffer, VideoCommon::FixSmallVectorADL(vk_copies)); + if (foreign) { + const VkBufferMemoryBarrier release{ + .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .dstAccessMask = 0, + .srcQueueFamilyIndex = queue_family, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT, + .buffer = dst_buffer, + .offset = covered_begin, + .size = covered_end - covered_begin, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0, release); + } + static constexpr VkMemoryBarrier HOST_BARRIER{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_HOST_READ_BIT, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, 0, + HOST_BARRIER); + }); +} + StagingBufferRef BufferCacheRuntime::UploadStagingBuffer(size_t size) { return staging_pool.Request(size, MemoryUsage::Upload); } diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index 9cf435a8ac..e14ce55b4a 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -101,7 +101,7 @@ public: void TryEnableUnifiedMemory(void* base, size_t size, std::span hardware_buffers, - size_t hardware_buffer_window); + size_t hardware_buffer_window, size_t hardware_buffer_base); [[nodiscard]] bool HasUnifiedMemory() const noexcept { return unified_memory != nullptr && unified_memory->IsValid(); @@ -111,13 +111,16 @@ public: return unified_memory ? unified_memory->GetSize() : 0; } + [[nodiscard]] u64 UnifiedMemoryBase() const noexcept { + return unified_memory ? unified_memory->GetBaseOffset() : 0; + } + [[nodiscard]] u64 UnifiedMemoryWindowSize() const noexcept { return unified_memory ? unified_memory->GetWindowSize() : 0; } - [[nodiscard]] VkBuffer UnifiedMemoryWindowBuffer(size_t index) const noexcept { - return unified_memory ? unified_memory->GetWindowBuffer(index) : VK_NULL_HANDLE; - } + void CopyToUnifiedMemory(size_t window_index, VkBuffer src_buffer, + std::span copies); u64 CurrentTick(); diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index f7046777db..9773fd32ac 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -239,11 +239,12 @@ RasterizerVulkan::RasterizerVulkan(Core::Frontend::EmuWindow& emu_window_, Tegra fence_manager(*this, gpu, texture_cache, buffer_cache, query_cache, device, scheduler), wfi_event(device.GetLogical().CreateEvent()) { scheduler.SetQueryCache(query_cache); - if (Settings::values.use_unified_memory.GetValue()) { + if (Settings::values.use_unified_memory.GetValue() && device_memory.IsBackingShared()) { buffer_cache_runtime.TryEnableUnifiedMemory( device_memory.GetPhysicalBase(), device_memory.GetPhysicalSize(), device_memory.GetBackingHardwareBuffers(), - device_memory.GetBackingHardwareBufferWindowSize()); + device_memory.GetBackingHardwareBufferWindowSize(), + device_memory.GetBackingHardwareBufferBase()); } memory_allocator.SetReclaimCallback([this](u64 bytes) -> u64 { u64 freed = staging_pool.ReclaimMemory(bytes); diff --git a/src/video_core/vulkan_common/vulkan_device.cpp b/src/video_core/vulkan_common/vulkan_device.cpp index 5b0b247375..5552ccc18f 100644 --- a/src/video_core/vulkan_common/vulkan_device.cpp +++ b/src/video_core/vulkan_common/vulkan_device.cpp @@ -1256,6 +1256,11 @@ bool Device::GetSuitability(bool requires_swapchain) { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT; SetNext(next, properties.external_memory_host); } + if (extensions.maintenance3 || instance_version >= VK_API_VERSION_1_1) { + properties.maintenance3.sType = + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_3_PROPERTIES; + SetNext(next, properties.maintenance3); + } if (extensions.maintenance4 || features.maintenance4.maintenance4) { properties.maintenance4.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_4_PROPERTIES; diff --git a/src/video_core/vulkan_common/vulkan_device.h b/src/video_core/vulkan_common/vulkan_device.h index 89238a4015..5c618784e8 100644 --- a/src/video_core/vulkan_common/vulkan_device.h +++ b/src/video_core/vulkan_common/vulkan_device.h @@ -947,6 +947,10 @@ FN_MAX_LIMIT_LIST return properties.maintenance4.maxBufferSize; } + u64 GetMaxMemoryAllocationSize() const { + return properties.maintenance3.maxMemoryAllocationSize; + } + bool HasTimelineSemaphore() const; /// Returns true if the device supports VK_KHR_synchronization2. @@ -1295,6 +1299,7 @@ private: VkPhysicalDeviceDescriptorBufferPropertiesEXT descriptor_buffer{}; VkPhysicalDeviceSubgroupSizeControlProperties subgroup_size_control{}; VkPhysicalDeviceTransformFeedbackPropertiesEXT transform_feedback{}; + VkPhysicalDeviceMaintenance3Properties maintenance3{}; VkPhysicalDeviceMaintenance4Properties maintenance4{}; VkPhysicalDeviceMaintenance5PropertiesKHR maintenance5{}; VkPhysicalDeviceDepthStencilResolveProperties depth_stencil_resolve{}; diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp index 0ad90ccafd..7a641e0bcb 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp @@ -95,15 +95,22 @@ namespace Vulkan { HostMemoryImport::HostMemoryImport(const Device &device_, void *base, size_t size, std::span hardware_buffers, - size_t hardware_buffer_window) + size_t hardware_buffer_window, size_t hardware_buffer_base) : device{device_} { - if (ImportHostPointer(base, size)) { + if (ImportHardwareBuffers(hardware_buffers, hardware_buffer_window, hardware_buffer_base, + size)) { return; } - ImportHardwareBuffers(hardware_buffers, hardware_buffer_window, size); - if (windows.empty()) { - LOG_INFO(Render_Vulkan, "Unified memory disabled, no host memory import path"); + if (!hardware_buffers.empty()) { + LOG_INFO(Render_Vulkan, + "Unified memory disabled, guest memory is backed by hardware buffers that " + "could not be imported"); + return; + } + if (ImportHostPointer(base, size)) { + return; } + LOG_INFO(Render_Vulkan, "Unified memory disabled, no host memory import path"); } bool HostMemoryImport::ImportHostPointer(void *base, size_t size) { @@ -124,8 +131,13 @@ namespace Vulkan { VkDeviceSize candidate_window = 1_GiB; const u64 max_buffer_size = device.GetMaxBufferSize(); if (max_buffer_size != 0 && max_buffer_size < candidate_window) { - candidate_window = Common::AlignDown(max_buffer_size, alignment); + candidate_window = max_buffer_size; + } + const u64 max_allocation_size = device.GetMaxMemoryAllocationSize(); + if (max_allocation_size != 0 && max_allocation_size < candidate_window) { + candidate_window = max_allocation_size; } + candidate_window = Common::AlignDown(candidate_window, alignment); if (candidate_window == 0) { return false; } @@ -226,19 +238,31 @@ namespace Vulkan { return true; } - void HostMemoryImport::ImportHardwareBuffers( + bool HostMemoryImport::ImportHardwareBuffers( [[maybe_unused]] std::span hardware_buffers, - [[maybe_unused]] size_t hardware_buffer_window, [[maybe_unused]] size_t size) { + [[maybe_unused]] size_t hardware_buffer_window, + [[maybe_unused]] size_t hardware_buffer_base, [[maybe_unused]] size_t size) { #ifdef __ANDROID__ if (hardware_buffers.empty() || hardware_buffer_window == 0 || !device.IsExtExternalMemoryAhbSupported()) { - return; + return false; + } + const u64 max_allocation_size = device.GetMaxMemoryAllocationSize(); + if (max_allocation_size != 0 && hardware_buffer_window > max_allocation_size) { + LOG_WARNING(Render_Vulkan, + "Hardware buffer windows of {} MiB exceed the {} MiB allocation limit", + hardware_buffer_window >> 20, max_allocation_size >> 20); + return false; + } + if (hardware_buffer_base >= size) { + return false; } const auto &logical = device.GetLogical(); const auto memory_props = device.GetPhysical().GetMemoryProperties().memoryProperties; window_size = hardware_buffer_window; + base_offset = hardware_buffer_base; for (size_t i = 0; i < hardware_buffers.size(); ++i) { - const size_t offset = i * hardware_buffer_window; + const size_t offset = hardware_buffer_base + i * hardware_buffer_window; if (offset >= size) { break; } @@ -320,11 +344,19 @@ namespace Vulkan { }); imported_size += static_cast(window_len); } - if (!windows.empty()) { - LOG_INFO(Render_Vulkan, - "Imported {} MiB of guest memory via hardware buffers in {} windows", - imported_size >> 20, windows.size()); + if (windows.empty()) { + LOG_INFO(Render_Vulkan, "Hardware buffer import failed"); + window_size = 0; + base_offset = 0; + return false; } + foreign_ownership = true; + LOG_INFO(Render_Vulkan, + "Imported {} MiB of guest memory at {:#x} via hardware buffers in {} windows", + imported_size >> 20, base_offset, windows.size()); + return true; +#else + return false; #endif } diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.h b/src/video_core/vulkan_common/vulkan_memory_allocator.h index 411836fe85..92a2c82a07 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.h +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.h @@ -46,7 +46,7 @@ namespace Vulkan { public: explicit HostMemoryImport(const Device &device_, void *base, size_t size, std::span hardware_buffers, - size_t hardware_buffer_window); + size_t hardware_buffer_window, size_t hardware_buffer_base); ~HostMemoryImport(); @@ -62,6 +62,14 @@ namespace Vulkan { return imported_size; } + [[nodiscard]] size_t GetBaseOffset() const noexcept { + return base_offset; + } + + [[nodiscard]] bool NeedsForeignOwnershipTransfer() const noexcept { + return foreign_ownership; + } + [[nodiscard]] VkDeviceSize GetWindowSize() const noexcept { return window_size; } @@ -82,13 +90,16 @@ namespace Vulkan { bool ImportHostPointer(void *base, size_t size); - void ImportHardwareBuffers(std::span hardware_buffers, - size_t hardware_buffer_window, size_t size); + bool ImportHardwareBuffers(std::span hardware_buffers, + size_t hardware_buffer_window, size_t hardware_buffer_base, + size_t size); const Device &device; std::vector windows; VkDeviceSize window_size{}; size_t imported_size{}; + size_t base_offset{}; + bool foreign_ownership{}; }; /// Memory allocator container.