From b6e509116007bafa58ef368c8b5d11d57d75c043 Mon Sep 17 00:00:00 2001 From: Exverge Date: Tue, 28 Jul 2026 15:53:21 -0400 Subject: [PATCH] [common/core] Reimplement virtual buffers --- src/common/CMakeLists.txt | 4 +- src/common/fiber.cpp | 1 - src/common/host_memory.cpp | 39 ++++- src/common/host_memory.h | 3 +- src/common/page_table.cpp | 2 +- src/common/page_table.h | 4 +- src/common/sparse_large_vector.cpp | 135 ++++++++++++++++ src/common/sparse_large_vector.h | 181 ++++++++++++++++++++++ src/common/virtual_buffer.cpp | 44 ------ src/common/virtual_buffer.h | 84 ---------- src/core/arm/dynarmic/arm_dynarmic_32.cpp | 4 +- src/core/arm/dynarmic/arm_dynarmic_64.cpp | 4 +- src/core/device_memory_manager.h | 8 +- src/core/device_memory_manager.inc | 38 +++-- src/core/memory.cpp | 23 +-- src/video_core/memory_manager.cpp | 4 +- src/video_core/memory_manager.h | 4 +- 17 files changed, 403 insertions(+), 179 deletions(-) create mode 100644 src/common/sparse_large_vector.cpp create mode 100644 src/common/sparse_large_vector.h delete mode 100644 src/common/virtual_buffer.cpp delete mode 100644 src/common/virtual_buffer.h diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 87c4642f04..e165e19cde 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -109,6 +109,8 @@ add_library( settings_setting.h slot_vector.h socket_types.h + sparse_large_vector.cpp + sparse_large_vector.h spin_lock.h stb.cpp stb.h @@ -136,8 +138,6 @@ add_library( uuid.cpp uuid.h vector_math.h - virtual_buffer.cpp - virtual_buffer.h zstd_compression.cpp zstd_compression.h fs/ryujinx_compat.h fs/ryujinx_compat.cpp diff --git a/src/common/fiber.cpp b/src/common/fiber.cpp index 69eca732eb..0e7f28c5b9 100644 --- a/src/common/fiber.cpp +++ b/src/common/fiber.cpp @@ -9,7 +9,6 @@ #include "common/assert.h" #include "common/fiber.h" -#include "common/virtual_buffer.h" #include diff --git a/src/common/host_memory.cpp b/src/common/host_memory.cpp index 445d34c444..f438f3cb1e 100644 --- a/src/common/host_memory.cpp +++ b/src/common/host_memory.cpp @@ -179,6 +179,14 @@ public: Release(); } + void* Allocate(size_t size) { + auto* ptr = VirtualAlloc(nullptr, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); + if (ptr == nullptr) { + LOG_CRITICAL(HW_Memory, "Failed to allocate fallback buffer with size {:#x}, error {}", size, GetLastError()); + } + return VirtualAlloc(nullptr, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); + } + void Map(size_t virtual_offset, size_t host_offset, size_t length, MemoryPermission perms) { std::unique_lock lock{placeholder_mutex}; if (!IsNiechePlaceholder(virtual_offset, length)) { @@ -571,6 +579,14 @@ public: Release(); } + void* Allocate(size_t size) { + auto* ptr = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + if (ptr == MAP_FAILED) { + LOG_CRITICAL(HW_Memory, "Failed to allocate fallback buffer with size {:#x}, {}", size, strerror(errno)); + } + return ptr; + } + void Map(size_t virtual_offset, size_t host_offset, size_t length, MemoryPermission perms) { // Intersect the range with our address space. AdjustMap(&virtual_offset, &length); @@ -691,8 +707,7 @@ HostMemory::HostMemory(size_t backing_size_, size_t virtual_size_) { #if defined(__OPENORBIS__) || defined(__managarm__) LOG_WARNING(HW_Memory, "Platform doesn't support fastmem"); - fallback_buffer.emplace(backing_size); - backing_base = fallback_buffer->data(); + backing_base = malloc(backing_size); virtual_base = nullptr; #else // Try to allocate a fastmem arena. @@ -707,16 +722,28 @@ HostMemory::HostMemory(size_t backing_size_, size_t virtual_size_) virtual_base_offset = virtual_base - impl->virtual_base; } } else { - impl.reset(); LOG_WARNING(HW_Memory, "Platform can support fastmem, but can't create it"); - fallback_buffer.emplace(backing_size); - backing_base = fallback_buffer->data(); + fallback_buffer = true; + backing_base = static_cast(impl->Allocate(backing_size)); virtual_base = nullptr; + impl.reset(); } #endif } -HostMemory::~HostMemory() = default; +HostMemory::~HostMemory() { +#if defined(__OPENORBIS__) || defined(__managarm__) + free(backing_base); +#elif _WIN32 + if (fallback_buffer) { + VirtualFree(backing_base, backing_size, MEM_RELEASE); + } +#else + if (fallback_buffer) { + munmap(backing_base, backing_size); + } +#endif +} HostMemory::HostMemory(HostMemory&&) noexcept = default; diff --git a/src/common/host_memory.h b/src/common/host_memory.h index 24f4670732..c85ea1ffab 100644 --- a/src/common/host_memory.h +++ b/src/common/host_memory.h @@ -10,7 +10,6 @@ #include #include "common/common_funcs.h" #include "common/common_types.h" -#include "common/virtual_buffer.h" namespace Common { @@ -86,7 +85,7 @@ private: u8* virtual_base{}; size_t virtual_base_offset{}; // Windows requires it for kernels whom lack proper support for some functions! - std::optional> fallback_buffer; + bool fallback_buffer; }; } // namespace Common diff --git a/src/common/page_table.cpp b/src/common/page_table.cpp index d34ba89993..740aa0660f 100644 --- a/src/common/page_table.cpp +++ b/src/common/page_table.cpp @@ -43,7 +43,7 @@ bool PageTable::ContinueTraversal(TraversalEntry* out_entry, TraversalContext* c void PageTable::Resize(std::size_t address_space_width_in_bits, std::size_t page_size_in_bits) { auto const num_page_table_entries = 1ULL << (address_space_width_in_bits - page_size_in_bits); - entries.resize(num_page_table_entries); + entries.ResizeAndClear(num_page_table_entries); current_address_space_width_in_bits = address_space_width_in_bits; page_size = 1ULL << page_size_in_bits; } diff --git a/src/common/page_table.h b/src/common/page_table.h index 0dfd152331..4965aee8c1 100644 --- a/src/common/page_table.h +++ b/src/common/page_table.h @@ -9,8 +9,8 @@ #include #include "common/common_types.h" +#include "common/sparse_large_vector.h" #include "common/typed_address.h" -#include "common/virtual_buffer.h" namespace Common { @@ -139,7 +139,7 @@ struct PageTable { u64 addr; u64 padding; }; - VirtualBuffer entries; + SparseLargeVector entries; static_assert(sizeof(PageEntryData) == 32); u8* fastmem_arena{}; diff --git a/src/common/sparse_large_vector.cpp b/src/common/sparse_large_vector.cpp new file mode 100644 index 0000000000..d63ee28331 --- /dev/null +++ b/src/common/sparse_large_vector.cpp @@ -0,0 +1,135 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +/* virtual_buffer.cpp */ +// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#ifdef _WIN32 +#include +#include +#else +#include +#endif + +#include "common/alignment.h" +#include "common/assert.h" +#include "common/sparse_large_vector.h" + +namespace Common { + +#ifdef _WIN32 +static std::vector> vector_regions; + +// Workaround for handling non-commited memory accessed by Dynarmic; usually result of an error +static LONG WINAPI FakePageFaultHandler(PEXCEPTION_POINTERS info) { + DWORD code = info->ExceptionRecord->ExceptionCode; + u64 exception_addr = reinterpret_cast(info->ExceptionRecord->ExceptionAddress); + + if (code != EXCEPTION_ACCESS_VIOLATION) { + // Not our problem + return EXCEPTION_CONTINUE_SEARCH; + } + + u64 addr = 0, addr2 = 0; + + for (auto region: vector_regions) { + auto addr_shifted = exception_addr >> HostPageBits; + if (region.first <= addr_shifted && addr_shifted <= region.second) { + addr = addr_shifted; + } + + // Page-boundary accesses + if (auto addr_ = (exception_addr + 0x40) >> HostPageBits; addr_ != addr_shifted && region.first <= addr_ && addr_ <= region.second) { + addr2 = addr_; + } + } + + if (addr == 0 && addr2 == 0) { + // Not our problem + return EXCEPTION_CONTINUE_SEARCH; + } + + LOG_ERROR(HW_Memory, "Accessing an unallocated region of a LargeVector at {:#x}; this shouldn't happen and is likely a Dynarmic error!", exception_addr); + + // Commit this region + if (addr != 0) { + if (!CommitVectorPage(addr << HostPageBits, false)) { + return EXCEPTION_CONTINUE_SEARCH; + } + } + // Commit next region if needed + if (addr2 != 0) { + if (!CommitVectorPage(addr2 << HostPageBits, false)) { + return EXCEPTION_CONTINUE_SEARCH; + } + } + + return EXCEPTION_CONTINUE_EXECUTION; +} + +bool CommitVectorPage(uintptr_t addr, bool write) noexcept { + MEMORY_BASIC_INFORMATION info {}; + auto res = VirtualQuery(reinterpret_cast(addr), &info, sizeof(info)); + if (res == 0) { + LOG_CRITICAL(HW_Memory, "Failed to query large buffer region at {:#x} with error {}, will try committing anyway", addr, GetLastError()); + } else if (info.State != MEM_RESERVE) { + LOG_ERROR(HW_Memory, "Tried to commit an unreserved large buffer region at {:#x} that is not mapped or is already committed (state {:#x})", addr, info.State); + return false; + } + + auto perm = write ? PAGE_READWRITE : PAGE_READONLY; + void* res2 = VirtualAlloc(reinterpret_cast(addr), HostPageSize, MEM_COMMIT, perm); + if (res2 == nullptr) { + LOG_ERROR(HW_Memory, "Failed to commit large buffer region at {:#x}, error {}", addr, GetLastError()); + return false; + } + + return true; +} +#endif + +void* AllocateMemoryPages(std::size_t size) noexcept { + if (auto page = HostPageSize; size % page != 0) { + LOG_WARNING(HW_Memory, "Allocating unaligned large vector with size {:#x}; aligning to {} page size", size, page); + size = AlignUp(size, page); + } + +#ifdef _WIN32 + // We will never use this memory entirely so instead of committing it up front let's just reserve it and commit each page individually + void* base = VirtualAlloc(nullptr, size, MEM_RESERVE, PAGE_READWRITE); + + if (base != nullptr) { + vector_regions.emplace_back(reinterpret_cast(base), reinterpret_cast(base) + size); + + static std::once_flag flag; + std::call_once(flag, []() { AddVectoredExceptionHandler(1, FakePageFaultHandler); }); + } else { + // Try committing everything instead?? + LOG_WARNING(HW_Memory, "Failed to reserve large vector region with error {}, trying to commit instead..", GetLastError()); + base = VirtualAlloc(nullptr, size, MEM_COMMIT, PAGE_READWRITE); + } + ASSERT_MSG(base, "Failed to reserve {:#x} sized region with error {}", size, GetLastError()); +#else + void* base = mmap(nullptr, size, PROT_READ, MAP_ANON | MAP_PRIVATE, -1, 0); + if (base == MAP_FAILED) + base = nullptr; + ASSERT_MSG(base, "Failed to allocate {:#x} sized region with error {}", size, strerror(errno)); +#endif + return base; +} + +void FreeMemoryPages(void* base, [[maybe_unused]] std::size_t size) noexcept { + if (auto page = HostPageSize; size % page != 0) { + size = AlignUp(size, page); + } + if (!base) + return; +#ifdef _WIN32 + ASSERT(VirtualFree(base, 0, MEM_RELEASE)); +#else + ASSERT(munmap(base, size) == 0); +#endif +} + +} // namespace Common diff --git a/src/common/sparse_large_vector.h b/src/common/sparse_large_vector.h new file mode 100644 index 0000000000..b1e9636840 --- /dev/null +++ b/src/common/sparse_large_vector.h @@ -0,0 +1,181 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +/* virtual_buffer.h */ +// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#pragma once + +#include +#include +#include +#include + +#ifndef _WIN32 +#include +#include +#endif + +#include "common/assert.h" + +namespace Common { + +#ifdef _WIN32 +constexpr u64 HostPageSize = 0x1000; +constexpr u64 HostPageBits = 12; +constexpr u64 HostPageMask = ~(HostPageSize - 1); +bool CommitVectorPage(uintptr_t addr, bool write) noexcept; +#else +const u64 HostPageSize = sysconf(_SC_PAGESIZE); +const u64 HostPageBits = std::countr_zero(HostPageSize); +const u64 HostPageMask = ~(HostPageSize - 1); +#endif + +void* AllocateMemoryPages(std::size_t size) noexcept; +void FreeMemoryPages(void* base, std::size_t size) noexcept; + +/// A large page-aligned buffer that has optimized memory usage for zero-writes. +template +requires std::is_trivially_copyable_v +class SparseLargeVector final { +public: + constexpr SparseLargeVector() = default; + + explicit SparseLargeVector(std::size_t count) noexcept + : alloc_size{count * sizeof(T)} + { + base_ptr = static_cast(AllocateMemoryPages(alloc_size)); + + // each item in vector holds information for 64 pages + auto denom = HostPageSize * 64; + committed_pages = std::vector>((alloc_size + denom - 1) / denom); + } + + ~SparseLargeVector() noexcept { + FreeMemoryPages(base_ptr, alloc_size); + } + + SparseLargeVector(const SparseLargeVector&) = delete; + SparseLargeVector& operator=(const SparseLargeVector&) = delete; + + SparseLargeVector(SparseLargeVector&& other) noexcept + : alloc_size{std::exchange(other.alloc_size, 0)} + , base_ptr{std::exchange(other.base_ptr, nullptr)} + , committed_pages{std::exchange(other.base_ptr, nullptr)} + {} + + SparseLargeVector& operator=(SparseLargeVector&& other) noexcept { + alloc_size = std::exchange(other.alloc_size, 0); + base_ptr = std::exchange(other.base_ptr, nullptr); + committed_pages = std::exchange(other.base_ptr, nullptr); + return *this; + } + + void ResizeAndClear(std::size_t count) noexcept { + if (auto const new_size = count * sizeof(T); new_size != alloc_size) { + FreeMemoryPages(base_ptr, alloc_size); + alloc_size = new_size; + base_ptr = static_cast(AllocateMemoryPages(alloc_size)); + + auto denom = HostPageSize * 64; + committed_pages = std::vector>((alloc_size + denom - 1) / denom); + } + } + + /// Returns a pointer to the value of the requested index if that page has been allocated, or otherwise return nullptr. + T* GetNoFault(std::size_t index) const noexcept { + if (!IsCommittedPage(index)) { + return nullptr; + } + return &base_ptr[index]; + } + + /// Returns a reference to the value of the requested index and allocates memory if needed. + T& GetAndFault(std::size_t index) noexcept { + if (index > alloc_size / sizeof(T)) { + UNREACHABLE_MSG("Out of bounds RW access on SparseLargeVector @ {}", index); + } + + if (!IsCommittedPage(index)) { + CommitPage(index); + } + return base_ptr[index]; + } + + /// Returns a reference to the value of the requested index if initialized, or will otherwise return a zero-initialized object. + const T& GetOrDefault(std::size_t index) const { +#ifdef _WIN32 + if (!IsCommittedPage(index)) { + return *reinterpret_cast(&default_val); + } +#endif + // On non-Windows, OS page table should optimize this by pointing to a zero page if unallocated. + return base_ptr[index]; + } + + void Set(std::size_t index, const T& value) noexcept { + if (index > alloc_size / sizeof(T)) { + LOG_CRITICAL(Common_Memory, "Out of bounds write on SparseLargeVector @ {}", index); + return; + } + if (!IsCommittedPage(index)) + CommitPage(index); + base_ptr[index] = value; + } + + void Zero(std::size_t index) noexcept { + if (!IsCommittedPage(index)) { + return; + } + // reinterpret_cast because C++ doesn't like memset'ing, but this should be valid + // because of std::is_trivially_copyable_v + std::memset(reinterpret_cast(&base_ptr[index]), 0, sizeof(T)); + } + + [[nodiscard]] constexpr const T& operator[](std::size_t index) const noexcept { + return GetOrDefault(index); + } + + [[nodiscard]] constexpr const T* data() const noexcept { + return base_ptr; + } + + [[nodiscard]] constexpr std::size_t size() const noexcept { + return alloc_size / sizeof(T); + } + +private: + [[nodiscard]] constexpr bool IsCommittedPage(std::size_t index) const noexcept { + if (index > alloc_size / sizeof(T)) { + LOG_CRITICAL(Common_Memory, "Out of bounds access on large vector @ {}", index); + return false; + } + + auto page = (index * sizeof(T)) >> HostPageBits; + auto val = committed_pages[page >> 6].load(std::memory_order_acquire); + return (val >> (page & 63)) & 1; + } + + constexpr void CommitPage(std::size_t index) noexcept { + auto page_index = (index * sizeof(T)) >> HostPageBits; + auto page = reinterpret_cast(base_ptr + index) & HostPageMask; +#ifdef _WIN32 + CommitVectorPage(page, true); +#else + mprotect(reinterpret_cast(page), HostPageSize, PROT_READ | PROT_WRITE); +#endif + + committed_pages[page_index >> 6].fetch_or(1ULL << (page_index & 63), std::memory_order_release); + } + + std::size_t alloc_size{}; + T* base_ptr{}; + + std::vector> committed_pages{}; +#ifdef _WIN32 + const std::array default_val{}; +#endif +}; + +} // namespace Common diff --git a/src/common/virtual_buffer.cpp b/src/common/virtual_buffer.cpp deleted file mode 100644 index 8a4265e3e6..0000000000 --- a/src/common/virtual_buffer.cpp +++ /dev/null @@ -1,44 +0,0 @@ -// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project -// SPDX-License-Identifier: GPL-3.0-or-later - -// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project -// SPDX-License-Identifier: GPL-2.0-or-later - -#ifdef _WIN32 -#include -#else -#include -#endif - -#include "common/assert.h" -#include "common/virtual_buffer.h" - -namespace Common { - -void* AllocateMemoryPages(std::size_t size) noexcept { -#ifdef _WIN32 - void* base = VirtualAlloc(nullptr, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); - if (base == nullptr) { - // Probably failing to reserve is less likely than failing to commit - base = VirtualAlloc(nullptr, size, MEM_COMMIT, PAGE_READWRITE); - } -#else - void* base = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0); - if (base == MAP_FAILED) - base = nullptr; -#endif - ASSERT(base); - return base; -} - -void FreeMemoryPages(void* base, [[maybe_unused]] std::size_t size) noexcept { - if (!base) - return; -#ifdef _WIN32 - ASSERT(VirtualFree(base, 0, MEM_RELEASE)); -#else - ASSERT(munmap(base, size) == 0); -#endif -} - -} // namespace Common diff --git a/src/common/virtual_buffer.h b/src/common/virtual_buffer.h deleted file mode 100644 index d6386e2a4d..0000000000 --- a/src/common/virtual_buffer.h +++ /dev/null @@ -1,84 +0,0 @@ -// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project -// SPDX-License-Identifier: GPL-3.0-or-later - -// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project -// SPDX-License-Identifier: GPL-2.0-or-later - -#pragma once - -#include - -namespace Common { - -void* AllocateMemoryPages(std::size_t size) noexcept; -void FreeMemoryPages(void* base, std::size_t size) noexcept; - -template -class VirtualBuffer final { -public: - // TODO: Uncomment this and change Common::PageTable::PageInfo to be trivially constructible - // using std::atomic_ref once libc++ has support for it - // static_assert( - // std::is_trivially_constructible_v, - // "T must be trivially constructible, as non-trivial constructors will not be executed " - // "with the current allocator"); - - constexpr VirtualBuffer() = default; - explicit VirtualBuffer(std::size_t count) noexcept - : alloc_size{count * sizeof(T)} - { - base_ptr = reinterpret_cast(AllocateMemoryPages(alloc_size)); - } - - ~VirtualBuffer() noexcept { - FreeMemoryPages(base_ptr, alloc_size); - } - - VirtualBuffer(const VirtualBuffer&) = delete; - VirtualBuffer& operator=(const VirtualBuffer&) = delete; - - VirtualBuffer(VirtualBuffer&& other) noexcept - : alloc_size{std::exchange(other.alloc_size, 0)} - , base_ptr{std::exchange(other.base_ptr, nullptr)} - {} - - VirtualBuffer& operator=(VirtualBuffer&& other) noexcept { - alloc_size = std::exchange(other.alloc_size, 0); - base_ptr = std::exchange(other.base_ptr, nullptr); - return *this; - } - - void resize(std::size_t count) noexcept { - if (auto const new_size = count * sizeof(T); new_size != alloc_size) { - FreeMemoryPages(base_ptr, alloc_size); - alloc_size = new_size; - base_ptr = reinterpret_cast(AllocateMemoryPages(alloc_size)); - } - } - - [[nodiscard]] constexpr const T& operator[](std::size_t index) const noexcept { - return base_ptr[index]; - } - - [[nodiscard]] constexpr T& operator[](std::size_t index) noexcept { - return base_ptr[index]; - } - - [[nodiscard]] constexpr T* data() noexcept { - return base_ptr; - } - - [[nodiscard]] constexpr const T* data() const noexcept { - return base_ptr; - } - - [[nodiscard]] constexpr std::size_t size() const noexcept { - return alloc_size / sizeof(T); - } - -private: - std::size_t alloc_size{}; - T* base_ptr{}; -}; - -} // namespace Common diff --git a/src/core/arm/dynarmic/arm_dynarmic_32.cpp b/src/core/arm/dynarmic/arm_dynarmic_32.cpp index e434a31d5d..c9551c01d1 100644 --- a/src/core/arm/dynarmic/arm_dynarmic_32.cpp +++ b/src/core/arm/dynarmic/arm_dynarmic_32.cpp @@ -175,7 +175,9 @@ void ArmDynarmic32::MakeJit(Common::PageTable* page_table) { constexpr size_t PageLog2Stride = 5; static_assert(1 << PageLog2Stride == sizeof(Common::PageTable::PageEntryData)); - config.page_table = reinterpret_cast*>(page_table->entries.data()); + // Dynarmic will not write to the page table, const_cast is safe here + config.page_table = reinterpret_cast*>( + const_cast(page_table->entries.data())); config.page_table_pointer_mask_bits = Common::PageTable::ATTRIBUTE_BITS; config.page_table_log2_stride = PageLog2Stride; config.absolute_offset_page_table = true; diff --git a/src/core/arm/dynarmic/arm_dynarmic_64.cpp b/src/core/arm/dynarmic/arm_dynarmic_64.cpp index e6c7e61265..1a27e8f381 100644 --- a/src/core/arm/dynarmic/arm_dynarmic_64.cpp +++ b/src/core/arm/dynarmic/arm_dynarmic_64.cpp @@ -214,7 +214,9 @@ void ArmDynarmic64::MakeJit(Common::PageTable* page_table, std::size_t address_s constexpr size_t PageLog2Stride = 5; static_assert(1 << PageLog2Stride == sizeof(Common::PageTable::PageEntryData)); - config.page_table = reinterpret_cast(page_table->entries.data()); + // Dynarmic will not write to the page table, const_cast is safe here + config.page_table = reinterpret_cast( + const_cast(page_table->entries.data())); config.page_table_address_space_bits = std::uint32_t(address_space_bits); config.page_table_pointer_mask_bits = Common::PageTable::ATTRIBUTE_BITS; config.page_table_log2_stride = PageLog2Stride; diff --git a/src/core/device_memory_manager.h b/src/core/device_memory_manager.h index 3d97fdcc5c..0ef014155b 100644 --- a/src/core/device_memory_manager.h +++ b/src/core/device_memory_manager.h @@ -18,7 +18,7 @@ #include "common/common_types.h" #include "common/range_mutex.h" #include "common/scratch_buffer.h" -#include "common/virtual_buffer.h" +#include "common/sparse_large_vector.h" namespace Core { @@ -178,8 +178,8 @@ private: u32 continuity_tracker; u32 compressed_physical_ptr; }; - Common::VirtualBuffer compressed_device_addr; - Common::VirtualBuffer tracked_entries; + Common::SparseLargeVector compressed_device_addr; + Common::SparseLargeVector tracked_entries; // Process memory interfaces @@ -201,7 +201,7 @@ private: } void InsertCPUBacking(size_t page_index, VAddr address, Asid asid) { - tracked_entries[page_index].cpu_backing_address = address | (asid.id << asid_start_bit); + tracked_entries.GetAndFault(page_index).cpu_backing_address = address | (asid.id << asid_start_bit); } std::array t_slot{}; diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index 9866f3a4f7..484f5bf5e7 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -179,14 +179,17 @@ DeviceMemoryManager::DeviceMemoryManager(const DeviceMemory& device_memo cached_pages = std::make_unique(); const size_t total_virtual = device_as_size >> Memory::YUZU_PAGEBITS; + + auto virtual_entry = TrackedEntry {}; + virtual_entry.compressed_physical_ptr = 0; + virtual_entry.continuity_tracker = 1; + virtual_entry.cpu_backing_address = 0; for (size_t i = 0; i < total_virtual; i++) { - tracked_entries[i].compressed_physical_ptr = 0; - tracked_entries[i].continuity_tracker = 1; - tracked_entries[i].cpu_backing_address = 0; + tracked_entries.Set(i, virtual_entry); } const size_t total_phys = 1ULL << ((Settings::values.memory_layout_mode.GetValue() == Settings::MemoryLayout::Memory_4Gb ? physical_min_bits : physical_max_bits) - Memory::YUZU_PAGEBITS); for (size_t i = 0; i < total_phys; i++) { - compressed_device_addr[i] = 0; + compressed_device_addr.Zero(i); } } @@ -224,22 +227,24 @@ void DeviceMemoryManager::Map(DAddr address, VAddr virtual_address, size const VAddr new_vaddress = virtual_address + i * Memory::YUZU_PAGESIZE; auto* ptr = process_memory->GetPointerSilent(Common::ProcessAddress(new_vaddress)); if (ptr == nullptr) [[unlikely]] { - tracked_entries[start_page_d + i].compressed_physical_ptr = 0; + if (auto v = tracked_entries.GetNoFault(start_page_d + i); v) { + v->compressed_physical_ptr = 0; + } continue; } auto phys_addr = static_cast(GetRawPhysicalAddr(ptr) >> Memory::YUZU_PAGEBITS) + 1U; - tracked_entries[start_page_d + i].compressed_physical_ptr = phys_addr; + tracked_entries.GetAndFault(start_page_d + i).compressed_physical_ptr = phys_addr; InsertCPUBacking(start_page_d + i, new_vaddress, asid); const u32 base_dev = compressed_device_addr[phys_addr - 1U]; const u32 new_dev = static_cast(start_page_d + i); if (base_dev == 0) [[likely]] { - compressed_device_addr[phys_addr - 1U] = new_dev; + compressed_device_addr.GetAndFault(phys_addr - 1U) = new_dev; continue; } u32 start_id = base_dev & MULTI_MASK; if ((base_dev >> MULTI_FLAG_BITS) == 0) { start_id = impl->multi_dev_address.Register(base_dev); - compressed_device_addr[phys_addr - 1U] = MULTI_FLAG | start_id; + compressed_device_addr.GetAndFault(phys_addr - 1U) = MULTI_FLAG | start_id; } impl->multi_dev_address.Register(new_dev, start_id); } @@ -256,23 +261,24 @@ void DeviceMemoryManager::Unmap(DAddr address, size_t size) { device_inter->InvalidateRegion(address, size); std::scoped_lock lk(mapping_guard); for (size_t i = 0; i < num_pages; i++) { - auto phys_addr = tracked_entries[start_page_d + i].compressed_physical_ptr; - tracked_entries[start_page_d + i].compressed_physical_ptr = 0; - tracked_entries[start_page_d + i].cpu_backing_address = 0; + auto& entry = tracked_entries.GetAndFault(start_page_d + i); + auto phys_addr = entry.compressed_physical_ptr; + entry.compressed_physical_ptr = 0; + entry.cpu_backing_address = 0; if (phys_addr != 0) [[likely]] { const u32 base_dev = compressed_device_addr[phys_addr - 1U]; if ((base_dev >> MULTI_FLAG_BITS) == 0) [[likely]] { - compressed_device_addr[phys_addr - 1] = 0; + compressed_device_addr.Zero(phys_addr - 1); continue; } const auto [more_entries, new_start] = impl->multi_dev_address.Unregister( static_cast(start_page_d + i), base_dev & MULTI_MASK); if (!more_entries) { - compressed_device_addr[phys_addr - 1] = - impl->multi_dev_address.ReleaseEntry(new_start); + compressed_device_addr.Set(phys_addr - 1, + impl->multi_dev_address.ReleaseEntry(new_start)); continue; } - compressed_device_addr[phys_addr - 1] = new_start | MULTI_FLAG; + compressed_device_addr.Set(phys_addr - 1, new_start | MULTI_FLAG); } } t_slot = {}; @@ -296,7 +302,7 @@ void DeviceMemoryManager::TrackContinuityImpl(DAddr address, VAddr virtu page_count = 1; } last_ptr = new_ptr; - tracked_entries[start_page_d + index].continuity_tracker = static_cast(page_count); + tracked_entries.GetAndFault(start_page_d + index).continuity_tracker = static_cast(page_count); } } template diff --git a/src/core/memory.cpp b/src/core/memory.cpp index a5e6b8592c..1e6c6b6bac 100644 --- a/src/core/memory.cpp +++ b/src/core/memory.cpp @@ -417,7 +417,7 @@ struct Memory::Impl { // Page is already marked. break; case Common::PageType::Memory: - current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(0, Common::PageType::DebugMemory); + current_page_table->entries.GetAndFault(vaddr >> YUZU_PAGEBITS).ptr.Store(0, Common::PageType::DebugMemory); break; default: UNREACHABLE(); @@ -434,7 +434,7 @@ struct Memory::Impl { break; case Common::PageType::DebugMemory: { u8* const pointer = GetPointerFromDebugMemory(vaddr & ~YUZU_PAGEMASK); - current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(uintptr_t(pointer) - (vaddr & ~YUZU_PAGEMASK), Common::PageType::Memory); + current_page_table->entries.GetAndFault(vaddr >> YUZU_PAGEBITS).ptr.Store(uintptr_t(pointer) - (vaddr & ~YUZU_PAGEMASK), Common::PageType::Memory); break; } default: @@ -477,7 +477,7 @@ struct Memory::Impl { break; case Common::PageType::DebugMemory: case Common::PageType::Memory: - current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(0, Common::PageType::RasterizerCachedMemory); + current_page_table->entries.GetAndFault(vaddr >> YUZU_PAGEBITS).ptr.Store(0, Common::PageType::RasterizerCachedMemory); break; case Common::PageType::RasterizerCachedMemory: // There can be more than one GPU region mapped per CPU region, so it's common @@ -503,9 +503,9 @@ struct Memory::Impl { // It's possible that this function has been called while updating the // pagetable after unmapping a VMA. In that case the underlying VMA will no // longer exist, and we should just leave the pagetable entry blank. - current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(0, Common::PageType::Unmapped); + current_page_table->entries.GetAndFault(vaddr >> YUZU_PAGEBITS).ptr.Store(0, Common::PageType::Unmapped); } else { - current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(uintptr_t(pointer) - (vaddr & ~YUZU_PAGEMASK), Common::PageType::Memory); + current_page_table->entries.GetAndFault(vaddr >> YUZU_PAGEBITS).ptr.Store(uintptr_t(pointer) - (vaddr & ~YUZU_PAGEMASK), Common::PageType::Memory); } break; } @@ -540,9 +540,8 @@ struct Memory::Impl { "Mapping memory page without a pointer @ {:016x}", base * YUZU_PAGESIZE); while (base != end) { - page_table.entries[base].ptr.Store(0, type); - page_table.entries[base].addr = 0; - page_table.entries[base].block = 0; + // TODO: add a ZeroRegion function + page_table.entries.Zero(base); base += 1; } } else { @@ -550,9 +549,11 @@ struct Memory::Impl { while (base != end) { auto host_ptr = uintptr_t(system.DeviceMemory().GetPointer(target)) - (base << YUZU_PAGEBITS); auto backing = GetInteger(target) - (base << YUZU_PAGEBITS); - page_table.entries[base].ptr.Store(host_ptr, type); - page_table.entries[base].addr = backing; - page_table.entries[base].block = orig_base << YUZU_PAGEBITS; + + auto& entry = page_table.entries.GetAndFault(base); + entry.ptr.Store(host_ptr, type); + entry.addr = backing; + entry.block = orig_base << YUZU_PAGEBITS; ASSERT_MSG(page_table.entries[base].ptr.Pointer(), "memory mapping base yield a nullptr within the table"); diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index b9b136f6b8..d5838a387d 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -46,8 +46,8 @@ MemoryManager::MemoryManager(Core::System& system_, MaxwellDeviceMemoryManager& page_table_mask = page_table_size - 1; big_page_table_mask = big_page_table_size - 1; + big_page_table_dev.ResizeAndClear(big_page_table_size); big_entries.resize(big_page_table_size / 32, 0); - big_page_table_dev.resize(big_page_table_size); big_page_continuous.resize(big_page_table_size / continuous_bits, 0); entries.resize(page_table_size / 32, 0); } @@ -143,7 +143,7 @@ GPUVAddr MemoryManager::BigPageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr const DAddr current_dev_addr = dev_addr + offset; const auto index = PageEntryIndex(current_gpu_addr, true); const u32 sub_value = static_cast(current_dev_addr >> cpu_page_bits); - big_page_table_dev[index] = sub_value; + big_page_table_dev.Set(index, sub_value); const bool is_continuous = ([&] { uintptr_t base_ptr{ reinterpret_cast(memory.GetPointer(current_dev_addr))}; diff --git a/src/video_core/memory_manager.h b/src/video_core/memory_manager.h index f9fddd177a..08978567d2 100644 --- a/src/video_core/memory_manager.h +++ b/src/video_core/memory_manager.h @@ -17,7 +17,7 @@ #include "common/multi_level_page_table.h" #include "common/range_map.h" #include "common/scratch_buffer.h" -#include "common/virtual_buffer.h" +#include "common/sparse_large_vector.h" #include "video_core/invalidation_accumulator.h" #include "video_core/cache_types.h" #include "video_core/host1x/gpu_device_memory_manager.h" @@ -214,7 +214,7 @@ private: Common::MultiLevelPageTable page_table; Common::RangeMap kind_map; - Common::VirtualBuffer big_page_table_dev; + Common::SparseLargeVector big_page_table_dev; std::vector big_page_continuous; boost::container::small_vector, 32> page_stash{};