From 8465e9a8c9e4def2780f2bb86e3cc2709e2ee9b5 Mon Sep 17 00:00:00 2001 From: Exverge Date: Thu, 6 Aug 2026 21:03:54 -0400 Subject: [PATCH] [common/dynarmic] Rewrite virtual buffers and optimize page table allocations (#4219) --- src/common/CMakeLists.txt | 4 +- src/common/fiber.cpp | 1 - src/common/host_memory.cpp | 39 +++- src/common/host_memory.h | 20 +- src/common/page_table.cpp | 38 +--- src/common/page_table.h | 108 +++++----- src/common/sparse_large_vector.cpp | 135 +++++++++++++ src/common/sparse_large_vector.h | 188 ++++++++++++++++++ src/common/virtual_buffer.cpp | 45 ----- src/common/virtual_buffer.h | 84 -------- src/core/arm/dynarmic/arm_dynarmic_32.cpp | 8 +- src/core/arm/dynarmic/arm_dynarmic_64.cpp | 9 +- src/core/device_memory.h | 10 +- src/core/device_memory_manager.h | 10 +- src/core/device_memory_manager.inc | 46 +++-- src/core/hle/kernel/k_page_table_base.cpp | 110 ++++++---- src/core/hle/kernel/k_page_table_base.h | 13 +- src/core/memory.cpp | 79 ++++---- .../backend/arm64/a32_address_space.cpp | 1 + .../backend/arm64/a64_address_space.cpp | 1 + .../src/dynarmic/backend/arm64/emit_arm64.h | 1 + .../backend/arm64/emit_arm64_memory.cpp | 7 + .../dynarmic/backend/x64/emit_x64_memory.h | 14 ++ .../src/dynarmic/interface/A32/config.h | 9 +- .../src/dynarmic/interface/A64/config.h | 9 +- src/video_core/memory_manager.cpp | 4 +- src/video_core/memory_manager.h | 4 +- 27 files changed, 641 insertions(+), 356 deletions(-) create mode 100644 src/common/sparse_large_vector.cpp create mode 100644 src/common/sparse_large_vector.h delete mode 100644 src/common/virtual_buffer.cpp delete mode 100644 src/common/virtual_buffer.h diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 87c4642f04..e165e19cde 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -109,6 +109,8 @@ add_library( settings_setting.h slot_vector.h socket_types.h + sparse_large_vector.cpp + sparse_large_vector.h spin_lock.h stb.cpp stb.h @@ -136,8 +138,6 @@ add_library( uuid.cpp uuid.h vector_math.h - virtual_buffer.cpp - virtual_buffer.h zstd_compression.cpp zstd_compression.h fs/ryujinx_compat.h fs/ryujinx_compat.cpp diff --git a/src/common/fiber.cpp b/src/common/fiber.cpp index 69eca732eb..0e7f28c5b9 100644 --- a/src/common/fiber.cpp +++ b/src/common/fiber.cpp @@ -9,7 +9,6 @@ #include "common/assert.h" #include "common/fiber.h" -#include "common/virtual_buffer.h" #include diff --git a/src/common/host_memory.cpp b/src/common/host_memory.cpp index 59942ee391..84045d1b40 100644 --- a/src/common/host_memory.cpp +++ b/src/common/host_memory.cpp @@ -194,6 +194,14 @@ public: Release(); } + void* Allocate(size_t size) { + auto* ptr = VirtualAlloc(nullptr, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); + if (ptr == nullptr) { + LOG_CRITICAL(HW_Memory, "Failed to allocate fallback buffer with size {:#x}, error {}", size, GetLastError()); + } + return ptr; + } + void Map(size_t virtual_offset, size_t host_offset, size_t length, MemoryPermission perms) { // If we are direct mapping, intersect the range with our address space. if (virtual_base == nullptr) { @@ -662,6 +670,14 @@ public: Release(); } + void* Allocate(size_t size) { + auto* ptr = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + if (ptr == MAP_FAILED) { + LOG_CRITICAL(HW_Memory, "Failed to allocate fallback buffer with size {:#x}, {}", size, strerror(errno)); + } + return ptr; + } + void Map(size_t virtual_offset, size_t host_offset, size_t length, MemoryPermission perms) { // If we are direct mapping, intersect the range with our address space. if (virtual_base == nullptr) { @@ -767,8 +783,7 @@ HostMemory::HostMemory(size_t backing_size_, size_t virtual_size_) { #if defined(__OPENORBIS__) || defined(__managarm__) LOG_WARNING(HW_Memory, "Platform doesn't support fastmem"); - fallback_buffer.emplace(backing_size); - backing_base = fallback_buffer->data(); + backing_base = static_cast(malloc(backing_size)); virtual_base = nullptr; #else // Try to allocate a fastmem arena. @@ -784,16 +799,28 @@ HostMemory::HostMemory(size_t backing_size_, size_t virtual_size_) virtual_base_offset = virtual_base - impl->virtual_base; } } else { - impl.reset(); LOG_WARNING(HW_Memory, "Platform can support fastmem, but can't create it"); - fallback_buffer.emplace(backing_size); - backing_base = fallback_buffer->data(); + fallback_buffer = true; + backing_base = static_cast(impl->Allocate(backing_size)); virtual_base = nullptr; + impl.reset(); } #endif } -HostMemory::~HostMemory() = default; +HostMemory::~HostMemory() { +#if defined(__OPENORBIS__) || defined(__managarm__) + free(backing_base); +#elif _WIN32 + if (fallback_buffer) { + VirtualFree(backing_base, backing_size, MEM_RELEASE); + } +#else + if (fallback_buffer) { + munmap(backing_base, backing_size); + } +#endif +} HostMemory::HostMemory(HostMemory&&) noexcept = default; diff --git a/src/common/host_memory.h b/src/common/host_memory.h index 9b536cb5c2..fe5d86a4be 100644 --- a/src/common/host_memory.h +++ b/src/common/host_memory.h @@ -15,18 +15,22 @@ #include "common/common_funcs.h" #include "common/common_types.h" -#include "common/virtual_buffer.h" -#include "core/memory.h" namespace Common { -#ifndef _WIN32 -const size_t HostPageSize = sysconf(_SC_PAGESIZE); +#ifndef ARCHITECTURE_x86_64 +const u64 HostPageSize = sysconf(_SC_PAGESIZE); +const u64 HostPageBits = std::countr_zero(HostPageSize); +const u64 HostPageMask = ~(HostPageSize - 1); +const u64 GuestHostAlignment = HostPageSize / 0x1000; #else -constexpr size_t HostPageSize = 0x1000; +constexpr u64 HostPageSize = 0x1000; +constexpr u64 HostPageBits = 12; +constexpr u64 HostPageMask = ~(HostPageSize - 1); +constexpr u64 GuestHostAlignment = 1; #endif -const size_t GuestHostAlignment = HostPageSize / 0x1000; -constexpr size_t HugePageSize = 0x200000; + +constexpr u64 HugePageSize = 0x200000; enum class MemoryPermission : u32 { Read = 1 << 0, @@ -117,7 +121,7 @@ private: u8* virtual_base{}; size_t virtual_base_offset{}; // Windows requires it for kernels whom lack proper support for some functions! - std::optional> fallback_buffer; + bool fallback_buffer{false}; }; } // namespace Common diff --git a/src/common/page_table.cpp b/src/common/page_table.cpp index d34ba89993..73278cfc59 100644 --- a/src/common/page_table.cpp +++ b/src/common/page_table.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: Copyright 2019 yuzu Emulator Project @@ -13,39 +13,11 @@ PageTable::PageTable() = default; PageTable::~PageTable() noexcept = default; -bool PageTable::BeginTraversal(TraversalEntry* out_entry, TraversalContext* out_context, - Common::ProcessAddress address) const { - out_context->next_offset = GetInteger(address); - out_context->next_page = address / page_size; - - return this->ContinueTraversal(out_entry, out_context); -} - -bool PageTable::ContinueTraversal(TraversalEntry* out_entry, TraversalContext* context) const { - // Setup invalid defaults. - out_entry->phys_addr = 0; - out_entry->block_size = page_size; - // Validate that we can read the actual entry. - if (auto const page = context->next_page; page < entries.size()) { - // Validate that the entry is mapped. - if (auto const paddr = entries[page].addr; paddr != 0) { - // Populate the results. - out_entry->phys_addr = paddr + context->next_offset; - context->next_page += 1; - context->next_offset += page_size; - return true; - } - } - context->next_page += 1; - context->next_offset += page_size; - return false; -} - -void PageTable::Resize(std::size_t address_space_width_in_bits, std::size_t page_size_in_bits) { - auto const num_page_table_entries = 1ULL << (address_space_width_in_bits - page_size_in_bits); - entries.resize(num_page_table_entries); +void PageTable::Resize(std::size_t address_space_width_in_bits, std::size_t page_bits) { + auto const num_page_table_entries = 1ULL << (address_space_width_in_bits - page_bits); + entries.ResizeAndClear(num_page_table_entries); current_address_space_width_in_bits = address_space_width_in_bits; - page_size = 1ULL << page_size_in_bits; + current_page_bits = page_bits; } } // namespace Common diff --git a/src/common/page_table.h b/src/common/page_table.h index 0dfd152331..88e071a74e 100644 --- a/src/common/page_table.h +++ b/src/common/page_table.h @@ -9,22 +9,22 @@ #include #include "common/common_types.h" +#include "common/sparse_large_vector.h" #include "common/typed_address.h" -#include "common/virtual_buffer.h" namespace Common { enum class PageType : u8 { /// Page is unmapped and should cause an access error. - Unmapped, + Unmapped = 0b00, /// Page is mapped to regular memory. This is the only type you can get pointers to. - Memory, + Memory = 0b01, /// Page is mapped to regular memory, but inaccessible from CPU fastmem and must use /// the callbacks. - DebugMemory, + DebugMemory = 0b10, /// Page is mapped to regular memory, but also needs to check for rasterizer cache flushing and /// invalidation - RasterizerCachedMemory, + RasterizerCachedMemory = 0b11, }; /** @@ -44,55 +44,74 @@ struct PageTable { /// Number of bits reserved for attribute tagging. /// This can be at most the guaranteed alignment of the pointers in the page table. - static constexpr int ATTRIBUTE_BITS = 2; + static constexpr int ATTRIBUTE_BITS = 12; /** - * Pair of host pointer and page type attribute. - * This uses the lower bits of a given pointer to store the attribute tag. + * Atomic tuple of host pointer, page type, and block id. + * This uses the lower bits of a given pointer to store the attributes. * Writing and reading the pointer attribute pair is guaranteed to be atomic for the same method * call. In other words, they are guaranteed to be synchronized at all times. */ - class PageInfo { + class PageEntryData { public: + struct Data { + Data(bool marked_, PageType type_, u16 block_, u64 page_) + : marked(static_cast(marked_) & 0b1) + , type(static_cast(type_) & ((1ULL << 2) - 1)) + , block(static_cast(block_) & ((1ULL << 9) - 1)) + , page((page_ >> ATTRIBUTE_BITS) & ((1ULL << 52) - 1)) {} + u64 marked : 1; + u64 type : 2; + u64 block : 9; // TODO: is 9 bits to little? we can use the upper 8 bits if needed + u64 page : 52; + }; + + [[nodiscard]] Data Raw() const noexcept { + return std::bit_cast(data_raw.load(std::memory_order_relaxed)); + } + /// Returns the page pointer - [[nodiscard]] uintptr_t Pointer() const noexcept { - return ExtractPointer(raw.load(std::memory_order_relaxed)); + [[nodiscard]] uintptr_t Pointer(bool ignored_marked = false) const noexcept { + return ExtractPointer(std::bit_cast(data_raw.load(std::memory_order_relaxed)), ignored_marked); } /// Returns the page type attribute [[nodiscard]] PageType Type() const noexcept { - return ExtractType(raw.load(std::memory_order_relaxed)); + return static_cast(std::bit_cast(data_raw.load(std::memory_order_relaxed)).type); + } + + /// Returns the block identifier. + [[nodiscard]] u16 Block() const noexcept { + return static_cast(std::bit_cast(data_raw.load(std::memory_order_relaxed)).block); } /// Returns the page pointer and attribute pair, extracted from the same atomic read - [[nodiscard]] std::pair PointerType() const noexcept { - const uintptr_t non_atomic_raw = raw.load(std::memory_order_relaxed); - return {ExtractPointer(non_atomic_raw), ExtractType(non_atomic_raw)}; + [[nodiscard]] std::tuple PointerTypeBlock(bool ignore_marked = false) const noexcept { + const auto non_atomic_raw = std::bit_cast(data_raw.load(std::memory_order_relaxed)); + return {ExtractPointer(non_atomic_raw, ignore_marked), static_cast(non_atomic_raw.type), static_cast(non_atomic_raw.block)}; } - /// Returns the raw representation of the page information. - /// Use ExtractPointer and ExtractType to unpack the value. - [[nodiscard]] uintptr_t Raw() const noexcept { - return raw.load(std::memory_order_relaxed); + /// Write page info atomically + constexpr void Store(bool marked, PageType type, u16 block, uintptr_t pointer) noexcept { + data_raw.store(std::bit_cast(Data{marked, type, block, pointer})); } - /// Write a page pointer and type pair atomically - void Store(uintptr_t pointer, PageType type) noexcept { - raw.store(pointer | uintptr_t(type)); + constexpr void MarkRasterizerCached() noexcept { + data_raw.fetch_or(0b111); } - /// Unpack a pointer from a page info raw representation - [[nodiscard]] static uintptr_t ExtractPointer(uintptr_t raw) noexcept { - return raw & (~uintptr_t{0} << ATTRIBUTE_BITS); + constexpr void MarkDebug(u64 ptr, u16 block) noexcept { + Store(true, PageType::RasterizerCachedMemory, block, ptr); } - /// Unpack a page type from a page info raw representation - [[nodiscard]] static PageType ExtractType(uintptr_t raw) noexcept { - return static_cast(raw & ((uintptr_t{1} << ATTRIBUTE_BITS) - 1)); + /// Unpack a pointer from a page info raw representation + [[nodiscard]] static uintptr_t ExtractPointer(Data raw, bool ignore_marked = false) noexcept { + return raw.marked && !ignore_marked ? 0 : raw.page << ATTRIBUTE_BITS; } private: - std::atomic raw; + std::atomic data_raw; + static_assert(sizeof(Data) == sizeof(std::atomic)); }; PageTable(); @@ -100,13 +119,8 @@ struct PageTable { PageTable(const PageTable&) = delete; PageTable& operator=(const PageTable&) = delete; - - PageTable(PageTable&&) noexcept = default; - PageTable& operator=(PageTable&&) noexcept = default; - - bool BeginTraversal(TraversalEntry* out_entry, TraversalContext* out_context, - Common::ProcessAddress address) const; - bool ContinueTraversal(TraversalEntry* out_entry, TraversalContext* context) const; + PageTable(PageTable&&) noexcept = delete; + PageTable& operator=(PageTable&&) noexcept = delete; /** * Resizes the page table to be able to accommodate enough pages within @@ -121,30 +135,14 @@ struct PageTable { return current_address_space_width_in_bits; } - bool GetPhysicalAddress(Common::PhysicalAddress* out_phys_addr, - Common::ProcessAddress virt_addr) const { - if (virt_addr > (1ULL << this->GetAddressSpaceBits())) { - return false; - } - - *out_phys_addr = entries[virt_addr / page_size].addr + GetInteger(virt_addr); - return true; - } - /// Vector of memory pointers backing each page. An entry can only be non-null if the /// corresponding attribute element is of type `Memory`. - struct PageEntryData { - PageInfo ptr; - u64 block; - u64 addr; - u64 padding; - }; - VirtualBuffer entries; - static_assert(sizeof(PageEntryData) == 32); + SparseLargeVector entries; + static_assert(sizeof(PageEntryData) == 8); u8* fastmem_arena{}; std::size_t current_address_space_width_in_bits{}; - std::size_t page_size{}; + std::size_t current_page_bits{}; }; } // namespace Common diff --git a/src/common/sparse_large_vector.cpp b/src/common/sparse_large_vector.cpp new file mode 100644 index 0000000000..14b97edf2b --- /dev/null +++ b/src/common/sparse_large_vector.cpp @@ -0,0 +1,135 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +/* virtual_buffer.cpp */ +// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#ifdef _WIN32 +#include +#include +#else +#include +#endif + +#include "common/alignment.h" +#include "common/assert.h" +#include "common/sparse_large_vector.h" + +namespace Common { + +#ifdef _WIN32 +static std::vector> vector_regions {}; + +// Workaround for handling non-commited memory accessed by Dynarmic; usually result of an error +static LONG WINAPI FakePageFaultHandler(PEXCEPTION_POINTERS info) { + DWORD code = info->ExceptionRecord->ExceptionCode; + u64 exception_addr = reinterpret_cast(info->ExceptionRecord->ExceptionAddress); + + if (code != EXCEPTION_ACCESS_VIOLATION) { + // Not our problem + return EXCEPTION_CONTINUE_SEARCH; + } + + u64 addr = 0, addr2 = 0; + + for (auto region: vector_regions) { + auto addr_shifted = exception_addr >> HostPageBits; + if (region.first <= addr_shifted && addr_shifted <= region.second) { + addr = addr_shifted; + } + + // Page-boundary accesses + if (auto addr_ = (exception_addr + 0x40) >> HostPageBits; addr_ != addr_shifted && region.first <= addr_ && addr_ <= region.second) { + addr2 = addr_; + } + } + + if (addr == 0 && addr2 == 0) { + // Not our problem + return EXCEPTION_CONTINUE_SEARCH; + } + + LOG_ERROR(HW_Memory, "Accessing an unallocated region of a LargeVector at {:#x}; this shouldn't happen and is likely a Dynarmic error!", exception_addr); + + // Commit this region + if (addr != 0) { + if (!CommitVectorPage(addr << HostPageBits, false)) { + return EXCEPTION_CONTINUE_SEARCH; + } + } + // Commit next region if needed + if (addr2 != 0) { + if (!CommitVectorPage(addr2 << HostPageBits, false)) { + return EXCEPTION_CONTINUE_SEARCH; + } + } + + return EXCEPTION_CONTINUE_EXECUTION; +} + +bool CommitVectorPage(uintptr_t addr, bool write) noexcept { + MEMORY_BASIC_INFORMATION info {}; + auto res = VirtualQuery(reinterpret_cast(addr), &info, sizeof(info)); + if (res == 0) { + LOG_CRITICAL(HW_Memory, "Failed to query large buffer region at {:#x} with error {}, will try committing anyway", addr, GetLastError()); + } else if (info.State != MEM_RESERVE) { + LOG_ERROR(HW_Memory, "Tried to commit an unreserved large buffer region at {:#x} that is not mapped or is already committed (state {:#x})", addr, info.State); + return false; + } + + auto perm = write ? PAGE_READWRITE : PAGE_READONLY; + void* res2 = VirtualAlloc(reinterpret_cast(addr), HostPageSize, MEM_COMMIT, perm); + if (res2 == nullptr) { + LOG_ERROR(HW_Memory, "Failed to commit large buffer region at {:#x}, error {}", addr, GetLastError()); + return false; + } + + return true; +} +#endif + +void* AllocateMemoryPages(std::size_t size) noexcept { + if (auto page = HostPageSize; size % page != 0) { + LOG_WARNING(HW_Memory, "Allocating unaligned large vector with size {:#x}; aligning to {} page size", size, page); + size = AlignUp(size, page); + } + +#ifdef _WIN32 + // We will never use this memory entirely so instead of committing it up front let's just reserve it and commit each page individually + void* base = VirtualAlloc(nullptr, size, MEM_RESERVE, PAGE_READWRITE); + + if (base != nullptr) { + vector_regions.emplace_back(reinterpret_cast(base), reinterpret_cast(base) + size); + + static std::once_flag flag; + std::call_once(flag, []() { AddVectoredExceptionHandler(1, FakePageFaultHandler); }); + } else { + // Try committing everything instead?? + LOG_WARNING(HW_Memory, "Failed to reserve large vector region with error {}, trying to commit instead..", GetLastError()); + base = VirtualAlloc(nullptr, size, MEM_COMMIT, PAGE_READWRITE); + } + ASSERT_MSG(base, "Failed to reserve {:#x} sized region with error {}", size, GetLastError()); +#else + void* base = mmap(nullptr, size, PROT_READ, MAP_ANON | MAP_PRIVATE, -1, 0); + if (base == MAP_FAILED) + base = nullptr; + ASSERT_MSG(base, "Failed to allocate {:#x} sized region with error {}", size, strerror(errno)); +#endif + return base; +} + +void FreeMemoryPages(void* base, [[maybe_unused]] std::size_t size) noexcept { + if (auto page = HostPageSize; size % page != 0) { + size = AlignUp(size, page); + } + if (!base) + return; +#ifdef _WIN32 + ASSERT(VirtualFree(base, 0, MEM_RELEASE)); +#else + ASSERT(munmap(base, size) == 0); +#endif +} + +} // namespace Common diff --git a/src/common/sparse_large_vector.h b/src/common/sparse_large_vector.h new file mode 100644 index 0000000000..03a7aaf98d --- /dev/null +++ b/src/common/sparse_large_vector.h @@ -0,0 +1,188 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +/* virtual_buffer.h */ +// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#pragma once + +#include +#include +#include +#include + +#ifndef _WIN32 +#include +#include +#endif + +#include "common/alignment.h" +#include "common/assert.h" +#include "common/host_memory.h" + +namespace Common { + +#ifdef _WIN32 +bool CommitVectorPage(uintptr_t addr, bool write) noexcept; +#endif + +void* AllocateMemoryPages(std::size_t size) noexcept; +void FreeMemoryPages(void* base, std::size_t size) noexcept; + +/// A large page-aligned buffer that has optimized memory usage for zero-writes. +template +requires std::is_trivially_copyable_v +class SparseLargeVector final { +public: + constexpr SparseLargeVector() = default; + + explicit SparseLargeVector(std::size_t count) noexcept + : alloc_size{count * sizeof(T)} + { + base_ptr = static_cast(AllocateMemoryPages(alloc_size)); + + // each item in vector holds information for 64 pages + auto denom = HostPageSize * 64; + committed_pages = std::vector>((alloc_size + denom - 1) / denom); + } + + ~SparseLargeVector() noexcept { + FreeMemoryPages(base_ptr, alloc_size); + } + + SparseLargeVector(const SparseLargeVector&) = delete; + SparseLargeVector& operator=(const SparseLargeVector&) = delete; + SparseLargeVector(SparseLargeVector&& other) = delete; + SparseLargeVector& operator=(SparseLargeVector&& other) = delete; + + void ResizeAndClear(std::size_t count) noexcept { + if (auto const new_size = count * sizeof(T); new_size != alloc_size) { + FreeMemoryPages(base_ptr, alloc_size); + alloc_size = new_size; + base_ptr = static_cast(AllocateMemoryPages(alloc_size)); + + auto denom = HostPageSize * 64; + committed_pages = std::vector>((alloc_size + denom - 1) / denom); + } + } + + /// Returns a reference to the value of the requested index and allocates memory if needed. + T& GetAndFault(std::size_t index) noexcept { + if (index > alloc_size / sizeof(T)) { + UNREACHABLE_MSG("Out of bounds RW access on SparseLargeVector @ {}", index); + } + + if (!IsCommittedPage(index)) { + CommitPage(index); + } + return base_ptr[index]; + } + + /// Returns a reference to the value of the requested index if initialized, or will otherwise return a zero-initialized object. + const T& GetOrDefault(std::size_t index) const { +#ifdef _WIN32 + if (!IsCommittedPage(index)) { + return *reinterpret_cast(&default_val); + } +#endif + // On non-Windows, OS page table should optimize this by pointing to a zero page if unallocated. + return base_ptr[index]; + } + + void Set(std::size_t index, const T& value) noexcept { + if (index > alloc_size / sizeof(T)) { + LOG_CRITICAL(Common_Memory, "Out of bounds write on SparseLargeVector @ {}", index); + return; + } + if (!IsCommittedPage(index)) + CommitPage(index); + base_ptr[index] = value; + } + + void ZeroRegion(std::size_t start, std::size_t end_) noexcept { + u64 base = reinterpret_cast(&base_ptr[start]); + const u64 end = reinterpret_cast(&base_ptr[end_]); + + const u64 end_page = AlignUp(base, HostPageSize); + const u64 first_size = (std::min)(end_page, end) - base; + + if (IsCommittedPage(start / sizeof(T))) { + std::memset(reinterpret_cast(base), 0, first_size); + } + + if (end <= end_page) + return; + + base = end_page; + + for (u64 page = base; page < end; page += HostPageSize) { + if (!IsCommittedPage((page - reinterpret_cast(base_ptr)) / sizeof(T))) { + continue; + } + + std::memset(reinterpret_cast(page), 0, (std::min)(HostPageSize, end - page)); + } + } + + constexpr void CommitRegion(size_t index, size_t end_) { + const u64 base = static_cast(index) * sizeof(T); + const u64 end = static_cast(end_) * sizeof(T); + + for (u64 page = AlignDown(base, HostPageSize); page < end; page += HostPageSize) { + if (!IsCommittedPage(page / sizeof(T))) { + CommitPage(page / sizeof(T)); + } + } + } + + constexpr T& GetUnchecked(size_t index) { + return base_ptr[index]; + } + + [[nodiscard]] constexpr const T& operator[](std::size_t index) const noexcept { + return GetOrDefault(index); + } + + [[nodiscard]] constexpr const T* data() const noexcept { + return base_ptr; + } + + [[nodiscard]] constexpr std::size_t size() const noexcept { + return alloc_size / sizeof(T); + } + +private: + [[nodiscard]] constexpr bool IsCommittedPage(std::size_t index) const noexcept { + if (index > alloc_size / sizeof(T)) { + LOG_CRITICAL(Common_Memory, "Out of bounds access on large vector @ {}", index); + return false; + } + + auto page = (index * sizeof(T)) >> HostPageBits; + auto val = committed_pages[page >> 6].load(std::memory_order_acquire); + return (val >> (page & 63)) & 1; + } + + constexpr void CommitPage(std::size_t index) noexcept { + auto page_index = (index * sizeof(T)) >> HostPageBits; + auto page = reinterpret_cast(base_ptr + index) & HostPageMask; +#ifdef _WIN32 + CommitVectorPage(page, true); +#else + mprotect(reinterpret_cast(page), HostPageSize, PROT_READ | PROT_WRITE); +#endif + + committed_pages[page_index >> 6].fetch_or(1ULL << (page_index & 63), std::memory_order_release); + } + + std::size_t alloc_size{}; + T* base_ptr{}; + + std::vector> committed_pages{}; +#ifdef _WIN32 + const std::array default_val{}; +#endif +}; + +} // namespace Common diff --git a/src/common/virtual_buffer.cpp b/src/common/virtual_buffer.cpp deleted file mode 100644 index 3017e32775..0000000000 --- a/src/common/virtual_buffer.cpp +++ /dev/null @@ -1,45 +0,0 @@ -// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project -// SPDX-License-Identifier: GPL-3.0-or-later - -// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project -// SPDX-License-Identifier: GPL-2.0-or-later - -#ifdef _WIN32 -#include -#else -#include -#endif - -#include "common/assert.h" -#include "common/virtual_buffer.h" - -namespace Common { - -void* AllocateMemoryPages(std::size_t size) noexcept { -#ifdef _WIN32 - void* base = VirtualAlloc(nullptr, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); - if (base == nullptr) { - // Probably failing to reserve is less likely than failing to commit - base = VirtualAlloc(nullptr, size, MEM_COMMIT, PAGE_READWRITE); - } - ASSERT_MSG(base, "Failed to allocate {} pages, error {}", size, GetLastError()); -#else - void* base = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0); - if (base == MAP_FAILED) - base = nullptr; - ASSERT_MSG(base, "Failed to allocate {} pages, error {}", size, strerror(errno)); -#endif - return base; -} - -void FreeMemoryPages(void* base, [[maybe_unused]] std::size_t size) noexcept { - if (!base) - return; -#ifdef _WIN32 - ASSERT(VirtualFree(base, 0, MEM_RELEASE)); -#else - ASSERT(munmap(base, size) == 0); -#endif -} - -} // namespace Common diff --git a/src/common/virtual_buffer.h b/src/common/virtual_buffer.h deleted file mode 100644 index d6386e2a4d..0000000000 --- a/src/common/virtual_buffer.h +++ /dev/null @@ -1,84 +0,0 @@ -// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project -// SPDX-License-Identifier: GPL-3.0-or-later - -// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project -// SPDX-License-Identifier: GPL-2.0-or-later - -#pragma once - -#include - -namespace Common { - -void* AllocateMemoryPages(std::size_t size) noexcept; -void FreeMemoryPages(void* base, std::size_t size) noexcept; - -template -class VirtualBuffer final { -public: - // TODO: Uncomment this and change Common::PageTable::PageInfo to be trivially constructible - // using std::atomic_ref once libc++ has support for it - // static_assert( - // std::is_trivially_constructible_v, - // "T must be trivially constructible, as non-trivial constructors will not be executed " - // "with the current allocator"); - - constexpr VirtualBuffer() = default; - explicit VirtualBuffer(std::size_t count) noexcept - : alloc_size{count * sizeof(T)} - { - base_ptr = reinterpret_cast(AllocateMemoryPages(alloc_size)); - } - - ~VirtualBuffer() noexcept { - FreeMemoryPages(base_ptr, alloc_size); - } - - VirtualBuffer(const VirtualBuffer&) = delete; - VirtualBuffer& operator=(const VirtualBuffer&) = delete; - - VirtualBuffer(VirtualBuffer&& other) noexcept - : alloc_size{std::exchange(other.alloc_size, 0)} - , base_ptr{std::exchange(other.base_ptr, nullptr)} - {} - - VirtualBuffer& operator=(VirtualBuffer&& other) noexcept { - alloc_size = std::exchange(other.alloc_size, 0); - base_ptr = std::exchange(other.base_ptr, nullptr); - return *this; - } - - void resize(std::size_t count) noexcept { - if (auto const new_size = count * sizeof(T); new_size != alloc_size) { - FreeMemoryPages(base_ptr, alloc_size); - alloc_size = new_size; - base_ptr = reinterpret_cast(AllocateMemoryPages(alloc_size)); - } - } - - [[nodiscard]] constexpr const T& operator[](std::size_t index) const noexcept { - return base_ptr[index]; - } - - [[nodiscard]] constexpr T& operator[](std::size_t index) noexcept { - return base_ptr[index]; - } - - [[nodiscard]] constexpr T* data() noexcept { - return base_ptr; - } - - [[nodiscard]] constexpr const T* data() const noexcept { - return base_ptr; - } - - [[nodiscard]] constexpr std::size_t size() const noexcept { - return alloc_size / sizeof(T); - } - -private: - std::size_t alloc_size{}; - T* base_ptr{}; -}; - -} // namespace Common diff --git a/src/core/arm/dynarmic/arm_dynarmic_32.cpp b/src/core/arm/dynarmic/arm_dynarmic_32.cpp index e81e21ff13..4266df613d 100644 --- a/src/core/arm/dynarmic/arm_dynarmic_32.cpp +++ b/src/core/arm/dynarmic/arm_dynarmic_32.cpp @@ -172,12 +172,12 @@ void ArmDynarmic32::MakeJit(Common::PageTable* page_table) { if (page_table) { constexpr size_t PageBits = 12; constexpr size_t NumPageTableEntries = 1 << (32 - PageBits); - constexpr size_t PageLog2Stride = 5; - static_assert(1 << PageLog2Stride == sizeof(Common::PageTable::PageEntryData)); - config.page_table = reinterpret_cast*>(page_table->entries.data()); + // Dynarmic will not write to the page table, const_cast is safe here + config.page_table = reinterpret_cast*>( + const_cast(page_table->entries.data())); config.page_table_pointer_mask_bits = Common::PageTable::ATTRIBUTE_BITS; - config.page_table_log2_stride = PageLog2Stride; + config.page_table_marked_bit = 0; config.absolute_offset_page_table = true; config.detect_misaligned_access_via_page_table = 16 | 32 | 64 | 128; config.only_detect_misalignment_via_page_table_on_page_boundary = true; diff --git a/src/core/arm/dynarmic/arm_dynarmic_64.cpp b/src/core/arm/dynarmic/arm_dynarmic_64.cpp index 5a6f5b045a..3549295835 100644 --- a/src/core/arm/dynarmic/arm_dynarmic_64.cpp +++ b/src/core/arm/dynarmic/arm_dynarmic_64.cpp @@ -211,13 +211,12 @@ void ArmDynarmic64::MakeJit(Common::PageTable* page_table, std::size_t address_s // Memory if (page_table) { - constexpr size_t PageLog2Stride = 5; - static_assert(1 << PageLog2Stride == sizeof(Common::PageTable::PageEntryData)); - - config.page_table = reinterpret_cast(page_table->entries.data()); + // Dynarmic will not write to the page table, const_cast is safe here + config.page_table = reinterpret_cast( + const_cast(page_table->entries.data())); config.page_table_address_space_bits = std::uint32_t(address_space_bits); config.page_table_pointer_mask_bits = Common::PageTable::ATTRIBUTE_BITS; - config.page_table_log2_stride = PageLog2Stride; + config.page_table_marked_bit = 0; config.silently_mirror_page_table = false; config.absolute_offset_page_table = true; config.detect_misaligned_access_via_page_table = 16 | 32 | 64 | 128; diff --git a/src/core/device_memory.h b/src/core/device_memory.h index 11bf0e3268..b5103e23bc 100644 --- a/src/core/device_memory.h +++ b/src/core/device_memory.h @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + // SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later @@ -26,8 +29,11 @@ public: template Common::PhysicalAddress GetPhysicalAddr(const T* ptr) const { - return (reinterpret_cast(ptr) - - reinterpret_cast(buffer.BackingBasePointer())) + + return GetPhysicalAddr(reinterpret_cast(ptr)); + } + + Common::PhysicalAddress GetPhysicalAddr(uintptr_t ptr) const { + return (ptr - reinterpret_cast(buffer.BackingBasePointer())) + DramMemoryMap::Base; } diff --git a/src/core/device_memory_manager.h b/src/core/device_memory_manager.h index 3d97fdcc5c..b4b3b46088 100644 --- a/src/core/device_memory_manager.h +++ b/src/core/device_memory_manager.h @@ -18,7 +18,7 @@ #include "common/common_types.h" #include "common/range_mutex.h" #include "common/scratch_buffer.h" -#include "common/virtual_buffer.h" +#include "common/sparse_large_vector.h" namespace Core { @@ -178,8 +178,8 @@ private: u32 continuity_tracker; u32 compressed_physical_ptr; }; - Common::VirtualBuffer compressed_device_addr; - Common::VirtualBuffer tracked_entries; + Common::SparseLargeVector compressed_device_addr; + Common::SparseLargeVector tracked_entries; // Process memory interfaces @@ -200,8 +200,8 @@ private: return std::make_pair(asid, address); } - void InsertCPUBacking(size_t page_index, VAddr address, Asid asid) { - tracked_entries[page_index].cpu_backing_address = address | (asid.id << asid_start_bit); + constexpr void InsertCPUBacking(size_t page_index, VAddr address, Asid asid) { + tracked_entries.GetUnchecked(page_index).cpu_backing_address = address | (asid.id << asid_start_bit); } std::array t_slot{}; diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index 9866f3a4f7..4906b0715b 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -179,14 +179,14 @@ DeviceMemoryManager::DeviceMemoryManager(const DeviceMemory& device_memo cached_pages = std::make_unique(); const size_t total_virtual = device_as_size >> Memory::YUZU_PAGEBITS; + + // TODO: this is stupid, make continuity_tracker default to 0 so we can benefit from SparseLargeVector + auto virtual_entry = TrackedEntry {}; + virtual_entry.compressed_physical_ptr = 0; + virtual_entry.continuity_tracker = 1; + virtual_entry.cpu_backing_address = 0; for (size_t i = 0; i < total_virtual; i++) { - tracked_entries[i].compressed_physical_ptr = 0; - tracked_entries[i].continuity_tracker = 1; - tracked_entries[i].cpu_backing_address = 0; - } - const size_t total_phys = 1ULL << ((Settings::values.memory_layout_mode.GetValue() == Settings::MemoryLayout::Memory_4Gb ? physical_min_bits : physical_max_bits) - Memory::YUZU_PAGEBITS); - for (size_t i = 0; i < total_phys; i++) { - compressed_device_addr[i] = 0; + tracked_entries.Set(i, virtual_entry); } } @@ -220,26 +220,28 @@ void DeviceMemoryManager::Map(DAddr address, VAddr virtual_address, size size_t start_page_d = address >> Memory::YUZU_PAGEBITS; size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS; std::scoped_lock lk(mapping_guard); + + tracked_entries.CommitRegion(start_page_d, start_page_d + num_pages); for (size_t i = 0; i < num_pages; i++) { const VAddr new_vaddress = virtual_address + i * Memory::YUZU_PAGESIZE; auto* ptr = process_memory->GetPointerSilent(Common::ProcessAddress(new_vaddress)); if (ptr == nullptr) [[unlikely]] { - tracked_entries[start_page_d + i].compressed_physical_ptr = 0; + tracked_entries.GetUnchecked(start_page_d + i).compressed_physical_ptr = 0; continue; } auto phys_addr = static_cast(GetRawPhysicalAddr(ptr) >> Memory::YUZU_PAGEBITS) + 1U; - tracked_entries[start_page_d + i].compressed_physical_ptr = phys_addr; + tracked_entries.GetUnchecked(start_page_d + i).compressed_physical_ptr = phys_addr; InsertCPUBacking(start_page_d + i, new_vaddress, asid); const u32 base_dev = compressed_device_addr[phys_addr - 1U]; const u32 new_dev = static_cast(start_page_d + i); if (base_dev == 0) [[likely]] { - compressed_device_addr[phys_addr - 1U] = new_dev; + compressed_device_addr.GetAndFault(phys_addr - 1U) = new_dev; continue; } u32 start_id = base_dev & MULTI_MASK; if ((base_dev >> MULTI_FLAG_BITS) == 0) { start_id = impl->multi_dev_address.Register(base_dev); - compressed_device_addr[phys_addr - 1U] = MULTI_FLAG | start_id; + compressed_device_addr.GetAndFault(phys_addr - 1U) = MULTI_FLAG | start_id; } impl->multi_dev_address.Register(new_dev, start_id); } @@ -255,24 +257,26 @@ void DeviceMemoryManager::Unmap(DAddr address, size_t size) { size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS; device_inter->InvalidateRegion(address, size); std::scoped_lock lk(mapping_guard); + + tracked_entries.CommitRegion(start_page_d, start_page_d + num_pages); // should already be committed, but just in case for (size_t i = 0; i < num_pages; i++) { - auto phys_addr = tracked_entries[start_page_d + i].compressed_physical_ptr; - tracked_entries[start_page_d + i].compressed_physical_ptr = 0; - tracked_entries[start_page_d + i].cpu_backing_address = 0; + auto& entry = tracked_entries.GetUnchecked(start_page_d + i); + auto phys_addr = entry.compressed_physical_ptr; + entry.compressed_physical_ptr = 0; + entry.cpu_backing_address = 0; if (phys_addr != 0) [[likely]] { - const u32 base_dev = compressed_device_addr[phys_addr - 1U]; + u32& base_dev = compressed_device_addr.GetAndFault(phys_addr - 1U); if ((base_dev >> MULTI_FLAG_BITS) == 0) [[likely]] { - compressed_device_addr[phys_addr - 1] = 0; + base_dev = 0; continue; } const auto [more_entries, new_start] = impl->multi_dev_address.Unregister( static_cast(start_page_d + i), base_dev & MULTI_MASK); if (!more_entries) { - compressed_device_addr[phys_addr - 1] = - impl->multi_dev_address.ReleaseEntry(new_start); + base_dev = impl->multi_dev_address.ReleaseEntry(new_start); continue; } - compressed_device_addr[phys_addr - 1] = new_start | MULTI_FLAG; + base_dev = new_start | MULTI_FLAG; } } t_slot = {}; @@ -285,6 +289,8 @@ void DeviceMemoryManager::TrackContinuityImpl(DAddr address, VAddr virtu size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS; uintptr_t last_ptr = 0; size_t page_count = 1; + + tracked_entries.CommitRegion(start_page_d, start_page_d + num_pages); for (size_t i = num_pages; i > 0; i--) { size_t index = i - 1; const VAddr new_vaddress = virtual_address + index * Memory::YUZU_PAGESIZE; @@ -296,7 +302,7 @@ void DeviceMemoryManager::TrackContinuityImpl(DAddr address, VAddr virtu page_count = 1; } last_ptr = new_ptr; - tracked_entries[start_page_d + index].continuity_tracker = static_cast(page_count); + tracked_entries.GetUnchecked(start_page_d + index).continuity_tracker = static_cast(page_count); } } template diff --git a/src/core/hle/kernel/k_page_table_base.cpp b/src/core/hle/kernel/k_page_table_base.cpp index 2b35d16c15..575929ed44 100644 --- a/src/core/hle/kernel/k_page_table_base.cpp +++ b/src/core/hle/kernel/k_page_table_base.cpp @@ -635,6 +635,36 @@ Result KPageTableBase::CheckMemoryState(const KMemoryInfo& info, KMemoryState st R_SUCCEED(); } +bool KPageTableBase::BeginTraversal(const Common::PageTable &impl, TraversalEntry *out_entry, TraversalContext *out_context, + Common::ProcessAddress address) const { + out_context->next_offset = GetInteger(address); + out_context->next_page = GetInteger(address) >> PageBits; + + return ContinueTraversal(impl, out_entry, out_context); +} + +bool KPageTableBase::ContinueTraversal(const Common::PageTable &impl, TraversalEntry *out_entry, + TraversalContext *context) const { + // Setup invalid defaults. + out_entry->phys_addr = 0; + out_entry->block_size = PageSize; + // Validate that we can read the actual entry. + if (auto const page = context->next_page; page < impl.entries.size()) { + // Validate that the entry is mapped. + if (auto const paddr = impl.entries[page].Pointer(true); paddr != 0) { + // Populate the results and return true + out_entry->phys_addr = GetInteger(m_system.DeviceMemory().GetPhysicalAddr(paddr + context->next_offset)); + context->next_page += 1; + context->next_offset += PageSize; + return true; + } + } + context->next_page += 1; + context->next_offset += PageSize; + // Otherwise return false + return false; +} + Result KPageTableBase::CheckMemoryStateContiguous(size_t* out_blocks_needed, KProcessAddress addr, size_t size, KMemoryState state_mask, KMemoryState state, KMemoryPermission perm_mask, @@ -940,7 +970,7 @@ Result KPageTableBase::QueryMappingImpl(KProcessAddress* out, KPhysicalAddress a size_t tot_size = 0; next_valid = - impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), region_start); + BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), region_start); next_entry.block_size = (next_entry.block_size - (GetInteger(region_start) & (next_entry.block_size - 1))); @@ -976,7 +1006,7 @@ Result KPageTableBase::QueryMappingImpl(KProcessAddress* out, KPhysicalAddress a break; } - next_valid = impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); + next_valid = ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)); } // Check the last entry. @@ -1754,7 +1784,7 @@ Result KPageTableBase::MakePageGroup(KPageGroup& pg, KProcessAddress addr, size_ // Begin traversal. TraversalContext context; TraversalEntry next_entry; - R_UNLESS(impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), addr), + R_UNLESS(BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), addr), ResultInvalidCurrentMemory); // Prepare tracking variables. @@ -1764,7 +1794,7 @@ Result KPageTableBase::MakePageGroup(KPageGroup& pg, KProcessAddress addr, size_ // Iterate, adding to group as we go. while (tot_size < size) { - R_UNLESS(impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)), + R_UNLESS(ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)), ResultInvalidCurrentMemory); if (next_entry.phys_addr != (cur_addr + cur_size)) { @@ -1828,7 +1858,7 @@ bool KPageTableBase::IsValidPageGroup(const KPageGroup& pg, KProcessAddress addr // Begin traversal. TraversalContext context; TraversalEntry next_entry; - if (!impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), addr)) { + if (!BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), addr)) { return false; } @@ -1839,7 +1869,7 @@ bool KPageTableBase::IsValidPageGroup(const KPageGroup& pg, KProcessAddress addr // Iterate, comparing expected to actual. while (tot_size < size) { - if (!impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context))) { + if (!ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context))) { return false; } @@ -1896,7 +1926,7 @@ Result KPageTableBase::GetContiguousMemoryRangeWithState( // Begin a traversal. TraversalContext context; TraversalEntry cur_entry = {.phys_addr = 0, .block_size = 0}; - R_UNLESS(impl.BeginTraversal(std::addressof(cur_entry), std::addressof(context), address), + R_UNLESS(BeginTraversal(impl, std::addressof(cur_entry), std::addressof(context), address), ResultInvalidCurrentMemory); // Traverse until we have enough size or we aren't contiguous any more. @@ -1905,7 +1935,7 @@ Result KPageTableBase::GetContiguousMemoryRangeWithState( for (contig_size = cur_entry.block_size - (GetInteger(phys_address) & (cur_entry.block_size - 1)); contig_size < size; contig_size += cur_entry.block_size) { - if (!impl.ContinueTraversal(std::addressof(cur_entry), std::addressof(context))) { + if (!ContinueTraversal(impl, std::addressof(cur_entry), std::addressof(context))) { break; } if (cur_entry.phys_addr != phys_address + contig_size) { @@ -2334,7 +2364,7 @@ Result KPageTableBase::QueryPhysicalAddress(Svc::lp64::PhysicalMemoryInfo* out, TraversalContext context; TraversalEntry next_entry; bool traverse_valid = - m_impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), virt_addr); + BeginTraversal(m_impl, std::addressof(next_entry), std::addressof(context), virt_addr); R_UNLESS(traverse_valid, ResultInvalidCurrentMemory); // Set tracking variables. @@ -2345,7 +2375,7 @@ Result KPageTableBase::QueryPhysicalAddress(Svc::lp64::PhysicalMemoryInfo* out, while (true) { // Continue the traversal. traverse_valid = - m_impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); + ContinueTraversal(m_impl, std::addressof(next_entry), std::addressof(context)); if (!traverse_valid) { break; } @@ -2567,7 +2597,7 @@ Result KPageTableBase::UnmapIoRegion(KProcessAddress dst_address, KPhysicalAddre TraversalContext context; TraversalEntry next_entry; ASSERT( - impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), dst_address)); + BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), dst_address)); // Check that the physical region matches. R_UNLESS(next_entry.phys_addr == phys_addr, ResultInvalidMemoryRegion); @@ -2577,7 +2607,7 @@ Result KPageTableBase::UnmapIoRegion(KProcessAddress dst_address, KPhysicalAddre next_entry.block_size - (GetInteger(phys_addr) & (next_entry.block_size - 1)); checked_size < size; checked_size += next_entry.block_size) { // Continue the traversal. - ASSERT(impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context))); + ASSERT(ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context))); // Check that the physical region matches. R_UNLESS(next_entry.phys_addr == phys_addr + checked_size, ResultInvalidMemoryRegion); @@ -3029,7 +3059,7 @@ Result KPageTableBase::InvalidateProcessDataCache(KProcessAddress address, size_ TraversalContext context; TraversalEntry next_entry; bool traverse_valid = - impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), address); + BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), address); R_UNLESS(traverse_valid, ResultInvalidCurrentMemory); // Prepare tracking variables. @@ -3041,7 +3071,7 @@ Result KPageTableBase::InvalidateProcessDataCache(KProcessAddress address, size_ while (tot_size < size) { // Continue the traversal. traverse_valid = - impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); + ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)); R_UNLESS(traverse_valid, ResultInvalidCurrentMemory); if (next_entry.phys_addr != (cur_addr + cur_size)) { @@ -3129,7 +3159,7 @@ Result KPageTableBase::ReadDebugMemory(KProcessAddress dst_address, KProcessAddr TraversalContext context; TraversalEntry next_entry; bool traverse_valid = - impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), src_address); + BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), src_address); R_UNLESS(traverse_valid, ResultInvalidCurrentMemory); // Prepare tracking variables. @@ -3167,7 +3197,7 @@ Result KPageTableBase::ReadDebugMemory(KProcessAddress dst_address, KProcessAddr while (tot_size < size) { // Continue the traversal. traverse_valid = - impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); + ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)); ASSERT(traverse_valid); if (next_entry.phys_addr != (cur_addr + cur_size)) { @@ -3225,7 +3255,7 @@ Result KPageTableBase::WriteDebugMemory(KProcessAddress dst_address, KProcessAdd TraversalContext context; TraversalEntry next_entry; bool traverse_valid = - impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), dst_address); + BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), dst_address); R_UNLESS(traverse_valid, ResultInvalidCurrentMemory); // Prepare tracking variables. @@ -3267,7 +3297,7 @@ Result KPageTableBase::WriteDebugMemory(KProcessAddress dst_address, KProcessAdd while (tot_size < size) { // Continue the traversal. traverse_valid = - impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); + ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)); ASSERT(traverse_valid); if (next_entry.phys_addr != (cur_addr + cur_size)) { @@ -3728,7 +3758,7 @@ Result KPageTableBase::CopyMemoryFromLinearToUser( TraversalContext context; TraversalEntry next_entry; bool traverse_valid = - impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), src_addr); + BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), src_addr); ASSERT(traverse_valid); // Prepare tracking variables. @@ -3768,7 +3798,7 @@ Result KPageTableBase::CopyMemoryFromLinearToUser( while (tot_size < size) { // Continue the traversal. traverse_valid = - impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); + ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)); ASSERT(traverse_valid); if (next_entry.phys_addr != (cur_addr + cur_size)) { @@ -3822,7 +3852,7 @@ Result KPageTableBase::CopyMemoryFromLinearToKernel( TraversalContext context; TraversalEntry next_entry; bool traverse_valid = - impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), src_addr); + BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), src_addr); ASSERT(traverse_valid); // Prepare tracking variables. @@ -3845,7 +3875,7 @@ Result KPageTableBase::CopyMemoryFromLinearToKernel( while (tot_size < size) { // Continue the traversal. traverse_valid = - impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); + ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)); ASSERT(traverse_valid); if (next_entry.phys_addr != (cur_addr + cur_size)) { @@ -3902,7 +3932,7 @@ Result KPageTableBase::CopyMemoryFromUserToLinear( TraversalContext context; TraversalEntry next_entry; bool traverse_valid = - impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), dst_addr); + BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), dst_addr); ASSERT(traverse_valid); // Prepare tracking variables. @@ -3941,7 +3971,7 @@ Result KPageTableBase::CopyMemoryFromUserToLinear( while (tot_size < size) { // Continue the traversal. traverse_valid = - impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); + ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)); ASSERT(traverse_valid); if (next_entry.phys_addr != (cur_addr + cur_size)) { @@ -3997,7 +4027,7 @@ Result KPageTableBase::CopyMemoryFromKernelToLinear(KProcessAddress dst_addr, si TraversalContext context; TraversalEntry next_entry; bool traverse_valid = - impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), dst_addr); + BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), dst_addr); ASSERT(traverse_valid); // Prepare tracking variables. @@ -4020,7 +4050,7 @@ Result KPageTableBase::CopyMemoryFromKernelToLinear(KProcessAddress dst_addr, si while (tot_size < size) { // Continue the traversal. traverse_valid = - impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); + ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)); ASSERT(traverse_valid); if (next_entry.phys_addr != (cur_addr + cur_size)) { @@ -4089,10 +4119,10 @@ Result KPageTableBase::CopyMemoryFromHeapToHeap( bool traverse_valid; // Begin traversal. - traverse_valid = src_impl.BeginTraversal(std::addressof(src_next_entry), + traverse_valid = BeginTraversal(src_impl, std::addressof(src_next_entry), std::addressof(src_context), src_addr); ASSERT(traverse_valid); - traverse_valid = dst_impl.BeginTraversal(std::addressof(dst_next_entry), + traverse_valid = BeginTraversal(dst_impl, std::addressof(dst_next_entry), std::addressof(dst_context), dst_addr); ASSERT(traverse_valid); @@ -4127,7 +4157,7 @@ Result KPageTableBase::CopyMemoryFromHeapToHeap( if (ofs + cur_copy_size != size) { if (cur_src_addr + cur_min_size == cur_src_block_addr + cur_src_size) { // Continue the src traversal. - traverse_valid = src_impl.ContinueTraversal(std::addressof(src_next_entry), + traverse_valid = ContinueTraversal(src_impl, std::addressof(src_next_entry), std::addressof(src_context)); ASSERT(traverse_valid); @@ -4138,7 +4168,7 @@ Result KPageTableBase::CopyMemoryFromHeapToHeap( if (cur_dst_addr + cur_min_size == dst_next_entry.phys_addr + dst_next_entry.block_size) { // Continue the dst traversal. - traverse_valid = dst_impl.ContinueTraversal(std::addressof(dst_next_entry), + traverse_valid = ContinueTraversal(dst_impl, std::addressof(dst_next_entry), std::addressof(dst_context)); ASSERT(traverse_valid); @@ -4223,10 +4253,10 @@ Result KPageTableBase::CopyMemoryFromHeapToHeapWithoutCheckDestination( bool traverse_valid; // Begin traversal. - traverse_valid = src_impl.BeginTraversal(std::addressof(src_next_entry), + traverse_valid = BeginTraversal(src_impl, std::addressof(src_next_entry), std::addressof(src_context), src_addr); ASSERT(traverse_valid); - traverse_valid = dst_impl.BeginTraversal(std::addressof(dst_next_entry), + traverse_valid = BeginTraversal(dst_impl, std::addressof(dst_next_entry), std::addressof(dst_context), dst_addr); ASSERT(traverse_valid); @@ -4261,7 +4291,7 @@ Result KPageTableBase::CopyMemoryFromHeapToHeapWithoutCheckDestination( if (ofs + cur_copy_size != size) { if (cur_src_addr + cur_min_size == cur_src_block_addr + cur_src_size) { // Continue the src traversal. - traverse_valid = src_impl.ContinueTraversal(std::addressof(src_next_entry), + traverse_valid = ContinueTraversal(src_impl, std::addressof(src_next_entry), std::addressof(src_context)); ASSERT(traverse_valid); @@ -4272,7 +4302,7 @@ Result KPageTableBase::CopyMemoryFromHeapToHeapWithoutCheckDestination( if (cur_dst_addr + cur_min_size == dst_next_entry.phys_addr + dst_next_entry.block_size) { // Continue the dst traversal. - traverse_valid = dst_impl.ContinueTraversal(std::addressof(dst_next_entry), + traverse_valid = ContinueTraversal(dst_impl, std::addressof(dst_next_entry), std::addressof(dst_context)); ASSERT(traverse_valid); @@ -4547,7 +4577,7 @@ Result KPageTableBase::SetupForIpcServer(KProcessAddress* out_addr, size_t size, // Begin traversal. TraversalContext context; TraversalEntry next_entry; - bool traverse_valid = src_impl.BeginTraversal(std::addressof(next_entry), + bool traverse_valid = BeginTraversal(src_impl, std::addressof(next_entry), std::addressof(context), aligned_src_start); ASSERT(traverse_valid); @@ -4597,7 +4627,7 @@ Result KPageTableBase::SetupForIpcServer(KProcessAddress* out_addr, size_t size, // If the block's size was one page, we may need to continue traversal. if (cur_block_size == 0 && aligned_src_size > PageSize) { traverse_valid = - src_impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); + ContinueTraversal(src_impl, std::addressof(next_entry), std::addressof(context)); ASSERT(traverse_valid); cur_block_addr = next_entry.phys_addr; @@ -4610,7 +4640,7 @@ Result KPageTableBase::SetupForIpcServer(KProcessAddress* out_addr, size_t size, while (aligned_src_start + tot_block_size < mapping_src_end) { // Continue the traversal. traverse_valid = - src_impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); + ContinueTraversal(src_impl, std::addressof(next_entry), std::addressof(context)); ASSERT(traverse_valid); // Process the block. @@ -4653,7 +4683,7 @@ Result KPageTableBase::SetupForIpcServer(KProcessAddress* out_addr, size_t size, if (mapped_block_end + cur_block_size < aligned_src_end && cur_block_size == last_block_size) { traverse_valid = - src_impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); + ContinueTraversal(src_impl, std::addressof(next_entry), std::addressof(context)); ASSERT(traverse_valid); cur_block_addr = next_entry.phys_addr; @@ -5601,7 +5631,7 @@ Result KPageTableBase::UnmapProcessMemory(KProcessAddress dst_address, size_t si ContiguousRangeInfo(KPageTableBase& pt, KProcessAddress address, size_t size) : m_pt(pt), m_remaining_size(size) { // Begin a traversal. - ASSERT(m_pt.GetImpl().BeginTraversal(std::addressof(m_entry), + ASSERT(m_pt.BeginTraversal(m_pt.GetImpl(), std::addressof(m_entry), std::addressof(m_context), address)); // Setup tracking fields. @@ -5632,7 +5662,7 @@ Result KPageTableBase::UnmapProcessMemory(KProcessAddress dst_address, size_t si void DetermineContiguousBlockExtents() { // Continue traversing until we're not contiguous, or we have enough. while (m_cur_size < m_remaining_size) { - ASSERT(m_pt.GetImpl().ContinueTraversal(std::addressof(m_entry), + ASSERT(m_pt.ContinueTraversal(m_pt.GetImpl(), std::addressof(m_entry), std::addressof(m_context))); // If we're not contiguous, we're done. diff --git a/src/core/hle/kernel/k_page_table_base.h b/src/core/hle/kernel/k_page_table_base.h index 160e722985..887ee15df6 100644 --- a/src/core/hle/kernel/k_page_table_base.h +++ b/src/core/hle/kernel/k_page_table_base.h @@ -369,6 +369,10 @@ private: size_t num_pages, size_t alignment, size_t offset, size_t guard_pages) const; + bool BeginTraversal(const Common::PageTable& impl, TraversalEntry* out_entry, TraversalContext* out_context, + Common::ProcessAddress address) const; + bool ContinueTraversal(const Common::PageTable& impl, TraversalEntry* out_entry, TraversalContext* context) const; + Result CheckMemoryStateContiguous(size_t* out_blocks_needed, KProcessAddress addr, size_t size, KMemoryState state_mask, KMemoryState state, KMemoryPermission perm_mask, KMemoryPermission perm, @@ -473,7 +477,14 @@ private: // Validate pre-conditions. ASSERT(this->IsLockedByCurrentThread()); - return this->GetImpl().GetPhysicalAddress(out, virt_addr); + if (virt_addr > (1ULL << m_address_space_width)) { + return false; + } + + *out = m_system.DeviceMemory().GetPhysicalAddr( + this->GetImpl().entries[GetInteger(virt_addr) >> PageBits].Pointer(true) + GetInteger(virt_addr)); + + return true; } public: diff --git a/src/core/memory.cpp b/src/core/memory.cpp index 7f453a11ca..7a8cbda05e 100644 --- a/src/core/memory.cpp +++ b/src/core/memory.cpp @@ -119,8 +119,10 @@ struct Memory::Impl { } u64 protect_bytes = 0, protect_begin = 0; + + current_page_table->entries.CommitRegion(vaddr >> YUZU_PAGEBITS, (vaddr + size) >> YUZU_PAGEBITS); for (u64 addr = vaddr; addr < vaddr + size; addr += YUZU_PAGESIZE) { - const Common::PageType page_type = current_page_table->entries[addr >> YUZU_PAGEBITS].ptr.Type(); + const Common::PageType page_type = current_page_table->entries.GetUnchecked(addr >> YUZU_PAGEBITS).Type(); switch (page_type) { case Common::PageType::RasterizerCachedMemory: if (protect_bytes > 0) { @@ -141,16 +143,14 @@ struct Memory::Impl { } [[nodiscard]] u8* GetPointerFromRasterizerCachedMemory(u64 vaddr) const { - Common::PhysicalAddress const paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].addr; - if (paddr) - return system.DeviceMemory().GetPointer(paddr + vaddr); + if (u64 paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].Pointer(true); paddr) + return reinterpret_cast(paddr) + vaddr; return {}; } [[nodiscard]] u8* GetPointerFromDebugMemory(u64 vaddr) const { - const Common::PhysicalAddress paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].addr; - if (paddr != 0) - return system.DeviceMemory().GetPointer(paddr + vaddr); + if (u64 paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].Pointer(true); paddr) + return reinterpret_cast(paddr) + vaddr; return {}; } @@ -261,10 +261,12 @@ struct Memory::Impl { std::size_t page_index = addr >> YUZU_PAGEBITS; std::size_t page_offset = addr & YUZU_PAGEMASK; bool user_accessible = true; + + current_page_table->entries.CommitRegion(page_index, page_index + (size >> YUZU_PAGEBITS) + 1); while (remaining_size != 0) { const std::size_t copy_amount = (std::min)(std::size_t(YUZU_PAGESIZE) - page_offset, remaining_size); const auto current_vaddr = u64((page_index << YUZU_PAGEBITS) + page_offset); - const auto [pointer, type] = current_page_table->entries[page_index].ptr.PointerType(); + const auto [pointer, type, _] = current_page_table->entries.GetUnchecked(page_index).PointerTypeBlock(); switch (type) { case Common::PageType::Unmapped: { user_accessible = false; @@ -315,10 +317,10 @@ struct Memory::Impl { } [[nodiscard]] inline const u8* GetSpan(const VAddr addr, const std::size_t size) const noexcept { - return (current_page_table->entries[addr >> YUZU_PAGEBITS].block == current_page_table->entries[(addr + size) >> YUZU_PAGEBITS].block) ? GetPointerSilent(addr) : nullptr; + return (current_page_table->entries[addr >> YUZU_PAGEBITS].Block() == current_page_table->entries[(addr + size) >> YUZU_PAGEBITS].Block()) ? GetPointerSilent(addr) : nullptr; } [[nodiscard]] inline u8* GetSpan(const VAddr addr, const std::size_t size) noexcept { - return (current_page_table->entries[addr >> YUZU_PAGEBITS].block == current_page_table->entries[(addr + size) >> YUZU_PAGEBITS].block) ? GetPointerSilent(addr) : nullptr; + return (current_page_table->entries[addr >> YUZU_PAGEBITS].Block() == current_page_table->entries[(addr + size) >> YUZU_PAGEBITS].Block()) ? GetPointerSilent(addr) : nullptr; } bool WriteBlockImpl(const Common::ProcessAddress addr, const void* buffer, const std::size_t size, bool unsafe) { @@ -422,11 +424,14 @@ struct Memory::Impl { // The region is at a granularity of CPU pages. const u64 num_pages = ((vaddr + size - 1) >> YUZU_PAGEBITS) - (vaddr >> YUZU_PAGEBITS) + 1; + + current_page_table->entries.CommitRegion(vaddr >> YUZU_PAGEBITS, (vaddr >> YUZU_PAGEBITS) + num_pages); for (u64 i = 0; i < num_pages; ++i, vaddr += YUZU_PAGESIZE) { - const Common::PageType page_type = current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Type(); + auto& entry = current_page_table->entries.GetUnchecked(vaddr >> YUZU_PAGEBITS); + const auto [pointer, type, block] = entry.PointerTypeBlock(true); if (debug) { // Switch page type to debug if now debug - switch (page_type) { + switch (type) { case Common::PageType::Unmapped: ASSERT(false && "Attempted to mark unmapped pages as debug"); break; @@ -435,14 +440,14 @@ struct Memory::Impl { // Page is already marked. break; case Common::PageType::Memory: - current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(0, Common::PageType::DebugMemory); + entry.MarkDebug(pointer, block); break; default: UNREACHABLE(); } } else { // Switch page type to non-debug if now non-debug - switch (page_type) { + switch (type) { case Common::PageType::Unmapped: ASSERT(false && "Attempted to mark unmapped pages as non-debug"); break; @@ -451,8 +456,7 @@ struct Memory::Impl { // Don't mess with already non-debug or rasterizer memory. break; case Common::PageType::DebugMemory: { - u8* const pointer = GetPointerFromDebugMemory(vaddr & ~YUZU_PAGEMASK); - current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(uintptr_t(pointer) - (vaddr & ~YUZU_PAGEMASK), Common::PageType::Memory); + entry.Store(false, Common::PageType::Memory, block, pointer); break; } default: @@ -484,8 +488,10 @@ struct Memory::Impl { // is different). This assumes the specified GPU address region is contiguous as well. const u64 num_pages = ((vaddr + size - 1) >> YUZU_PAGEBITS) - (vaddr >> YUZU_PAGEBITS) + 1; + current_page_table->entries.CommitRegion(vaddr >> YUZU_PAGEBITS, (vaddr >> YUZU_PAGEBITS) + num_pages); for (u64 i = 0; i < num_pages; ++i, vaddr += YUZU_PAGESIZE) { - const Common::PageType page_type= current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Type(); + auto& entry = current_page_table->entries.GetUnchecked(vaddr >> YUZU_PAGEBITS); + const Common::PageType page_type = entry.Type(); if (cached) { // Switch page type to cached if now cached switch (page_type) { @@ -495,7 +501,7 @@ struct Memory::Impl { break; case Common::PageType::DebugMemory: case Common::PageType::Memory: - current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(0, Common::PageType::RasterizerCachedMemory); + entry.MarkRasterizerCached(); break; case Common::PageType::RasterizerCachedMemory: // There can be more than one GPU region mapped per CPU region, so it's common @@ -517,13 +523,13 @@ struct Memory::Impl { // that this area is already unmarked as cached. break; case Common::PageType::RasterizerCachedMemory: { - if (u8* const pointer = GetPointerFromRasterizerCachedMemory(vaddr & ~YUZU_PAGEMASK); pointer == nullptr) { + if (auto [ptr, _, block] = entry.PointerTypeBlock(true); ptr == 0) { // It's possible that this function has been called while updating the // pagetable after unmapping a VMA. In that case the underlying VMA will no // longer exist, and we should just leave the pagetable entry blank. - current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(0, Common::PageType::Unmapped); + entry.Store(false, Common::PageType::Unmapped, block, 0); } else { - current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(uintptr_t(pointer) - (vaddr & ~YUZU_PAGEMASK), Common::PageType::Memory); + entry.Store(false, Common::PageType::Memory, block, ptr); } break; } @@ -558,12 +564,7 @@ struct Memory::Impl { "Mapping memory page without a pointer @ {:016x}", base * YUZU_PAGESIZE); // TODO: remove extra pages - while (base != end) { - page_table.entries[base].ptr.Store(0, type); - page_table.entries[base].addr = 0; - page_table.entries[base].block = 0; - base += 1; - } + page_table.entries.ZeroRegion(base, end); return {false, false}; } else { std::pair out = {false, false}; @@ -577,6 +578,7 @@ struct Memory::Impl { auto e = base - off; for (u64 i = 0; i < off; ++i, ++e) { + // TODO: store data in a set in Memory::Impl, we can't use this trick anymore (probably?) if (page_table.entries[e].addr == 0 && page_table.entries[e].block == 0) { page_table.entries[e].block = (GetInteger(target) >> YUZU_PAGEBITS) - off + i; } else { @@ -599,7 +601,11 @@ struct Memory::Impl { } } - auto orig_base = base; + static std::atomic block = 0; + auto current_block = block.fetch_add(1); + ASSERT(current_block <= 512); + + page_table.entries.CommitRegion(base, end); while (base != end) { auto target_paddr = target; if (auto real_paddr = page_table.entries[base].block; real_paddr != 0 && page_table.entries[base].addr == 0) { @@ -609,12 +615,10 @@ struct Memory::Impl { target_paddr = real_paddr << YUZU_PAGEBITS; } auto host_ptr = uintptr_t(system.DeviceMemory().GetPointer(target_paddr)) - (base << YUZU_PAGEBITS); - auto backing = GetInteger(target_paddr) - (base << YUZU_PAGEBITS); - page_table.entries[base].ptr.Store(host_ptr, type); - page_table.entries[base].addr = backing; - page_table.entries[base].block = orig_base << YUZU_PAGEBITS; + auto& entry = page_table.entries.GetUnchecked(base); - ASSERT_MSG(page_table.entries[base].ptr.Pointer(), + entry.Store(false, type, current_block, host_ptr); + ASSERT_MSG(page_table.entries[base].Pointer(), "memory mapping base yield a nullptr within the table"); base += 1; @@ -630,11 +634,11 @@ struct Memory::Impl { vaddr &= 0xffffffffffffULL; if (AddressSpaceContains(*current_page_table, vaddr, 1)) [[likely]] { // Avoid adding any extra logic to this fast-path block - const uintptr_t raw_pointer = current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Raw(); - if (const uintptr_t pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) [[likely]] { + const auto raw = current_page_table->entries[vaddr >> YUZU_PAGEBITS].Raw(); + if (auto pointer = Common::PageTable::PageEntryData::ExtractPointer(raw); pointer) [[likely]] { return reinterpret_cast(pointer + vaddr); } else { - switch (Common::PageTable::PageInfo::ExtractType(raw_pointer)) { + switch (static_cast(raw.type)) { case Common::PageType::Memory: ASSERT_MSG(false, "Mapped memory page without a pointer @ {:#016x}", vaddr); return nullptr; @@ -646,6 +650,7 @@ struct Memory::Impl { return host_ptr; } case Common::PageType::Unmapped: [[unlikely]] { + __builtin_debugtrap(); on_unmapped(); return nullptr; } @@ -872,7 +877,7 @@ bool Memory::IsValidVirtualAddress(const Common::ProcessAddress vaddr) const { if (page >= page_table.entries.size()) { return false; } - const auto [pointer, type] = page_table.entries[page].ptr.PointerType(); + const auto [pointer, type, _] = page_table.entries[page].PointerTypeBlock(); return pointer != 0 || type == Common::PageType::RasterizerCachedMemory || type == Common::PageType::DebugMemory; } diff --git a/src/dynarmic/src/dynarmic/backend/arm64/a32_address_space.cpp b/src/dynarmic/src/dynarmic/backend/arm64/a32_address_space.cpp index da51220e9c..64dd0b85ed 100644 --- a/src/dynarmic/src/dynarmic/backend/arm64/a32_address_space.cpp +++ b/src/dynarmic/src/dynarmic/backend/arm64/a32_address_space.cpp @@ -373,6 +373,7 @@ EmitConfig A32AddressSpace::GetEmitConfig() { .page_table_address_space_bits = 32, .page_table_pointer_mask_bits = conf.page_table_pointer_mask_bits, .page_table_log2_stride = conf.page_table_log2_stride, + .page_table_marked_bit = conf.page_table_marked_bit, .silently_mirror_page_table = true, .absolute_offset_page_table = conf.absolute_offset_page_table, .detect_misaligned_access_via_page_table = conf.detect_misaligned_access_via_page_table, diff --git a/src/dynarmic/src/dynarmic/backend/arm64/a64_address_space.cpp b/src/dynarmic/src/dynarmic/backend/arm64/a64_address_space.cpp index 2c71ffe282..e65c1d90f4 100644 --- a/src/dynarmic/src/dynarmic/backend/arm64/a64_address_space.cpp +++ b/src/dynarmic/src/dynarmic/backend/arm64/a64_address_space.cpp @@ -547,6 +547,7 @@ EmitConfig A64AddressSpace::GetEmitConfig() { .page_table_address_space_bits = conf.page_table_address_space_bits, .page_table_pointer_mask_bits = conf.page_table_pointer_mask_bits, .page_table_log2_stride = conf.page_table_log2_stride, + .page_table_marked_bit = conf.page_table_marked_bit, .silently_mirror_page_table = conf.silently_mirror_page_table, .absolute_offset_page_table = conf.absolute_offset_page_table, .detect_misaligned_access_via_page_table = conf.detect_misaligned_access_via_page_table, diff --git a/src/dynarmic/src/dynarmic/backend/arm64/emit_arm64.h b/src/dynarmic/src/dynarmic/backend/arm64/emit_arm64.h index fefbcd8f5a..8016429392 100644 --- a/src/dynarmic/src/dynarmic/backend/arm64/emit_arm64.h +++ b/src/dynarmic/src/dynarmic/backend/arm64/emit_arm64.h @@ -130,6 +130,7 @@ struct EmitConfig { std::size_t page_table_address_space_bits; int page_table_pointer_mask_bits; std::size_t page_table_log2_stride; + std::optional page_table_marked_bit; bool silently_mirror_page_table; bool absolute_offset_page_table; u8 detect_misaligned_access_via_page_table; diff --git a/src/dynarmic/src/dynarmic/backend/arm64/emit_arm64_memory.cpp b/src/dynarmic/src/dynarmic/backend/arm64/emit_arm64_memory.cpp index 143244d60b..7cb7f0a630 100644 --- a/src/dynarmic/src/dynarmic/backend/arm64/emit_arm64_memory.cpp +++ b/src/dynarmic/src/dynarmic/backend/arm64/emit_arm64_memory.cpp @@ -273,6 +273,13 @@ std::pair InlinePageTableEmitVAddrLookup(oaknut::Cod // load x0 = *<(u8*)pagetable + index> code.LDR(Xscratch0, Xpagetable, Xscratch0); + if (ctx.conf.page_table_marked_bit) { + // check for marked bit + code.TST(Xscratch0, 1ULL << *ctx.conf.page_table_marked_bit); + // if marked, view this page as unmapped + code.CSEL(Xscratch0, Xscratch0, XZR, EQ); + } + if (ctx.conf.page_table_pointer_mask_bits != 0) { const u64 mask = u64(~u64(0)) << ctx.conf.page_table_pointer_mask_bits; code.AND(Xscratch0, Xscratch0, mask); diff --git a/src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.h b/src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.h index 3ac078f1d7..71d1f6a7e3 100644 --- a/src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.h +++ b/src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.h @@ -88,6 +88,20 @@ template<> code.shr(tmp, int(page_table_const_bits)); code.shl(tmp, int(ctx.conf.page_table_log2_stride)); code.mov(page, qword[r14 + tmp.cvt64()]); + + // check for marked bit, use as unmapped if marked + if (ctx.conf.page_table_marked_bit) { + // zero page, we can use it as scratch register before it's initialized + code.xor_(page, page); + if (*ctx.conf.page_table_marked_bit >= 30) { + code.bt(tmp, *ctx.conf.page_table_marked_bit); + code.cmovc(tmp, page); + } else { + code.test(tmp, 1ULL << *ctx.conf.page_table_marked_bit); + code.cmovnz(tmp, page); + } + } + // mask away attributes if (ctx.conf.page_table_pointer_mask_bits == 0) { code.test(page, page); } else { diff --git a/src/dynarmic/src/dynarmic/interface/A32/config.h b/src/dynarmic/src/dynarmic/interface/A32/config.h index 5a97fb69f3..6f99a51e55 100644 --- a/src/dynarmic/src/dynarmic/interface/A32/config.h +++ b/src/dynarmic/src/dynarmic/interface/A32/config.h @@ -165,8 +165,13 @@ struct UserConfig { /// If the configured value is 3, all pointers will be forcefully aligned to 8 bytes. std::int32_t page_table_pointer_mask_bits = 0; - // Log2 of the size per page entry, value should be either 3 or 4 - std::size_t page_table_log2_stride = 3; + /// Log2 of the size per page entry, value should be either 3 or 4 + std::uint32_t page_table_log2_stride = 3; + + /// Setting this value has Dynarmic check the specified bit of the page pointer provided by page table. + /// If the bit is set to 1, Dynarmic will treat it as unmapped. + /// This bit should be included as part of `page_table_pointer_mask_bits`. + std::optional page_table_marked_bit = std::nullopt; /// Select the architecture version to use. /// There are minor behavioural differences between versions. diff --git a/src/dynarmic/src/dynarmic/interface/A64/config.h b/src/dynarmic/src/dynarmic/interface/A64/config.h index 83c1593fd8..e74b41c259 100644 --- a/src/dynarmic/src/dynarmic/interface/A64/config.h +++ b/src/dynarmic/src/dynarmic/interface/A64/config.h @@ -179,8 +179,13 @@ struct UserConfig { /// If the configured value is 3, all pointers will be forcefully aligned to 8 bytes. std::int32_t page_table_pointer_mask_bits = 0; - // Log2 of the size per page entry, value should be either 3 or 4 - std::size_t page_table_log2_stride = 3; + /// Log2 of the size per page entry, value should be either 3 or 4 + std::uint32_t page_table_log2_stride = 3; + + /// Setting this value has Dynarmic check the specified bit of the page pointer provided by page table. + /// If the bit is set to 1, Dynarmic will treat it as unmapped. + /// This bit should be included as part of `page_table_pointer_mask_bits`. + std::optional page_table_marked_bit = std::nullopt; /// Counter-timer frequency register. The value of the register is not interpreted by /// dynarmic. diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index b9b136f6b8..d5838a387d 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -46,8 +46,8 @@ MemoryManager::MemoryManager(Core::System& system_, MaxwellDeviceMemoryManager& page_table_mask = page_table_size - 1; big_page_table_mask = big_page_table_size - 1; + big_page_table_dev.ResizeAndClear(big_page_table_size); big_entries.resize(big_page_table_size / 32, 0); - big_page_table_dev.resize(big_page_table_size); big_page_continuous.resize(big_page_table_size / continuous_bits, 0); entries.resize(page_table_size / 32, 0); } @@ -143,7 +143,7 @@ GPUVAddr MemoryManager::BigPageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr const DAddr current_dev_addr = dev_addr + offset; const auto index = PageEntryIndex(current_gpu_addr, true); const u32 sub_value = static_cast(current_dev_addr >> cpu_page_bits); - big_page_table_dev[index] = sub_value; + big_page_table_dev.Set(index, sub_value); const bool is_continuous = ([&] { uintptr_t base_ptr{ reinterpret_cast(memory.GetPointer(current_dev_addr))}; diff --git a/src/video_core/memory_manager.h b/src/video_core/memory_manager.h index f9fddd177a..08978567d2 100644 --- a/src/video_core/memory_manager.h +++ b/src/video_core/memory_manager.h @@ -17,7 +17,7 @@ #include "common/multi_level_page_table.h" #include "common/range_map.h" #include "common/scratch_buffer.h" -#include "common/virtual_buffer.h" +#include "common/sparse_large_vector.h" #include "video_core/invalidation_accumulator.h" #include "video_core/cache_types.h" #include "video_core/host1x/gpu_device_memory_manager.h" @@ -214,7 +214,7 @@ private: Common::MultiLevelPageTable page_table; Common::RangeMap kind_map; - Common::VirtualBuffer big_page_table_dev; + Common::SparseLargeVector big_page_table_dev; std::vector big_page_continuous; boost::container::small_vector, 32> page_stash{};