Browse Source
[common/dynarmic] Rewrite virtual buffers and optimize page table allocations (#4219)
pull/4181/head
[common/dynarmic] Rewrite virtual buffers and optimize page table allocations (#4219)
pull/4181/head
Exverge
6 days ago
No known key found for this signature in database
GPG Key ID: DAD399BCC5FB77E4
27 changed files with 641 additions and 356 deletions
-
4src/common/CMakeLists.txt
-
1src/common/fiber.cpp
-
39src/common/host_memory.cpp
-
20src/common/host_memory.h
-
38src/common/page_table.cpp
-
108src/common/page_table.h
-
135src/common/sparse_large_vector.cpp
-
188src/common/sparse_large_vector.h
-
45src/common/virtual_buffer.cpp
-
84src/common/virtual_buffer.h
-
8src/core/arm/dynarmic/arm_dynarmic_32.cpp
-
9src/core/arm/dynarmic/arm_dynarmic_64.cpp
-
10src/core/device_memory.h
-
10src/core/device_memory_manager.h
-
46src/core/device_memory_manager.inc
-
110src/core/hle/kernel/k_page_table_base.cpp
-
13src/core/hle/kernel/k_page_table_base.h
-
79src/core/memory.cpp
-
1src/dynarmic/src/dynarmic/backend/arm64/a32_address_space.cpp
-
1src/dynarmic/src/dynarmic/backend/arm64/a64_address_space.cpp
-
1src/dynarmic/src/dynarmic/backend/arm64/emit_arm64.h
-
7src/dynarmic/src/dynarmic/backend/arm64/emit_arm64_memory.cpp
-
14src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.h
-
9src/dynarmic/src/dynarmic/interface/A32/config.h
-
9src/dynarmic/src/dynarmic/interface/A64/config.h
-
4src/video_core/memory_manager.cpp
-
4src/video_core/memory_manager.h
@ -0,0 +1,135 @@ |
|||
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
|
|||
// SPDX-License-Identifier: GPL-3.0-or-later
|
|||
|
|||
/* virtual_buffer.cpp */ |
|||
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
|
|||
// SPDX-License-Identifier: GPL-2.0-or-later
|
|||
|
|||
#ifdef _WIN32
|
|||
#include <windows.h>
|
|||
#include <mutex>
|
|||
#else
|
|||
#include <sys/mman.h>
|
|||
#endif
|
|||
|
|||
#include "common/alignment.h"
|
|||
#include "common/assert.h"
|
|||
#include "common/sparse_large_vector.h"
|
|||
|
|||
namespace Common { |
|||
|
|||
#ifdef _WIN32
|
|||
static std::vector<std::pair<u64, u64>> vector_regions {}; |
|||
|
|||
// Workaround for handling non-commited memory accessed by Dynarmic; usually result of an error
|
|||
static LONG WINAPI FakePageFaultHandler(PEXCEPTION_POINTERS info) { |
|||
DWORD code = info->ExceptionRecord->ExceptionCode; |
|||
u64 exception_addr = reinterpret_cast<u64>(info->ExceptionRecord->ExceptionAddress); |
|||
|
|||
if (code != EXCEPTION_ACCESS_VIOLATION) { |
|||
// Not our problem
|
|||
return EXCEPTION_CONTINUE_SEARCH; |
|||
} |
|||
|
|||
u64 addr = 0, addr2 = 0; |
|||
|
|||
for (auto region: vector_regions) { |
|||
auto addr_shifted = exception_addr >> HostPageBits; |
|||
if (region.first <= addr_shifted && addr_shifted <= region.second) { |
|||
addr = addr_shifted; |
|||
} |
|||
|
|||
// Page-boundary accesses
|
|||
if (auto addr_ = (exception_addr + 0x40) >> HostPageBits; addr_ != addr_shifted && region.first <= addr_ && addr_ <= region.second) { |
|||
addr2 = addr_; |
|||
} |
|||
} |
|||
|
|||
if (addr == 0 && addr2 == 0) { |
|||
// Not our problem
|
|||
return EXCEPTION_CONTINUE_SEARCH; |
|||
} |
|||
|
|||
LOG_ERROR(HW_Memory, "Accessing an unallocated region of a LargeVector at {:#x}; this shouldn't happen and is likely a Dynarmic error!", exception_addr); |
|||
|
|||
// Commit this region
|
|||
if (addr != 0) { |
|||
if (!CommitVectorPage(addr << HostPageBits, false)) { |
|||
return EXCEPTION_CONTINUE_SEARCH; |
|||
} |
|||
} |
|||
// Commit next region if needed
|
|||
if (addr2 != 0) { |
|||
if (!CommitVectorPage(addr2 << HostPageBits, false)) { |
|||
return EXCEPTION_CONTINUE_SEARCH; |
|||
} |
|||
} |
|||
|
|||
return EXCEPTION_CONTINUE_EXECUTION; |
|||
} |
|||
|
|||
bool CommitVectorPage(uintptr_t addr, bool write) noexcept { |
|||
MEMORY_BASIC_INFORMATION info {}; |
|||
auto res = VirtualQuery(reinterpret_cast<void*>(addr), &info, sizeof(info)); |
|||
if (res == 0) { |
|||
LOG_CRITICAL(HW_Memory, "Failed to query large buffer region at {:#x} with error {}, will try committing anyway", addr, GetLastError()); |
|||
} else if (info.State != MEM_RESERVE) { |
|||
LOG_ERROR(HW_Memory, "Tried to commit an unreserved large buffer region at {:#x} that is not mapped or is already committed (state {:#x})", addr, info.State); |
|||
return false; |
|||
} |
|||
|
|||
auto perm = write ? PAGE_READWRITE : PAGE_READONLY; |
|||
void* res2 = VirtualAlloc(reinterpret_cast<LPVOID>(addr), HostPageSize, MEM_COMMIT, perm); |
|||
if (res2 == nullptr) { |
|||
LOG_ERROR(HW_Memory, "Failed to commit large buffer region at {:#x}, error {}", addr, GetLastError()); |
|||
return false; |
|||
} |
|||
|
|||
return true; |
|||
} |
|||
#endif
|
|||
|
|||
void* AllocateMemoryPages(std::size_t size) noexcept { |
|||
if (auto page = HostPageSize; size % page != 0) { |
|||
LOG_WARNING(HW_Memory, "Allocating unaligned large vector with size {:#x}; aligning to {} page size", size, page); |
|||
size = AlignUp(size, page); |
|||
} |
|||
|
|||
#ifdef _WIN32
|
|||
// We will never use this memory entirely so instead of committing it up front let's just reserve it and commit each page individually
|
|||
void* base = VirtualAlloc(nullptr, size, MEM_RESERVE, PAGE_READWRITE); |
|||
|
|||
if (base != nullptr) { |
|||
vector_regions.emplace_back(reinterpret_cast<u64>(base), reinterpret_cast<u64>(base) + size); |
|||
|
|||
static std::once_flag flag; |
|||
std::call_once(flag, []() { AddVectoredExceptionHandler(1, FakePageFaultHandler); }); |
|||
} else { |
|||
// Try committing everything instead??
|
|||
LOG_WARNING(HW_Memory, "Failed to reserve large vector region with error {}, trying to commit instead..", GetLastError()); |
|||
base = VirtualAlloc(nullptr, size, MEM_COMMIT, PAGE_READWRITE); |
|||
} |
|||
ASSERT_MSG(base, "Failed to reserve {:#x} sized region with error {}", size, GetLastError()); |
|||
#else
|
|||
void* base = mmap(nullptr, size, PROT_READ, MAP_ANON | MAP_PRIVATE, -1, 0); |
|||
if (base == MAP_FAILED) |
|||
base = nullptr; |
|||
ASSERT_MSG(base, "Failed to allocate {:#x} sized region with error {}", size, strerror(errno)); |
|||
#endif
|
|||
return base; |
|||
} |
|||
|
|||
void FreeMemoryPages(void* base, [[maybe_unused]] std::size_t size) noexcept { |
|||
if (auto page = HostPageSize; size % page != 0) { |
|||
size = AlignUp(size, page); |
|||
} |
|||
if (!base) |
|||
return; |
|||
#ifdef _WIN32
|
|||
ASSERT(VirtualFree(base, 0, MEM_RELEASE)); |
|||
#else
|
|||
ASSERT(munmap(base, size) == 0); |
|||
#endif
|
|||
} |
|||
|
|||
} // namespace Common
|
|||
@ -0,0 +1,188 @@ |
|||
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project |
|||
// SPDX-License-Identifier: GPL-3.0-or-later |
|||
|
|||
/* virtual_buffer.h */ |
|||
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project |
|||
// SPDX-License-Identifier: GPL-2.0-or-later |
|||
|
|||
#pragma once |
|||
|
|||
#include <atomic> |
|||
#include <bit> |
|||
#include <utility> |
|||
#include <vector> |
|||
|
|||
#ifndef _WIN32 |
|||
#include <unistd.h> |
|||
#include <sys/mman.h> |
|||
#endif |
|||
|
|||
#include "common/alignment.h" |
|||
#include "common/assert.h" |
|||
#include "common/host_memory.h" |
|||
|
|||
namespace Common { |
|||
|
|||
#ifdef _WIN32 |
|||
bool CommitVectorPage(uintptr_t addr, bool write) noexcept; |
|||
#endif |
|||
|
|||
void* AllocateMemoryPages(std::size_t size) noexcept; |
|||
void FreeMemoryPages(void* base, std::size_t size) noexcept; |
|||
|
|||
/// A large page-aligned buffer that has optimized memory usage for zero-writes. |
|||
template <typename T> |
|||
requires std::is_trivially_copyable_v<T> |
|||
class SparseLargeVector final { |
|||
public: |
|||
constexpr SparseLargeVector() = default; |
|||
|
|||
explicit SparseLargeVector(std::size_t count) noexcept |
|||
: alloc_size{count * sizeof(T)} |
|||
{ |
|||
base_ptr = static_cast<T*>(AllocateMemoryPages(alloc_size)); |
|||
|
|||
// each item in vector holds information for 64 pages |
|||
auto denom = HostPageSize * 64; |
|||
committed_pages = std::vector<std::atomic<u64>>((alloc_size + denom - 1) / denom); |
|||
} |
|||
|
|||
~SparseLargeVector() noexcept { |
|||
FreeMemoryPages(base_ptr, alloc_size); |
|||
} |
|||
|
|||
SparseLargeVector(const SparseLargeVector&) = delete; |
|||
SparseLargeVector& operator=(const SparseLargeVector&) = delete; |
|||
SparseLargeVector(SparseLargeVector&& other) = delete; |
|||
SparseLargeVector& operator=(SparseLargeVector&& other) = delete; |
|||
|
|||
void ResizeAndClear(std::size_t count) noexcept { |
|||
if (auto const new_size = count * sizeof(T); new_size != alloc_size) { |
|||
FreeMemoryPages(base_ptr, alloc_size); |
|||
alloc_size = new_size; |
|||
base_ptr = static_cast<T*>(AllocateMemoryPages(alloc_size)); |
|||
|
|||
auto denom = HostPageSize * 64; |
|||
committed_pages = std::vector<std::atomic<u64>>((alloc_size + denom - 1) / denom); |
|||
} |
|||
} |
|||
|
|||
/// Returns a reference to the value of the requested index and allocates memory if needed. |
|||
T& GetAndFault(std::size_t index) noexcept { |
|||
if (index > alloc_size / sizeof(T)) { |
|||
UNREACHABLE_MSG("Out of bounds RW access on SparseLargeVector @ {}", index); |
|||
} |
|||
|
|||
if (!IsCommittedPage(index)) { |
|||
CommitPage(index); |
|||
} |
|||
return base_ptr[index]; |
|||
} |
|||
|
|||
/// Returns a reference to the value of the requested index if initialized, or will otherwise return a zero-initialized object. |
|||
const T& GetOrDefault(std::size_t index) const { |
|||
#ifdef _WIN32 |
|||
if (!IsCommittedPage(index)) { |
|||
return *reinterpret_cast<const T*>(&default_val); |
|||
} |
|||
#endif |
|||
// On non-Windows, OS page table should optimize this by pointing to a zero page if unallocated. |
|||
return base_ptr[index]; |
|||
} |
|||
|
|||
void Set(std::size_t index, const T& value) noexcept { |
|||
if (index > alloc_size / sizeof(T)) { |
|||
LOG_CRITICAL(Common_Memory, "Out of bounds write on SparseLargeVector @ {}", index); |
|||
return; |
|||
} |
|||
if (!IsCommittedPage(index)) |
|||
CommitPage(index); |
|||
base_ptr[index] = value; |
|||
} |
|||
|
|||
void ZeroRegion(std::size_t start, std::size_t end_) noexcept { |
|||
u64 base = reinterpret_cast<u64>(&base_ptr[start]); |
|||
const u64 end = reinterpret_cast<u64>(&base_ptr[end_]); |
|||
|
|||
const u64 end_page = AlignUp(base, HostPageSize); |
|||
const u64 first_size = (std::min)(end_page, end) - base; |
|||
|
|||
if (IsCommittedPage(start / sizeof(T))) { |
|||
std::memset(reinterpret_cast<void*>(base), 0, first_size); |
|||
} |
|||
|
|||
if (end <= end_page) |
|||
return; |
|||
|
|||
base = end_page; |
|||
|
|||
for (u64 page = base; page < end; page += HostPageSize) { |
|||
if (!IsCommittedPage((page - reinterpret_cast<u64>(base_ptr)) / sizeof(T))) { |
|||
continue; |
|||
} |
|||
|
|||
std::memset(reinterpret_cast<void*>(page), 0, (std::min)(HostPageSize, end - page)); |
|||
} |
|||
} |
|||
|
|||
constexpr void CommitRegion(size_t index, size_t end_) { |
|||
const u64 base = static_cast<u64>(index) * sizeof(T); |
|||
const u64 end = static_cast<u64>(end_) * sizeof(T); |
|||
|
|||
for (u64 page = AlignDown(base, HostPageSize); page < end; page += HostPageSize) { |
|||
if (!IsCommittedPage(page / sizeof(T))) { |
|||
CommitPage(page / sizeof(T)); |
|||
} |
|||
} |
|||
} |
|||
|
|||
constexpr T& GetUnchecked(size_t index) { |
|||
return base_ptr[index]; |
|||
} |
|||
|
|||
[[nodiscard]] constexpr const T& operator[](std::size_t index) const noexcept { |
|||
return GetOrDefault(index); |
|||
} |
|||
|
|||
[[nodiscard]] constexpr const T* data() const noexcept { |
|||
return base_ptr; |
|||
} |
|||
|
|||
[[nodiscard]] constexpr std::size_t size() const noexcept { |
|||
return alloc_size / sizeof(T); |
|||
} |
|||
|
|||
private: |
|||
[[nodiscard]] constexpr bool IsCommittedPage(std::size_t index) const noexcept { |
|||
if (index > alloc_size / sizeof(T)) { |
|||
LOG_CRITICAL(Common_Memory, "Out of bounds access on large vector @ {}", index); |
|||
return false; |
|||
} |
|||
|
|||
auto page = (index * sizeof(T)) >> HostPageBits; |
|||
auto val = committed_pages[page >> 6].load(std::memory_order_acquire); |
|||
return (val >> (page & 63)) & 1; |
|||
} |
|||
|
|||
constexpr void CommitPage(std::size_t index) noexcept { |
|||
auto page_index = (index * sizeof(T)) >> HostPageBits; |
|||
auto page = reinterpret_cast<uintptr_t>(base_ptr + index) & HostPageMask; |
|||
#ifdef _WIN32 |
|||
CommitVectorPage(page, true); |
|||
#else |
|||
mprotect(reinterpret_cast<void*>(page), HostPageSize, PROT_READ | PROT_WRITE); |
|||
#endif |
|||
|
|||
committed_pages[page_index >> 6].fetch_or(1ULL << (page_index & 63), std::memory_order_release); |
|||
} |
|||
|
|||
std::size_t alloc_size{}; |
|||
T* base_ptr{}; |
|||
|
|||
std::vector<std::atomic<u64>> committed_pages{}; |
|||
#ifdef _WIN32 |
|||
const std::array<u8, sizeof(T)> default_val{}; |
|||
#endif |
|||
}; |
|||
|
|||
} // namespace Common |
|||
@ -1,45 +0,0 @@ |
|||
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
|
|||
// SPDX-License-Identifier: GPL-3.0-or-later
|
|||
|
|||
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
|
|||
// SPDX-License-Identifier: GPL-2.0-or-later
|
|||
|
|||
#ifdef _WIN32
|
|||
#include <windows.h>
|
|||
#else
|
|||
#include <sys/mman.h>
|
|||
#endif
|
|||
|
|||
#include "common/assert.h"
|
|||
#include "common/virtual_buffer.h"
|
|||
|
|||
namespace Common { |
|||
|
|||
void* AllocateMemoryPages(std::size_t size) noexcept { |
|||
#ifdef _WIN32
|
|||
void* base = VirtualAlloc(nullptr, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); |
|||
if (base == nullptr) { |
|||
// Probably failing to reserve is less likely than failing to commit
|
|||
base = VirtualAlloc(nullptr, size, MEM_COMMIT, PAGE_READWRITE); |
|||
} |
|||
ASSERT_MSG(base, "Failed to allocate {} pages, error {}", size, GetLastError()); |
|||
#else
|
|||
void* base = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0); |
|||
if (base == MAP_FAILED) |
|||
base = nullptr; |
|||
ASSERT_MSG(base, "Failed to allocate {} pages, error {}", size, strerror(errno)); |
|||
#endif
|
|||
return base; |
|||
} |
|||
|
|||
void FreeMemoryPages(void* base, [[maybe_unused]] std::size_t size) noexcept { |
|||
if (!base) |
|||
return; |
|||
#ifdef _WIN32
|
|||
ASSERT(VirtualFree(base, 0, MEM_RELEASE)); |
|||
#else
|
|||
ASSERT(munmap(base, size) == 0); |
|||
#endif
|
|||
} |
|||
|
|||
} // namespace Common
|
|||
@ -1,84 +0,0 @@ |
|||
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project |
|||
// SPDX-License-Identifier: GPL-3.0-or-later |
|||
|
|||
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project |
|||
// SPDX-License-Identifier: GPL-2.0-or-later |
|||
|
|||
#pragma once |
|||
|
|||
#include <utility> |
|||
|
|||
namespace Common { |
|||
|
|||
void* AllocateMemoryPages(std::size_t size) noexcept; |
|||
void FreeMemoryPages(void* base, std::size_t size) noexcept; |
|||
|
|||
template <typename T> |
|||
class VirtualBuffer final { |
|||
public: |
|||
// TODO: Uncomment this and change Common::PageTable::PageInfo to be trivially constructible |
|||
// using std::atomic_ref once libc++ has support for it |
|||
// static_assert( |
|||
// std::is_trivially_constructible_v<T>, |
|||
// "T must be trivially constructible, as non-trivial constructors will not be executed " |
|||
// "with the current allocator"); |
|||
|
|||
constexpr VirtualBuffer() = default; |
|||
explicit VirtualBuffer(std::size_t count) noexcept |
|||
: alloc_size{count * sizeof(T)} |
|||
{ |
|||
base_ptr = reinterpret_cast<T*>(AllocateMemoryPages(alloc_size)); |
|||
} |
|||
|
|||
~VirtualBuffer() noexcept { |
|||
FreeMemoryPages(base_ptr, alloc_size); |
|||
} |
|||
|
|||
VirtualBuffer(const VirtualBuffer&) = delete; |
|||
VirtualBuffer& operator=(const VirtualBuffer&) = delete; |
|||
|
|||
VirtualBuffer(VirtualBuffer&& other) noexcept |
|||
: alloc_size{std::exchange(other.alloc_size, 0)} |
|||
, base_ptr{std::exchange(other.base_ptr, nullptr)} |
|||
{} |
|||
|
|||
VirtualBuffer& operator=(VirtualBuffer&& other) noexcept { |
|||
alloc_size = std::exchange(other.alloc_size, 0); |
|||
base_ptr = std::exchange(other.base_ptr, nullptr); |
|||
return *this; |
|||
} |
|||
|
|||
void resize(std::size_t count) noexcept { |
|||
if (auto const new_size = count * sizeof(T); new_size != alloc_size) { |
|||
FreeMemoryPages(base_ptr, alloc_size); |
|||
alloc_size = new_size; |
|||
base_ptr = reinterpret_cast<T*>(AllocateMemoryPages(alloc_size)); |
|||
} |
|||
} |
|||
|
|||
[[nodiscard]] constexpr const T& operator[](std::size_t index) const noexcept { |
|||
return base_ptr[index]; |
|||
} |
|||
|
|||
[[nodiscard]] constexpr T& operator[](std::size_t index) noexcept { |
|||
return base_ptr[index]; |
|||
} |
|||
|
|||
[[nodiscard]] constexpr T* data() noexcept { |
|||
return base_ptr; |
|||
} |
|||
|
|||
[[nodiscard]] constexpr const T* data() const noexcept { |
|||
return base_ptr; |
|||
} |
|||
|
|||
[[nodiscard]] constexpr std::size_t size() const noexcept { |
|||
return alloc_size / sizeof(T); |
|||
} |
|||
|
|||
private: |
|||
std::size_t alloc_size{}; |
|||
T* base_ptr{}; |
|||
}; |
|||
|
|||
} // namespace Common |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue