Browse Source

[dynarmic/core] Half page table entries sizes using a marked bit

pull/4219/head
Exverge 1 week ago
parent
commit
eb9a04a291
No known key found for this signature in database GPG Key ID: DAD399BCC5FB77E4
  1. 30
      src/common/page_table.cpp
  2. 106
      src/common/page_table.h
  3. 4
      src/core/arm/dynarmic/arm_dynarmic_32.cpp
  4. 5
      src/core/arm/dynarmic/arm_dynarmic_64.cpp
  5. 10
      src/core/device_memory.h
  6. 110
      src/core/hle/kernel/k_page_table_base.cpp
  7. 13
      src/core/hle/kernel/k_page_table_base.h
  8. 70
      src/core/memory.cpp
  9. 1
      src/dynarmic/src/dynarmic/backend/arm64/a32_address_space.cpp
  10. 1
      src/dynarmic/src/dynarmic/backend/arm64/a64_address_space.cpp
  11. 1
      src/dynarmic/src/dynarmic/backend/arm64/emit_arm64.h
  12. 7
      src/dynarmic/src/dynarmic/backend/arm64/emit_arm64_memory.cpp
  13. 14
      src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.h
  14. 9
      src/dynarmic/src/dynarmic/interface/A32/config.h
  15. 9
      src/dynarmic/src/dynarmic/interface/A64/config.h

30
src/common/page_table.cpp

@ -13,36 +13,6 @@ PageTable::PageTable() = default;
PageTable::~PageTable() noexcept = default;
bool PageTable::BeginTraversal(TraversalEntry* out_entry, TraversalContext* out_context,
Common::ProcessAddress address) const {
out_context->next_offset = GetInteger(address);
out_context->next_page = GetInteger(address) >> current_page_bits;
return this->ContinueTraversal(out_entry, out_context);
}
bool PageTable::ContinueTraversal(TraversalEntry* out_entry, TraversalContext* context) const {
auto page_size = 1 << current_page_bits;
// Setup invalid defaults.
out_entry->phys_addr = 0;
out_entry->block_size = page_size;
// Validate that we can read the actual entry.
if (auto const page = context->next_page; page < entries.size()) {
// Validate that the entry is mapped.
if (auto const paddr = entries[page].addr; paddr != 0) {
// Populate the results and return true
out_entry->phys_addr = entries[page].GetPhysOffset(current_page_bits) + context->next_offset;
context->next_page += 1;
context->next_offset += page_size;
return true;
}
}
context->next_page += 1;
context->next_offset += page_size;
// Otherwise return false
return false;
}
void PageTable::Resize(std::size_t address_space_width_in_bits, std::size_t page_bits) {
auto const num_page_table_entries = 1ULL << (address_space_width_in_bits - page_bits);
entries.ResizeAndClear(num_page_table_entries);

106
src/common/page_table.h

@ -16,15 +16,15 @@ namespace Common {
enum class PageType : u8 {
/// Page is unmapped and should cause an access error.
Unmapped,
Unmapped = 0b00,
/// Page is mapped to regular memory. This is the only type you can get pointers to.
Memory,
Memory = 0b01,
/// Page is mapped to regular memory, but inaccessible from CPU fastmem and must use
/// the callbacks.
DebugMemory,
DebugMemory = 0b10,
/// Page is mapped to regular memory, but also needs to check for rasterizer cache flushing and
/// invalidation
RasterizerCachedMemory,
RasterizerCachedMemory = 0b11,
};
/**
@ -44,55 +44,75 @@ struct PageTable {
/// Number of bits reserved for attribute tagging.
/// This can be at most the guaranteed alignment of the pointers in the page table.
static constexpr int ATTRIBUTE_BITS = 2;
static constexpr int ATTRIBUTE_BITS = 12;
/**
* Pair of host pointer and page type attribute.
* This uses the lower bits of a given pointer to store the attribute tag.
* Atomic tuple of host pointer, page type, and block id.
* This uses the lower bits of a given pointer to store the attributes.
* Writing and reading the pointer attribute pair is guaranteed to be atomic for the same method
* call. In other words, they are guaranteed to be synchronized at all times.
*/
class PageInfo {
class PageEntryData {
public:
struct Data {
Data(bool marked_, PageType type_, u16 block_, u64 page)
: marked(marked_), type(static_cast<u64>(type_)), block(block_), page(page >> ATTRIBUTE_BITS) {}
u64 marked : 1;
u64 type : 2;
u64 block : 9; // TODO: is 9 bits to little? we can use the upper 8 bits if needed
u64 page : 52;
};
[[nodiscard]] Data Raw() const noexcept {
return data.load(std::memory_order_relaxed);
}
/// Returns the page pointer
[[nodiscard]] uintptr_t Pointer() const noexcept {
return ExtractPointer(raw.load(std::memory_order_relaxed));
[[nodiscard]] uintptr_t Pointer(bool ignored_marked = false) const noexcept {
return ExtractPointer(data.load(std::memory_order_relaxed), ignored_marked);
}
/// Returns the page type attribute
[[nodiscard]] PageType Type() const noexcept {
return ExtractType(raw.load(std::memory_order_relaxed));
return static_cast<PageType>(data.load(std::memory_order_relaxed).type);
}
/// Returns the block identifier.
[[nodiscard]] u16 Block() const noexcept {
return static_cast<u16>(data.load(std::memory_order_relaxed).block);
}
/// Returns the page pointer and attribute pair, extracted from the same atomic read
[[nodiscard]] std::pair<uintptr_t, PageType> PointerType() const noexcept {
const uintptr_t non_atomic_raw = raw.load(std::memory_order_relaxed);
return {ExtractPointer(non_atomic_raw), ExtractType(non_atomic_raw)};
[[nodiscard]] std::tuple<uintptr_t, PageType, u16> PointerTypeBlock() const noexcept {
const Data non_atomic_raw = data.load(std::memory_order_relaxed);
return {ExtractPointer(non_atomic_raw), static_cast<PageType>(non_atomic_raw.type), static_cast<u16>(non_atomic_raw.block)};
}
/// Returns the raw representation of the page information.
/// Use ExtractPointer and ExtractType to unpack the value.
[[nodiscard]] uintptr_t Raw() const noexcept {
return raw.load(std::memory_order_relaxed);
/// Write page info atomically
constexpr void Store(bool marked, PageType type, u16 block, uintptr_t pointer) noexcept {
data.store({marked, type, block, pointer});
}
/// Write a page pointer and type pair atomically
void Store(uintptr_t pointer, PageType type) noexcept {
raw.store(pointer | uintptr_t(type));
constexpr void MarkRasterizerCached() noexcept {
data_raw.fetch_or(0b111);
}
/// Unpack a pointer from a page info raw representation
[[nodiscard]] static uintptr_t ExtractPointer(uintptr_t raw) noexcept {
return raw & (~uintptr_t{0} << ATTRIBUTE_BITS);
constexpr void MarkDebug(u64 ptr, u16 block) noexcept {
Store(true, PageType::RasterizerCachedMemory, block, ptr);
}
/// Unpack a page type from a page info raw representation
[[nodiscard]] static PageType ExtractType(uintptr_t raw) noexcept {
return static_cast<PageType>(raw & ((uintptr_t{1} << ATTRIBUTE_BITS) - 1));
/// Unpack a pointer from a page info raw representation
[[nodiscard]] static uintptr_t ExtractPointer(Data raw, bool ignore_marked = false) noexcept {
return raw.marked && !ignore_marked ? 0 : raw.page << ATTRIBUTE_BITS;
}
private:
std::atomic<uintptr_t> raw;
union {
std::atomic<Data> data;
std::atomic<u64> data_raw;
};
static_assert(sizeof(std::atomic<Data>) == 8);
static_assert(std::atomic<Data>::is_always_lock_free);
};
PageTable();
@ -103,10 +123,6 @@ struct PageTable {
PageTable(PageTable&&) noexcept = delete;
PageTable& operator=(PageTable&&) noexcept = delete;
bool BeginTraversal(TraversalEntry* out_entry, TraversalContext* out_context,
Common::ProcessAddress address) const;
bool ContinueTraversal(TraversalEntry* out_entry, TraversalContext* context) const;
/**
* Resizes the page table to be able to accommodate enough pages within
* a given address space.
@ -120,34 +136,10 @@ struct PageTable {
return current_address_space_width_in_bits;
}
bool GetPhysicalAddress(Common::PhysicalAddress* out_phys_addr,
Common::ProcessAddress virt_addr) const {
if (virt_addr > (1ULL << this->GetAddressSpaceBits())) {
return false;
}
*out_phys_addr = entries[GetInteger(virt_addr) >> current_page_bits].GetPhysOffset(current_page_bits) + GetInteger(virt_addr);
return true;
}
/// Vector of memory pointers backing each page. An entry can only be non-null if the
/// corresponding attribute element is of type `Memory`.
struct PageEntryData {
PageInfo ptr;
u32 block;
u32 addr;
constexpr u64 GetPhysOffset(u64 page_bits) const {
// TODO: For whatever reason, when storing the "physical address" of a large page group, yuzu code writes it as:
// `addr = base_phys_addr - vaddr`, where base_phys_addr is the base address of the first entry in the page group.
// This 90% of the time results in a negative pointer. However, as of #4219, `addr` is stored as a u32 instead of a u64,
// so we use sign extension to work around this issue.
s64 result = (static_cast<s64>(static_cast<s32>(addr))) << page_bits;
return static_cast<u64>(result);
}
};
SparseLargeVector<PageEntryData> entries;
static_assert(sizeof(PageEntryData) == 16);
static_assert(sizeof(PageEntryData) == 8);
u8* fastmem_arena{};
std::size_t current_address_space_width_in_bits{};

4
src/core/arm/dynarmic/arm_dynarmic_32.cpp

@ -172,14 +172,12 @@ void ArmDynarmic32::MakeJit(Common::PageTable* page_table) {
if (page_table) {
constexpr size_t PageBits = 12;
constexpr size_t NumPageTableEntries = 1 << (32 - PageBits);
constexpr size_t PageLog2Stride = 4;
static_assert(1 << PageLog2Stride == sizeof(Common::PageTable::PageEntryData));
// Dynarmic will not write to the page table, const_cast is safe here
config.page_table = reinterpret_cast<std::array<std::uint8_t*, NumPageTableEntries>*>(
const_cast<Common::PageTable::PageEntryData*>(page_table->entries.data()));
config.page_table_pointer_mask_bits = Common::PageTable::ATTRIBUTE_BITS;
config.page_table_log2_stride = PageLog2Stride;
config.page_table_marked_bit = 0;
config.absolute_offset_page_table = true;
config.detect_misaligned_access_via_page_table = 16 | 32 | 64 | 128;
config.only_detect_misalignment_via_page_table_on_page_boundary = true;

5
src/core/arm/dynarmic/arm_dynarmic_64.cpp

@ -211,15 +211,12 @@ void ArmDynarmic64::MakeJit(Common::PageTable* page_table, std::size_t address_s
// Memory
if (page_table) {
constexpr size_t PageLog2Stride = 4;
static_assert(1 << PageLog2Stride == sizeof(Common::PageTable::PageEntryData));
// Dynarmic will not write to the page table, const_cast is safe here
config.page_table = reinterpret_cast<void**>(
const_cast<Common::PageTable::PageEntryData*>(page_table->entries.data()));
config.page_table_address_space_bits = std::uint32_t(address_space_bits);
config.page_table_pointer_mask_bits = Common::PageTable::ATTRIBUTE_BITS;
config.page_table_log2_stride = PageLog2Stride;
config.page_table_marked_bit = 0;
config.silently_mirror_page_table = false;
config.absolute_offset_page_table = true;
config.detect_misaligned_access_via_page_table = 16 | 32 | 64 | 128;

10
src/core/device_memory.h

@ -1,3 +1,6 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
@ -26,8 +29,11 @@ public:
template <typename T>
Common::PhysicalAddress GetPhysicalAddr(const T* ptr) const {
return (reinterpret_cast<uintptr_t>(ptr) -
reinterpret_cast<uintptr_t>(buffer.BackingBasePointer())) +
return GetPhysicalAddr(reinterpret_cast<uintptr_t>(ptr));
}
Common::PhysicalAddress GetPhysicalAddr(uintptr_t ptr) const {
return (ptr - reinterpret_cast<uintptr_t>(buffer.BackingBasePointer())) +
DramMemoryMap::Base;
}

110
src/core/hle/kernel/k_page_table_base.cpp

@ -635,6 +635,36 @@ Result KPageTableBase::CheckMemoryState(const KMemoryInfo& info, KMemoryState st
R_SUCCEED();
}
bool KPageTableBase::BeginTraversal(const Common::PageTable &impl, TraversalEntry *out_entry, TraversalContext *out_context,
Common::ProcessAddress address) const {
out_context->next_offset = GetInteger(address);
out_context->next_page = GetInteger(address) >> PageBits;
return ContinueTraversal(impl, out_entry, out_context);
}
bool KPageTableBase::ContinueTraversal(const Common::PageTable &impl, TraversalEntry *out_entry,
TraversalContext *context) const {
// Setup invalid defaults.
out_entry->phys_addr = 0;
out_entry->block_size = PageSize;
// Validate that we can read the actual entry.
if (auto const page = context->next_page; page < impl.entries.size()) {
// Validate that the entry is mapped.
if (auto const paddr = impl.entries[page].Pointer(); paddr != 0) {
// Populate the results and return true
out_entry->phys_addr = GetInteger(m_system.DeviceMemory().GetPhysicalAddr(paddr)) + context->next_offset % PageSize;
context->next_page += 1;
context->next_offset += PageSize;
return true;
}
}
context->next_page += 1;
context->next_offset += PageSize;
// Otherwise return false
return false;
}
Result KPageTableBase::CheckMemoryStateContiguous(size_t* out_blocks_needed, KProcessAddress addr,
size_t size, KMemoryState state_mask,
KMemoryState state, KMemoryPermission perm_mask,
@ -940,7 +970,7 @@ Result KPageTableBase::QueryMappingImpl(KProcessAddress* out, KPhysicalAddress a
size_t tot_size = 0;
next_valid =
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), region_start);
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), region_start);
next_entry.block_size =
(next_entry.block_size - (GetInteger(region_start) & (next_entry.block_size - 1)));
@ -976,7 +1006,7 @@ Result KPageTableBase::QueryMappingImpl(KProcessAddress* out, KPhysicalAddress a
break;
}
next_valid = impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
next_valid = ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
}
// Check the last entry.
@ -1754,7 +1784,7 @@ Result KPageTableBase::MakePageGroup(KPageGroup& pg, KProcessAddress addr, size_
// Begin traversal.
TraversalContext context;
TraversalEntry next_entry;
R_UNLESS(impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), addr),
R_UNLESS(BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), addr),
ResultInvalidCurrentMemory);
// Prepare tracking variables.
@ -1764,7 +1794,7 @@ Result KPageTableBase::MakePageGroup(KPageGroup& pg, KProcessAddress addr, size_
// Iterate, adding to group as we go.
while (tot_size < size) {
R_UNLESS(impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)),
R_UNLESS(ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)),
ResultInvalidCurrentMemory);
if (next_entry.phys_addr != (cur_addr + cur_size)) {
@ -1828,7 +1858,7 @@ bool KPageTableBase::IsValidPageGroup(const KPageGroup& pg, KProcessAddress addr
// Begin traversal.
TraversalContext context;
TraversalEntry next_entry;
if (!impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), addr)) {
if (!BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), addr)) {
return false;
}
@ -1839,7 +1869,7 @@ bool KPageTableBase::IsValidPageGroup(const KPageGroup& pg, KProcessAddress addr
// Iterate, comparing expected to actual.
while (tot_size < size) {
if (!impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context))) {
if (!ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context))) {
return false;
}
@ -1896,7 +1926,7 @@ Result KPageTableBase::GetContiguousMemoryRangeWithState(
// Begin a traversal.
TraversalContext context;
TraversalEntry cur_entry = {.phys_addr = 0, .block_size = 0};
R_UNLESS(impl.BeginTraversal(std::addressof(cur_entry), std::addressof(context), address),
R_UNLESS(BeginTraversal(impl, std::addressof(cur_entry), std::addressof(context), address),
ResultInvalidCurrentMemory);
// Traverse until we have enough size or we aren't contiguous any more.
@ -1905,7 +1935,7 @@ Result KPageTableBase::GetContiguousMemoryRangeWithState(
for (contig_size =
cur_entry.block_size - (GetInteger(phys_address) & (cur_entry.block_size - 1));
contig_size < size; contig_size += cur_entry.block_size) {
if (!impl.ContinueTraversal(std::addressof(cur_entry), std::addressof(context))) {
if (!ContinueTraversal(impl, std::addressof(cur_entry), std::addressof(context))) {
break;
}
if (cur_entry.phys_addr != phys_address + contig_size) {
@ -2334,7 +2364,7 @@ Result KPageTableBase::QueryPhysicalAddress(Svc::lp64::PhysicalMemoryInfo* out,
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid =
m_impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), virt_addr);
BeginTraversal(m_impl, std::addressof(next_entry), std::addressof(context), virt_addr);
R_UNLESS(traverse_valid, ResultInvalidCurrentMemory);
// Set tracking variables.
@ -2345,7 +2375,7 @@ Result KPageTableBase::QueryPhysicalAddress(Svc::lp64::PhysicalMemoryInfo* out,
while (true) {
// Continue the traversal.
traverse_valid =
m_impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ContinueTraversal(m_impl, std::addressof(next_entry), std::addressof(context));
if (!traverse_valid) {
break;
}
@ -2567,7 +2597,7 @@ Result KPageTableBase::UnmapIoRegion(KProcessAddress dst_address, KPhysicalAddre
TraversalContext context;
TraversalEntry next_entry;
ASSERT(
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), dst_address));
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), dst_address));
// Check that the physical region matches.
R_UNLESS(next_entry.phys_addr == phys_addr, ResultInvalidMemoryRegion);
@ -2577,7 +2607,7 @@ Result KPageTableBase::UnmapIoRegion(KProcessAddress dst_address, KPhysicalAddre
next_entry.block_size - (GetInteger(phys_addr) & (next_entry.block_size - 1));
checked_size < size; checked_size += next_entry.block_size) {
// Continue the traversal.
ASSERT(impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)));
ASSERT(ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)));
// Check that the physical region matches.
R_UNLESS(next_entry.phys_addr == phys_addr + checked_size, ResultInvalidMemoryRegion);
@ -3029,7 +3059,7 @@ Result KPageTableBase::InvalidateProcessDataCache(KProcessAddress address, size_
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid =
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), address);
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), address);
R_UNLESS(traverse_valid, ResultInvalidCurrentMemory);
// Prepare tracking variables.
@ -3041,7 +3071,7 @@ Result KPageTableBase::InvalidateProcessDataCache(KProcessAddress address, size_
while (tot_size < size) {
// Continue the traversal.
traverse_valid =
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
R_UNLESS(traverse_valid, ResultInvalidCurrentMemory);
if (next_entry.phys_addr != (cur_addr + cur_size)) {
@ -3129,7 +3159,7 @@ Result KPageTableBase::ReadDebugMemory(KProcessAddress dst_address, KProcessAddr
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid =
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), src_address);
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), src_address);
R_UNLESS(traverse_valid, ResultInvalidCurrentMemory);
// Prepare tracking variables.
@ -3167,7 +3197,7 @@ Result KPageTableBase::ReadDebugMemory(KProcessAddress dst_address, KProcessAddr
while (tot_size < size) {
// Continue the traversal.
traverse_valid =
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
if (next_entry.phys_addr != (cur_addr + cur_size)) {
@ -3225,7 +3255,7 @@ Result KPageTableBase::WriteDebugMemory(KProcessAddress dst_address, KProcessAdd
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid =
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), dst_address);
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), dst_address);
R_UNLESS(traverse_valid, ResultInvalidCurrentMemory);
// Prepare tracking variables.
@ -3267,7 +3297,7 @@ Result KPageTableBase::WriteDebugMemory(KProcessAddress dst_address, KProcessAdd
while (tot_size < size) {
// Continue the traversal.
traverse_valid =
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
if (next_entry.phys_addr != (cur_addr + cur_size)) {
@ -3728,7 +3758,7 @@ Result KPageTableBase::CopyMemoryFromLinearToUser(
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid =
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), src_addr);
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), src_addr);
ASSERT(traverse_valid);
// Prepare tracking variables.
@ -3768,7 +3798,7 @@ Result KPageTableBase::CopyMemoryFromLinearToUser(
while (tot_size < size) {
// Continue the traversal.
traverse_valid =
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
if (next_entry.phys_addr != (cur_addr + cur_size)) {
@ -3822,7 +3852,7 @@ Result KPageTableBase::CopyMemoryFromLinearToKernel(
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid =
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), src_addr);
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), src_addr);
ASSERT(traverse_valid);
// Prepare tracking variables.
@ -3845,7 +3875,7 @@ Result KPageTableBase::CopyMemoryFromLinearToKernel(
while (tot_size < size) {
// Continue the traversal.
traverse_valid =
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
if (next_entry.phys_addr != (cur_addr + cur_size)) {
@ -3902,7 +3932,7 @@ Result KPageTableBase::CopyMemoryFromUserToLinear(
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid =
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), dst_addr);
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), dst_addr);
ASSERT(traverse_valid);
// Prepare tracking variables.
@ -3941,7 +3971,7 @@ Result KPageTableBase::CopyMemoryFromUserToLinear(
while (tot_size < size) {
// Continue the traversal.
traverse_valid =
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
if (next_entry.phys_addr != (cur_addr + cur_size)) {
@ -3997,7 +4027,7 @@ Result KPageTableBase::CopyMemoryFromKernelToLinear(KProcessAddress dst_addr, si
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid =
impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), dst_addr);
BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), dst_addr);
ASSERT(traverse_valid);
// Prepare tracking variables.
@ -4020,7 +4050,7 @@ Result KPageTableBase::CopyMemoryFromKernelToLinear(KProcessAddress dst_addr, si
while (tot_size < size) {
// Continue the traversal.
traverse_valid =
impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
if (next_entry.phys_addr != (cur_addr + cur_size)) {
@ -4089,10 +4119,10 @@ Result KPageTableBase::CopyMemoryFromHeapToHeap(
bool traverse_valid;
// Begin traversal.
traverse_valid = src_impl.BeginTraversal(std::addressof(src_next_entry),
traverse_valid = BeginTraversal(src_impl, std::addressof(src_next_entry),
std::addressof(src_context), src_addr);
ASSERT(traverse_valid);
traverse_valid = dst_impl.BeginTraversal(std::addressof(dst_next_entry),
traverse_valid = BeginTraversal(dst_impl, std::addressof(dst_next_entry),
std::addressof(dst_context), dst_addr);
ASSERT(traverse_valid);
@ -4127,7 +4157,7 @@ Result KPageTableBase::CopyMemoryFromHeapToHeap(
if (ofs + cur_copy_size != size) {
if (cur_src_addr + cur_min_size == cur_src_block_addr + cur_src_size) {
// Continue the src traversal.
traverse_valid = src_impl.ContinueTraversal(std::addressof(src_next_entry),
traverse_valid = ContinueTraversal(src_impl, std::addressof(src_next_entry),
std::addressof(src_context));
ASSERT(traverse_valid);
@ -4138,7 +4168,7 @@ Result KPageTableBase::CopyMemoryFromHeapToHeap(
if (cur_dst_addr + cur_min_size ==
dst_next_entry.phys_addr + dst_next_entry.block_size) {
// Continue the dst traversal.
traverse_valid = dst_impl.ContinueTraversal(std::addressof(dst_next_entry),
traverse_valid = ContinueTraversal(dst_impl, std::addressof(dst_next_entry),
std::addressof(dst_context));
ASSERT(traverse_valid);
@ -4223,10 +4253,10 @@ Result KPageTableBase::CopyMemoryFromHeapToHeapWithoutCheckDestination(
bool traverse_valid;
// Begin traversal.
traverse_valid = src_impl.BeginTraversal(std::addressof(src_next_entry),
traverse_valid = BeginTraversal(src_impl, std::addressof(src_next_entry),
std::addressof(src_context), src_addr);
ASSERT(traverse_valid);
traverse_valid = dst_impl.BeginTraversal(std::addressof(dst_next_entry),
traverse_valid = BeginTraversal(dst_impl, std::addressof(dst_next_entry),
std::addressof(dst_context), dst_addr);
ASSERT(traverse_valid);
@ -4261,7 +4291,7 @@ Result KPageTableBase::CopyMemoryFromHeapToHeapWithoutCheckDestination(
if (ofs + cur_copy_size != size) {
if (cur_src_addr + cur_min_size == cur_src_block_addr + cur_src_size) {
// Continue the src traversal.
traverse_valid = src_impl.ContinueTraversal(std::addressof(src_next_entry),
traverse_valid = ContinueTraversal(src_impl, std::addressof(src_next_entry),
std::addressof(src_context));
ASSERT(traverse_valid);
@ -4272,7 +4302,7 @@ Result KPageTableBase::CopyMemoryFromHeapToHeapWithoutCheckDestination(
if (cur_dst_addr + cur_min_size ==
dst_next_entry.phys_addr + dst_next_entry.block_size) {
// Continue the dst traversal.
traverse_valid = dst_impl.ContinueTraversal(std::addressof(dst_next_entry),
traverse_valid = ContinueTraversal(dst_impl, std::addressof(dst_next_entry),
std::addressof(dst_context));
ASSERT(traverse_valid);
@ -4547,7 +4577,7 @@ Result KPageTableBase::SetupForIpcServer(KProcessAddress* out_addr, size_t size,
// Begin traversal.
TraversalContext context;
TraversalEntry next_entry;
bool traverse_valid = src_impl.BeginTraversal(std::addressof(next_entry),
bool traverse_valid = BeginTraversal(src_impl, std::addressof(next_entry),
std::addressof(context), aligned_src_start);
ASSERT(traverse_valid);
@ -4597,7 +4627,7 @@ Result KPageTableBase::SetupForIpcServer(KProcessAddress* out_addr, size_t size,
// If the block's size was one page, we may need to continue traversal.
if (cur_block_size == 0 && aligned_src_size > PageSize) {
traverse_valid =
src_impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ContinueTraversal(src_impl, std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
cur_block_addr = next_entry.phys_addr;
@ -4610,7 +4640,7 @@ Result KPageTableBase::SetupForIpcServer(KProcessAddress* out_addr, size_t size,
while (aligned_src_start + tot_block_size < mapping_src_end) {
// Continue the traversal.
traverse_valid =
src_impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ContinueTraversal(src_impl, std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
// Process the block.
@ -4653,7 +4683,7 @@ Result KPageTableBase::SetupForIpcServer(KProcessAddress* out_addr, size_t size,
if (mapped_block_end + cur_block_size < aligned_src_end &&
cur_block_size == last_block_size) {
traverse_valid =
src_impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context));
ContinueTraversal(src_impl, std::addressof(next_entry), std::addressof(context));
ASSERT(traverse_valid);
cur_block_addr = next_entry.phys_addr;
@ -5601,7 +5631,7 @@ Result KPageTableBase::UnmapProcessMemory(KProcessAddress dst_address, size_t si
ContiguousRangeInfo(KPageTableBase& pt, KProcessAddress address, size_t size)
: m_pt(pt), m_remaining_size(size) {
// Begin a traversal.
ASSERT(m_pt.GetImpl().BeginTraversal(std::addressof(m_entry),
ASSERT(m_pt.BeginTraversal(m_pt.GetImpl(), std::addressof(m_entry),
std::addressof(m_context), address));
// Setup tracking fields.
@ -5632,7 +5662,7 @@ Result KPageTableBase::UnmapProcessMemory(KProcessAddress dst_address, size_t si
void DetermineContiguousBlockExtents() {
// Continue traversing until we're not contiguous, or we have enough.
while (m_cur_size < m_remaining_size) {
ASSERT(m_pt.GetImpl().ContinueTraversal(std::addressof(m_entry),
ASSERT(m_pt.ContinueTraversal(m_pt.GetImpl(), std::addressof(m_entry),
std::addressof(m_context)));
// If we're not contiguous, we're done.

13
src/core/hle/kernel/k_page_table_base.h

@ -369,6 +369,10 @@ private:
size_t num_pages, size_t alignment, size_t offset,
size_t guard_pages) const;
bool BeginTraversal(const Common::PageTable& impl, TraversalEntry* out_entry, TraversalContext* out_context,
Common::ProcessAddress address) const;
bool ContinueTraversal(const Common::PageTable& impl, TraversalEntry* out_entry, TraversalContext* context) const;
Result CheckMemoryStateContiguous(size_t* out_blocks_needed, KProcessAddress addr, size_t size,
KMemoryState state_mask, KMemoryState state,
KMemoryPermission perm_mask, KMemoryPermission perm,
@ -473,7 +477,14 @@ private:
// Validate pre-conditions.
ASSERT(this->IsLockedByCurrentThread());
return this->GetImpl().GetPhysicalAddress(out, virt_addr);
if (virt_addr > (1ULL << m_address_space_width)) {
return false;
}
*out = m_system.DeviceMemory().GetPhysicalAddr(
this->GetImpl().entries[virt_addr >> PageBits].Pointer());
return true;
}
public:

70
src/core/memory.cpp

@ -104,7 +104,7 @@ struct Memory::Impl {
current_page_table->entries.CommitRegion(vaddr >> YUZU_PAGEBITS, (vaddr + size) >> YUZU_PAGEBITS);
for (u64 addr = vaddr; addr < vaddr + size; addr += YUZU_PAGESIZE) {
const Common::PageType page_type = current_page_table->entries.GetUnchecked(addr >> YUZU_PAGEBITS).ptr.Type();
const Common::PageType page_type = current_page_table->entries.GetUnchecked(addr >> YUZU_PAGEBITS).Type();
switch (page_type) {
case Common::PageType::RasterizerCachedMemory:
if (protect_bytes > 0) {
@ -125,16 +125,14 @@ struct Memory::Impl {
}
[[nodiscard]] u8* GetPointerFromRasterizerCachedMemory(u64 vaddr) const {
Common::PhysicalAddress const paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].GetPhysOffset(YUZU_PAGEBITS);
if (paddr)
return system.DeviceMemory().GetPointer<u8>(paddr + vaddr);
if (u64 paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].Pointer(true); paddr)
return reinterpret_cast<u8*>(paddr) + (vaddr % YUZU_PAGESIZE);
return {};
}
[[nodiscard]] u8* GetPointerFromDebugMemory(u64 vaddr) const {
const Common::PhysicalAddress paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].GetPhysOffset(YUZU_PAGEBITS);
if (paddr != 0)
return system.DeviceMemory().GetPointer<u8>(paddr + vaddr);
if (u64 paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].Pointer(true); paddr)
return reinterpret_cast<u8*>(paddr) + (vaddr % YUZU_PAGESIZE);
return {};
}
@ -250,7 +248,7 @@ struct Memory::Impl {
while (remaining_size != 0) {
const std::size_t copy_amount = (std::min)(std::size_t(YUZU_PAGESIZE) - page_offset, remaining_size);
const auto current_vaddr = u64((page_index << YUZU_PAGEBITS) + page_offset);
const auto [pointer, type] = current_page_table->entries.GetUnchecked(page_index).ptr.PointerType();
const auto [pointer, type, _] = current_page_table->entries.GetUnchecked(page_index).PointerTypeBlock();
switch (type) {
case Common::PageType::Unmapped: {
user_accessible = false;
@ -258,7 +256,7 @@ struct Memory::Impl {
break;
}
case Common::PageType::Memory: {
u8* mem_ptr = reinterpret_cast<u8*>(pointer + page_offset + (page_index << YUZU_PAGEBITS));
u8* mem_ptr = reinterpret_cast<u8*>(pointer + page_offset);
on_memory(offset, copy_amount, mem_ptr);
break;
}
@ -301,10 +299,10 @@ struct Memory::Impl {
}
[[nodiscard]] inline const u8* GetSpan(const VAddr addr, const std::size_t size) const noexcept {
return (current_page_table->entries[addr >> YUZU_PAGEBITS].block == current_page_table->entries[(addr + size) >> YUZU_PAGEBITS].block) ? GetPointerSilent(addr) : nullptr;
return (current_page_table->entries[addr >> YUZU_PAGEBITS].Block() == current_page_table->entries[(addr + size) >> YUZU_PAGEBITS].Block()) ? GetPointerSilent(addr) : nullptr;
}
[[nodiscard]] inline u8* GetSpan(const VAddr addr, const std::size_t size) noexcept {
return (current_page_table->entries[addr >> YUZU_PAGEBITS].block == current_page_table->entries[(addr + size) >> YUZU_PAGEBITS].block) ? GetPointerSilent(addr) : nullptr;
return (current_page_table->entries[addr >> YUZU_PAGEBITS].Block() == current_page_table->entries[(addr + size) >> YUZU_PAGEBITS].Block()) ? GetPointerSilent(addr) : nullptr;
}
bool WriteBlockImpl(const Common::ProcessAddress addr, const void* buffer, const std::size_t size, bool unsafe) {
@ -412,10 +410,10 @@ struct Memory::Impl {
current_page_table->entries.CommitRegion(vaddr >> YUZU_PAGEBITS, (vaddr >> YUZU_PAGEBITS) + num_pages);
for (u64 i = 0; i < num_pages; ++i, vaddr += YUZU_PAGESIZE) {
auto& entry = current_page_table->entries.GetUnchecked(vaddr >> YUZU_PAGEBITS);
const Common::PageType page_type = entry.ptr.Type();
const auto [pointer, type, block] = entry.PointerTypeBlock();
if (debug) {
// Switch page type to debug if now debug
switch (page_type) {
switch (type) {
case Common::PageType::Unmapped:
ASSERT(false && "Attempted to mark unmapped pages as debug");
break;
@ -424,14 +422,14 @@ struct Memory::Impl {
// Page is already marked.
break;
case Common::PageType::Memory:
entry.ptr.Store(0, Common::PageType::DebugMemory);
entry.MarkDebug(pointer, block);
break;
default:
UNREACHABLE();
}
} else {
// Switch page type to non-debug if now non-debug
switch (page_type) {
switch (type) {
case Common::PageType::Unmapped:
ASSERT(false && "Attempted to mark unmapped pages as non-debug");
break;
@ -440,8 +438,7 @@ struct Memory::Impl {
// Don't mess with already non-debug or rasterizer memory.
break;
case Common::PageType::DebugMemory: {
u8* const pointer = GetPointerFromDebugMemory(vaddr & ~YUZU_PAGEMASK);
entry.ptr.Store(uintptr_t(pointer) - (vaddr & ~YUZU_PAGEMASK), Common::PageType::Memory);
entry.Store(false, Common::PageType::Memory, block, pointer);
break;
}
default:
@ -476,7 +473,7 @@ struct Memory::Impl {
current_page_table->entries.CommitRegion(vaddr >> YUZU_PAGEBITS, (vaddr >> YUZU_PAGEBITS) + num_pages);
for (u64 i = 0; i < num_pages; ++i, vaddr += YUZU_PAGESIZE) {
auto& entry = current_page_table->entries.GetUnchecked(vaddr >> YUZU_PAGEBITS);
const Common::PageType page_type = entry.ptr.Type();
const Common::PageType page_type = entry.Type();
if (cached) {
// Switch page type to cached if now cached
switch (page_type) {
@ -486,7 +483,7 @@ struct Memory::Impl {
break;
case Common::PageType::DebugMemory:
case Common::PageType::Memory:
entry.ptr.Store(0, Common::PageType::RasterizerCachedMemory);
entry.MarkRasterizerCached();
break;
case Common::PageType::RasterizerCachedMemory:
// There can be more than one GPU region mapped per CPU region, so it's common
@ -508,13 +505,14 @@ struct Memory::Impl {
// that this area is already unmarked as cached.
break;
case Common::PageType::RasterizerCachedMemory: {
if (u8* const pointer = GetPointerFromRasterizerCachedMemory(vaddr & ~YUZU_PAGEMASK); pointer == nullptr) {
if (auto [ptr, _, block] = entry.PointerTypeBlock(); ptr == 0) {
// It's possible that this function has been called while updating the
// pagetable after unmapping a VMA. In that case the underlying VMA will no
// longer exist, and we should just leave the pagetable entry blank.
entry.ptr.Store(0, Common::PageType::Unmapped);
// TODO: can this just set entry to 0?
entry.Store(true, Common::PageType::Unmapped, block, ptr);
} else {
entry.ptr.Store(uintptr_t(pointer) - (vaddr & ~YUZU_PAGEMASK), Common::PageType::Memory);
entry.Store(false, Common::PageType::Memory, block, ptr);
}
break;
}
@ -550,19 +548,17 @@ struct Memory::Impl {
page_table.entries.ZeroRegion(base, end);
} else {
auto orig_base = base;
static std::atomic<u16> block = 0;
auto current_block = block.fetch_add(1);
ASSERT(current_block <= 512);
page_table.entries.CommitRegion(base, end);
while (base != end) {
auto host_ptr = uintptr_t(system.DeviceMemory().GetPointer<u8>(target)) - (base << YUZU_PAGEBITS);
auto host_ptr = reinterpret_cast<u64>(system.DeviceMemory().GetPointer<u8>(target));
auto& entry = page_table.entries.GetUnchecked(base);
entry.ptr.Store(host_ptr, type);
// TODO: see comments in PageTable::GetPhysOffset
entry.addr = static_cast<u32>(GetInteger(target) >> YUZU_PAGEBITS) - static_cast<u32>(base);
entry.block = static_cast<u32>(orig_base);
ASSERT_MSG(page_table.entries[base].ptr.Pointer(),
entry.Store(false, type, current_block, host_ptr);
ASSERT_MSG(page_table.entries[base].Pointer(),
"memory mapping base yield a nullptr within the table");
base += 1;
@ -577,11 +573,11 @@ struct Memory::Impl {
vaddr &= 0xffffffffffffULL;
if (AddressSpaceContains(*current_page_table, vaddr, 1)) [[likely]] {
// Avoid adding any extra logic to this fast-path block
const uintptr_t raw_pointer = current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Raw();
if (const uintptr_t pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) [[likely]] {
return reinterpret_cast<u8*>(pointer + vaddr);
const auto raw = current_page_table->entries[vaddr >> YUZU_PAGEBITS].Raw();
if (auto pointer = Common::PageTable::PageEntryData::ExtractPointer(raw); pointer) [[likely]] {
return reinterpret_cast<u8*>(pointer + vaddr % YUZU_PAGESIZE);
} else {
switch (Common::PageTable::PageInfo::ExtractType(raw_pointer)) {
switch (static_cast<Common::PageType>(raw.type)) {
case Common::PageType::Memory:
ASSERT_MSG(false, "Mapped memory page without a pointer @ {:#016x}", vaddr);
return nullptr;
@ -819,8 +815,10 @@ bool Memory::IsValidVirtualAddress(const Common::ProcessAddress vaddr) const {
if (page >= page_table.entries.size()) {
return false;
}
const auto [pointer, type] = page_table.entries[page].ptr.PointerType();
return pointer != 0 || type == Common::PageType::RasterizerCachedMemory ||
const auto raw = page_table.entries[page].Raw();
const auto type = static_cast<Common::PageType>(raw.type);
return raw.page != 0 || type == Common::PageType::RasterizerCachedMemory ||
type == Common::PageType::DebugMemory;
}

1
src/dynarmic/src/dynarmic/backend/arm64/a32_address_space.cpp

@ -373,6 +373,7 @@ EmitConfig A32AddressSpace::GetEmitConfig() {
.page_table_address_space_bits = 32,
.page_table_pointer_mask_bits = conf.page_table_pointer_mask_bits,
.page_table_log2_stride = conf.page_table_log2_stride,
.page_table_marked_bit = conf.page_table_marked_bit,
.silently_mirror_page_table = true,
.absolute_offset_page_table = conf.absolute_offset_page_table,
.detect_misaligned_access_via_page_table = conf.detect_misaligned_access_via_page_table,

1
src/dynarmic/src/dynarmic/backend/arm64/a64_address_space.cpp

@ -547,6 +547,7 @@ EmitConfig A64AddressSpace::GetEmitConfig() {
.page_table_address_space_bits = conf.page_table_address_space_bits,
.page_table_pointer_mask_bits = conf.page_table_pointer_mask_bits,
.page_table_log2_stride = conf.page_table_log2_stride,
.page_table_marked_bit = conf.page_table_marked_bit,
.silently_mirror_page_table = conf.silently_mirror_page_table,
.absolute_offset_page_table = conf.absolute_offset_page_table,
.detect_misaligned_access_via_page_table = conf.detect_misaligned_access_via_page_table,

1
src/dynarmic/src/dynarmic/backend/arm64/emit_arm64.h

@ -130,6 +130,7 @@ struct EmitConfig {
std::size_t page_table_address_space_bits;
int page_table_pointer_mask_bits;
std::size_t page_table_log2_stride;
std::optional<std::uint16_t> page_table_marked_bit;
bool silently_mirror_page_table;
bool absolute_offset_page_table;
u8 detect_misaligned_access_via_page_table;

7
src/dynarmic/src/dynarmic/backend/arm64/emit_arm64_memory.cpp

@ -273,6 +273,13 @@ std::pair<oaknut::XReg, oaknut::XReg> InlinePageTableEmitVAddrLookup(oaknut::Cod
// load x0 = *<(u8*)pagetable + index>
code.LDR(Xscratch0, Xpagetable, Xscratch0);
if (ctx.conf.page_table_marked_bit) {
// check for marked bit
code.TST(Xscratch0, 1ULL << *ctx.conf.page_table_marked_bit);
// if marked, view this page as unmapped
code.CSEL(Xscratch0, Xscratch0, XZR, EQ);
}
if (ctx.conf.page_table_pointer_mask_bits != 0) {
const u64 mask = u64(~u64(0)) << ctx.conf.page_table_pointer_mask_bits;
code.AND(Xscratch0, Xscratch0, mask);

14
src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.h

@ -88,6 +88,20 @@ template<>
code.shr(tmp, int(page_table_const_bits));
code.shl(tmp, int(ctx.conf.page_table_log2_stride));
code.mov(page, qword[r14 + tmp.cvt64()]);
// check for marked bit, use as unmapped if marked
if (ctx.conf.page_table_marked_bit) {
// zero page, we can use it as scratch register before it's initialized
code.xor_(page, page);
if (*ctx.conf.page_table_marked_bit >= 30) {
code.bt(tmp, *ctx.conf.page_table_marked_bit);
code.cmovc(tmp, page);
} else {
code.test(tmp, 1ULL << *ctx.conf.page_table_marked_bit);
code.cmovnz(tmp, page);
}
}
// mask away attributes
if (ctx.conf.page_table_pointer_mask_bits == 0) {
code.test(page, page);
} else {

9
src/dynarmic/src/dynarmic/interface/A32/config.h

@ -165,8 +165,13 @@ struct UserConfig {
/// If the configured value is 3, all pointers will be forcefully aligned to 8 bytes.
std::int32_t page_table_pointer_mask_bits = 0;
// Log2 of the size per page entry, value should be either 3 or 4
std::size_t page_table_log2_stride = 3;
/// Log2 of the size per page entry, value should be either 3 or 4
std::uint32_t page_table_log2_stride = 3;
/// Setting this value has Dynarmic check the specified bit of the page pointer provided by page table.
/// If the bit is set to 1, Dynarmic will treat it as unmapped.
/// This bit should be included as part of `page_table_pointer_mask_bits`.
std::optional<std::uint16_t> page_table_marked_bit = std::nullopt;
/// Select the architecture version to use.
/// There are minor behavioural differences between versions.

9
src/dynarmic/src/dynarmic/interface/A64/config.h

@ -179,8 +179,13 @@ struct UserConfig {
/// If the configured value is 3, all pointers will be forcefully aligned to 8 bytes.
std::int32_t page_table_pointer_mask_bits = 0;
// Log2 of the size per page entry, value should be either 3 or 4
std::size_t page_table_log2_stride = 3;
/// Log2 of the size per page entry, value should be either 3 or 4
std::uint32_t page_table_log2_stride = 3;
/// Setting this value has Dynarmic check the specified bit of the page pointer provided by page table.
/// If the bit is set to 1, Dynarmic will treat it as unmapped.
/// This bit should be included as part of `page_table_pointer_mask_bits`.
std::optional<std::uint16_t> page_table_marked_bit = std::nullopt;
/// Counter-timer frequency register. The value of the register is not interpreted by
/// dynarmic.

Loading…
Cancel
Save