diff --git a/src/common/page_table.h b/src/common/page_table.h index 88e071a74e..35c76bb58c 100644 --- a/src/common/page_table.h +++ b/src/common/page_table.h @@ -42,9 +42,8 @@ struct PageTable { u64 next_offset{}; }; - /// Number of bits reserved for attribute tagging. - /// This can be at most the guaranteed alignment of the pointers in the page table. - static constexpr int ATTRIBUTE_BITS = 12; + /// Masks out bits reserved for attribute tagging. + static constexpr u64 ATTRIBUTE_MASK = ((1ULL << 44) - 1) << 12; /** * Atomic tuple of host pointer, page type, and block id. @@ -55,15 +54,17 @@ struct PageTable { class PageEntryData { public: struct Data { - Data(bool marked_, PageType type_, u16 block_, u64 page_) - : marked(static_cast(marked_) & 0b1) - , type(static_cast(type_) & ((1ULL << 2) - 1)) - , block(static_cast(block_) & ((1ULL << 9) - 1)) - , page((page_ >> ATTRIBUTE_BITS) & ((1ULL << 52) - 1)) {} + Data(bool marked_, PageType type_, u32 block_, u64 page_) + : marked(static_cast(marked_) & 0b1) + , type(static_cast(type_) & ((1ULL << 2) - 1)) + , block(static_cast(block_) & ((1ULL << 9) - 1)) + , page((page_ >> 12) & ((1ULL << 44) - 1)) + , block2((static_cast(block_) << 9) & ((1ULL << 8) - 1)) {} u64 marked : 1; u64 type : 2; - u64 block : 9; // TODO: is 9 bits to little? we can use the upper 8 bits if needed - u64 page : 52; + u64 block : 9; + u64 page : 44; // first 12 bits are page offset, last 8 bits are architecturally reserved + u64 block2 : 8; }; [[nodiscard]] Data Raw() const noexcept { @@ -81,18 +82,18 @@ struct PageTable { } /// Returns the block identifier. - [[nodiscard]] u16 Block() const noexcept { - return static_cast(std::bit_cast(data_raw.load(std::memory_order_relaxed)).block); + [[nodiscard]] u32 Block() const noexcept { + return ExtractBlock(std::bit_cast(data_raw.load(std::memory_order_relaxed))); } /// Returns the page pointer and attribute pair, extracted from the same atomic read [[nodiscard]] std::tuple PointerTypeBlock(bool ignore_marked = false) const noexcept { const auto non_atomic_raw = std::bit_cast(data_raw.load(std::memory_order_relaxed)); - return {ExtractPointer(non_atomic_raw, ignore_marked), static_cast(non_atomic_raw.type), static_cast(non_atomic_raw.block)}; + return {ExtractPointer(non_atomic_raw, ignore_marked), static_cast(non_atomic_raw.type), ExtractBlock(non_atomic_raw)}; } /// Write page info atomically - constexpr void Store(bool marked, PageType type, u16 block, uintptr_t pointer) noexcept { + constexpr void Store(bool marked, PageType type, u32 block, uintptr_t pointer) noexcept { data_raw.store(std::bit_cast(Data{marked, type, block, pointer})); } @@ -106,7 +107,11 @@ struct PageTable { /// Unpack a pointer from a page info raw representation [[nodiscard]] static uintptr_t ExtractPointer(Data raw, bool ignore_marked = false) noexcept { - return raw.marked && !ignore_marked ? 0 : raw.page << ATTRIBUTE_BITS; + return raw.marked && !ignore_marked ? 0 : raw.page << 12; + } + + [[nodiscard]] static u32 ExtractBlock(Data raw) noexcept { + return raw.block | (raw.block2 << 9); } private: diff --git a/src/core/arm/dynarmic/arm_dynarmic_32.cpp b/src/core/arm/dynarmic/arm_dynarmic_32.cpp index 4266df613d..886ebab44b 100644 --- a/src/core/arm/dynarmic/arm_dynarmic_32.cpp +++ b/src/core/arm/dynarmic/arm_dynarmic_32.cpp @@ -176,7 +176,7 @@ void ArmDynarmic32::MakeJit(Common::PageTable* page_table) { // Dynarmic will not write to the page table, const_cast is safe here config.page_table = reinterpret_cast*>( const_cast(page_table->entries.data())); - config.page_table_pointer_mask_bits = Common::PageTable::ATTRIBUTE_BITS; + config.page_table_pointer_mask = Common::PageTable::ATTRIBUTE_MASK; config.page_table_marked_bit = 0; config.absolute_offset_page_table = true; config.detect_misaligned_access_via_page_table = 16 | 32 | 64 | 128; diff --git a/src/core/arm/dynarmic/arm_dynarmic_64.cpp b/src/core/arm/dynarmic/arm_dynarmic_64.cpp index 3549295835..51e2e70b87 100644 --- a/src/core/arm/dynarmic/arm_dynarmic_64.cpp +++ b/src/core/arm/dynarmic/arm_dynarmic_64.cpp @@ -215,7 +215,7 @@ void ArmDynarmic64::MakeJit(Common::PageTable* page_table, std::size_t address_s config.page_table = reinterpret_cast( const_cast(page_table->entries.data())); config.page_table_address_space_bits = std::uint32_t(address_space_bits); - config.page_table_pointer_mask_bits = Common::PageTable::ATTRIBUTE_BITS; + config.page_table_pointer_mask = Common::PageTable::ATTRIBUTE_MASK; config.page_table_marked_bit = 0; config.silently_mirror_page_table = false; config.absolute_offset_page_table = true; diff --git a/src/core/memory.cpp b/src/core/memory.cpp index 1158f27d79..11da2eec0e 100644 --- a/src/core/memory.cpp +++ b/src/core/memory.cpp @@ -606,9 +606,9 @@ struct Memory::Impl { } } - static std::atomic block = 0; - auto current_block = block.fetch_add(1); - ASSERT(current_block <= 512); + static std::atomic block = 0; + auto current_block = block.fetch_add(1, std::memory_order_relaxed); + ASSERT(current_block <= 131071); page_table.entries.CommitRegion(base, end); while (base != end) { diff --git a/src/dynarmic/src/dynarmic/backend/arm64/a32_address_space.cpp b/src/dynarmic/src/dynarmic/backend/arm64/a32_address_space.cpp index 64dd0b85ed..81931dce67 100644 --- a/src/dynarmic/src/dynarmic/backend/arm64/a32_address_space.cpp +++ b/src/dynarmic/src/dynarmic/backend/arm64/a32_address_space.cpp @@ -371,7 +371,7 @@ EmitConfig A32AddressSpace::GetEmitConfig() { .page_table_pointer = std::bit_cast(conf.page_table), .page_table_address_space_bits = 32, - .page_table_pointer_mask_bits = conf.page_table_pointer_mask_bits, + .page_table_pointer_mask = conf.page_table_pointer_mask, .page_table_log2_stride = conf.page_table_log2_stride, .page_table_marked_bit = conf.page_table_marked_bit, .silently_mirror_page_table = true, diff --git a/src/dynarmic/src/dynarmic/backend/arm64/a64_address_space.cpp b/src/dynarmic/src/dynarmic/backend/arm64/a64_address_space.cpp index e65c1d90f4..966a22a942 100644 --- a/src/dynarmic/src/dynarmic/backend/arm64/a64_address_space.cpp +++ b/src/dynarmic/src/dynarmic/backend/arm64/a64_address_space.cpp @@ -545,7 +545,7 @@ EmitConfig A64AddressSpace::GetEmitConfig() { .page_table_pointer = std::bit_cast(conf.page_table), .page_table_address_space_bits = conf.page_table_address_space_bits, - .page_table_pointer_mask_bits = conf.page_table_pointer_mask_bits, + .page_table_pointer_mask = conf.page_table_pointer_mask, .page_table_log2_stride = conf.page_table_log2_stride, .page_table_marked_bit = conf.page_table_marked_bit, .silently_mirror_page_table = conf.silently_mirror_page_table, diff --git a/src/dynarmic/src/dynarmic/backend/arm64/emit_arm64.h b/src/dynarmic/src/dynarmic/backend/arm64/emit_arm64.h index 8016429392..73d04cebf8 100644 --- a/src/dynarmic/src/dynarmic/backend/arm64/emit_arm64.h +++ b/src/dynarmic/src/dynarmic/backend/arm64/emit_arm64.h @@ -128,7 +128,7 @@ struct EmitConfig { // Page table u64 page_table_pointer; std::size_t page_table_address_space_bits; - int page_table_pointer_mask_bits; + u64 page_table_pointer_mask; std::size_t page_table_log2_stride; std::optional page_table_marked_bit; bool silently_mirror_page_table; diff --git a/src/dynarmic/src/dynarmic/backend/arm64/emit_arm64_memory.cpp b/src/dynarmic/src/dynarmic/backend/arm64/emit_arm64_memory.cpp index 7cb7f0a630..b0da74d4f2 100644 --- a/src/dynarmic/src/dynarmic/backend/arm64/emit_arm64_memory.cpp +++ b/src/dynarmic/src/dynarmic/backend/arm64/emit_arm64_memory.cpp @@ -280,9 +280,8 @@ std::pair InlinePageTableEmitVAddrLookup(oaknut::Cod code.CSEL(Xscratch0, Xscratch0, XZR, EQ); } - if (ctx.conf.page_table_pointer_mask_bits != 0) { - const u64 mask = u64(~u64(0)) << ctx.conf.page_table_pointer_mask_bits; - code.AND(Xscratch0, Xscratch0, mask); + if (ctx.conf.page_table_pointer_mask != 0) { + code.AND(Xscratch0, Xscratch0, ctx.conf.page_table_pointer_mask); } code.CBZ(Xscratch0, *fallback); diff --git a/src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.h b/src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.h index 71d1f6a7e3..8cb8f7b34f 100644 --- a/src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.h +++ b/src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.h @@ -102,10 +102,11 @@ template<> } } // mask away attributes - if (ctx.conf.page_table_pointer_mask_bits == 0) { + if (ctx.conf.page_table_pointer_mask == 0) { code.test(page, page); } else { - code.and_(page, ~u32(0) << ctx.conf.page_table_pointer_mask_bits); + code.mov(tmp, ctx.conf.page_table_pointer_mask); + code.and_(page, tmp); } code.jz(abort, code.T_NEAR); if (ctx.conf.absolute_offset_page_table) { @@ -157,10 +158,25 @@ template<> code.shl(tmp, int(ctx.conf.page_table_log2_stride)); code.mov(page, qword[r14 + tmp]); - if (ctx.conf.page_table_pointer_mask_bits == 0) { + + // check for marked bit, use as unmapped if marked + if (ctx.conf.page_table_marked_bit) { + // zero page, we can use it as scratch register before it's initialized + code.xor_(page, page); + if (*ctx.conf.page_table_marked_bit >= 30) { + code.bt(tmp, *ctx.conf.page_table_marked_bit); + code.cmovc(tmp, page); + } else { + code.test(tmp, 1ULL << *ctx.conf.page_table_marked_bit); + code.cmovnz(tmp, page); + } + } + // mask away attributes + if (ctx.conf.page_table_pointer_mask == 0) { code.test(page, page); } else { - code.and_(page, ~u32(0) << ctx.conf.page_table_pointer_mask_bits); + code.mov(tmp, ctx.conf.page_table_pointer_mask); + code.and_(page, tmp); } code.jz(abort, code.T_NEAR); if (ctx.conf.absolute_offset_page_table) { diff --git a/src/dynarmic/src/dynarmic/interface/A32/config.h b/src/dynarmic/src/dynarmic/interface/A32/config.h index 6f99a51e55..a4988e7b45 100644 --- a/src/dynarmic/src/dynarmic/interface/A32/config.h +++ b/src/dynarmic/src/dynarmic/interface/A32/config.h @@ -159,11 +159,11 @@ struct UserConfig { /// Maximum size is limited by the maximum length of a x86_64 / arm64 jump. std::uint32_t code_cache_size = 128 * 1024 * 1024; // bytes - /// Masks out the first N bits in host pointers from the page table. + /// Applies a bit mask to the bits in host pointers from the page table. /// The intention behind this is to allow users of Dynarmic to pack attributes in the /// same integer and update the pointer attribute pair atomically. - /// If the configured value is 3, all pointers will be forcefully aligned to 8 bytes. - std::int32_t page_table_pointer_mask_bits = 0; + /// If the configured value is ~(0b111ULL), all pointers will be forcefully aligned to 8 bytes. + std::uint64_t page_table_pointer_mask = 0; /// Log2 of the size per page entry, value should be either 3 or 4 std::uint32_t page_table_log2_stride = 3; diff --git a/src/dynarmic/src/dynarmic/interface/A64/config.h b/src/dynarmic/src/dynarmic/interface/A64/config.h index e74b41c259..834b4dc5b3 100644 --- a/src/dynarmic/src/dynarmic/interface/A64/config.h +++ b/src/dynarmic/src/dynarmic/interface/A64/config.h @@ -173,18 +173,18 @@ struct UserConfig { /// This is only used if page_table is not nullptr. std::uint32_t page_table_address_space_bits = 36; - /// Masks out the first N bits in host pointers from the page table. + /// Applies a bit mask to the bits in host pointers from the page table. /// The intention behind this is to allow users of Dynarmic to pack attributes in the /// same integer and update the pointer attribute pair atomically. - /// If the configured value is 3, all pointers will be forcefully aligned to 8 bytes. - std::int32_t page_table_pointer_mask_bits = 0; + /// If the configured value is ~(0b111ULL), all pointers will be forcefully aligned to 8 bytes. + std::uint64_t page_table_pointer_mask = 0; /// Log2 of the size per page entry, value should be either 3 or 4 std::uint32_t page_table_log2_stride = 3; /// Setting this value has Dynarmic check the specified bit of the page pointer provided by page table. /// If the bit is set to 1, Dynarmic will treat it as unmapped. - /// This bit should be included as part of `page_table_pointer_mask_bits`. + /// This bit should be included as part of `page_table_pointer_mask`. std::optional page_table_marked_bit = std::nullopt; /// Counter-timer frequency register. The value of the register is not interpreted by