diff --git a/src/common/sparse_large_vector.h b/src/common/sparse_large_vector.h index 917607118f..fb58b11277 100644 --- a/src/common/sparse_large_vector.h +++ b/src/common/sparse_large_vector.h @@ -17,6 +17,7 @@ #include #endif +#include "common/alignment.h" #include "common/assert.h" namespace Common { @@ -122,6 +123,20 @@ public: std::memset(reinterpret_cast(&base_ptr[index]), 0, sizeof(T)); } + constexpr void CommitRegion(size_t index, size_t end_) { + auto base = index * sizeof(T); + auto end = end_ * sizeof(T); + + while (base < end) { + CommitPage(base); + base = AlignDown(base, HostPageSize) + HostPageSize; + } + } + + constexpr T& GetUnchecked(size_t index) { + return base_ptr[index]; + } + [[nodiscard]] constexpr const T& operator[](std::size_t index) const noexcept { return GetOrDefault(index); } diff --git a/src/core/device_memory_manager.h b/src/core/device_memory_manager.h index 0ef014155b..b4b3b46088 100644 --- a/src/core/device_memory_manager.h +++ b/src/core/device_memory_manager.h @@ -200,8 +200,8 @@ private: return std::make_pair(asid, address); } - void InsertCPUBacking(size_t page_index, VAddr address, Asid asid) { - tracked_entries.GetAndFault(page_index).cpu_backing_address = address | (asid.id << asid_start_bit); + constexpr void InsertCPUBacking(size_t page_index, VAddr address, Asid asid) { + tracked_entries.GetUnchecked(page_index).cpu_backing_address = address | (asid.id << asid_start_bit); } std::array t_slot{}; diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index 484f5bf5e7..4906b0715b 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -180,6 +180,7 @@ DeviceMemoryManager::DeviceMemoryManager(const DeviceMemory& device_memo const size_t total_virtual = device_as_size >> Memory::YUZU_PAGEBITS; + // TODO: this is stupid, make continuity_tracker default to 0 so we can benefit from SparseLargeVector auto virtual_entry = TrackedEntry {}; virtual_entry.compressed_physical_ptr = 0; virtual_entry.continuity_tracker = 1; @@ -187,10 +188,6 @@ DeviceMemoryManager::DeviceMemoryManager(const DeviceMemory& device_memo for (size_t i = 0; i < total_virtual; i++) { tracked_entries.Set(i, virtual_entry); } - const size_t total_phys = 1ULL << ((Settings::values.memory_layout_mode.GetValue() == Settings::MemoryLayout::Memory_4Gb ? physical_min_bits : physical_max_bits) - Memory::YUZU_PAGEBITS); - for (size_t i = 0; i < total_phys; i++) { - compressed_device_addr.Zero(i); - } } template @@ -223,17 +220,17 @@ void DeviceMemoryManager::Map(DAddr address, VAddr virtual_address, size size_t start_page_d = address >> Memory::YUZU_PAGEBITS; size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS; std::scoped_lock lk(mapping_guard); + + tracked_entries.CommitRegion(start_page_d, start_page_d + num_pages); for (size_t i = 0; i < num_pages; i++) { const VAddr new_vaddress = virtual_address + i * Memory::YUZU_PAGESIZE; auto* ptr = process_memory->GetPointerSilent(Common::ProcessAddress(new_vaddress)); if (ptr == nullptr) [[unlikely]] { - if (auto v = tracked_entries.GetNoFault(start_page_d + i); v) { - v->compressed_physical_ptr = 0; - } + tracked_entries.GetUnchecked(start_page_d + i).compressed_physical_ptr = 0; continue; } auto phys_addr = static_cast(GetRawPhysicalAddr(ptr) >> Memory::YUZU_PAGEBITS) + 1U; - tracked_entries.GetAndFault(start_page_d + i).compressed_physical_ptr = phys_addr; + tracked_entries.GetUnchecked(start_page_d + i).compressed_physical_ptr = phys_addr; InsertCPUBacking(start_page_d + i, new_vaddress, asid); const u32 base_dev = compressed_device_addr[phys_addr - 1U]; const u32 new_dev = static_cast(start_page_d + i); @@ -260,25 +257,26 @@ void DeviceMemoryManager::Unmap(DAddr address, size_t size) { size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS; device_inter->InvalidateRegion(address, size); std::scoped_lock lk(mapping_guard); + + tracked_entries.CommitRegion(start_page_d, start_page_d + num_pages); // should already be committed, but just in case for (size_t i = 0; i < num_pages; i++) { - auto& entry = tracked_entries.GetAndFault(start_page_d + i); + auto& entry = tracked_entries.GetUnchecked(start_page_d + i); auto phys_addr = entry.compressed_physical_ptr; entry.compressed_physical_ptr = 0; entry.cpu_backing_address = 0; if (phys_addr != 0) [[likely]] { - const u32 base_dev = compressed_device_addr[phys_addr - 1U]; + u32& base_dev = compressed_device_addr.GetAndFault(phys_addr - 1U); if ((base_dev >> MULTI_FLAG_BITS) == 0) [[likely]] { - compressed_device_addr.Zero(phys_addr - 1); + base_dev = 0; continue; } const auto [more_entries, new_start] = impl->multi_dev_address.Unregister( static_cast(start_page_d + i), base_dev & MULTI_MASK); if (!more_entries) { - compressed_device_addr.Set(phys_addr - 1, - impl->multi_dev_address.ReleaseEntry(new_start)); + base_dev = impl->multi_dev_address.ReleaseEntry(new_start); continue; } - compressed_device_addr.Set(phys_addr - 1, new_start | MULTI_FLAG); + base_dev = new_start | MULTI_FLAG; } } t_slot = {}; @@ -291,6 +289,8 @@ void DeviceMemoryManager::TrackContinuityImpl(DAddr address, VAddr virtu size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS; uintptr_t last_ptr = 0; size_t page_count = 1; + + tracked_entries.CommitRegion(start_page_d, start_page_d + num_pages); for (size_t i = num_pages; i > 0; i--) { size_t index = i - 1; const VAddr new_vaddress = virtual_address + index * Memory::YUZU_PAGESIZE; @@ -302,7 +302,7 @@ void DeviceMemoryManager::TrackContinuityImpl(DAddr address, VAddr virtu page_count = 1; } last_ptr = new_ptr; - tracked_entries.GetAndFault(start_page_d + index).continuity_tracker = static_cast(page_count); + tracked_entries.GetUnchecked(start_page_d + index).continuity_tracker = static_cast(page_count); } } template diff --git a/src/core/memory.cpp b/src/core/memory.cpp index 9f9c0f6585..9b4e0d878b 100644 --- a/src/core/memory.cpp +++ b/src/core/memory.cpp @@ -101,8 +101,10 @@ struct Memory::Impl { } u64 protect_bytes = 0, protect_begin = 0; + + current_page_table->entries.CommitRegion(vaddr >> YUZU_PAGEBITS, (vaddr + size) >> YUZU_PAGEBITS); for (u64 addr = vaddr; addr < vaddr + size; addr += YUZU_PAGESIZE) { - const Common::PageType page_type = current_page_table->entries[addr >> YUZU_PAGEBITS].ptr.Type(); + const Common::PageType page_type = current_page_table->entries.GetUnchecked(addr >> YUZU_PAGEBITS).ptr.Type(); switch (page_type) { case Common::PageType::RasterizerCachedMemory: if (protect_bytes > 0) { @@ -243,10 +245,12 @@ struct Memory::Impl { std::size_t page_index = addr >> YUZU_PAGEBITS; std::size_t page_offset = addr & YUZU_PAGEMASK; bool user_accessible = true; + + current_page_table->entries.CommitRegion(page_index, page_index + (size >> YUZU_PAGEBITS) + 1); while (remaining_size != 0) { const std::size_t copy_amount = (std::min)(std::size_t(YUZU_PAGESIZE) - page_offset, remaining_size); const auto current_vaddr = u64((page_index << YUZU_PAGEBITS) + page_offset); - const auto [pointer, type] = current_page_table->entries[page_index].ptr.PointerType(); + const auto [pointer, type] = current_page_table->entries.GetUnchecked(page_index).ptr.PointerType(); switch (type) { case Common::PageType::Unmapped: { user_accessible = false; @@ -404,8 +408,11 @@ struct Memory::Impl { // The region is at a granularity of CPU pages. const u64 num_pages = ((vaddr + size - 1) >> YUZU_PAGEBITS) - (vaddr >> YUZU_PAGEBITS) + 1; + + current_page_table->entries.CommitRegion(vaddr >> YUZU_PAGEBITS, (vaddr >> YUZU_PAGEBITS) + num_pages); for (u64 i = 0; i < num_pages; ++i, vaddr += YUZU_PAGESIZE) { - const Common::PageType page_type = current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Type(); + auto& entry = current_page_table->entries.GetUnchecked(vaddr >> YUZU_PAGEBITS); + const Common::PageType page_type = entry.ptr.Type(); if (debug) { // Switch page type to debug if now debug switch (page_type) { @@ -417,7 +424,7 @@ struct Memory::Impl { // Page is already marked. break; case Common::PageType::Memory: - current_page_table->entries.GetAndFault(vaddr >> YUZU_PAGEBITS).ptr.Store(0, Common::PageType::DebugMemory); + entry.ptr.Store(0, Common::PageType::DebugMemory); break; default: UNREACHABLE(); @@ -434,7 +441,7 @@ struct Memory::Impl { break; case Common::PageType::DebugMemory: { u8* const pointer = GetPointerFromDebugMemory(vaddr & ~YUZU_PAGEMASK); - current_page_table->entries.GetAndFault(vaddr >> YUZU_PAGEBITS).ptr.Store(uintptr_t(pointer) - (vaddr & ~YUZU_PAGEMASK), Common::PageType::Memory); + entry.ptr.Store(uintptr_t(pointer) - (vaddr & ~YUZU_PAGEMASK), Common::PageType::Memory); break; } default: @@ -466,8 +473,10 @@ struct Memory::Impl { // is different). This assumes the specified GPU address region is contiguous as well. const u64 num_pages = ((vaddr + size - 1) >> YUZU_PAGEBITS) - (vaddr >> YUZU_PAGEBITS) + 1; + current_page_table->entries.CommitRegion(vaddr >> YUZU_PAGEBITS, (vaddr >> YUZU_PAGEBITS) + num_pages); for (u64 i = 0; i < num_pages; ++i, vaddr += YUZU_PAGESIZE) { - const Common::PageType page_type= current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Type(); + auto& entry = current_page_table->entries.GetUnchecked(vaddr >> YUZU_PAGEBITS); + const Common::PageType page_type = entry.ptr.Type(); if (cached) { // Switch page type to cached if now cached switch (page_type) { @@ -477,7 +486,7 @@ struct Memory::Impl { break; case Common::PageType::DebugMemory: case Common::PageType::Memory: - current_page_table->entries.GetAndFault(vaddr >> YUZU_PAGEBITS).ptr.Store(0, Common::PageType::RasterizerCachedMemory); + entry.ptr.Store(0, Common::PageType::RasterizerCachedMemory); break; case Common::PageType::RasterizerCachedMemory: // There can be more than one GPU region mapped per CPU region, so it's common @@ -503,9 +512,9 @@ struct Memory::Impl { // It's possible that this function has been called while updating the // pagetable after unmapping a VMA. In that case the underlying VMA will no // longer exist, and we should just leave the pagetable entry blank. - current_page_table->entries.GetAndFault(vaddr >> YUZU_PAGEBITS).ptr.Store(0, Common::PageType::Unmapped); + entry.ptr.Store(0, Common::PageType::Unmapped); } else { - current_page_table->entries.GetAndFault(vaddr >> YUZU_PAGEBITS).ptr.Store(uintptr_t(pointer) - (vaddr & ~YUZU_PAGEMASK), Common::PageType::Memory); + entry.ptr.Store(uintptr_t(pointer) - (vaddr & ~YUZU_PAGEMASK), Common::PageType::Memory); } break; } @@ -546,17 +555,17 @@ struct Memory::Impl { } } else { auto orig_base = base; + + page_table.entries.CommitRegion(base, end); while (base != end) { auto host_ptr = uintptr_t(system.DeviceMemory().GetPointer(target)) - (base << YUZU_PAGEBITS); - auto& entry = page_table.entries.GetAndFault(base); + auto& entry = page_table.entries.GetUnchecked(base); entry.ptr.Store(host_ptr, type); // TODO: see comments in PageTable::GetPhysOffset entry.addr = static_cast(GetInteger(target) >> YUZU_PAGEBITS) - static_cast(base); entry.block = static_cast(orig_base); - ASSERT_MSG(entry.GetPhysOffset(YUZU_PAGEBITS) == GetInteger(target) - (base << YUZU_PAGEBITS), "assert failed; {:#x} == {:#x}", entry.GetPhysOffset(YUZU_PAGEBITS), GetInteger(target) - (base << YUZU_PAGEBITS)); - ASSERT_MSG(page_table.entries[base].ptr.Pointer(), "memory mapping base yield a nullptr within the table");