eden-emu
/
eden
mirror of https://git.eden-emu.dev/eden-emu/eden.git
1
0
Fork 0
Code Releases Wiki Activity
Browse Source

[large_vector] don't atomically check in bulk

pull/4219/head
Exverge 2 weeks ago
parent
795eca2e0e
commit
ed04589fe5
No known key found for this signature in database GPG Key ID: DAD399BCC5FB77E4
4 changed files with 53 additions and 29 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 15
      src/common/sparse_large_vector.h
  2. 4
      src/core/device_memory_manager.h
  3. 30
      src/core/device_memory_manager.inc
  4. 33
      src/core/memory.cpp

15
src/common/sparse_large_vector.h
View File

@ -17,6 +17,7 @@
#include <sys/mman.h> #include <sys/mman.h>
#endif #endif
#include "common/alignment.h"
#include "common/assert.h" #include "common/assert.h"
namespace Common { namespace Common {
@ -122,6 +123,20 @@ public:
std::memset(reinterpret_cast<void*>(&base_ptr[index]), 0, sizeof(T)); std::memset(reinterpret_cast<void*>(&base_ptr[index]), 0, sizeof(T));
} }
constexpr void CommitRegion(size_t index, size_t end_) {
auto base = index * sizeof(T);
auto end = end_ * sizeof(T);
while (base < end) {
CommitPage(base);
base = AlignDown(base, HostPageSize) + HostPageSize;
}
}
constexpr T& GetUnchecked(size_t index) {
return base_ptr[index];
}
[[nodiscard]] constexpr const T& operator[](std::size_t index) const noexcept { [[nodiscard]] constexpr const T& operator[](std::size_t index) const noexcept {
return GetOrDefault(index); return GetOrDefault(index);
} }

4
src/core/device_memory_manager.h
View File

@ -200,8 +200,8 @@ private:
return std::make_pair(asid, address); return std::make_pair(asid, address);
} }
void InsertCPUBacking(size_t page_index, VAddr address, Asid asid) {
tracked_entries.GetAndFault(page_index).cpu_backing_address = address | (asid.id << asid_start_bit);
constexpr void InsertCPUBacking(size_t page_index, VAddr address, Asid asid) {
tracked_entries.GetUnchecked(page_index).cpu_backing_address = address | (asid.id << asid_start_bit);
} }
std::array<TranslationEntry, 4> t_slot{}; std::array<TranslationEntry, 4> t_slot{};

30
src/core/device_memory_manager.inc
View File

@ -180,6 +180,7 @@ DeviceMemoryManager<Traits>::DeviceMemoryManager(const DeviceMemory& device_memo
const size_t total_virtual = device_as_size >> Memory::YUZU_PAGEBITS; const size_t total_virtual = device_as_size >> Memory::YUZU_PAGEBITS;
// TODO: this is stupid, make continuity_tracker default to 0 so we can benefit from SparseLargeVector
auto virtual_entry = TrackedEntry {}; auto virtual_entry = TrackedEntry {};
virtual_entry.compressed_physical_ptr = 0; virtual_entry.compressed_physical_ptr = 0;
virtual_entry.continuity_tracker = 1; virtual_entry.continuity_tracker = 1;
@ -187,10 +188,6 @@ DeviceMemoryManager<Traits>::DeviceMemoryManager(const DeviceMemory& device_memo
for (size_t i = 0; i < total_virtual; i++) { for (size_t i = 0; i < total_virtual; i++) {
tracked_entries.Set(i, virtual_entry); tracked_entries.Set(i, virtual_entry);
} }
const size_t total_phys = 1ULL << ((Settings::values.memory_layout_mode.GetValue() == Settings::MemoryLayout::Memory_4Gb ? physical_min_bits : physical_max_bits) - Memory::YUZU_PAGEBITS);
for (size_t i = 0; i < total_phys; i++) {
compressed_device_addr.Zero(i);
}
} }
template <typename Traits> template <typename Traits>
@ -223,17 +220,17 @@ void DeviceMemoryManager<Traits>::Map(DAddr address, VAddr virtual_address, size
size_t start_page_d = address >> Memory::YUZU_PAGEBITS; size_t start_page_d = address >> Memory::YUZU_PAGEBITS;
size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS; size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS;
std::scoped_lock lk(mapping_guard); std::scoped_lock lk(mapping_guard);
tracked_entries.CommitRegion(start_page_d, start_page_d + num_pages);
for (size_t i = 0; i < num_pages; i++) { for (size_t i = 0; i < num_pages; i++) {
const VAddr new_vaddress = virtual_address + i * Memory::YUZU_PAGESIZE; const VAddr new_vaddress = virtual_address + i * Memory::YUZU_PAGESIZE;
auto* ptr = process_memory->GetPointerSilent(Common::ProcessAddress(new_vaddress)); auto* ptr = process_memory->GetPointerSilent(Common::ProcessAddress(new_vaddress));
if (ptr == nullptr) [[unlikely]] { if (ptr == nullptr) [[unlikely]] {
if (auto v = tracked_entries.GetNoFault(start_page_d + i); v) {
v->compressed_physical_ptr = 0;
}
tracked_entries.GetUnchecked(start_page_d + i).compressed_physical_ptr = 0;
continue; continue;
} }
auto phys_addr = static_cast<u32>(GetRawPhysicalAddr(ptr) >> Memory::YUZU_PAGEBITS) + 1U; auto phys_addr = static_cast<u32>(GetRawPhysicalAddr(ptr) >> Memory::YUZU_PAGEBITS) + 1U;
tracked_entries.GetAndFault(start_page_d + i).compressed_physical_ptr = phys_addr;
tracked_entries.GetUnchecked(start_page_d + i).compressed_physical_ptr = phys_addr;
InsertCPUBacking(start_page_d + i, new_vaddress, asid); InsertCPUBacking(start_page_d + i, new_vaddress, asid);
const u32 base_dev = compressed_device_addr[phys_addr - 1U]; const u32 base_dev = compressed_device_addr[phys_addr - 1U];
const u32 new_dev = static_cast<u32>(start_page_d + i); const u32 new_dev = static_cast<u32>(start_page_d + i);
@ -260,25 +257,26 @@ void DeviceMemoryManager<Traits>::Unmap(DAddr address, size_t size) {
size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS; size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS;
device_inter->InvalidateRegion(address, size); device_inter->InvalidateRegion(address, size);
std::scoped_lock lk(mapping_guard); std::scoped_lock lk(mapping_guard);
tracked_entries.CommitRegion(start_page_d, start_page_d + num_pages); // should already be committed, but just in case
for (size_t i = 0; i < num_pages; i++) { for (size_t i = 0; i < num_pages; i++) {
auto& entry = tracked_entries.GetAndFault(start_page_d + i);
auto& entry = tracked_entries.GetUnchecked(start_page_d + i);
auto phys_addr = entry.compressed_physical_ptr; auto phys_addr = entry.compressed_physical_ptr;
entry.compressed_physical_ptr = 0; entry.compressed_physical_ptr = 0;
entry.cpu_backing_address = 0; entry.cpu_backing_address = 0;
if (phys_addr != 0) [[likely]] { if (phys_addr != 0) [[likely]] {
const u32 base_dev = compressed_device_addr[phys_addr - 1U];
u32& base_dev = compressed_device_addr.GetAndFault(phys_addr - 1U);
if ((base_dev >> MULTI_FLAG_BITS) == 0) [[likely]] { if ((base_dev >> MULTI_FLAG_BITS) == 0) [[likely]] {
compressed_device_addr.Zero(phys_addr - 1);
base_dev = 0;
continue; continue;
} }
const auto [more_entries, new_start] = impl->multi_dev_address.Unregister( const auto [more_entries, new_start] = impl->multi_dev_address.Unregister(
static_cast<u32>(start_page_d + i), base_dev & MULTI_MASK); static_cast<u32>(start_page_d + i), base_dev & MULTI_MASK);
if (!more_entries) { if (!more_entries) {
compressed_device_addr.Set(phys_addr - 1,
impl->multi_dev_address.ReleaseEntry(new_start));
base_dev = impl->multi_dev_address.ReleaseEntry(new_start);
continue; continue;
} }
compressed_device_addr.Set(phys_addr - 1, new_start | MULTI_FLAG);
base_dev = new_start | MULTI_FLAG;
} }
} }
t_slot = {}; t_slot = {};
@ -291,6 +289,8 @@ void DeviceMemoryManager<Traits>::TrackContinuityImpl(DAddr address, VAddr virtu
size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS; size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS;
uintptr_t last_ptr = 0; uintptr_t last_ptr = 0;
size_t page_count = 1; size_t page_count = 1;
tracked_entries.CommitRegion(start_page_d, start_page_d + num_pages);
for (size_t i = num_pages; i > 0; i--) { for (size_t i = num_pages; i > 0; i--) {
size_t index = i - 1; size_t index = i - 1;
const VAddr new_vaddress = virtual_address + index * Memory::YUZU_PAGESIZE; const VAddr new_vaddress = virtual_address + index * Memory::YUZU_PAGESIZE;
@ -302,7 +302,7 @@ void DeviceMemoryManager<Traits>::TrackContinuityImpl(DAddr address, VAddr virtu
page_count = 1; page_count = 1;
} }
last_ptr = new_ptr; last_ptr = new_ptr;
tracked_entries.GetAndFault(start_page_d + index).continuity_tracker = static_cast<u32>(page_count);
tracked_entries.GetUnchecked(start_page_d + index).continuity_tracker = static_cast<u32>(page_count);
} }
} }
template <typename Traits> template <typename Traits>

33
src/core/memory.cpp
View File

@ -101,8 +101,10 @@ struct Memory::Impl {
} }
u64 protect_bytes = 0, protect_begin = 0; u64 protect_bytes = 0, protect_begin = 0;
current_page_table->entries.CommitRegion(vaddr >> YUZU_PAGEBITS, (vaddr + size) >> YUZU_PAGEBITS);
for (u64 addr = vaddr; addr < vaddr + size; addr += YUZU_PAGESIZE) { for (u64 addr = vaddr; addr < vaddr + size; addr += YUZU_PAGESIZE) {
const Common::PageType page_type = current_page_table->entries[addr >> YUZU_PAGEBITS].ptr.Type();
const Common::PageType page_type = current_page_table->entries.GetUnchecked(addr >> YUZU_PAGEBITS).ptr.Type();
switch (page_type) { switch (page_type) {
case Common::PageType::RasterizerCachedMemory: case Common::PageType::RasterizerCachedMemory:
if (protect_bytes > 0) { if (protect_bytes > 0) {
@ -243,10 +245,12 @@ struct Memory::Impl {
std::size_t page_index = addr >> YUZU_PAGEBITS; std::size_t page_index = addr >> YUZU_PAGEBITS;
std::size_t page_offset = addr & YUZU_PAGEMASK; std::size_t page_offset = addr & YUZU_PAGEMASK;
bool user_accessible = true; bool user_accessible = true;
current_page_table->entries.CommitRegion(page_index, page_index + (size >> YUZU_PAGEBITS) + 1);
while (remaining_size != 0) { while (remaining_size != 0) {
const std::size_t copy_amount = (std::min)(std::size_t(YUZU_PAGESIZE) - page_offset, remaining_size); const std::size_t copy_amount = (std::min)(std::size_t(YUZU_PAGESIZE) - page_offset, remaining_size);
const auto current_vaddr = u64((page_index << YUZU_PAGEBITS) + page_offset); const auto current_vaddr = u64((page_index << YUZU_PAGEBITS) + page_offset);
const auto [pointer, type] = current_page_table->entries[page_index].ptr.PointerType();
const auto [pointer, type] = current_page_table->entries.GetUnchecked(page_index).ptr.PointerType();
switch (type) { switch (type) {
case Common::PageType::Unmapped: { case Common::PageType::Unmapped: {
user_accessible = false; user_accessible = false;
@ -404,8 +408,11 @@ struct Memory::Impl {
// The region is at a granularity of CPU pages. // The region is at a granularity of CPU pages.
const u64 num_pages = ((vaddr + size - 1) >> YUZU_PAGEBITS) - (vaddr >> YUZU_PAGEBITS) + 1; const u64 num_pages = ((vaddr + size - 1) >> YUZU_PAGEBITS) - (vaddr >> YUZU_PAGEBITS) + 1;
current_page_table->entries.CommitRegion(vaddr >> YUZU_PAGEBITS, (vaddr >> YUZU_PAGEBITS) + num_pages);
for (u64 i = 0; i < num_pages; ++i, vaddr += YUZU_PAGESIZE) { for (u64 i = 0; i < num_pages; ++i, vaddr += YUZU_PAGESIZE) {
const Common::PageType page_type = current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Type();
auto& entry = current_page_table->entries.GetUnchecked(vaddr >> YUZU_PAGEBITS);
const Common::PageType page_type = entry.ptr.Type();
if (debug) { if (debug) {
// Switch page type to debug if now debug // Switch page type to debug if now debug
switch (page_type) { switch (page_type) {
@ -417,7 +424,7 @@ struct Memory::Impl {
// Page is already marked. // Page is already marked.
break; break;
case Common::PageType::Memory: case Common::PageType::Memory:
current_page_table->entries.GetAndFault(vaddr >> YUZU_PAGEBITS).ptr.Store(0, Common::PageType::DebugMemory);
entry.ptr.Store(0, Common::PageType::DebugMemory);
break; break;
default: default:
UNREACHABLE(); UNREACHABLE();
@ -434,7 +441,7 @@ struct Memory::Impl {
break; break;
case Common::PageType::DebugMemory: { case Common::PageType::DebugMemory: {
u8* const pointer = GetPointerFromDebugMemory(vaddr & ~YUZU_PAGEMASK); u8* const pointer = GetPointerFromDebugMemory(vaddr & ~YUZU_PAGEMASK);
current_page_table->entries.GetAndFault(vaddr >> YUZU_PAGEBITS).ptr.Store(uintptr_t(pointer) - (vaddr & ~YUZU_PAGEMASK), Common::PageType::Memory);
entry.ptr.Store(uintptr_t(pointer) - (vaddr & ~YUZU_PAGEMASK), Common::PageType::Memory);
break; break;
} }
default: default:
@ -466,8 +473,10 @@ struct Memory::Impl {
// is different). This assumes the specified GPU address region is contiguous as well. // is different). This assumes the specified GPU address region is contiguous as well.
const u64 num_pages = ((vaddr + size - 1) >> YUZU_PAGEBITS) - (vaddr >> YUZU_PAGEBITS) + 1; const u64 num_pages = ((vaddr + size - 1) >> YUZU_PAGEBITS) - (vaddr >> YUZU_PAGEBITS) + 1;
current_page_table->entries.CommitRegion(vaddr >> YUZU_PAGEBITS, (vaddr >> YUZU_PAGEBITS) + num_pages);
for (u64 i = 0; i < num_pages; ++i, vaddr += YUZU_PAGESIZE) { for (u64 i = 0; i < num_pages; ++i, vaddr += YUZU_PAGESIZE) {
const Common::PageType page_type= current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Type();
auto& entry = current_page_table->entries.GetUnchecked(vaddr >> YUZU_PAGEBITS);
const Common::PageType page_type = entry.ptr.Type();
if (cached) { if (cached) {
// Switch page type to cached if now cached // Switch page type to cached if now cached
switch (page_type) { switch (page_type) {
@ -477,7 +486,7 @@ struct Memory::Impl {
break; break;
case Common::PageType::DebugMemory: case Common::PageType::DebugMemory:
case Common::PageType::Memory: case Common::PageType::Memory:
current_page_table->entries.GetAndFault(vaddr >> YUZU_PAGEBITS).ptr.Store(0, Common::PageType::RasterizerCachedMemory);
entry.ptr.Store(0, Common::PageType::RasterizerCachedMemory);
break; break;
case Common::PageType::RasterizerCachedMemory: case Common::PageType::RasterizerCachedMemory:
// There can be more than one GPU region mapped per CPU region, so it's common // There can be more than one GPU region mapped per CPU region, so it's common
@ -503,9 +512,9 @@ struct Memory::Impl {
// It's possible that this function has been called while updating the // It's possible that this function has been called while updating the
// pagetable after unmapping a VMA. In that case the underlying VMA will no // pagetable after unmapping a VMA. In that case the underlying VMA will no
// longer exist, and we should just leave the pagetable entry blank. // longer exist, and we should just leave the pagetable entry blank.
current_page_table->entries.GetAndFault(vaddr >> YUZU_PAGEBITS).ptr.Store(0, Common::PageType::Unmapped);
entry.ptr.Store(0, Common::PageType::Unmapped);
} else { } else {
current_page_table->entries.GetAndFault(vaddr >> YUZU_PAGEBITS).ptr.Store(uintptr_t(pointer) - (vaddr & ~YUZU_PAGEMASK), Common::PageType::Memory);
entry.ptr.Store(uintptr_t(pointer) - (vaddr & ~YUZU_PAGEMASK), Common::PageType::Memory);
} }
break; break;
} }
@ -546,17 +555,17 @@ struct Memory::Impl {
} }
} else { } else {
auto orig_base = base; auto orig_base = base;
page_table.entries.CommitRegion(base, end);
while (base != end) { while (base != end) {
auto host_ptr = uintptr_t(system.DeviceMemory().GetPointer<u8>(target)) - (base << YUZU_PAGEBITS); auto host_ptr = uintptr_t(system.DeviceMemory().GetPointer<u8>(target)) - (base << YUZU_PAGEBITS);
auto& entry = page_table.entries.GetAndFault(base);
auto& entry = page_table.entries.GetUnchecked(base);
entry.ptr.Store(host_ptr, type); entry.ptr.Store(host_ptr, type);
// TODO: see comments in PageTable::GetPhysOffset // TODO: see comments in PageTable::GetPhysOffset
entry.addr = static_cast<u32>(GetInteger(target) >> YUZU_PAGEBITS) - static_cast<u32>(base); entry.addr = static_cast<u32>(GetInteger(target) >> YUZU_PAGEBITS) - static_cast<u32>(base);
entry.block = static_cast<u32>(orig_base); entry.block = static_cast<u32>(orig_base);
ASSERT_MSG(entry.GetPhysOffset(YUZU_PAGEBITS) == GetInteger(target) - (base << YUZU_PAGEBITS), "assert failed; {:#x} == {:#x}", entry.GetPhysOffset(YUZU_PAGEBITS), GetInteger(target) - (base << YUZU_PAGEBITS));
ASSERT_MSG(page_table.entries[base].ptr.Pointer(), ASSERT_MSG(page_table.entries[base].ptr.Pointer(),
"memory mapping base yield a nullptr within the table"); "memory mapping base yield a nullptr within the table");

Write Preview
Loading…
Cancel
Save