diff --git a/src/common/page_table.cpp b/src/common/page_table.cpp
index 5814688bc9..f3aa6b58c0 100644
--- a/src/common/page_table.cpp
+++ b/src/common/page_table.cpp
@@ -22,18 +22,19 @@ bool PageTable::BeginTraversal(TraversalEntry* out_entry, TraversalContext* out_
 }
 
 bool PageTable::ContinueTraversal(TraversalEntry* out_entry, TraversalContext* context) const {
+    auto page_size = 1 << current_page_bits;
     // Setup invalid defaults.
     out_entry->phys_addr = 0;
-    out_entry->block_size = 1 << current_page_bits;
-    // Setup context
+    out_entry->block_size = page_size;
+    // Setup return context
     context->next_page += 1;
-    context->next_offset += 1 << current_page_bits;
+    context->next_offset += page_size;
     // Validate that we can read the actual entry.
     if (auto const page = context->next_page; page < entries.size()) {
         // Validate that the entry is mapped.
         if (auto const paddr = entries[page].addr; paddr != 0) {
             // Populate the results and return true
-            out_entry->phys_addr = (paddr << current_page_bits) + context->next_offset;
+            out_entry->phys_addr = (entries[page].GetPhysOffset(current_page_bits)) + context->next_offset;
             return true;
         }
     }
diff --git a/src/common/page_table.h b/src/common/page_table.h
index dd8671c51f..2b6dca152c 100644
--- a/src/common/page_table.h
+++ b/src/common/page_table.h
@@ -127,7 +127,7 @@ struct PageTable {
             return false;
         }
 
-        *out_phys_addr = (entries[GetInteger(virt_addr) >> current_page_bits].addr >> current_page_bits) + GetInteger(virt_addr);
+        *out_phys_addr = entries[GetInteger(virt_addr) >> current_page_bits].GetPhysOffset(current_page_bits) + GetInteger(virt_addr);
         return true;
     }
 
@@ -137,6 +137,15 @@ struct PageTable {
         PageInfo ptr;
         u32 block;
         u32 addr;
+
+        constexpr u64 GetPhysOffset(u64 page_bits) const {
+            // TODO: For whatever reason, when storing the "physical address" of a large page group, yuzu code writes it as:
+            // `addr = base_phys_addr - vaddr`, where base_phys_addr is the base address of the first entry in the page group.
+            // This 90% of the time results in a negative pointer. However, as of #4219, `addr` is stored as a u32 instead of a u64,
+            // so we use sign extension to work around this issue.
+            s64 result = (static_cast<s64>(static_cast<s32>(addr))) << page_bits;
+            return static_cast<u64>(result);
+        }
     };
     SparseLargeVector<PageEntryData> entries;
     static_assert(sizeof(PageEntryData) == 16);
diff --git a/src/core/memory.cpp b/src/core/memory.cpp
index 3ba4b39168..5c634cb611 100644
--- a/src/core/memory.cpp
+++ b/src/core/memory.cpp
@@ -123,14 +123,14 @@ struct Memory::Impl {
     }
 
     [[nodiscard]] u8* GetPointerFromRasterizerCachedMemory(u64 vaddr) const {
-        Common::PhysicalAddress const paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].addr;
+        Common::PhysicalAddress const paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].GetPhysOffset(YUZU_PAGEBITS);
         if (paddr)
             return system.DeviceMemory().GetPointer<u8>((paddr << YUZU_PAGEBITS) + vaddr);
         return {};
     }
 
     [[nodiscard]] u8* GetPointerFromDebugMemory(u64 vaddr) const {
-        const Common::PhysicalAddress paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].addr;
+        const Common::PhysicalAddress paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].GetPhysOffset(YUZU_PAGEBITS);
         if (paddr != 0)
             return system.DeviceMemory().GetPointer<u8>((paddr << YUZU_PAGEBITS) + vaddr);
         return {};
@@ -551,7 +551,8 @@ struct Memory::Impl {
 
                 auto& entry = page_table.entries.GetAndFault(base);
                 entry.ptr.Store(host_ptr, type);
-                entry.addr = static_cast<u32>((GetInteger(target) >> YUZU_PAGEBITS) - base);
+                // TODO: see comments in PageTable::GetPhysOffset
+                entry.addr = static_cast<u32>(GetInteger(target) >> YUZU_PAGEBITS) - static_cast<u32>(base);
                 entry.block = static_cast<u32>(orig_base);
 
                 ASSERT_MSG(page_table.entries[base].ptr.Pointer(),