diff --git a/src/common/assert.h b/src/common/assert.h
index da7f090d6b..fa1def6cba 100644
--- a/src/common/assert.h
+++ b/src/common/assert.h
@@ -24,7 +24,7 @@ void AssertFailSoftImpl();
 #define YUZU_NO_INLINE
 #endif
 
-#if !defined(__clang__) && !defined(__GNUC__)
+#if defined(__clang__) || defined(__GNUC__)
 #define YUZU_ALWAYS_INLINE __attribute__((always_inline))
 #elif defined(_MSC_VER)
 #define YUZU_ALWAYS_INLINE [[msvc::forceinline]]
diff --git a/src/common/host_memory.cpp b/src/common/host_memory.cpp
index bf776036f0..5c30d34519 100644
--- a/src/common/host_memory.cpp
+++ b/src/common/host_memory.cpp
@@ -777,43 +777,90 @@ HostMemory::HostMemory(HostMemory&&) noexcept = default;
 
 HostMemory& HostMemory::operator=(HostMemory&&) noexcept = default;
 
-bool HostMemory::IsIrregularlyMappedAddress(size_t addr) noexcept {
-    auto index = addr / Core::Memory::YUZU_PAGESIZE;
+HostMemory::IrregularMapping* HostMemory::GetUnalignedMappingFromVirtual(u64 offset) {
+    auto index = offset >> Core::Memory::YUZU_PAGEBITS;
+    constexpr auto cmp = [](const u64 a, const IrregularMapping b) {
+        return a < b.vaddr;
+    };
+    auto i = unaligned_mappings.upper_bound(index, cmp);
+
+    if (i == unaligned_mappings.begin())
+        return nullptr;
+    --i;
+
+    return index < i->vaddr + i->size ? &*i : nullptr;
+}
+
+const HostMemory::IrregularMapping* HostMemory::GetIrregularMappingFromFakePhysical(u64 offset) {
+    auto index = offset >> Core::Memory::YUZU_PAGEBITS;
+    constexpr auto cmp = [](const u64 a, const IrregularMapping b) {
+        return a < b.vaddr;
+    };
+    auto i = irregular_mappings.upper_bound(index, cmp);
 
-    auto i = irregular_mappings.upper_bound(index);
     if (i == irregular_mappings.begin())
-        return false;
+        return nullptr;
     --i;
 
-    return index < i->first + i->second;
+    return index < i->fake_paddr + i->size ? &*i : nullptr;
 }
 
 void HostMemory::Map(size_t virtual_offset, size_t host_offset, size_t length, MemoryPermission perms, bool separate_heap) {
 #if !(defined(__OPENORBIS__) || defined(__managarm__))
+    // TODO: offset paddr with vaddr to align with page table?
     if (virtual_offset % HostPageSize != 0) {
-        if (IsIrregularlyMappedAddress(virtual_offset)) {
+        if (virtual_offset % 0x1000 != 0) [[unlikely]] {
+            UNREACHABLE_MSG("Attempted to map virtual addresses {:#x}-{:#x} which is unaligned to guest page size", virtual_offset, virtual_offset + length);
+        }
+
+        if (auto map = GetUnalignedMappingFromVirtual(virtual_offset); map) {
+            ASSERT(map->fake_paddr == 0); // sanity check
             auto aligned = AlignUp(virtual_offset, HostPageSize);
-            LOG_WARNING(HW_Memory, "Irregularly mapped virtual addresses {:#x}-{:#x} will not have a valid physical address",
-                virtual_offset, aligned);
-            length -= aligned - virtual_offset;
+
+            // TODO: is fake_paddr right here?
+            map->fake_paddr = host_offset;
+            irregular_mappings.insert(*map);
+
+            LOG_WARNING(HW_Memory, "Irregularly mapped virtual addresses {:#x}-{:#x} will not have a valid physical address (fake: {:#x}, real: {:#x})",
+                virtual_offset, aligned, map->fake_paddr, map->real_paddr);
+            length -= map->size;
             virtual_offset = aligned;
         } else {
             auto aligned = AlignDown(virtual_offset, HostPageSize);
-            irregular_mappings.emplace(aligned / Core::Memory::YUZU_PAGESIZE, virtual_offset - aligned);
-            length += virtual_offset - aligned;
+
+            // TODO: is host_offset right here?
+            auto* mapping = new IrregularMapping {aligned, host_offset, virtual_offset - aligned};
+            unaligned_mappings.insert(*mapping);
+
+            length += mapping->size;
             virtual_offset = aligned;
         }
     }
 
     if (length % HostPageSize != 0) {
-        if (IsIrregularlyMappedAddress(virtual_offset + length)) {
+        if (length % 0x1000 != 0) [[unlikely]] {
+            // TODO: why does this happen??
+            length = AlignDown(length, 0x1000);
+            //UNREACHABLE_MSG("Attempted to map virtual addresses {:#x}-{:#x} which is unaligned to guest page size", virtual_offset, virtual_offset + length);
+        }
+
+        if (auto map = GetUnalignedMappingFromVirtual(virtual_offset + length); map) {
+            ASSERT(map->fake_paddr == 0); // sanity check TODO: handle multiple misaligned mappings in one page boundary
             auto aligned = AlignDown(length, HostPageSize);
-            LOG_WARNING(HW_Memory, "Irregularly mapped virtual addresses {:#x}-{:#x} will not have a valid physical address",
-                virtual_offset + aligned, virtual_offset + length);
+
+            // TODO: is fake_paddr right here?
+            map->fake_paddr = host_offset + length;
+            irregular_mappings.insert(*map);
+
+            LOG_WARNING(HW_Memory, "Irregularly mapped virtual addresses {:#x}-{:#x} will not have a valid physical address (fake: {:#x}, real: {:#x})",
+                virtual_offset + aligned, virtual_offset + length, map->fake_paddr, map->real_paddr);
             length = aligned;
         } else {
             auto aligned = AlignUp(length, HostPageSize);
-            irregular_mappings.emplace((virtual_offset + aligned) / Core::Memory::YUZU_PAGESIZE, aligned - length);
+            // TODO: is host_offset right here?
+            auto mapping = new IrregularMapping { virtual_offset + length, host_offset + length, aligned - length };
+            unaligned_mappings.insert(*mapping);
+
             length = aligned;
         }
     }
@@ -849,6 +896,10 @@ void HostMemory::Protect(size_t virtual_offset, size_t length, MemoryPermission
     bool execute = True(perm & MemoryPermission::Execute);
 
     if (length % HostPageSize != 0 || virtual_offset % HostPageSize != 0) {
+        if (virtual_offset % 0x1000 != 0 || length % 0x1000 != 0) [[unlikely]] {
+            UNREACHABLE_MSG("Attempted to protect virtual addresses {:#x}-{:#x} which is unaligned to guest page size", virtual_offset, virtual_offset + length);
+        }
+
         // todo: make this actually inherit most permissive
         LOG_WARNING(HW_Memory, "Memory is unaligned to page size, surrounding pages will inherit most permissive permissions");
         auto aligned = AlignDown(virtual_offset, HostPageSize);
diff --git a/src/common/host_memory.h b/src/common/host_memory.h
index 315f5f0bfe..596060a7ed 100644
--- a/src/common/host_memory.h
+++ b/src/common/host_memory.h
@@ -9,6 +9,10 @@
 #include <memory>
 #include <optional>
 
+#include <boost/intrusive/set.hpp>
+
+namespace bi = boost::intrusive;
+
 #ifndef _WIN32
 #include <unistd.h>
 #endif
@@ -18,6 +22,7 @@
 #include "common/common_funcs.h"
 #include "common/common_types.h"
 #include "common/virtual_buffer.h"
+#include "core/memory.h"
 
 namespace Common {
 
@@ -84,10 +89,30 @@ public:
         return virtual_base;
     }
 
-    bool IsInVirtualRange(void* address) const noexcept {
+    bool IsInVirtualRange(const void* address) const noexcept {
         return address >= virtual_base && address < virtual_base + virtual_size;
     }
 
+    using by_vaddr = bi::set_base_hook<bi::tag<struct _by_vaddr>>;
+    using by_fake_paddr = bi::set_base_hook<bi::tag<struct _by_fake_paddr>>;
+
+    struct IrregularMapping : by_vaddr, by_fake_paddr {
+
+        IrregularMapping(u64 vaddr, u64 paddr, u64 size) : vaddr(vaddr >> Core::Memory::YUZU_PAGEBITS),
+                                                           size(size >> Core::Memory::YUZU_PAGEBITS),
+                                                           real_paddr(paddr >> Core::Memory::YUZU_PAGEBITS) {}
+        u64 vaddr;
+        u64 size;
+
+        // Real backing memory linked to this mapping
+        u64 real_paddr;
+        // Memory address stored by the page table that mapped this mapping
+        u64 fake_paddr {0};
+    };
+
+    IrregularMapping* GetUnalignedMappingFromVirtual(u64 offset);
+    const IrregularMapping* GetIrregularMappingFromFakePhysical(u64 offset);
+
 private:
     size_t backing_size{};
     size_t virtual_size{};
@@ -100,12 +125,20 @@ private:
     u8* backing_base{};
     u8* virtual_base{};
     size_t virtual_base_offset{};
-    // todo: include actual paddr for host ops?
-    std::map<size_t, size_t> irregular_mappings{};
     // Windows requires it for kernels whom lack proper support for some functions!
-    std::optional<Common::VirtualBuffer<u8>> fallback_buffer;
-
-    bool IsIrregularlyMappedAddress(size_t addr) noexcept;
+    std::optional<VirtualBuffer<u8>> fallback_buffer;
+
+    static inline auto cmp_vaddr = [](const IrregularMapping& a, const IrregularMapping& b) {
+        return a.vaddr < b.vaddr;
+    };
+    static inline auto cmp_paddr = [](const IrregularMapping& a, const IrregularMapping& b) {
+        return a.fake_paddr < b.fake_paddr;
+    };
+
+    // Mappings that have mapped more memory than needed due to page-size limitations
+    bi::set<IrregularMapping, bi::base_hook<by_vaddr>, bi::compare<decltype(cmp_vaddr)>> unaligned_mappings;
+    // Mappings in `unaligned_mappings` that have a fake physical address due to them being mapped again.
+    bi::set<IrregularMapping, bi::base_hook<by_fake_paddr>, bi::compare<decltype(cmp_paddr)>> irregular_mappings;
 };
 
 } // namespace Common