diff --git a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp
index 140c6eb6e3..c227acb944 100644
--- a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp
+++ b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp
@@ -409,21 +409,19 @@ NvResult nvhost_as_gpu::MapBufferEx(IoctlMapBufferEx& params) {
         mapping_map[params.offset] = mapping;
     }
 
-    map_buffer_offsets.insert(params.offset);
-
     return NvResult::Success;
 }
 
 NvResult nvhost_as_gpu::UnmapBuffer(IoctlUnmapBuffer& params) {
-    if (map_buffer_offsets.find(params.offset) != map_buffer_offsets.end()) {
-        LOG_DEBUG(Service_NVDRV, "called, offset={:#X}", params.offset);
+    LOG_DEBUG(Service_NVDRV, "called, offset={:#X}", params.offset);
 
-        std::scoped_lock lock(mutex);
+    std::scoped_lock lock(mutex);
 
-        if (!vm.initialised) {
-            return NvResult::BadValue;
-        }
+    if (!vm.initialised) {
+        return NvResult::BadValue;
+    }
 
+    try {
         auto mapping{mapping_map.at(params.offset)};
 
         if (!mapping->fixed) {
@@ -445,8 +443,10 @@ NvResult nvhost_as_gpu::UnmapBuffer(IoctlUnmapBuffer& params) {
         nvmap.UnpinHandle(mapping->handle);
 
         mapping_map.erase(params.offset);
-        map_buffer_offsets.erase(params.offset);
+    } catch (const std::out_of_range&) {
+        LOG_WARNING(Service_NVDRV, "Couldn't find region to unmap at 0x{:X}", params.offset);
     }
+
     return NvResult::Success;
 }
 
diff --git a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h
index c9a6737ba9..63fb38c851 100644
--- a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h
+++ b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h
@@ -13,7 +13,6 @@
 #include <memory>
 #include <mutex>
 #include <optional>
-#include <unordered_set>
 #include <vector>
 
 #include "common/address_space.h"
@@ -113,8 +112,6 @@ private:
     };
     static_assert(sizeof(IoctlRemapEntry) == 20, "IoctlRemapEntry is incorrect size");
 
-    std::unordered_set<s64_le> map_buffer_offsets{};
-
     struct IoctlMapBufferEx {
         MappingFlags flags{}; // bit0: fixed_offset, bit2: cacheable
         u32_le kind{};        // -1 is default