diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h
index b6d0852b4e..ffeb1e0a7e 100644
--- a/src/video_core/buffer_cache/buffer_cache.h
+++ b/src/video_core/buffer_cache/buffer_cache.h
@@ -121,6 +121,15 @@ void BufferCache<P>::WriteMemory(DAddr device_addr, u64 size) {
     memory_tracker.MarkRegionAsCpuModified(device_addr, size);
 }
 
+template <class P>
+void BufferCache<P>::CpuWriteInvalidate(DAddr device_addr, u64 size) {
+    if (!memory_tracker.CpuMarkIfNotGpuModified(device_addr, size)) {
+        return;
+    }
+    std::scoped_lock lock{mutex};
+    WriteMemory(device_addr, size);
+}
+
 template <class P>
 void BufferCache<P>::CachedWriteMemory(DAddr device_addr, u64 size) {
     const bool is_dirty = IsRegionRegistered(device_addr, size);
diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h
index 14ab3e6ebc..3b900b95f9 100644
--- a/src/video_core/buffer_cache/buffer_cache_base.h
+++ b/src/video_core/buffer_cache/buffer_cache_base.h
@@ -217,6 +217,8 @@ public:
 
     void WriteMemory(DAddr device_addr, u64 size);
 
+    void CpuWriteInvalidate(DAddr device_addr, u64 size);
+
     void CachedWriteMemory(DAddr device_addr, u64 size);
 
     bool OnCPUWrite(DAddr device_addr, u64 size);
diff --git a/src/video_core/buffer_cache/memory_tracker_base.h b/src/video_core/buffer_cache/memory_tracker_base.h
index 2c571271d6..887b4ff149 100644
--- a/src/video_core/buffer_cache/memory_tracker_base.h
+++ b/src/video_core/buffer_cache/memory_tracker_base.h
@@ -10,6 +10,7 @@
 #include <bit>
 #include <deque>
 #include <limits>
+#include <mutex>
 #include <type_traits>
 #include <ankerl/unordered_dense.h>
 #include <utility>
@@ -130,12 +131,28 @@ public:
     }
 
     void FlushCachedWrites() noexcept {
+        std::scoped_lock lk{tracker_mutex};
         for (auto id : cached_pages) {
             top_tier[id]->FlushCachedWrites();
         }
         cached_pages.clear();
     }
 
+    [[nodiscard]] bool CpuMarkIfNotGpuModified(VAddr addr, u64 size) {
+        std::scoped_lock lk{tracker_mutex};
+        const bool gpu = IteratePagesNoLock<false>(
+            addr, size, [](Manager* manager, u64 offset, size_t sz) {
+                return manager->IsRegionModified(Type::GPU, offset, sz);
+            });
+        if (gpu) {
+            return true;
+        }
+        IteratePagesNoLock<true>(addr, size, [](Manager* manager, u64 offset, size_t sz) {
+            manager->ChangeRegionState(Type::CPU, true, manager->cpu_addr + offset, sz);
+        });
+        return false;
+    }
+
     /// Call 'func' for each CPU modified range and unmark those pages as CPU modified
     template <typename Func>
     void ForEachUploadRange(VAddr query_cpu_range, u64 query_size, Func&& func) {
@@ -162,6 +179,12 @@ public:
 private:
     template <bool create_region_on_fail, typename Func>
     bool IteratePages(VAddr cpu_address, size_t size, Func&& func) {
+        std::scoped_lock lk{tracker_mutex};
+        return IteratePagesNoLock<create_region_on_fail>(cpu_address, size, std::forward<Func>(func));
+    }
+
+    template <bool create_region_on_fail, typename Func>
+    bool IteratePagesNoLock(VAddr cpu_address, size_t size, Func&& func) {
         using FuncReturn = typename std::invoke_result<Func, Manager*, u64, size_t>::type;
         static constexpr bool BOOL_BREAK = std::is_same_v<FuncReturn, bool>;
         std::size_t remaining_size{size};
@@ -199,6 +222,7 @@ private:
 
     template <bool create_region_on_fail, typename Func>
     std::pair<u64, u64> IteratePairs(VAddr cpu_address, size_t size, Func&& func) {
+        std::scoped_lock lk{tracker_mutex};
         std::size_t remaining_size{size};
         std::size_t page_index{cpu_address >> HIGHER_PAGE_BITS};
         u64 page_offset{cpu_address & HIGHER_PAGE_MASK};
@@ -259,6 +283,7 @@ private:
     std::deque<Manager*> free_managers;
     ankerl::unordered_dense::set<u32> cached_pages;
     DeviceTracker* device_tracker = nullptr;
+    std::mutex tracker_mutex;
 };
 
 } // namespace VideoCommon
diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
index c5ceebf6cb..e7120fe8bb 100644
--- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp
+++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
@@ -773,6 +773,7 @@ bool RasterizerVulkan::OnCPUWrite(DAddr addr, u64 size) {
 }
 
 static constexpr bool ENABLE_TEXTURE_CACHE_INVALIDATION_SKIP = true;
+static constexpr bool ENABLE_FINE_GRAINED_TRACKER_LOCK = true;
 
 void RasterizerVulkan::OnCacheInvalidation(DAddr addr, u64 size) {
     if (addr == 0 || size == 0) {
@@ -784,7 +785,9 @@ void RasterizerVulkan::OnCacheInvalidation(DAddr addr, u64 size) {
         std::scoped_lock lock{texture_cache.mutex};
         texture_cache.WriteMemory(addr, size);
     }
-    {
+    if (ENABLE_FINE_GRAINED_TRACKER_LOCK) {
+        buffer_cache.CpuWriteInvalidate(addr, size);
+    } else {
         std::scoped_lock lock{buffer_cache.mutex};
         buffer_cache.WriteMemory(addr, size);
     }