[TEST] Extend coalescing to more cpu regions

src/video_core/buffer_cache/buffer_cache.h

@@ -1721,6 +1721,9 @@ void BufferCache<P>::TouchBuffer(Buffer& buffer, BufferId buffer_id) noexcept {
 
 template <class P>
 bool BufferCache<P>::SynchronizeBuffer(Buffer& buffer, DAddr device_addr, u32 size) {
+    if (!memory_tracker.HasCpuModifiedCheap(device_addr, size)) {
+        return true;
+    }
     upload_copies.clear();
     u64 total_size_bytes = 0;
     u64 largest_copy = 0;

src/video_core/buffer_cache/memory_tracker_base.h

@@ -7,6 +7,7 @@
 #pragma once
 
 #include <algorithm>
+#include <atomic>
 #include <bit>
 #include <deque>
 #include <limits>
@@ -50,6 +51,25 @@ public:
         });
     }
 
+    [[nodiscard]] bool HasCpuModifiedCheap(VAddr query_cpu_addr, u64 query_size) noexcept {
+        std::size_t remaining_size{query_size};
+        std::size_t page_index{query_cpu_addr >> HIGHER_PAGE_BITS};
+        u64 page_offset{query_cpu_addr & HIGHER_PAGE_MASK};
+        while (remaining_size > 0) {
+            const std::size_t copy_amount{
+                std::min<std::size_t>(HIGHER_PAGE_SIZE - page_offset, remaining_size)};
+            const Manager* manager =
+                std::atomic_ref<Manager*>(top_tier[page_index]).load(std::memory_order_acquire);
+            if (manager == nullptr || manager->CpuModifiedPageCount() != 0) {
+                return true;
+            }
+            page_index++;
+            page_offset = 0;
+            remaining_size -= copy_amount;
+        }
+        return false;
+    }
+
     /// Returns true if a region has been modified from the CPU
     [[nodiscard]] bool IsRegionCpuModified(VAddr query_cpu_addr, u64 query_size) noexcept {
         return IteratePages<true>(query_cpu_addr, query_size, [](Manager* manager, u64 offset, size_t size) {
@@ -260,7 +280,8 @@ private:
 
     void CreateRegion(std::size_t page_index) {
         const VAddr base_cpu_addr = page_index << HIGHER_PAGE_BITS;
-        top_tier[page_index] = GetNewManager(base_cpu_addr);
+        std::atomic_ref<Manager*>(top_tier[page_index])
+            .store(GetNewManager(base_cpu_addr), std::memory_order_release);
     }
 
     Manager* GetNewManager(VAddr base_cpu_address) {

src/video_core/buffer_cache/word_manager.h

@@ -7,6 +7,7 @@
 #pragma once
 
 #include <algorithm>
+#include <atomic>
 #include <bit>
 #include <limits>
 #include <span>
@@ -48,6 +49,11 @@ struct WordManager {
         u64 const last_word = (~u64{0} << shift) >> shift;
         heap[num_words * size_t(Type::CPU) + num_words - 1] = last_word;
         heap[num_words * size_t(Type::Untracked) + num_words - 1] = last_word;
+        u32 cpu_pages = 0;
+        for (size_t i = 0; i < num_words; ++i) {
+            cpu_pages += static_cast<u32>(std::popcount(heap[num_words * size_t(Type::CPU) + i]));
+        }
+        cpu_modified_pages.store(cpu_pages, std::memory_order_relaxed);
     }
     explicit WordManager() = default;
 
@@ -120,10 +126,15 @@ struct WordManager {
         [[maybe_unused]] std::span<u64> untracked_words = Span(Type::Untracked);
         [[maybe_unused]] std::span<u64> cached_words = Span(Type::CachedCPU);
         std::vector<std::pair<VAddr, u64>> ranges;
+        s64 cpu_delta = 0;
         IterateWords(dirty_addr - cpu_addr, size, [&](size_t index, u64 mask) {
             if (type == Type::CPU || type == Type::CachedCPU) {
                 CollectChangedRanges(!enable, index, untracked_words[index], mask, ranges);
             }
+            if (type == Type::CPU) {
+                const u64 old = state_words[index];
+                cpu_delta += enable ? std::popcount(~old & mask) : -std::popcount(old & mask);
+            }
             if (enable) {
                 state_words[index] |= mask;
                 if (type == Type::CPU || type == Type::CachedCPU)
@@ -138,6 +149,9 @@ struct WordManager {
                     untracked_words[index] &= ~mask;
             }
         });
+        if (cpu_delta != 0) {
+            cpu_modified_pages.fetch_add(static_cast<u32>(cpu_delta), std::memory_order_release);
+        }
         if (!ranges.empty()) {
             ApplyCollectedRanges(ranges, (!enable) ? 1 : -1);
         }
@@ -165,6 +179,7 @@ struct WordManager {
                  (pending_pointer - pending_offset) * BYTES_PER_PAGE);
         };
         std::vector<std::pair<VAddr, u64>> ranges;
+        s64 cpu_delta = 0;
         IterateWords(offset, size, [&](size_t index, u64 mask) {
             if (type == Type::GPU)
                 mask &= ~untracked_words[index];
@@ -173,6 +188,8 @@ struct WordManager {
                 if (type == Type::CPU || type == Type::CachedCPU) {
                     CollectChangedRanges(true, index, untracked_words[index], mask, ranges);
                 }
+                if (type == Type::CPU)
+                    cpu_delta -= std::popcount(word);
                 state_words[index] &= ~mask;
                 if (type == Type::CPU || type == Type::CachedCPU)
                     untracked_words[index] &= ~mask;
@@ -194,6 +211,9 @@ struct WordManager {
                 }
             });
         });
+        if (cpu_delta != 0) {
+            cpu_modified_pages.fetch_add(static_cast<u32>(cpu_delta), std::memory_order_release);
+        }
         if (pending) {
             release();
         }
@@ -248,13 +268,18 @@ struct WordManager {
         auto const untracked_words = Span(Type::Untracked);
         auto const cpu_words = Span(Type::CPU);
         std::vector<std::pair<VAddr, u64>> ranges;
+        s64 cpu_delta = 0;
         for (u64 word_index = 0; word_index < num_words; ++word_index) {
             const u64 cached_bits = cached_words[word_index];
             CollectChangedRanges(false, word_index, untracked_words[word_index], cached_bits, ranges);
+            cpu_delta += std::popcount(~cpu_words[word_index] & cached_bits);
             untracked_words[word_index] |= cached_bits;
             cpu_words[word_index] |= cached_bits;
             cached_words[word_index] = 0;
         }
+        if (cpu_delta != 0) {
+            cpu_modified_pages.fetch_add(static_cast<u32>(cpu_delta), std::memory_order_release);
+        }
         if (!ranges.empty()) {
             ApplyCollectedRanges(ranges, -1);
         }
@@ -319,9 +344,14 @@ struct WordManager {
         return std::span<const u64>(heap.data() + num_words * size_t(type), num_words);
     }
 
+    [[nodiscard]] u32 CpuModifiedPageCount() const noexcept {
+        return cpu_modified_pages.load(std::memory_order_acquire);
+    }
+
     std::array<u64, size_t(Type::Max) * num_words> heap = {};
     DeviceTracker* tracker = nullptr;
     VAddr cpu_addr = 0;
+    std::atomic<u32> cpu_modified_pages{0};
 };
 
 } // namespace VideoCommon