eden-emu
/
eden
mirror of https://git.eden-emu.dev/eden-emu/eden.git
1
0
Fork 0
Code Releases Wiki Activity
Browse Source

Merge pull request #10084 from FernandoS27/yuzu-goes-broom-broom 

Y.F.C Buffer Cache Revamp
pull/15/merge
Morph 3 years ago
committed by GitHub
parent
1f079d9566 d6f565e5da
commit
47938541c2
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 2255 additions and 1727 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      src/tests/CMakeLists.txt
  2. 549
      src/tests/video_core/buffer_base.cpp
  3. 549
      src/tests/video_core/memory_tracker.cpp
  4. 5
      src/video_core/CMakeLists.txt
  5. 518
      src/video_core/buffer_cache/buffer_base.h
  6. 4
      src/video_core/buffer_cache/buffer_cache.cpp
  7. 1002
      src/video_core/buffer_cache/buffer_cache.h
  8. 580
      src/video_core/buffer_cache/buffer_cache_base.h
  9. 271
      src/video_core/buffer_cache/memory_tracker_base.h
  10. 462
      src/video_core/buffer_cache/word_manager.h
  11. 4
      src/video_core/renderer_opengl/gl_buffer_cache.h
  12. 9
      src/video_core/renderer_opengl/gl_buffer_cache_base.cpp
  13. 8
      src/video_core/renderer_vulkan/vk_buffer_cache.cpp
  14. 10
      src/video_core/renderer_vulkan/vk_buffer_cache.h
  15. 9
      src/video_core/renderer_vulkan/vk_buffer_cache_base.cpp

2
src/tests/CMakeLists.txt
View File

@ -15,7 +15,7 @@ add_executable(tests
core/core_timing.cpp
core/internal_network/network.cpp
precompiled_headers.h
video_core/buffer_base.cpp
video_core/memory_tracker.cpp
input_common/calibration_configuration_job.cpp
)

549
src/tests/video_core/buffer_base.cpp
View File

@ -1,549 +0,0 @@
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include <stdexcept>
#include <unordered_map>
#include <catch2/catch_test_macros.hpp>
#include "common/alignment.h"
#include "common/common_types.h"
#include "video_core/buffer_cache/buffer_base.h"
namespace {
using VideoCommon::BufferBase;
using Range = std::pair<u64, u64>;
constexpr u64 PAGE = 4096;
constexpr u64 WORD = 4096 * 64;
constexpr VAddr c = 0x1328914000;
class RasterizerInterface {
public:
void UpdatePagesCachedCount(VAddr addr, u64 size, int delta) {
const u64 page_start{addr >> Core::Memory::YUZU_PAGEBITS};
const u64 page_end{(addr + size + Core::Memory::YUZU_PAGESIZE - 1) >>
Core::Memory::YUZU_PAGEBITS};
for (u64 page = page_start; page < page_end; ++page) {
int& value = page_table[page];
value += delta;
if (value < 0) {
throw std::logic_error{"negative page"};
}
if (value == 0) {
page_table.erase(page);
}
}
}
[[nodiscard]] int Count(VAddr addr) const noexcept {
const auto it = page_table.find(addr >> Core::Memory::YUZU_PAGEBITS);
return it == page_table.end() ? 0 : it->second;
}
[[nodiscard]] unsigned Count() const noexcept {
unsigned count = 0;
for (const auto& [index, value] : page_table) {
count += value;
}
return count;
}
private:
std::unordered_map<u64, int> page_table;
};
} // Anonymous namespace
TEST_CASE("BufferBase: Small buffer", "[video_core]") {
RasterizerInterface rasterizer;
BufferBase buffer(rasterizer, c, WORD);
REQUIRE(rasterizer.Count() == 0);
buffer.UnmarkRegionAsCpuModified(c, WORD);
REQUIRE(rasterizer.Count() == WORD / PAGE);
REQUIRE(buffer.ModifiedCpuRegion(c, WORD) == Range{0, 0});
buffer.MarkRegionAsCpuModified(c + PAGE, 1);
REQUIRE(buffer.ModifiedCpuRegion(c, WORD) == Range{PAGE * 1, PAGE * 2});
}
TEST_CASE("BufferBase: Large buffer", "[video_core]") {
RasterizerInterface rasterizer;
BufferBase buffer(rasterizer, c, WORD * 32);
buffer.UnmarkRegionAsCpuModified(c, WORD * 32);
buffer.MarkRegionAsCpuModified(c + 4096, WORD * 4);
REQUIRE(buffer.ModifiedCpuRegion(c, WORD + PAGE * 2) == Range{PAGE, WORD + PAGE * 2});
REQUIRE(buffer.ModifiedCpuRegion(c + PAGE * 2, PAGE * 6) == Range{PAGE * 2, PAGE * 8});
REQUIRE(buffer.ModifiedCpuRegion(c, WORD * 32) == Range{PAGE, WORD * 4 + PAGE});
REQUIRE(buffer.ModifiedCpuRegion(c + WORD * 4, PAGE) == Range{WORD * 4, WORD * 4 + PAGE});
REQUIRE(buffer.ModifiedCpuRegion(c + WORD * 3 + PAGE * 63, PAGE) ==
Range{WORD * 3 + PAGE * 63, WORD * 4});
buffer.MarkRegionAsCpuModified(c + WORD * 5 + PAGE * 6, PAGE);
buffer.MarkRegionAsCpuModified(c + WORD * 5 + PAGE * 8, PAGE);
REQUIRE(buffer.ModifiedCpuRegion(c + WORD * 5, WORD) ==
Range{WORD * 5 + PAGE * 6, WORD * 5 + PAGE * 9});
buffer.UnmarkRegionAsCpuModified(c + WORD * 5 + PAGE * 8, PAGE);
REQUIRE(buffer.ModifiedCpuRegion(c + WORD * 5, WORD) ==
Range{WORD * 5 + PAGE * 6, WORD * 5 + PAGE * 7});
buffer.MarkRegionAsCpuModified(c + PAGE, WORD * 31 + PAGE * 63);
REQUIRE(buffer.ModifiedCpuRegion(c, WORD * 32) == Range{PAGE, WORD * 32});
buffer.UnmarkRegionAsCpuModified(c + PAGE * 4, PAGE);
buffer.UnmarkRegionAsCpuModified(c + PAGE * 6, PAGE);
buffer.UnmarkRegionAsCpuModified(c, WORD * 32);
REQUIRE(buffer.ModifiedCpuRegion(c, WORD * 32) == Range{0, 0});
}
TEST_CASE("BufferBase: Rasterizer counting", "[video_core]") {
RasterizerInterface rasterizer;
BufferBase buffer(rasterizer, c, PAGE * 2);
REQUIRE(rasterizer.Count() == 0);
buffer.UnmarkRegionAsCpuModified(c, PAGE);
REQUIRE(rasterizer.Count() == 1);
buffer.MarkRegionAsCpuModified(c, PAGE * 2);
REQUIRE(rasterizer.Count() == 0);
buffer.UnmarkRegionAsCpuModified(c, PAGE);
buffer.UnmarkRegionAsCpuModified(c + PAGE, PAGE);
REQUIRE(rasterizer.Count() == 2);
buffer.MarkRegionAsCpuModified(c, PAGE * 2);
REQUIRE(rasterizer.Count() == 0);
}
TEST_CASE("BufferBase: Basic range", "[video_core]") {
RasterizerInterface rasterizer;
BufferBase buffer(rasterizer, c, WORD);
buffer.UnmarkRegionAsCpuModified(c, WORD);
buffer.MarkRegionAsCpuModified(c, PAGE);
int num = 0;
buffer.ForEachUploadRange(c, WORD, [&](u64 offset, u64 size) {
REQUIRE(offset == 0U);
REQUIRE(size == PAGE);
++num;
});
REQUIRE(num == 1U);
}
TEST_CASE("BufferBase: Border upload", "[video_core]") {
RasterizerInterface rasterizer;
BufferBase buffer(rasterizer, c, WORD * 2);
buffer.UnmarkRegionAsCpuModified(c, WORD * 2);
buffer.MarkRegionAsCpuModified(c + WORD - PAGE, PAGE * 2);
buffer.ForEachUploadRange(c, WORD * 2, [](u64 offset, u64 size) {
REQUIRE(offset == WORD - PAGE);
REQUIRE(size == PAGE * 2);
});
}
TEST_CASE("BufferBase: Border upload range", "[video_core]") {
RasterizerInterface rasterizer;
BufferBase buffer(rasterizer, c, WORD * 2);
buffer.UnmarkRegionAsCpuModified(c, WORD * 2);
buffer.MarkRegionAsCpuModified(c + WORD - PAGE, PAGE * 2);
buffer.ForEachUploadRange(c + WORD - PAGE, PAGE * 2, [](u64 offset, u64 size) {
REQUIRE(offset == WORD - PAGE);
REQUIRE(size == PAGE * 2);
});
buffer.MarkRegionAsCpuModified(c + WORD - PAGE, PAGE * 2);
buffer.ForEachUploadRange(c + WORD - PAGE, PAGE, [](u64 offset, u64 size) {
REQUIRE(offset == WORD - PAGE);
REQUIRE(size == PAGE);
});
buffer.ForEachUploadRange(c + WORD, PAGE, [](u64 offset, u64 size) {
REQUIRE(offset == WORD);
REQUIRE(size == PAGE);
});
}
TEST_CASE("BufferBase: Border upload partial range", "[video_core]") {
RasterizerInterface rasterizer;
BufferBase buffer(rasterizer, c, WORD * 2);
buffer.UnmarkRegionAsCpuModified(c, WORD * 2);
buffer.MarkRegionAsCpuModified(c + WORD - PAGE, PAGE * 2);
buffer.ForEachUploadRange(c + WORD - 1, 2, [](u64 offset, u64 size) {
REQUIRE(offset == WORD - PAGE);
REQUIRE(size == PAGE * 2);
});
buffer.MarkRegionAsCpuModified(c + WORD - PAGE, PAGE * 2);
buffer.ForEachUploadRange(c + WORD - 1, 1, [](u64 offset, u64 size) {
REQUIRE(offset == WORD - PAGE);
REQUIRE(size == PAGE);
});
buffer.ForEachUploadRange(c + WORD + 50, 1, [](u64 offset, u64 size) {
REQUIRE(offset == WORD);
REQUIRE(size == PAGE);
});
}
TEST_CASE("BufferBase: Partial word uploads", "[video_core]") {
RasterizerInterface rasterizer;
BufferBase buffer(rasterizer, c, 0x9d000);
int num = 0;
buffer.ForEachUploadRange(c, WORD, [&](u64 offset, u64 size) {
REQUIRE(offset == 0U);
REQUIRE(size == WORD);
++num;
});
REQUIRE(num == 1);
buffer.ForEachUploadRange(c + WORD, WORD, [&](u64 offset, u64 size) {
REQUIRE(offset == WORD);
REQUIRE(size == WORD);
++num;
});
REQUIRE(num == 2);
buffer.ForEachUploadRange(c + 0x79000, 0x24000, [&](u64 offset, u64 size) {
REQUIRE(offset == WORD * 2);
REQUIRE(size == PAGE * 0x1d);
++num;
});
REQUIRE(num == 3);
}
TEST_CASE("BufferBase: Partial page upload", "[video_core]") {
RasterizerInterface rasterizer;
BufferBase buffer(rasterizer, c, WORD);
buffer.UnmarkRegionAsCpuModified(c, WORD);
int num = 0;
buffer.MarkRegionAsCpuModified(c + PAGE * 2, PAGE);
buffer.MarkRegionAsCpuModified(c + PAGE * 9, PAGE);
buffer.ForEachUploadRange(c, PAGE * 3, [&](u64 offset, u64 size) {
REQUIRE(offset == PAGE * 2);
REQUIRE(size == PAGE);
++num;
});
REQUIRE(num == 1);
buffer.ForEachUploadRange(c + PAGE * 7, PAGE * 3, [&](u64 offset, u64 size) {
REQUIRE(offset == PAGE * 9);
REQUIRE(size == PAGE);
++num;
});
REQUIRE(num == 2);
}
TEST_CASE("BufferBase: Partial page upload with multiple words on the right") {
RasterizerInterface rasterizer;
BufferBase buffer(rasterizer, c, WORD * 8);
buffer.UnmarkRegionAsCpuModified(c, WORD * 8);
buffer.MarkRegionAsCpuModified(c + PAGE * 13, WORD * 7);
int num = 0;
buffer.ForEachUploadRange(c + PAGE * 10, WORD * 7, [&](u64 offset, u64 size) {
REQUIRE(offset == PAGE * 13);
REQUIRE(size == WORD * 7 - PAGE * 3);
++num;
});
REQUIRE(num == 1);
buffer.ForEachUploadRange(c + PAGE, WORD * 8, [&](u64 offset, u64 size) {
REQUIRE(offset == WORD * 7 + PAGE * 10);
REQUIRE(size == PAGE * 3);
++num;
});
REQUIRE(num == 2);
}
TEST_CASE("BufferBase: Partial page upload with multiple words on the left", "[video_core]") {
RasterizerInterface rasterizer;
BufferBase buffer(rasterizer, c, WORD * 8);
buffer.UnmarkRegionAsCpuModified(c, WORD * 8);
buffer.MarkRegionAsCpuModified(c + PAGE * 13, WORD * 7);
int num = 0;
buffer.ForEachUploadRange(c + PAGE * 16, WORD * 7, [&](u64 offset, u64 size) {
REQUIRE(offset == PAGE * 16);
REQUIRE(size == WORD * 7 - PAGE * 3);
++num;
});
REQUIRE(num == 1);
buffer.ForEachUploadRange(c + PAGE, WORD, [&](u64 offset, u64 size) {
REQUIRE(offset == PAGE * 13);
REQUIRE(size == PAGE * 3);
++num;
});
REQUIRE(num == 2);
}
TEST_CASE("BufferBase: Partial page upload with multiple words in the middle", "[video_core]") {
RasterizerInterface rasterizer;
BufferBase buffer(rasterizer, c, WORD * 8);
buffer.UnmarkRegionAsCpuModified(c, WORD * 8);
buffer.MarkRegionAsCpuModified(c + PAGE * 13, PAGE * 140);
int num = 0;
buffer.ForEachUploadRange(c + PAGE * 16, WORD, [&](u64 offset, u64 size) {
REQUIRE(offset == PAGE * 16);
REQUIRE(size == WORD);
++num;
});
REQUIRE(num == 1);
buffer.ForEachUploadRange(c, WORD, [&](u64 offset, u64 size) {
REQUIRE(offset == PAGE * 13);
REQUIRE(size == PAGE * 3);
++num;
});
REQUIRE(num == 2);
buffer.ForEachUploadRange(c, WORD * 8, [&](u64 offset, u64 size) {
REQUIRE(offset == WORD + PAGE * 16);
REQUIRE(size == PAGE * 73);
++num;
});
REQUIRE(num == 3);
}
TEST_CASE("BufferBase: Empty right bits", "[video_core]") {
RasterizerInterface rasterizer;
BufferBase buffer(rasterizer, c, WORD * 2048);
buffer.UnmarkRegionAsCpuModified(c, WORD * 2048);
buffer.MarkRegionAsCpuModified(c + WORD - PAGE, PAGE * 2);
buffer.ForEachUploadRange(c, WORD * 2048, [](u64 offset, u64 size) {
REQUIRE(offset == WORD - PAGE);
REQUIRE(size == PAGE * 2);
});
}
TEST_CASE("BufferBase: Out of bound ranges 1", "[video_core]") {
RasterizerInterface rasterizer;
BufferBase buffer(rasterizer, c, WORD);
buffer.UnmarkRegionAsCpuModified(c, WORD);
buffer.MarkRegionAsCpuModified(c, PAGE);
int num = 0;
buffer.ForEachUploadRange(c - WORD, WORD, [&](u64 offset, u64 size) { ++num; });
buffer.ForEachUploadRange(c + WORD, WORD, [&](u64 offset, u64 size) { ++num; });
buffer.ForEachUploadRange(c - PAGE, PAGE, [&](u64 offset, u64 size) { ++num; });
REQUIRE(num == 0);
buffer.ForEachUploadRange(c - PAGE, PAGE * 2, [&](u64 offset, u64 size) { ++num; });
REQUIRE(num == 1);
buffer.MarkRegionAsCpuModified(c, WORD);
REQUIRE(rasterizer.Count() == 0);
}
TEST_CASE("BufferBase: Out of bound ranges 2", "[video_core]") {
RasterizerInterface rasterizer;
BufferBase buffer(rasterizer, c, 0x22000);
REQUIRE_NOTHROW(buffer.UnmarkRegionAsCpuModified(c + 0x22000, PAGE));
REQUIRE_NOTHROW(buffer.UnmarkRegionAsCpuModified(c + 0x28000, PAGE));
REQUIRE(rasterizer.Count() == 0);
REQUIRE_NOTHROW(buffer.UnmarkRegionAsCpuModified(c + 0x21100, PAGE - 0x100));
REQUIRE(rasterizer.Count() == 1);
REQUIRE_NOTHROW(buffer.UnmarkRegionAsCpuModified(c - 0x1000, PAGE * 2));
buffer.UnmarkRegionAsCpuModified(c - 0x3000, PAGE * 2);
buffer.UnmarkRegionAsCpuModified(c - 0x2000, PAGE * 2);
REQUIRE(rasterizer.Count() == 2);
}
TEST_CASE("BufferBase: Out of bound ranges 3", "[video_core]") {
RasterizerInterface rasterizer;
BufferBase buffer(rasterizer, c, 0x310720);
buffer.UnmarkRegionAsCpuModified(c, 0x310720);
REQUIRE(rasterizer.Count(c) == 1);
REQUIRE(rasterizer.Count(c + PAGE) == 1);
REQUIRE(rasterizer.Count(c + WORD) == 1);
REQUIRE(rasterizer.Count(c + WORD + PAGE) == 1);
}
TEST_CASE("BufferBase: Sparse regions 1", "[video_core]") {
RasterizerInterface rasterizer;
BufferBase buffer(rasterizer, c, WORD);
buffer.UnmarkRegionAsCpuModified(c, WORD);
buffer.MarkRegionAsCpuModified(c + PAGE * 1, PAGE);
buffer.MarkRegionAsCpuModified(c + PAGE * 3, PAGE * 4);
buffer.ForEachUploadRange(c, WORD, [i = 0](u64 offset, u64 size) mutable {
static constexpr std::array<u64, 2> offsets{PAGE, PAGE * 3};
static constexpr std::array<u64, 2> sizes{PAGE, PAGE * 4};
REQUIRE(offset == offsets.at(i));
REQUIRE(size == sizes.at(i));
++i;
});
}
TEST_CASE("BufferBase: Sparse regions 2", "[video_core]") {
RasterizerInterface rasterizer;
BufferBase buffer(rasterizer, c, 0x22000);
buffer.UnmarkRegionAsCpuModified(c, 0x22000);
REQUIRE(rasterizer.Count() == 0x22);
buffer.MarkRegionAsCpuModified(c + PAGE * 0x1B, PAGE);
buffer.MarkRegionAsCpuModified(c + PAGE * 0x21, PAGE);
buffer.ForEachUploadRange(c, WORD, [i = 0](u64 offset, u64 size) mutable {
static constexpr std::array<u64, 2> offsets{PAGE * 0x1B, PAGE * 0x21};
static constexpr std::array<u64, 2> sizes{PAGE, PAGE};
REQUIRE(offset == offsets.at(i));
REQUIRE(size == sizes.at(i));
++i;
});
}
TEST_CASE("BufferBase: Single page modified range", "[video_core]") {
RasterizerInterface rasterizer;
BufferBase buffer(rasterizer, c, PAGE);
REQUIRE(buffer.IsRegionCpuModified(c, PAGE));
buffer.UnmarkRegionAsCpuModified(c, PAGE);
REQUIRE(!buffer.IsRegionCpuModified(c, PAGE));
}
TEST_CASE("BufferBase: Two page modified range", "[video_core]") {
RasterizerInterface rasterizer;
BufferBase buffer(rasterizer, c, PAGE * 2);
REQUIRE(buffer.IsRegionCpuModified(c, PAGE));
REQUIRE(buffer.IsRegionCpuModified(c + PAGE, PAGE));
REQUIRE(buffer.IsRegionCpuModified(c, PAGE * 2));
buffer.UnmarkRegionAsCpuModified(c, PAGE);
REQUIRE(!buffer.IsRegionCpuModified(c, PAGE));
}
TEST_CASE("BufferBase: Multi word modified ranges", "[video_core]") {
for (int offset = 0; offset < 4; ++offset) {
const VAddr address = c + WORD * offset;
RasterizerInterface rasterizer;
BufferBase buffer(rasterizer, address, WORD * 4);
REQUIRE(buffer.IsRegionCpuModified(address, PAGE));
REQUIRE(buffer.IsRegionCpuModified(address + PAGE * 48, PAGE));
REQUIRE(buffer.IsRegionCpuModified(address + PAGE * 56, PAGE));
buffer.UnmarkRegionAsCpuModified(address + PAGE * 32, PAGE);
REQUIRE(buffer.IsRegionCpuModified(address + PAGE, WORD));
REQUIRE(buffer.IsRegionCpuModified(address + PAGE * 31, PAGE));
REQUIRE(!buffer.IsRegionCpuModified(address + PAGE * 32, PAGE));
REQUIRE(buffer.IsRegionCpuModified(address + PAGE * 33, PAGE));
REQUIRE(buffer.IsRegionCpuModified(address + PAGE * 31, PAGE * 2));
REQUIRE(buffer.IsRegionCpuModified(address + PAGE * 32, PAGE * 2));
buffer.UnmarkRegionAsCpuModified(address + PAGE * 33, PAGE);
REQUIRE(!buffer.IsRegionCpuModified(address + PAGE * 32, PAGE * 2));
}
}
TEST_CASE("BufferBase: Single page in large buffer", "[video_core]") {
RasterizerInterface rasterizer;
BufferBase buffer(rasterizer, c, WORD * 16);
buffer.UnmarkRegionAsCpuModified(c, WORD * 16);
REQUIRE(!buffer.IsRegionCpuModified(c, WORD * 16));
buffer.MarkRegionAsCpuModified(c + WORD * 12 + PAGE * 8, PAGE);
REQUIRE(buffer.IsRegionCpuModified(c, WORD * 16));
REQUIRE(buffer.IsRegionCpuModified(c + WORD * 10, WORD * 2));
REQUIRE(buffer.IsRegionCpuModified(c + WORD * 11, WORD * 2));
REQUIRE(buffer.IsRegionCpuModified(c + WORD * 12, WORD * 2));
REQUIRE(buffer.IsRegionCpuModified(c + WORD * 12 + PAGE * 4, PAGE * 8));
REQUIRE(buffer.IsRegionCpuModified(c + WORD * 12 + PAGE * 6, PAGE * 8));
REQUIRE(!buffer.IsRegionCpuModified(c + WORD * 12 + PAGE * 6, PAGE));
REQUIRE(buffer.IsRegionCpuModified(c + WORD * 12 + PAGE * 7, PAGE * 2));
REQUIRE(buffer.IsRegionCpuModified(c + WORD * 12 + PAGE * 8, PAGE * 2));
}
TEST_CASE("BufferBase: Out of bounds region query") {
RasterizerInterface rasterizer;
BufferBase buffer(rasterizer, c, WORD * 16);
REQUIRE(!buffer.IsRegionCpuModified(c - PAGE, PAGE));
REQUIRE(!buffer.IsRegionCpuModified(c - PAGE * 2, PAGE));
REQUIRE(!buffer.IsRegionCpuModified(c + WORD * 16, PAGE));
REQUIRE(buffer.IsRegionCpuModified(c + WORD * 16 - PAGE, WORD * 64));
REQUIRE(!buffer.IsRegionCpuModified(c + WORD * 16, WORD * 64));
}
TEST_CASE("BufferBase: Wrap word regions") {
RasterizerInterface rasterizer;
BufferBase buffer(rasterizer, c, WORD * 2);
buffer.UnmarkRegionAsCpuModified(c, WORD * 2);
buffer.MarkRegionAsCpuModified(c + PAGE * 63, PAGE * 2);
REQUIRE(buffer.IsRegionCpuModified(c, WORD * 2));
REQUIRE(!buffer.IsRegionCpuModified(c + PAGE * 62, PAGE));
REQUIRE(buffer.IsRegionCpuModified(c + PAGE * 63, PAGE));
REQUIRE(buffer.IsRegionCpuModified(c + PAGE * 64, PAGE));
REQUIRE(buffer.IsRegionCpuModified(c + PAGE * 63, PAGE * 2));
REQUIRE(buffer.IsRegionCpuModified(c + PAGE * 63, PAGE * 8));
REQUIRE(buffer.IsRegionCpuModified(c + PAGE * 60, PAGE * 8));
REQUIRE(!buffer.IsRegionCpuModified(c + PAGE * 127, WORD * 16));
buffer.MarkRegionAsCpuModified(c + PAGE * 127, PAGE);
REQUIRE(buffer.IsRegionCpuModified(c + PAGE * 127, WORD * 16));
REQUIRE(buffer.IsRegionCpuModified(c + PAGE * 127, PAGE));
REQUIRE(!buffer.IsRegionCpuModified(c + PAGE * 126, PAGE));
REQUIRE(buffer.IsRegionCpuModified(c + PAGE * 126, PAGE * 2));
REQUIRE(!buffer.IsRegionCpuModified(c + PAGE * 128, WORD * 16));
}
TEST_CASE("BufferBase: Unaligned page region query") {
RasterizerInterface rasterizer;
BufferBase buffer(rasterizer, c, WORD);
buffer.UnmarkRegionAsCpuModified(c, WORD);
buffer.MarkRegionAsCpuModified(c + 4000, 1000);
REQUIRE(buffer.IsRegionCpuModified(c, PAGE));
REQUIRE(buffer.IsRegionCpuModified(c + PAGE, PAGE));
REQUIRE(buffer.IsRegionCpuModified(c + 4000, 1000));
REQUIRE(buffer.IsRegionCpuModified(c + 4000, 1));
}
TEST_CASE("BufferBase: Cached write") {
RasterizerInterface rasterizer;
BufferBase buffer(rasterizer, c, WORD);
buffer.UnmarkRegionAsCpuModified(c, WORD);
buffer.CachedCpuWrite(c + PAGE, PAGE);
REQUIRE(!buffer.IsRegionCpuModified(c + PAGE, PAGE));
buffer.FlushCachedWrites();
REQUIRE(buffer.IsRegionCpuModified(c + PAGE, PAGE));
buffer.MarkRegionAsCpuModified(c, WORD);
REQUIRE(rasterizer.Count() == 0);
}
TEST_CASE("BufferBase: Multiple cached write") {
RasterizerInterface rasterizer;
BufferBase buffer(rasterizer, c, WORD);
buffer.UnmarkRegionAsCpuModified(c, WORD);
buffer.CachedCpuWrite(c + PAGE, PAGE);
buffer.CachedCpuWrite(c + PAGE * 3, PAGE);
REQUIRE(!buffer.IsRegionCpuModified(c + PAGE, PAGE));
REQUIRE(!buffer.IsRegionCpuModified(c + PAGE * 3, PAGE));
buffer.FlushCachedWrites();
REQUIRE(buffer.IsRegionCpuModified(c + PAGE, PAGE));
REQUIRE(buffer.IsRegionCpuModified(c + PAGE * 3, PAGE));
buffer.MarkRegionAsCpuModified(c, WORD);
REQUIRE(rasterizer.Count() == 0);
}
TEST_CASE("BufferBase: Cached write unmarked") {
RasterizerInterface rasterizer;
BufferBase buffer(rasterizer, c, WORD);
buffer.UnmarkRegionAsCpuModified(c, WORD);
buffer.CachedCpuWrite(c + PAGE, PAGE);
buffer.UnmarkRegionAsCpuModified(c + PAGE, PAGE);
REQUIRE(!buffer.IsRegionCpuModified(c + PAGE, PAGE));
buffer.FlushCachedWrites();
REQUIRE(buffer.IsRegionCpuModified(c + PAGE, PAGE));
buffer.MarkRegionAsCpuModified(c, WORD);
REQUIRE(rasterizer.Count() == 0);
}
TEST_CASE("BufferBase: Cached write iterated") {
RasterizerInterface rasterizer;
BufferBase buffer(rasterizer, c, WORD);
buffer.UnmarkRegionAsCpuModified(c, WORD);
buffer.CachedCpuWrite(c + PAGE, PAGE);
int num = 0;
buffer.ForEachUploadRange(c, WORD, [&](u64 offset, u64 size) { ++num; });
REQUIRE(num == 0);
REQUIRE(!buffer.IsRegionCpuModified(c + PAGE, PAGE));
buffer.FlushCachedWrites();
REQUIRE(buffer.IsRegionCpuModified(c + PAGE, PAGE));
buffer.MarkRegionAsCpuModified(c, WORD);
REQUIRE(rasterizer.Count() == 0);
}
TEST_CASE("BufferBase: Cached write downloads") {
RasterizerInterface rasterizer;
BufferBase buffer(rasterizer, c, WORD);
buffer.UnmarkRegionAsCpuModified(c, WORD);
REQUIRE(rasterizer.Count() == 64);
buffer.CachedCpuWrite(c + PAGE, PAGE);
REQUIRE(rasterizer.Count() == 63);
buffer.MarkRegionAsGpuModified(c + PAGE, PAGE);
int num = 0;
buffer.ForEachDownloadRangeAndClear(c, WORD, [&](u64 offset, u64 size) { ++num; });
buffer.ForEachUploadRange(c, WORD, [&](u64 offset, u64 size) { ++num; });
REQUIRE(num == 0);
REQUIRE(!buffer.IsRegionCpuModified(c + PAGE, PAGE));
REQUIRE(!buffer.IsRegionGpuModified(c + PAGE, PAGE));
buffer.FlushCachedWrites();
REQUIRE(buffer.IsRegionCpuModified(c + PAGE, PAGE));
REQUIRE(!buffer.IsRegionGpuModified(c + PAGE, PAGE));
buffer.MarkRegionAsCpuModified(c, WORD);
REQUIRE(rasterizer.Count() == 0);
}

549
src/tests/video_core/memory_tracker.cpp
View File

@ -0,0 +1,549 @@
// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
#include <memory>
#include <stdexcept>
#include <unordered_map>
#include <catch2/catch_test_macros.hpp>
#include "common/alignment.h"
#include "common/common_types.h"
#include "video_core/buffer_cache/memory_tracker_base.h"
namespace {
using Range = std::pair<u64, u64>;
constexpr u64 PAGE = 4096;
constexpr u64 WORD = 4096 * 64;
constexpr u64 HIGH_PAGE_BITS = 22;
constexpr u64 HIGH_PAGE_SIZE = 1ULL << HIGH_PAGE_BITS;
constexpr VAddr c = 16 * HIGH_PAGE_SIZE;
class RasterizerInterface {
public:
void UpdatePagesCachedCount(VAddr addr, u64 size, int delta) {
const u64 page_start{addr >> Core::Memory::YUZU_PAGEBITS};
const u64 page_end{(addr + size + Core::Memory::YUZU_PAGESIZE - 1) >>
Core::Memory::YUZU_PAGEBITS};
for (u64 page = page_start; page < page_end; ++page) {
int& value = page_table[page];
value += delta;
if (value < 0) {
throw std::logic_error{"negative page"};
}
if (value == 0) {
page_table.erase(page);
}
}
}
[[nodiscard]] int Count(VAddr addr) const noexcept {
const auto it = page_table.find(addr >> Core::Memory::YUZU_PAGEBITS);
return it == page_table.end() ? 0 : it->second;
}
[[nodiscard]] unsigned Count() const noexcept {
unsigned count = 0;
for (const auto& [index, value] : page_table) {
count += value;
}
return count;
}
private:
std::unordered_map<u64, int> page_table;
};
} // Anonymous namespace
using MemoryTracker = VideoCommon::MemoryTrackerBase<RasterizerInterface>;
TEST_CASE("MemoryTracker: Small region", "[video_core]") {
RasterizerInterface rasterizer;
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
REQUIRE(rasterizer.Count() == 0);
memory_track->UnmarkRegionAsCpuModified(c, WORD);
REQUIRE(rasterizer.Count() == WORD / PAGE);
REQUIRE(memory_track->ModifiedCpuRegion(c, WORD) == Range{0, 0});
memory_track->MarkRegionAsCpuModified(c + PAGE, 1);
REQUIRE(memory_track->ModifiedCpuRegion(c, WORD) == Range{c + PAGE * 1, c + PAGE * 2});
}
TEST_CASE("MemoryTracker: Large region", "[video_core]") {
RasterizerInterface rasterizer;
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
memory_track->UnmarkRegionAsCpuModified(c, WORD * 32);
memory_track->MarkRegionAsCpuModified(c + 4096, WORD * 4);
REQUIRE(memory_track->ModifiedCpuRegion(c, WORD + PAGE * 2) ==
Range{c + PAGE, c + WORD + PAGE * 2});
REQUIRE(memory_track->ModifiedCpuRegion(c + PAGE * 2, PAGE * 6) ==
Range{c + PAGE * 2, c + PAGE * 8});
REQUIRE(memory_track->ModifiedCpuRegion(c, WORD * 32) == Range{c + PAGE, c + WORD * 4 + PAGE});
REQUIRE(memory_track->ModifiedCpuRegion(c + WORD * 4, PAGE) ==
Range{c + WORD * 4, c + WORD * 4 + PAGE});
REQUIRE(memory_track->ModifiedCpuRegion(c + WORD * 3 + PAGE * 63, PAGE) ==
Range{c + WORD * 3 + PAGE * 63, c + WORD * 4});
memory_track->MarkRegionAsCpuModified(c + WORD * 5 + PAGE * 6, PAGE);
memory_track->MarkRegionAsCpuModified(c + WORD * 5 + PAGE * 8, PAGE);
REQUIRE(memory_track->ModifiedCpuRegion(c + WORD * 5, WORD) ==
Range{c + WORD * 5 + PAGE * 6, c + WORD * 5 + PAGE * 9});
memory_track->UnmarkRegionAsCpuModified(c + WORD * 5 + PAGE * 8, PAGE);
REQUIRE(memory_track->ModifiedCpuRegion(c + WORD * 5, WORD) ==
Range{c + WORD * 5 + PAGE * 6, c + WORD * 5 + PAGE * 7});
memory_track->MarkRegionAsCpuModified(c + PAGE, WORD * 31 + PAGE * 63);
REQUIRE(memory_track->ModifiedCpuRegion(c, WORD * 32) == Range{c + PAGE, c + WORD * 32});
memory_track->UnmarkRegionAsCpuModified(c + PAGE * 4, PAGE);
memory_track->UnmarkRegionAsCpuModified(c + PAGE * 6, PAGE);
memory_track->UnmarkRegionAsCpuModified(c, WORD * 32);
REQUIRE(memory_track->ModifiedCpuRegion(c, WORD * 32) == Range{0, 0});
}
TEST_CASE("MemoryTracker: Rasterizer counting", "[video_core]") {
RasterizerInterface rasterizer;
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
REQUIRE(rasterizer.Count() == 0);
memory_track->UnmarkRegionAsCpuModified(c, PAGE);
REQUIRE(rasterizer.Count() == 1);
memory_track->MarkRegionAsCpuModified(c, PAGE * 2);
REQUIRE(rasterizer.Count() == 0);
memory_track->UnmarkRegionAsCpuModified(c, PAGE);
memory_track->UnmarkRegionAsCpuModified(c + PAGE, PAGE);
REQUIRE(rasterizer.Count() == 2);
memory_track->MarkRegionAsCpuModified(c, PAGE * 2);
REQUIRE(rasterizer.Count() == 0);
}
TEST_CASE("MemoryTracker: Basic range", "[video_core]") {
RasterizerInterface rasterizer;
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
memory_track->UnmarkRegionAsCpuModified(c, WORD);
memory_track->MarkRegionAsCpuModified(c, PAGE);
int num = 0;
memory_track->ForEachUploadRange(c, WORD, [&](u64 offset, u64 size) {
REQUIRE(offset == c);
REQUIRE(size == PAGE);
++num;
});
REQUIRE(num == 1U);
}
TEST_CASE("MemoryTracker: Border upload", "[video_core]") {
RasterizerInterface rasterizer;
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
memory_track->UnmarkRegionAsCpuModified(c, WORD * 2);
memory_track->MarkRegionAsCpuModified(c + WORD - PAGE, PAGE * 2);
memory_track->ForEachUploadRange(c, WORD * 2, [](u64 offset, u64 size) {
REQUIRE(offset == c + WORD - PAGE);
REQUIRE(size == PAGE * 2);
});
}
TEST_CASE("MemoryTracker: Border upload range", "[video_core]") {
RasterizerInterface rasterizer;
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
memory_track->UnmarkRegionAsCpuModified(c, WORD * 2);
memory_track->MarkRegionAsCpuModified(c + WORD - PAGE, PAGE * 2);
memory_track->ForEachUploadRange(c + WORD - PAGE, PAGE * 2, [](u64 offset, u64 size) {
REQUIRE(offset == c + WORD - PAGE);
REQUIRE(size == PAGE * 2);
});
memory_track->MarkRegionAsCpuModified(c + WORD - PAGE, PAGE * 2);
memory_track->ForEachUploadRange(c + WORD - PAGE, PAGE, [](u64 offset, u64 size) {
REQUIRE(offset == c + WORD - PAGE);
REQUIRE(size == PAGE);
});
memory_track->ForEachUploadRange(c + WORD, PAGE, [](u64 offset, u64 size) {
REQUIRE(offset == c + WORD);
REQUIRE(size == PAGE);
});
}
TEST_CASE("MemoryTracker: Border upload partial range", "[video_core]") {
RasterizerInterface rasterizer;
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
memory_track->UnmarkRegionAsCpuModified(c, WORD * 2);
memory_track->MarkRegionAsCpuModified(c + WORD - PAGE, PAGE * 2);
memory_track->ForEachUploadRange(c + WORD - 1, 2, [](u64 offset, u64 size) {
REQUIRE(offset == c + WORD - PAGE);
REQUIRE(size == PAGE * 2);
});
memory_track->MarkRegionAsCpuModified(c + WORD - PAGE, PAGE * 2);
memory_track->ForEachUploadRange(c + WORD - 1, 1, [](u64 offset, u64 size) {
REQUIRE(offset == c + WORD - PAGE);
REQUIRE(size == PAGE);
});
memory_track->ForEachUploadRange(c + WORD + 50, 1, [](u64 offset, u64 size) {
REQUIRE(offset == c + WORD);
REQUIRE(size == PAGE);
});
}
TEST_CASE("MemoryTracker: Partial word uploads", "[video_core]") {
RasterizerInterface rasterizer;
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
int num = 0;
memory_track->ForEachUploadRange(c, WORD, [&](u64 offset, u64 size) {
REQUIRE(offset == c);
REQUIRE(size == WORD);
++num;
});
REQUIRE(num == 1);
memory_track->ForEachUploadRange(c + WORD, WORD, [&](u64 offset, u64 size) {
REQUIRE(offset == c + WORD);
REQUIRE(size == WORD);
++num;
});
REQUIRE(num == 2);
memory_track->ForEachUploadRange(c + 0x79000, 0x24000, [&](u64 offset, u64 size) {
REQUIRE(offset == c + WORD * 2);
REQUIRE(size == PAGE * 0x1d);
++num;
});
REQUIRE(num == 3);
}
TEST_CASE("MemoryTracker: Partial page upload", "[video_core]") {
RasterizerInterface rasterizer;
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
memory_track->UnmarkRegionAsCpuModified(c, WORD);
int num = 0;
memory_track->MarkRegionAsCpuModified(c + PAGE * 2, PAGE);
memory_track->MarkRegionAsCpuModified(c + PAGE * 9, PAGE);
memory_track->ForEachUploadRange(c, PAGE * 3, [&](u64 offset, u64 size) {
REQUIRE(offset == c + PAGE * 2);
REQUIRE(size == PAGE);
++num;
});
REQUIRE(num == 1);
memory_track->ForEachUploadRange(c + PAGE * 7, PAGE * 3, [&](u64 offset, u64 size) {
REQUIRE(offset == c + PAGE * 9);
REQUIRE(size == PAGE);
++num;
});
REQUIRE(num == 2);
}
TEST_CASE("MemoryTracker: Partial page upload with multiple words on the right") {
RasterizerInterface rasterizer;
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
memory_track->UnmarkRegionAsCpuModified(c, WORD * 9);
memory_track->MarkRegionAsCpuModified(c + PAGE * 13, WORD * 7);
int num = 0;
memory_track->ForEachUploadRange(c + PAGE * 10, WORD * 7, [&](u64 offset, u64 size) {
REQUIRE(offset == c + PAGE * 13);
REQUIRE(size == WORD * 7 - PAGE * 3);
++num;
});
REQUIRE(num == 1);
memory_track->ForEachUploadRange(c + PAGE, WORD * 8, [&](u64 offset, u64 size) {
REQUIRE(offset == c + WORD * 7 + PAGE * 10);
REQUIRE(size == PAGE * 3);
++num;
});
REQUIRE(num == 2);
}
TEST_CASE("MemoryTracker: Partial page upload with multiple words on the left", "[video_core]") {
RasterizerInterface rasterizer;
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
memory_track->UnmarkRegionAsCpuModified(c, WORD * 8);
memory_track->MarkRegionAsCpuModified(c + PAGE * 13, WORD * 7);
int num = 0;
memory_track->ForEachUploadRange(c + PAGE * 16, WORD * 7, [&](u64 offset, u64 size) {
REQUIRE(offset == c + PAGE * 16);
REQUIRE(size == WORD * 7 - PAGE * 3);
++num;
});
REQUIRE(num == 1);
memory_track->ForEachUploadRange(c + PAGE, WORD, [&](u64 offset, u64 size) {
REQUIRE(offset == c + PAGE * 13);
REQUIRE(size == PAGE * 3);
++num;
});
REQUIRE(num == 2);
}
TEST_CASE("MemoryTracker: Partial page upload with multiple words in the middle", "[video_core]") {
RasterizerInterface rasterizer;
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
memory_track->UnmarkRegionAsCpuModified(c, WORD * 8);
memory_track->MarkRegionAsCpuModified(c + PAGE * 13, PAGE * 140);
int num = 0;
memory_track->ForEachUploadRange(c + PAGE * 16, WORD, [&](u64 offset, u64 size) {
REQUIRE(offset == c + PAGE * 16);
REQUIRE(size == WORD);
++num;
});
REQUIRE(num == 1);
memory_track->ForEachUploadRange(c, WORD, [&](u64 offset, u64 size) {
REQUIRE(offset == c + PAGE * 13);
REQUIRE(size == PAGE * 3);
++num;
});
REQUIRE(num == 2);
memory_track->ForEachUploadRange(c, WORD * 8, [&](u64 offset, u64 size) {
REQUIRE(offset == c + WORD + PAGE * 16);
REQUIRE(size == PAGE * 73);
++num;
});
REQUIRE(num == 3);
}
TEST_CASE("MemoryTracker: Empty right bits", "[video_core]") {
RasterizerInterface rasterizer;
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
memory_track->UnmarkRegionAsCpuModified(c, WORD * 2048);
memory_track->MarkRegionAsCpuModified(c + WORD - PAGE, PAGE * 2);
memory_track->ForEachUploadRange(c, WORD * 2048, [](u64 offset, u64 size) {
REQUIRE(offset == c + WORD - PAGE);
REQUIRE(size == PAGE * 2);
});
}
TEST_CASE("MemoryTracker: Out of bound ranges 1", "[video_core]") {
RasterizerInterface rasterizer;
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
memory_track->UnmarkRegionAsCpuModified(c - WORD, 3 * WORD);
memory_track->MarkRegionAsCpuModified(c, PAGE);
REQUIRE(rasterizer.Count() == (3 * WORD - PAGE) / PAGE);
int num = 0;
memory_track->ForEachUploadRange(c - WORD, WORD, [&](u64 offset, u64 size) { ++num; });
memory_track->ForEachUploadRange(c + WORD, WORD, [&](u64 offset, u64 size) { ++num; });
memory_track->ForEachUploadRange(c - PAGE, PAGE, [&](u64 offset, u64 size) { ++num; });
REQUIRE(num == 0);
memory_track->ForEachUploadRange(c - PAGE, PAGE * 2, [&](u64 offset, u64 size) { ++num; });
REQUIRE(num == 1);
memory_track->MarkRegionAsCpuModified(c, WORD);
REQUIRE(rasterizer.Count() == 2 * WORD / PAGE);
}
TEST_CASE("MemoryTracker: Out of bound ranges 2", "[video_core]") {
RasterizerInterface rasterizer;
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
REQUIRE_NOTHROW(memory_track->UnmarkRegionAsCpuModified(c + 0x22000, PAGE));
REQUIRE_NOTHROW(memory_track->UnmarkRegionAsCpuModified(c + 0x28000, PAGE));
REQUIRE(rasterizer.Count() == 2);
REQUIRE_NOTHROW(memory_track->UnmarkRegionAsCpuModified(c + 0x21100, PAGE - 0x100));
REQUIRE(rasterizer.Count() == 3);
REQUIRE_NOTHROW(memory_track->UnmarkRegionAsCpuModified(c - PAGE, PAGE * 2));
memory_track->UnmarkRegionAsCpuModified(c - PAGE * 3, PAGE * 2);
memory_track->UnmarkRegionAsCpuModified(c - PAGE * 2, PAGE * 2);
REQUIRE(rasterizer.Count() == 7);
}
TEST_CASE("MemoryTracker: Out of bound ranges 3", "[video_core]") {
RasterizerInterface rasterizer;
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
memory_track->UnmarkRegionAsCpuModified(c, 0x310720);
REQUIRE(rasterizer.Count(c) == 1);
REQUIRE(rasterizer.Count(c + PAGE) == 1);
REQUIRE(rasterizer.Count(c + WORD) == 1);
REQUIRE(rasterizer.Count(c + WORD + PAGE) == 1);
}
TEST_CASE("MemoryTracker: Sparse regions 1", "[video_core]") {
RasterizerInterface rasterizer;
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
memory_track->UnmarkRegionAsCpuModified(c, WORD);
memory_track->MarkRegionAsCpuModified(c + PAGE * 1, PAGE);
memory_track->MarkRegionAsCpuModified(c + PAGE * 3, PAGE * 4);
memory_track->ForEachUploadRange(c, WORD, [i = 0](u64 offset, u64 size) mutable {
static constexpr std::array<u64, 2> offsets{c + PAGE, c + PAGE * 3};
static constexpr std::array<u64, 2> sizes{PAGE, PAGE * 4};
REQUIRE(offset == offsets.at(i));
REQUIRE(size == sizes.at(i));
++i;
});
}
TEST_CASE("MemoryTracker: Sparse regions 2", "[video_core]") {
RasterizerInterface rasterizer;
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
memory_track->UnmarkRegionAsCpuModified(c, PAGE * 0x23);
REQUIRE(rasterizer.Count() == 0x23);
memory_track->MarkRegionAsCpuModified(c + PAGE * 0x1B, PAGE);
memory_track->MarkRegionAsCpuModified(c + PAGE * 0x21, PAGE);
memory_track->ForEachUploadRange(c, PAGE * 0x23, [i = 0](u64 offset, u64 size) mutable {
static constexpr std::array<u64, 3> offsets{c + PAGE * 0x1B, c + PAGE * 0x21};
static constexpr std::array<u64, 3> sizes{PAGE, PAGE};
REQUIRE(offset == offsets.at(i));
REQUIRE(size == sizes.at(i));
++i;
});
}
TEST_CASE("MemoryTracker: Single page modified range", "[video_core]") {
RasterizerInterface rasterizer;
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
REQUIRE(memory_track->IsRegionCpuModified(c, PAGE));
memory_track->UnmarkRegionAsCpuModified(c, PAGE);
REQUIRE(!memory_track->IsRegionCpuModified(c, PAGE));
}
TEST_CASE("MemoryTracker: Two page modified range", "[video_core]") {
RasterizerInterface rasterizer;
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
REQUIRE(memory_track->IsRegionCpuModified(c, PAGE));
REQUIRE(memory_track->IsRegionCpuModified(c + PAGE, PAGE));
REQUIRE(memory_track->IsRegionCpuModified(c, PAGE * 2));
memory_track->UnmarkRegionAsCpuModified(c, PAGE);
REQUIRE(!memory_track->IsRegionCpuModified(c, PAGE));
}
TEST_CASE("MemoryTracker: Multi word modified ranges", "[video_core]") {
for (int offset = 0; offset < 4; ++offset) {
const VAddr address = c + WORD * offset;
RasterizerInterface rasterizer;
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
REQUIRE(memory_track->IsRegionCpuModified(address, PAGE));
REQUIRE(memory_track->IsRegionCpuModified(address + PAGE * 48, PAGE));
REQUIRE(memory_track->IsRegionCpuModified(address + PAGE * 56, PAGE));
memory_track->UnmarkRegionAsCpuModified(address + PAGE * 32, PAGE);
REQUIRE(memory_track->IsRegionCpuModified(address + PAGE, WORD));
REQUIRE(memory_track->IsRegionCpuModified(address + PAGE * 31, PAGE));
REQUIRE(!memory_track->IsRegionCpuModified(address + PAGE * 32, PAGE));
REQUIRE(memory_track->IsRegionCpuModified(address + PAGE * 33, PAGE));
REQUIRE(memory_track->IsRegionCpuModified(address + PAGE * 31, PAGE * 2));
REQUIRE(memory_track->IsRegionCpuModified(address + PAGE * 32, PAGE * 2));
memory_track->UnmarkRegionAsCpuModified(address + PAGE * 33, PAGE);
REQUIRE(!memory_track->IsRegionCpuModified(address + PAGE * 32, PAGE * 2));
}
}
TEST_CASE("MemoryTracker: Single page in large region", "[video_core]") {
RasterizerInterface rasterizer;
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
memory_track->UnmarkRegionAsCpuModified(c, WORD * 16);
REQUIRE(!memory_track->IsRegionCpuModified(c, WORD * 16));
memory_track->MarkRegionAsCpuModified(c + WORD * 12 + PAGE * 8, PAGE);
REQUIRE(memory_track->IsRegionCpuModified(c, WORD * 16));
REQUIRE(!memory_track->IsRegionCpuModified(c + WORD * 10, WORD * 2));
REQUIRE(memory_track->IsRegionCpuModified(c + WORD * 11, WORD * 2));
REQUIRE(memory_track->IsRegionCpuModified(c + WORD * 12, WORD * 2));
REQUIRE(memory_track->IsRegionCpuModified(c + WORD * 12 + PAGE * 4, PAGE * 8));
REQUIRE(memory_track->IsRegionCpuModified(c + WORD * 12 + PAGE * 6, PAGE * 8));
REQUIRE(!memory_track->IsRegionCpuModified(c + WORD * 12 + PAGE * 6, PAGE));
REQUIRE(memory_track->IsRegionCpuModified(c + WORD * 12 + PAGE * 7, PAGE * 2));
REQUIRE(memory_track->IsRegionCpuModified(c + WORD * 12 + PAGE * 8, PAGE * 2));
}
TEST_CASE("MemoryTracker: Wrap word regions") {
RasterizerInterface rasterizer;
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
memory_track->UnmarkRegionAsCpuModified(c, WORD * 32);
memory_track->MarkRegionAsCpuModified(c + PAGE * 63, PAGE * 2);
REQUIRE(memory_track->IsRegionCpuModified(c, WORD * 2));
REQUIRE(!memory_track->IsRegionCpuModified(c + PAGE * 62, PAGE));
REQUIRE(memory_track->IsRegionCpuModified(c + PAGE * 63, PAGE));
REQUIRE(memory_track->IsRegionCpuModified(c + PAGE * 64, PAGE));
REQUIRE(memory_track->IsRegionCpuModified(c + PAGE * 63, PAGE * 2));
REQUIRE(memory_track->IsRegionCpuModified(c + PAGE * 63, PAGE * 8));
REQUIRE(memory_track->IsRegionCpuModified(c + PAGE * 60, PAGE * 8));
REQUIRE(!memory_track->IsRegionCpuModified(c + PAGE * 127, WORD * 16));
memory_track->MarkRegionAsCpuModified(c + PAGE * 127, PAGE);
REQUIRE(memory_track->IsRegionCpuModified(c + PAGE * 127, WORD * 16));
REQUIRE(memory_track->IsRegionCpuModified(c + PAGE * 127, PAGE));
REQUIRE(!memory_track->IsRegionCpuModified(c + PAGE * 126, PAGE));
REQUIRE(memory_track->IsRegionCpuModified(c + PAGE * 126, PAGE * 2));
REQUIRE(!memory_track->IsRegionCpuModified(c + PAGE * 128, WORD * 16));
}
TEST_CASE("MemoryTracker: Unaligned page region query") {
RasterizerInterface rasterizer;
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
memory_track->UnmarkRegionAsCpuModified(c, WORD);
memory_track->MarkRegionAsCpuModified(c + 4000, 1000);
REQUIRE(memory_track->IsRegionCpuModified(c, PAGE));
REQUIRE(memory_track->IsRegionCpuModified(c + PAGE, PAGE));
REQUIRE(memory_track->IsRegionCpuModified(c + 4000, 1000));
REQUIRE(memory_track->IsRegionCpuModified(c + 4000, 1));
}
TEST_CASE("MemoryTracker: Cached write") {
RasterizerInterface rasterizer;
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
memory_track->UnmarkRegionAsCpuModified(c, WORD);
memory_track->CachedCpuWrite(c + PAGE, c + PAGE);
REQUIRE(!memory_track->IsRegionCpuModified(c + PAGE, PAGE));
memory_track->FlushCachedWrites();
REQUIRE(memory_track->IsRegionCpuModified(c + PAGE, PAGE));
memory_track->MarkRegionAsCpuModified(c, WORD);
REQUIRE(rasterizer.Count() == 0);
}
TEST_CASE("MemoryTracker: Multiple cached write") {
RasterizerInterface rasterizer;
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
memory_track->UnmarkRegionAsCpuModified(c, WORD);
memory_track->CachedCpuWrite(c + PAGE, PAGE);
memory_track->CachedCpuWrite(c + PAGE * 3, PAGE);
REQUIRE(!memory_track->IsRegionCpuModified(c + PAGE, PAGE));
REQUIRE(!memory_track->IsRegionCpuModified(c + PAGE * 3, PAGE));
memory_track->FlushCachedWrites();
REQUIRE(memory_track->IsRegionCpuModified(c + PAGE, PAGE));
REQUIRE(memory_track->IsRegionCpuModified(c + PAGE * 3, PAGE));
memory_track->MarkRegionAsCpuModified(c, WORD);
REQUIRE(rasterizer.Count() == 0);
}
TEST_CASE("MemoryTracker: Cached write unmarked") {
RasterizerInterface rasterizer;
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
memory_track->UnmarkRegionAsCpuModified(c, WORD);
memory_track->CachedCpuWrite(c + PAGE, PAGE);
memory_track->UnmarkRegionAsCpuModified(c + PAGE, PAGE);
REQUIRE(!memory_track->IsRegionCpuModified(c + PAGE, PAGE));
memory_track->FlushCachedWrites();
REQUIRE(memory_track->IsRegionCpuModified(c + PAGE, PAGE));
memory_track->MarkRegionAsCpuModified(c, WORD);
REQUIRE(rasterizer.Count() == 0);
}
TEST_CASE("MemoryTracker: Cached write iterated") {
RasterizerInterface rasterizer;
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
memory_track->UnmarkRegionAsCpuModified(c, WORD);
memory_track->CachedCpuWrite(c + PAGE, PAGE);
int num = 0;
memory_track->ForEachUploadRange(c, WORD, [&](u64 offset, u64 size) { ++num; });
REQUIRE(num == 0);
REQUIRE(!memory_track->IsRegionCpuModified(c + PAGE, PAGE));
memory_track->FlushCachedWrites();
REQUIRE(memory_track->IsRegionCpuModified(c + PAGE, PAGE));
memory_track->MarkRegionAsCpuModified(c, WORD);
REQUIRE(rasterizer.Count() == 0);
}
TEST_CASE("MemoryTracker: Cached write downloads") {
RasterizerInterface rasterizer;
std::unique_ptr<MemoryTracker> memory_track(std::make_unique<MemoryTracker>(rasterizer));
memory_track->UnmarkRegionAsCpuModified(c, WORD);
REQUIRE(rasterizer.Count() == 64);
memory_track->CachedCpuWrite(c + PAGE, PAGE);
REQUIRE(rasterizer.Count() == 63);
memory_track->MarkRegionAsGpuModified(c + PAGE, PAGE);
int num = 0;
memory_track->ForEachDownloadRangeAndClear(c, WORD, [&](u64 offset, u64 size) { ++num; });
REQUIRE(num == 1);
num = 0;
memory_track->ForEachUploadRange(c, WORD, [&](u64 offset, u64 size) { ++num; });
REQUIRE(num == 0);
REQUIRE(!memory_track->IsRegionCpuModified(c + PAGE, PAGE));
REQUIRE(!memory_track->IsRegionGpuModified(c + PAGE, PAGE));
memory_track->FlushCachedWrites();
REQUIRE(memory_track->IsRegionCpuModified(c + PAGE, PAGE));
REQUIRE(!memory_track->IsRegionGpuModified(c + PAGE, PAGE));
memory_track->MarkRegionAsCpuModified(c, WORD);
REQUIRE(rasterizer.Count() == 0);
}

5
src/video_core/CMakeLists.txt
View File

@ -11,8 +11,11 @@ endif()
add_library(video_core STATIC
buffer_cache/buffer_base.h
buffer_cache/buffer_cache_base.h
buffer_cache/buffer_cache.cpp
buffer_cache/buffer_cache.h
buffer_cache/memory_tracker_base.h
buffer_cache/word_manager.h
cache_types.h
cdma_pusher.cpp
cdma_pusher.h
@ -104,6 +107,7 @@ add_library(video_core STATIC
renderer_null/renderer_null.h
renderer_opengl/blit_image.cpp
renderer_opengl/blit_image.h
renderer_opengl/gl_buffer_cache_base.cpp
renderer_opengl/gl_buffer_cache.cpp
renderer_opengl/gl_buffer_cache.h
renderer_opengl/gl_compute_pipeline.cpp
@ -154,6 +158,7 @@ add_library(video_core STATIC
renderer_vulkan/renderer_vulkan.cpp
renderer_vulkan/vk_blit_screen.cpp
renderer_vulkan/vk_blit_screen.h
renderer_vulkan/vk_buffer_cache_base.cpp
renderer_vulkan/vk_buffer_cache.cpp
renderer_vulkan/vk_buffer_cache.h
renderer_vulkan/vk_command_pool.cpp

518
src/video_core/buffer_cache/buffer_base.h
View File

@ -1,5 +1,5 @@
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
#pragma once
@ -11,9 +11,7 @@
#include "common/alignment.h"
#include "common/common_funcs.h"
#include "common/common_types.h"
#include "common/div_ceil.h"
#include "common/settings.h"
#include "core/memory.h"
#include "video_core/buffer_cache/word_manager.h"
namespace VideoCommon {
@ -36,116 +34,12 @@ struct NullBufferParams {};
*/
template <class RasterizerInterface>
class BufferBase {
static constexpr u64 PAGES_PER_WORD = 64;
static constexpr u64 BYTES_PER_PAGE = Core::Memory::YUZU_PAGESIZE;
static constexpr u64 BYTES_PER_WORD = PAGES_PER_WORD * BYTES_PER_PAGE;
/// Vector tracking modified pages tightly packed with small vector optimization
union WordsArray {
/// Returns the pointer to the words state
[[nodiscard]] const u64* Pointer(bool is_short) const noexcept {
return is_short ? &stack : heap;
}
/// Returns the pointer to the words state
[[nodiscard]] u64* Pointer(bool is_short) noexcept {
return is_short ? &stack : heap;
}
u64 stack = 0; ///< Small buffers storage
u64* heap; ///< Not-small buffers pointer to the storage
};
struct Words {
explicit Words() = default;
explicit Words(u64 size_bytes_) : size_bytes{size_bytes_} {
if (IsShort()) {
cpu.stack = ~u64{0};
gpu.stack = 0;
cached_cpu.stack = 0;
untracked.stack = ~u64{0};
} else {
// Share allocation between CPU and GPU pages and set their default values
const size_t num_words = NumWords();
u64* const alloc = new u64[num_words * 4];
cpu.heap = alloc;
gpu.heap = alloc + num_words;
cached_cpu.heap = alloc + num_words * 2;
untracked.heap = alloc + num_words * 3;
std::fill_n(cpu.heap, num_words, ~u64{0});
std::fill_n(gpu.heap, num_words, 0);
std::fill_n(cached_cpu.heap, num_words, 0);
std::fill_n(untracked.heap, num_words, ~u64{0});
}
// Clean up tailing bits
const u64 last_word_size = size_bytes % BYTES_PER_WORD;
const u64 last_local_page = Common::DivCeil(last_word_size, BYTES_PER_PAGE);
const u64 shift = (PAGES_PER_WORD - last_local_page) % PAGES_PER_WORD;
const u64 last_word = (~u64{0} << shift) >> shift;
cpu.Pointer(IsShort())[NumWords() - 1] = last_word;
untracked.Pointer(IsShort())[NumWords() - 1] = last_word;
}
~Words() {
Release();
}
Words& operator=(Words&& rhs) noexcept {
Release();
size_bytes = rhs.size_bytes;
cpu = rhs.cpu;
gpu = rhs.gpu;
cached_cpu = rhs.cached_cpu;
untracked = rhs.untracked;
rhs.cpu.heap = nullptr;
return *this;
}
Words(Words&& rhs) noexcept
: size_bytes{rhs.size_bytes}, cpu{rhs.cpu}, gpu{rhs.gpu},
cached_cpu{rhs.cached_cpu}, untracked{rhs.untracked} {
rhs.cpu.heap = nullptr;
}
Words& operator=(const Words&) = delete;
Words(const Words&) = delete;
/// Returns true when the buffer fits in the small vector optimization
[[nodiscard]] bool IsShort() const noexcept {
return size_bytes <= BYTES_PER_WORD;
}
/// Returns the number of words of the buffer
[[nodiscard]] size_t NumWords() const noexcept {
return Common::DivCeil(size_bytes, BYTES_PER_WORD);
}
/// Release buffer resources
void Release() {
if (!IsShort()) {
// CPU written words is the base for the heap allocation
delete[] cpu.heap;
}
}
u64 size_bytes = 0;
WordsArray cpu;
WordsArray gpu;
WordsArray cached_cpu;
WordsArray untracked;
};
enum class Type {
CPU,
GPU,
CachedCPU,
Untracked,
};
public:
explicit BufferBase(RasterizerInterface& rasterizer_, VAddr cpu_addr_, u64 size_bytes)
: rasterizer{&rasterizer_}, cpu_addr{Common::AlignDown(cpu_addr_, BYTES_PER_PAGE)},
words(Common::AlignUp(size_bytes + (cpu_addr_ - cpu_addr), BYTES_PER_PAGE)) {}
static constexpr u64 BASE_PAGE_BITS = 16;
static constexpr u64 BASE_PAGE_SIZE = 1ULL << BASE_PAGE_BITS;
explicit BufferBase(RasterizerInterface& rasterizer_, VAddr cpu_addr_, u64 size_bytes_)
: cpu_addr{cpu_addr_}, size_bytes{size_bytes_} {}
explicit BufferBase(NullBufferParams) {}
@ -155,100 +49,6 @@ public:
BufferBase& operator=(BufferBase&&) = default;
BufferBase(BufferBase&&) = default;
/// Returns the inclusive CPU modified range in a begin end pair
[[nodiscard]] std::pair<u64, u64> ModifiedCpuRegion(VAddr query_cpu_addr,
u64 query_size) const noexcept {
const u64 offset = query_cpu_addr - cpu_addr;
return ModifiedRegion<Type::CPU>(offset, query_size);
}
/// Returns the inclusive GPU modified range in a begin end pair
[[nodiscard]] std::pair<u64, u64> ModifiedGpuRegion(VAddr query_cpu_addr,
u64 query_size) const noexcept {
const u64 offset = query_cpu_addr - cpu_addr;
return ModifiedRegion<Type::GPU>(offset, query_size);
}
/// Returns true if a region has been modified from the CPU
[[nodiscard]] bool IsRegionCpuModified(VAddr query_cpu_addr, u64 query_size) const noexcept {
const u64 offset = query_cpu_addr - cpu_addr;
return IsRegionModified<Type::CPU>(offset, query_size);
}
/// Returns true if a region has been modified from the GPU
[[nodiscard]] bool IsRegionGpuModified(VAddr query_cpu_addr, u64 query_size) const noexcept {
const u64 offset = query_cpu_addr - cpu_addr;
return IsRegionModified<Type::GPU>(offset, query_size);
}
/// Mark region as CPU modified, notifying the rasterizer about this change
void MarkRegionAsCpuModified(VAddr dirty_cpu_addr, u64 size) {
ChangeRegionState<Type::CPU, true>(dirty_cpu_addr, size);
}
/// Unmark region as CPU modified, notifying the rasterizer about this change
void UnmarkRegionAsCpuModified(VAddr dirty_cpu_addr, u64 size) {
ChangeRegionState<Type::CPU, false>(dirty_cpu_addr, size);
}
/// Mark region as modified from the host GPU
void MarkRegionAsGpuModified(VAddr dirty_cpu_addr, u64 size) noexcept {
ChangeRegionState<Type::GPU, true>(dirty_cpu_addr, size);
}
/// Unmark region as modified from the host GPU
void UnmarkRegionAsGpuModified(VAddr dirty_cpu_addr, u64 size) noexcept {
ChangeRegionState<Type::GPU, false>(dirty_cpu_addr, size);
}
/// Mark region as modified from the CPU
/// but don't mark it as modified until FlusHCachedWrites is called.
void CachedCpuWrite(VAddr dirty_cpu_addr, u64 size) {
flags |= BufferFlagBits::CachedWrites;
ChangeRegionState<Type::CachedCPU, true>(dirty_cpu_addr, size);
}
/// Flushes cached CPU writes, and notify the rasterizer about the deltas
void FlushCachedWrites() noexcept {
flags &= ~BufferFlagBits::CachedWrites;
const u64 num_words = NumWords();
u64* const cached_words = Array<Type::CachedCPU>();
u64* const untracked_words = Array<Type::Untracked>();
u64* const cpu_words = Array<Type::CPU>();
for (u64 word_index = 0; word_index < num_words; ++word_index) {
const u64 cached_bits = cached_words[word_index];
NotifyRasterizer<false>(word_index, untracked_words[word_index], cached_bits);
untracked_words[word_index] |= cached_bits;
cpu_words[word_index] |= cached_bits;
if (!Settings::values.use_pessimistic_flushes) {
cached_words[word_index] = 0;
}
}
}
/// Call 'func' for each CPU modified range and unmark those pages as CPU modified
template <typename Func>
void ForEachUploadRange(VAddr query_cpu_range, u64 size, Func&& func) {
ForEachModifiedRange<Type::CPU>(query_cpu_range, size, true, func);
}
/// Call 'func' for each GPU modified range and unmark those pages as GPU modified
template <typename Func>
void ForEachDownloadRange(VAddr query_cpu_range, u64 size, bool clear, Func&& func) {
ForEachModifiedRange<Type::GPU>(query_cpu_range, size, clear, func);
}
template <typename Func>
void ForEachDownloadRangeAndClear(VAddr query_cpu_range, u64 size, Func&& func) {
ForEachModifiedRange<Type::GPU>(query_cpu_range, size, true, func);
}
/// Call 'func' for each GPU modified range and unmark those pages as GPU modified
template <typename Func>
void ForEachDownloadRange(Func&& func) {
ForEachModifiedRange<Type::GPU>(cpu_addr, SizeBytes(), true, func);
}
/// Mark buffer as picked
void Pick() noexcept {
flags |= BufferFlagBits::Picked;
@ -295,11 +95,6 @@ public:
return static_cast<u32>(other_cpu_addr - cpu_addr);
}
/// Returns the size in bytes of the buffer
[[nodiscard]] u64 SizeBytes() const noexcept {
return words.size_bytes;
}
size_t getLRUID() const noexcept {
return lru_id;
}
@ -308,305 +103,16 @@ public:
lru_id = lru_id_;
}
private:
template <Type type>
u64* Array() noexcept {
if constexpr (type == Type::CPU) {
return words.cpu.Pointer(IsShort());
} else if constexpr (type == Type::GPU) {
return words.gpu.Pointer(IsShort());
} else if constexpr (type == Type::CachedCPU) {
return words.cached_cpu.Pointer(IsShort());
} else if constexpr (type == Type::Untracked) {
return words.untracked.Pointer(IsShort());
}
}
template <Type type>
const u64* Array() const noexcept {
if constexpr (type == Type::CPU) {
return words.cpu.Pointer(IsShort());
} else if constexpr (type == Type::GPU) {
return words.gpu.Pointer(IsShort());
} else if constexpr (type == Type::CachedCPU) {
return words.cached_cpu.Pointer(IsShort());
} else if constexpr (type == Type::Untracked) {
return words.untracked.Pointer(IsShort());
}
}
/**
* Change the state of a range of pages
*
* @param dirty_addr Base address to mark or unmark as modified
* @param size Size in bytes to mark or unmark as modified
*/
template <Type type, bool enable>
void ChangeRegionState(u64 dirty_addr, s64 size) noexcept(type == Type::GPU) {
const s64 difference = dirty_addr - cpu_addr;
const u64 offset = std::max<s64>(difference, 0);
size += std::min<s64>(difference, 0);
if (offset >= SizeBytes() || size < 0) {
return;
}
u64* const untracked_words = Array<Type::Untracked>();
u64* const state_words = Array<type>();
const u64 offset_end = std::min(offset + size, SizeBytes());
const u64 begin_page_index = offset / BYTES_PER_PAGE;
const u64 begin_word_index = begin_page_index / PAGES_PER_WORD;
const u64 end_page_index = Common::DivCeil(offset_end, BYTES_PER_PAGE);
const u64 end_word_index = Common::DivCeil(end_page_index, PAGES_PER_WORD);
u64 page_index = begin_page_index % PAGES_PER_WORD;
u64 word_index = begin_word_index;
while (word_index < end_word_index) {
const u64 next_word_first_page = (word_index + 1) * PAGES_PER_WORD;
const u64 left_offset =
std::min(next_word_first_page - end_page_index, PAGES_PER_WORD) % PAGES_PER_WORD;
const u64 right_offset = page_index;
u64 bits = ~u64{0};
bits = (bits >> right_offset) << right_offset;
bits = (bits << left_offset) >> left_offset;
if constexpr (type == Type::CPU || type == Type::CachedCPU) {
NotifyRasterizer<!enable>(word_index, untracked_words[word_index], bits);
}
if constexpr (enable) {
state_words[word_index] |= bits;
if constexpr (type == Type::CPU || type == Type::CachedCPU) {
untracked_words[word_index] |= bits;
}
} else {
state_words[word_index] &= ~bits;
if constexpr (type == Type::CPU || type == Type::CachedCPU) {
untracked_words[word_index] &= ~bits;
}
}
page_index = 0;
++word_index;
}
}
/**
* Notify rasterizer about changes in the CPU tracking state of a word in the buffer
*
* @param word_index Index to the word to notify to the rasterizer
* @param current_bits Current state of the word
* @param new_bits New state of the word
*
* @tparam add_to_rasterizer True when the rasterizer should start tracking the new pages
*/
template <bool add_to_rasterizer>
void NotifyRasterizer(u64 word_index, u64 current_bits, u64 new_bits) const {
u64 changed_bits = (add_to_rasterizer ? current_bits : ~current_bits) & new_bits;
VAddr addr = cpu_addr + word_index * BYTES_PER_WORD;
while (changed_bits != 0) {
const int empty_bits = std::countr_zero(changed_bits);
addr += empty_bits * BYTES_PER_PAGE;
changed_bits >>= empty_bits;
const u32 continuous_bits = std::countr_one(changed_bits);
const u64 size = continuous_bits * BYTES_PER_PAGE;
const VAddr begin_addr = addr;
addr += size;
changed_bits = continuous_bits < PAGES_PER_WORD ? (changed_bits >> continuous_bits) : 0;
rasterizer->UpdatePagesCachedCount(begin_addr, size, add_to_rasterizer ? 1 : -1);
}
}
/**
* Loop over each page in the given range, turn off those bits and notify the rasterizer if
* needed. Call the given function on each turned off range.
*
* @param query_cpu_range Base CPU address to loop over
* @param size Size in bytes of the CPU range to loop over
* @param func Function to call for each turned off region
*/
template <Type type, typename Func>
void ForEachModifiedRange(VAddr query_cpu_range, s64 size, bool clear, Func&& func) {
static_assert(type != Type::Untracked);
const s64 difference = query_cpu_range - cpu_addr;
const u64 query_begin = std::max<s64>(difference, 0);
size += std::min<s64>(difference, 0);
if (query_begin >= SizeBytes() || size < 0) {
return;
}
u64* const untracked_words = Array<Type::Untracked>();
u64* const state_words = Array<type>();
const u64 query_end = query_begin + std::min(static_cast<u64>(size), SizeBytes());
u64* const words_begin = state_words + query_begin / BYTES_PER_WORD;
u64* const words_end = state_words + Common::DivCeil(query_end, BYTES_PER_WORD);
const auto modified = [](u64 word) { return word != 0; };
const auto first_modified_word = std::find_if(words_begin, words_end, modified);
if (first_modified_word == words_end) {
// Exit early when the buffer is not modified
return;
}
const auto last_modified_word = std::find_if_not(first_modified_word, words_end, modified);
const u64 word_index_begin = std::distance(state_words, first_modified_word);
const u64 word_index_end = std::distance(state_words, last_modified_word);
const unsigned local_page_begin = std::countr_zero(*first_modified_word);
const unsigned local_page_end =
static_cast<unsigned>(PAGES_PER_WORD) - std::countl_zero(last_modified_word[-1]);
const u64 word_page_begin = word_index_begin * PAGES_PER_WORD;
const u64 word_page_end = (word_index_end - 1) * PAGES_PER_WORD;
const u64 query_page_begin = query_begin / BYTES_PER_PAGE;
const u64 query_page_end = Common::DivCeil(query_end, BYTES_PER_PAGE);
const u64 page_index_begin = std::max(word_page_begin + local_page_begin, query_page_begin);
const u64 page_index_end = std::min(word_page_end + local_page_end, query_page_end);
const u64 first_word_page_begin = page_index_begin % PAGES_PER_WORD;
const u64 last_word_page_end = (page_index_end - 1) % PAGES_PER_WORD + 1;
u64 page_begin = first_word_page_begin;
u64 current_base = 0;
u64 current_size = 0;
bool on_going = false;
for (u64 word_index = word_index_begin; word_index < word_index_end; ++word_index) {
const bool is_last_word = word_index + 1 == word_index_end;
const u64 page_end = is_last_word ? last_word_page_end : PAGES_PER_WORD;
const u64 right_offset = page_begin;
const u64 left_offset = PAGES_PER_WORD - page_end;
u64 bits = ~u64{0};
bits = (bits >> right_offset) << right_offset;
bits = (bits << left_offset) >> left_offset;
const u64 current_word = state_words[word_index] & bits;
if (clear) {
state_words[word_index] &= ~bits;
}
if constexpr (type == Type::CPU) {
const u64 current_bits = untracked_words[word_index] & bits;
untracked_words[word_index] &= ~bits;
NotifyRasterizer<true>(word_index, current_bits, ~u64{0});
}
// Exclude CPU modified pages when visiting GPU pages
const u64 word = current_word & ~(type == Type::GPU ? untracked_words[word_index] : 0);
u64 page = page_begin;
page_begin = 0;
while (page < page_end) {
const int empty_bits = std::countr_zero(word >> page);
if (on_going && empty_bits != 0) {
InvokeModifiedRange(func, current_size, current_base);
current_size = 0;
on_going = false;
}
if (empty_bits == PAGES_PER_WORD) {
break;
}
page += empty_bits;
const int continuous_bits = std::countr_one(word >> page);
if (!on_going && continuous_bits != 0) {
current_base = word_index * PAGES_PER_WORD + page;
on_going = true;
}
current_size += continuous_bits;
page += continuous_bits;
}
}
if (on_going && current_size > 0) {
InvokeModifiedRange(func, current_size, current_base);
}
}
template <typename Func>
void InvokeModifiedRange(Func&& func, u64 current_size, u64 current_base) {
const u64 current_size_bytes = current_size * BYTES_PER_PAGE;
const u64 offset_begin = current_base * BYTES_PER_PAGE;
const u64 offset_end = std::min(offset_begin + current_size_bytes, SizeBytes());
func(offset_begin, offset_end - offset_begin);
size_t SizeBytes() const {
return size_bytes;
}
/**
* Returns true when a region has been modified
*
* @param offset Offset in bytes from the start of the buffer
* @param size Size in bytes of the region to query for modifications
*/
template <Type type>
[[nodiscard]] bool IsRegionModified(u64 offset, u64 size) const noexcept {
static_assert(type != Type::Untracked);
const u64* const untracked_words = Array<Type::Untracked>();
const u64* const state_words = Array<type>();
const u64 num_query_words = size / BYTES_PER_WORD + 1;
const u64 word_begin = offset / BYTES_PER_WORD;
const u64 word_end = std::min<u64>(word_begin + num_query_words, NumWords());
const u64 page_limit = Common::DivCeil(offset + size, BYTES_PER_PAGE);
u64 page_index = (offset / BYTES_PER_PAGE) % PAGES_PER_WORD;
for (u64 word_index = word_begin; word_index < word_end; ++word_index, page_index = 0) {
const u64 off_word = type == Type::GPU ? untracked_words[word_index] : 0;
const u64 word = state_words[word_index] & ~off_word;
if (word == 0) {
continue;
}
const u64 page_end = std::min((word_index + 1) * PAGES_PER_WORD, page_limit);
const u64 local_page_end = page_end % PAGES_PER_WORD;
const u64 page_end_shift = (PAGES_PER_WORD - local_page_end) % PAGES_PER_WORD;
if (((word >> page_index) << page_index) << page_end_shift != 0) {
return true;
}
}
return false;
}
/**
* Returns a begin end pair with the inclusive modified region
*
* @param offset Offset in bytes from the start of the buffer
* @param size Size in bytes of the region to query for modifications
*/
template <Type type>
[[nodiscard]] std::pair<u64, u64> ModifiedRegion(u64 offset, u64 size) const noexcept {
static_assert(type != Type::Untracked);
const u64* const untracked_words = Array<Type::Untracked>();
const u64* const state_words = Array<type>();
const u64 num_query_words = size / BYTES_PER_WORD + 1;
const u64 word_begin = offset / BYTES_PER_WORD;
const u64 word_end = std::min<u64>(word_begin + num_query_words, NumWords());
const u64 page_base = offset / BYTES_PER_PAGE;
const u64 page_limit = Common::DivCeil(offset + size, BYTES_PER_PAGE);
u64 begin = std::numeric_limits<u64>::max();
u64 end = 0;
for (u64 word_index = word_begin; word_index < word_end; ++word_index) {
const u64 off_word = type == Type::GPU ? untracked_words[word_index] : 0;
const u64 word = state_words[word_index] & ~off_word;
if (word == 0) {
continue;
}
const u64 local_page_begin = std::countr_zero(word);
const u64 local_page_end = PAGES_PER_WORD - std::countl_zero(word);
const u64 page_index = word_index * PAGES_PER_WORD;
const u64 page_begin = std::max(page_index + local_page_begin, page_base);
const u64 page_end = std::min(page_index + local_page_end, page_limit);
begin = std::min(begin, page_begin);
end = std::max(end, page_end);
}
static constexpr std::pair<u64, u64> EMPTY{0, 0};
return begin < end ? std::make_pair(begin * BYTES_PER_PAGE, end * BYTES_PER_PAGE) : EMPTY;
}
/// Returns the number of words of the buffer
[[nodiscard]] size_t NumWords() const noexcept {
return words.NumWords();
}
/// Returns true when the buffer fits in the small vector optimization
[[nodiscard]] bool IsShort() const noexcept {
return words.IsShort();
}
RasterizerInterface* rasterizer = nullptr;
private:
VAddr cpu_addr = 0;
Words words;
BufferFlagBits flags{};
int stream_score = 0;
size_t lru_id = SIZE_MAX;
size_t size_bytes = 0;
};
} // namespace VideoCommon

4
src/video_core/buffer_cache/buffer_cache.cpp
View File

@ -1,5 +1,5 @@
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
#include "common/microprofile.h"

1002
src/video_core/buffer_cache/buffer_cache.h
File diff suppressed because it is too large
View File

580
src/video_core/buffer_cache/buffer_cache_base.h
View File

@ -0,0 +1,580 @@
// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
#pragma once
#include <algorithm>
#include <array>
#include <functional>
#include <memory>
#include <mutex>
#include <numeric>
#include <span>
#include <unordered_map>
#include <vector>
#include <boost/container/small_vector.hpp>
#define BOOST_NO_MT
#include <boost/pool/detail/mutex.hpp>
#undef BOOST_NO_MT
#include <boost/icl/interval.hpp>
#include <boost/icl/interval_base_set.hpp>
#include <boost/icl/interval_set.hpp>
#include <boost/icl/split_interval_map.hpp>
#include <boost/pool/pool.hpp>
#include <boost/pool/pool_alloc.hpp>
#include <boost/pool/poolfwd.hpp>
#include "common/common_types.h"
#include "common/div_ceil.h"
#include "common/literals.h"
#include "common/lru_cache.h"
#include "common/microprofile.h"
#include "common/scope_exit.h"
#include "common/settings.h"
#include "core/memory.h"
#include "video_core/buffer_cache/buffer_base.h"
#include "video_core/control/channel_state_cache.h"
#include "video_core/delayed_destruction_ring.h"
#include "video_core/dirty_flags.h"
#include "video_core/engines/draw_manager.h"
#include "video_core/engines/kepler_compute.h"
#include "video_core/engines/maxwell_3d.h"
#include "video_core/memory_manager.h"
#include "video_core/rasterizer_interface.h"
#include "video_core/surface.h"
#include "video_core/texture_cache/slot_vector.h"
#include "video_core/texture_cache/types.h"
namespace boost {
template <typename T>
class fast_pool_allocator<T, default_user_allocator_new_delete, details::pool::null_mutex, 4096, 0>;
}
namespace VideoCommon {
MICROPROFILE_DECLARE(GPU_PrepareBuffers);
MICROPROFILE_DECLARE(GPU_BindUploadBuffers);
MICROPROFILE_DECLARE(GPU_DownloadMemory);
using BufferId = SlotId;
using VideoCore::Surface::PixelFormat;
using namespace Common::Literals;
constexpr u32 NUM_VERTEX_BUFFERS = 32;
constexpr u32 NUM_TRANSFORM_FEEDBACK_BUFFERS = 4;
constexpr u32 NUM_GRAPHICS_UNIFORM_BUFFERS = 18;
constexpr u32 NUM_COMPUTE_UNIFORM_BUFFERS = 8;
constexpr u32 NUM_STORAGE_BUFFERS = 16;
constexpr u32 NUM_TEXTURE_BUFFERS = 16;
constexpr u32 NUM_STAGES = 5;
using UniformBufferSizes = std::array<std::array<u32, NUM_GRAPHICS_UNIFORM_BUFFERS>, NUM_STAGES>;
using ComputeUniformBufferSizes = std::array<u32, NUM_COMPUTE_UNIFORM_BUFFERS>;
enum class ObtainBufferSynchronize : u32 {
NoSynchronize = 0,
FullSynchronize = 1,
SynchronizeNoDirty = 2,
};
enum class ObtainBufferOperation : u32 {
DoNothing = 0,
MarkAsWritten = 1,
DiscardWrite = 2,
MarkQuery = 3,
};
template <typename P>
class BufferCache : public VideoCommon::ChannelSetupCaches<VideoCommon::ChannelInfo> {
// Page size for caching purposes.
// This is unrelated to the CPU page size and it can be changed as it seems optimal.
static constexpr u32 CACHING_PAGEBITS = 16;
static constexpr u64 CACHING_PAGESIZE = u64{1} << CACHING_PAGEBITS;
static constexpr bool IS_OPENGL = P::IS_OPENGL;
static constexpr bool HAS_PERSISTENT_UNIFORM_BUFFER_BINDINGS =
P::HAS_PERSISTENT_UNIFORM_BUFFER_BINDINGS;
static constexpr bool HAS_FULL_INDEX_AND_PRIMITIVE_SUPPORT =
P::HAS_FULL_INDEX_AND_PRIMITIVE_SUPPORT;
static constexpr bool NEEDS_BIND_UNIFORM_INDEX = P::NEEDS_BIND_UNIFORM_INDEX;
static constexpr bool NEEDS_BIND_STORAGE_INDEX = P::NEEDS_BIND_STORAGE_INDEX;
static constexpr bool USE_MEMORY_MAPS = P::USE_MEMORY_MAPS;
static constexpr bool SEPARATE_IMAGE_BUFFERS_BINDINGS = P::SEPARATE_IMAGE_BUFFER_BINDINGS;
static constexpr bool IMPLEMENTS_ASYNC_DOWNLOADS = P::IMPLEMENTS_ASYNC_DOWNLOADS;
static constexpr BufferId NULL_BUFFER_ID{0};
static constexpr s64 DEFAULT_EXPECTED_MEMORY = 512_MiB;
static constexpr s64 DEFAULT_CRITICAL_MEMORY = 1_GiB;
static constexpr s64 TARGET_THRESHOLD = 4_GiB;
// Debug Flags.
static constexpr bool DISABLE_DOWNLOADS = true;
using Maxwell = Tegra::Engines::Maxwell3D::Regs;
using Runtime = typename P::Runtime;
using Buffer = typename P::Buffer;
using Async_Buffer = typename P::Async_Buffer;
using MemoryTracker = typename P::MemoryTracker;
using IntervalCompare = std::less<VAddr>;
using IntervalInstance = boost::icl::interval_type_default<VAddr, std::less>;
using IntervalAllocator = boost::fast_pool_allocator<VAddr>;
using IntervalSet = boost::icl::interval_set<VAddr>;
using IntervalType = typename IntervalSet::interval_type;
template <typename Type>
struct counter_add_functor : public boost::icl::identity_based_inplace_combine<Type> {
// types
typedef counter_add_functor<Type> type;
typedef boost::icl::identity_based_inplace_combine<Type> base_type;
// public member functions
void operator()(Type& current, const Type& added) const {
current += added;
if (current < base_type::identity_element()) {
current = base_type::identity_element();
}
}
// public static functions
static void version(Type&){};
};
using OverlapCombine = counter_add_functor<int>;
using OverlapSection = boost::icl::inter_section<int>;
using OverlapCounter = boost::icl::split_interval_map<VAddr, int>;
struct Empty {};
struct OverlapResult {
std::vector<BufferId> ids;
VAddr begin;
VAddr end;
bool has_stream_leap = false;
};
struct Binding {
VAddr cpu_addr{};
u32 size{};
BufferId buffer_id;
};
struct TextureBufferBinding : Binding {
PixelFormat format;
};
static constexpr Binding NULL_BINDING{
.cpu_addr = 0,
.size = 0,
.buffer_id = NULL_BUFFER_ID,
};
public:
static constexpr u32 DEFAULT_SKIP_CACHE_SIZE = static_cast<u32>(4_KiB);
explicit BufferCache(VideoCore::RasterizerInterface& rasterizer_,
Core::Memory::Memory& cpu_memory_, Runtime& runtime_);
void TickFrame();
void WriteMemory(VAddr cpu_addr, u64 size);
void CachedWriteMemory(VAddr cpu_addr, u64 size);
void DownloadMemory(VAddr cpu_addr, u64 size);
bool InlineMemory(VAddr dest_address, size_t copy_size, std::span<const u8> inlined_buffer);
void BindGraphicsUniformBuffer(size_t stage, u32 index, GPUVAddr gpu_addr, u32 size);
void DisableGraphicsUniformBuffer(size_t stage, u32 index);
void UpdateGraphicsBuffers(bool is_indexed);
void UpdateComputeBuffers();
void BindHostGeometryBuffers(bool is_indexed);
void BindHostStageBuffers(size_t stage);
void BindHostComputeBuffers();
void SetUniformBuffersState(const std::array<u32, NUM_STAGES>& mask,
const UniformBufferSizes* sizes);
void SetComputeUniformBufferState(u32 mask, const ComputeUniformBufferSizes* sizes);
void UnbindGraphicsStorageBuffers(size_t stage);
void BindGraphicsStorageBuffer(size_t stage, size_t ssbo_index, u32 cbuf_index, u32 cbuf_offset,
bool is_written);
void UnbindGraphicsTextureBuffers(size_t stage);
void BindGraphicsTextureBuffer(size_t stage, size_t tbo_index, GPUVAddr gpu_addr, u32 size,
PixelFormat format, bool is_written, bool is_image);
void UnbindComputeStorageBuffers();
void BindComputeStorageBuffer(size_t ssbo_index, u32 cbuf_index, u32 cbuf_offset,
bool is_written);
void UnbindComputeTextureBuffers();
void BindComputeTextureBuffer(size_t tbo_index, GPUVAddr gpu_addr, u32 size, PixelFormat format,
bool is_written, bool is_image);
[[nodiscard]] std::pair<Buffer*, u32> ObtainBuffer(GPUVAddr gpu_addr, u32 size,
ObtainBufferSynchronize sync_info,
ObtainBufferOperation post_op);
void FlushCachedWrites();
/// Return true when there are uncommitted buffers to be downloaded
[[nodiscard]] bool HasUncommittedFlushes() const noexcept;
void AccumulateFlushes();
/// Return true when the caller should wait for async downloads
[[nodiscard]] bool ShouldWaitAsyncFlushes() const noexcept;
/// Commit asynchronous downloads
void CommitAsyncFlushes();
void CommitAsyncFlushesHigh();
/// Pop asynchronous downloads
void PopAsyncFlushes();
void PopAsyncBuffers();
bool DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 amount);
bool DMAClear(GPUVAddr src_address, u64 amount, u32 value);
/// Return true when a CPU region is modified from the GPU
[[nodiscard]] bool IsRegionGpuModified(VAddr addr, size_t size);
/// Return true when a region is registered on the cache
[[nodiscard]] bool IsRegionRegistered(VAddr addr, size_t size);
/// Return true when a CPU region is modified from the CPU
[[nodiscard]] bool IsRegionCpuModified(VAddr addr, size_t size);
void SetDrawIndirect(
const Tegra::Engines::DrawManager::IndirectParams* current_draw_indirect_) {
current_draw_indirect = current_draw_indirect_;
}
[[nodiscard]] std::pair<Buffer*, u32> GetDrawIndirectCount();
[[nodiscard]] std::pair<Buffer*, u32> GetDrawIndirectBuffer();
std::recursive_mutex mutex;
Runtime& runtime;
private:
template <typename Func>
static void ForEachEnabledBit(u32 enabled_mask, Func&& func) {
for (u32 index = 0; enabled_mask != 0; ++index, enabled_mask >>= 1) {
const int disabled_bits = std::countr_zero(enabled_mask);
index += disabled_bits;
enabled_mask >>= disabled_bits;
func(index);
}
}
template <typename Func>
void ForEachBufferInRange(VAddr cpu_addr, u64 size, Func&& func) {
const u64 page_end = Common::DivCeil(cpu_addr + size, CACHING_PAGESIZE);
for (u64 page = cpu_addr >> CACHING_PAGEBITS; page < page_end;) {
const BufferId buffer_id = page_table[page];
if (!buffer_id) {
++page;
continue;
}
Buffer& buffer = slot_buffers[buffer_id];
func(buffer_id, buffer);
const VAddr end_addr = buffer.CpuAddr() + buffer.SizeBytes();
page = Common::DivCeil(end_addr, CACHING_PAGESIZE);
}
}
template <typename Func>
void ForEachInRangeSet(IntervalSet& current_range, VAddr cpu_addr, u64 size, Func&& func) {
const VAddr start_address = cpu_addr;
const VAddr end_address = start_address + size;
const IntervalType search_interval{start_address, end_address};
auto it = current_range.lower_bound(search_interval);
if (it == current_range.end()) {
return;
}
auto end_it = current_range.upper_bound(search_interval);
for (; it != end_it; it++) {
VAddr inter_addr_end = it->upper();
VAddr inter_addr = it->lower();
if (inter_addr_end > end_address) {
inter_addr_end = end_address;
}
if (inter_addr < start_address) {
inter_addr = start_address;
}
func(inter_addr, inter_addr_end);
}
}
template <typename Func>
void ForEachInOverlapCounter(OverlapCounter& current_range, VAddr cpu_addr, u64 size,
Func&& func) {
const VAddr start_address = cpu_addr;
const VAddr end_address = start_address + size;
const IntervalType search_interval{start_address, end_address};
auto it = current_range.lower_bound(search_interval);
if (it == current_range.end()) {
return;
}
auto end_it = current_range.upper_bound(search_interval);
for (; it != end_it; it++) {
auto& inter = it->first;
VAddr inter_addr_end = inter.upper();
VAddr inter_addr = inter.lower();
if (inter_addr_end > end_address) {
inter_addr_end = end_address;
}
if (inter_addr < start_address) {
inter_addr = start_address;
}
func(inter_addr, inter_addr_end, it->second);
}
}
void RemoveEachInOverlapCounter(OverlapCounter& current_range,
const IntervalType search_interval, int subtract_value) {
bool any_removals = false;
current_range.add(std::make_pair(search_interval, subtract_value));
do {
any_removals = false;
auto it = current_range.lower_bound(search_interval);
if (it == current_range.end()) {
return;
}
auto end_it = current_range.upper_bound(search_interval);
for (; it != end_it; it++) {
if (it->second <= 0) {
any_removals = true;
current_range.erase(it);
break;
}
}
} while (any_removals);
}
static bool IsRangeGranular(VAddr cpu_addr, size_t size) {
return (cpu_addr & ~Core::Memory::YUZU_PAGEMASK) ==
((cpu_addr + size) & ~Core::Memory::YUZU_PAGEMASK);
}
void RunGarbageCollector();
void WaitOnAsyncFlushes(VAddr cpu_addr, u64 size);
void BindHostIndexBuffer();
void BindHostVertexBuffers();
void BindHostDrawIndirectBuffers();
void BindHostGraphicsUniformBuffers(size_t stage);
void BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 binding_index, bool needs_bind);
void BindHostGraphicsStorageBuffers(size_t stage);
void BindHostGraphicsTextureBuffers(size_t stage);
void BindHostTransformFeedbackBuffers();
void BindHostComputeUniformBuffers();
void BindHostComputeStorageBuffers();
void BindHostComputeTextureBuffers();
void DoUpdateGraphicsBuffers(bool is_indexed);
void DoUpdateComputeBuffers();
void UpdateIndexBuffer();
void UpdateVertexBuffers();
void UpdateVertexBuffer(u32 index);
void UpdateDrawIndirect();
void UpdateUniformBuffers(size_t stage);
void UpdateStorageBuffers(size_t stage);
void UpdateTextureBuffers(size_t stage);
void UpdateTransformFeedbackBuffers();
void UpdateTransformFeedbackBuffer(u32 index);
void UpdateComputeUniformBuffers();
void UpdateComputeStorageBuffers();
void UpdateComputeTextureBuffers();
void MarkWrittenBuffer(BufferId buffer_id, VAddr cpu_addr, u32 size);
[[nodiscard]] BufferId FindBuffer(VAddr cpu_addr, u32 size);
[[nodiscard]] OverlapResult ResolveOverlaps(VAddr cpu_addr, u32 wanted_size);
void JoinOverlap(BufferId new_buffer_id, BufferId overlap_id, bool accumulate_stream_score);
[[nodiscard]] BufferId CreateBuffer(VAddr cpu_addr, u32 wanted_size);
void Register(BufferId buffer_id);
void Unregister(BufferId buffer_id);
template <bool insert>
void ChangeRegister(BufferId buffer_id);
void TouchBuffer(Buffer& buffer, BufferId buffer_id) noexcept;
bool SynchronizeBuffer(Buffer& buffer, VAddr cpu_addr, u32 size);
bool SynchronizeBufferImpl(Buffer& buffer, VAddr cpu_addr, u32 size);
bool SynchronizeBufferNoModified(Buffer& buffer, VAddr cpu_addr, u32 size);
void UploadMemory(Buffer& buffer, u64 total_size_bytes, u64 largest_copy,
std::span<BufferCopy> copies);
void ImmediateUploadMemory(Buffer& buffer, u64 largest_copy,
std::span<const BufferCopy> copies);
void MappedUploadMemory(Buffer& buffer, u64 total_size_bytes, std::span<BufferCopy> copies);
void DownloadBufferMemory(Buffer& buffer_id);
void DownloadBufferMemory(Buffer& buffer_id, VAddr cpu_addr, u64 size);
void DeleteBuffer(BufferId buffer_id, bool do_not_mark = false);
void NotifyBufferDeletion();
[[nodiscard]] Binding StorageBufferBinding(GPUVAddr ssbo_addr, u32 cbuf_index,
bool is_written) const;
[[nodiscard]] TextureBufferBinding GetTextureBufferBinding(GPUVAddr gpu_addr, u32 size,
PixelFormat format);
[[nodiscard]] std::span<const u8> ImmediateBufferWithData(VAddr cpu_addr, size_t size);
[[nodiscard]] std::span<u8> ImmediateBuffer(size_t wanted_capacity);
[[nodiscard]] bool HasFastUniformBufferBound(size_t stage, u32 binding_index) const noexcept;
void ClearDownload(IntervalType subtract_interval);
VideoCore::RasterizerInterface& rasterizer;
Core::Memory::Memory& cpu_memory;
SlotVector<Buffer> slot_buffers;
DelayedDestructionRing<Buffer, 8> delayed_destruction_ring;
const Tegra::Engines::DrawManager::IndirectParams* current_draw_indirect{};
u32 last_index_count = 0;
Binding index_buffer;
std::array<Binding, NUM_VERTEX_BUFFERS> vertex_buffers;
std::array<std::array<Binding, NUM_GRAPHICS_UNIFORM_BUFFERS>, NUM_STAGES> uniform_buffers;
std::array<std::array<Binding, NUM_STORAGE_BUFFERS>, NUM_STAGES> storage_buffers;
std::array<std::array<TextureBufferBinding, NUM_TEXTURE_BUFFERS>, NUM_STAGES> texture_buffers;
std::array<Binding, NUM_TRANSFORM_FEEDBACK_BUFFERS> transform_feedback_buffers;
Binding count_buffer_binding;
Binding indirect_buffer_binding;
std::array<Binding, NUM_COMPUTE_UNIFORM_BUFFERS> compute_uniform_buffers;
std::array<Binding, NUM_STORAGE_BUFFERS> compute_storage_buffers;
std::array<TextureBufferBinding, NUM_TEXTURE_BUFFERS> compute_texture_buffers;
std::array<u32, NUM_STAGES> enabled_uniform_buffer_masks{};
u32 enabled_compute_uniform_buffer_mask = 0;
const UniformBufferSizes* uniform_buffer_sizes{};
const ComputeUniformBufferSizes* compute_uniform_buffer_sizes{};
std::array<u32, NUM_STAGES> enabled_storage_buffers{};
std::array<u32, NUM_STAGES> written_storage_buffers{};
u32 enabled_compute_storage_buffers = 0;
u32 written_compute_storage_buffers = 0;
std::array<u32, NUM_STAGES> enabled_texture_buffers{};
std::array<u32, NUM_STAGES> written_texture_buffers{};
std::array<u32, NUM_STAGES> image_texture_buffers{};
u32 enabled_compute_texture_buffers = 0;
u32 written_compute_texture_buffers = 0;
u32 image_compute_texture_buffers = 0;
std::array<u32, 16> uniform_cache_hits{};
std::array<u32, 16> uniform_cache_shots{};
u32 uniform_buffer_skip_cache_size = DEFAULT_SKIP_CACHE_SIZE;
bool has_deleted_buffers = false;
std::conditional_t<HAS_PERSISTENT_UNIFORM_BUFFER_BINDINGS, std::array<u32, NUM_STAGES>, Empty>
dirty_uniform_buffers{};
std::conditional_t<IS_OPENGL, std::array<u32, NUM_STAGES>, Empty> fast_bound_uniform_buffers{};
std::conditional_t<HAS_PERSISTENT_UNIFORM_BUFFER_BINDINGS,
std::array<std::array<u32, NUM_GRAPHICS_UNIFORM_BUFFERS>, NUM_STAGES>, Empty>
uniform_buffer_binding_sizes{};
std::vector<BufferId> cached_write_buffer_ids;
MemoryTracker memory_tracker;
IntervalSet uncommitted_ranges;
IntervalSet common_ranges;
IntervalSet cached_ranges;
IntervalSet pending_ranges;
std::deque<IntervalSet> committed_ranges;
// Async Buffers
OverlapCounter async_downloads;
std::deque<std::optional<Async_Buffer>> async_buffers;
std::deque<boost::container::small_vector<BufferCopy, 4>> pending_downloads;
std::optional<Async_Buffer> current_buffer;
std::deque<Async_Buffer> async_buffers_death_ring;
size_t immediate_buffer_capacity = 0;
Common::ScratchBuffer<u8> immediate_buffer_alloc;
struct LRUItemParams {
using ObjectType = BufferId;
using TickType = u64;
};
Common::LeastRecentlyUsedCache<LRUItemParams> lru_cache;
u64 frame_tick = 0;
u64 total_used_memory = 0;
u64 minimum_memory = 0;
u64 critical_memory = 0;
BufferId inline_buffer_id;
bool active_async_buffers = false;
std::array<BufferId, ((1ULL << 39) >> CACHING_PAGEBITS)> page_table;
};
} // namespace VideoCommon

271
src/video_core/buffer_cache/memory_tracker_base.h
View File

@ -0,0 +1,271 @@
// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
#pragma once
#include <algorithm>
#include <bit>
#include <deque>
#include <limits>
#include <type_traits>
#include <unordered_set>
#include <utility>
#include "common/alignment.h"
#include "common/common_types.h"
#include "video_core/buffer_cache/word_manager.h"
namespace VideoCommon {
template <class RasterizerInterface>
class MemoryTrackerBase {
static constexpr size_t MAX_CPU_PAGE_BITS = 39;
static constexpr size_t HIGHER_PAGE_BITS = 22;
static constexpr size_t HIGHER_PAGE_SIZE = 1ULL << HIGHER_PAGE_BITS;
static constexpr size_t HIGHER_PAGE_MASK = HIGHER_PAGE_SIZE - 1ULL;
static constexpr size_t NUM_HIGH_PAGES = 1ULL << (MAX_CPU_PAGE_BITS - HIGHER_PAGE_BITS);
static constexpr size_t MANAGER_POOL_SIZE = 32;
static constexpr size_t WORDS_STACK_NEEDED = HIGHER_PAGE_SIZE / BYTES_PER_WORD;
using Manager = WordManager<RasterizerInterface, WORDS_STACK_NEEDED>;
public:
MemoryTrackerBase(RasterizerInterface& rasterizer_) : rasterizer{&rasterizer_} {}
~MemoryTrackerBase() = default;
/// Returns the inclusive CPU modified range in a begin end pair
[[nodiscard]] std::pair<u64, u64> ModifiedCpuRegion(VAddr query_cpu_addr,
u64 query_size) noexcept {
return IteratePairs<true>(
query_cpu_addr, query_size, [](Manager* manager, u64 offset, size_t size) {
return manager->template ModifiedRegion<Type::CPU>(offset, size);
});
}
/// Returns the inclusive GPU modified range in a begin end pair
[[nodiscard]] std::pair<u64, u64> ModifiedGpuRegion(VAddr query_cpu_addr,
u64 query_size) noexcept {
return IteratePairs<false>(
query_cpu_addr, query_size, [](Manager* manager, u64 offset, size_t size) {
return manager->template ModifiedRegion<Type::GPU>(offset, size);
});
}
/// Returns true if a region has been modified from the CPU
[[nodiscard]] bool IsRegionCpuModified(VAddr query_cpu_addr, u64 query_size) noexcept {
return IteratePages<true>(
query_cpu_addr, query_size, [](Manager* manager, u64 offset, size_t size) {
return manager->template IsRegionModified<Type::CPU>(offset, size);
});
}
/// Returns true if a region has been modified from the GPU
[[nodiscard]] bool IsRegionGpuModified(VAddr query_cpu_addr, u64 query_size) noexcept {
return IteratePages<false>(
query_cpu_addr, query_size, [](Manager* manager, u64 offset, size_t size) {
return manager->template IsRegionModified<Type::GPU>(offset, size);
});
}
/// Mark region as CPU modified, notifying the rasterizer about this change
void MarkRegionAsCpuModified(VAddr dirty_cpu_addr, u64 query_size) {
IteratePages<true>(dirty_cpu_addr, query_size,
[](Manager* manager, u64 offset, size_t size) {
manager->template ChangeRegionState<Type::CPU, true>(
manager->GetCpuAddr() + offset, size);
});
}
/// Unmark region as CPU modified, notifying the rasterizer about this change
void UnmarkRegionAsCpuModified(VAddr dirty_cpu_addr, u64 query_size) {
IteratePages<true>(dirty_cpu_addr, query_size,
[](Manager* manager, u64 offset, size_t size) {
manager->template ChangeRegionState<Type::CPU, false>(
manager->GetCpuAddr() + offset, size);
});
}
/// Mark region as modified from the host GPU
void MarkRegionAsGpuModified(VAddr dirty_cpu_addr, u64 query_size) noexcept {
IteratePages<true>(dirty_cpu_addr, query_size,
[](Manager* manager, u64 offset, size_t size) {
manager->template ChangeRegionState<Type::GPU, true>(
manager->GetCpuAddr() + offset, size);
});
}
/// Unmark region as modified from the host GPU
void UnmarkRegionAsGpuModified(VAddr dirty_cpu_addr, u64 query_size) noexcept {
IteratePages<true>(dirty_cpu_addr, query_size,
[](Manager* manager, u64 offset, size_t size) {
manager->template ChangeRegionState<Type::GPU, false>(
manager->GetCpuAddr() + offset, size);
});
}
/// Mark region as modified from the CPU
/// but don't mark it as modified until FlusHCachedWrites is called.
void CachedCpuWrite(VAddr dirty_cpu_addr, u64 query_size) {
IteratePages<true>(
dirty_cpu_addr, query_size, [this](Manager* manager, u64 offset, size_t size) {
const VAddr cpu_address = manager->GetCpuAddr() + offset;
manager->template ChangeRegionState<Type::CachedCPU, true>(cpu_address, size);
cached_pages.insert(static_cast<u32>(cpu_address >> HIGHER_PAGE_BITS));
});
}
/// Flushes cached CPU writes, and notify the rasterizer about the deltas
void FlushCachedWrites(VAddr query_cpu_addr, u64 query_size) noexcept {
IteratePages<false>(query_cpu_addr, query_size,
[](Manager* manager, [[maybe_unused]] u64 offset,
[[maybe_unused]] size_t size) { manager->FlushCachedWrites(); });
}
void FlushCachedWrites() noexcept {
for (auto id : cached_pages) {
top_tier[id]->FlushCachedWrites();
}
cached_pages.clear();
}
/// Call 'func' for each CPU modified range and unmark those pages as CPU modified
template <typename Func>
void ForEachUploadRange(VAddr query_cpu_range, u64 query_size, Func&& func) {
IteratePages<true>(query_cpu_range, query_size,
[&func](Manager* manager, u64 offset, size_t size) {
manager->template ForEachModifiedRange<Type::CPU, true>(
manager->GetCpuAddr() + offset, size, func);
});
}
/// Call 'func' for each GPU modified range and unmark those pages as GPU modified
template <typename Func>
void ForEachDownloadRange(VAddr query_cpu_range, u64 query_size, bool clear, Func&& func) {
IteratePages<false>(query_cpu_range, query_size,
[&func, clear](Manager* manager, u64 offset, size_t size) {
if (clear) {
manager->template ForEachModifiedRange<Type::GPU, true>(
manager->GetCpuAddr() + offset, size, func);
} else {
manager->template ForEachModifiedRange<Type::GPU, false>(
manager->GetCpuAddr() + offset, size, func);
}
});
}
template <typename Func>
void ForEachDownloadRangeAndClear(VAddr query_cpu_range, u64 query_size, Func&& func) {
IteratePages<false>(query_cpu_range, query_size,
[&func](Manager* manager, u64 offset, size_t size) {
manager->template ForEachModifiedRange<Type::GPU, true>(
manager->GetCpuAddr() + offset, size, func);
});
}
private:
template <bool create_region_on_fail, typename Func>
bool IteratePages(VAddr cpu_address, size_t size, Func&& func) {
using FuncReturn = typename std::invoke_result<Func, Manager*, u64, size_t>::type;
static constexpr bool BOOL_BREAK = std::is_same_v<FuncReturn, bool>;
std::size_t remaining_size{size};
std::size_t page_index{cpu_address >> HIGHER_PAGE_BITS};
u64 page_offset{cpu_address & HIGHER_PAGE_MASK};
while (remaining_size > 0) {
const std::size_t copy_amount{std::min(HIGHER_PAGE_SIZE - page_offset, remaining_size)};
auto* manager{top_tier[page_index]};
if (manager) {
if constexpr (BOOL_BREAK) {
if (func(manager, page_offset, copy_amount)) {
return true;
}
} else {
func(manager, page_offset, copy_amount);
}
} else if constexpr (create_region_on_fail) {
CreateRegion(page_index);
manager = top_tier[page_index];
if constexpr (BOOL_BREAK) {
if (func(manager, page_offset, copy_amount)) {
return true;
}
} else {
func(manager, page_offset, copy_amount);
}
}
page_index++;
page_offset = 0;
remaining_size -= copy_amount;
}
return false;
}
template <bool create_region_on_fail, typename Func>
std::pair<u64, u64> IteratePairs(VAddr cpu_address, size_t size, Func&& func) {
std::size_t remaining_size{size};
std::size_t page_index{cpu_address >> HIGHER_PAGE_BITS};
u64 page_offset{cpu_address & HIGHER_PAGE_MASK};
u64 begin = std::numeric_limits<u64>::max();
u64 end = 0;
while (remaining_size > 0) {
const std::size_t copy_amount{std::min(HIGHER_PAGE_SIZE - page_offset, remaining_size)};
auto* manager{top_tier[page_index]};
const auto execute = [&] {
auto [new_begin, new_end] = func(manager, page_offset, copy_amount);
if (new_begin != 0 || new_end != 0) {
const u64 base_address = page_index << HIGHER_PAGE_BITS;
begin = std::min(new_begin + base_address, begin);
end = std::max(new_end + base_address, end);
}
};
if (manager) {
execute();
} else if constexpr (create_region_on_fail) {
CreateRegion(page_index);
manager = top_tier[page_index];
execute();
}
page_index++;
page_offset = 0;
remaining_size -= copy_amount;
}
if (begin < end) {
return std::make_pair(begin, end);
} else {
return std::make_pair(0ULL, 0ULL);
}
}
void CreateRegion(std::size_t page_index) {
const VAddr base_cpu_addr = page_index << HIGHER_PAGE_BITS;
top_tier[page_index] = GetNewManager(base_cpu_addr);
}
Manager* GetNewManager(VAddr base_cpu_addess) {
const auto on_return = [&] {
auto* new_manager = free_managers.front();
new_manager->SetCpuAddress(base_cpu_addess);
free_managers.pop_front();
return new_manager;
};
if (!free_managers.empty()) {
return on_return();
}
manager_pool.emplace_back();
auto& last_pool = manager_pool.back();
for (size_t i = 0; i < MANAGER_POOL_SIZE; i++) {
new (&last_pool[i]) Manager(0, *rasterizer, HIGHER_PAGE_SIZE);
free_managers.push_back(&last_pool[i]);
}
return on_return();
}
std::deque<std::array<Manager, MANAGER_POOL_SIZE>> manager_pool;
std::deque<Manager*> free_managers;
std::array<Manager*, NUM_HIGH_PAGES> top_tier{};
std::unordered_set<u32> cached_pages;
RasterizerInterface* rasterizer = nullptr;
};
} // namespace VideoCommon

462
src/video_core/buffer_cache/word_manager.h
View File

@ -0,0 +1,462 @@
// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
#pragma once
#include <algorithm>
#include <bit>
#include <limits>
#include <span>
#include <utility>
#include "common/alignment.h"
#include "common/common_funcs.h"
#include "common/common_types.h"
#include "common/div_ceil.h"
#include "core/memory.h"
namespace VideoCommon {
constexpr u64 PAGES_PER_WORD = 64;
constexpr u64 BYTES_PER_PAGE = Core::Memory::YUZU_PAGESIZE;
constexpr u64 BYTES_PER_WORD = PAGES_PER_WORD * BYTES_PER_PAGE;
enum class Type {
CPU,
GPU,
CachedCPU,
Untracked,
};
/// Vector tracking modified pages tightly packed with small vector optimization
template <size_t stack_words = 1>
struct WordsArray {
/// Returns the pointer to the words state
[[nodiscard]] const u64* Pointer(bool is_short) const noexcept {
return is_short ? stack.data() : heap;
}
/// Returns the pointer to the words state
[[nodiscard]] u64* Pointer(bool is_short) noexcept {
return is_short ? stack.data() : heap;
}
std::array<u64, stack_words> stack{}; ///< Small buffers storage
u64* heap; ///< Not-small buffers pointer to the storage
};
template <size_t stack_words = 1>
struct Words {
explicit Words() = default;
explicit Words(u64 size_bytes_) : size_bytes{size_bytes_} {
num_words = Common::DivCeil(size_bytes, BYTES_PER_WORD);
if (IsShort()) {
cpu.stack.fill(~u64{0});
gpu.stack.fill(0);
cached_cpu.stack.fill(0);
untracked.stack.fill(~u64{0});
} else {
// Share allocation between CPU and GPU pages and set their default values
u64* const alloc = new u64[num_words * 4];
cpu.heap = alloc;
gpu.heap = alloc + num_words;
cached_cpu.heap = alloc + num_words * 2;
untracked.heap = alloc + num_words * 3;
std::fill_n(cpu.heap, num_words, ~u64{0});
std::fill_n(gpu.heap, num_words, 0);
std::fill_n(cached_cpu.heap, num_words, 0);
std::fill_n(untracked.heap, num_words, ~u64{0});
}
// Clean up tailing bits
const u64 last_word_size = size_bytes % BYTES_PER_WORD;
const u64 last_local_page = Common::DivCeil(last_word_size, BYTES_PER_PAGE);
const u64 shift = (PAGES_PER_WORD - last_local_page) % PAGES_PER_WORD;
const u64 last_word = (~u64{0} << shift) >> shift;
cpu.Pointer(IsShort())[NumWords() - 1] = last_word;
untracked.Pointer(IsShort())[NumWords() - 1] = last_word;
}
~Words() {
Release();
}
Words& operator=(Words&& rhs) noexcept {
Release();
size_bytes = rhs.size_bytes;
num_words = rhs.num_words;
cpu = rhs.cpu;
gpu = rhs.gpu;
cached_cpu = rhs.cached_cpu;
untracked = rhs.untracked;
rhs.cpu.heap = nullptr;
return *this;
}
Words(Words&& rhs) noexcept
: size_bytes{rhs.size_bytes}, num_words{rhs.num_words}, cpu{rhs.cpu}, gpu{rhs.gpu},
cached_cpu{rhs.cached_cpu}, untracked{rhs.untracked} {
rhs.cpu.heap = nullptr;
}
Words& operator=(const Words&) = delete;
Words(const Words&) = delete;
/// Returns true when the buffer fits in the small vector optimization
[[nodiscard]] bool IsShort() const noexcept {
return num_words <= stack_words;
}
/// Returns the number of words of the buffer
[[nodiscard]] size_t NumWords() const noexcept {
return num_words;
}
/// Release buffer resources
void Release() {
if (!IsShort()) {
// CPU written words is the base for the heap allocation
delete[] cpu.heap;
}
}
template <Type type>
std::span<u64> Span() noexcept {
if constexpr (type == Type::CPU) {
return std::span<u64>(cpu.Pointer(IsShort()), num_words);
} else if constexpr (type == Type::GPU) {
return std::span<u64>(gpu.Pointer(IsShort()), num_words);
} else if constexpr (type == Type::CachedCPU) {
return std::span<u64>(cached_cpu.Pointer(IsShort()), num_words);
} else if constexpr (type == Type::Untracked) {
return std::span<u64>(untracked.Pointer(IsShort()), num_words);
}
}
template <Type type>
std::span<const u64> Span() const noexcept {
if constexpr (type == Type::CPU) {
return std::span<const u64>(cpu.Pointer(IsShort()), num_words);
} else if constexpr (type == Type::GPU) {
return std::span<const u64>(gpu.Pointer(IsShort()), num_words);
} else if constexpr (type == Type::CachedCPU) {
return std::span<const u64>(cached_cpu.Pointer(IsShort()), num_words);
} else if constexpr (type == Type::Untracked) {
return std::span<const u64>(untracked.Pointer(IsShort()), num_words);
}
}
u64 size_bytes = 0;
size_t num_words = 0;
WordsArray<stack_words> cpu;
WordsArray<stack_words> gpu;
WordsArray<stack_words> cached_cpu;
WordsArray<stack_words> untracked;
};
template <class RasterizerInterface, size_t stack_words = 1>
class WordManager {
public:
explicit WordManager(VAddr cpu_addr_, RasterizerInterface& rasterizer_, u64 size_bytes)
: cpu_addr{cpu_addr_}, rasterizer{&rasterizer_}, words{size_bytes} {}
explicit WordManager() = default;
void SetCpuAddress(VAddr new_cpu_addr) {
cpu_addr = new_cpu_addr;
}
VAddr GetCpuAddr() const {
return cpu_addr;
}
static u64 ExtractBits(u64 word, size_t page_start, size_t page_end) {
constexpr size_t number_bits = sizeof(u64) * 8;
const size_t limit_page_end = number_bits - std::min(page_end, number_bits);
u64 bits = (word >> page_start) << page_start;
bits = (bits << limit_page_end) >> limit_page_end;
return bits;
}
static std::pair<size_t, size_t> GetWordPage(VAddr address) {
const size_t converted_address = static_cast<size_t>(address);
const size_t word_number = converted_address / BYTES_PER_WORD;
const size_t amount_pages = converted_address % BYTES_PER_WORD;
return std::make_pair(word_number, amount_pages / BYTES_PER_PAGE);
}
template <typename Func>
void IterateWords(size_t offset, size_t size, Func&& func) const {
using FuncReturn = std::invoke_result_t<Func, std::size_t, u64>;
static constexpr bool BOOL_BREAK = std::is_same_v<FuncReturn, bool>;
const size_t start = static_cast<size_t>(std::max<s64>(static_cast<s64>(offset), 0LL));
const size_t end = static_cast<size_t>(std::max<s64>(static_cast<s64>(offset + size), 0LL));
if (start >= SizeBytes() || end <= start) {
return;
}
auto [start_word, start_page] = GetWordPage(start);
auto [end_word, end_page] = GetWordPage(end + BYTES_PER_PAGE - 1ULL);
const size_t num_words = NumWords();
start_word = std::min(start_word, num_words);
end_word = std::min(end_word, num_words);
const size_t diff = end_word - start_word;
end_word += (end_page + PAGES_PER_WORD - 1ULL) / PAGES_PER_WORD;
end_word = std::min(end_word, num_words);
end_page += diff * PAGES_PER_WORD;
constexpr u64 base_mask{~0ULL};
for (size_t word_index = start_word; word_index < end_word; word_index++) {
const u64 mask = ExtractBits(base_mask, start_page, end_page);
start_page = 0;
end_page -= PAGES_PER_WORD;
if constexpr (BOOL_BREAK) {
if (func(word_index, mask)) {
return;
}
} else {
func(word_index, mask);
}
}
}
template <typename Func>
void IteratePages(u64 mask, Func&& func) const {
size_t offset = 0;
while (mask != 0) {
const size_t empty_bits = std::countr_zero(mask);
offset += empty_bits;
mask = mask >> empty_bits;
const size_t continuous_bits = std::countr_one(mask);
func(offset, continuous_bits);
mask = continuous_bits < PAGES_PER_WORD ? (mask >> continuous_bits) : 0;
offset += continuous_bits;
}
}
/**
* Change the state of a range of pages
*
* @param dirty_addr Base address to mark or unmark as modified
* @param size Size in bytes to mark or unmark as modified
*/
template <Type type, bool enable>
void ChangeRegionState(u64 dirty_addr, u64 size) noexcept(type == Type::GPU) {
std::span<u64> state_words = words.template Span<type>();
[[maybe_unused]] std::span<u64> untracked_words = words.template Span<Type::Untracked>();
[[maybe_unused]] std::span<u64> cached_words = words.template Span<Type::CachedCPU>();
IterateWords(dirty_addr - cpu_addr, size, [&](size_t index, u64 mask) {
if constexpr (type == Type::CPU || type == Type::CachedCPU) {
NotifyRasterizer<!enable>(index, untracked_words[index], mask);
}
if constexpr (enable) {
state_words[index] |= mask;
if constexpr (type == Type::CPU || type == Type::CachedCPU) {
untracked_words[index] |= mask;
}
if constexpr (type == Type::CPU) {
cached_words[index] &= ~mask;
}
} else {
if constexpr (type == Type::CPU) {
const u64 word = state_words[index] & mask;
cached_words[index] &= ~word;
}
state_words[index] &= ~mask;
if constexpr (type == Type::CPU || type == Type::CachedCPU) {
untracked_words[index] &= ~mask;
}
}
});
}
/**
* Loop over each page in the given range, turn off those bits and notify the rasterizer if
* needed. Call the given function on each turned off range.
*
* @param query_cpu_range Base CPU address to loop over
* @param size Size in bytes of the CPU range to loop over
* @param func Function to call for each turned off region
*/
template <Type type, bool clear, typename Func>
void ForEachModifiedRange(VAddr query_cpu_range, s64 size, Func&& func) {
static_assert(type != Type::Untracked);
std::span<u64> state_words = words.template Span<type>();
[[maybe_unused]] std::span<u64> untracked_words = words.template Span<Type::Untracked>();
[[maybe_unused]] std::span<u64> cached_words = words.template Span<Type::CachedCPU>();
const size_t offset = query_cpu_range - cpu_addr;
bool pending = false;
size_t pending_offset{};
size_t pending_pointer{};
const auto release = [&]() {
func(cpu_addr + pending_offset * BYTES_PER_PAGE,
(pending_pointer - pending_offset) * BYTES_PER_PAGE);
};
IterateWords(offset, size, [&](size_t index, u64 mask) {
const u64 word = state_words[index] & mask;
if constexpr (clear) {
if constexpr (type == Type::CPU || type == Type::CachedCPU) {
NotifyRasterizer<true>(index, untracked_words[index], mask);
}
state_words[index] &= ~mask;
if constexpr (type == Type::CPU || type == Type::CachedCPU) {
untracked_words[index] &= ~mask;
}
if constexpr (type == Type::CPU) {
cached_words[index] &= ~word;
}
}
const size_t base_offset = index * PAGES_PER_WORD;
IteratePages(word, [&](size_t pages_offset, size_t pages_size) {
const auto reset = [&]() {
pending_offset = base_offset + pages_offset;
pending_pointer = base_offset + pages_offset + pages_size;
};
if (!pending) {
reset();
pending = true;
return;
}
if (pending_pointer == base_offset + pages_offset) {
pending_pointer += pages_size;
return;
}
release();
reset();
});
});
if (pending) {
release();
}
}
/**
* Returns true when a region has been modified
*
* @param offset Offset in bytes from the start of the buffer
* @param size Size in bytes of the region to query for modifications
*/
template <Type type>
[[nodiscard]] bool IsRegionModified(u64 offset, u64 size) const noexcept {
static_assert(type != Type::Untracked);
const std::span<const u64> state_words = words.template Span<type>();
bool result = false;
IterateWords(offset, size, [&](size_t index, u64 mask) {
const u64 word = state_words[index] & mask;
if (word != 0) {
result = true;
return true;
}
return false;
});
return result;
}
/**
* Returns a begin end pair with the inclusive modified region
*
* @param offset Offset in bytes from the start of the buffer
* @param size Size in bytes of the region to query for modifications
*/
template <Type type>
[[nodiscard]] std::pair<u64, u64> ModifiedRegion(u64 offset, u64 size) const noexcept {
static_assert(type != Type::Untracked);
const std::span<const u64> state_words = words.template Span<type>();
u64 begin = std::numeric_limits<u64>::max();
u64 end = 0;
IterateWords(offset, size, [&](size_t index, u64 mask) {
const u64 word = state_words[index] & mask;
if (word == 0) {
return;
}
const u64 local_page_begin = std::countr_zero(word);
const u64 local_page_end = PAGES_PER_WORD - std::countl_zero(word);
const u64 page_index = index * PAGES_PER_WORD;
begin = std::min(begin, page_index + local_page_begin);
end = page_index + local_page_end;
});
static constexpr std::pair<u64, u64> EMPTY{0, 0};
return begin < end ? std::make_pair(begin * BYTES_PER_PAGE, end * BYTES_PER_PAGE) : EMPTY;
}
/// Returns the number of words of the manager
[[nodiscard]] size_t NumWords() const noexcept {
return words.NumWords();
}
/// Returns the size in bytes of the manager
[[nodiscard]] u64 SizeBytes() const noexcept {
return words.size_bytes;
}
/// Returns true when the buffer fits in the small vector optimization
[[nodiscard]] bool IsShort() const noexcept {
return words.IsShort();
}
void FlushCachedWrites() noexcept {
const u64 num_words = NumWords();
u64* const cached_words = Array<Type::CachedCPU>();
u64* const untracked_words = Array<Type::Untracked>();
u64* const cpu_words = Array<Type::CPU>();
for (u64 word_index = 0; word_index < num_words; ++word_index) {
const u64 cached_bits = cached_words[word_index];
NotifyRasterizer<false>(word_index, untracked_words[word_index], cached_bits);
untracked_words[word_index] |= cached_bits;
cpu_words[word_index] |= cached_bits;
cached_words[word_index] = 0;
}
}
private:
template <Type type>
u64* Array() noexcept {
if constexpr (type == Type::CPU) {
return words.cpu.Pointer(IsShort());
} else if constexpr (type == Type::GPU) {
return words.gpu.Pointer(IsShort());
} else if constexpr (type == Type::CachedCPU) {
return words.cached_cpu.Pointer(IsShort());
} else if constexpr (type == Type::Untracked) {
return words.untracked.Pointer(IsShort());
}
}
template <Type type>
const u64* Array() const noexcept {
if constexpr (type == Type::CPU) {
return words.cpu.Pointer(IsShort());
} else if constexpr (type == Type::GPU) {
return words.gpu.Pointer(IsShort());
} else if constexpr (type == Type::CachedCPU) {
return words.cached_cpu.Pointer(IsShort());
} else if constexpr (type == Type::Untracked) {
return words.untracked.Pointer(IsShort());
}
}
/**
* Notify rasterizer about changes in the CPU tracking state of a word in the buffer
*
* @param word_index Index to the word to notify to the rasterizer
* @param current_bits Current state of the word
* @param new_bits New state of the word
*
* @tparam add_to_rasterizer True when the rasterizer should start tracking the new pages
*/
template <bool add_to_rasterizer>
void NotifyRasterizer(u64 word_index, u64 current_bits, u64 new_bits) const {
u64 changed_bits = (add_to_rasterizer ? current_bits : ~current_bits) & new_bits;
VAddr addr = cpu_addr + word_index * BYTES_PER_WORD;
IteratePages(changed_bits, [&](size_t offset, size_t size) {
rasterizer->UpdatePagesCachedCount(addr + offset * BYTES_PER_PAGE,
size * BYTES_PER_PAGE, add_to_rasterizer ? 1 : -1);
});
}
VAddr cpu_addr = 0;
RasterizerInterface* rasterizer = nullptr;
Words<stack_words> words;
};
} // namespace VideoCommon

4
src/video_core/renderer_opengl/gl_buffer_cache.h
View File

@ -8,6 +8,7 @@
#include "common/common_types.h"
#include "video_core/buffer_cache/buffer_cache.h"
#include "video_core/buffer_cache/memory_tracker_base.h"
#include "video_core/rasterizer_interface.h"
#include "video_core/renderer_opengl/gl_device.h"
#include "video_core/renderer_opengl/gl_resource_manager.h"
@ -200,6 +201,8 @@ private:
struct BufferCacheParams {
using Runtime = OpenGL::BufferCacheRuntime;
using Buffer = OpenGL::Buffer;
using Async_Buffer = u32;
using MemoryTracker = VideoCommon::MemoryTrackerBase<VideoCore::RasterizerInterface>;
static constexpr bool IS_OPENGL = true;
static constexpr bool HAS_PERSISTENT_UNIFORM_BUFFER_BINDINGS = true;
@ -208,6 +211,7 @@ struct BufferCacheParams {
static constexpr bool NEEDS_BIND_STORAGE_INDEX = true;
static constexpr bool USE_MEMORY_MAPS = false;
static constexpr bool SEPARATE_IMAGE_BUFFER_BINDINGS = true;
static constexpr bool IMPLEMENTS_ASYNC_DOWNLOADS = false;
};
using BufferCache = VideoCommon::BufferCache<BufferCacheParams>;

9
src/video_core/renderer_opengl/gl_buffer_cache_base.cpp
View File

@ -0,0 +1,9 @@
// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
#include "video_core/buffer_cache/buffer_cache.h"
#include "video_core/renderer_opengl/gl_buffer_cache.h"
namespace VideoCommon {
template class VideoCommon::BufferCache<OpenGL::BufferCacheParams>;
}

8
src/video_core/renderer_vulkan/vk_buffer_cache.cpp
View File

@ -314,8 +314,12 @@ StagingBufferRef BufferCacheRuntime::UploadStagingBuffer(size_t size) {
return staging_pool.Request(size, MemoryUsage::Upload);
}
StagingBufferRef BufferCacheRuntime::DownloadStagingBuffer(size_t size) {
return staging_pool.Request(size, MemoryUsage::Download);
StagingBufferRef BufferCacheRuntime::DownloadStagingBuffer(size_t size, bool deferred) {
return staging_pool.Request(size, MemoryUsage::Download, deferred);
}
void BufferCacheRuntime::FreeDeferredStagingBuffer(StagingBufferRef& ref) {
staging_pool.FreeDeferred(ref);
}
u64 BufferCacheRuntime::GetDeviceLocalMemory() const {

10
src/video_core/renderer_vulkan/vk_buffer_cache.h
View File

@ -3,7 +3,8 @@
#pragma once
#include "video_core/buffer_cache/buffer_cache.h"
#include "video_core/buffer_cache/buffer_cache_base.h"
#include "video_core/buffer_cache/memory_tracker_base.h"
#include "video_core/engines/maxwell_3d.h"
#include "video_core/renderer_vulkan/vk_compute_pass.h"
#include "video_core/renderer_vulkan/vk_staging_buffer_pool.h"
@ -75,7 +76,9 @@ public:
[[nodiscard]] StagingBufferRef UploadStagingBuffer(size_t size);
[[nodiscard]] StagingBufferRef DownloadStagingBuffer(size_t size);
[[nodiscard]] StagingBufferRef DownloadStagingBuffer(size_t size, bool deferred = false);
void FreeDeferredStagingBuffer(StagingBufferRef& ref);
void PreCopyBarrier();
@ -142,6 +145,8 @@ private:
struct BufferCacheParams {
using Runtime = Vulkan::BufferCacheRuntime;
using Buffer = Vulkan::Buffer;
using Async_Buffer = Vulkan::StagingBufferRef;
using MemoryTracker = VideoCommon::MemoryTrackerBase<VideoCore::RasterizerInterface>;
static constexpr bool IS_OPENGL = false;
static constexpr bool HAS_PERSISTENT_UNIFORM_BUFFER_BINDINGS = false;
@ -150,6 +155,7 @@ struct BufferCacheParams {
static constexpr bool NEEDS_BIND_STORAGE_INDEX = false;
static constexpr bool USE_MEMORY_MAPS = true;
static constexpr bool SEPARATE_IMAGE_BUFFER_BINDINGS = false;
static constexpr bool IMPLEMENTS_ASYNC_DOWNLOADS = true;
};
using BufferCache = VideoCommon::BufferCache<BufferCacheParams>;

9
src/video_core/renderer_vulkan/vk_buffer_cache_base.cpp
View File

@ -0,0 +1,9 @@
// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include "video_core/buffer_cache/buffer_cache.h"
#include "video_core/renderer_vulkan/vk_buffer_cache.h"
namespace VideoCommon {
template class VideoCommon::BufferCache<Vulkan::BufferCacheParams>;
}
Write Preview
Loading…
Cancel
Save