diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h
index fbdeba9346..51e880ad36 100644
--- a/src/video_core/texture_cache/texture_cache.h
+++ b/src/video_core/texture_cache/texture_cache.h
@@ -133,6 +133,9 @@ void TextureCache<P>::RunGarbageCollector() {
         if (True(image.flags & ImageFlagBits::IsDecoding)) {
             return false;
         }
+        if (!high_priority_mode && True(image.flags & ImageFlagBits::CostlyLoad)) {
+            return false;
+        }
         if (image.IsSafeDownload() && False(image.flags & ImageFlagBits::BadOverlap)) {
             auto map = runtime.DownloadStagingBuffer(image.unswizzled_size_bytes);
             const auto copies = FixSmallVectorADL(FullDownloadCopies(image.info));