diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h
index 1ee8c62e84..685810337f 100644
--- a/src/video_core/texture_cache/texture_cache.h
+++ b/src/video_core/texture_cache/texture_cache.h
@@ -1495,6 +1495,10 @@ void TextureCache<P>::TickAsyncUnswizzle() {
         }
 
         task.initialized = true;
+        task.was_rescaled = image.IsRescaled();
+        if (task.was_rescaled) {
+            image.ScaleDown(true);
+        }
     }
 
     // Read data
@@ -1625,8 +1629,11 @@ void TextureCache<P>::TickAsyncUnswizzle() {
 
     if (is_final_batch && all_submitted) {
         runtime.FreeDeferredStagingBuffer(task.staging_buffer);
-        image.flags &= ~ImageFlagBits::IsDecoding;
         runtime.ReleaseSparseUnswizzleBuffer(image);
+        image.flags &= ~ImageFlagBits::IsDecoding;
+        if (task.was_rescaled) {
+            image.ScaleUp();
+        }
         unswizzle_queue.pop_front();
     }
 }
diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h
index 755085ec43..69c494254b 100644
--- a/src/video_core/texture_cache/texture_cache_base.h
+++ b/src/video_core/texture_cache/texture_cache_base.h
@@ -141,6 +141,7 @@ class TextureCache : public VideoCommon::ChannelSetupCaches<TextureCacheChannelI
         size_t last_submitted_offset = 0;
         size_t bytes_per_slice;
         bool initialized = false;
+        bool was_rescaled = false;
         bool is_sparse = false;
         std::vector<u8> slice_has_data;
         std::vector<VideoCommon::Accelerated::SliceBBox> slice_bounds;
@@ -439,18 +440,6 @@ private:
     void QueueAsyncUnswizzle(Image& image, ImageId image_id);
     void TickAsyncUnswizzle();
 
-    struct CompletedSparseImage {
-        ImageId image_id;
-        VideoCommon::ImageInfo info;
-        GPUVAddr gpu_addr;
-        size_t guest_size_bytes;
-        std::vector<std::pair<GPUVAddr, size_t>> last_segments;
-        std::vector<u8> slice_uploaded;
-        size_t bytes_per_slice;
-        u64 swizzled_slice_size;
-        u32 swizzle_block_depth;
-    };
-
     Runtime& runtime;
 
     Tegra::MaxwellDeviceMemoryManager& device_memory;