[VK] Hybrid memory allocation with robust alignment (#325)

Combines robust OOM handling with progressive size reduction and spec-compliant alignment:

This hybrid approach maximizes allocation success rates through multiple recovery tries.

Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/325
Reviewed-by: CamilleLaVey <camillelavey99@gmail.com>

Credits to Wildcard
Co-authored-by: wildcard <wildcard@eden-emu.dev>
Co-committed-by: wildcard <wildcard@eden-emu.dev>

src/video_core/vulkan_common/vulkan_memory_allocator.cpp

@@ -143,6 +143,10 @@ public:
         return (flags & property_flags) == flags && (type_mask & shifted_memory_type) != 0;
     }
 
+    [[nodiscard]] bool IsEmpty() const noexcept {
+        return commits.empty();
+    }
+
 private:
     [[nodiscard]] static constexpr u32 ShiftType(u32 type) {
         return 1U << type;
@@ -290,36 +294,117 @@ MemoryCommit MemoryAllocator::Commit(const VkMemoryRequirements& requirements, M
     if (std::optional<MemoryCommit> commit = TryCommit(requirements, flags)) {
         return std::move(*commit);
     }
-    // Commit has failed, allocate more memory.
-    const u64 chunk_size = AllocationChunkSize(requirements.size);
-    if (!TryAllocMemory(flags, type_mask, chunk_size)) {
-        // TODO(Rodrigo): Handle out of memory situations in some way like flushing to guest memory.
-        throw vk::Exception(VK_ERROR_OUT_OF_DEVICE_MEMORY);
+
+    // Commit has failed, try progressive fallback strategy
+    u64 chunk_size = AllocationChunkSize(requirements.size);
+    const u64 minimum_size = std::max<u64>(requirements.size, 4ULL << 20); // 4MB minimum
+
+    // try 1: Try allocating with original chunk size
+    if (TryAllocMemory(flags, type_mask, chunk_size)) {
+        return TryCommit(requirements, flags).value();
     }
-    // Commit again, this time it won't fail since there's a fresh allocation above.
-    // If it does, there's a bug.
+
+    // try 2: Clean up empty allocations and try again
+    bool cleaned_up = false;
+    for (auto it = allocations.begin(); it != allocations.end();) {
+        if ((*it)->IsEmpty()) {
+            it = allocations.erase(it);
+            cleaned_up = true;
+        } else {
+            ++it;
+        }
+    }
+
+    if (cleaned_up && TryAllocMemory(flags, type_mask, chunk_size)) {
+        LOG_INFO(Render_Vulkan, "Memory allocation succeeded after cleanup");
         return TryCommit(requirements, flags).value();
+    }
+
+    // try 3: Progressive size reduction with cleanup between attempts
+    while (chunk_size > minimum_size) {
+        chunk_size >>= 1; // Halve the chunk size
+        chunk_size = std::max(chunk_size, minimum_size);
+
+        if (TryAllocMemory(flags, type_mask, chunk_size)) {
+            LOG_WARNING(Render_Vulkan, "Memory allocation succeeded with reduced chunk size: {} MB",
+                       chunk_size >> 20);
+            return TryCommit(requirements, flags).value();
+        }
+
+        // Clean up again between size reduction attempts
+        for (auto it = allocations.begin(); it != allocations.end();) {
+            if ((*it)->IsEmpty()) {
+                it = allocations.erase(it);
+            } else {
+                ++it;
+            }
+        }
+    }
+
+    // try 4: Try minimum size allocation
+    if (chunk_size <= minimum_size && TryAllocMemory(flags, type_mask, minimum_size)) {
+        LOG_WARNING(Render_Vulkan, "Memory allocation succeeded with minimum size: {} MB",
+                    minimum_size >> 20);
+        return TryCommit(requirements, flags).value();
+    }
+    // try 5: Fallback to non-device-local memory if original was device-local
+    if (flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) {
+            const VkMemoryPropertyFlags fallback_flags = flags & ~VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
+
+            // Try with original chunk size first
+            u64 fallback_chunk_size = AllocationChunkSize(requirements.size);
+            if (TryAllocMemory(fallback_flags, type_mask, fallback_chunk_size)) {
+                if (auto commit = TryCommit(requirements, fallback_flags)) {
+                    LOG_WARNING(Render_Vulkan, "Falling back to non-device-local memory due to OOM");
+                    return std::move(*commit);
+                }
+            }
+
+            // Progressive size reduction for non-device-local memory
+            while (fallback_chunk_size > minimum_size) {
+                fallback_chunk_size >>= 1;
+                fallback_chunk_size = std::max(fallback_chunk_size, minimum_size);
+
+                if (TryAllocMemory(fallback_flags, type_mask, fallback_chunk_size)) {
+                    if (auto commit = TryCommit(requirements, fallback_flags)) {
+                        LOG_WARNING(Render_Vulkan,
+                                   "Falling back to non-device-local memory with reduced size: {} MB",
+                                   fallback_chunk_size >> 20);
+                        return std::move(*commit);
+                    }
+                }
+            }
+    }
+
+
+    LOG_CRITICAL(Render_Vulkan, "Vulkan memory allocation failed - exhausted all strategies");
+    throw vk::Exception(VK_ERROR_OUT_OF_DEVICE_MEMORY);
 }
 
 bool MemoryAllocator::TryAllocMemory(VkMemoryPropertyFlags flags, u32 type_mask, u64 size) {
-    const u32 type = FindType(flags, type_mask).value();
+    const auto type_opt = FindType(flags, type_mask);
+    if (!type_opt) {
+        return false;
+    }
+
+    // Adreno requires 4KB alignment(subject to review)
+    const u64 aligned_size = (device.GetDriverID() == VK_DRIVER_ID_QUALCOMM_PROPRIETARY) ?
+                            Common::AlignUp(size, 4096) :
+                            size;
+
     vk::DeviceMemory memory = device.GetLogical().TryAllocateMemory({
         .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
         .pNext = nullptr,
-        .allocationSize = size,
-        .memoryTypeIndex = type,
+        .allocationSize = aligned_size,
+        .memoryTypeIndex = *type_opt,
     });
+
     if (!memory) {
-        if ((flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0) {
-            // Try to allocate non device local memory
-            return TryAllocMemory(flags & ~VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, type_mask, size);
-        } else {
-            // RIP
         return false;
     }
-    }
+
     allocations.push_back(
-        std::make_unique<MemoryAllocation>(this, std::move(memory), flags, size, type));
+        std::make_unique<MemoryAllocation>(this, std::move(memory), flags, aligned_size, *type_opt));
     return true;
 }
 
@@ -331,11 +416,25 @@ void MemoryAllocator::ReleaseMemory(MemoryAllocation* alloc) {
 
 std::optional<MemoryCommit> MemoryAllocator::TryCommit(const VkMemoryRequirements& requirements,
                                                        VkMemoryPropertyFlags flags) {
+    // Conservative, spec-compliant alignment for suballocation
+    VkDeviceSize eff_align = requirements.alignment;
+    const auto& limits = device.GetPhysical().GetProperties().limits;
+    if ((flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) &&
+        !(flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) {
+        // Non-coherent memory must be invalidated on atom boundary
+        if (limits.nonCoherentAtomSize > eff_align) eff_align = limits.nonCoherentAtomSize;
+    }
+    // Separate buffers to avoid stalls on tilers
+    if (buffer_image_granularity > eff_align) {
+        eff_align = buffer_image_granularity;
+    }
+    eff_align = std::bit_ceil(eff_align);
+
     for (auto& allocation : allocations) {
         if (!allocation->IsCompatible(flags, requirements.memoryTypeBits)) {
             continue;
         }
-        if (auto commit = allocation->Commit(requirements.size, requirements.alignment)) {
+        if (auto commit = allocation->Commit(requirements.size, eff_align)) {
             return commit;
         }
     }