From ade34bb59da916d6fff1846ba1729e18248b2669 Mon Sep 17 00:00:00 2001
From: baldurk <baldurk@baldurk.org>
Date: Mon, 15 Apr 2024 16:22:45 +0100
Subject: [PATCH] Disallow dedicated memory on external images where memory
 reqs change

---
 renderdoc/driver/vulkan/vk_image_states.cpp   |  2 +-
 renderdoc/driver/vulkan/vk_resources.h        |  5 +++
 .../vulkan/wrappers/vk_resource_funcs.cpp     | 38 +++++++++++++++++++
 3 files changed, 44 insertions(+), 1 deletion(-)

diff --git a/renderdoc/driver/vulkan/vk_image_states.cpp b/renderdoc/driver/vulkan/vk_image_states.cpp
index 21e8c4566..48624f899 100644
--- a/renderdoc/driver/vulkan/vk_image_states.cpp
+++ b/renderdoc/driver/vulkan/vk_image_states.cpp
@@ -1013,7 +1013,7 @@ void ImageState::RecordBarrier(VkImageMemoryBarrier barrier, uint32_t queueFamil
      barrier.dstQueueFamilyIndex == VK_QUEUE_FAMILY_EXTERNAL ||
      barrier.dstQueueFamilyIndex == VK_QUEUE_FAMILY_FOREIGN_EXT)
   {
-    RDCERR("External/foreign queue families are not supported");
+    RDCDEBUG("External/foreign queue families are not supported");
     return;
   }
   if(GetImageInfo().sharingMode == VK_SHARING_MODE_CONCURRENT)
diff --git a/renderdoc/driver/vulkan/vk_resources.h b/renderdoc/driver/vulkan/vk_resources.h
index 398f7fa39..56df08bb8 100644
--- a/renderdoc/driver/vulkan/vk_resources.h
+++ b/renderdoc/driver/vulkan/vk_resources.h
@@ -995,6 +995,11 @@ struct ResourceInfo
   // METADATA) we put them in the array.
   Sparse::PageTable sparseTable;
   rdcarray<AspectSparseTable> altSparseAspects;
+
+  // for external images if we query both external and non-external and the sizes are different, we
+  // can't allow dedicated memory as it is required to precisely match in size.
+  bool banDedicated = false;
+
   VkImageAspectFlags sparseAspect;
 
   ResourceId dedicatedMemory;
diff --git a/renderdoc/driver/vulkan/wrappers/vk_resource_funcs.cpp b/renderdoc/driver/vulkan/wrappers/vk_resource_funcs.cpp
index 6caf2b5f8..b45446e67 100644
--- a/renderdoc/driver/vulkan/wrappers/vk_resource_funcs.cpp
+++ b/renderdoc/driver/vulkan/wrappers/vk_resource_funcs.cpp
@@ -307,6 +307,23 @@ bool WrappedVulkan::Serialise_vkAllocateMemory(SerialiserType &ser, VkDevice dev
 
     UnwrapNextChain(m_State, "VkMemoryAllocateInfo", tempMem, (VkBaseInStructure *)&patched);
 
+    // remove dedicated memory struct if it is not allowed due to changing memory sizes
+    {
+      VkMemoryDedicatedAllocateInfo *dedicated = (VkMemoryDedicatedAllocateInfo *)FindNextStruct(
+          &AllocateInfo, VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO);
+      if(dedicated && dedicated->image != VK_NULL_HANDLE)
+      {
+        VkMemoryRequirements mrq = {};
+        ObjDisp(device)->GetImageMemoryRequirements(Unwrap(device), Unwrap(dedicated->image), &mrq);
+
+        if(mrq.size != AllocateInfo.allocationSize)
+        {
+          RDCDEBUG("Removing dedicated allocation for incompatible size");
+          RemoveNextStruct(&patched, VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO);
+        }
+      }
+    }
+
     if(patched.memoryTypeIndex >= m_PhysicalDeviceData.memProps.memoryTypeCount)
     {
       SET_ERROR_RESULT(
@@ -496,6 +513,19 @@ VkResult WrappedVulkan::vkAllocateMemory(VkDevice device, const VkMemoryAllocate
   if(IsCaptureMode(m_State) && memFlags && (memFlags->flags & VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT))
     memFlags->flags |= VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_CAPTURE_REPLAY_BIT;
 
+  // remove dedicated memory struct if it is not allowed
+  {
+    VkMemoryDedicatedAllocateInfo *dedicated = (VkMemoryDedicatedAllocateInfo *)FindNextStruct(
+        pAllocateInfo, VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO);
+    if(dedicated && dedicated->image != VK_NULL_HANDLE)
+    {
+      VkResourceRecord *imageRecord = GetRecord(dedicated->image);
+
+      if(imageRecord->resInfo->banDedicated)
+        RemoveNextStruct(&unwrapped, VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO);
+    }
+  }
+
   VkResult ret;
   SERIALISE_TIME_CALL(
       ret = ObjDisp(device)->AllocateMemory(Unwrap(device), &unwrapped, NULL, pMemory));
@@ -2507,6 +2537,14 @@ VkResult WrappedVulkan::vkCreateImage(VkDevice device, const VkImageCreateInfo *
                   "Non-external version requires %llu bytes at %llu alignment, in %x memory types",
                   mrq.size, mrq.alignment, mrq.memoryTypeBits);
 
+              if(resInfo.memreqs.size != mrq.size)
+              {
+                RDCWARN(
+                    "Required size changed on image between external/non-external, banning "
+                    "dedicated memory");
+                resInfo.banDedicated = true;
+              }
+
               resInfo.memreqs.size = RDCMAX(resInfo.memreqs.size, mrq.size);
               resInfo.memreqs.alignment = RDCMAX(resInfo.memreqs.alignment, mrq.alignment);