Fix self-capture for vulkan captures using ASs

* The problem here is that due to design flaws in the extension when ASs are in
  use we don't know whether a memory allocation will need BDA or not an the
  application doesn't have to set any flag - unlike for normal buffer BDA. So we
  promote (almost) all memory allocations to BDA when using ASs even if they're
  not needed.
* This normally works fine except if during self-capture the replay process
  allocates some normal memory before all application replayed allocations have
  been made, the self-capturing will promote it to BDA and request a replayable
  address that might clash with a later address the application had used and
  would be needed.
* To solve this, we ensure that during capture we don't create wrapped
  allocations more than necessary - to avoid causing clashes - as well as
  ensuring that on replay we only create new allocations after all replayed
  allocations.
* We also take advantage of dedicated allocations for fake swapchain images,
  since dedicated image allocations will not be promoted to BDA.

renderdoc/driver/vulkan/vk_common.cpp

@@ -171,6 +171,11 @@ VkObjectType objType<VkBuffer>()
   return VK_OBJECT_TYPE_BUFFER;
 }
 template <>
+VkObjectType objType<VkDeviceMemory>()
+{
+  return VK_OBJECT_TYPE_DEVICE_MEMORY;
+}
+template <>
 VkObjectType objType<VkImage>()
 {
   return VK_OBJECT_TYPE_IMAGE;
@@ -270,7 +275,7 @@ void GPUBuffer::Create(WrappedVulkan *driver, VkDevice dev, VkDeviceSize size, u
   vkr = ObjDisp(dev)->BindBufferMemory(Unwrap(dev), buf, mem, 0);
   CHECK_VKR(driver, vkr);
 
-  if(flags & eGPUBufferAddressable)
+  if(useBufferAddressKHR && (flags & eGPUBufferAddressable))
   {
     RDCCOMPILE_ASSERT(VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO ==
                           VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO_EXT,

renderdoc/driver/vulkan/vk_core.cpp

@@ -3214,6 +3214,14 @@ RDResult WrappedVulkan::ReadLogInitialisation(RDCFile *rdc, bool storeStructured
 
     if((SystemChunk)context == SystemChunk::CaptureScope)
     {
+      // create most internal resources now, after having created all application resources. This
+      // means that in a self-capture scenario we don't risk screwing up BDA allocations by having a
+      // non-BDA buffer that's then promoted to BDA during self capture and steals some application
+      // reserved addresses.
+      m_DebugManager = new VulkanDebugManager(this);
+
+      m_Replay->CreateResources();
+
       GetReplay()->WriteFrameRecord().frameInfo.fileOffset = offsetStart;
 
       // read the remaining data into memory and pass to immediate context

renderdoc/driver/vulkan/vk_debug.cpp

@@ -474,12 +474,10 @@ VulkanDebugManager::VulkanDebugManager(WrappedVulkan *driver)
         VK_IMAGE_LAYOUT_UNDEFINED,
     };
 
-    vkr = driver->vkCreateImage(driver->GetDev(), &imInfo, NULL, &m_DummyDepthImage);
+    vkr = ObjDisp(dev)->CreateImage(Unwrap(dev), &imInfo, NULL, &m_UnwrappedDummyDepthImage);
     CHECK_VKR(m_pDriver, vkr);
 
-    NameVulkanObject(m_DummyDepthImage, "m_DummyDepthImage");
-
-    rm->SetInternalResource(GetResID(m_DummyDepthImage));
+    NameUnwrappedVulkanObject(m_UnwrappedDummyDepthImage, "m_UnwrappedDummyDepthImage");
   }
 
   // need a dummy UINT texture to fill the binding when we don't have a stencil aspect to copy.
@@ -489,6 +487,8 @@ VulkanDebugManager::VulkanDebugManager(WrappedVulkan *driver)
                                VK_FORMAT_S8_UINT,           VK_FORMAT_D32_SFLOAT_S8_UINT,
                                VK_FORMAT_D24_UNORM_S8_UINT, VK_FORMAT_D16_UNORM_S8_UINT};
 
+  // the dummy objects are allocated as unwrapped so that they doesn't go through BDA promotion for
+  // their memory when ASs are enabled :(
   for(VkFormat f : attemptFormats)
   {
     VkImageAspectFlags viewAspectMask =
@@ -552,18 +552,16 @@ VulkanDebugManager::VulkanDebugManager(WrappedVulkan *driver)
 
     RDCASSERT(imgprops.sampleCounts & imInfo.samples, imgprops.sampleCounts, imInfo.samples);
 
-    vkr = driver->vkCreateImage(driver->GetDev(), &imInfo, NULL, &m_DummyStencilImage);
+    vkr = ObjDisp(dev)->CreateImage(Unwrap(dev), &imInfo, NULL, &m_UnwrappedDummyStencilImage);
     CHECK_VKR(m_pDriver, vkr);
 
-    NameVulkanObject(m_DummyStencilImage, "m_DummyStencilImage");
-
-    rm->SetInternalResource(GetResID(m_DummyStencilImage));
+    NameUnwrappedVulkanObject(m_UnwrappedDummyStencilImage, "m_UnwrappedDummyStencilImage");
 
     VkMemoryRequirements depthmrq = {};
-    driver->vkGetImageMemoryRequirements(driver->GetDev(), m_DummyDepthImage, &depthmrq);
+    ObjDisp(dev)->GetImageMemoryRequirements(Unwrap(dev), m_UnwrappedDummyDepthImage, &depthmrq);
 
     VkMemoryRequirements mrq = {};
-    driver->vkGetImageMemoryRequirements(driver->GetDev(), m_DummyStencilImage, &mrq);
+    ObjDisp(dev)->GetImageMemoryRequirements(Unwrap(dev), m_UnwrappedDummyStencilImage, &mrq);
 
     // assume we can combine these images into one allocation
     RDCASSERT((mrq.memoryTypeBits & depthmrq.memoryTypeBits) != 0, mrq.memoryTypeBits,
@@ -586,27 +584,27 @@ VulkanDebugManager::VulkanDebugManager(WrappedVulkan *driver)
         driver->GetGPULocalMemoryIndex(mrq.memoryTypeBits),
     };
 
-    vkr = driver->vkAllocateMemory(driver->GetDev(), &allocInfo, NULL, &m_DummyMemory);
+    vkr = ObjDisp(dev)->AllocateMemory(Unwrap(dev), &allocInfo, NULL, &m_UnwrappedDummyMemory);
     CHECK_VKR(m_pDriver, vkr);
 
     if(vkr != VK_SUCCESS)
       return;
 
-    rm->SetInternalResource(GetResID(m_DummyMemory));
+    NameUnwrappedVulkanObject(m_UnwrappedDummyMemory, "m_UnwrappedDummyMemory");
 
-    NameVulkanObject(m_DummyStencilImage, "m_DummyMemory");
-
-    vkr = driver->vkBindImageMemory(driver->GetDev(), m_DummyStencilImage, m_DummyMemory, 0);
+    vkr = ObjDisp(dev)->BindImageMemory(Unwrap(dev), m_UnwrappedDummyStencilImage,
+                                        m_UnwrappedDummyMemory, 0);
     CHECK_VKR(m_pDriver, vkr);
 
-    vkr = driver->vkBindImageMemory(driver->GetDev(), m_DummyDepthImage, m_DummyMemory, mrq.size);
+    vkr = ObjDisp(dev)->BindImageMemory(Unwrap(dev), m_UnwrappedDummyDepthImage,
+                                        m_UnwrappedDummyMemory, mrq.size);
     CHECK_VKR(m_pDriver, vkr);
 
     VkImageViewCreateInfo viewInfo = {
         VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
         NULL,
         0,
-        m_DummyStencilImage,
+        m_UnwrappedDummyStencilImage,
         VK_IMAGE_VIEW_TYPE_2D_ARRAY,
         f,
         {VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY,
@@ -620,22 +618,18 @@ VulkanDebugManager::VulkanDebugManager(WrappedVulkan *driver)
         },
     };
 
-    vkr = driver->vkCreateImageView(driver->GetDev(), &viewInfo, NULL, &m_DummyStencilView);
+    vkr = ObjDisp(dev)->CreateImageView(Unwrap(dev), &viewInfo, NULL, &m_UnwrappedDummyStencilView);
     CHECK_VKR(m_pDriver, vkr);
 
-    NameVulkanObject(m_DummyStencilView, "m_DummyStencilView");
+    NameUnwrappedVulkanObject(m_UnwrappedDummyStencilView, "m_UnwrappedDummyStencilView");
 
-    rm->SetInternalResource(GetResID(m_DummyStencilView));
-
-    viewInfo.image = m_DummyDepthImage;
+    viewInfo.image = m_UnwrappedDummyDepthImage;
     viewInfo.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
     viewInfo.format = VK_FORMAT_R8G8B8A8_UNORM;
-    vkr = driver->vkCreateImageView(driver->GetDev(), &viewInfo, NULL, &m_DummyDepthView);
+    vkr = ObjDisp(dev)->CreateImageView(Unwrap(dev), &viewInfo, NULL, &m_UnwrappedDummyDepthView);
     CHECK_VKR(m_pDriver, vkr);
 
-    NameVulkanObject(m_DummyDepthView, "m_DummyDepthView");
-
-    rm->SetInternalResource(GetResID(m_DummyDepthView));
+    NameUnwrappedVulkanObject(m_UnwrappedDummyDepthView, "m_UnwrappedDummyDepthView");
 
     VkCommandBuffer cmd = driver->GetNextCmd();
 
@@ -658,13 +652,13 @@ VulkanDebugManager::VulkanDebugManager(WrappedVulkan *driver)
         VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL,
         VK_QUEUE_FAMILY_IGNORED,
         VK_QUEUE_FAMILY_IGNORED,
-        Unwrap(m_DummyStencilImage),
+        m_UnwrappedDummyStencilImage,
         {barrierAspectMask, 0, 1, 0, 1},
     };
 
     DoPipelineBarrier(cmd, 1, &barrier);
 
-    barrier.image = Unwrap(m_DummyDepthImage);
+    barrier.image = m_UnwrappedDummyDepthImage;
     barrierAspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
     DoPipelineBarrier(cmd, 1, &barrier);
 
@@ -673,7 +667,7 @@ VulkanDebugManager::VulkanDebugManager(WrappedVulkan *driver)
     break;
   }
 
-  if(m_DummyStencilImage == VK_NULL_HANDLE)
+  if(m_UnwrappedDummyStencilImage == VK_NULL_HANDLE)
   {
     RDCERR("Couldn't find any integer format we could generate a dummy multisampled image with");
   }
@@ -882,11 +876,11 @@ VulkanDebugManager::~VulkanDebugManager()
   for(VkDescriptorPool pool : m_BufferMSDescriptorPools)
     m_pDriver->vkDestroyDescriptorPool(dev, pool, NULL);
 
-  m_pDriver->vkDestroyImageView(dev, m_DummyDepthView, NULL);
-  m_pDriver->vkDestroyImage(dev, m_DummyDepthImage, NULL);
-  m_pDriver->vkDestroyImageView(dev, m_DummyStencilView, NULL);
-  m_pDriver->vkDestroyImage(dev, m_DummyStencilImage, NULL);
-  m_pDriver->vkFreeMemory(dev, m_DummyMemory, NULL);
+  ObjDisp(dev)->DestroyImageView(Unwrap(dev), m_UnwrappedDummyDepthView, NULL);
+  ObjDisp(dev)->DestroyImage(Unwrap(dev), m_UnwrappedDummyDepthImage, NULL);
+  ObjDisp(dev)->DestroyImageView(Unwrap(dev), m_UnwrappedDummyStencilView, NULL);
+  ObjDisp(dev)->DestroyImage(Unwrap(dev), m_UnwrappedDummyStencilImage, NULL);
+  ObjDisp(dev)->FreeMemory(Unwrap(dev), m_UnwrappedDummyMemory, NULL);
 
   m_pDriver->vkDestroyDescriptorSetLayout(dev, m_BufferMSDescSetLayout, NULL);
   m_pDriver->vkDestroyPipelineLayout(dev, m_BufferMSPipeLayout, NULL);

renderdoc/driver/vulkan/vk_debug.h

@@ -148,11 +148,11 @@ private:
   VkPipeline m_DepthMS2BufferPipe = VK_NULL_HANDLE;
 
   // MSAA dummy images
-  VkDeviceMemory m_DummyMemory = VK_NULL_HANDLE;
-  VkImage m_DummyDepthImage = {VK_NULL_HANDLE};
-  VkImageView m_DummyDepthView = {VK_NULL_HANDLE};
-  VkImage m_DummyStencilImage = {VK_NULL_HANDLE};
-  VkImageView m_DummyStencilView = {VK_NULL_HANDLE};
+  VkDeviceMemory m_UnwrappedDummyMemory = VK_NULL_HANDLE;
+  VkImage m_UnwrappedDummyDepthImage = {VK_NULL_HANDLE};
+  VkImageView m_UnwrappedDummyDepthView = {VK_NULL_HANDLE};
+  VkImage m_UnwrappedDummyStencilImage = {VK_NULL_HANDLE};
+  VkImageView m_UnwrappedDummyStencilView = {VK_NULL_HANDLE};
 
   // dummy pipeline
   VkPipelineLayout m_DummyPipelineLayout = VK_NULL_HANDLE;

renderdoc/driver/vulkan/vk_memory.cpp

@@ -446,7 +446,10 @@ MemoryAllocation WrappedVulkan::AllocateMemoryForResource(bool buffer, VkMemoryR
     ret.mem = VK_NULL_HANDLE;
 
     if(vkr != VK_SUCCESS)
+    {
+      RDCERR("Failed allocating internal memory: %s", ToStr(vkr).c_str());
       return ret;
+    }
 
     GetResourceManager()->WrapResource(Unwrap(d), chunk.mem);
 

renderdoc/driver/vulkan/vk_msaa_buffer_conv.cpp

@@ -294,9 +294,9 @@ void VulkanDebugManager::CopyDepthTex2DMSToBuffer(VkCommandBuffer cmd, VkBuffer
 
   if((aspectFlags & VK_IMAGE_ASPECT_DEPTH_BIT) == 0)
   {
-    if(m_DummyDepthView != VK_NULL_HANDLE)
+    if(m_UnwrappedDummyDepthView != VK_NULL_HANDLE)
     {
-      srcdesc[0].imageView = Unwrap(m_DummyDepthView);
+      srcdesc[0].imageView = m_UnwrappedDummyDepthView;
     }
     else
     {
@@ -309,9 +309,9 @@ void VulkanDebugManager::CopyDepthTex2DMSToBuffer(VkCommandBuffer cmd, VkBuffer
 
   if((aspectFlags & VK_IMAGE_ASPECT_STENCIL_BIT) == 0)
   {
-    if(m_DummyStencilView != VK_NULL_HANDLE)
+    if(m_UnwrappedDummyStencilView != VK_NULL_HANDLE)
     {
-      srcdesc[1].imageView = Unwrap(m_DummyStencilView);
+      srcdesc[1].imageView = m_UnwrappedDummyStencilView;
     }
     else
     {

renderdoc/driver/vulkan/vk_rendertext.cpp

@@ -31,13 +31,23 @@
 #define VULKAN 1
 #include "data/glsl/glsl_ubos_cpp.h"
 
+namespace
+{
+const rdcarray<VkFormat> BBFormats = {
+    VK_FORMAT_R8G8B8A8_SRGB,
+    VK_FORMAT_R8G8B8A8_UNORM,
+    VK_FORMAT_B8G8R8A8_SRGB,
+    VK_FORMAT_B8G8R8A8_UNORM,
+    VK_FORMAT_A2B10G10R10_UNORM_PACK32,
+    VK_FORMAT_R16G16B16A16_SFLOAT,
+    VK_FORMAT_R5G6B5_UNORM_PACK16,
+};
+};
+
 VulkanTextRenderer::VulkanTextRenderer(WrappedVulkan *driver)
 {
-  m_pDriver = driver;
   m_Device = driver->GetDev();
 
-  VulkanResourceManager *rm = driver->GetResourceManager();
-
   VkDevice dev = m_Device;
 
   VkResult vkr = VK_SUCCESS;
@@ -53,11 +63,9 @@ VulkanTextRenderer::VulkanTextRenderer(WrappedVulkan *driver)
       VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
   sampInfo.maxLod = 128.0f;
 
-  vkr = m_pDriver->vkCreateSampler(dev, &sampInfo, NULL, &m_LinearSampler);
+  vkr = ObjDisp(dev)->CreateSampler(Unwrap(dev), &sampInfo, NULL, &m_LinearSampler);
   RDCASSERTEQUAL(vkr, VK_SUCCESS);
 
-  rm->SetInternalResource(GetResID(m_LinearSampler));
-
   // just need enough for text rendering
   VkDescriptorPoolSize captureDescPoolTypes[] = {
       {VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 1},
@@ -75,11 +83,9 @@ VulkanTextRenderer::VulkanTextRenderer(WrappedVulkan *driver)
   };
 
   // create descriptor pool
-  vkr = m_pDriver->vkCreateDescriptorPool(dev, &descpoolInfo, NULL, &m_DescriptorPool);
+  vkr = ObjDisp(dev)->CreateDescriptorPool(Unwrap(dev), &descpoolInfo, NULL, &m_DescriptorPool);
   RDCASSERTEQUAL(vkr, VK_SUCCESS);
 
-  rm->SetInternalResource(GetResID(m_DescriptorPool));
-
   // declare some common creation info structs
   VkPipelineLayoutCreateInfo pipeLayoutInfo = {VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO};
   pipeLayoutInfo.setLayoutCount = 1;
@@ -87,58 +93,13 @@ VulkanTextRenderer::VulkanTextRenderer(WrappedVulkan *driver)
   VkDescriptorSetAllocateInfo descSetAllocInfo = {VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO,
                                                   NULL, m_DescriptorPool, 1, NULL};
 
-  // compatible render passes for creating pipelines.
-  VkRenderPass RGBA8sRGBRP = VK_NULL_HANDLE;
-  VkRenderPass RGBA8LinearRP = VK_NULL_HANDLE;
-  VkRenderPass BGRA8sRGBRP = VK_NULL_HANDLE;
-  VkRenderPass BGRA8LinearRP = VK_NULL_HANDLE;
-
-  {
-    VkAttachmentDescription attDesc = {0,
-                                       VK_FORMAT_R8G8B8A8_SRGB,
-                                       VK_SAMPLE_COUNT_1_BIT,
-                                       VK_ATTACHMENT_LOAD_OP_LOAD,
-                                       VK_ATTACHMENT_STORE_OP_STORE,
-                                       VK_ATTACHMENT_LOAD_OP_DONT_CARE,
-                                       VK_ATTACHMENT_STORE_OP_DONT_CARE,
-                                       VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
-                                       VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL};
-
-    VkAttachmentReference attRef = {0, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL};
-
-    VkSubpassDescription sub = {0};
-
-    sub.colorAttachmentCount = 1;
-    sub.pColorAttachments = &attRef;
-
-    VkRenderPassCreateInfo rpinfo = {
-        VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO, NULL, 0, 1, &attDesc, 1, &sub,
-    };
-
-    attDesc.format = VK_FORMAT_R8G8B8A8_SRGB;
-    m_pDriver->vkCreateRenderPass(dev, &rpinfo, NULL, &RGBA8sRGBRP);
-    rm->SetInternalResource(GetResID(RGBA8sRGBRP));
-
-    attDesc.format = VK_FORMAT_R8G8B8A8_UNORM;
-    m_pDriver->vkCreateRenderPass(dev, &rpinfo, NULL, &RGBA8LinearRP);
-    rm->SetInternalResource(GetResID(RGBA8LinearRP));
-
-    attDesc.format = VK_FORMAT_B8G8R8A8_SRGB;
-    m_pDriver->vkCreateRenderPass(dev, &rpinfo, NULL, &BGRA8sRGBRP);
-    rm->SetInternalResource(GetResID(BGRA8sRGBRP));
-
-    attDesc.format = VK_FORMAT_B8G8R8A8_UNORM;
-    m_pDriver->vkCreateRenderPass(dev, &rpinfo, NULL, &BGRA8LinearRP);
-    rm->SetInternalResource(GetResID(BGRA8LinearRP));
-  }
-
   // declare the pipeline creation info and all of its sub-structures
   // these are modified as appropriate for each pipeline we create
   VkPipelineShaderStageCreateInfo stages[2] = {
       {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, NULL, 0, VK_SHADER_STAGE_VERTEX_BIT,
-       shaderCache->GetBuiltinModule(BuiltinShader::TextVS), "main", NULL},
+       Unwrap(shaderCache->GetBuiltinModule(BuiltinShader::TextVS)), "main", NULL},
       {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, NULL, 0, VK_SHADER_STAGE_FRAGMENT_BIT,
-       shaderCache->GetBuiltinModule(BuiltinShader::TextFS), "main", NULL},
+       Unwrap(shaderCache->GetBuiltinModule(BuiltinShader::TextFS)), "main", NULL},
   };
 
   VkPipelineVertexInputStateCreateInfo vi = {
@@ -238,24 +199,19 @@ VulkanTextRenderer::VulkanTextRenderer(WrappedVulkan *driver)
       &layoutBinding[0],
   };
 
-  vkr = m_pDriver->vkCreateDescriptorSetLayout(dev, &descsetLayoutInfo, NULL, &m_TextDescSetLayout);
+  vkr = ObjDisp(dev)->CreateDescriptorSetLayout(Unwrap(dev), &descsetLayoutInfo, NULL,
+                                                &m_TextDescSetLayout);
   RDCASSERTEQUAL(vkr, VK_SUCCESS);
 
-  rm->SetInternalResource(GetResID(m_TextDescSetLayout));
-
   pipeLayoutInfo.pSetLayouts = &m_TextDescSetLayout;
 
-  vkr = m_pDriver->vkCreatePipelineLayout(dev, &pipeLayoutInfo, NULL, &m_TextPipeLayout);
+  vkr = ObjDisp(dev)->CreatePipelineLayout(Unwrap(dev), &pipeLayoutInfo, NULL, &m_TextPipeLayout);
   RDCASSERTEQUAL(vkr, VK_SUCCESS);
 
-  rm->SetInternalResource(GetResID(m_TextPipeLayout));
-
   descSetAllocInfo.pSetLayouts = &m_TextDescSetLayout;
-  vkr = m_pDriver->vkAllocateDescriptorSets(dev, &descSetAllocInfo, &m_TextDescSet);
+  vkr = ObjDisp(dev)->AllocateDescriptorSets(Unwrap(dev), &descSetAllocInfo, &m_TextDescSet);
   RDCASSERTEQUAL(vkr, VK_SUCCESS);
 
-  rm->SetInternalResource(GetResID(m_TextDescSet));
-
   // make the ring conservatively large to handle many lines of text * several frames
   m_TextGeneralUBO.Create(driver, dev, 128, 100, 0);
   m_TextGeneralUBO.Name("m_TextGeneralUBO");
@@ -269,33 +225,44 @@ VulkanTextRenderer::VulkanTextRenderer(WrappedVulkan *driver)
 
   pipeInfo.layout = m_TextPipeLayout;
 
-  pipeInfo.renderPass = RGBA8sRGBRP;
-  vkr = m_pDriver->vkCreateGraphicsPipelines(dev, VK_NULL_HANDLE, 1, &pipeInfo, NULL,
-                                             &m_TextPipeline[0]);
-  RDCASSERTEQUAL(vkr, VK_SUCCESS);
+  {
+    VkAttachmentDescription attDesc = {0,
+                                       VK_FORMAT_R8G8B8A8_SRGB,
+                                       VK_SAMPLE_COUNT_1_BIT,
+                                       VK_ATTACHMENT_LOAD_OP_LOAD,
+                                       VK_ATTACHMENT_STORE_OP_STORE,
+                                       VK_ATTACHMENT_LOAD_OP_DONT_CARE,
+                                       VK_ATTACHMENT_STORE_OP_DONT_CARE,
+                                       VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
+                                       VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL};
 
-  rm->SetInternalResource(GetResID(m_TextPipeline[0]));
+    VkAttachmentReference attRef = {0, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL};
 
-  pipeInfo.renderPass = RGBA8LinearRP;
-  vkr = m_pDriver->vkCreateGraphicsPipelines(dev, VK_NULL_HANDLE, 1, &pipeInfo, NULL,
-                                             &m_TextPipeline[1]);
-  RDCASSERTEQUAL(vkr, VK_SUCCESS);
+    VkSubpassDescription sub = {0};
 
-  rm->SetInternalResource(GetResID(m_TextPipeline[1]));
+    sub.colorAttachmentCount = 1;
+    sub.pColorAttachments = &attRef;
 
-  pipeInfo.renderPass = BGRA8sRGBRP;
-  vkr = m_pDriver->vkCreateGraphicsPipelines(dev, VK_NULL_HANDLE, 1, &pipeInfo, NULL,
-                                             &m_TextPipeline[2]);
-  RDCASSERTEQUAL(vkr, VK_SUCCESS);
+    VkRenderPassCreateInfo rpinfo = {
+        VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO, NULL, 0, 1, &attDesc, 1, &sub,
+    };
 
-  rm->SetInternalResource(GetResID(m_TextPipeline[2]));
+    RDCASSERTEQUAL(BBFormats.size(), VulkanTextRenderer::NUM_BB_FORMATS);
 
-  pipeInfo.renderPass = BGRA8LinearRP;
-  vkr = m_pDriver->vkCreateGraphicsPipelines(dev, VK_NULL_HANDLE, 1, &pipeInfo, NULL,
-                                             &m_TextPipeline[3]);
-  RDCASSERTEQUAL(vkr, VK_SUCCESS);
+    for(size_t i = 0; i < BBFormats.size(); i++)
+    {
+      VkRenderPass rp;
+      attDesc.format = BBFormats[i];
+      ObjDisp(dev)->CreateRenderPass(Unwrap(dev), &rpinfo, NULL, &rp);
 
-  rm->SetInternalResource(GetResID(m_TextPipeline[3]));
+      pipeInfo.renderPass = rp;
+      vkr = ObjDisp(dev)->CreateGraphicsPipelines(Unwrap(dev), VK_NULL_HANDLE, 1, &pipeInfo, NULL,
+                                                  &m_TextPipeline[i]);
+      RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+      ObjDisp(dev)->DestroyRenderPass(Unwrap(dev), rp, NULL);
+    }
+  }
 
   // create the actual font texture data and glyph data, for upload
   {
@@ -348,15 +315,13 @@ VulkanTextRenderer::VulkanTextRenderer(WrappedVulkan *driver)
 
     // create and fill image
     {
-      vkr = m_pDriver->vkCreateImage(dev, &imInfo, NULL, &m_TextAtlas);
+      vkr = ObjDisp(dev)->CreateImage(Unwrap(dev), &imInfo, NULL, &m_TextAtlas);
       RDCASSERTEQUAL(vkr, VK_SUCCESS);
 
-      NameVulkanObject(m_TextAtlas, "m_TextAtlas");
-
-      rm->SetInternalResource(GetResID(m_TextAtlas));
+      NameUnwrappedVulkanObject(m_TextAtlas, "m_TextAtlas");
 
       VkMemoryRequirements mrq = {0};
-      m_pDriver->vkGetImageMemoryRequirements(dev, m_TextAtlas, &mrq);
+      ObjDisp(dev)->GetImageMemoryRequirements(Unwrap(dev), m_TextAtlas, &mrq);
 
       // allocate readback memory
       VkMemoryAllocateInfo allocInfo = {
@@ -366,12 +331,10 @@ VulkanTextRenderer::VulkanTextRenderer(WrappedVulkan *driver)
           driver->GetGPULocalMemoryIndex(mrq.memoryTypeBits),
       };
 
-      vkr = m_pDriver->vkAllocateMemory(dev, &allocInfo, NULL, &m_TextAtlasMem);
+      vkr = ObjDisp(dev)->AllocateMemory(Unwrap(dev), &allocInfo, NULL, &m_TextAtlasMem);
       RDCASSERTEQUAL(vkr, VK_SUCCESS);
 
-      rm->SetInternalResource(GetResID(m_TextAtlasMem));
-
-      vkr = m_pDriver->vkBindImageMemory(dev, m_TextAtlas, m_TextAtlasMem, 0);
+      vkr = ObjDisp(dev)->BindImageMemory(Unwrap(dev), m_TextAtlas, m_TextAtlasMem, 0);
       RDCASSERTEQUAL(vkr, VK_SUCCESS);
 
       VkImageViewCreateInfo viewInfo = {
@@ -386,11 +349,9 @@ VulkanTextRenderer::VulkanTextRenderer(WrappedVulkan *driver)
           {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1},
       };
 
-      vkr = m_pDriver->vkCreateImageView(dev, &viewInfo, NULL, &m_TextAtlasView);
+      vkr = ObjDisp(dev)->CreateImageView(Unwrap(dev), &viewInfo, NULL, &m_TextAtlasView);
       RDCASSERTEQUAL(vkr, VK_SUCCESS);
 
-      rm->SetInternalResource(GetResID(m_TextAtlasView));
-
       // create temporary memory and buffer to upload atlas
       // doesn't need to be ring'd, as it's static
       m_TextAtlasUpload.Create(driver, dev, 32768, 1, 0);
@@ -448,7 +409,7 @@ VulkanTextRenderer::VulkanTextRenderer(WrappedVulkan *driver)
         VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
         VK_QUEUE_FAMILY_IGNORED,
         VK_QUEUE_FAMILY_IGNORED,
-        Unwrap(m_TextAtlas),
+        m_TextAtlas,
         {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1},
     };
 
@@ -485,8 +446,7 @@ VulkanTextRenderer::VulkanTextRenderer(WrappedVulkan *driver)
     // copy to image
     ObjDisp(textAtlasUploadCmd)
         ->CmdCopyBufferToImage(Unwrap(textAtlasUploadCmd), m_TextAtlasUpload.UnwrappedBuffer(),
-                               Unwrap(m_TextAtlas), VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1,
-                               &bufRegion);
+                               m_TextAtlas, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &bufRegion);
 
     VkImageMemoryBarrier copydonebarrier = {
         VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
@@ -497,7 +457,7 @@ VulkanTextRenderer::VulkanTextRenderer(WrappedVulkan *driver)
         VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL,
         VK_QUEUE_FAMILY_IGNORED,
         VK_QUEUE_FAMILY_IGNORED,
-        Unwrap(m_TextAtlas),
+        m_TextAtlas,
         {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1},
     };
 
@@ -516,44 +476,39 @@ VulkanTextRenderer::VulkanTextRenderer(WrappedVulkan *driver)
 
   VkDescriptorImageInfo atlasImInfo;
   atlasImInfo.imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
-  atlasImInfo.imageView = Unwrap(m_TextAtlasView);
-  atlasImInfo.sampler = Unwrap(m_LinearSampler);
+  atlasImInfo.imageView = m_TextAtlasView;
+  atlasImInfo.sampler = m_LinearSampler;
 
   VkWriteDescriptorSet textSetWrites[] = {
-      {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, NULL, Unwrap(m_TextDescSet), 0, 0, 1,
+      {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, NULL, m_TextDescSet, 0, 0, 1,
        VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC, NULL, &bufInfo[0], NULL},
-      {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, NULL, Unwrap(m_TextDescSet), 1, 0, 1,
+      {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, NULL, m_TextDescSet, 1, 0, 1,
        VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, NULL, &bufInfo[1], NULL},
-      {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, NULL, Unwrap(m_TextDescSet), 2, 0, 1,
+      {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, NULL, m_TextDescSet, 2, 0, 1,
        VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC, NULL, &bufInfo[2], NULL},
-      {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, NULL, Unwrap(m_TextDescSet), 3, 0, 1,
+      {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, NULL, m_TextDescSet, 3, 0, 1,
        VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, &atlasImInfo, NULL, NULL},
   };
 
   ObjDisp(dev)->UpdateDescriptorSets(Unwrap(dev), ARRAY_COUNT(textSetWrites), textSetWrites, 0, NULL);
-
-  m_pDriver->vkDestroyRenderPass(dev, RGBA8sRGBRP, NULL);
-  m_pDriver->vkDestroyRenderPass(dev, RGBA8LinearRP, NULL);
-  m_pDriver->vkDestroyRenderPass(dev, BGRA8sRGBRP, NULL);
-  m_pDriver->vkDestroyRenderPass(dev, BGRA8LinearRP, NULL);
 }
 
 VulkanTextRenderer::~VulkanTextRenderer()
 {
   VkDevice dev = m_Device;
 
-  m_pDriver->vkDestroyDescriptorPool(dev, m_DescriptorPool, NULL);
+  ObjDisp(dev)->DestroyDescriptorPool(Unwrap(dev), m_DescriptorPool, NULL);
 
-  m_pDriver->vkDestroySampler(dev, m_LinearSampler, NULL);
+  ObjDisp(dev)->DestroySampler(Unwrap(dev), m_LinearSampler, NULL);
 
-  m_pDriver->vkDestroyDescriptorSetLayout(dev, m_TextDescSetLayout, NULL);
-  m_pDriver->vkDestroyPipelineLayout(dev, m_TextPipeLayout, NULL);
+  ObjDisp(dev)->DestroyDescriptorSetLayout(Unwrap(dev), m_TextDescSetLayout, NULL);
+  ObjDisp(dev)->DestroyPipelineLayout(Unwrap(dev), m_TextPipeLayout, NULL);
   for(size_t i = 0; i < ARRAY_COUNT(m_TextPipeline); i++)
-    m_pDriver->vkDestroyPipeline(dev, m_TextPipeline[i], NULL);
+    ObjDisp(dev)->DestroyPipeline(Unwrap(dev), m_TextPipeline[i], NULL);
 
-  m_pDriver->vkDestroyImageView(dev, m_TextAtlasView, NULL);
-  m_pDriver->vkDestroyImage(dev, m_TextAtlas, NULL);
-  m_pDriver->vkFreeMemory(dev, m_TextAtlasMem, NULL);
+  ObjDisp(dev)->DestroyImageView(Unwrap(dev), m_TextAtlasView, NULL);
+  ObjDisp(dev)->DestroyImage(Unwrap(dev), m_TextAtlas, NULL);
+  ObjDisp(dev)->FreeMemory(Unwrap(dev), m_TextAtlasMem, NULL);
 
   m_TextGeneralUBO.Destroy();
   m_TextGlyphUBO.Destroy();
@@ -583,15 +538,13 @@ void VulkanTextRenderer::BeginText(const TextPrintState &textstate)
 
   VkPipeline pipe = m_TextPipeline[0];
 
-  if(textstate.fmt == VK_FORMAT_R8G8B8A8_UNORM)
-    pipe = m_TextPipeline[1];
-  else if(textstate.fmt == VK_FORMAT_B8G8R8A8_SRGB)
-    pipe = m_TextPipeline[2];
-  else if(textstate.fmt == VK_FORMAT_B8G8R8A8_UNORM)
-    pipe = m_TextPipeline[3];
+  int idx = BBFormats.indexOf(textstate.fmt);
+  if(idx >= 0)
+    pipe = m_TextPipeline[idx];
+  else
+    RDCERR("Unexpected backbuffer format %s", ToStr(textstate.fmt).c_str());
 
-  ObjDisp(textstate.cmd)
-      ->CmdBindPipeline(Unwrap(textstate.cmd), VK_PIPELINE_BIND_POINT_GRAPHICS, Unwrap(pipe));
+  ObjDisp(textstate.cmd)->CmdBindPipeline(Unwrap(textstate.cmd), VK_PIPELINE_BIND_POINT_GRAPHICS, pipe);
 
   VkViewport viewport = {0.0f, 0.0f, (float)textstate.w, (float)textstate.h, 0.0f, 1.0f};
   ObjDisp(textstate.cmd)->CmdSetViewport(Unwrap(textstate.cmd), 0, 1, &viewport);
@@ -648,7 +601,7 @@ void VulkanTextRenderer::RenderTextInternal(const TextPrintState &textstate, flo
 
   ObjDisp(textstate.cmd)
       ->CmdBindDescriptorSets(Unwrap(textstate.cmd), VK_PIPELINE_BIND_POINT_GRAPHICS,
-                              Unwrap(m_TextPipeLayout), 0, 1, UnwrapPtr(m_TextDescSet), 2, offsets);
+                              m_TextPipeLayout, 0, 1, &m_TextDescSet, 2, offsets);
 
   ObjDisp(textstate.cmd)->CmdDraw(Unwrap(textstate.cmd), 6 * (uint32_t)len, 1, 0, 0);
 }

renderdoc/driver/vulkan/vk_rendertext.h

@@ -53,7 +53,6 @@ private:
   static const uint32_t FONT_TEX_WIDTH = 256;
   static const uint32_t FONT_TEX_HEIGHT = 128;
 
-  WrappedVulkan *m_pDriver = NULL;
   VkDevice m_Device = VK_NULL_HANDLE;
 
   float m_FontCharAspect = 1.0f;
@@ -63,8 +62,8 @@ private:
   VkPipelineLayout m_TextPipeLayout = VK_NULL_HANDLE;
   VkDescriptorSet m_TextDescSet = VK_NULL_HANDLE;
 
-  // 0 - RGBA8_SRGB, 1 - RGBA8, 2 - BGRA8_SRGB, 3 - BGRA8
-  VkPipeline m_TextPipeline[4] = {VK_NULL_HANDLE};
+  static const int NUM_BB_FORMATS = 7;
+  VkPipeline m_TextPipeline[NUM_BB_FORMATS] = {};
 
   VkSampler m_LinearSampler = VK_NULL_HANDLE;
   VkDescriptorPool m_DescriptorPool = VK_NULL_HANDLE;

renderdoc/driver/vulkan/wrappers/vk_device_funcs.cpp

@@ -4298,10 +4298,6 @@ bool WrappedVulkan::Serialise_vkCreateDevice(SerialiserType &ser, VkPhysicalDevi
 
     m_ShaderCache = new VulkanShaderCache(this);
 
-    m_DebugManager = new VulkanDebugManager(this);
-
-    m_Replay->CreateResources();
-
     SetDebugMessageSink(sink);
   }
 

renderdoc/driver/vulkan/wrappers/vk_resource_funcs.cpp

@@ -318,8 +318,19 @@ bool WrappedVulkan::Serialise_vkAllocateMemory(SerialiserType &ser, VkDevice dev
 
         if(mrq.size != AllocateInfo.allocationSize)
         {
-          RDCDEBUG("Removing dedicated allocation for incompatible size");
-          RemoveNextStruct(&patched, VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO);
+          RDCDEBUG("Patching dedicated allocation for incompatible size");
+
+          // if acceleration structures are used, we promote all non-dedicated memory to be BDA as
+          // we can't know if it will be used for an AS or not during capture. That means that
+          // during self-capture if we just remove the dedicated allocation structure here without
+          // any other changes the self-capture layer will promote it to BDA and potentially cause
+          // clashes with reserved addresses elsewhere.
+          // instead we do the more dangerous thing of adjusting the allocation size to match the
+          // image's memory requirements and keep the dedicated allocation.
+          if(AccelerationStructures())
+            patched.allocationSize = mrq.size;
+          else
+            RemoveNextStruct(&patched, VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO);
         }
       }
     }

renderdoc/driver/vulkan/wrappers/vk_wsi_funcs.cpp

@@ -429,6 +429,26 @@ bool WrappedVulkan::Serialise_vkCreateSwapchainKHR(SerialiserType &ser, VkDevice
           GetGPULocalMemoryIndex(mrq.memoryTypeBits),
       };
 
+      VkMemoryDedicatedAllocateInfo dedicated = {
+          VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO,
+          NULL,
+          Unwrap(im),
+      };
+
+      // if the acceleration structures feature is enabled, we _must_ be on at least vulkan 1.1 (the
+      // extension requires it). Vulkan 1.1 unconditionally allows the use of dedicated image
+      // allocations without any feature bits (the original extension didn't have any either).
+      //
+      // we do this because without it, the extra memory allocation may be promoted to BDA by
+      // self-capturing and cause problems with address space clashes. We don't need the dedicated
+      // allocation but it avoids that behaviour as it may not be legal to query the address of a
+      // dedicated image memory allocation and in any case we know that it can't be legally used for
+      // BDA or AS backing memory
+      if(AccelerationStructures())
+      {
+        allocInfo.pNext = &dedicated;
+      }
+
       vkr = ObjDisp(device)->AllocateMemory(Unwrap(device), &allocInfo, NULL, &mem);
       CHECK_VKR(this, vkr);