Vulkan Pixel History: secondary command buffers

Also:
- support for choosing mip level and slice
- tests for secondary command buffers
- adds a callback around vkCmdExecuteCommands
- refactors pixel history occlusion callback into its own
callback. Allows processing fewer events later on, and getting colour
information separately.
- keep track of subpassContents for vkCmdBeginRenderPass (inline or
  secondary)

renderdoc/driver/vulkan/vk_core.h

@@ -237,6 +237,20 @@ struct VulkanDrawcallCallback
   // callbacks above for the first EID, then call this function for the others
   // to indicate that they are the same.
   virtual void AliasEvent(uint32_t primary, uint32_t alias) = 0;
+
+  // Returns true if vkCmdExecuteCommands should be called separately for every
+  // command buffer in pCommandBuffers.
+  virtual bool SplitSecondary() = 0;
+
+  // called before vkCmdExecuteCommands with a range for the draws inside the
+  // secondary command buffer.
+  virtual void PreCmdExecute(uint32_t baseEid, uint32_t secondaryFirst, uint32_t secondaryLast,
+                             VkCommandBuffer cmd) = 0;
+
+  // called after vkCmdExecuteCommands with a range for the draw inside the
+  // seocndary command buffer.
+  virtual void PostCmdExecute(uint32_t baseEid, uint32_t secondaryFirst, uint32_t secondaryLast,
+                              VkCommandBuffer cmd) = 0;
 };
 
 class WrappedVulkan : public IFrameCapturer
@@ -925,6 +939,14 @@ public:
     return record ? *record->instDevInfo : m_EnabledExtensions;
   }
 
+  bool IsCmdPrimary()
+  {
+    RDCASSERT(m_LastCmdBufferID != ResourceId());
+    auto it = m_BakedCmdBufferInfo.find(m_LastCmdBufferID);
+    RDCASSERT(it != m_BakedCmdBufferInfo.end());
+    return it->second.level == VK_COMMAND_BUFFER_LEVEL_PRIMARY;
+  }
+
   VulkanRenderState &GetCmdRenderState()
   {
     RDCASSERT(m_LastCmdBufferID != ResourceId());

renderdoc/driver/vulkan/vk_counters.cpp

@@ -400,6 +400,15 @@ struct VulkanAMDDrawCallback : public VulkanDrawcallCallback
   {
     m_AliasEvents.push_back(make_rdcpair(primary, alias));
   }
+  bool SplitSecondary() override { return false; }
+  void PreCmdExecute(uint32_t baseEid, uint32_t secondaryFirst, uint32_t secondaryLast,
+                     VkCommandBuffer cmd) override
+  {
+  }
+  void PostCmdExecute(uint32_t baseEid, uint32_t secondaryFirst, uint32_t secondaryLast,
+                      VkCommandBuffer cmd) override
+  {
+  }
 
   uint32_t *m_pSampleId;
   WrappedVulkan *m_pDriver;
@@ -556,6 +565,15 @@ struct VulkanKHRCallback : public VulkanDrawcallCallback
   {
     m_AliasEvents.push_back(std::make_pair(primary, alias));
   }
+  bool SplitSecondary() override { return false; }
+  void PreCmdExecute(uint32_t baseEid, uint32_t secondaryFirst, uint32_t secondaryLast,
+                     VkCommandBuffer cmd) override
+  {
+  }
+  void PostCmdExecute(uint32_t baseEid, uint32_t secondaryFirst, uint32_t secondaryLast,
+                      VkCommandBuffer cmd) override
+  {
+  }
 
   void PreEndCommandBuffer(VkCommandBuffer cmd) override {}
   WrappedVulkan *m_pDriver;
@@ -758,6 +776,15 @@ struct VulkanGPUTimerCallback : public VulkanDrawcallCallback
   {
     m_AliasEvents.push_back(make_rdcpair(primary, alias));
   }
+  bool SplitSecondary() override { return false; }
+  void PreCmdExecute(uint32_t baseEid, uint32_t secondaryFirst, uint32_t secondaryLast,
+                     VkCommandBuffer cmd) override
+  {
+  }
+  void PostCmdExecute(uint32_t baseEid, uint32_t secondaryFirst, uint32_t secondaryLast,
+                      VkCommandBuffer cmd) override
+  {
+  }
 
   void PreEndCommandBuffer(VkCommandBuffer cmd) override {}
   WrappedVulkan *m_pDriver;

renderdoc/driver/vulkan/vk_overlay.cpp

@@ -255,6 +255,15 @@ struct VulkanQuadOverdrawCallback : public VulkanDrawcallCallback
   {
     // don't care
   }
+  bool SplitSecondary() { return false; }
+  void PreCmdExecute(uint32_t baseEid, uint32_t secondaryFirst, uint32_t secondaryLast,
+                     VkCommandBuffer cmd)
+  {
+  }
+  void PostCmdExecute(uint32_t baseEid, uint32_t secondaryFirst, uint32_t secondaryLast,
+                      VkCommandBuffer cmd)
+  {
+  }
 
   WrappedVulkan *m_pDriver;
   VkDescriptorSetLayout m_DescSetLayout;

renderdoc/driver/vulkan/vk_pixelhistory.cpp

@@ -70,6 +70,8 @@ struct CopyPixelParams
   VkFormat srcImageFormat;
   VkImageLayout srcImageLayout;
   VkOffset3D imageOffset;
+  uint32_t slice;
+  uint32_t mipLevel;
 
   VkBuffer dstBuffer;
 };
@@ -673,35 +675,207 @@ protected:
   rdcarray<VkFramebuffer> m_FbsToDestroy;
 };
 
-struct VulkanOcclusionAndStencilCallback : public VulkanPixelHistoryCallback
+// VulkanOcclusionCallback callback is used to determine which draw events might have
+// modified the pixel by doing an occlusion query.
+struct VulkanOcclusionCallback : public VulkanPixelHistoryCallback
 {
-  VulkanOcclusionAndStencilCallback(WrappedVulkan *vk, PixelHistoryShaderCache *shaderCache,
-                                    uint32_t x, uint32_t y, VkImage image, VkFormat format,
-                                    uint32_t sampleMask, VkQueryPool occlusionPool,
-                                    VkImageView stencilImageView, VkImage stencilImage,
-                                    VkBuffer dstBuffer, const rdcarray<EventUsage> &events)
-      : VulkanPixelHistoryCallback(vk, shaderCache, x, y, sampleMask, occlusionPool),
-        m_Image(image),
-        m_Format(format),
-        m_DstBuffer(dstBuffer),
-        m_StencilImageView(stencilImageView),
-        m_StencilImage(stencilImage)
+  VulkanOcclusionCallback(WrappedVulkan *vk, PixelHistoryShaderCache *shaderCache, VkImage image,
+                          uint32_t x, uint32_t y, uint32_t sampleMask, VkQueryPool occlusionPool,
+                          const rdcarray<EventUsage> &allEvents)
+      : VulkanPixelHistoryCallback(vk, shaderCache, x, y, sampleMask, occlusionPool), m_Image(image)
   {
-    for(size_t i = 0; i < events.size(); i++)
-      m_Events.insert(std::make_pair(events[i].eventId, events[i]));
+    for(size_t i = 0; i < allEvents.size(); i++)
+      m_Events.push_back(allEvents[i].eventId);
+  }
+
+  ~VulkanOcclusionCallback()
+  {
+    for(auto it = m_PipeCache.begin(); it != m_PipeCache.end(); ++it)
+      m_pDriver->vkDestroyPipeline(m_pDriver->GetDev(), it->second, NULL);
   }
 
   void PreDraw(uint32_t eid, VkCommandBuffer cmd)
   {
-    auto it = m_Events.find(eid);
-    if(it == m_Events.end())
+    if(!m_Events.contains(eid))
       return;
-    EventUsage event = it->second;
     VulkanRenderState prevState = m_pDriver->GetCmdRenderState();
+    VulkanRenderState &pipestate = m_pDriver->GetCmdRenderState();
+    const VulkanCreationInfo::Pipeline &p =
+        m_pDriver->GetDebugManager()->GetPipelineInfo(pipestate.graphics.pipeline);
+
+    uint32_t framebufferIndex = 0;
+    const rdcarray<ResourceId> &atts = pipestate.GetFramebufferAttachments();
+    for(uint32_t i = 0; i < atts.size(); i++)
+    {
+      ResourceId img = m_pDriver->GetDebugManager()->GetImageViewInfo(atts[i]).image;
+      if(img == GetResID(m_Image))
+      {
+        framebufferIndex = i;
+        break;
+      }
+    }
+    VkPipeline pipe = GetPixelOcclusionPipeline(eid, prevState.graphics.pipeline, framebufferIndex);
+    if(p.dynamicStates[VkDynamicScissor])
+      for(uint32_t i = 0; i < pipestate.views.size(); i++)
+        ScissorToPixel(pipestate.views[i], pipestate.scissors[i]);
+    pipestate.graphics.pipeline = GetResID(pipe);
+    ReplayDrawWithQuery(cmd, eid);
+
+    m_pDriver->GetCmdRenderState() = prevState;
+    m_pDriver->GetCmdRenderState().BindPipeline(m_pDriver, cmd, VulkanRenderState::BindGraphics,
+                                                true);
+  }
+
+  bool PostDraw(uint32_t eid, VkCommandBuffer cmd) { return false; }
+  void PostRedraw(uint32_t eid, VkCommandBuffer cmd) {}
+  void PreDispatch(uint32_t eid, VkCommandBuffer cmd) { return; }
+  bool PostDispatch(uint32_t eid, VkCommandBuffer cmd) { return false; }
+  void PostRedispatch(uint32_t eid, VkCommandBuffer cmd) {}
+  void PreMisc(uint32_t eid, DrawFlags flags, VkCommandBuffer cmd) { return; }
+  bool PostMisc(uint32_t eid, DrawFlags flags, VkCommandBuffer cmd) { return false; }
+  void PostRemisc(uint32_t eid, DrawFlags flags, VkCommandBuffer cmd) {}
+  void PreEndCommandBuffer(VkCommandBuffer cmd) {}
+  void AliasEvent(uint32_t primary, uint32_t alias) {}
+  bool SplitSecondary() { return false; }
+  void PreCmdExecute(uint32_t baseEid, uint32_t secondaryFirst, uint32_t secondaryLast,
+                     VkCommandBuffer cmd)
+  {
+  }
+  void PostCmdExecute(uint32_t baseEid, uint32_t secondaryFirst, uint32_t secondaryLast,
+                      VkCommandBuffer cmd)
+  {
+  }
+
+  void FetchOcclusionResults()
+  {
+    if(m_OcclusionQueries.size() == 0)
+      return;
+
+    m_OcclusionResults.resize(m_OcclusionQueries.size());
+    VkResult vkr = ObjDisp(m_pDriver->GetDev())
+                       ->GetQueryPoolResults(Unwrap(m_pDriver->GetDev()), m_OcclusionPool, 0,
+                                             (uint32_t)m_OcclusionResults.size(),
+                                             m_OcclusionResults.byteSize(),
+                                             m_OcclusionResults.data(), sizeof(uint64_t),
+                                             VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WAIT_BIT);
+    RDCASSERTEQUAL(vkr, VK_SUCCESS);
+  }
+
+  uint64_t GetOcclusionResult(uint32_t eventId)
+  {
+    auto it = m_OcclusionQueries.find(eventId);
+    if(it == m_OcclusionQueries.end())
+      return 0;
+    RDCASSERT(it->second < m_OcclusionResults.size());
+    return m_OcclusionResults[it->second];
+  }
+
+private:
+  // ReplayDrawWithQuery binds the pipeline in the current state, and replays a single
+  // draw with an occlusion query.
+  void ReplayDrawWithQuery(VkCommandBuffer cmd, uint32_t eventId)
+  {
+    const DrawcallDescription *drawcall = m_pDriver->GetDrawcall(eventId);
+    m_pDriver->GetCmdRenderState().BindPipeline(m_pDriver, cmd, VulkanRenderState::BindGraphics,
+                                                true);
+
+    uint32_t occlIndex = (uint32_t)m_OcclusionQueries.size();
+    ObjDisp(cmd)->CmdBeginQuery(Unwrap(cmd), m_OcclusionPool, occlIndex, 0);
+
+    if(drawcall->flags & DrawFlags::Indexed)
+      ObjDisp(cmd)->CmdDrawIndexed(Unwrap(cmd), drawcall->numIndices, drawcall->numInstances,
+                                   drawcall->indexOffset, drawcall->baseVertex,
+                                   drawcall->instanceOffset);
+    else
+      ObjDisp(cmd)->CmdDraw(Unwrap(cmd), drawcall->numIndices, drawcall->numInstances,
+                            drawcall->vertexOffset, drawcall->instanceOffset);
+
+    ObjDisp(cmd)->CmdEndQuery(Unwrap(cmd), m_OcclusionPool, occlIndex);
+    m_OcclusionQueries.insert(std::make_pair(eventId, occlIndex));
+  }
+
+  VkPipeline GetPixelOcclusionPipeline(uint32_t eid, ResourceId pipeline, uint32_t framebufferIndex)
+  {
+    auto it = m_PipeCache.find(pipeline);
+    if(it != m_PipeCache.end())
+      return it->second;
+
+    VkGraphicsPipelineCreateInfo pipeCreateInfo = {};
+    rdcarray<VkPipelineShaderStageCreateInfo> stages;
+    MakeAllPassIncrementStencilPipelineCI(eid, pipeline, pipeCreateInfo, stages);
+    {
+      // We just need to determine if something attempted to write to pixel.
+      // Disable actual color modifications.
+      VkPipelineColorBlendStateCreateInfo *cbs =
+          (VkPipelineColorBlendStateCreateInfo *)pipeCreateInfo.pColorBlendState;
+      VkPipelineColorBlendAttachmentState *atts =
+          (VkPipelineColorBlendAttachmentState *)cbs->pAttachments;
+      for(uint32_t i = 0; i < cbs->attachmentCount; i++)
+        atts[i].colorWriteMask = 0;
+    }
+    for(uint32_t i = 0; i < pipeCreateInfo.stageCount; i++)
+    {
+      if(stages[i].stage == VK_SHADER_STAGE_FRAGMENT_BIT)
+      {
+        stages[i].module = m_ShaderCache->GetFixedColShader(framebufferIndex);
+        stages[i].pName = "main";
+        break;
+      }
+    }
+    VkPipeline pipe;
+    VkResult vkr = m_pDriver->vkCreateGraphicsPipelines(m_pDriver->GetDev(), VK_NULL_HANDLE, 1,
+                                                        &pipeCreateInfo, NULL, &pipe);
+    RDCASSERTEQUAL(vkr, VK_SUCCESS);
+    m_PipeCache.insert(std::make_pair(pipeline, pipe));
+    return pipe;
+  }
+
+private:
+  VkImage m_Image;
+  std::map<ResourceId, VkPipeline> m_PipeCache;
+  rdcarray<uint32_t> m_Events;
+  // Key is event ID, and value is an index of where the occlusion result.
+  std::map<uint32_t, uint32_t> m_OcclusionQueries;
+  rdcarray<uint64_t> m_OcclusionResults;
+};
+
+struct VulkanColorAndStencilCallback : public VulkanPixelHistoryCallback
+{
+  VulkanColorAndStencilCallback(WrappedVulkan *vk, PixelHistoryShaderCache *shaderCache, uint32_t x,
+                                uint32_t y, VkImage image, VkFormat format, const Subresource &sub,
+                                uint32_t sampleMask, VkImageView stencilImageView,
+                                VkImage stencilImage, VkBuffer dstBuffer,
+                                const rdcarray<uint32_t> &events)
+      : VulkanPixelHistoryCallback(vk, shaderCache, x, y, sampleMask, VK_NULL_HANDLE),
+        m_Image(image),
+        m_Format(format),
+        m_DstBuffer(dstBuffer),
+        m_StencilImageView(stencilImageView),
+        m_StencilImage(stencilImage),
+        m_Subresource(sub),
+        m_Events(events)
+  {
+  }
+
+  ~VulkanColorAndStencilCallback()
+  {
+    for(auto it = m_PipeCache.begin(); it != m_PipeCache.end(); ++it)
+    {
+      m_pDriver->vkDestroyPipeline(m_pDriver->GetDev(), it->second.fixedShaderStencil, NULL);
+      m_pDriver->vkDestroyPipeline(m_pDriver->GetDev(), it->second.originalShaderStencil, NULL);
+    }
+  }
+
+  void PreDraw(uint32_t eid, VkCommandBuffer cmd)
+  {
+    if(!m_Events.contains(eid) || !m_pDriver->IsCmdPrimary())
+      return;
 
-    // TODO: handle secondary command buffers.
     // TODO: can't end renderpass if we are not on the last subpass.
-    m_pDriver->GetCmdRenderState().EndRenderPass(cmd);
+    VulkanRenderState prevState = m_pDriver->GetCmdRenderState();
+    VulkanRenderState &pipestate = m_pDriver->GetCmdRenderState();
+
+    pipestate.EndRenderPass(cmd);
 
     // Get pre-modification values
     size_t storeOffset = m_EventIndices.size() * sizeof(EventInfo);
@@ -718,12 +892,10 @@ struct VulkanOcclusionAndStencilCallback : public VulkanPixelHistoryCallback
 
     CopyPixel(m_Image, m_Format, depthImage, depthFormat, cmd, storeOffset);
 
-    VulkanRenderState &pipestate = m_pDriver->GetCmdRenderState();
     ResourceId prevRenderpass = pipestate.renderPass;
     ResourceId prevFramebuffer = pipestate.GetFramebuffer();
     rdcarray<ResourceId> prevFBattachments = pipestate.GetFramebufferAttachments();
-    const VulkanCreationInfo::Pipeline &p =
-        m_pDriver->GetDebugManager()->GetPipelineInfo(pipestate.graphics.pipeline);
+
     uint32_t prevSubpass = pipestate.subpass;
 
     {
@@ -745,23 +917,20 @@ struct VulkanOcclusionAndStencilCallback : public VulkanPixelHistoryCallback
       PipelineReplacements replacements =
           GetPipelineReplacements(eid, pipestate.graphics.pipeline, newRp, framebufferIndex);
 
-      if(p.dynamicStates[VkDynamicViewport])
+      const VulkanCreationInfo::Pipeline &p =
+          m_pDriver->GetDebugManager()->GetPipelineInfo(pipestate.graphics.pipeline);
+      if(p.dynamicStates[VkDynamicScissor])
         for(uint32_t i = 0; i < pipestate.views.size(); i++)
           ScissorToPixel(pipestate.views[i], pipestate.scissors[i]);
 
-      // Replay the draw with a fixed color shader that never discards, stencil
-      // increment to count number of fragments, and an occlusion query around
-      // the draw. We will get occlusion data to figure out if anything wrote to
-      // the pixel, as well as number of fragments not accounting for potential
-      // shader discard.
+      // Replay the draw with a fixed color shader that never discards, and stencil
+      // increment to count number of fragments. We will get the number of fragments
+      // not accounting for shader discard.
       pipestate.SetFramebuffer(m_pDriver, GetResID(newFb));
       pipestate.renderPass = GetResID(newRp);
       pipestate.subpass = 0;
       pipestate.graphics.pipeline = GetResID(replacements.fixedShaderStencil);
-      ReplayDraw(cmd, m_OcclusionQueries.size(), eid, true, true);
-
-      m_OcclusionQueries.insert(
-          std::pair<uint32_t, uint32_t>(eid, (uint32_t)m_OcclusionQueries.size()));
+      ReplayDraw(cmd, eid, true);
 
       CopyPixelParams params = {};
       params.multisampled = false;
@@ -774,6 +943,8 @@ struct VulkanOcclusionAndStencilCallback : public VulkanPixelHistoryCallback
       params.dstBuffer = m_DstBuffer;
       params.depthCopy = true;
       params.stencilOnly = true;
+      params.mipLevel = m_Subresource.mip;
+      params.slice = m_Subresource.slice;
       // Copy stencil value that indicates the number of fragments ignoring
       // shader discard.
       m_pDriver->GetDebugManager()->PixelHistoryCopyPixel(
@@ -782,7 +953,7 @@ struct VulkanOcclusionAndStencilCallback : public VulkanPixelHistoryCallback
       // Replay the draw with the original fragment shader to get the actual number
       // of fragments, accounting for potential shader discard.
       pipestate.graphics.pipeline = GetResID(replacements.originalShaderStencil);
-      ReplayDraw(cmd, 0, eid, false, true);
+      ReplayDraw(cmd, eid, true);
 
       m_pDriver->GetDebugManager()->PixelHistoryCopyPixel(
           cmd, params, storeOffset + offsetof(struct EventInfo, dsWithShaderDiscard));
@@ -801,7 +972,7 @@ struct VulkanOcclusionAndStencilCallback : public VulkanPixelHistoryCallback
 
   bool PostDraw(uint32_t eid, VkCommandBuffer cmd)
   {
-    if(m_Events.find(eid) == m_Events.end())
+    if(!m_Events.contains(eid) || !m_pDriver->IsCmdPrimary())
       return false;
 
     m_pDriver->GetCmdRenderState().EndRenderPass(cmd);
@@ -834,9 +1005,74 @@ struct VulkanOcclusionAndStencilCallback : public VulkanPixelHistoryCallback
     // nothing to do
   }
 
+  void PreCmdExecute(uint32_t baseEid, uint32_t secondaryFirst, uint32_t secondaryLast,
+                     VkCommandBuffer cmd)
+  {
+    uint32_t eventId = 0;
+    if(m_Events.size() == 0)
+      return;
+    for(size_t i = 0; i < m_Events.size(); i++)
+    {
+      // Find the first event in range
+      if(m_Events[i] >= secondaryFirst && m_Events[i] <= secondaryLast)
+      {
+        eventId = m_Events[i];
+        break;
+      }
+    }
+    if(eventId == 0)
+      return;
+
+    m_pDriver->GetCmdRenderState().EndRenderPass(cmd);
+
+    // Copy
+    size_t storeOffset = m_EventIndices.size() * sizeof(EventInfo);
+    CopyPixel(m_Image, m_Format, VK_NULL_HANDLE, VK_FORMAT_UNDEFINED, cmd, storeOffset);
+    m_EventIndices.insert(std::make_pair(eventId, m_EventIndices.size()));
+
+    m_pDriver->GetCmdRenderState().BeginRenderPassAndApplyState(m_pDriver, cmd,
+                                                                VulkanRenderState::BindNone);
+  }
+
+  void PostCmdExecute(uint32_t baseEid, uint32_t secondaryFirst, uint32_t secondaryLast,
+                      VkCommandBuffer cmd)
+  {
+    uint32_t eventId = 0;
+    if(m_Events.size() == 0)
+      return;
+    for(int32_t i = (int32_t)m_Events.size() - 1; i >= 0; i--)
+    {
+      // Find the last event in range.
+      if(m_Events[i] >= secondaryFirst && m_Events[i] <= secondaryLast)
+      {
+        eventId = m_Events[i];
+        break;
+      }
+    }
+    if(eventId == 0)
+      return;
+
+    m_pDriver->GetCmdRenderState().EndRenderPass(cmd);
+    size_t storeOffset = 0;
+    auto it = m_EventIndices.find(eventId);
+    if(it != m_EventIndices.end())
+    {
+      storeOffset = it->second * sizeof(EventInfo);
+    }
+    else
+    {
+      storeOffset = m_EventIndices.size() * sizeof(EventInfo);
+      m_EventIndices.insert(std::make_pair(eventId, m_EventIndices.size()));
+    }
+    CopyPixel(m_Image, m_Format, VK_NULL_HANDLE, VK_FORMAT_UNDEFINED, cmd,
+              storeOffset + offsetof(struct EventInfo, postmod));
+    m_pDriver->GetCmdRenderState().BeginRenderPassAndApplyState(m_pDriver, cmd,
+                                                                VulkanRenderState::BindNone);
+  }
+
   void PreDispatch(uint32_t eid, VkCommandBuffer cmd)
   {
-    if(m_Events.find(eid) == m_Events.end())
+    if(!m_Events.contains(eid))
       return;
 
     size_t storeOffset = m_EventIndices.size() * sizeof(EventInfo);
@@ -844,7 +1080,7 @@ struct VulkanOcclusionAndStencilCallback : public VulkanPixelHistoryCallback
   }
   bool PostDispatch(uint32_t eid, VkCommandBuffer cmd)
   {
-    if(m_Events.find(eid) == m_Events.end())
+    if(!m_Events.contains(eid))
       return false;
 
     size_t storeOffset = m_EventIndices.size() * sizeof(EventInfo);
@@ -857,7 +1093,7 @@ struct VulkanOcclusionAndStencilCallback : public VulkanPixelHistoryCallback
   void PreMisc(uint32_t eid, DrawFlags flags, VkCommandBuffer cmd) { PreDispatch(eid, cmd); }
   bool PostMisc(uint32_t eid, DrawFlags flags, VkCommandBuffer cmd)
   {
-    if(m_Events.find(eid) == m_Events.end())
+    if(!m_Events.contains(eid))
       return false;
     if(flags & DrawFlags::BeginPass)
       m_pDriver->GetCmdRenderState().EndRenderPass(cmd);
@@ -866,46 +1102,31 @@ struct VulkanOcclusionAndStencilCallback : public VulkanPixelHistoryCallback
 
     if(flags & DrawFlags::BeginPass)
       m_pDriver->GetCmdRenderState().BeginRenderPassAndApplyState(m_pDriver, cmd,
-                                                                  VulkanRenderState::BindGraphics);
+                                                                  VulkanRenderState::BindNone);
 
     return ret;
   }
 
+  bool SplitSecondary() { return true; }
   void PostRemisc(uint32_t eid, DrawFlags flags, VkCommandBuffer cmd) {}
   void PreEndCommandBuffer(VkCommandBuffer cmd) {}
   void AliasEvent(uint32_t primary, uint32_t alias)
   {
-    // TODO: handle aliased events.
+    RDCWARN(
+        "Alised events are not supported, results might be inaccurate. Primary event id: %u, "
+        "alias: %u.",
+        primary, alias);
   }
 
-  void FetchOcclusionResults()
-  {
-    if(m_OcclusionQueries.size() == 0)
-      return;
-
-    m_OcclusionResults.resize(m_OcclusionQueries.size());
-    VkResult vkr = ObjDisp(m_pDriver->GetDev())
-                       ->GetQueryPoolResults(Unwrap(m_pDriver->GetDev()), m_OcclusionPool, 0,
-                                             (uint32_t)m_OcclusionResults.size(),
-                                             m_OcclusionResults.byteSize(),
-                                             m_OcclusionResults.data(), sizeof(uint64_t),
-                                             VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WAIT_BIT);
-    RDCASSERTEQUAL(vkr, VK_SUCCESS);
-  }
-
-  uint64_t GetOcclusionResult(uint32_t eventId)
-  {
-    auto it = m_OcclusionQueries.find(eventId);
-    RDCASSERT(it != m_OcclusionQueries.end());
-    RDCASSERT(it->second < m_OcclusionResults.size());
-    return m_OcclusionResults[it->second];
-  }
-
-  size_t GetEventIndex(uint32_t eventId)
+  int32_t GetEventIndex(uint32_t eventId)
   {
     auto it = m_EventIndices.find(eventId);
+    if(it == m_EventIndices.end())
+      // Most likely a secondary command buffer event for which there is no
+      // information.
+      return -1;
     RDCASSERT(it != m_EventIndices.end());
-    return it->second;
+    return (int32_t)it->second;
   }
 
 private:
@@ -922,6 +1143,8 @@ private:
     colourCopyParams.imageOffset.y = int32_t(m_Y);
     colourCopyParams.imageOffset.z = 0;
     colourCopyParams.dstBuffer = m_DstBuffer;
+    colourCopyParams.slice = m_Subresource.slice;
+    colourCopyParams.mipLevel = m_Subresource.mip;
 
     m_pDriver->GetDebugManager()->PixelHistoryCopyPixel(cmd, colourCopyParams, offset);
 
@@ -939,10 +1162,8 @@ private:
 
   // ReplayDraw begins renderpass, executes a single draw defined by the eventId and
   // ends the renderpass.
-  void ReplayDraw(VkCommandBuffer cmd, size_t eventIndex, int eventId, bool doQuery,
-                  bool clear = false)
+  void ReplayDraw(VkCommandBuffer cmd, uint32_t eventId, bool clear = false)
   {
-    const DrawcallDescription *drawcall = m_pDriver->GetDrawcall(eventId);
     m_pDriver->GetCmdRenderState().BeginRenderPassAndApplyState(m_pDriver, cmd,
                                                                 VulkanRenderState::BindGraphics);
 
@@ -960,9 +1181,7 @@ private:
       ObjDisp(cmd)->CmdClearAttachments(Unwrap(cmd), 1, &att, 1, &rect);
     }
 
-    if(doQuery)
-      ObjDisp(cmd)->CmdBeginQuery(Unwrap(cmd), m_OcclusionPool, (uint32_t)eventIndex, 0);
-
+    const DrawcallDescription *drawcall = m_pDriver->GetDrawcall(eventId);
     if(drawcall->flags & DrawFlags::Indexed)
       ObjDisp(cmd)->CmdDrawIndexed(Unwrap(cmd), drawcall->numIndices, drawcall->numInstances,
                                    drawcall->indexOffset, drawcall->baseVertex,
@@ -971,9 +1190,6 @@ private:
       ObjDisp(cmd)->CmdDraw(Unwrap(cmd), drawcall->numIndices, drawcall->numInstances,
                             drawcall->vertexOffset, drawcall->instanceOffset);
 
-    if(doQuery)
-      ObjDisp(cmd)->CmdEndQuery(Unwrap(cmd), m_OcclusionPool, (uint32_t)eventIndex);
-
     m_pDriver->GetCmdRenderState().EndRenderPass(cmd);
   }
 
@@ -987,8 +1203,8 @@ private:
     // which shaders need to be modified. Those flags are based on the shaders bound,
     // so in theory all events should share those flags if they are using the same
     // pipeline.
-    auto pipeIt = m_PipelineCache.find(pipeline);
-    if(pipeIt != m_PipelineCache.end())
+    auto pipeIt = m_PipeCache.find(pipeline);
+    if(pipeIt != m_PipeCache.end())
       return pipeIt->second;
 
     VkGraphicsPipelineCreateInfo pipeCreateInfo = {};
@@ -1030,7 +1246,7 @@ private:
                                                &replacements.fixedShaderStencil);
     RDCASSERTEQUAL(vkr, VK_SUCCESS);
 
-    m_PipelineCache.insert(std::make_pair(pipeline, replacements));
+    m_PipeCache.insert(std::make_pair(pipeline, replacements));
 
     return replacements;
   }
@@ -1038,17 +1254,15 @@ private:
   VkImage m_Image;
   VkFormat m_Format;
   VkBuffer m_DstBuffer;
+  Subresource m_Subresource;
 
   VkImageView m_StencilImageView;
   VkImage m_StencilImage;
 
-  std::map<ResourceId, PipelineReplacements> m_PipelineCache;
-  std::map<uint32_t, EventUsage> m_Events;
+  std::map<ResourceId, PipelineReplacements> m_PipeCache;
+  rdcarray<uint32_t> m_Events;
   // Key is event ID, and value is an index of where the event data is stored.
   std::map<uint32_t, size_t> m_EventIndices;
-  // Key is event ID, and value is an index of where the occlusion result.
-  std::map<uint32_t, uint32_t> m_OcclusionQueries;
-  rdcarray<uint64_t> m_OcclusionResults;
 };
 
 // TestsFailedCallback replays draws to figure out which tests failed (for ex., depth,
@@ -1120,6 +1334,15 @@ struct TestsFailedCallback : public VulkanPixelHistoryCallback
   void PreMisc(uint32_t eid, DrawFlags flags, VkCommandBuffer cmd) {}
   bool PostMisc(uint32_t eid, DrawFlags flags, VkCommandBuffer cmd) { return false; }
   void PostRemisc(uint32_t eid, DrawFlags flags, VkCommandBuffer cmd) {}
+  bool SplitSecondary() { return false; }
+  void PreCmdExecute(uint32_t baseEid, uint32_t secondaryFirst, uint32_t secondaryLast,
+                     VkCommandBuffer cmd)
+  {
+  }
+  void PostCmdExecute(uint32_t baseEid, uint32_t secondaryFirst, uint32_t secondaryLast,
+                      VkCommandBuffer cmd)
+  {
+  }
   void PreEndCommandBuffer(VkCommandBuffer cmd) {}
   uint32_t GetEventFlags(uint32_t eventId)
   {
@@ -1533,7 +1756,8 @@ private:
 struct VulkanPixelHistoryPerFragmentCallback : VulkanPixelHistoryCallback
 {
   VulkanPixelHistoryPerFragmentCallback(WrappedVulkan *vk, PixelHistoryShaderCache *shaderCache,
-                                        uint32_t x, uint32_t y, uint32_t sampleMask,
+                                        uint32_t x, uint32_t y, const Subresource &sub,
+                                        uint32_t sampleMask,
                                         std::map<uint32_t, uint32_t> eventFragments,
                                         VkImage originalImage, VkFormat srcFormat, VkBuffer dstBuffer,
                                         VkImage newImage, VkImageView newImageView, VkImage dsImage,
@@ -1542,6 +1766,7 @@ struct VulkanPixelHistoryPerFragmentCallback : VulkanPixelHistoryCallback
         m_EventFragments(eventFragments),
         m_Image(originalImage),
         m_SrcFormat(srcFormat),
+        m_Subresource(sub),
         m_DstBuffer(dstBuffer),
         m_PerFragmentImage(newImage),
         m_PerFragmentImageView(newImageView),
@@ -1617,6 +1842,8 @@ struct VulkanPixelHistoryPerFragmentCallback : VulkanPixelHistoryCallback
     colourCopyParams.imageOffset.y = int32_t(m_Y);
     colourCopyParams.imageOffset.z = 0;
     colourCopyParams.dstBuffer = m_DstBuffer;
+    colourCopyParams.slice = m_Subresource.slice;
+    colourCopyParams.mipLevel = m_Subresource.mip;
 
     const VulkanCreationInfo::Pipeline &p =
         m_pDriver->GetDebugManager()->GetPipelineInfo(prevState.graphics.pipeline);
@@ -1937,6 +2164,16 @@ struct VulkanPixelHistoryPerFragmentCallback : VulkanPixelHistoryCallback
   void PostRemisc(uint32_t eid, DrawFlags flags, VkCommandBuffer cmd) {}
   void PreEndCommandBuffer(VkCommandBuffer cmd) {}
   void AliasEvent(uint32_t primary, uint32_t alias) {}
+  bool SplitSecondary() { return false; }
+  void PreCmdExecute(uint32_t baseEid, uint32_t secondaryFirst, uint32_t secondaryLast,
+                     VkCommandBuffer cmd)
+  {
+  }
+  void PostCmdExecute(uint32_t baseEid, uint32_t secondaryFirst, uint32_t secondaryLast,
+                      VkCommandBuffer cmd)
+  {
+  }
+
   uint32_t GetEventOffset(uint32_t eid)
   {
     auto it = m_EventIndices.find(eid);
@@ -1962,6 +2199,7 @@ private:
   VkImage m_PerFragmentImage;
   VkImage m_StencilImage;
   VkImageView m_StencilImageView;
+  Subresource m_Subresource;
 
   rdcarray<VkPipeline> m_PipesToDestroy;
 };
@@ -2089,6 +2327,16 @@ struct VulkanPixelHistoryDiscardedFragmentsCallback : VulkanPixelHistoryCallback
   void PostRemisc(uint32_t eid, DrawFlags flags, VkCommandBuffer cmd) {}
   void PreEndCommandBuffer(VkCommandBuffer cmd) {}
   void AliasEvent(uint32_t primary, uint32_t alias) {}
+  bool SplitSecondary() { return false; }
+  void PreCmdExecute(uint32_t baseEid, uint32_t secondaryFirst, uint32_t secondaryLast,
+                     VkCommandBuffer cmd)
+  {
+  }
+  void PostCmdExecute(uint32_t baseEid, uint32_t secondaryFirst, uint32_t secondaryLast,
+                      VkCommandBuffer cmd)
+  {
+  }
+
 private:
   std::map<rdcpair<uint32_t, uint32_t>, uint32_t> m_OcclusionIndices;
   rdcarray<uint64_t> m_OcclusionResults;
@@ -2206,7 +2454,7 @@ bool VulkanDebugManager::PixelHistorySetupResources(PixelHistoryResources &resou
 
   vkr = ObjDisp(dev)->BeginCommandBuffer(Unwrap(cmd), &beginInfo);
   RDCASSERTEQUAL(vkr, VK_SUCCESS);
-
+  ObjDisp(cmd)->CmdFillBuffer(Unwrap(cmd), Unwrap(dstBuffer), 0, VK_WHOLE_SIZE, 0);
   colorImageState.InlineTransition(
       cmd, m_pDriver->m_QueueFamilyIdx, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, 0,
       VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, m_pDriver->GetImageTransitionInfo());
@@ -2266,19 +2514,22 @@ void VulkanDebugManager::PixelHistoryCopyPixel(VkCommandBuffer cmd, CopyPixelPar
   region.imageExtent.depth = 1U;
   if(!p.depthCopy)
   {
-    region.imageSubresource = VkImageSubresourceLayers{VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
+    region.imageSubresource =
+        VkImageSubresourceLayers{VK_IMAGE_ASPECT_COLOR_BIT, p.mipLevel, p.slice, 1};
     regions.push_back(region);
     aspectFlags = VkImageAspectFlags(VK_IMAGE_ASPECT_COLOR_BIT);
   }
   else if(p.stencilOnly)
   {
-    region.imageSubresource = VkImageSubresourceLayers{VK_IMAGE_ASPECT_STENCIL_BIT, 0, 0, 1};
+    region.imageSubresource =
+        VkImageSubresourceLayers{VK_IMAGE_ASPECT_STENCIL_BIT, p.mipLevel, p.slice, 1};
     aspectFlags = VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT;
     regions.push_back(region);
   }
   else
   {
-    region.imageSubresource = VkImageSubresourceLayers{VK_IMAGE_ASPECT_DEPTH_BIT, 0, 0, 1};
+    region.imageSubresource =
+        VkImageSubresourceLayers{VK_IMAGE_ASPECT_DEPTH_BIT, p.mipLevel, p.slice, 1};
     if(IsDepthOnlyFormat(p.srcImageFormat) || IsDepthAndStencilFormat(p.srcImageFormat))
     {
       regions.push_back(region);
@@ -2493,18 +2744,17 @@ rdcarray<PixelModification> VulkanReplay::PixelHistory(rdcarray<EventUsage> even
 
   PixelHistoryShaderCache *shaderCache = new PixelHistoryShaderCache(m_pDriver);
 
-  VulkanOcclusionAndStencilCallback cb(
-      m_pDriver, shaderCache, x, y, GetResourceManager()->GetCurrentHandle<VkImage>(target),
-      imginfo.format, sampleMask, occlusionPool, resources.stencilImageView, resources.stencilImage,
-      resources.dstBuffer, events);
+  VkImage targetImage = GetResourceManager()->GetCurrentHandle<VkImage>(target);
+  VulkanOcclusionCallback occlCb(m_pDriver, shaderCache, targetImage, x, y, sampleMask,
+                                 occlusionPool, events);
   m_pDriver->ReplayLog(0, events.back().eventId, eReplay_Full);
   m_pDriver->SubmitCmds();
   m_pDriver->FlushQ();
-
-  cb.FetchOcclusionResults();
+  occlCb.FetchOcclusionResults();
 
   // Gather all draw events that could have written to pixel for another replay pass,
   // to determine if these draws failed for some reason (for ex., depth test).
+  rdcarray<uint32_t> modEvents;
   rdcarray<uint32_t> drawEvents;
   for(size_t ev = 0; ev < events.size(); ev++)
   {
@@ -2516,14 +2766,28 @@ rdcarray<PixelModification> VulkanReplay::PixelHistory(rdcarray<EventUsage> even
       // TODO
     }
 
-    if(!directWrite && !clear)
+    if(directWrite || clear)
     {
-      uint64_t occlData = cb.GetOcclusionResult((uint32_t)events[ev].eventId);
+      modEvents.push_back(events[ev].eventId);
+    }
+    else
+    {
+      uint64_t occlData = occlCb.GetOcclusionResult((uint32_t)events[ev].eventId);
       if(occlData > 0)
+      {
         drawEvents.push_back(events[ev].eventId);
+        modEvents.push_back(events[ev].eventId);
+      }
     }
   }
 
+  VulkanColorAndStencilCallback cb(m_pDriver, shaderCache, x, y, targetImage, imginfo.format, sub,
+                                   sampleMask, resources.stencilImageView, resources.stencilImage,
+                                   resources.dstBuffer, modEvents);
+  m_pDriver->ReplayLog(0, events.back().eventId, eReplay_Full);
+  m_pDriver->SubmitCmds();
+  m_pDriver->FlushQ();
+
   // If there are any draw events, do another replay pass, in order to figure out
   // which tests failed for each draw event.
   TestsFailedCallback *tfCb = NULL;
@@ -2532,9 +2796,8 @@ rdcarray<PixelModification> VulkanReplay::PixelHistory(rdcarray<EventUsage> even
     VkQueryPool tfOcclusionPool;
     CreateOcclusionPool(m_pDriver, (uint32_t)drawEvents.size() * 6, &tfOcclusionPool);
 
-    tfCb = new TestsFailedCallback(m_pDriver, shaderCache, x, y,
-                                   GetResourceManager()->GetCurrentHandle<VkImage>(target),
-                                   sampleMask, tfOcclusionPool, drawEvents);
+    tfCb = new TestsFailedCallback(m_pDriver, shaderCache, x, y, targetImage, sampleMask,
+                                   tfOcclusionPool, drawEvents);
     m_pDriver->ReplayLog(0, events.back().eventId, eReplay_Full);
     m_pDriver->SubmitCmds();
     m_pDriver->FlushQ();
@@ -2589,7 +2852,14 @@ rdcarray<PixelModification> VulkanReplay::PixelHistory(rdcarray<EventUsage> even
   {
     PixelModification &mod = history[h];
 
-    const EventInfo &ei = eventsInfo[cb.GetEventIndex(mod.eventId)];
+    int32_t eventIndex = cb.GetEventIndex(mod.eventId);
+    if(eventIndex == -1)
+    {
+      // There is no information, skip the event.
+      h++;
+      continue;
+    }
+    const EventInfo &ei = eventsInfo[eventIndex];
     FillInColor(fmt, ei.premod, mod.preMod);
     FillInColor(fmt, ei.postmod, mod.postMod);
     mod.preMod.depth = ei.premod.depth.fdepth;
@@ -2603,6 +2873,8 @@ rdcarray<PixelModification> VulkanReplay::PixelHistory(rdcarray<EventUsage> even
     mod.shaderOut.col.intValue[1] = fragsClipped;
     bool someFragsClipped = (fragsClipped < frags);
     mod.primitiveID = someFragsClipped;
+    // Draws in secondary command buffers will fail this check,
+    // so nothing else needs to be checked in the callback itself.
     if(frags > 0)
       eventsWithFrags[mod.eventId] = frags;
 
@@ -2625,8 +2897,7 @@ rdcarray<PixelModification> VulkanReplay::PixelHistory(rdcarray<EventUsage> even
     // Replay to get shader output value, post modification value and primitive ID for every
     // fragment.
     VulkanPixelHistoryPerFragmentCallback perFragmentCB(
-        m_pDriver, shaderCache, x, y, sampleMask, eventsWithFrags,
-        GetResourceManager()->GetCurrentHandle<VkImage>(target), imginfo.format,
+        m_pDriver, shaderCache, x, y, sub, sampleMask, eventsWithFrags, targetImage, imginfo.format,
         resources.dstBuffer, resources.colorImage, resources.colorImageView, resources.stencilImage,
         resources.stencilImageView);
     m_pDriver->ReplayLog(0, eventsWithFrags.rbegin()->first, eReplay_Full);
@@ -2711,7 +2982,6 @@ rdcarray<PixelModification> VulkanReplay::PixelHistory(rdcarray<EventUsage> even
     }
   }
 
-  m_pDriver->vkUnmapMemory(dev, resources.bufferMemory);
   GetDebugManager()->PixelHistoryDestroyResources(resources);
   ObjDisp(dev)->DestroyQueryPool(Unwrap(dev), occlusionPool, NULL);
   delete shaderCache;

renderdoc/driver/vulkan/vk_postvs.cpp

@@ -3300,6 +3300,15 @@ struct VulkanInitPostVSCallback : public VulkanDrawcallCallback
     if(m_Events.contains(primary))
       m_pDriver->GetReplay()->AliasPostVSBuffers(primary, alias);
   }
+  bool SplitSecondary() { return false; }
+  void PreCmdExecute(uint32_t baseEid, uint32_t secondaryFirst, uint32_t secondaryLast,
+                     VkCommandBuffer cmd)
+  {
+  }
+  void PostCmdExecute(uint32_t baseEid, uint32_t secondaryFirst, uint32_t secondaryLast,
+                      VkCommandBuffer cmd)
+  {
+  }
 
   WrappedVulkan *m_pDriver;
   const rdcarray<uint32_t> &m_Events;

renderdoc/driver/vulkan/vk_state.cpp

@@ -77,7 +77,7 @@ void VulkanRenderState::BeginRenderPassAndApplyState(WrappedVulkan *vk, VkComman
     imagelessAttachments.pAttachments = imagelessViews.data();
   }
 
-  ObjDisp(cmd)->CmdBeginRenderPass(Unwrap(cmd), &rpbegin, VK_SUBPASS_CONTENTS_INLINE);
+  ObjDisp(cmd)->CmdBeginRenderPass(Unwrap(cmd), &rpbegin, subpassContents);
 
   BindPipeline(vk, cmd, binding, true);
 

renderdoc/driver/vulkan/vk_state.h

@@ -109,6 +109,7 @@ struct VulkanRenderState
 
   ResourceId renderPass;
   uint32_t subpass = 0;
+  VkSubpassContents subpassContents;
 
   // framebuffer accessors - to allow for imageless framebuffers and prevent accidentally changing
   // only the framebuffer without updating the attachments

renderdoc/driver/vulkan/wrappers/vk_cmd_funcs.cpp

@@ -854,8 +854,17 @@ bool WrappedVulkan::Serialise_vkBeginCommandBuffer(SerialiserType &ser, VkComman
         // add one-time submit flag as this partial cmd buffer will only be submitted once
         BeginInfo.flags |= VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
         if(AllocateInfo.level == VK_COMMAND_BUFFER_LEVEL_SECONDARY)
+        {
           BeginInfo.flags |= VK_COMMAND_BUFFER_USAGE_SIMULTANEOUS_USE_BIT;
 
+          if(BeginInfo.pInheritanceInfo->renderPass != VK_NULL_HANDLE)
+            m_BakedCmdBufferInfo[BakedCommandBuffer].state.renderPass =
+                GetResID(BeginInfo.pInheritanceInfo->renderPass);
+          if(BeginInfo.pInheritanceInfo->framebuffer != VK_NULL_HANDLE)
+            m_BakedCmdBufferInfo[BakedCommandBuffer].state.SetFramebuffer(
+                this, GetResID(BeginInfo.pInheritanceInfo->framebuffer));
+        }
+
         ObjDisp(cmd)->BeginCommandBuffer(Unwrap(cmd), &unwrappedBeginInfo);
       }
 
@@ -1278,6 +1287,7 @@ bool WrappedVulkan::Serialise_vkCmdBeginRenderPass(SerialiserType &ser, VkComman
           renderstate.subpass = 0;
           renderstate.renderPass = GetResID(RenderPassBegin.renderPass);
           renderstate.renderArea = RenderPassBegin.renderArea;
+          renderstate.subpassContents = contents;
 
           const VkRenderPassAttachmentBeginInfo *attachmentsInfo =
               (const VkRenderPassAttachmentBeginInfo *)FindNextStruct(
@@ -1656,6 +1666,7 @@ bool WrappedVulkan::Serialise_vkCmdEndRenderPass(SerialiserType &ser, VkCommandB
           VulkanRenderState &renderstate = GetCmdRenderState();
           renderstate.renderPass = ResourceId();
           renderstate.SetFramebuffer(ResourceId(), rdcarray<ResourceId>());
+          renderstate.subpassContents = VK_SUBPASS_CONTENTS_MAX_ENUM;
         }
 
         DrawFlags drawFlags = DrawFlags::PassBoundary | DrawFlags::EndPass;
@@ -1787,6 +1798,7 @@ bool WrappedVulkan::Serialise_vkCmdBeginRenderPass2(SerialiserType &ser,
           renderstate.subpass = 0;
           renderstate.renderPass = GetResID(RenderPassBegin.renderPass);
           renderstate.renderArea = RenderPassBegin.renderArea;
+          renderstate.subpassContents = SubpassBegin.contents;
 
           const VkRenderPassAttachmentBeginInfo *attachmentsInfo =
               (const VkRenderPassAttachmentBeginInfo *)FindNextStruct(
@@ -2161,6 +2173,7 @@ bool WrappedVulkan::Serialise_vkCmdEndRenderPass2(SerialiserType &ser, VkCommand
           VulkanRenderState &renderstate = GetCmdRenderState();
           renderstate.renderPass = ResourceId();
           renderstate.SetFramebuffer(ResourceId(), rdcarray<ResourceId>());
+          renderstate.subpassContents = VK_SUBPASS_CONTENTS_MAX_ENUM;
         }
 
         DrawFlags drawFlags = DrawFlags::PassBoundary | DrawFlags::EndPass;
@@ -3532,6 +3545,7 @@ bool WrappedVulkan::Serialise_vkCmdExecuteCommands(SerialiserType &ser, VkComman
               parentCmdBufInfo.state.GetFramebuffer(),
               parentCmdBufInfo.state.GetFramebufferAttachments());
           m_BakedCmdBufferInfo[cmd].state.renderArea = parentCmdBufInfo.state.renderArea;
+          m_BakedCmdBufferInfo[cmd].state.subpassContents = parentCmdBufInfo.state.subpassContents;
 
           // 2 extra for the virtual labels around the command buffer
           parentCmdBufInfo.curEventID += 2 + m_BakedCmdBufferInfo[cmd].eventCount;
@@ -3608,9 +3622,47 @@ bool WrappedVulkan::Serialise_vkCmdExecuteCommands(SerialiserType &ser, VkComman
 #endif
 
           if(!rerecordedCmds.empty())
-            ObjDisp(commandBuffer)
-                ->CmdExecuteCommands(Unwrap(commandBuffer), (uint32_t)rerecordedCmds.size(),
-                                     rerecordedCmds.data());
+          {
+            if(m_DrawcallCallback && m_DrawcallCallback->SplitSecondary())
+            {
+              DrawcallUse use(m_CurChunkOffset, 0);
+              auto it = std::lower_bound(m_DrawcallUses.begin(), m_DrawcallUses.end(), use);
+              if(it != m_DrawcallUses.end())
+              {
+                uint32_t eventId = it->eventId + 2;
+
+                for(uint32_t i = 0; i < (uint32_t)rerecordedCmds.size(); i++)
+                {
+                  ResourceId cmd = GetResourceManager()->GetOriginalID(GetResID(pCommandBuffers[i]));
+                  BakedCmdBufferInfo &info = m_BakedCmdBufferInfo[cmd];
+                  if(info.draw && info.draw->children.size() > 0)
+                  {
+                    uint32_t firstEventId = eventId + info.draw->children.front().draw.eventId;
+                    uint32_t lastEventId = eventId + info.draw->children.back().draw.eventId;
+                    m_DrawcallCallback->PreCmdExecute(eventId, firstEventId, lastEventId,
+                                                      commandBuffer);
+                    ObjDisp(commandBuffer)
+                        ->CmdExecuteCommands(Unwrap(commandBuffer), 1, &rerecordedCmds[i]);
+                    m_DrawcallCallback->PostCmdExecute(eventId, firstEventId, lastEventId,
+                                                       commandBuffer);
+                  }
+                  else
+                  {
+                    ObjDisp(commandBuffer)
+                        ->CmdExecuteCommands(Unwrap(commandBuffer), 1, &rerecordedCmds[i]);
+                  }
+
+                  eventId += 2 + m_BakedCmdBufferInfo[cmd].eventCount;
+                }
+              }
+            }
+            else
+            {
+              ObjDisp(commandBuffer)
+                  ->CmdExecuteCommands(Unwrap(commandBuffer), (uint32_t)rerecordedCmds.size(),
+                                       rerecordedCmds.data());
+            }
+          }
         }
       }
     }

util/test/demos/vk/vk_pixel_history_test.cpp

@@ -113,9 +113,9 @@ void main()
         {Vec3f(0.0f, 0.0f, 0.9f), Vec4f(1.0f, 0.0f, 0.0f, 1.0f), Vec2f(1.0f, 0.0f)},
 
         // this triangle is just in the background to contribute to overdraw
-        {Vec3f(-0.9f, 0.9f, 0.95f), Vec4f(0.1f, 0.1f, 0.1f, 1.0f), Vec2f(0.0f, 0.0f)},
-        {Vec3f(0.0f, -0.9f, 0.95f), Vec4f(0.1f, 0.1f, 0.1f, 1.0f), Vec2f(0.0f, 1.0f)},
-        {Vec3f(0.9f, 0.9f, 0.95f), Vec4f(0.1f, 0.1f, 0.1f, 1.0f), Vec2f(1.0f, 0.0f)},
+        {Vec3f(-0.9f, 0.9f, 0.95f), Vec4f(1.0f, 0.0f, 1.0f, 1.0f), Vec2f(0.0f, 0.0f)},
+        {Vec3f(0.0f, -0.9f, 0.95f), Vec4f(1.0f, 0.0f, 1.0f, 1.0f), Vec2f(0.0f, 1.0f)},
+        {Vec3f(0.9f, 0.9f, 0.95f), Vec4f(1.0f, 0.0f, 1.0f, 1.0f), Vec2f(1.0f, 0.0f)},
 
         // the draw has a few triangles, main one that is occluded for depth, another that is
         // adding to overdraw complexity, one that is backface culled, then a few more of various
@@ -255,14 +255,13 @@ void main()
 
     pipeCreateInfo.rasterizationState.cullMode = VK_CULL_MODE_FRONT_BIT;
     VkPipeline cullFrontPipe = createGraphicsPipeline(pipeCreateInfo);
+    pipeCreateInfo.rasterizationState.cullMode = VK_CULL_MODE_BACK_BIT;
 
     renderPassCreateInfo.attachments.pop_back();
     renderPassCreateInfo.subpasses[0].pDepthStencilAttachment = NULL;
 
     VkRenderPass subrp = createRenderPass(renderPassCreateInfo);
 
-    pipeCreateInfo.stages[1] =
-        CompileShaderModule(whitepixel, ShaderLang::glsl, ShaderStage::frag, "main");
     pipeCreateInfo.renderPass = subrp;
     pipeCreateInfo.depthStencilState.stencilTestEnable = VK_FALSE;
     pipeCreateInfo.depthStencilState.depthCompareOp = VK_COMPARE_OP_ALWAYS;
@@ -358,17 +357,36 @@ void main()
       s.extent.width /= 4;
       s.extent.height /= 4;
 
-      vkCmdSetViewport(cmd, 0, 1, &v);
-      vkCmdSetScissor(cmd, 0, 1, &s);
-
+      setMarker(cmd, "Begin RenderPass Secondary");
       vkCmdBeginRenderPass(
-          cmd, vkh::RenderPassBeginInfo(subrp, subfb, s, {vkh::ClearValue(0.0f, 0.0f, 0.0f, 1.0f)}),
-          VK_SUBPASS_CONTENTS_INLINE);
+          cmd, vkh::RenderPassBeginInfo(subrp, subfb, s, {vkh::ClearValue(0.f, 1.0f, 0.f, 1.0f)}),
+          VK_SUBPASS_CONTENTS_SECONDARY_COMMAND_BUFFERS);
 
-      vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, whitepipe);
+      std::vector<VkCommandBuffer> secondaries;
+      // Record the first secondary command buffer.
+      {
+        VkCommandBuffer cmd2 = GetCommandBuffer(VK_COMMAND_BUFFER_LEVEL_SECONDARY);
+        secondaries.push_back(cmd2);
+        vkBeginCommandBuffer(
+            cmd2, vkh::CommandBufferBeginInfo(VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT,
+                                              vkh::CommandBufferInheritanceInfo(subrp, 0, subfb)));
+        vkh::cmdBindVertexBuffers(cmd2, 0, {vb.buffer}, {0});
+        vkCmdBindPipeline(cmd2, VK_PIPELINE_BIND_POINT_GRAPHICS, whitepipe);
+        vkCmdSetViewport(cmd2, 0, 1, &v);
+        vkCmdSetScissor(cmd2, 0, 1, &s);
+        setMarker(cmd2, "Secondary: background");
+        vkCmdDraw(cmd2, 6, 1, 3, 0);
+        setMarker(cmd2, "Secondary: culled");
+        vkCmdDraw(cmd2, 6, 1, 12, 0);
+        setMarker(cmd2, "Secondary: pink");
+        vkCmdDraw(cmd2, 9, 1, 24, 0);
+        setMarker(cmd2, "Secondary: red and blue");
+        vkCmdDraw(cmd2, 6, 1, 0, 0);
+        vkEndCommandBuffer(cmd2);
+      }
 
-      setMarker(cmd, "Subresources");
-      vkCmdDraw(cmd, 24, 1, 9, 0);
+      setMarker(cmd, "Secondary Test");
+      vkCmdExecuteCommands(cmd, (uint32_t)secondaries.size(), secondaries.data());
 
       vkCmdEndRenderPass(cmd);
 

util/test/tests/Vulkan/Vk_Pixel_History.py

@@ -9,6 +9,7 @@ def depth_test_failed(x): return x.depthTestFailed
 def stencil_test_failed(x): return x.stencilTestFailed
 def shader_discarded(x): return x.shaderDiscarded
 def shader_out_col(x): return value_selector(x.shaderOut.col)
+def pre_mod_col(x): return value_selector(x.preMod.col)
 def post_mod_col(x): return value_selector(x.postMod.col)
 def primitive_id(x): return x.primitiveID
 
@@ -23,6 +24,10 @@ class VK_Pixel_History(rdtest.TestCase):
             rdtest.log.print("Vulkan pixel history not tested")
             return
 
+        self.primary_test()
+        self.secondary_cmd_test()
+
+    def primary_test(self):
         test_marker: rd.DrawcallDescription = self.find_draw("Test")
         self.controller.SetFrameEvent(test_marker.next.eventId, True)
 
@@ -30,8 +35,6 @@ class VK_Pixel_History(rdtest.TestCase):
 
         rt: rd.BoundResource = pipe.GetOutputTargets()[0]
 
-        vp: rd.Viewport = pipe.GetViewport(0)
-
         tex = rt.resourceId
         tex_details = self.get_texture(tex)
 
@@ -58,7 +61,7 @@ class VK_Pixel_History(rdtest.TestCase):
             [[event_id, background_eid], [depth_test_failed, True], [post_mod_col, (1.0, 0.0, 0.0, 1.0)]],
             [[event_id, test_eid], [stencil_test_failed, True]],
         ]
-        self.check_events(events, modifs)
+        self.check_events(events, modifs, False)
         self.check_pixel_value(tex, x, y, value_selector(modifs[-1].postMod.col), sub=sub, cast=rt.typeCast)
 
         x, y = 190, 150
@@ -71,7 +74,7 @@ class VK_Pixel_History(rdtest.TestCase):
             [[event_id, cull_eid], [culled, True]],
             [[event_id, test_eid], [depth_test_failed, True]],
         ]
-        self.check_events(events, modifs)
+        self.check_events(events, modifs, False)
         self.check_pixel_value(tex, x, y, value_selector(modifs[-1].postMod.col), sub=sub, cast=rt.typeCast)
 
         x, y = 200, 50
@@ -82,7 +85,7 @@ class VK_Pixel_History(rdtest.TestCase):
             [[event_id, background_eid], [passed, True]],
             [[event_id, test_eid], [passed, True], [primitive_id, 7]],
         ]
-        self.check_events(events, modifs)
+        self.check_events(events, modifs, False)
         self.check_pixel_value(tex, x, y, value_selector(modifs[-1].postMod.col), sub=sub, cast=rt.typeCast)
 
         x, y = 150, 250
@@ -92,13 +95,71 @@ class VK_Pixel_History(rdtest.TestCase):
             [[event_id, begin_renderpass_eid], [passed, True]],
             [[event_id, background_eid], [shader_discarded, True]],
         ]
-        self.check_events(events, modifs)
+        self.check_events(events, modifs, False)
         self.check_pixel_value(tex, x, y, value_selector(modifs[-1].postMod.col), sub=sub, cast=rt.typeCast)
 
-    def check_events(self, events, modifs):
-        self.check(len(modifs) == len(events))
-        # Check for consistency first
-        self.check_modifs_consistent(modifs)
+    def secondary_cmd_test(self):
+        secondary_marker: rd.DrawcallDescription = self.find_draw("Secondary: red and blue")
+        self.controller.SetFrameEvent(secondary_marker.next.eventId, True)
+
+        pipe: rd.PipeState = self.controller.GetPipelineState()
+        rt: rd.BoundResource = pipe.GetOutputTargets()[0]
+        sub = rd.Subresource()
+        tex = rt.resourceId
+        tex_details = self.get_texture(tex)
+        if tex_details.arraysize > 1:
+            sub.slice = rt.firstSlice
+        if tex_details.mips > 1:
+            sub.mip = rt.firstMip
+
+        sec_beg_renderpass_eid = self.find_draw("Begin RenderPass Secondary").next.eventId
+        background_eid = self.find_draw("Secondary: background").next.eventId
+        culled_eid = self.find_draw("Secondary: culled").next.eventId
+        sec_red_and_blue = self.find_draw("Secondary: red and blue").next.eventId
+
+        # Test culling
+        x, y = 70, 40
+        rdtest.log.print("Testing pixel {}, {}".format(x, y))
+        modifs: List[rd.PixelModification] = self.controller.PixelHistory(tex, x, y, sub, rt.typeCast)
+        events = [
+            [[event_id, sec_beg_renderpass_eid], [passed, True], [post_mod_col, (0.0, 1.0, 0.0, 1.0)]],
+            [[event_id, background_eid], [passed, True], [pre_mod_col, (0.0, 1.0, 0.0, 1.0)]],
+            [[event_id, culled_eid], [passed, False], [culled, True]],
+        ]
+        self.check_events(events, modifs, True)
+        self.check_pixel_value(tex, x, y, value_selector(modifs[-1].postMod.col), sub=sub, cast=rt.typeCast)
+
+        # Blue triangle
+        x, y = 40, 40
+        rdtest.log.print("Testing pixel {}, {}".format(x, y))
+        modifs: List[rd.PixelModification] = self.controller.PixelHistory(tex, x, y, sub, rt.typeCast)
+        events = [
+            [[event_id, sec_beg_renderpass_eid], [passed, True], [post_mod_col, (0.0, 1.0, 0.0, 1.0)]],
+            # This is the first event in the command buffer, should have pre-mod
+            [[event_id, background_eid], [passed, True], [pre_mod_col, (0.0, 1.0, 0.0, 1.0)]],
+            # This is the last event in the command buffer, should have post-mod
+            [[event_id, sec_red_and_blue], [passed, True], [post_mod_col, (0.0, 0.0, 1.0, 1.0)]],
+        ]
+        self.check_events(events, modifs, True)
+        self.check_pixel_value(tex, x, y, value_selector(modifs[-1].postMod.col), sub=sub, cast=rt.typeCast)
+
+        # Didn't get post mod for background_eid
+        self.controller.SetFrameEvent(background_eid, True)
+        modifs: List[rd.PixelModification] = self.controller.PixelHistory(tex, x, y, sub, rt.typeCast)
+        events = [
+            [[event_id, sec_beg_renderpass_eid]],
+            # The only event, should have both pre and post mod.
+            [[event_id, background_eid], [passed, True], [pre_mod_col, (0.0, 1.0, 0.0, 1.0)], [post_mod_col, (1.0, 0.0, 1.0, 1.0)]],
+        ]
+        self.check_events(events, modifs, True)
+        self.check_pixel_value(tex, x, y, value_selector(modifs[-1].postMod.col), sub=sub, cast=rt.typeCast)
+
+    def check_events(self, events, modifs, hasSecondary):
+        self.check(len(modifs) == len(events), "Expected {} events, got {}".format(len(events), len(modifs)))
+        # Check for consistency first. For secondary command buffers,
+        # might not have all information, so don't check for consistency
+        if not hasSecondary:
+            self.check_modifs_consistent(modifs)
         for i in range(len(modifs)):
             for c in range(len(events[i])):
                 expected = events[i][c][1]