Added support for VK_EXT_nested_command_buffer

Serialisation functions were updated for new structs and enum values. CaptureQueueSubmit was updated to recurse nested command buffer records so that command buffers and their resources are referenced in the capture. The logic for rebasing secondary command buffer events was updated to support nesting. The PartialReplayData struct was updated so all partial submissions can be tracked during active replay.

renderdoc/driver/vulkan/extension_support.md

@@ -76,6 +76,7 @@ Maintainers can update this file by updating vk.xml in this folder and running `
 * `VK_EXT_metal_surface`
 * `VK_EXT_multisampled_render_to_single_sampled`
 * `VK_EXT_mutable_descriptor_type`
+* `VK_EXT_nested_command_buffer`
 * `VK_EXT_non_seamless_cube_map`
 * `VK_EXT_pageable_device_local_memory`
 * `VK_EXT_pci_bus_info`
@@ -288,7 +289,6 @@ Ray tracing extensions are now standard and will likely be supported at some poi
 * `VK_EXT_legacy_dithering`
 * `VK_EXT_metal_objects`
 * `VK_EXT_multi_draw`
-* `VK_EXT_nested_command_buffer`
 * `VK_EXT_opacity_micromap`
 * `VK_EXT_physical_device_drm`
 * `VK_EXT_pipeline_library_group_handles`

renderdoc/driver/vulkan/vk_common.h

@@ -1368,6 +1368,8 @@ DECLARE_REFLECTION_STRUCT(VkPhysicalDeviceMultisampledRenderToSingleSampledFeatu
 DECLARE_REFLECTION_STRUCT(VkPhysicalDeviceMultiviewFeatures);
 DECLARE_REFLECTION_STRUCT(VkPhysicalDeviceMultiviewProperties);
 DECLARE_REFLECTION_STRUCT(VkPhysicalDeviceMutableDescriptorTypeFeaturesEXT);
+DECLARE_REFLECTION_STRUCT(VkPhysicalDeviceNestedCommandBufferFeaturesEXT);
+DECLARE_REFLECTION_STRUCT(VkPhysicalDeviceNestedCommandBufferPropertiesEXT);
 DECLARE_REFLECTION_STRUCT(VkPhysicalDeviceNonSeamlessCubeMapFeaturesEXT);
 DECLARE_REFLECTION_STRUCT(VkPhysicalDevicePageableDeviceLocalMemoryFeaturesEXT);
 DECLARE_REFLECTION_STRUCT(VkPhysicalDevicePCIBusInfoPropertiesEXT);
@@ -1797,6 +1799,8 @@ DECLARE_DESERIALISE_TYPE(VkPhysicalDeviceMultisampledRenderToSingleSampledFeatur
 DECLARE_DESERIALISE_TYPE(VkPhysicalDeviceMultiviewFeatures);
 DECLARE_DESERIALISE_TYPE(VkPhysicalDeviceMultiviewProperties);
 DECLARE_DESERIALISE_TYPE(VkPhysicalDeviceMutableDescriptorTypeFeaturesEXT);
+DECLARE_DESERIALISE_TYPE(VkPhysicalDeviceNestedCommandBufferFeaturesEXT);
+DECLARE_DESERIALISE_TYPE(VkPhysicalDeviceNestedCommandBufferPropertiesEXT);
 DECLARE_DESERIALISE_TYPE(VkPhysicalDeviceNonSeamlessCubeMapFeaturesEXT);
 DECLARE_DESERIALISE_TYPE(VkPhysicalDevicePageableDeviceLocalMemoryFeaturesEXT);
 DECLARE_DESERIALISE_TYPE(VkPhysicalDevicePCIBusInfoPropertiesEXT);

renderdoc/driver/vulkan/vk_core.cpp

@@ -205,6 +205,9 @@ WrappedVulkan::~WrappedVulkan()
     delete m_ThreadTempMem[i];
   }
 
+  for(size_t i = 0; i < m_Partial.commandTree.size(); i++)
+    delete m_Partial.commandTree[i];
+
   delete m_Replay;
 }
 
@@ -1113,6 +1116,10 @@ static const VkExtensionProperties supportedExtensions[] = {
         VK_EXT_MUTABLE_DESCRIPTOR_TYPE_EXTENSION_NAME,
         VK_EXT_MUTABLE_DESCRIPTOR_TYPE_SPEC_VERSION,
     },
+    {
+        VK_EXT_NESTED_COMMAND_BUFFER_EXTENSION_NAME,
+        VK_EXT_NESTED_COMMAND_BUFFER_SPEC_VERSION,
+    },
     {
         VK_EXT_NON_SEAMLESS_CUBE_MAP_EXTENSION_NAME,
         VK_EXT_NON_SEAMLESS_CUBE_MAP_SPEC_VERSION,
@@ -4201,8 +4208,7 @@ void WrappedVulkan::ReplayLog(uint32_t startEventID, uint32_t endEventID, Replay
   {
     if(!partial)
     {
-      m_Partial[Primary].Reset();
-      m_Partial[Secondary].Reset();
+      m_Partial.Reset();
       m_RenderState = VulkanRenderState();
       for(auto it = m_BakedCmdBufferInfo.begin(); it != m_BakedCmdBufferInfo.end(); it++)
         it->second.state = VulkanRenderState();
@@ -4215,7 +4221,7 @@ void WrappedVulkan::ReplayLog(uint32_t startEventID, uint32_t endEventID, Replay
 
     VkResult vkr = VK_SUCCESS;
 
-    bool rpWasActive[2] = {};
+    rdcarray<CommandBufferNode> cacheNodes = m_Partial.partialStack;
 
     // we'll need our own command buffer if we're replaying just a subsection
     // of events within a single command buffer record - always if it's only
@@ -4244,10 +4250,9 @@ void WrappedVulkan::ReplayLog(uint32_t startEventID, uint32_t endEventID, Replay
       m_RenderState.subpassContents = VK_SUBPASS_CONTENTS_INLINE;
       m_RenderState.dynamicRendering.flags &= ~VK_RENDERING_CONTENTS_SECONDARY_COMMAND_BUFFERS_BIT;
 
-      rpWasActive[Primary] = m_Partial[Primary].renderPassActive;
-      rpWasActive[Secondary] = m_Partial[Secondary].renderPassActive;
+      bool rpActive = IsPartialRenderPassActive();
 
-      if(rpWasActive[Primary] || rpWasActive[Secondary])
+      if(rpActive)
       {
         const ActionDescription *action = GetAction(endEventID);
 
@@ -4327,14 +4332,13 @@ void WrappedVulkan::ReplayLog(uint32_t startEventID, uint32_t endEventID, Replay
       // even if it wasn't before (if the above event was a CmdBeginRenderPass).
       // If we began our own custom single-action loadrp, and it was ended by a CmdEndRenderPass,
       // we need to reverse the virtual transitions we did above, as it won't happen otherwise
-      if(m_Partial[Primary].renderPassActive || m_Partial[Secondary].renderPassActive)
+      if(IsPartialRenderPassActive())
         m_RenderState.EndRenderPass(cmd);
 
       // we might have replayed a CmdBeginRenderPass or CmdEndRenderPass,
       // but we want to keep the partial replay data state intact, so restore
       // whether or not a render pass was active.
-      m_Partial[Primary].renderPassActive = rpWasActive[Primary];
-      m_Partial[Secondary].renderPassActive = rpWasActive[Secondary];
+      m_Partial.partialStack = cacheNodes;
 
       ObjDisp(cmd)->EndCommandBuffer(Unwrap(cmd));
 
@@ -4814,6 +4818,109 @@ const VkFormatProperties &WrappedVulkan::GetFormatProperties(VkFormat f)
   return m_PhysicalDeviceData.fmtProps[f];
 }
 
+bool WrappedVulkan::IsCommandBufferPartial(ResourceId cmdId)
+{
+  for(const CommandBufferNode &cmdNode : m_Partial.partialStack)
+  {
+    // a given command buffer should appear at most once in the partial stack.
+    if(cmdNode.cmdId == cmdId)
+      return true;
+  }
+
+  return false;
+}
+
+bool WrappedVulkan::IsCommandBufferPartialPrimary(ResourceId cmdId)
+{
+  if(m_Partial.partialStack.empty())
+    return false;
+
+  return (m_Partial.partialStack.front().cmdId == cmdId) &&
+         (m_BakedCmdBufferInfo[cmdId].level == VK_COMMAND_BUFFER_LEVEL_PRIMARY);
+}
+
+void WrappedVulkan::SetPartialStack(const CommandBufferNode *targetNode, uint32_t curEvent)
+{
+  // If a command buffer is in the partial stack, it is either a parent of the deepest command buffer
+  // we have seen in this active replay, or it is the deepest.  Either way, don't change the stack.
+  if(IsCommandBufferPartial(targetNode->cmdId))
+    return;
+
+  BuildPartialStackUpToTarget(targetNode->rootNode, targetNode, curEvent);
+}
+
+void WrappedVulkan::BuildPartialStackUpToTarget(const CommandBufferNode *curNode,
+                                                const CommandBufferNode *targetNode,
+                                                uint32_t curEvent)
+{
+  // If the command node is not already in the list, add it
+  // We add a copy instead of the original so renderPassActive can be modified and cached freely
+  if(!IsCommandBufferPartial(curNode->cmdId))
+    m_Partial.partialStack.push_back(*curNode);
+
+  if(curNode->cmdId == targetNode->cmdId)
+    return;
+
+  // We need to recurse the chain of command submits that are currently partial until we hit the target
+  for(const CommandBufferNode *cmdNode : curNode->childCmdNodes)
+  {
+    if(IsEventInCommandBuffer(cmdNode, curEvent, m_BakedCmdBufferInfo[cmdNode->cmdId].eventCount))
+    {
+      BuildPartialStackUpToTarget(cmdNode, targetNode, curEvent);
+    }
+  }
+}
+
+bool WrappedVulkan::IsEventInCommandBuffer(const CommandBufferNode *cmdNode, uint32_t ev,
+                                           uint32_t eventCount)
+{
+  return RDCMAX(1U, cmdNode->beginEvent) - 1 <= ev && ev < (cmdNode->beginEvent + eventCount);
+}
+
+bool WrappedVulkan::IsCommandBufferDeepestPartial(ResourceId cmdId)
+{
+  if(m_Partial.partialStack.empty())
+    return false;
+
+  return m_Partial.partialStack.back().cmdId == cmdId;
+}
+
+WrappedVulkan::CommandBufferNode *WrappedVulkan::GetCommandBufferPartialSubmission(ResourceId cmdId)
+{
+  for(CommandBufferNode &cmdNode : m_Partial.partialStack)
+  {
+    if(cmdNode.cmdId == cmdId)
+      return &cmdNode;
+  }
+
+  return NULL;
+}
+
+bool WrappedVulkan::IsPartialRenderPassActive()
+{
+  for(const CommandBufferNode &cmdNode : m_Partial.partialStack)
+  {
+    if(cmdNode.renderPassActive)
+      return true;
+  }
+
+  return false;
+}
+
+bool WrappedVulkan::ShouldUpdateRenderpassActive(ResourceId cmdId, bool dynamicRendering)
+{
+  if(m_OutsideCmdBuffer != VK_NULL_HANDLE)
+    return true;
+
+  // If we're opening or closing a dynamic renderpass, we only want to track the state if
+  // we are in the most deeply nested command buffer
+  if(dynamicRendering)
+    return IsCommandBufferDeepestPartial(cmdId);
+
+  // Otherwise we are in a non-dynamic renderpass and state should only be tracked for the primary
+  return IsCommandBufferPartialPrimary(cmdId);
+}
+
 bool WrappedVulkan::InRerecordRange(ResourceId cmdid)
 {
   // if we have an outside command buffer, assume the range is valid and we're replaying all events
@@ -4823,12 +4930,11 @@ bool WrappedVulkan::InRerecordRange(ResourceId cmdid)
 
   // if not, check if we're one of the actual partial command buffers and check to see if we're in
   // the range for their partial replay.
-  for(int p = 0; p < ePartialNum; p++)
+  for(const CommandBufferNode &cmdNode : m_Partial.partialStack)
   {
-    if(cmdid == m_Partial[p].partialParent)
+    if(cmdNode.cmdId == cmdid)
     {
-      return m_BakedCmdBufferInfo[m_Partial[p].partialParent].curEventID + m_Partial[p].baseEvent <=
-             m_LastEventID;
+      return m_BakedCmdBufferInfo[cmdid].curEventID + cmdNode.beginEvent <= m_LastEventID;
     }
   }
 
@@ -4845,45 +4951,21 @@ bool WrappedVulkan::HasRerecordCmdBuf(ResourceId cmdid)
   return m_RerecordCmds.find(cmdid) != m_RerecordCmds.end();
 }
 
-bool WrappedVulkan::ShouldUpdateRenderState(ResourceId cmdid, bool forcePrimary)
-{
-  if(m_OutsideCmdBuffer != VK_NULL_HANDLE)
-    return true;
-
-  // if forcePrimary is set we're tracking renderpass activity that only happens in the primary
-  // command buffer. So even if a secondary is partial, we still want to check it.
-  if(forcePrimary)
-    return m_Partial[Primary].partialParent == cmdid;
-
-  // otherwise, if a secondary command buffer is partial we want to *ignore* any state setting
-  // happening in the primary buffer as fortunately no state is inherited (so we don't need to
-  // worry about any state before the execute) and any state setting recorded afterwards would
-  // incorrectly override what we have.
-  if(m_Partial[Secondary].partialParent != ResourceId())
-    return cmdid == m_Partial[Secondary].partialParent;
-
-  return cmdid == m_Partial[Primary].partialParent;
-}
-
 bool WrappedVulkan::IsRenderpassOpen(ResourceId cmdid)
 {
   if(m_OutsideCmdBuffer != VK_NULL_HANDLE)
     return true;
 
-  // if not, check if we're one of the actual partial command buffers and check to see if we're in
-  // the range for their partial replay.
-  for(int p = 0; p < ePartialNum; p++)
+  for(const CommandBufferNode &cmdNode : m_Partial.partialStack)
   {
-    if(cmdid == m_Partial[p].partialParent)
-    {
+    if(cmdNode.cmdId == cmdid)
       return m_BakedCmdBufferInfo[cmdid].renderPassOpen;
-    }
   }
 
   return false;
 }
 
-VkCommandBuffer WrappedVulkan::RerecordCmdBuf(ResourceId cmdid, PartialReplayIndex partialType)
+VkCommandBuffer WrappedVulkan::RerecordCmdBuf(ResourceId cmdid)
 {
   if(m_OutsideCmdBuffer != VK_NULL_HANDLE)
     return m_OutsideCmdBuffer;
@@ -4901,9 +4983,10 @@ VkCommandBuffer WrappedVulkan::RerecordCmdBuf(ResourceId cmdid, PartialReplayInd
 
 ResourceId WrappedVulkan::GetPartialCommandBuffer()
 {
-  if(m_Partial[Secondary].partialParent != ResourceId())
-    return m_Partial[Secondary].partialParent;
-  return m_Partial[Primary].partialParent;
+  if(m_Partial.partialStack.empty())
+    return ResourceId();
+
+  return m_Partial.partialStack.back().cmdId;
 }
 
 void WrappedVulkan::AddAction(const ActionDescription &a)

renderdoc/driver/vulkan/vk_core.h

@@ -653,6 +653,27 @@ private:
 
   const VkFormatProperties &GetFormatProperties(VkFormat f);
 
+  // CommandBufferNodes represent a specific execution of a command buffer in a frame. During a
+  // replay, they are used to track the set of partially-submitted command buffers.
+  struct CommandBufferNode
+  {
+    ResourceId cmdId = ResourceId();
+    uint32_t beginEvent = 0;
+    rdcarray<CommandBufferNode *> childCmdNodes;
+    CommandBufferNode *rootNode = NULL;
+    bool renderPassActive = false;
+  };
+
+  // CommandBufferExecuteInfo tracks the position of the execution of a secondary command buffer
+  // relative to the beginning of the parent command buffer. At the end of a replay's initial
+  // loading stage, these are used to build the tree of CommandBufferNodes that track a command
+  // buffer execution's absolute position in the frame.
+  struct CommandBufferExecuteInfo
+  {
+    ResourceId cmdId = ResourceId();
+    uint32_t relPos = 0;
+  };
+
   struct BakedCmdBufferInfo
   {
     BakedCmdBufferInfo()
@@ -735,59 +756,72 @@ private:
   };
   rdcarray<ActionUse> m_ActionUses;
 
-  enum PartialReplayIndex
-  {
-    Primary,
-    Secondary,
-    ePartialNum
-  };
-
-  struct Submission
-  {
-    Submission(uint32_t eid) : baseEvent(eid), rebased(false) {}
-    uint32_t baseEvent = 0;
-    bool rebased = false;
-  };
-
-  // by definition, when replaying we must have N completely submitted command buffers, and at most
-  // two partially-submitted command buffers. One primary, that we're part-way through, and then
-  // if we're part-way through a vkCmdExecuteCommandBuffers inside that primary then there's one
-  // secondary.
+  // during active replay, command buffers may be partially-submitted if the selected event occurs
+  // within the range of the command buffer. If secondary command buffers are used and the selected
+  // event occurs within the range of a secondary, both the secondary and the command buffer that
+  // executed it are considered partially-submitted. If VK_EXT_nested_command_buffer is enabled,
+  // there may be up to N partially-submitted command buffers, where N is the maximum command
+  // nesting depth used in the capture.
   struct PartialReplayData
   {
     PartialReplayData() { Reset(); }
-    void Reset()
-    {
-      partialParent = ResourceId();
-      baseEvent = 0;
-      renderPassActive = false;
-    }
+    void Reset() { partialStack.clear(); }
 
-    // this records where in the frame a command buffer was submitted, so that we know if our replay
-    // range ends in one of these ranges we need to construct a partial command buffer for future
-    // replaying. Note that we always have the complete command buffer around - it's the bakeID
-    // itself.
-    // Since we only ever record a bakeID once the key is unique - note that the same command buffer
-    // could be recorded multiple times a frame, so the parent command buffer ID (the one recorded
-    // in vkCmd chunks) is NOT unique.
-    // However, a single baked command list can be submitted multiple times - so we have to have a
-    // list of base events
-    // Note in the case of secondary command buffers we mark when these are rebased to 'absolute'
-    // event IDs, since they could be submitted multiple times in the frame and we don't want to
-    // rebase all of them each time.
-    // Map from bakeID -> vector<Submission>
-    std::map<ResourceId, rdcarray<Submission>> cmdBufferSubmits;
+    // partialStack tracks the set of command buffers that are partially submitted during an active
+    // replay. when vkBeginCommandBuffer is replayed, we determine whether the command buffer is
+    // partial by checking whether the last event of the replay occurs within the event range of the
+    // command buffer. The last entry of partialStack should always be the node for the submission
+    // of the most deeply nested command buffer encountered in the replay so far. It may be empty if
+    // no command buffer is partial, or we have not yet replayed a partial command buffer.
+    rdcarray<CommandBufferNode> partialStack;
+    // commandTree represents the hierarchy of command buffer submissions in a frame. Each top level
+    // CommandBufferNode in the array should represent a primary command buffer, which may contain
+    // child nodes representing secondary command buffer executions. If VK_EXT_nested_command_buffer
+    // is enabled, these secondary nodes may also have child nodes, up to
+    // maxCommandBufferNestingLevel returned in VkPhysicalDeviceNestedCommandBufferPropertiesEXT.
+    // These trees are used during replay to update the partialStack.
+    rdcarray<CommandBufferNode *> commandTree;
+    // submitLookup maps a given command buffer ID to all the CommandBufferNodes representing
+    // different submissions of the same command buffer in the frame.
+    rdcflatmap<ResourceId, rdcarray<CommandBufferNode *>> submitLookup;
+  } m_Partial;
 
-    // identifies the baked ID of the command buffer that's actually partial at each level.
-    ResourceId partialParent;
+  // tracks secondary command buffer executions during initial replay loading. At the end of
+  // loading, these executions are rebased into the command nodes in PartialReplayData.
+  rdcflatmap<ResourceId, rdcarray<CommandBufferExecuteInfo>> m_CommandBufferExecutes;
 
-    // the base even of the submission that's partial, as defined above in partialParent
-    uint32_t baseEvent;
+  // in active replay, determines whether the given command buffer is partially submitted.
+  // a command buffer is partially submitted if the last event in the replayed range lies within the command buffer.
+  bool IsCommandBufferPartial(ResourceId cmdId);
 
-    // whether a renderpass is currently active in the partial recording - as with baseEvent, only
-    // valid for the command buffer referred to by partialParent.
-    bool renderPassActive;
-  } m_Partial[ePartialNum];
+  // in active replay, determines whether the given command buffer is a partially submitted primary command buffer.
+  bool IsCommandBufferPartialPrimary(ResourceId cmdId);
+
+  // helper function used to build the stack of partial command buffer nodes up to the target node.
+  // this is called in Serialise_vkBeginCommandBuffer. When we replay a given command buffer for the
+  // first time, we determine it is partial then build the partial stack up to it.
+  void SetPartialStack(const CommandBufferNode *targetNode, uint32_t curEvent);
+
+  // helper recursive function for SetPartialStack
+  void BuildPartialStackUpToTarget(const CommandBufferNode *curNode,
+                                   const CommandBufferNode *targetNode, uint32_t curEvent);
+
+  // helper function that determines whether a given event lies within the scope of a command buffer submission.
+  bool IsEventInCommandBuffer(const CommandBufferNode *cmdNode, uint32_t ev, uint32_t eventCount);
+
+  // in active replay, determines whether the given command buffer is the deepest nested partial command buffer.
+  bool IsCommandBufferDeepestPartial(ResourceId cmdId);
+
+  // for a given command buffer, gets the command node representing its partial submission in the
+  // partial stack. this function does not check whether the command buffer is in the partial stack.
+  // It will return null if it is not.
+  CommandBufferNode *GetCommandBufferPartialSubmission(ResourceId cmdId);
+
+  // determines whether a render pass is active for any node within the partial stack.
+  bool IsPartialRenderPassActive();
+
+  // determines whether we should track the open/close state of a renderpass.
+  bool ShouldUpdateRenderpassActive(ResourceId cmdId, bool dynamicRendering = false);
 
   // if we're replaying just a single action or a particular command
   // buffer subsection of command events, we don't go through the
@@ -812,12 +846,13 @@ private:
 
   bool InRerecordRange(ResourceId cmdid);
   bool HasRerecordCmdBuf(ResourceId cmdid);
-  bool ShouldUpdateRenderState(ResourceId cmdid, bool forcePrimary = false);
   bool IsRenderpassOpen(ResourceId cmdid);
-  VkCommandBuffer RerecordCmdBuf(ResourceId cmdid, PartialReplayIndex partialType = ePartialNum);
+  VkCommandBuffer RerecordCmdBuf(ResourceId cmdid);
 
   ResourceId GetPartialCommandBuffer();
 
+  void UpdateRenderStateForSecondaries(BakedCmdBufferInfo &ancestorCB, BakedCmdBufferInfo &currentCB);
+
   // this info is stored in the record on capture, but we
   // need it on replay too
   struct DescriptorSetInfo
@@ -1031,8 +1066,17 @@ private:
   bool PatchIndirectDraw(size_t drawIndex, uint32_t paramStride, VkIndirectPatchType type,
                          ActionDescription &action, byte *&argptr, byte *argend);
   void InsertActionsAndRefreshIDs(BakedCmdBufferInfo &cmdBufInfo);
+  void AddReferencesForSecondaries(VkResourceRecord *record,
+                                   rdcarray<VkResourceRecord *> &cmdsWithReferences,
+                                   std::unordered_set<ResourceId> &refdIDs);
+  void AddRecordsForSecondaries(VkResourceRecord *record);
+  void UpdateImageStatesForSecondaries(VkResourceRecord *record);
   void CaptureQueueSubmit(VkQueue queue, const rdcarray<VkCommandBuffer> &commandBuffers,
                           VkFence fence);
+
+  CommandBufferNode *BuildSubmitTree(ResourceId cmdId, uint32_t curEvent,
+                                     CommandBufferNode *rootNode = NULL);
+
   void ReplayQueueSubmit(VkQueue queue, VkSubmitInfo2 submitInfo, rdcstr basename);
   void InternalFlushMemoryRange(VkDevice device, const VkMappedMemoryRange &memRange,
                                 bool internalFlush, bool capframe);

renderdoc/driver/vulkan/vk_hookset_defs.h

@@ -557,7 +557,8 @@
   DeclExt(KHR_calibrated_timestamps);                  \
   DeclExt(KHR_deferred_host_operations);               \
   DeclExt(KHR_acceleration_structure);                 \
-  DeclExt(KHR_ray_query);
+  DeclExt(KHR_ray_query);                              \
+  DeclExt(EXT_nested_command_buffer);
 
 // for simplicity and since the check itself is platform agnostic,
 // these aren't protected in platform defines
@@ -676,6 +677,7 @@
   CheckExt(EXT_pageable_device_local_memory, VKXX);           \
   CheckExt(EXT_swapchain_maintenance1, VKXX);                 \
   CheckExt(EXT_provoking_vertex, VKXX);                       \
+  CheckExt(EXT_nested_command_buffer, VKXX);                  \
   CheckExt(EXT_attachment_feedback_loop_dynamic_state, VKXX); \
   CheckExt(EXT_extended_dynamic_state3, VKXX);                \
   CheckExt(EXT_mesh_shader, VKXX);                            \

renderdoc/driver/vulkan/vk_next_chains.cpp

@@ -367,6 +367,10 @@ static void AppendModifiedChainedStruct(byte *&tempMem, VkStruct *outputStruct,
               VkPhysicalDeviceMultiviewProperties);                                                  \
   COPY_STRUCT(VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MUTABLE_DESCRIPTOR_TYPE_FEATURES_EXT,                \
               VkPhysicalDeviceMutableDescriptorTypeFeaturesEXT);                                     \
+  COPY_STRUCT(VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_NESTED_COMMAND_BUFFER_FEATURES_EXT,                  \
+              VkPhysicalDeviceNestedCommandBufferFeaturesEXT);                                       \
+  COPY_STRUCT(VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_NESTED_COMMAND_BUFFER_PROPERTIES_EXT,                \
+              VkPhysicalDeviceNestedCommandBufferPropertiesEXT);                                     \
   COPY_STRUCT(VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_NON_SEAMLESS_CUBE_MAP_FEATURES_EXT,                  \
               VkPhysicalDeviceNonSeamlessCubeMapFeaturesEXT);                                        \
   COPY_STRUCT(VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_LINE_RASTERIZATION_FEATURES_EXT,                     \
@@ -1023,8 +1027,6 @@ static void AppendModifiedChainedStruct(byte *&tempMem, VkStruct *outputStruct,
   case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MULTIVIEW_PER_VIEW_ATTRIBUTES_PROPERTIES_NVX:      \
   case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MULTIVIEW_PER_VIEW_RENDER_AREAS_FEATURES_QCOM:     \
   case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MULTIVIEW_PER_VIEW_VIEWPORTS_FEATURES_QCOM:        \
-  case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_NESTED_COMMAND_BUFFER_FEATURES_EXT:                \
-  case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_NESTED_COMMAND_BUFFER_PROPERTIES_EXT:              \
   case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_OPACITY_MICROMAP_FEATURES_EXT:                     \
   case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_OPACITY_MICROMAP_PROPERTIES_EXT:                   \
   case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_OPTICAL_FLOW_FEATURES_NV:                          \

renderdoc/driver/vulkan/vk_overlay.cpp

@@ -731,9 +731,10 @@ ResourceId VulkanReplay::RenderOverlay(ResourceId texid, FloatVector clearCol, D
   const VulkanCreationInfo::Pipeline &pipeInfo =
       m_pDriver->m_CreationInfo.m_Pipeline[state.graphics.pipeline];
 
+  bool rpActive = m_pDriver->IsPartialRenderPassActive();
+
   if((mainDraw && !(mainDraw->flags & (ActionFlags::MeshDispatch | ActionFlags::Drawcall))) ||
-     (!m_pDriver->m_Partial[WrappedVulkan::Primary].renderPassActive &&
-      !m_pDriver->m_Partial[WrappedVulkan::Secondary].renderPassActive))
+     !rpActive)
   {
     // don't do anything, no action capable of making overlays selected
     float black[] = {0.0f, 0.0f, 0.0f, 0.0f};

renderdoc/driver/vulkan/vk_serialise.cpp

@@ -786,6 +786,12 @@ SERIALISE_VK_HANDLES();
   PNEXT_STRUCT(VK_STRUCTURE_TYPE_MUTABLE_DESCRIPTOR_TYPE_CREATE_INFO_EXT,                              \
                VkMutableDescriptorTypeCreateInfoEXT)                                                   \
                                                                                                        \
+  /* VK_EXT_nested_command_buffer */                                                                   \
+  PNEXT_STRUCT(VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_NESTED_COMMAND_BUFFER_FEATURES_EXT,                   \
+               VkPhysicalDeviceNestedCommandBufferFeaturesEXT)                                         \
+  PNEXT_STRUCT(VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_NESTED_COMMAND_BUFFER_PROPERTIES_EXT,                 \
+               VkPhysicalDeviceNestedCommandBufferPropertiesEXT)                                       \
+                                                                                                       \
   /* VK_EXT_non_seamless_cube_map */                                                                   \
   PNEXT_STRUCT(VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_NON_SEAMLESS_CUBE_MAP_FEATURES_EXT,                   \
                VkPhysicalDeviceNonSeamlessCubeMapFeaturesEXT)                                          \
@@ -1617,10 +1623,6 @@ SERIALISE_VK_HANDLES();
   PNEXT_UNSUPPORTED(VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MULTI_DRAW_FEATURES_EXT)                         \
   PNEXT_UNSUPPORTED(VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MULTI_DRAW_PROPERTIES_EXT)                       \
                                                                                                        \
-  /* VK_EXT_nested_command_buffer */                                                                   \
-  PNEXT_UNSUPPORTED(VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_NESTED_COMMAND_BUFFER_FEATURES_EXT)              \
-  PNEXT_UNSUPPORTED(VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_NESTED_COMMAND_BUFFER_PROPERTIES_EXT)            \
-                                                                                                       \
   /* VK_EXT_opacity_micromap */                                                                        \
   PNEXT_UNSUPPORTED(VK_STRUCTURE_TYPE_MICROMAP_BUILD_INFO_EXT)                                         \
   PNEXT_UNSUPPORTED(VK_STRUCTURE_TYPE_MICROMAP_CREATE_INFO_EXT)                                        \
@@ -6802,6 +6804,40 @@ void Deserialise(const VkPhysicalDeviceMultiviewProperties &el)
   DeserialiseNext(el.pNext);
 }
 
+template <typename SerialiserType>
+void DoSerialise(SerialiserType &ser, VkPhysicalDeviceNestedCommandBufferFeaturesEXT &el)
+{
+  RDCASSERT(ser.IsReading() ||
+            el.sType == VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_NESTED_COMMAND_BUFFER_FEATURES_EXT);
+  SerialiseNext(ser, el.sType, el.pNext);
+
+  SERIALISE_MEMBER(nestedCommandBuffer);
+  SERIALISE_MEMBER(nestedCommandBufferRendering);
+  SERIALISE_MEMBER(nestedCommandBufferSimultaneousUse);
+}
+
+template <>
+void Deserialise(const VkPhysicalDeviceNestedCommandBufferFeaturesEXT &el)
+{
+  DeserialiseNext(el.pNext);
+}
+
+template <typename SerialiserType>
+void DoSerialise(SerialiserType &ser, VkPhysicalDeviceNestedCommandBufferPropertiesEXT &el)
+{
+  RDCASSERT(ser.IsReading() ||
+            el.sType == VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_NESTED_COMMAND_BUFFER_PROPERTIES_EXT);
+  SerialiseNext(ser, el.sType, el.pNext);
+
+  SERIALISE_MEMBER(maxCommandBufferNestingLevel);
+}
+
+template <>
+void Deserialise(const VkPhysicalDeviceNestedCommandBufferPropertiesEXT &el)
+{
+  DeserialiseNext(el.pNext);
+}
+
 template <typename SerialiserType>
 void DoSerialise(SerialiserType &ser, VkPhysicalDevicePipelineCreationCacheControlFeatures &el)
 {
@@ -12346,6 +12382,8 @@ INSTANTIATE_SERIALISE_TYPE(VkPhysicalDeviceMultisampledRenderToSingleSampledFeat
 INSTANTIATE_SERIALISE_TYPE(VkPhysicalDeviceMultiviewFeatures);
 INSTANTIATE_SERIALISE_TYPE(VkPhysicalDeviceMultiviewProperties);
 INSTANTIATE_SERIALISE_TYPE(VkPhysicalDeviceMutableDescriptorTypeFeaturesEXT);
+INSTANTIATE_SERIALISE_TYPE(VkPhysicalDeviceNestedCommandBufferFeaturesEXT);
+INSTANTIATE_SERIALISE_TYPE(VkPhysicalDeviceNestedCommandBufferPropertiesEXT);
 INSTANTIATE_SERIALISE_TYPE(VkPhysicalDeviceNonSeamlessCubeMapFeaturesEXT);
 INSTANTIATE_SERIALISE_TYPE(VkPhysicalDevicePageableDeviceLocalMemoryFeaturesEXT);
 INSTANTIATE_SERIALISE_TYPE(VkPhysicalDevicePCIBusInfoPropertiesEXT);

renderdoc/driver/vulkan/wrappers/vk_cmd_funcs.cpp

@@ -1169,50 +1169,33 @@ bool WrappedVulkan::Serialise_vkBeginCommandBuffer(SerialiserType &ser, VkComman
 
     if(IsActiveReplaying(m_State))
     {
-      const uint32_t length = m_BakedCmdBufferInfo[BakedCommandBuffer].eventCount;
+      const rdcarray<CommandBufferNode *> &submits = m_Partial.submitLookup[BakedCommandBuffer];
 
       bool rerecord = false;
-      bool partial = false;
-      int partialType = ePartialNum;
 
       // check for partial execution of this command buffer
-      for(int p = 0; p < ePartialNum; p++)
+      for(const CommandBufferNode *submit : submits)
       {
-        const rdcarray<Submission> &submissions = m_Partial[p].cmdBufferSubmits[BakedCommandBuffer];
-
-        for(auto it = submissions.begin(); it != submissions.end(); ++it)
+        if(IsEventInCommandBuffer(submit, m_LastEventID,
+                                  m_BakedCmdBufferInfo[BakedCommandBuffer].eventCount))
+        {
+          SetPartialStack(submit, m_LastEventID);
+
+          GetCmdRenderState().xfbcounters.clear();
+          GetCmdRenderState().conditionalRendering.buffer = ResourceId();
+
+          rerecord = true;
+        }
+        else if(submit->beginEvent <= m_LastEventID)
         {
-          if(RDCMAX(1U, it->baseEvent) - 1 <= m_LastEventID &&
-             m_LastEventID < (it->baseEvent + length))
-          {
 #if ENABLED(VERBOSE_PARTIAL_REPLAY)
-            RDCDEBUG("vkBegin - partial detected %u < %u < %u, %s -> %s", it->baseEvent,
-                     m_LastEventID, it->baseEvent + length, ToStr(CommandBuffer).c_str(),
-                     ToStr(BakedCommandBuffer).c_str());
+          RDCDEBUG("vkBegin - full re-record detected %u < %u <= %u, %s -> %s", it->baseEvent,
+                   it->baseEvent + length, m_LastEventID, ToStr(CommandBuffer).c_str(),
+                   ToStr(BakedCommandBuffer).c_str());
 #endif
 
-            m_Partial[p].partialParent = BakedCommandBuffer;
-            m_Partial[p].baseEvent = it->baseEvent;
-            m_Partial[p].renderPassActive = false;
-
-            GetCmdRenderState().xfbcounters.clear();
-            GetCmdRenderState().conditionalRendering.buffer = ResourceId();
-
-            rerecord = true;
-            partial = true;
-            partialType = p;
-          }
-          else if(it->baseEvent <= m_LastEventID)
-          {
-#if ENABLED(VERBOSE_PARTIAL_REPLAY)
-            RDCDEBUG("vkBegin - full re-record detected %u < %u <= %u, %s -> %s", it->baseEvent,
-                     it->baseEvent + length, m_LastEventID, ToStr(CommandBuffer).c_str(),
-                     ToStr(BakedCommandBuffer).c_str());
-#endif
-
-            // this submission is completely within the range, so it should still be re-recorded
-            rerecord = true;
-          }
+          // this submission is completely within the range, so it should still be re-recorded
+          rerecord = true;
         }
       }
 
@@ -1472,22 +1455,19 @@ bool WrappedVulkan::Serialise_vkEndCommandBuffer(SerialiserType &ser, VkCommandB
 #endif
 
         VulkanRenderState &renderstate = GetCmdRenderState();
-        if(m_Partial[Primary].partialParent == BakedCommandBuffer && !renderstate.xfbcounters.empty())
-        {
-          renderstate.EndTransformFeedback(this, commandBuffer);
-        }
 
-        if(m_Partial[Primary].partialParent == BakedCommandBuffer &&
-           renderstate.IsConditionalRenderingEnabled())
+        if(IsCommandBufferPartialPrimary(BakedCommandBuffer))
         {
-          renderstate.EndConditionalRendering(commandBuffer);
+          if(!renderstate.xfbcounters.empty())
+            renderstate.EndTransformFeedback(this, commandBuffer);
+
+          if(renderstate.IsConditionalRenderingEnabled())
+            renderstate.EndConditionalRendering(commandBuffer);
         }
 
         // finish any render pass that was still active in the primary partial parent
-        if((m_Partial[Primary].partialParent == BakedCommandBuffer &&
-            m_Partial[Primary].renderPassActive) ||
-           (m_Partial[Secondary].partialParent == BakedCommandBuffer &&
-            m_Partial[Secondary].renderPassActive))
+        if(IsCommandBufferPartial(m_LastCmdBufferID) &&
+           GetCommandBufferPartialSubmission(m_LastCmdBufferID)->renderPassActive)
         {
           if(m_BakedCmdBufferInfo[m_LastCmdBufferID].renderPassOpen)
           {
@@ -1780,10 +1760,10 @@ bool WrappedVulkan::Serialise_vkCmdBeginRenderPass(SerialiserType &ser, VkComman
         commandBuffer = RerecordCmdBuf(m_LastCmdBufferID);
 
         // only if we're partially recording do we update this state
-        if(ShouldUpdateRenderState(m_LastCmdBufferID, true))
+        if(ShouldUpdateRenderpassActive(m_LastCmdBufferID, false))
         {
-          m_Partial[Primary].renderPassActive = true;
-          m_BakedCmdBufferInfo[m_LastCmdBufferID].renderPassOpen = true;
+          GetCommandBufferPartialSubmission(m_LastCmdBufferID)->renderPassActive =
+              m_BakedCmdBufferInfo[m_LastCmdBufferID].renderPassOpen = true;
         }
 
         m_BakedCmdBufferInfo[m_LastCmdBufferID].activeSubpass = 0;
@@ -2236,9 +2216,10 @@ bool WrappedVulkan::Serialise_vkCmdEndRenderPass(SerialiserType &ser, VkCommandB
 
         rdcarray<VkImageMemoryBarrier> imgBarriers = GetImplicitRenderPassBarriers(~0U);
 
-        if(ShouldUpdateRenderState(m_LastCmdBufferID, true))
+        // only if we're partially recording do we update this state
+        if(ShouldUpdateRenderpassActive(m_LastCmdBufferID, false))
         {
-          m_Partial[Primary].renderPassActive =
+          GetCommandBufferPartialSubmission(m_LastCmdBufferID)->renderPassActive =
               m_BakedCmdBufferInfo[m_LastCmdBufferID].renderPassOpen = false;
         }
 
@@ -2441,9 +2422,9 @@ bool WrappedVulkan::Serialise_vkCmdBeginRenderPass2(SerialiserType &ser,
         commandBuffer = RerecordCmdBuf(m_LastCmdBufferID);
 
         // only if we're partially recording do we update this state
-        if(ShouldUpdateRenderState(m_LastCmdBufferID, true))
+        if(ShouldUpdateRenderpassActive(m_LastCmdBufferID, false))
         {
-          m_Partial[Primary].renderPassActive =
+          GetCommandBufferPartialSubmission(m_LastCmdBufferID)->renderPassActive =
               m_BakedCmdBufferInfo[m_LastCmdBufferID].renderPassOpen = true;
         }
 
@@ -2930,9 +2911,10 @@ bool WrappedVulkan::Serialise_vkCmdEndRenderPass2(SerialiserType &ser, VkCommand
 
         rdcarray<VkImageMemoryBarrier> imgBarriers = GetImplicitRenderPassBarriers(~0U);
 
-        if(ShouldUpdateRenderState(m_LastCmdBufferID, true))
+        // only if we're partially recording do we update this state
+        if(ShouldUpdateRenderpassActive(m_LastCmdBufferID, false))
         {
-          m_Partial[Primary].renderPassActive =
+          GetCommandBufferPartialSubmission(m_LastCmdBufferID)->renderPassActive =
               m_BakedCmdBufferInfo[m_LastCmdBufferID].renderPassOpen = false;
         }
 
@@ -4704,6 +4686,24 @@ void WrappedVulkan::vkCmdResetQueryPool(VkCommandBuffer commandBuffer, VkQueryPo
   }
 }
 
+void WrappedVulkan::UpdateRenderStateForSecondaries(BakedCmdBufferInfo &ancestorCB,
+                                                    BakedCmdBufferInfo &currentCB)
+{
+  currentCB.state.SetRenderPass(ancestorCB.state.GetRenderPass());
+  currentCB.state.subpass = ancestorCB.state.subpass;
+  currentCB.state.dynamicRendering = ancestorCB.state.dynamicRendering;
+  currentCB.state.SetFramebuffer(ancestorCB.state.GetFramebuffer(),
+                                 ancestorCB.state.GetFramebufferAttachments());
+  currentCB.state.renderArea = ancestorCB.state.renderArea;
+  currentCB.state.subpassContents = ancestorCB.state.subpassContents;
+
+  if(currentCB.action)
+  {
+    for(const ResourceId &childCB : currentCB.action->executedCmds)
+      UpdateRenderStateForSecondaries(ancestorCB, m_BakedCmdBufferInfo[childCB]);
+  }
+}
+
 template <typename SerialiserType>
 bool WrappedVulkan::Serialise_vkCmdExecuteCommands(SerialiserType &ser, VkCommandBuffer commandBuffer,
                                                    uint32_t commandBufferCount,
@@ -4814,8 +4814,13 @@ bool WrappedVulkan::Serialise_vkCmdExecuteCommands(SerialiserType &ser, VkComman
           parentCmdBufInfo.debugMessages.back().eventId += parentCmdBufInfo.curEventID;
         }
 
-        // only primary command buffers can be submitted
-        m_Partial[Secondary].cmdBufferSubmits[cmd].push_back(parentCmdBufInfo.curEventID);
+        // Record execution of the secondary command buffer in the parent's CommandBufferNode
+        // Only primary command buffers can be submitted
+        CommandBufferExecuteInfo execInfo;
+        execInfo.cmdId = cmd;
+        execInfo.relPos = parentCmdBufInfo.curEventID;
+
+        m_CommandBufferExecutes[m_LastCmdBufferID].push_back(execInfo);
 
         parentCmdBufInfo.action->executedCmds.push_back(cmd);
 
@@ -4875,12 +4880,14 @@ bool WrappedVulkan::Serialise_vkCmdExecuteCommands(SerialiserType &ser, VkComman
         parentCmdBufInfo.curEventID++;
 
         bool fullRecord = false;
-        uint32_t startEID = parentCmdBufInfo.curEventID + m_Partial[Primary].baseEvent;
+        uint32_t startEID = parentCmdBufInfo.curEventID;
+        if(IsCommandBufferPartial(m_LastCmdBufferID))
+          startEID += GetCommandBufferPartialSubmission(m_LastCmdBufferID)->beginEvent;
 
         // if we're in the re-record range and this command buffer isn't partial, we execute all
         // command buffers because m_Partial[Primary].baseEvent above is only valid for the partial
         // command buffer
-        if(m_Partial[Primary].partialParent != m_LastCmdBufferID)
+        if(!IsCommandBufferPartial(m_LastCmdBufferID))
         {
 #if ENABLED(VERBOSE_PARTIAL_REPLAY)
           RDCDEBUG("Fully re-recording non-partial execute in command buffer %s for %s",
@@ -4898,15 +4905,7 @@ bool WrappedVulkan::Serialise_vkCmdExecuteCommands(SerialiserType &ser, VkComman
           // activated inside the secondary which we should not overwrite.
           if(parentCmdBufInfo.state.ActiveRenderPass())
           {
-            m_BakedCmdBufferInfo[cmd].state.SetRenderPass(parentCmdBufInfo.state.GetRenderPass());
-            m_BakedCmdBufferInfo[cmd].state.subpass = parentCmdBufInfo.state.subpass;
-            m_BakedCmdBufferInfo[cmd].state.dynamicRendering =
-                parentCmdBufInfo.state.dynamicRendering;
-            m_BakedCmdBufferInfo[cmd].state.SetFramebuffer(
-                parentCmdBufInfo.state.GetFramebuffer(),
-                parentCmdBufInfo.state.GetFramebufferAttachments());
-            m_BakedCmdBufferInfo[cmd].state.renderArea = parentCmdBufInfo.state.renderArea;
-            m_BakedCmdBufferInfo[cmd].state.subpassContents = parentCmdBufInfo.state.subpassContents;
+            UpdateRenderStateForSecondaries(parentCmdBufInfo, m_BakedCmdBufferInfo[cmd]);
           }
 
           // 2 extra for the virtual labels around the command buffer
@@ -6988,14 +6987,10 @@ bool WrappedVulkan::Serialise_vkCmdBeginRendering(SerialiserType &ser, VkCommand
         commandBuffer = RerecordCmdBuf(m_LastCmdBufferID);
 
         // only if we're partially recording do we update this state
-        if(ShouldUpdateRenderState(m_LastCmdBufferID))
+        if(ShouldUpdateRenderpassActive(m_LastCmdBufferID, true))
         {
-          if(m_Partial[Primary].partialParent == m_LastCmdBufferID)
-            m_Partial[Primary].renderPassActive = true;
-          else if(m_Partial[Secondary].partialParent == m_LastCmdBufferID)
-            m_Partial[Secondary].renderPassActive = true;
-
-          m_BakedCmdBufferInfo[m_LastCmdBufferID].renderPassOpen = true;
+          GetCommandBufferPartialSubmission(m_LastCmdBufferID)->renderPassActive =
+              m_BakedCmdBufferInfo[m_LastCmdBufferID].renderPassOpen = true;
         }
         m_BakedCmdBufferInfo[m_LastCmdBufferID].activeSubpass = 0;
 
@@ -7392,17 +7387,14 @@ bool WrappedVulkan::Serialise_vkCmdEndRendering(SerialiserType &ser, VkCommandBu
 
         bool suspending = (renderstate.dynamicRendering.flags & VK_RENDERING_SUSPENDING_BIT) != 0;
 
-        if(ShouldUpdateRenderState(m_LastCmdBufferID))
+        if(ShouldUpdateRenderpassActive(m_LastCmdBufferID, true))
         {
           m_BakedCmdBufferInfo[m_LastCmdBufferID].renderPassOpen = false;
 
           // if this rendering is just being suspended, the pass is still active
-          if(!suspending)
+          if(!suspending && IsCommandBufferPartial(m_LastCmdBufferID))
           {
-            if(m_Partial[Primary].partialParent == m_LastCmdBufferID)
-              m_Partial[Primary].renderPassActive = false;
-            else if(m_Partial[Secondary].partialParent == m_LastCmdBufferID)
-              m_Partial[Secondary].renderPassActive = false;
+            GetCommandBufferPartialSubmission(m_LastCmdBufferID)->renderPassActive = false;
           }
         }
 

renderdoc/driver/vulkan/wrappers/vk_device_funcs.cpp

@@ -3303,6 +3303,15 @@ bool WrappedVulkan::Serialise_vkCreateDevice(SerialiserType &ser, VkPhysicalDevi
         }
       }
       END_PHYS_EXT_CHECK();
+
+      BEGIN_PHYS_EXT_CHECK(VkPhysicalDeviceNestedCommandBufferFeaturesEXT,
+                           VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_NESTED_COMMAND_BUFFER_FEATURES_EXT);
+      {
+        CHECK_PHYS_EXT_FEATURE(nestedCommandBuffer);
+        CHECK_PHYS_EXT_FEATURE(nestedCommandBufferRendering);
+        CHECK_PHYS_EXT_FEATURE(nestedCommandBufferSimultaneousUse);
+      }
+      END_PHYS_EXT_CHECK();
     }
 
     if(availFeatures.depthClamp)

renderdoc/driver/vulkan/wrappers/vk_draw_funcs.cpp

@@ -401,10 +401,8 @@ bool WrappedVulkan::Serialise_vkCmdDrawIndirect(SerialiserType &ser, VkCommandBu
           {
             curEID = m_BakedCmdBufferInfo[m_LastCmdBufferID].curEventID;
 
-            if(m_Partial[Primary].partialParent == m_LastCmdBufferID)
-              curEID += m_Partial[Primary].baseEvent;
-            else if(m_Partial[Secondary].partialParent == m_LastCmdBufferID)
-              curEID += m_Partial[Secondary].baseEvent;
+            if(IsCommandBufferPartial(m_LastCmdBufferID))
+              curEID += GetCommandBufferPartialSubmission(m_LastCmdBufferID)->beginEvent;
           }
 
           ActionUse use(m_CurChunkOffset, 0);
@@ -811,10 +809,8 @@ bool WrappedVulkan::Serialise_vkCmdDrawIndexedIndirect(SerialiserType &ser,
           {
             curEID = m_BakedCmdBufferInfo[m_LastCmdBufferID].curEventID;
 
-            if(m_Partial[Primary].partialParent == m_LastCmdBufferID)
-              curEID += m_Partial[Primary].baseEvent;
-            else if(m_Partial[Secondary].partialParent == m_LastCmdBufferID)
-              curEID += m_Partial[Secondary].baseEvent;
+            if(IsCommandBufferPartial(m_LastCmdBufferID))
+              curEID += GetCommandBufferPartialSubmission(m_LastCmdBufferID)->beginEvent;
           }
 
           ActionUse use(m_CurChunkOffset, 0);
@@ -2778,10 +2774,8 @@ bool WrappedVulkan::Serialise_vkCmdDrawIndirectCount(SerialiserType &ser,
         {
           curEID = m_BakedCmdBufferInfo[m_LastCmdBufferID].curEventID;
 
-          if(m_Partial[Primary].partialParent == m_LastCmdBufferID)
-            curEID += m_Partial[Primary].baseEvent;
-          else if(m_Partial[Secondary].partialParent == m_LastCmdBufferID)
-            curEID += m_Partial[Secondary].baseEvent;
+          if(IsCommandBufferPartial(m_LastCmdBufferID))
+            curEID += GetCommandBufferPartialSubmission(m_LastCmdBufferID)->beginEvent;
         }
 
         ActionUse use(m_CurChunkOffset, 0);
@@ -3125,10 +3119,8 @@ bool WrappedVulkan::Serialise_vkCmdDrawIndexedIndirectCount(
         {
           curEID = m_BakedCmdBufferInfo[m_LastCmdBufferID].curEventID;
 
-          if(m_Partial[Primary].partialParent == m_LastCmdBufferID)
-            curEID += m_Partial[Primary].baseEvent;
-          else if(m_Partial[Secondary].partialParent == m_LastCmdBufferID)
-            curEID += m_Partial[Secondary].baseEvent;
+          if(IsCommandBufferPartial(m_LastCmdBufferID))
+            curEID += GetCommandBufferPartialSubmission(m_LastCmdBufferID)->beginEvent;
         }
 
         ActionUse use(m_CurChunkOffset, 0);
@@ -4488,10 +4480,8 @@ bool WrappedVulkan::Serialise_vkCmdDrawMeshTasksIndirectEXT(SerialiserType &ser,
           {
             curEID = m_BakedCmdBufferInfo[m_LastCmdBufferID].curEventID;
 
-            if(m_Partial[Primary].partialParent == m_LastCmdBufferID)
-              curEID += m_Partial[Primary].baseEvent;
-            else if(m_Partial[Secondary].partialParent == m_LastCmdBufferID)
-              curEID += m_Partial[Secondary].baseEvent;
+            if(IsCommandBufferPartial(m_LastCmdBufferID))
+              curEID += GetCommandBufferPartialSubmission(m_LastCmdBufferID)->beginEvent;
           }
 
           ActionUse use(m_CurChunkOffset, 0);
@@ -4889,10 +4879,8 @@ bool WrappedVulkan::Serialise_vkCmdDrawMeshTasksIndirectCountEXT(
         {
           curEID = m_BakedCmdBufferInfo[m_LastCmdBufferID].curEventID;
 
-          if(m_Partial[Primary].partialParent == m_LastCmdBufferID)
-            curEID += m_Partial[Primary].baseEvent;
-          else if(m_Partial[Secondary].partialParent == m_LastCmdBufferID)
-            curEID += m_Partial[Secondary].baseEvent;
+          if(IsCommandBufferPartial(m_LastCmdBufferID))
+            curEID += GetCommandBufferPartialSubmission(m_LastCmdBufferID)->beginEvent;
         }
 
         ActionUse use(m_CurChunkOffset, 0);

renderdoc/driver/vulkan/wrappers/vk_queue_funcs.cpp

@@ -249,6 +249,34 @@ void WrappedVulkan::DoSubmit(VkQueue queue, VkSubmitInfo2 submitInfo)
   }
 }
 
+WrappedVulkan::CommandBufferNode *WrappedVulkan::BuildSubmitTree(ResourceId cmdId, uint32_t curEvent,
+                                                                 CommandBufferNode *rootNode)
+{
+  CommandBufferNode *cmdNode = new CommandBufferNode();
+  cmdNode->cmdId = cmdId;
+  cmdNode->beginEvent = curEvent;
+
+  // setting the root node of the primary to itself simplifies building the tree here, as well as
+  // building the partial stack during active replay.
+  if(rootNode == NULL)
+    rootNode = cmdNode;
+
+  cmdNode->rootNode = rootNode;
+
+  m_Partial.submitLookup[cmdId].push_back(cmdNode);
+
+  const rdcarray<CommandBufferExecuteInfo> &executedCmds = m_CommandBufferExecutes[cmdId];
+
+  for(const CommandBufferExecuteInfo &childExecuteInfo : executedCmds)
+  {
+    CommandBufferNode *rebaseChild = BuildSubmitTree(
+        childExecuteInfo.cmdId, cmdNode->beginEvent + childExecuteInfo.relPos, rootNode);
+    cmdNode->childCmdNodes.push_back(rebaseChild);
+  }
+
+  return cmdNode;
+}
+
 void WrappedVulkan::ReplayQueueSubmit(VkQueue queue, VkSubmitInfo2 submitInfo, rdcstr basename)
 {
   if(IsLoading(m_State))
@@ -307,20 +335,10 @@ void WrappedVulkan::ReplayQueueSubmit(VkQueue queue, VkSubmitInfo2 submitInfo, r
       // and drawIDs
       InsertActionsAndRefreshIDs(cmdBufInfo);
 
-      for(size_t e = 0; e < cmdBufInfo.action->executedCmds.size(); e++)
-      {
-        rdcarray<Submission> &submits =
-            m_Partial[Secondary].cmdBufferSubmits[cmdBufInfo.action->executedCmds[e]];
+      // only primary command buffers can be submitted
+      CommandBufferNode *rebaseNode = BuildSubmitTree(cmd, m_RootEventID);
 
-        for(size_t s = 0; s < submits.size(); s++)
-        {
-          if(!submits[s].rebased)
-          {
-            submits[s].baseEvent += m_RootEventID;
-            submits[s].rebased = true;
-          }
-        }
-      }
+      m_Partial.commandTree.push_back(rebaseNode);
 
       for(size_t i = 0; i < cmdBufInfo.debugMessages.size(); i++)
       {
@@ -328,9 +346,6 @@ void WrappedVulkan::ReplayQueueSubmit(VkQueue queue, VkSubmitInfo2 submitInfo, r
         m_DebugMessages.back().eventId += m_RootEventID;
       }
 
-      // only primary command buffers can be submitted
-      m_Partial[Primary].cmdBufferSubmits[cmd].push_back(Submission(m_RootEventID));
-
       m_RootEventID += cmdBufInfo.eventCount;
       m_RootActionID += cmdBufInfo.actionCount;
 
@@ -733,18 +748,6 @@ void WrappedVulkan::InsertActionsAndRefreshIDs(BakedCmdBufferInfo &cmdBufInfo)
             m_BakedCmdBufferInfo[n.indirectPatch.commandBuffer].actionCount += eidShift;
           }
 
-          for(size_t e = 0; e < cmdBufInfo.action->executedCmds.size(); e++)
-          {
-            rdcarray<Submission> &submits =
-                m_Partial[Secondary].cmdBufferSubmits[cmdBufInfo.action->executedCmds[e]];
-
-            for(size_t s = 0; s < submits.size(); s++)
-            {
-              if(submits[s].baseEvent >= cmdBufNodes[i].action.eventId + 2)
-                submits[s].baseEvent += eidShift;
-            }
-          }
-
           RDCASSERT(cmdBufNodes[i + 1].action.events.size() == 1);
           uint32_t chunkIndex = cmdBufNodes[i + 1].action.events[0].chunkIndex;
 
@@ -906,6 +909,55 @@ void WrappedVulkan::InsertActionsAndRefreshIDs(BakedCmdBufferInfo &cmdBufInfo)
   }
 }
 
+void WrappedVulkan::AddReferencesForSecondaries(VkResourceRecord *record,
+                                                rdcarray<VkResourceRecord *> &cmdsWithReferences,
+                                                std::unordered_set<ResourceId> &refdIDs)
+{
+  // cannot add references here until after we've done descriptor sets later, see comment
+  // in CaptureQueueSubmit.
+  // bakedSubcmds->AddResourceReferences(GetResourceManager());
+  // GetResourceManager()->MergeReferencedMemory(bakedSubcmds->cmdInfo->memFrameRefs);
+  // UpdateImageStates(bakedSubcmds->cmdInfo->imageStates);
+  const rdcarray<VkResourceRecord *> &subcmds = record->bakedCommands->cmdInfo->subcmds;
+
+  for(VkResourceRecord *subcmd : subcmds)
+  {
+    cmdsWithReferences.push_back(subcmd->bakedCommands);
+
+    subcmd->bakedCommands->AddReferencedIDs(refdIDs);
+
+    GetResourceManager()->MarkResourceFrameReferenced(subcmd->cmdInfo->allocRecord->GetResourceID(),
+                                                      eFrameRef_Read);
+
+    subcmd->bakedCommands->AddRef();
+
+    AddReferencesForSecondaries(subcmd, cmdsWithReferences, refdIDs);
+  }
+}
+
+void WrappedVulkan::AddRecordsForSecondaries(VkResourceRecord *record)
+{
+  const rdcarray<VkResourceRecord *> &subcmds = record->bakedCommands->cmdInfo->subcmds;
+
+  for(VkResourceRecord *subcmd : subcmds)
+  {
+    m_CmdBufferRecords.push_back(subcmd->bakedCommands);
+    AddRecordsForSecondaries(subcmd);
+  }
+}
+
+void WrappedVulkan::UpdateImageStatesForSecondaries(VkResourceRecord *record)
+{
+  const rdcarray<VkResourceRecord *> &subcmds = record->bakedCommands->cmdInfo->subcmds;
+
+  for(VkResourceRecord *subcmd : subcmds)
+  {
+    subcmd->bakedCommands->AddResourceReferences(GetResourceManager());
+    UpdateImageStates(subcmd->bakedCommands->cmdInfo->imageStates);
+    UpdateImageStatesForSecondaries(subcmd);
+  }
+}
+
 void WrappedVulkan::CaptureQueueSubmit(VkQueue queue,
                                        const rdcarray<VkCommandBuffer> &commandBuffers, VkFence fence)
 {
@@ -972,31 +1024,17 @@ void WrappedVulkan::CaptureQueueSubmit(VkQueue queue,
       GetResourceManager()->MarkResourceFrameReferenced(
           record->cmdInfo->allocRecord->GetResourceID(), eFrameRef_Read);
 
-      const rdcarray<VkResourceRecord *> &subcmds = record->bakedCommands->cmdInfo->subcmds;
-
-      for(size_t sub = 0; sub < subcmds.size(); sub++)
-      {
-        VkResourceRecord *bakedSubcmds = subcmds[sub]->bakedCommands;
-
-        // cannot add references here until after we've done descriptor sets later, see comment
-        // above
-        // bakedSubcmds->AddResourceReferences(GetResourceManager());
-        // GetResourceManager()->MergeReferencedMemory(bakedSubcmds->cmdInfo->memFrameRefs);
-        // UpdateImageStates(bakedSubcmds->cmdInfo->imageStates);
-        cmdsWithReferences.push_back(bakedSubcmds);
-
-        bakedSubcmds->AddReferencedIDs(refdIDs);
-        GetResourceManager()->MarkResourceFrameReferenced(
-            subcmds[sub]->cmdInfo->allocRecord->GetResourceID(), eFrameRef_Read);
-
-        bakedSubcmds->AddRef();
-      }
+      // cannot add references here until after we've done descriptor sets later, see comment
+      // above
+      // bakedSubcmds->AddResourceReferences(GetResourceManager());
+      // GetResourceManager()->MergeReferencedMemory(bakedSubcmds->cmdInfo->memFrameRefs);
+      // UpdateImageStates(bakedSubcmds->cmdInfo->imageStates);
+      AddReferencesForSecondaries(record, cmdsWithReferences, refdIDs);
 
       {
         SCOPED_LOCK(m_CmdBufferRecordsLock);
         m_CmdBufferRecords.push_back(record->bakedCommands);
-        for(size_t sub = 0; sub < subcmds.size(); sub++)
-          m_CmdBufferRecords.push_back(subcmds[sub]->bakedCommands);
+        AddRecordsForSecondaries(record);
       }
 
       record->bakedCommands->AddRef();
@@ -1025,12 +1063,7 @@ void WrappedVulkan::CaptureQueueSubmit(VkQueue queue,
 
       record->bakedCommands->AddResourceReferences(GetResourceManager());
       UpdateImageStates(record->bakedCommands->cmdInfo->imageStates);
-
-      for(VkResourceRecord *sub : record->bakedCommands->cmdInfo->subcmds)
-      {
-        sub->bakedCommands->AddResourceReferences(GetResourceManager());
-        UpdateImageStates(sub->bakedCommands->cmdInfo->imageStates);
-      }
+      UpdateImageStatesForSecondaries(record);
     }
 
     // every 20 submits clean background references, in case the application isn't presenting.