Do two-pass processing of pNext chains for strict validity. Closes #3093

* We first need to do a pass to find extension structs that affect validity of
  pointers (whether they're ignored or not), then do a pass to process.

renderdoc/driver/vulkan/vk_common.h

@@ -337,7 +337,17 @@ enum VkFlagWithNoBits
   FlagWithNoBits_Dummy_Bit = 1,
 };
 
+// global per-chain flags to use in a double-pass processing
+struct NextChainFlags
+{
+  // VkPipelineRenderingCreateInfoKHR provides a list of formats which is normally valid except if
+  // we're creating a pipeline library without the fragment output interface. We need to detect that
+  // first before processing it
+  bool dynRenderingFormatsValid = true;
+};
+
 size_t GetNextPatchSize(const void *next);
+void PreprocessNextChain(const VkBaseInStructure *nextInput, NextChainFlags &nextChainFlags);
 void UnwrapNextChain(CaptureState state, const char *structName, byte *&tempMem,
                      VkBaseInStructure *infoStruct);
 void CopyNextChainForPatching(const char *structName, byte *&tempMem, VkBaseInStructure *infoStruct);

renderdoc/driver/vulkan/vk_next_chains.cpp

@@ -1607,12 +1607,37 @@ size_t GetNextPatchSize(const void *pNext)
   return memSize;
 }
 
+void PreprocessNextChain(const VkBaseInStructure *nextInput, NextChainFlags &nextChainFlags)
+{
+  while(nextInput)
+  {
+    switch(nextInput->sType)
+    {
+      case VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_LIBRARY_CREATE_INFO_EXT:
+      {
+        const VkGraphicsPipelineLibraryCreateInfoEXT *libCreateInfo =
+            (const VkGraphicsPipelineLibraryCreateInfoEXT *)nextInput;
+        nextChainFlags.dynRenderingFormatsValid =
+            (libCreateInfo->flags &
+             VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_OUTPUT_INTERFACE_BIT_EXT) != 0;
+        break;
+      }
+      default: break;
+    }
+
+    nextInput = nextInput->pNext;
+  }
+}
+
 void UnwrapNextChain(CaptureState state, const char *structName, byte *&tempMem,
                      VkBaseInStructure *infoStruct)
 {
   if(!infoStruct)
     return;
 
+  NextChainFlags nextChainFlags;
+  PreprocessNextChain(infoStruct, nextChainFlags);
+
   // during capture, this walks the pNext chain and either copies structs that can be passed
   // straight through, or copies and modifies any with vulkan objects that need to be unwrapped.
   //
@@ -2287,8 +2312,11 @@ void UnwrapNextChain(CaptureState state, const char *structName, byte *&tempMem,
         *out = *in;
 
         out->pColorAttachmentFormats = outFormats;
-        for(uint32_t i = 0; i < in->colorAttachmentCount; i++)
-          outFormats[i] = in->pColorAttachmentFormats[i];
+        if(nextChainFlags.dynRenderingFormatsValid)
+        {
+          for(uint32_t i = 0; i < in->colorAttachmentCount; i++)
+            outFormats[i] = in->pColorAttachmentFormats[i];
+        }
 
         break;
       }

renderdoc/driver/vulkan/vk_serialise.cpp

@@ -3472,6 +3472,12 @@ void Deserialise(const VkImageMemoryBarrier &el)
 template <typename SerialiserType>
 void DoSerialise(SerialiserType &ser, VkGraphicsPipelineCreateInfo &el)
 {
+  NextChainFlags nextChainFlags;
+  if(ser.IsWriting())
+    PreprocessNextChain((const VkBaseInStructure *)el.pNext, nextChainFlags);
+
+  ser.SetStructArg((uint64_t)(uintptr_t)&nextChainFlags);
+
   RDCASSERT(ser.IsReading() || el.sType == VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO);
   SerialiseNext(ser, el.sType, el.pNext);
 
@@ -8843,8 +8849,19 @@ void DoSerialise(SerialiserType &ser, VkPipelineRenderingCreateInfo &el)
   SerialiseNext(ser, el.sType, el.pNext);
 
   SERIALISE_MEMBER(viewMask);
-  SERIALISE_MEMBER(colorAttachmentCount);
-  SERIALISE_MEMBER_ARRAY(pColorAttachmentFormats, colorAttachmentCount);
+
+  NextChainFlags *nextChainFlags = (NextChainFlags *)(uintptr_t)ser.GetStructArg();
+
+  if(nextChainFlags && nextChainFlags->dynRenderingFormatsValid)
+  {
+    SERIALISE_MEMBER(colorAttachmentCount);
+    SERIALISE_MEMBER_ARRAY(pColorAttachmentFormats, colorAttachmentCount);
+  }
+  else
+  {
+    SERIALISE_MEMBER_EMPTY(colorAttachmentCount);
+    SERIALISE_MEMBER_ARRAY_EMPTY(pColorAttachmentFormats);
+  }
   SERIALISE_MEMBER(depthAttachmentFormat);
   SERIALISE_MEMBER(stencilAttachmentFormat);
 }

util/test/demos/vk/vk_parameter_zoo.cpp

@@ -262,15 +262,18 @@ void main()
 
   void Prepare(int argc, char **argv)
   {
+    optDevExts.push_back(VK_EXT_GRAPHICS_PIPELINE_LIBRARY_EXTENSION_NAME);
     optDevExts.push_back(VK_EXT_TOOLING_INFO_EXTENSION_NAME);
-    optDevExts.push_back(VK_KHR_DESCRIPTOR_UPDATE_TEMPLATE_EXTENSION_NAME);
-    optDevExts.push_back(VK_KHR_PUSH_DESCRIPTOR_EXTENSION_NAME);
     optDevExts.push_back(VK_EXT_TRANSFORM_FEEDBACK_EXTENSION_NAME);
-    optDevExts.push_back(VK_KHR_TIMELINE_SEMAPHORE_EXTENSION_NAME);
     optDevExts.push_back(VK_KHR_BIND_MEMORY_2_EXTENSION_NAME);
-    optDevExts.push_back(VK_KHR_MULTIVIEW_EXTENSION_NAME);
+    optDevExts.push_back(VK_KHR_DESCRIPTOR_UPDATE_TEMPLATE_EXTENSION_NAME);
+    optDevExts.push_back(VK_KHR_DYNAMIC_RENDERING_EXTENSION_NAME);
     optDevExts.push_back(VK_KHR_MAINTENANCE2_EXTENSION_NAME);
+    optDevExts.push_back(VK_KHR_MULTIVIEW_EXTENSION_NAME);
+    optDevExts.push_back(VK_KHR_PIPELINE_LIBRARY_EXTENSION_NAME);
+    optDevExts.push_back(VK_KHR_PUSH_DESCRIPTOR_EXTENSION_NAME);
     optDevExts.push_back(VK_KHR_SYNCHRONIZATION_2_EXTENSION_NAME);
+    optDevExts.push_back(VK_KHR_TIMELINE_SEMAPHORE_EXTENSION_NAME);
 
     VulkanGraphicsTest::Prepare(argc, argv);
 
@@ -338,6 +341,28 @@ void main()
       sync2Features.pNext = (void *)devInfoNext;
       devInfoNext = &sync2Features;
     }
+
+    static VkPhysicalDeviceGraphicsPipelineLibraryFeaturesEXT gpl = {
+        VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GRAPHICS_PIPELINE_LIBRARY_FEATURES_EXT,
+    };
+
+    if(hasExt(VK_EXT_GRAPHICS_PIPELINE_LIBRARY_EXTENSION_NAME))
+    {
+      gpl.graphicsPipelineLibrary = VK_TRUE;
+      gpl.pNext = (void *)devInfoNext;
+      devInfoNext = &gpl;
+    }
+
+    static VkPhysicalDeviceDynamicRenderingFeaturesKHR dyn = {
+        VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DYNAMIC_RENDERING_FEATURES_KHR,
+    };
+
+    if(hasExt(VK_KHR_DYNAMIC_RENDERING_EXTENSION_NAME))
+    {
+      dyn.dynamicRendering = VK_TRUE;
+      dyn.pNext = (void *)devInfoNext;
+      devInfoNext = &dyn;
+    }
   }
 
   int main()
@@ -1431,6 +1456,64 @@ void main()
     vkDestroyCommandPool(device, cmdPool, NULL);
     vkDestroyDescriptorPool(device, descPool, NULL);
 
+    if(hasExt(VK_EXT_GRAPHICS_PIPELINE_LIBRARY_EXTENSION_NAME) &&
+       hasExt(VK_KHR_DYNAMIC_RENDERING_EXTENSION_NAME))
+    {
+      // create a vertex-input pipeline, with dynamic rendering enabled, and pass garbage in the
+      // dynamic rendering struct. Specify the vertex-pipeline flag *after* the dynamic rendering
+      // struct to force two-pass processing
+
+      VkGraphicsPipelineLibraryCreateInfoEXT libInfo = {};
+      libInfo.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_LIBRARY_CREATE_INFO_EXT;
+      libInfo.flags = VK_GRAPHICS_PIPELINE_LIBRARY_VERTEX_INPUT_INTERFACE_BIT_EXT;
+
+      // none of this should be used
+      VkPipelineRenderingCreateInfoKHR dynInfo = {};
+      dynInfo.sType = VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO_KHR;
+      dynInfo.viewMask = 0x12345678;
+      dynInfo.depthAttachmentFormat = VK_FORMAT_BC1_RGB_SRGB_BLOCK;
+      dynInfo.stencilAttachmentFormat = VK_FORMAT_ASTC_10x10_SFLOAT_BLOCK;
+      dynInfo.pColorAttachmentFormats = (VkFormat *)0x1234;
+      dynInfo.colorAttachmentCount = 1234;
+
+      vkh::GraphicsPipelineCreateInfo libCreateInfo;
+
+      libCreateInfo.layout = layout;
+
+      libCreateInfo.vertexInputState.vertexBindingDescriptions = {vkh::vertexBind(0, DefaultA2V)};
+      libCreateInfo.vertexInputState.vertexAttributeDescriptions = {
+          vkh::vertexAttr(0, 0, DefaultA2V, pos),
+          vkh::vertexAttr(1, 0, DefaultA2V, col),
+          vkh::vertexAttr(2, 0, DefaultA2V, uv),
+      };
+
+      libCreateInfo.pNext = &dynInfo;
+      dynInfo.pNext = &libInfo;
+
+      createGraphicsPipeline(libCreateInfo);
+
+      // for a pre-raster or fragment shader pipeline the viewmask is used, but not the formats still
+      dynInfo.viewMask = 1;
+
+      libInfo.flags = VK_GRAPHICS_PIPELINE_LIBRARY_PRE_RASTERIZATION_SHADERS_BIT_EXT;
+
+      libCreateInfo.vertexInputState.vertexBindingDescriptions.clear();
+      libCreateInfo.vertexInputState.vertexAttributeDescriptions.clear();
+
+      libCreateInfo.stages = {
+          CompileShaderModule(VKDefaultVertex, ShaderLang::glsl, ShaderStage::vert, "main"),
+      };
+
+      createGraphicsPipeline(libCreateInfo);
+
+      libInfo.flags = VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_SHADER_BIT_EXT;
+      libCreateInfo.stages = {
+          CompileShaderModule(VKDefaultPixel, ShaderLang::glsl, ShaderStage::frag, "main"),
+      };
+
+      createGraphicsPipeline(libCreateInfo);
+    }
+
     while(Running())
     {
       // acquire and clear the backbuffer