Add stubbed structures for RDAT decoding/encoding

* We have parallel structures that are mostly the same but are 'flattened' with
  indices for lookups for arrays or strings

renderdoc/driver/shaders/dxbc/dxbc_common.h

@@ -56,7 +56,7 @@ struct CountOffset
 };
 
 // matches D3D11_SHADER_VERSION_TYPE from d3d11shader.h
-enum class ShaderType
+enum class ShaderType : uint32_t
 {
   // D3D10 Shaders
   Pixel = 0,

renderdoc/driver/shaders/dxbc/dxbc_container.cpp

@@ -1701,12 +1701,13 @@ DXBCContainer::DXBCContainer(const bytebuf &ByteCode, const rdcstr &debugInfoPat
     }
     else if(*fourcc == FOURCC_RDAT)
     {
-      // runtime data
+      m_RDATOffset = chunkContents - data;
+      m_RDATSize = *chunkSize;
     }
     else if(*fourcc == FOURCC_PSV0)
     {
-      // this chunk contains some information we could use for reflection but it doesn't contain
-      // enough, and doesn't have anything else interesting so we skip it
+      m_PSVOffset = chunkContents - data;
+      m_PSVSize = *chunkSize;
     }
     else if(*fourcc == FOURCC_ISGN || *fourcc == FOURCC_OSGN || *fourcc == FOURCC_ISG1 ||
             *fourcc == FOURCC_OSG1 || *fourcc == FOURCC_OSG5 || *fourcc == FOURCC_PCSG ||

renderdoc/driver/shaders/dxbc/dxbc_container.h

@@ -45,6 +45,13 @@ bool IsPDBFile(void *data, size_t length);
 void UnwrapEmbeddedPDBData(bytebuf &bytes);
 };
 
+namespace DXIL
+{
+// defined in dxil_metadata.h as these have heavy dependency on dxil enums
+struct PSVData;
+struct RDATData;
+};
+
 // many thanks to winehq for information of format of RDEF, STAT and SIGN chunks:
 // http://source.winehq.org/git/wine.git/blob/HEAD:/dlls/d3dcompiler_43/reflection.c
 namespace DXBC
@@ -238,6 +245,12 @@ public:
   static rdcstr GetDebugBinaryPath(const void *ByteCode, size_t ByteCodeLength);
   static D3D_PRIMITIVE_TOPOLOGY GetOutputTopology(const void *ByteCode, size_t ByteCodeLength);
 
+  bool GetPipelineValidation(DXIL::PSVData &psv) const;
+  bool GetRuntimeData(DXIL::RDATData &rdat) const;
+
+  static void SetPipelineValidation(bytebuf &ByteCode, const DXIL::PSVData &psv);
+  static void SetRuntimeData(bytebuf &ByteCode, const DXIL::RDATData &rdat);
+
 private:
   void TryFetchSeparateDebugInfo(bytebuf &byteCode, const rdcstr &debugInfoPath);
 
@@ -262,6 +275,11 @@ private:
   size_t m_NonDebugDXILByteCodeOffset = 0;
   size_t m_NonDebugDXILByteCodeSize = 0;
 
+  size_t m_RDATOffset = 0;
+  size_t m_RDATSize = 0;
+  size_t m_PSVOffset = 0;
+  size_t m_PSVSize = 0;
+
   rdcflatmap<ShaderEntryPoint, rdcpair<CBufferVariableType, CBufferVariableType>> m_RayPayloads;
 
   ShaderStatistics m_ShaderStats;

renderdoc/driver/shaders/dxil/dxil_common.h

@@ -24,6 +24,12 @@
 
 #pragma once
 
+namespace DXBC
+{
+enum class ShaderType : uint32_t;
+enum class GlobalShaderFlags : int64_t;
+};
+
 namespace DXIL
 {
 enum class ResourceClass

renderdoc/driver/shaders/dxil/dxil_metadata.cpp

@@ -0,0 +1,296 @@
+/******************************************************************************
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2024 Baldur Karlsson
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ ******************************************************************************/
+
+#include "dxil_metadata.h"
+
+#include "driver/shaders/dxbc/dxbc_container.h"
+#include "serialise/streamio.h"
+
+// serialise/encode helpers
+namespace
+{
+
+struct RuntimePartHeader
+{
+  DXIL::RDATData::Part part;
+  uint32_t size;
+};
+
+struct RuntimePartTableHeader
+{
+  uint32_t count;
+  uint32_t stride;
+};
+
+// slightly type-safer way of returning an index/offset encoded as a uint
+struct IndexReference
+{
+  uint32_t offset;
+};
+
+struct BytesReference
+{
+  uint32_t offset;
+  uint32_t size;
+};
+
+static IndexReference MakeStringRef(rdcstr &stringblob, const rdcstr &str)
+{
+  // not efficient, we don't cache anything but do a straight linear search.
+  uint32_t offs = 0;
+
+  while(offs < stringblob.length())
+  {
+    const char *curstr = stringblob.c_str() + offs;
+    uint32_t curlen = (uint32_t)strlen(curstr);
+
+    if(curlen == str.length() && strcmp(curstr, str.c_str()) == 0)
+      return {offs};
+
+    // skip past the NULL terminator to the start of the next string
+    offs += curlen + 1;
+  }
+
+  uint32_t ret = (uint32_t)stringblob.length();
+  stringblob.append(str);
+  // we need to explicitly include the NULL terminators
+  stringblob.push_back('\0');
+  return {ret};
+}
+
+static BytesReference MakeBytesRef(rdcarray<bytebuf> &bytesblobs, const bytebuf &bytes)
+{
+  // ~0U indicates empty bytes
+  if(bytes.empty())
+    return {~0U};
+
+  // super inefficient but we don't expect there to be many bytes blobs (only root signatures)
+  int32_t idx = bytesblobs.indexOf(bytes);
+  if(idx < 0)
+  {
+    size_t offs = 0;
+    for(size_t i = 0; i < bytesblobs.size(); i++)
+      offs += bytesblobs[i].size();
+
+    bytesblobs.push_back(bytes);
+    return {(uint32_t)offs, (uint32_t)bytes.size()};
+  }
+
+  size_t offs = 0;
+  for(int32_t i = 0; i < idx; i++)
+    offs += bytesblobs[i].size();
+  return {(uint32_t)offs, (uint32_t)bytes.size()};
+}
+
+static IndexReference MakeIndexArrayRef(rdcarray<uint32_t> &idxArrays,
+                                        const rdcarray<uint32_t> &idxs, bool emptyIsNull)
+{
+  // ~0U indicates NULL, in some cases replaces an empty array
+  if(emptyIsNull && idxs.empty())
+    return {~0U};
+
+  // not efficient, we don't cache anything but do a straight linear search.
+  uint32_t offs = 0;
+
+  while(offs < idxArrays.size())
+  {
+    const uint32_t *curarray = idxArrays.data() + offs;
+    // length-prefix on array
+    uint32_t curlen = *curarray;
+    curarray++;
+
+    if(curlen == idxs.size())
+    {
+      bool allsame = true;
+      for(uint32_t i = 0; i < curlen && allsame; i++)
+        allsame &= idxs[i] == curarray[i];
+
+      if(allsame)
+        return {offs};
+    }
+
+    // skip past the length and the current array
+    offs += 1 + curlen;
+  }
+
+  uint32_t ret = (uint32_t)idxArrays.size();
+  // idx arrays are length prefixed
+  idxArrays.push_back((uint32_t)idxs.size());
+  idxArrays.append(idxs);
+  return {ret};
+}
+
+// serialised equivalent to RDAT::ResourceInfo
+struct EncodedResourceInfo
+{
+  DXIL::ResourceClass nspace;
+  DXIL::ResourceKind kind;
+  uint32_t LinearID;
+  uint32_t space;
+  uint32_t regStart;
+  uint32_t regEnd;
+  IndexReference name;
+  DXIL::RDATData::ResourceFlags flags;
+};
+
+// serialised equivalent to RDAT::FunctionInfo
+struct EncodedFunctionInfo
+{
+  IndexReference name;
+  IndexReference unmangledName;
+  IndexReference globalResourcesIndexArrayRef;
+  IndexReference functionDependenciesArrayRef;
+  DXBC::ShaderType type;
+  uint32_t payloadBytes;
+  uint32_t attribBytes;
+  // extremely annoyingly this is two 32-bit integers which is relevant since 64-bit alignment
+  // causes extra packing in the struct
+  uint32_t featureFlags[2];
+  uint32_t shaderCompatMask;    // bitmask based on DXBC::ShaderType enum of stages this function
+                                // could be used with.
+  uint16_t minShaderModel;
+  uint16_t minType;    // looks to always be equal to type above
+};
+
+// serialised equivalent to RDAT::FunctionInfo2
+struct EncodedFunctionInfo2
+{
+  EncodedFunctionInfo info1;
+
+  uint8_t minWaveCount;
+  uint8_t maxWaveCount;
+  DXIL::RDATData::ShaderBehaviourFlags shaderBehaviourFlags;
+
+  // below here is a stage-specific set of data containing e.g. signature elements. Currently
+  // DXC does not emit RDAT except for in library targets, so this will be unused. It would be an
+  // index into a table elsewhere of VSInfo, PSInfo, etc.
+  IndexReference extraInfoRef;
+};
+
+// serialised equivalent to RDAT::FunctionInfo2
+struct EncodedSubobjectInfo
+{
+  DXIL::RDATData::SubobjectInfo::SubobjectType type;
+  IndexReference name;
+
+  // we union members where possible but several contain arrays/strings which can't be unioned.
+
+  union
+  {
+    DXIL::RDATData::SubobjectInfo::StateConfig config;
+    DXIL::RDATData::SubobjectInfo::RTShaderConfig rtshaderconfig;
+    DXIL::RDATData::SubobjectInfo::RTPipeConfig1 rtpipeconfig;
+
+    struct
+    {
+      BytesReference data;
+    } rs;
+
+    struct
+    {
+      IndexReference subobject;
+      IndexReference exports;
+    } assoc;
+
+    struct
+    {
+      DXIL::RDATData::HitGroupType type;
+      IndexReference anyHit;
+      IndexReference closestHit;
+      IndexReference intersection;
+    } hitgroup;
+  };
+};
+
+static void BakeRuntimePart(rdcarray<bytebuf> &parts, DXIL::RDATData::Part part, void *data,
+                            uint32_t byteSize)
+{
+  // empty parts are skipped
+  if(byteSize == 0)
+    return;
+
+  const uint32_t alignedDataSize = (uint32_t)AlignUp4(byteSize);
+  const RuntimePartHeader header = {part, alignedDataSize};
+
+  bytebuf b;
+  b.resize(alignedDataSize + sizeof(header));
+  memcpy(b.data(), &header, sizeof(header));
+  memcpy(b.data() + sizeof(header), data, byteSize);
+  parts.push_back(std::move(b));
+}
+
+template <typename TableType>
+static void BakeRuntimeTablePart(rdcarray<bytebuf> &parts, DXIL::RDATData::Part part,
+                                 const rdcarray<TableType> &entries)
+{
+  // empty parts are skipped
+  if(entries.size() == 0)
+    return;
+
+  const uint32_t alignedEntriesSize = (uint32_t)AlignUp4(entries.byteSize());
+  const RuntimePartTableHeader tableHeader = {(uint32_t)entries.count(),
+                                              (uint32_t)AlignUp4(sizeof(TableType))};
+  const RuntimePartHeader header = {part, alignedEntriesSize + sizeof(tableHeader)};
+
+  bytebuf b;
+  b.resize(alignedEntriesSize + sizeof(header) + sizeof(tableHeader));
+  memcpy(b.data(), &header, sizeof(header));
+  memcpy(b.data() + sizeof(header), &tableHeader, sizeof(tableHeader));
+  memcpy(b.data() + sizeof(header) + sizeof(tableHeader), entries.data(), entries.byteSize());
+  parts.push_back(std::move(b));
+}
+
+};
+
+namespace DXBC
+{
+bool DXBCContainer::GetPipelineValidation(DXIL::PSVData &psv) const
+{
+  using namespace DXIL;
+
+  if(m_PSVOffset == 0)
+    return false;
+
+  return true;
+}
+
+void DXBCContainer::SetPipelineValidation(bytebuf &ByteCode, const DXIL::PSVData &psv)
+{
+}
+
+bool DXBCContainer::GetRuntimeData(DXIL::RDATData &rdat) const
+{
+  using namespace DXIL;
+
+  if(m_RDATOffset == 0)
+    return false;
+
+  return true;
+}
+
+void DXBCContainer::SetRuntimeData(bytebuf &ByteCode, const DXIL::RDATData &rdat)
+{
+}
+
+};    // namespace DXBC

renderdoc/driver/shaders/dxil/dxil_metadata.h

@@ -0,0 +1,237 @@
+/******************************************************************************
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2019-2024 Baldur Karlsson
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ ******************************************************************************/
+
+#pragma once
+
+#include "dxil_common.h"
+
+namespace DXBC
+{
+enum class ShaderType : uint32_t;
+enum class GlobalShaderFlags : int64_t;
+};
+
+namespace DXIL
+{
+
+struct PSVData
+{
+};
+
+struct RDATData
+{
+  enum
+  {
+    Version1_0 = 0x10
+  };
+
+  enum class Part
+  {
+    Invalid = 0,
+    StringBuffer = 1,
+    IndexArrays = 2,
+    ResourceTable = 3,
+    FunctionTable = 4,
+    RawBytes = 5,
+    SubobjectTable = 6,
+  };
+
+  enum class ResourceFlags
+  {
+    None = 0x0,
+    GloballyCoherent = 0x1,
+    HasCounter = 0x2,
+    ROV = 0x4,
+    // unused dynamic indexing flag? 0x8
+    Atomic64 = 0x10,
+  };
+
+  // name arbitrarily chosen to avoid the extremely generic "shader flags" naming
+  enum class ShaderBehaviourFlags : uint16_t
+  {
+    None = 0x0,
+    NodeProgramEntry = 0x1,
+    SVPositionOutput = 0x2,
+    SVDepthOutput = 0x4,
+    SampleRate = 0x8,
+    ViewID = 0x10,
+  };
+
+  struct ResourceInfo
+  {
+    ResourceClass nspace;    // SRV, UAV, Sampler, CB
+    ResourceKind kind;       // texture type (2D, 3D, etc) or other resource binding type
+    uint32_t resourceIndex;    // the 0-based ID of this resource within the class namespace (SRV, UAV, etc).
+    uint32_t space;       // register space
+    uint32_t regStart;    // start register (inclusive - for single register bind it's == reg)
+    uint32_t regEnd;      // end register (inclusive - for single register bind it's == reg)
+    rdcstr name;
+    ResourceFlags flags;
+
+    bool operator==(const ResourceInfo &o)
+    {
+      // use namespace and linear ID to look up resources
+      return nspace == o.nspace && resourceIndex == o.resourceIndex;
+    }
+
+    bool operator==(const rdcpair<DXIL::ResourceClass, uint32_t> &o)
+    {
+      return nspace == o.first && resourceIndex == o.second;
+    }
+  };
+
+  struct FunctionInfo
+  {
+    rdcstr name;
+    rdcstr unmangledName;
+    rdcarray<rdcpair<ResourceClass, uint32_t>> globalResources;
+    rdcarray<rdcstr> functionDependencies;
+    DXBC::ShaderType type;
+    uint32_t payloadBytes;
+    uint32_t attribBytes;
+    DXBC::GlobalShaderFlags featureFlags;
+    uint32_t shaderCompatMask;    // bitmask based on DXBC::ShaderType enum of stages this function could be used with.
+    uint16_t minShaderModel;
+    uint16_t minType;    // looks to always be equal to type above
+  };
+
+  struct FunctionInfo2 : FunctionInfo
+  {
+    FunctionInfo2(FunctionInfo &info) : FunctionInfo(info)
+    {
+      minWaveCount = maxWaveCount = 0;
+      shaderBehaviourFlags = ShaderBehaviourFlags::None;
+      extraInfoRef = ~0U;
+    }
+
+    uint8_t minWaveCount;
+    uint8_t maxWaveCount;
+    ShaderBehaviourFlags shaderBehaviourFlags;
+
+    // below here is a stage-specific set of data containing e.g. signature elements. Currently
+    // DXC does not emit RDAT except for in library targets, so this will be unused. It would be an
+    // index into a table elsewhere of VSInfo, PSInfo, etc.
+    uint32_t extraInfoRef;
+  };
+
+  enum class StateObjectFlags : uint32_t
+  {
+    None = 0x0,
+    LocalDepsOnExternals = 0x1,
+    ExternalDepsOnLocals = 0x2,
+    AllowAdditions = 0x4,
+  };
+
+  enum class HitGroupType : uint32_t
+  {
+    Triangle = 0,
+    ProceduralPrimitive = 1,
+  };
+
+  enum class RTPipeFlags : uint32_t
+  {
+    None = 0x0,
+    SkipTriangles = 0x100,
+    SkipProcedural = 0x200,
+  };
+
+  struct SubobjectInfo
+  {
+    // values match D3D12_STATE_SUBOBJECT_TYPE
+    enum class SubobjectType : uint32_t
+    {
+      StateConfig = 0,
+      GlobalRS = 1,
+      LocalRS = 2,
+      // missing enum values
+      SubobjectToExportsAssoc = 8,
+      RTShaderConfig = 9,
+      RTPipeConfig = 10,
+      Hitgroup = 11,
+      RTPipeConfig1 = 12,
+    } type;
+    rdcstr name;
+
+    // we union members where possible but several contain arrays/strings which can't be unioned.
+
+    struct StateConfig
+    {
+      StateObjectFlags flags;
+    };
+
+    struct RTShaderConfig
+    {
+      uint32_t maxPayloadBytes;
+      uint32_t maxAttribBytes;
+    };
+
+    struct RTPipeConfig1
+    {
+      uint32_t maxRecursion;
+      RTPipeFlags flags;
+    };
+
+    union
+    {
+      StateConfig config;
+      RTShaderConfig rtshaderconfig;
+      RTPipeConfig1 rtpipeconfig;
+    };
+
+    struct RootSig
+    {
+      bytebuf data;
+    } rs;
+
+    struct Assoc
+    {
+      rdcstr subobject;
+      rdcarray<rdcstr> exports;
+    } assoc;
+
+    struct Hitgroup
+    {
+      HitGroupType type;
+      rdcstr anyHit;
+      rdcstr closestHit;
+      rdcstr intersection;
+    } hitgroup;
+  };
+
+  enum class FunctionInfoVersion
+  {
+    Version1 = 1,
+    Version2,
+    VersionLatest = Version2,
+  } functionVersion = FunctionInfoVersion::VersionLatest;
+
+  rdcarray<ResourceInfo> resourceInfo;
+  rdcarray<FunctionInfo2> functionInfo;
+  rdcarray<SubobjectInfo> subobjectsInfo;
+};
+
+};
+
+BITMASK_OPERATORS(DXIL::RDATData::ResourceFlags);
+BITMASK_OPERATORS(DXIL::RDATData::ShaderBehaviourFlags);

renderdoc/driver/shaders/dxil/renderdoc_dxil.vcxproj

@@ -107,6 +107,7 @@
     <ClCompile Include="dxil_debug.cpp" />
     <ClCompile Include="dxil_debuginfo.cpp" />
     <ClCompile Include="dxil_disassemble.cpp" />
+    <ClCompile Include="dxil_metadata.cpp" />
     <ClCompile Include="dxil_reflect.cpp" />
     <ClCompile Include="dxil_stringise.cpp" />
     <ClCompile Include="llvm_decoder.cpp" />
@@ -121,6 +122,7 @@
     <ClInclude Include="dxil_common.h" />
     <ClInclude Include="dxil_debug.h" />
     <ClInclude Include="dxil_debuginfo.h" />
+    <ClInclude Include="dxil_metadata.h" />
     <ClInclude Include="llvm_bitreader.h" />
     <ClInclude Include="llvm_bitwriter.h" />
     <ClInclude Include="llvm_common.h" />

renderdoc/driver/shaders/dxil/renderdoc_dxil.vcxproj.filters

@@ -14,6 +14,7 @@
     <ClCompile Include="llvm_encoder.cpp" />
     <ClCompile Include="dxil_debug.cpp" />
     <ClCompile Include="dxil_stringise.cpp" />
+    <ClCompile Include="dxil_metadata.cpp" />
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="precompiled.h">
@@ -29,6 +30,7 @@
     <ClInclude Include="llvm_encoder.h" />
     <ClInclude Include="llvm_common.h" />
     <ClInclude Include="dxil_debug.h" />
+    <ClInclude Include="dxil_metadata.h" />
   </ItemGroup>
   <ItemGroup>
     <Filter Include="PCH">