Reflect multiple entry points & ray properties from DXIL shaders

qrenderdoc/Code/Interface/ShaderProcessingTool.cpp

@@ -279,7 +279,7 @@ ShaderToolOutput ShaderProcessingTool::DisassembleShader(QWidget *window,
       arg = output_file = tmpPath(lit("shader_output"));
     if(arg == lit("{entry_point}"))
     {
-      arg = shaderDetails->entryPoint;
+      arg = shaderDetails->debugInfo.entrySourceName;
       if(arg.isEmpty())
         arg = lit("main");
     }

qrenderdoc/Windows/PipelineState/D3D12PipelineStateViewer.cpp

@@ -1178,7 +1178,7 @@ void D3D12PipelineStateViewer::setShaderState(const D3D12Pipe::Shader &stage, RD
     int entryFile = qMax(0, dbg.entryLocation.fileIndex);
 
     shText += QFormatStr(": %1() - %2")
-                  .arg(shaderDetails->entryPoint)
+                  .arg(shaderDetails->debugInfo.entrySourceName)
                   .arg(QFileInfo(dbg.files[entryFile].filename).fileName());
   }
   shader->setText(shText);
@@ -2851,7 +2851,7 @@ void D3D12PipelineStateViewer::exportHTML(QXmlStreamWriter &xml, const D3D12Pipe
       int entryFile = qMax(0, dbg.entryLocation.fileIndex);
 
       shadername = QFormatStr("%1() - %2")
-                       .arg(shaderDetails->entryPoint)
+                       .arg(shaderDetails->debugInfo.entrySourceName)
                        .arg(QFileInfo(dbg.files[entryFile].filename).fileName());
     }
 

qrenderdoc/Windows/PipelineState/PipelineStateViewer.cpp

@@ -1021,8 +1021,8 @@ IShaderViewer *PipelineStateViewer::EditDecompiledSource(const ShaderProcessingT
     if(flag.name == "@spirver")
       flags.flags.push_back(flag);
 
-  IShaderViewer *sv = EditShader(id, shaderDetails->stage, shaderDetails->entryPoint, flags,
-                                 KnownShaderTool::Unknown, tool.output, files);
+  IShaderViewer *sv = EditShader(id, shaderDetails->stage, shaderDetails->debugInfo.entrySourceName,
+                                 flags, KnownShaderTool::Unknown, tool.output, files);
 
   sv->ShowErrors(out.log);
 

renderdoc/api/replay/rdcflatmap.h

@@ -125,7 +125,7 @@ struct rdcflatmap
 
     size_t idx = lower_bound_idx(val.first);
     bool inserted = false;
-    if(idx >= size() || storage.at(idx).first != val.first)
+    if(idx >= size() || !(storage.at(idx).first == val.first))
     {
       storage.insert(idx, val);
       inserted = true;
@@ -141,7 +141,7 @@ struct rdcflatmap
 
     size_t idx = lower_bound_idx(val.first);
     bool inserted = false;
-    if(idx >= size() || storage.at(idx).first != val.first)
+    if(idx >= size() || !(storage.at(idx).first == val.first))
     {
       storage.insert(idx, std::move(val));
       inserted = true;
@@ -203,7 +203,7 @@ private:
   iterator sorted_find(const Key &id)
   {
     size_t idx = lower_bound_idx(id);
-    if(idx >= size() || storage.at(idx).first != id)
+    if(idx >= size() || !(storage.at(idx).first == id))
       return end();
 
     return begin() + idx;
@@ -212,7 +212,7 @@ private:
   const_iterator sorted_find(const Key &id) const
   {
     size_t idx = lower_bound_idx(id);
-    if(idx >= size() || storage.at(idx).first != id)
+    if(idx >= size() || !(storage.at(idx).first == id))
       return end();
 
     return begin() + idx;
@@ -228,7 +228,7 @@ private:
   Value &sorted_at(const Key &id)
   {
     size_t idx = lower_bound_idx(id);
-    if(idx >= size() || storage.at(idx).first != id)
+    if(idx >= size() || !(storage.at(idx).first == id))
     {
       storage.insert(idx, {id, Value()});
     }

renderdoc/api/replay/shader_types.h

@@ -1753,6 +1753,26 @@ input payload (for mesh shaders)
 :type: ConstantBlock
 )");
   ConstantBlock taskPayload;
+
+  DOCUMENT(R"(The block layout of the ray payload.
+
+Only relevant for raytracing shaders, this gives the payload accessible for read and write by ray
+evaluation during the processing of the ray
+
+:type: ConstantBlock
+)");
+  ConstantBlock rayPayload;
+
+  DOCUMENT(R"(The block layout of the ray attributes structure.
+
+Only relevant for intersection shaders and closest/any hit shaders, this gives
+the attributes structure produced by a custom intersection shader which is available by hit shaders,
+or else the built-in structure if no intersection shader was used and a triangle intersection is
+reported.
+
+:type: ConstantBlock
+)");
+  ConstantBlock rayAttributes;
 };
 
 DECLARE_REFLECTION_STRUCT(ShaderReflection);

renderdoc/driver/d3d11/d3d11_resources.cpp

@@ -93,7 +93,7 @@ void WrappedShader::ShaderEntry::BuildReflection()
       D3Dx_IA_VERTEX_INPUT_RESOURCE_SLOT_COUNT == D3D11_IA_VERTEX_INPUT_RESOURCE_SLOT_COUNT,
       "Mismatched vertex input count");
 
-  MakeShaderReflection(m_DXBCFile, m_Details);
+  MakeShaderReflection(m_DXBCFile, {}, m_Details);
   m_Details->resourceId = m_ID;
 
   DescriptorAccess access;

renderdoc/driver/d3d12/d3d12_resources.cpp

@@ -598,7 +598,7 @@ void WrappedID3D12PipelineState::ShaderEntry::BuildReflection()
       D3Dx_IA_VERTEX_INPUT_RESOURCE_SLOT_COUNT == D3D12_IA_VERTEX_INPUT_RESOURCE_SLOT_COUNT,
       "Mismatched vertex input count");
 
-  MakeShaderReflection(m_DXBCFile, m_Details);
+  MakeShaderReflection(m_DXBCFile, {}, m_Details);
   m_Details->resourceId = GetResourceID();
 }
 

renderdoc/driver/shaders/dxbc/dxbc_container.h

@@ -27,6 +27,7 @@
 
 #include <map>
 #include "api/replay/rdcarray.h"
+#include "api/replay/rdcflatmap.h"
 #include "api/replay/rdcpair.h"
 #include "api/replay/rdcstr.h"
 #include "common/common.h"
@@ -184,6 +185,19 @@ public:
   const Reflection *GetReflection() const { return m_Reflection; }
   D3D_PRIMITIVE_TOPOLOGY GetOutputTopology();
 
+  CBufferVariableType GetRayPayload(const ShaderEntryPoint &entry)
+  {
+    if(m_RayPayloads.empty())
+      return {};
+    return m_RayPayloads[entry].first;
+  }
+  CBufferVariableType GetRayAttributes(const ShaderEntryPoint &entry)
+  {
+    if(m_RayPayloads.empty())
+      return {};
+    return m_RayPayloads[entry].second;
+  }
+
   rdcarray<ShaderEntryPoint> GetEntryPoints() const { return m_EntryPoints; }
 
   const rdcstr &GetDisassembly(bool dxcStyle);
@@ -246,6 +260,8 @@ private:
   size_t m_NonDebugDXILByteCodeOffset = 0;
   size_t m_NonDebugDXILByteCodeSize = 0;
 
+  rdcflatmap<ShaderEntryPoint, rdcpair<CBufferVariableType, CBufferVariableType>> m_RayPayloads;
+
   ShaderStatistics m_ShaderStats;
   DXBCBytecode::Program *m_DXBCByteCode = NULL;
   DXIL::Program *m_DXILByteCode = NULL;

renderdoc/driver/shaders/dxbc/dxbc_reflect.cpp

@@ -293,7 +293,8 @@ static void MakeResourceList(bool srv, DXBC::DXBCContainer *dxbc,
   }
 }
 
-void MakeShaderReflection(DXBC::DXBCContainer *dxbc, ShaderReflection *refl)
+void MakeShaderReflection(DXBC::DXBCContainer *dxbc, const ShaderEntryPoint &entry,
+                          ShaderReflection *refl)
 {
   if(dxbc == NULL || !RenderDoc::Inst().IsReplayApp())
     return;
@@ -308,6 +309,7 @@ void MakeShaderReflection(DXBC::DXBCContainer *dxbc, ShaderReflection *refl)
     case DXBC::ShaderType::Compute: refl->stage = ShaderStage::Compute; break;
     case DXBC::ShaderType::Amplification: refl->stage = ShaderStage::Amplification; break;
     case DXBC::ShaderType::Mesh: refl->stage = ShaderStage::Mesh; break;
+    case DXBC::ShaderType::Library: refl->stage = entry.stage; break;
     default:
       RDCERR("Unexpected DXBC shader type %u", dxbc->m_Type);
       refl->stage = ShaderStage::Vertex;
@@ -318,8 +320,6 @@ void MakeShaderReflection(DXBC::DXBCContainer *dxbc, ShaderReflection *refl)
 
   if(dxbc->GetDebugInfo())
   {
-    refl->debugInfo.entrySourceName = refl->entryPoint = dxbc->GetDebugInfo()->GetEntryFunction();
-
     refl->debugInfo.encoding = ShaderEncoding::HLSL;
 
     refl->debugInfo.sourceDebugInformation = true;
@@ -330,9 +330,22 @@ void MakeShaderReflection(DXBC::DXBCContainer *dxbc, ShaderReflection *refl)
 
     dxbc->GetDebugInfo()->GetLineInfo(~0U, ~0U, refl->debugInfo.entryLocation);
 
-    rdcstr entry = dxbc->GetDebugInfo()->GetEntryFunction();
-    if(entry.empty())
-      entry = "main";
+    rdcstr entryFunc = entry.name;
+    if(entryFunc.empty())
+      entryFunc = dxbc->GetDebugInfo()->GetEntryFunction();
+    if(entryFunc.empty())
+      entryFunc = "main";
+
+    refl->debugInfo.entrySourceName = refl->entryPoint = entryFunc;
+
+    // demangle DXIL source names for display
+    if(refl->debugInfo.entrySourceName.size() > 2 && refl->debugInfo.entrySourceName[0] == '\x1' &&
+       refl->debugInfo.entrySourceName[1] == '?')
+    {
+      int idx = refl->debugInfo.entrySourceName.indexOf('@');
+      if(idx > 2)
+        refl->debugInfo.entrySourceName = refl->debugInfo.entrySourceName.substr(2, idx - 2);
+    }
 
     // assume the debug info put the file with the entry point at the start. SDBG seems to do this
     // by default, and SPDB has an extra sorting step that probably maybe possibly does this.
@@ -349,6 +362,7 @@ void MakeShaderReflection(DXBC::DXBCContainer *dxbc, ShaderReflection *refl)
       case DXBC::ShaderType::Hull: profile = "hs"; break;
       case DXBC::ShaderType::Domain: profile = "ds"; break;
       case DXBC::ShaderType::Compute: profile = "cs"; break;
+      case DXBC::ShaderType::Library: profile = "lib"; break;
       default: profile = "xx"; break;
     }
     profile += StringFormat::Fmt("_%u_%u", dxbc->m_Version.Major, dxbc->m_Version.Minor);
@@ -469,4 +483,24 @@ void MakeShaderReflection(DXBC::DXBCContainer *dxbc, ShaderReflection *refl)
     refl->taskPayload.variables.push_back(
         MakeConstantBufferVariable(false, dxbcRefl->TaskPayload.members[v]));
   }
+
+  DXBC::CBufferVariableType RayPayload = dxbc->GetRayPayload(entry);
+  DXBC::CBufferVariableType RayAttributes = dxbc->GetRayAttributes(entry);
+
+  refl->rayPayload.bufferBacked = false;
+  refl->rayPayload.name = RayPayload.name;
+  refl->rayPayload.variables.reserve(RayPayload.members.size());
+  for(size_t v = 0; v < RayPayload.members.size(); v++)
+  {
+    refl->rayPayload.variables.push_back(MakeConstantBufferVariable(false, RayPayload.members[v]));
+  }
+
+  refl->rayAttributes.bufferBacked = false;
+  refl->rayAttributes.name = RayAttributes.name;
+  refl->rayAttributes.variables.reserve(RayAttributes.members.size());
+  for(size_t v = 0; v < RayAttributes.members.size(); v++)
+  {
+    refl->rayAttributes.variables.push_back(
+        MakeConstantBufferVariable(false, RayAttributes.members[v]));
+  }
 }

renderdoc/driver/shaders/dxbc/dxbc_reflect.h

@@ -30,7 +30,9 @@ class DXBCContainer;
 }
 
 struct ShaderReflection;
+struct ShaderEntryPoint;
 
 #define D3Dx_IA_VERTEX_INPUT_RESOURCE_SLOT_COUNT 32
 
-void MakeShaderReflection(DXBC::DXBCContainer *dxbc, ShaderReflection *refl);
+void MakeShaderReflection(DXBC::DXBCContainer *dxbc, const ShaderEntryPoint &entry,
+                          ShaderReflection *refl);

renderdoc/driver/shaders/dxil/dxil_bytecode.h

@@ -27,6 +27,7 @@
 #include <stdint.h>
 
 #include "api/replay/apidefs.h"
+#include "api/replay/rdcflatmap.h"
 #include "api/replay/rdcstr.h"
 #include "common/common.h"
 #include "driver/dx/official/d3dcommon.h"
@@ -1230,6 +1231,10 @@ public:
   void FetchComputeProperties(DXBC::Reflection *reflection);
   DXBC::Reflection *GetReflection();
   rdcarray<ShaderEntryPoint> GetEntryPoints();
+  void FillRayPayloads(
+      Program *executable,
+      rdcflatmap<ShaderEntryPoint, rdcpair<DXBC::CBufferVariableType, DXBC::CBufferVariableType>>
+          &rayPayloads);
 
   DXBC::ShaderType GetShaderType() const { return m_Type; }
   uint32_t GetMajorVersion() const { return m_Major; }

renderdoc/driver/shaders/dxil/dxil_reflect.cpp

@@ -703,6 +703,175 @@ void Program::FetchComputeProperties(DXBC::Reflection *reflection)
   reflection->DispatchThreadsDimension[2] = 1;
 }
 
+void Program::FillRayPayloads(
+    Program *executable,
+    rdcflatmap<ShaderEntryPoint, rdcpair<DXBC::CBufferVariableType, DXBC::CBufferVariableType>> &rayPayloads)
+{
+  if(m_Type != DXBC::ShaderType::Library)
+    return;
+
+  DXMeta dx(m_NamedMeta);
+
+  TypeInfo typeInfo(dx.typeAnnotations);
+
+  if(dx.entryPoints)
+  {
+    for(Metadata *entry : dx.entryPoints->children)
+    {
+      if(entry->children.size() > 2 && entry->children[0] != NULL)
+      {
+        ShaderEntryPoint entryPoint;
+        entryPoint.name = entry->children[1]->str;
+
+        Metadata *tags = entry->children[4];
+
+        for(size_t i = 0; i < tags->children.size(); i += 2)
+        {
+          // 8 is the type tag
+          if(getival<uint32_t>(tags->children[i]) == 8U)
+          {
+            entryPoint.stage =
+                GetShaderStage((DXBC::ShaderType)getival<uint32_t>(tags->children[i + 1]));
+            break;
+          }
+        }
+
+        Function *ownFunc = cast<Function>(entry->children[0]->value);
+        Function *executableFunc = NULL;
+
+        // locate the function in the executable program so we can iterate instructions.
+        for(Function *f : executable->m_Functions)
+        {
+          // assume names will match
+          if(f->name == ownFunc->name)
+          {
+            executableFunc = f;
+            break;
+          }
+        }
+
+        // intersection shaders only report attributes, they do not access the ray payload
+        if(entryPoint.stage == ShaderStage::Intersection)
+        {
+          // find the reportHit and grab the type from that
+          for(const Instruction *in : executableFunc->instructions)
+          {
+            const Instruction &inst = *in;
+
+            if(inst.op == Operation::Call && inst.getFuncCall()->name.beginsWith("dx.op.reportHit"))
+            {
+              if(inst.args.size() != 4)
+              {
+                RDCERR("Unexpected number of arguments to reportHit");
+                continue;
+              }
+              const Type *executableAttrType = inst.args[3]->type;
+              if(!executableAttrType)
+              {
+                RDCERR("Unexpected untyped payload argument to reportHit");
+                continue;
+              }
+
+              RDCASSERT(executableAttrType->type == Type::Pointer);
+              executableAttrType = (Type *)executableAttrType->inner;
+
+              Type *ownAttrType = NULL;
+
+              // we have the executable type but we can't use that to look up our type info. Try to
+              // go back by name
+              for(Type *t : m_Types)
+              {
+                if(t->type == executableAttrType->type && t->name == executableAttrType->name)
+                {
+                  ownAttrType = t;
+                  break;
+                }
+              }
+
+              if(ownAttrType)
+                rayPayloads[entryPoint].second = MakePayloadType(typeInfo, ownAttrType);
+              else
+                RDCERR("Couldn't find matching attribute type for '%s' by name",
+                       executableAttrType->name);
+              break;
+            }
+          }
+        }
+        // raygen shaders only use the ray payload, not attributes
+        else if(entryPoint.stage == ShaderStage::RayGen)
+        {
+          // find the reportHit and grab the type from that
+          for(const Instruction *in : executableFunc->instructions)
+          {
+            const Instruction &inst = *in;
+
+            if(inst.op == Operation::Call && inst.getFuncCall()->name.beginsWith("dx.op.traceRay"))
+            {
+              if(inst.args.size() != 16)
+              {
+                RDCERR("Unexpected number of arguments to traceRay");
+                continue;
+              }
+              const Type *executablePayloadType = inst.args[15]->type;
+              if(!executablePayloadType)
+              {
+                RDCERR("Unexpected untyped payload argument to traceRay");
+                continue;
+              }
+
+              RDCASSERT(executablePayloadType->type == Type::Pointer);
+              executablePayloadType = (Type *)executablePayloadType->inner;
+
+              Type *ownPayloadType = NULL;
+
+              // we have the executable type but we can't use that to look up our type info. Try to
+              // go back by name
+              for(Type *t : m_Types)
+              {
+                if(t->type == executablePayloadType->type && t->name == executablePayloadType->name)
+                {
+                  ownPayloadType = t;
+                  break;
+                }
+              }
+
+              if(ownPayloadType)
+                rayPayloads[entryPoint].first = MakePayloadType(typeInfo, ownPayloadType);
+              else
+                RDCERR("Couldn't find matching payload type for '%s' by name",
+                       executablePayloadType->name);
+
+              break;
+            }
+          }
+        }
+        else if(entryPoint.stage == ShaderStage::Miss || entryPoint.stage == ShaderStage::AnyHit ||
+                entryPoint.stage == ShaderStage::ClosestHit)
+        {
+          const Type *payloadType = ownFunc->type->members[0];
+          RDCASSERT(payloadType->type == Type::Pointer);
+          payloadType = (Type *)payloadType->inner;
+
+          rdcpair<DXBC::CBufferVariableType, DXBC::CBufferVariableType> &dst =
+              rayPayloads[entryPoint];
+
+          // miss shaders only use the payload, any-hit and closest-hit use both. The first
+          // parameter is the payload, the second is the attributes
+          dst.first = MakePayloadType(typeInfo, payloadType);
+          if(entryPoint.stage != ShaderStage::Miss)
+          {
+            const Type *attrType = ownFunc->type->members[1];
+            RDCASSERT(attrType->type == Type::Pointer);
+            attrType = (Type *)attrType->inner;
+
+            dst.second = MakePayloadType(typeInfo, attrType);
+          }
+        }
+      }
+    }
+  }
+}
+
 D3D_PRIMITIVE_TOPOLOGY Program::GetOutputTopology()
 {
   if(m_Type != DXBC::ShaderType::Geometry && m_Type != DXBC::ShaderType::Domain &&

renderdoc/replay/renderdoc_serialise.inl

@@ -295,8 +295,10 @@ void DoSerialise(SerialiserType &ser, ShaderReflection &el)
   SERIALISE_MEMBER(pointerTypes);
 
   SERIALISE_MEMBER(taskPayload);
+  SERIALISE_MEMBER(rayPayload);
+  SERIALISE_MEMBER(rayAttributes);
 
-  SIZE_CHECK(488);
+  SIZE_CHECK(632);
 }
 
 template <typename SerialiserType>