Allow querying the shader encoding supported by custom shaders

* This is distinct from target shaders in the remote proxy case - custom shaders
  are built locally.

qrenderdoc/Code/CaptureContext.cpp

@@ -820,6 +820,7 @@ void CaptureContext::LoadCaptureThreaded(const QString &captureFile, const QStri
 
     m_APIProps = r->GetAPIProperties();
 
+    m_CustomEncodings = r->GetCustomShaderEncodings();
     m_TargetEncodings = r->GetTargetShaderEncodings();
 
     m_PostloadProgress = 0.2f;

qrenderdoc/Code/CaptureContext.h

@@ -141,6 +141,7 @@ public:
   CaptureModifications GetCaptureModifications() override { return m_CaptureMods; }
   const FrameDescription &FrameInfo() override { return m_FrameInfo; }
   const APIProperties &APIProps() override { return m_APIProps; }
+  rdcarray<ShaderEncoding> CustomShaderEncodings() override { return m_CustomEncodings; }
   rdcarray<ShaderEncoding> TargetShaderEncodings() override { return m_TargetEncodings; }
   uint32_t CurSelectedEvent() override { return m_SelectedEventID; }
   uint32_t CurEvent() override { return m_EventID; }
@@ -332,7 +333,7 @@ private:
   const rdcarray<DrawcallDescription> *m_Drawcalls;
   rdcarray<DrawcallDescription> m_EmptyDraws;
 
-  rdcarray<ShaderEncoding> m_TargetEncodings;
+  rdcarray<ShaderEncoding> m_CustomEncodings, m_TargetEncodings;
   APIProperties m_APIProps;
   FrameDescription m_FrameInfo;
   const DrawcallDescription *m_FirstDrawcall = NULL;

qrenderdoc/Code/Interface/QRDInterface.h

@@ -532,7 +532,7 @@ DECLARE_REFLECTION_STRUCT(IPythonShell);
 
 DOCUMENT(R"(A shader window used for viewing, editing, or debugging.
 
-.. function:: SaveCallback(context, viewer, files)
+.. function:: SaveCallback(context, viewer, encoding, flags, entry, compiled)
 
   Not a member function - the signature for any ``SaveCallback`` callbacks.
 
@@ -1228,6 +1228,15 @@ building target shaders for the currently loaded capture. See
 )");
   virtual rdcarray<ShaderEncoding> TargetShaderEncodings() = 0;
 
+  DOCUMENT(R"(Retrieve the list of :class:`~renderdoc.ShaderEncoding` that are available for
+building custom shaders for the currently loaded capture. See
+:meth:`~renderdoc.ReplayController.BuildCustomShader`.
+
+:return: The available encodings.
+:rtype: ``list`` of :class:`~renderdoc.ShaderEncoding`
+)");
+  virtual rdcarray<ShaderEncoding> CustomShaderEncodings() = 0;
+
   DOCUMENT(R"(Retrieve the currently selected :data:`eventId <renderdoc.APIEvent.eventId>`.
 
 In most cases, prefer using :meth:`CurEvent`. See :meth:`CaptureViewer.OnSelectedEventChanged` for more

qrenderdoc/Windows/PythonShell.cpp

@@ -69,6 +69,10 @@ struct CaptureContextInvoker : ICaptureContext
   {
     return m_Ctx.TargetShaderEncodings();
   }
+  virtual rdcarray<ShaderEncoding> CustomShaderEncodings() override
+  {
+    return m_Ctx.CustomShaderEncodings();
+  }
   virtual uint32_t CurSelectedEvent() override { return m_Ctx.CurSelectedEvent(); }
   virtual uint32_t CurEvent() override { return m_Ctx.CurEvent(); }
   virtual const DrawcallDescription *CurSelectedDrawcall() override

renderdoc/api/replay/renderdoc_replay.h

@@ -1039,6 +1039,21 @@ of the compile process or using alternate/updated tools.
 )");
   virtual rdcarray<ShaderEncoding> GetTargetShaderEncodings() = 0;
 
+  DOCUMENT(R"(Retrieve the list of supported :class:`ShaderEncoding` which can be build using
+:meth:`BuildCustomShader`.
+
+The list is sorted in priority order, so if the caller has a shader in a form but could
+compile/translate it to another, prefer to satisfy the first encoding before later encodings.
+
+This typically means the 'native' encoding is listed first, and then subsequent encodings are
+compiled internally - so compiling externally could be preferable as it allows better customisation
+of the compile process or using alternate/updated tools.
+
+:return: The list of target shader encodings available.
+:rtype: ``list`` of :class:`ShaderEncoding`
+)");
+  virtual rdcarray<ShaderEncoding> GetCustomShaderEncodings() = 0;
+
   DOCUMENT(R"(Replace one resource with another for subsequent replay and analysis work.
 
 This is commonly used for modifying the capture by selectively replacing resources with newly

renderdoc/core/image_viewer.cpp

@@ -141,6 +141,10 @@ public:
   {
     return m_Proxy->GetTargetShaderEncodings();
   }
+  rdcarray<ShaderEncoding> GetCustomShaderEncodings()
+  {
+    return m_Proxy->GetCustomShaderEncodings();
+  }
   void BuildCustomShader(std::string source, std::string entry,
                          const ShaderCompileFlags &compileFlags, ShaderStage type, ResourceId *id,
                          std::string *errors)

renderdoc/core/replay_proxy.h

@@ -418,6 +418,14 @@ public:
     }
   }
 
+  rdcarray<ShaderEncoding> GetCustomShaderEncodings()
+  {
+    if(m_Proxy)
+      return m_Proxy->GetCustomShaderEncodings();
+
+    return {};
+  }
+
   void FreeCustomShader(ResourceId id)
   {
     if(m_Proxy)

renderdoc/driver/d3d11/d3d11_replay.h

@@ -181,6 +181,10 @@ public:
   void GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip,
                       const GetTextureDataParams &params, bytebuf &data);
 
+  rdcarray<ShaderEncoding> GetCustomShaderEncodings()
+  {
+    return {ShaderEncoding::DXBC, ShaderEncoding::HLSL};
+  }
   rdcarray<ShaderEncoding> GetTargetShaderEncodings()
   {
     return {ShaderEncoding::DXBC, ShaderEncoding::HLSL};

renderdoc/driver/d3d12/d3d12_replay.h

@@ -142,6 +142,10 @@ public:
   void GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip,
                       const GetTextureDataParams &params, bytebuf &data);
 
+  rdcarray<ShaderEncoding> GetCustomShaderEncodings()
+  {
+    return {ShaderEncoding::DXBC, ShaderEncoding::HLSL};
+  }
   rdcarray<ShaderEncoding> GetTargetShaderEncodings()
   {
     return {ShaderEncoding::DXBC, ShaderEncoding::HLSL};

renderdoc/driver/gl/gl_replay.h

@@ -171,6 +171,7 @@ public:
   void RenderMesh(uint32_t eventId, const std::vector<MeshFormat> &secondaryDraws,
                   const MeshDisplay &cfg);
 
+  rdcarray<ShaderEncoding> GetCustomShaderEncodings() { return {ShaderEncoding::GLSL}; }
   rdcarray<ShaderEncoding> GetTargetShaderEncodings() { return {ShaderEncoding::GLSL}; }
   void BuildTargetShader(ShaderEncoding sourceEncoding, bytebuf source, std::string entry,
                          const ShaderCompileFlags &compileFlags, ShaderStage type, ResourceId *id,

renderdoc/driver/vulkan/vk_replay.h

@@ -312,6 +312,10 @@ public:
   void RenderMesh(uint32_t eventId, const std::vector<MeshFormat> &secondaryDraws,
                   const MeshDisplay &cfg);
 
+  rdcarray<ShaderEncoding> GetCustomShaderEncodings()
+  {
+    return {ShaderEncoding::SPIRV, ShaderEncoding::GLSL};
+  }
   rdcarray<ShaderEncoding> GetTargetShaderEncodings()
   {
     return {ShaderEncoding::SPIRV, ShaderEncoding::GLSL};

renderdoc/replay/replay_controller.cpp

@@ -1908,6 +1908,13 @@ void ReplayController::Shutdown()
   delete this;
 }
 
+rdcarray<ShaderEncoding> ReplayController::GetCustomShaderEncodings()
+{
+  CHECK_REPLAY_THREAD();
+
+  return m_pDevice->GetCustomShaderEncodings();
+}
+
 rdcarray<ShaderEncoding> ReplayController::GetTargetShaderEncodings()
 {
   CHECK_REPLAY_THREAD();

renderdoc/replay/replay_controller.h

@@ -158,6 +158,7 @@ public:
                                                 ShaderStage type);
   void FreeCustomShader(ResourceId id);
 
+  rdcarray<ShaderEncoding> GetCustomShaderEncodings();
   rdcarray<ShaderEncoding> GetTargetShaderEncodings();
   rdcpair<ResourceId, rdcstr> BuildTargetShader(const char *entry, ShaderEncoding sourceEncoding,
                                                 bytebuf source,

renderdoc/replay/replay_driver.h

@@ -217,6 +217,7 @@ public:
   virtual void BuildCustomShader(std::string source, std::string entry,
                                  const ShaderCompileFlags &compileFlags, ShaderStage type,
                                  ResourceId *id, std::string *errors) = 0;
+  virtual rdcarray<ShaderEncoding> GetCustomShaderEncodings() = 0;
   virtual ResourceId ApplyCustomShader(ResourceId shader, ResourceId texid, uint32_t mip,
                                        uint32_t arrayIdx, uint32_t sampleIdx, CompType typeHint) = 0;
   virtual void FreeCustomShader(ResourceId id) = 0;