Pass through and allow different source shader encodings when editing

* This means e.g. the D3D11 back-end can accept DXBC directly if the UI can
  provide it, or compile from HLSL as before.
* More importantly, the Vulkan back-end can take SPIR-V compiled from any
  source, or compile from GLSL as before as a fall-back.

qrenderdoc/Code/CaptureContext.cpp

@@ -1459,13 +1459,14 @@ void CaptureContext::ShowResourceInspector()
   m_MainWindow->showResourceInspector();
 }
 
-IShaderViewer *CaptureContext::EditShader(bool customShader, const rdcstr &entryPoint,
-                                          const rdcstrpairs &files,
+IShaderViewer *CaptureContext::EditShader(bool customShader, ShaderStage stage,
+                                          const rdcstr &entryPoint, const rdcstrpairs &files,
+                                          ShaderEncoding shaderEncoding, ShaderCompileFlags flags,
                                           IShaderViewer::SaveCallback saveCallback,
                                           IShaderViewer::CloseCallback closeCallback)
 {
-  return ShaderViewer::EditShader(*this, customShader, entryPoint, files, saveCallback,
-                                  closeCallback, m_MainWindow->Widget());
+  return ShaderViewer::EditShader(*this, customShader, stage, entryPoint, files, shaderEncoding,
+                                  flags, saveCallback, closeCallback, m_MainWindow->Widget());
 }
 
 IShaderViewer *CaptureContext::DebugShader(const ShaderBindpointMapping *bind,

qrenderdoc/Code/CaptureContext.h

@@ -192,8 +192,9 @@ public:
   void ShowPythonShell() override;
   void ShowResourceInspector() override;
 
-  IShaderViewer *EditShader(bool customShader, const rdcstr &entryPoint, const rdcstrpairs &files,
-                            IShaderViewer::SaveCallback saveCallback,
+  IShaderViewer *EditShader(bool customShader, ShaderStage stage, const rdcstr &entryPoint,
+                            const rdcstrpairs &files, ShaderEncoding shaderEncoding,
+                            ShaderCompileFlags flags, IShaderViewer::SaveCallback saveCallback,
                             IShaderViewer::CloseCallback closeCallback) override;
 
   IShaderViewer *DebugShader(const ShaderBindpointMapping *bind, const ShaderReflection *shader,

qrenderdoc/Code/Interface/QRDInterface.h

@@ -510,8 +510,11 @@ DOCUMENT(R"(A shader window used for viewing, editing, or debugging.
 
   :param CaptureContext context: The current capture context.
   :param ShaderViewer viewer: The open shader viewer.
-  :param list files: A ``list`` with 2-tuples of ``str``, the first element being the filename and
-    the second element being the file contents.
+  :param ShaderEncoding encoding: The encoding of the files being passed.
+  :param ShaderCompileFlags flags: The flags to use during compilation.
+  :param str entryFunc: The name of the entry point.
+  :param bytes source: The byte buffer containing the source - may just be text depending on the
+    encoding.
 
 .. function:: CloseCallback(context)
 
@@ -523,7 +526,9 @@ DOCUMENT(R"(A shader window used for viewing, editing, or debugging.
 )");
 struct IShaderViewer
 {
-  typedef std::function<void(ICaptureContext *ctx, IShaderViewer *, const rdcstrpairs &)> SaveCallback;
+  typedef std::function<void(ICaptureContext *ctx, IShaderViewer *, ShaderEncoding,
+                             ShaderCompileFlags, rdcstr, bytebuf)>
+      SaveCallback;
   typedef std::function<void(ICaptureContext *ctx)> CloseCallback;
 
   DOCUMENT(
@@ -1721,9 +1726,12 @@ place if needed.
   DOCUMENT(R"(Show a new :class:`ShaderViewer` window, showing an editable view of a given shader.
 
 :param bool customShader: ``True`` if the shader being edited is a custom display shader.
+:param ~renderdoc.ShaderStage stage: The shader stage for this shader.
 :param str entryPoint: The entry point to be used when compiling the edited shader.
 :param list files: The files stored in a ``list`` with 2-tuples of ``str``. The first element being
   the filename and the second being the file contents.
+:param ~renderdoc.ShaderEncoding shaderEncoding: The encoding of the input files.
+:param ~renderdoc.ShaderCompileFlags flags: The flags originally used to compile the shader.
 :param ShaderViewer.SaveCallback saveCallback: The callback function to call when a save/update is
   triggered.
 :param ShaderViewer.CloseCallback closeCallback: The callback function to call when the shader
@@ -1731,8 +1739,9 @@ place if needed.
 :return: The new :class:`ShaderViewer` window opened but not shown for editing.
 :rtype: ShaderViewer
 )");
-  virtual IShaderViewer *EditShader(bool customShader, const rdcstr &entryPoint,
-                                    const rdcstrpairs &files,
+  virtual IShaderViewer *EditShader(bool customShader, ShaderStage stage, const rdcstr &entryPoint,
+                                    const rdcstrpairs &files, ShaderEncoding shaderEncoding,
+                                    ShaderCompileFlags flags,
                                     IShaderViewer::SaveCallback saveCallback,
                                     IShaderViewer::CloseCallback closeCallback) = 0;
 

qrenderdoc/Windows/PipelineState/D3D11PipelineStateViewer.cpp

@@ -2338,7 +2338,9 @@ void D3D11PipelineStateViewer::shaderEdit_clicked()
   if(files.empty())
     return;
 
-  m_Common.EditShader(stage->stage, stage->resourceId, shaderDetails, entryFunc, files);
+  // we always consider the input HLSL, either the stub or the original source
+  m_Common.EditShader(stage->stage, stage->resourceId, shaderDetails, entryFunc,
+                      ShaderEncoding::HLSL, files);
 }
 
 void D3D11PipelineStateViewer::shaderSave_clicked()

qrenderdoc/Windows/PipelineState/D3D12PipelineStateViewer.cpp

@@ -2166,7 +2166,9 @@ void D3D12PipelineStateViewer::shaderEdit_clicked()
   if(files.empty())
     return;
 
-  m_Common.EditShader(stage->stage, stage->resourceId, shaderDetails, entryFunc, files);
+  // we always consider the input HLSL, either the stub or the original source
+  m_Common.EditShader(stage->stage, stage->resourceId, shaderDetails, entryFunc,
+                      ShaderEncoding::HLSL, files);
 }
 
 void D3D12PipelineStateViewer::shaderSave_clicked()

qrenderdoc/Windows/PipelineState/GLPipelineStateViewer.cpp

@@ -2330,7 +2330,9 @@ void GLPipelineStateViewer::shaderEdit_clicked()
   if(files.empty())
     return;
 
-  m_Common.EditShader(stage->stage, stage->shaderResourceId, shaderDetails, entryFunc, files);
+  // we always consider the input GLSL
+  m_Common.EditShader(stage->stage, stage->shaderResourceId, shaderDetails, entryFunc,
+                      ShaderEncoding::GLSL, files);
 }
 
 void GLPipelineStateViewer::shaderSave_clicked()

qrenderdoc/Windows/PipelineState/PipelineStateViewer.cpp

@@ -719,132 +719,35 @@ QString PipelineStateViewer::GenerateHLSLStub(const ShaderReflection *shaderDeta
 }
 
 void PipelineStateViewer::EditShader(ShaderStage shaderType, ResourceId id,
-                                     const ShaderReflection *shaderDetails,
-                                     const QString &entryFunc, const rdcstrpairs &files)
+                                     const ShaderReflection *shaderDetails, const QString &entry,
+                                     ShaderEncoding encoding, const rdcstrpairs &files)
 {
+  if(!shaderDetails)
+    return;
+
   IShaderViewer *sv = m_Ctx.EditShader(
-      false, entryFunc, files,
+      false, shaderDetails->stage, entry, files, encoding, shaderDetails->debugInfo.compileFlags,
       // save callback
-      [entryFunc, shaderType, id, shaderDetails](ICaptureContext *ctx, IShaderViewer *viewer,
-                                                 const rdcstrpairs &updatedfiles) {
-        QString compileSource = updatedfiles[0].second;
+      [shaderType, id, shaderDetails](ICaptureContext *ctx, IShaderViewer *viewer,
+                                      ShaderEncoding shaderEncoding, ShaderCompileFlags flags,
+                                      rdcstr entryFunc, bytebuf shaderBytes) {
+
+        if(shaderBytes.isEmpty())
+          return;
 
         ANALYTIC_SET(UIFeatures.ShaderEditing, true);
 
-        // try and match up #includes against the files that we have. This isn't always
-        // possible as fxc only seems to include the source for files if something in
-        // that file was included in the compiled output. So you might end up with
-        // dangling #includes - we just have to ignore them
-        int offs = compileSource.indexOf(lit("#include"));
-
-        while(offs >= 0)
-        {
-          // search back to ensure this is a valid #include (ie. not in a comment).
-          // Must only see whitespace before, then a newline.
-          int ws = qMax(0, offs - 1);
-          while(ws >= 0 &&
-                (compileSource[ws] == QLatin1Char(' ') || compileSource[ws] == QLatin1Char('\t')))
-            ws--;
-
-          // not valid? jump to next.
-          if(ws > 0 && compileSource[ws] != QLatin1Char('\n'))
-          {
-            offs = compileSource.indexOf(lit("#include"), offs + 1);
-            continue;
-          }
-
-          int start = ws + 1;
-
-          bool tail = true;
-
-          int lineEnd = compileSource.indexOf(QLatin1Char('\n'), start + 1);
-          if(lineEnd == -1)
-          {
-            lineEnd = compileSource.length();
-            tail = false;
-          }
-
-          ws = offs + sizeof("#include") - 1;
-          while(compileSource[ws] == QLatin1Char(' ') || compileSource[ws] == QLatin1Char('\t'))
-            ws++;
-
-          QString line = compileSource.mid(offs, lineEnd - offs + 1);
-
-          if(compileSource[ws] != QLatin1Char('<') && compileSource[ws] != QLatin1Char('"'))
-          {
-            viewer->ShowErrors(lit("Invalid #include directive found:\r\n") + line);
-            return;
-          }
-
-          // find matching char, either <> or "";
-          int end = compileSource.indexOf(
-              compileSource[ws] == QLatin1Char('"') ? QLatin1Char('"') : QLatin1Char('>'), ws + 1);
-
-          if(end == -1)
-          {
-            viewer->ShowErrors(lit("Invalid #include directive found:\r\n") + line);
-            return;
-          }
-
-          QString fname = compileSource.mid(ws + 1, end - ws - 1);
-
-          QString fileText;
-
-          // look for exact match first
-          for(int i = 0; i < updatedfiles.count(); i++)
-          {
-            if(QString(updatedfiles[i].first) == fname)
-            {
-              fileText = updatedfiles[i].second;
-              break;
-            }
-          }
-
-          if(fileText.isEmpty())
-          {
-            QString search = QFileInfo(fname).fileName();
-
-            // if not, try and find the same filename (this is not proper include handling!)
-            for(const rdcstrpair &kv : updatedfiles)
-            {
-              if(QFileInfo(kv.first).fileName().compare(search, Qt::CaseInsensitive) == 0)
-              {
-                fileText = kv.second;
-                break;
-              }
-            }
-
-            if(fileText.isEmpty())
-              fileText = QFormatStr("// Can't find file %1\n").arg(fname);
-          }
-
-          compileSource = compileSource.left(offs) + lit("\n\n") + fileText + lit("\n\n") +
-                          (tail ? compileSource.mid(lineEnd + 1) : QString());
-
-          // need to start searching from the beginning - wasteful but allows nested includes to
-          // work
-          offs = compileSource.indexOf(lit("#include"));
-        }
-
-        for(const rdcstrpair &kv : updatedfiles)
-        {
-          if(kv.first == "@cmdline")
-            compileSource = QString(kv.second) + lit("\n\n") + compileSource;
-        }
-
         // invoke off to the ReplayController to replace the capture's shader
         // with our edited one
-        ctx->Replay().AsyncInvoke([ctx, entryFunc, compileSource, shaderType, id, shaderDetails,
-                                   viewer](IReplayController *r) {
+        ctx->Replay().AsyncInvoke([ctx, entryFunc, shaderBytes, shaderEncoding, flags, shaderType,
+                                   id, shaderDetails, viewer](IReplayController *r) {
           rdcstr errs;
 
-          const ShaderCompileFlags &flags = shaderDetails->debugInfo.compileFlags;
-
           ResourceId from = id;
           ResourceId to;
 
-          std::tie(to, errs) = r->BuildTargetShader(
-              entryFunc.toUtf8().data(), compileSource.toUtf8().data(), flags, shaderType);
+          std::tie(to, errs) = r->BuildTargetShader(entryFunc.c_str(), shaderEncoding, shaderBytes,
+                                                    flags, shaderType);
 
           GUIInvoke::call(viewer->Widget(), [viewer, errs]() { viewer->ShowErrors(errs); });
           if(to == ResourceId())

qrenderdoc/Windows/PipelineState/PipelineStateViewer.h

@@ -70,7 +70,7 @@ public:
                             rdcstrpairs &files);
   QString GenerateHLSLStub(const ShaderReflection *shaderDetails, const QString &entryFunc);
   void EditShader(ShaderStage shaderType, ResourceId id, const ShaderReflection *shaderDetails,
-                  const QString &entryFunc, const rdcstrpairs &files);
+                  const QString &entryFunc, ShaderEncoding encoding, const rdcstrpairs &files);
 
   void setTopologyDiagram(QLabel *diagram, Topology topo);
   void setMeshViewPixmap(RDLabel *meshView);

qrenderdoc/Windows/PipelineState/VulkanPipelineStateViewer.cpp

@@ -2434,6 +2434,7 @@ void VulkanPipelineStateViewer::shaderEdit_clicked()
   rdcstrpairs files;
 
   bool hasOrigSource = m_Common.PrepareShaderEditing(shaderDetails, entryFunc, files);
+  ShaderEncoding encoding = shaderDetails->debugInfo.encoding;
 
   if(hasOrigSource)
   {
@@ -2449,8 +2450,12 @@ void VulkanPipelineStateViewer::shaderEdit_clicked()
 
     if(!glsl.isEmpty())
     {
+      // if we decompiled, we expect the entry point name to be the same and assume GLSL
+      // decompilation for now
+      entryFunc = shaderDetails->entryPoint;
+      encoding = ShaderEncoding::GLSL;
       files.clear();
-      files.push_back(make_rdcpair<rdcstr, rdcstr>("generated.glsl", glsl));
+      files.push_back(make_rdcpair<rdcstr, rdcstr>("decompiled", glsl));
     }
     else
     {
@@ -2459,15 +2464,16 @@ void VulkanPipelineStateViewer::shaderEdit_clicked()
 
         GUIInvoke::call(this, [this, stage, shaderDetails, entryFunc, disasm]() {
           rdcstrpairs fileMap;
-          fileMap.push_back(make_rdcpair<rdcstr, rdcstr>("generated.glsl", disasm));
-          m_Common.EditShader(stage->stage, stage->resourceId, shaderDetails, entryFunc, fileMap);
+          fileMap.push_back(make_rdcpair<rdcstr, rdcstr>("pseudo_disassembly", disasm));
+          m_Common.EditShader(stage->stage, stage->resourceId, shaderDetails, entryFunc,
+                              ShaderEncoding::Unknown, fileMap);
         });
       });
       return;
     }
   }
 
-  m_Common.EditShader(stage->stage, stage->resourceId, shaderDetails, entryFunc, files);
+  m_Common.EditShader(stage->stage, stage->resourceId, shaderDetails, entryFunc, encoding, files);
 }
 
 void VulkanPipelineStateViewer::shaderSave_clicked()

qrenderdoc/Windows/PythonShell.cpp

@@ -373,13 +373,15 @@ struct CaptureContextInvoker : ICaptureContext
   {
     InvokeVoidFunction(&ICaptureContext::ShowResourceInspector);
   }
-  virtual IShaderViewer *EditShader(bool customShader, const rdcstr &entryPoint,
-                                    const rdcstrpairs &files,
+  virtual IShaderViewer *EditShader(bool customShader, ShaderStage stage, const rdcstr &entryPoint,
+                                    const rdcstrpairs &files, ShaderEncoding shaderEncoding,
+                                    ShaderCompileFlags flags,
                                     IShaderViewer::SaveCallback saveCallback,
                                     IShaderViewer::CloseCallback closeCallback) override
   {
-    return InvokeRetFunction<IShaderViewer *>(&ICaptureContext::EditShader, customShader,
-                                              entryPoint, files, saveCallback, closeCallback);
+    return InvokeRetFunction<IShaderViewer *>(&ICaptureContext::EditShader, customShader, stage,
+                                              entryPoint, files, shaderEncoding, flags,
+                                              saveCallback, closeCallback);
   }
 
   virtual IShaderViewer *DebugShader(const ShaderBindpointMapping *bind,

qrenderdoc/Windows/ShaderViewer.cpp

@@ -181,13 +181,20 @@ ShaderViewer::ShaderViewer(ICaptureContext &ctx, QWidget *parent)
   m_Ctx.AddCaptureViewer(this);
 }
 
-void ShaderViewer::editShader(bool customShader, const QString &entryPoint, const rdcstrpairs &files)
+void ShaderViewer::editShader(bool customShader, ShaderStage stage, const QString &entryPoint,
+                              const rdcstrpairs &files, ShaderEncoding shaderEncoding,
+                              ShaderCompileFlags flags)
 {
   m_Scintillas.removeOne(m_DisassemblyView);
   ui->docking->removeToolWindow(m_DisassemblyFrame);
 
   m_DisassemblyView = NULL;
 
+  m_Stage = stage;
+  m_Flags = flags;
+  m_Encoding = shaderEncoding;
+  m_EntryPoint = entryPoint;
+
   // hide debugging windows
   ui->watch->hide();
   ui->registers->hide();
@@ -3179,6 +3186,111 @@ void ShaderViewer::on_findReplace_clicked()
   m_FindReplace->takeFocus();
 }
 
+bool ShaderViewer::ProcessIncludeDirectives(QString &source, const rdcstrpairs &files)
+{
+  // try and match up #includes against the files that we have. This isn't always
+  // possible as fxc only seems to include the source for files if something in
+  // that file was included in the compiled output. So you might end up with
+  // dangling #includes - we just have to ignore them
+  int offs = source.indexOf(lit("#include"));
+
+  while(offs >= 0)
+  {
+    // search back to ensure this is a valid #include (ie. not in a comment).
+    // Must only see whitespace before, then a newline.
+    int ws = qMax(0, offs - 1);
+    while(ws >= 0 && (source[ws] == QLatin1Char(' ') || source[ws] == QLatin1Char('\t')))
+      ws--;
+
+    // not valid? jump to next.
+    if(ws > 0 && source[ws] != QLatin1Char('\n'))
+    {
+      offs = source.indexOf(lit("#include"), offs + 1);
+      continue;
+    }
+
+    int start = ws + 1;
+
+    bool tail = true;
+
+    int lineEnd = source.indexOf(QLatin1Char('\n'), start + 1);
+    if(lineEnd == -1)
+    {
+      lineEnd = source.length();
+      tail = false;
+    }
+
+    ws = offs + sizeof("#include") - 1;
+    while(source[ws] == QLatin1Char(' ') || source[ws] == QLatin1Char('\t'))
+      ws++;
+
+    QString line = source.mid(offs, lineEnd - offs + 1);
+
+    if(source[ws] != QLatin1Char('<') && source[ws] != QLatin1Char('"'))
+    {
+      ShowErrors(lit("Invalid #include directive found:\r\n") + line);
+      return false;
+    }
+
+    // find matching char, either <> or "";
+    int end =
+        source.indexOf(source[ws] == QLatin1Char('"') ? QLatin1Char('"') : QLatin1Char('>'), ws + 1);
+
+    if(end == -1)
+    {
+      ShowErrors(lit("Invalid #include directive found:\r\n") + line);
+      return false;
+    }
+
+    QString fname = source.mid(ws + 1, end - ws - 1);
+
+    QString fileText;
+
+    // look for exact match first
+    for(int i = 0; i < files.count(); i++)
+    {
+      if(QString(files[i].first) == fname)
+      {
+        fileText = files[i].second;
+        break;
+      }
+    }
+
+    if(fileText.isEmpty())
+    {
+      QString search = QFileInfo(fname).fileName();
+
+      // if not, try and find the same filename (this is not proper include handling!)
+      for(const rdcstrpair &kv : files)
+      {
+        if(QFileInfo(kv.first).fileName().compare(search, Qt::CaseInsensitive) == 0)
+        {
+          fileText = kv.second;
+          break;
+        }
+      }
+
+      if(fileText.isEmpty())
+        fileText = QFormatStr("// Can't find file %1\n").arg(fname);
+    }
+
+    source = source.left(offs) + lit("\n\n") + fileText + lit("\n\n") +
+             (tail ? source.mid(lineEnd + 1) : QString());
+
+    // need to start searching from the beginning - wasteful but allows nested includes to
+    // work
+    offs = source.indexOf(lit("#include"));
+  }
+
+  for(const rdcstrpair &kv : files)
+  {
+    if(kv.first == "@cmdline")
+      source = QString(kv.second) + lit("\n\n") + source;
+  }
+
+  return true;
+}
+
 void ShaderViewer::on_refresh_clicked()
 {
   if(m_Trace)
@@ -3189,6 +3301,8 @@ void ShaderViewer::on_refresh_clicked()
 
   if(m_SaveCallback)
   {
+    ShaderEncoding encoding = m_Encoding;
+
     rdcstrpairs files;
     for(ScintillaEdit *s : m_Scintillas)
     {
@@ -3196,7 +3310,22 @@ void ShaderViewer::on_refresh_clicked()
       files.push_back(make_rdcpair<rdcstr, rdcstr>(
           w->property("filename").toString(), QString::fromUtf8(s->getText(s->textLength() + 1))));
     }
-    m_SaveCallback(&m_Ctx, this, files);
+
+    if(files.isEmpty())
+      return;
+
+    QString source = files[0].second;
+
+    if(encoding == ShaderEncoding::HLSL || encoding == ShaderEncoding::GLSL)
+    {
+      bool success = ProcessIncludeDirectives(source, files);
+      if(!success)
+        return;
+    }
+
+    bytebuf shaderBytes(source.toUtf8());
+
+    m_SaveCallback(&m_Ctx, this, encoding, m_Flags, m_EntryPoint, shaderBytes);
   }
 }
 

qrenderdoc/Windows/ShaderViewer.h

@@ -61,15 +61,16 @@ class ShaderViewer : public QFrame, public IShaderViewer, public ICaptureViewer
   Q_OBJECT
 
 public:
-  static IShaderViewer *EditShader(ICaptureContext &ctx, bool customShader,
+  static IShaderViewer *EditShader(ICaptureContext &ctx, bool customShader, ShaderStage stage,
                                    const QString &entryPoint, const rdcstrpairs &files,
+                                   ShaderEncoding shaderEncoding, ShaderCompileFlags flags,
                                    IShaderViewer::SaveCallback saveCallback,
                                    IShaderViewer::CloseCallback closeCallback, QWidget *parent)
   {
     ShaderViewer *ret = new ShaderViewer(ctx, parent);
     ret->m_SaveCallback = saveCallback;
     ret->m_CloseCallback = closeCallback;
-    ret->editShader(customShader, entryPoint, files);
+    ret->editShader(customShader, stage, entryPoint, files, shaderEncoding, flags);
     return ret;
   }
 
@@ -153,11 +154,14 @@ public slots:
 
 private:
   explicit ShaderViewer(ICaptureContext &ctx, QWidget *parent = 0);
-  void editShader(bool customShader, const QString &entryPoint, const rdcstrpairs &files);
+  void editShader(bool customShader, ShaderStage stage, const QString &entryPoint,
+                  const rdcstrpairs &files, ShaderEncoding shaderEncoding, ShaderCompileFlags flags);
   void debugShader(const ShaderBindpointMapping *bind, const ShaderReflection *shader,
                    ResourceId pipeline, ShaderDebugTrace *trace, const QString &debugContext);
   bool eventFilter(QObject *watched, QEvent *event) override;
 
+  bool ProcessIncludeDirectives(QString &source, const rdcstrpairs &files);
+
   const rdcarray<ShaderVariable> *GetVariableList(VariableCategory varCat, int arrayIdx);
   void getRegisterFromWord(const QString &text, VariableCategory &varCat, int &varIdx, int &arrayIdx);
 
@@ -182,6 +186,9 @@ private:
   ICaptureContext &m_Ctx;
   const ShaderBindpointMapping *m_Mapping = NULL;
   const ShaderReflection *m_ShaderDetails = NULL;
+  ShaderEncoding m_Encoding;
+  rdcstr m_EntryPoint;
+  ShaderCompileFlags m_Flags;
   ShaderStage m_Stage;
   QString m_DebugContext;
   ResourceId m_Pipeline;

qrenderdoc/Windows/TextureViewer.cpp

@@ -3872,15 +3872,17 @@ void TextureViewer::on_customEdit_clicked()
   files.push_back(make_rdcpair<rdcstr, rdcstr>(filename, src));
 
   IShaderViewer *s = m_Ctx.EditShader(
-      true, lit("main"), files,
+      true, ShaderStage::Fragment, lit("main"), files,
+      IsD3D(m_Ctx.APIProps().localRenderer) ? ShaderEncoding::HLSL : ShaderEncoding::GLSL,
+      ShaderCompileFlags(),
       // Save Callback
-      [this, key, filename, path](ICaptureContext *ctx, IShaderViewer *viewer,
-                                  const rdcstrpairs &updatedfiles) {
+      [this, key, filename, path](ICaptureContext *ctx, IShaderViewer *viewer, ShaderEncoding encoding,
+                                  ShaderCompileFlags flags, rdcstr entryFunc, bytebuf bytes) {
         {
           QFile fileHandle(path);
           if(fileHandle.open(QFile::WriteOnly | QIODevice::Truncate | QIODevice::Text))
           {
-            fileHandle.write(updatedfiles[0].second.c_str());
+            fileHandle.write(QByteArray(bytes));
             fileHandle.close();
 
             // watcher doesn't trigger on internal modifications

renderdoc/api/replay/renderdoc_replay.h

@@ -881,14 +881,16 @@ See :meth:`BuildCustomShader`.
 The language used is native to the API's renderer - HLSL for D3D based renderers, GLSL otherwise.
 
 :param str entry: The entry point to use when compiling.
-:param str source: The source file.
+:param ShaderEncoding sourceEncoding: The encoding of the source data.
+:param bytes source: The source data itself.
 :param ShaderCompileFlags compileFlags: API-specific compilation flags.
 :param ShaderStage type: The stage that this shader will be executed at.
 :return: A ``tuple`` with the id of the new shader if compilation was successful,
   :meth:`ResourceId.Null` otherwise, and a ``str`` with any warnings/errors from compilation.
 :rtype: ``tuple`` of :class:`ResourceId` and ``str``.
 )");
-  virtual rdcpair<ResourceId, rdcstr> BuildTargetShader(const char *entry, const char *source,
+  virtual rdcpair<ResourceId, rdcstr> BuildTargetShader(const char *entry,
+                                                        ShaderEncoding sourceEncoding, bytebuf source,
                                                         const ShaderCompileFlags &flags,
                                                         ShaderStage type) = 0;
 

renderdoc/core/image_viewer.cpp

@@ -242,10 +242,15 @@ public:
     RDCEraseEl(ret);
     return ret;
   }
-  void BuildTargetShader(string source, string entry, const ShaderCompileFlags &compileFlags,
-                         ShaderStage type, ResourceId *id, string *errors)
-  {
   rdcarray<ShaderEncoding> GetTargetShaderEncodings() { return {}; }
+  void BuildTargetShader(ShaderEncoding sourceEncoding, bytebuf source, string entry,
+                         const ShaderCompileFlags &compileFlags, ShaderStage type, ResourceId *id,
+                         string *errors)
+  {
+    if(id)
+      *id = ResourceId();
+    if(errors)
+      *errors = "Building target shaders is unsupported";
   }
   void ReplaceResource(ResourceId from, ResourceId to) {}
   void RemoveReplacement(ResourceId id) {}

renderdoc/core/replay_proxy.cpp

@@ -987,8 +987,8 @@ rdcarray<ShaderEncoding> ReplayProxy::GetTargetShaderEncodings()
 
 template <typename ParamSerialiser, typename ReturnSerialiser>
 void ReplayProxy::Proxied_BuildTargetShader(ParamSerialiser &paramser, ReturnSerialiser &retser,
-                                            std::string source, std::string entry,
-                                            const ShaderCompileFlags &compileFlags,
+                                            ShaderEncoding sourceEncoding, bytebuf source,
+                                            std::string entry, const ShaderCompileFlags &compileFlags,
                                             ShaderStage type, ResourceId *id, std::string *errors)
 {
   const ReplayProxyPacket packet = eReplayProxy_BuildTargetShader;
@@ -997,6 +997,7 @@ void ReplayProxy::Proxied_BuildTargetShader(ParamSerialiser &paramser, ReturnSer
 
   {
     BEGIN_PARAMS();
+    SERIALISE_ELEMENT(sourceEncoding);
     SERIALISE_ELEMENT(source);
     SERIALISE_ELEMENT(entry);
     SERIALISE_ELEMENT(compileFlags);
@@ -1005,7 +1006,8 @@ void ReplayProxy::Proxied_BuildTargetShader(ParamSerialiser &paramser, ReturnSer
   }
 
   if(paramser.IsReading() && !paramser.IsErrored() && !m_IsErrored)
-    m_Remote->BuildTargetShader(source, entry, compileFlags, type, &ret_id, &ret_errors);
+    m_Remote->BuildTargetShader(sourceEncoding, source, entry, compileFlags, type, &ret_id,
+                                &ret_errors);
 
   {
     ReturnSerialiser &ser = retser;
@@ -1021,11 +1023,11 @@ void ReplayProxy::Proxied_BuildTargetShader(ParamSerialiser &paramser, ReturnSer
   }
 }
 
-void ReplayProxy::BuildTargetShader(std::string source, std::string entry,
-                                    const ShaderCompileFlags &compileFlags, ShaderStage type,
-                                    ResourceId *id, std::string *errors)
+void ReplayProxy::BuildTargetShader(ShaderEncoding sourceEncoding, bytebuf source,
+                                    std::string entry, const ShaderCompileFlags &compileFlags,
+                                    ShaderStage type, ResourceId *id, std::string *errors)
 {
-  PROXY_FUNCTION(BuildTargetShader, source, entry, compileFlags, type, id, errors);
+  PROXY_FUNCTION(BuildTargetShader, sourceEncoding, source, entry, compileFlags, type, id, errors);
 }
 
 template <typename ParamSerialiser, typename ReturnSerialiser>
@@ -2102,7 +2104,8 @@ bool ReplayProxy::Tick(int type)
     }
     case eReplayProxy_GetPostVS: GetPostVSBuffers(0, 0, 0, MeshDataStage::Unknown); break;
     case eReplayProxy_BuildTargetShader:
-      BuildTargetShader("", "", ShaderCompileFlags(), ShaderStage::Vertex, NULL, NULL);
+      BuildTargetShader(ShaderEncoding::Unknown, bytebuf(), "", ShaderCompileFlags(),
+                        ShaderStage::Vertex, NULL, NULL);
       break;
     case eReplayProxy_ReplaceResource: ReplaceResource(ResourceId(), ResourceId()); break;
     case eReplayProxy_RemoveReplacement: RemoveReplacement(ResourceId()); break;

renderdoc/core/replay_proxy.h

@@ -494,10 +494,10 @@ public:
   IMPLEMENT_FUNCTION_PROXIED(ShaderDebugTrace, DebugThread, uint32_t eventId,
                              const uint32_t groupid[3], const uint32_t threadid[3]);
 
-  IMPLEMENT_FUNCTION_PROXIED(void, BuildTargetShader, std::string source, std::string entry,
-                             const ShaderCompileFlags &compileFlags, ShaderStage type,
-                             ResourceId *id, std::string *errors);
   IMPLEMENT_FUNCTION_PROXIED(rdcarray<ShaderEncoding>, GetTargetShaderEncodings);
+  IMPLEMENT_FUNCTION_PROXIED(void, BuildTargetShader, ShaderEncoding sourceEncoding, bytebuf source,
+                             std::string entry, const ShaderCompileFlags &compileFlags,
+                             ShaderStage type, ResourceId *id, std::string *errors);
   IMPLEMENT_FUNCTION_PROXIED(void, ReplaceResource, ResourceId from, ResourceId to);
   IMPLEMENT_FUNCTION_PROXIED(void, RemoveReplacement, ResourceId id);
 

renderdoc/driver/d3d11/d3d11_replay.cpp

@@ -2210,12 +2210,10 @@ D3D11DebugManager *D3D11Replay::GetDebugManager()
   return m_pDevice->GetDebugManager();
 }
 
-void D3D11Replay::BuildShader(std::string source, std::string entry,
+void D3D11Replay::BuildShader(ShaderEncoding sourceEncoding, bytebuf source, std::string entry,
                               const ShaderCompileFlags &compileFlags, ShaderStage type,
                               ResourceId *id, std::string *errors)
 {
-  uint32_t flags = DXBC::DecodeFlags(compileFlags);
-
   if(id == NULL || errors == NULL)
   {
     if(id)
@@ -2223,30 +2221,44 @@ void D3D11Replay::BuildShader(std::string source, std::string entry,
     return;
   }
 
-  char *profile = NULL;
-
-  switch(type)
+  if(sourceEncoding == ShaderEncoding::HLSL)
   {
-    case ShaderStage::Vertex: profile = "vs_5_0"; break;
-    case ShaderStage::Hull: profile = "hs_5_0"; break;
-    case ShaderStage::Domain: profile = "ds_5_0"; break;
-    case ShaderStage::Geometry: profile = "gs_5_0"; break;
-    case ShaderStage::Pixel: profile = "ps_5_0"; break;
-    case ShaderStage::Compute: profile = "cs_5_0"; break;
-    default:
-      RDCERR("Unexpected type in BuildShader!");
+    uint32_t flags = DXBC::DecodeFlags(compileFlags);
+
+    char *profile = NULL;
+
+    switch(type)
+    {
+      case ShaderStage::Vertex: profile = "vs_5_0"; break;
+      case ShaderStage::Hull: profile = "hs_5_0"; break;
+      case ShaderStage::Domain: profile = "ds_5_0"; break;
+      case ShaderStage::Geometry: profile = "gs_5_0"; break;
+      case ShaderStage::Pixel: profile = "ps_5_0"; break;
+      case ShaderStage::Compute: profile = "cs_5_0"; break;
+      default:
+        RDCERR("Unexpected type in BuildShader!");
+        *id = ResourceId();
+        return;
+    }
+
+    std::string hlsl;
+    hlsl.assign((const char *)source.data(), source.size());
+
+    ID3DBlob *blob = NULL;
+
+    *errors = m_pDevice->GetShaderCache()->GetShaderBlob(hlsl.c_str(), entry.c_str(), flags,
+                                                         profile, &blob);
+
+    if(blob == NULL)
+    {
       *id = ResourceId();
       return;
-  }
+    }
 
-  ID3DBlob *blob = NULL;
-  *errors = m_pDevice->GetShaderCache()->GetShaderBlob(source.c_str(), entry.c_str(), flags,
-                                                       profile, &blob);
+    source.clear();
+    source.assign((byte *)blob->GetBufferPointer(), blob->GetBufferSize());
 
-  if(blob == NULL)
-  {
-    *id = ResourceId();
-    return;
+    SAFE_RELEASE(blob);
   }
 
   switch(type)
@@ -2254,105 +2266,124 @@ void D3D11Replay::BuildShader(std::string source, std::string entry,
     case ShaderStage::Vertex:
     {
       ID3D11VertexShader *sh = NULL;
-      m_pDevice->CreateVertexShader(blob->GetBufferPointer(), blob->GetBufferSize(), NULL, &sh);
-
-      SAFE_RELEASE(blob);
+      HRESULT hr = m_pDevice->CreateVertexShader(source.data(), source.size(), NULL, &sh);
 
       if(sh != NULL)
+      {
         *id = ((WrappedID3D11Shader<ID3D11VertexShader> *)sh)->GetResourceID();
+      }
       else
+      {
+        *errors = StringFormat::Fmt("Failed to create shader: %s", ToStr(hr).c_str());
         *id = ResourceId();
+      }
       return;
     }
     case ShaderStage::Hull:
     {
       ID3D11HullShader *sh = NULL;
-      m_pDevice->CreateHullShader(blob->GetBufferPointer(), blob->GetBufferSize(), NULL, &sh);
-
-      SAFE_RELEASE(blob);
+      HRESULT hr = m_pDevice->CreateHullShader(source.data(), source.size(), NULL, &sh);
 
       if(sh != NULL)
+      {
         *id = ((WrappedID3D11Shader<ID3D11HullShader> *)sh)->GetResourceID();
+      }
       else
+      {
+        *errors = StringFormat::Fmt("Failed to create shader: %s", ToStr(hr).c_str());
         *id = ResourceId();
+      }
       return;
     }
     case ShaderStage::Domain:
     {
       ID3D11DomainShader *sh = NULL;
-      m_pDevice->CreateDomainShader(blob->GetBufferPointer(), blob->GetBufferSize(), NULL, &sh);
-
-      SAFE_RELEASE(blob);
+      HRESULT hr = m_pDevice->CreateDomainShader(source.data(), source.size(), NULL, &sh);
 
       if(sh != NULL)
+      {
         *id = ((WrappedID3D11Shader<ID3D11DomainShader> *)sh)->GetResourceID();
+      }
       else
+      {
+        *errors = StringFormat::Fmt("Failed to create shader: %s", ToStr(hr).c_str());
         *id = ResourceId();
+      }
       return;
     }
     case ShaderStage::Geometry:
     {
       ID3D11GeometryShader *sh = NULL;
-      m_pDevice->CreateGeometryShader(blob->GetBufferPointer(), blob->GetBufferSize(), NULL, &sh);
-
-      SAFE_RELEASE(blob);
+      HRESULT hr = m_pDevice->CreateGeometryShader(source.data(), source.size(), NULL, &sh);
 
       if(sh != NULL)
+      {
         *id = ((WrappedID3D11Shader<ID3D11GeometryShader> *)sh)->GetResourceID();
+      }
       else
+      {
+        *errors = StringFormat::Fmt("Failed to create shader: %s", ToStr(hr).c_str());
         *id = ResourceId();
+      }
       return;
     }
     case ShaderStage::Pixel:
     {
       ID3D11PixelShader *sh = NULL;
-      m_pDevice->CreatePixelShader(blob->GetBufferPointer(), blob->GetBufferSize(), NULL, &sh);
-
-      SAFE_RELEASE(blob);
+      HRESULT hr = m_pDevice->CreatePixelShader(source.data(), source.size(), NULL, &sh);
 
       if(sh != NULL)
+      {
         *id = ((WrappedID3D11Shader<ID3D11PixelShader> *)sh)->GetResourceID();
+      }
       else
+      {
+        *errors = StringFormat::Fmt("Failed to create shader: %s", ToStr(hr).c_str());
         *id = ResourceId();
+      }
       return;
     }
     case ShaderStage::Compute:
     {
       ID3D11ComputeShader *sh = NULL;
-      m_pDevice->CreateComputeShader(blob->GetBufferPointer(), blob->GetBufferSize(), NULL, &sh);
-
-      SAFE_RELEASE(blob);
+      HRESULT hr = m_pDevice->CreateComputeShader(source.data(), source.size(), NULL, &sh);
 
       if(sh != NULL)
+      {
         *id = ((WrappedID3D11Shader<ID3D11ComputeShader> *)sh)->GetResourceID();
+      }
       else
+      {
+        *errors = StringFormat::Fmt("Failed to create shader: %s", ToStr(hr).c_str());
         *id = ResourceId();
+      }
       return;
     }
     default: break;
   }
 
-  SAFE_RELEASE(blob);
-
   RDCERR("Unexpected type in BuildShader!");
   *id = ResourceId();
 }
 
-void D3D11Replay::BuildTargetShader(std::string source, std::string entry,
-                                    const ShaderCompileFlags &compileFlags, ShaderStage type,
-                                    ResourceId *id, std::string *errors)
+void D3D11Replay::BuildTargetShader(ShaderEncoding sourceEncoding, bytebuf source,
+                                    std::string entry, const ShaderCompileFlags &compileFlags,
+                                    ShaderStage type, ResourceId *id, std::string *errors)
 {
   ShaderCompileFlags debugCompileFlags =
       DXBC::EncodeFlags(DXBC::DecodeFlags(compileFlags) | D3DCOMPILE_DEBUG);
 
-  BuildShader(source, entry, debugCompileFlags, type, id, errors);
+  BuildShader(sourceEncoding, source, entry, debugCompileFlags, type, id, errors);
 }
 
 void D3D11Replay::BuildCustomShader(std::string source, std::string entry,
                                     const ShaderCompileFlags &compileFlags, ShaderStage type,
                                     ResourceId *id, std::string *errors)
 {
-  BuildShader(source, entry, compileFlags, type, id, errors);
+  bytebuf buf;
+  buf.resize(source.size());
+  memcpy(buf.data(), source.c_str(), buf.size());
+  BuildShader(ShaderEncoding::HLSL, buf, entry, compileFlags, type, id, errors);
 }
 
 bool D3D11Replay::RenderTexture(TextureDisplay cfg)

renderdoc/driver/d3d11/d3d11_replay.h

@@ -176,12 +176,13 @@ public:
   void GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip,
                       const GetTextureDataParams &params, bytebuf &data);
 
-  void BuildTargetShader(string source, string entry, const ShaderCompileFlags &compileFlags,
-                         ShaderStage type, ResourceId *id, string *errors);
   rdcarray<ShaderEncoding> GetTargetShaderEncodings()
   {
     return {ShaderEncoding::DXBC, ShaderEncoding::HLSL};
   }
+  void BuildTargetShader(ShaderEncoding sourceEncoding, bytebuf source, string entry,
+                         const ShaderCompileFlags &compileFlags, ShaderStage type, ResourceId *id,
+                         string *errors);
   void ReplaceResource(ResourceId from, ResourceId to);
   void RemoveReplacement(ResourceId id);
 
@@ -242,8 +243,9 @@ private:
 
   D3D11DebugManager *GetDebugManager();
   // shared by BuildCustomShader and BuildTargetShader
-  void BuildShader(std::string source, std::string entry, const ShaderCompileFlags &compileFlags,
-                   ShaderStage type, ResourceId *id, std::string *errors);
+  void BuildShader(ShaderEncoding sourceEncoding, bytebuf source, std::string entry,
+                   const ShaderCompileFlags &compileFlags, ShaderStage type, ResourceId *id,
+                   std::string *errors);
 
   void ClearPostVSCache();
 

renderdoc/driver/d3d12/d3d12_replay.cpp

@@ -2688,12 +2688,10 @@ vector<DebugMessage> D3D12Replay::GetDebugMessages()
   return m_pDevice->GetDebugMessages();
 }
 
-void D3D12Replay::BuildShader(std::string source, std::string entry,
+void D3D12Replay::BuildShader(ShaderEncoding sourceEncoding, bytebuf source, std::string entry,
                               const ShaderCompileFlags &compileFlags, ShaderStage type,
                               ResourceId *id, std::string *errors)
 {
-  uint32_t flags = DXBC::DecodeFlags(compileFlags);
-
   if(id == NULL || errors == NULL)
   {
     if(id)
@@ -2701,51 +2699,62 @@ void D3D12Replay::BuildShader(std::string source, std::string entry,
     return;
   }
 
-  char *profile = NULL;
-
-  switch(type)
+  if(sourceEncoding == ShaderEncoding::HLSL)
   {
-    case ShaderStage::Vertex: profile = "vs_5_0"; break;
-    case ShaderStage::Hull: profile = "hs_5_0"; break;
-    case ShaderStage::Domain: profile = "ds_5_0"; break;
-    case ShaderStage::Geometry: profile = "gs_5_0"; break;
-    case ShaderStage::Pixel: profile = "ps_5_0"; break;
-    case ShaderStage::Compute: profile = "cs_5_0"; break;
-    default:
-      RDCERR("Unexpected type in BuildShader!");
+    uint32_t flags = DXBC::DecodeFlags(compileFlags);
+
+    char *profile = NULL;
+
+    switch(type)
+    {
+      case ShaderStage::Vertex: profile = "vs_5_0"; break;
+      case ShaderStage::Hull: profile = "hs_5_0"; break;
+      case ShaderStage::Domain: profile = "ds_5_0"; break;
+      case ShaderStage::Geometry: profile = "gs_5_0"; break;
+      case ShaderStage::Pixel: profile = "ps_5_0"; break;
+      case ShaderStage::Compute: profile = "cs_5_0"; break;
+      default:
+        RDCERR("Unexpected type in BuildShader!");
+        *id = ResourceId();
+        return;
+    }
+
+    std::string hlsl;
+    hlsl.assign((const char *)source.data(), source.size());
+
+    ID3DBlob *blob = NULL;
+    *errors = m_pDevice->GetShaderCache()->GetShaderBlob(hlsl.c_str(), entry.c_str(), flags,
+                                                         profile, &blob);
+
+    if(blob == NULL)
+    {
       *id = ResourceId();
       return;
-  }
+    }
 
-  ID3DBlob *blob = NULL;
-  *errors = m_pDevice->GetShaderCache()->GetShaderBlob(source.c_str(), entry.c_str(), flags,
-                                                       profile, &blob);
+    source.clear();
+    source.assign((byte *)blob->GetBufferPointer(), blob->GetBufferSize());
 
-  if(blob == NULL)
-  {
-    *id = ResourceId();
-    return;
+    SAFE_RELEASE(blob);
   }
 
   D3D12_SHADER_BYTECODE byteCode;
-  byteCode.BytecodeLength = blob->GetBufferSize();
-  byteCode.pShaderBytecode = blob->GetBufferPointer();
+  byteCode.BytecodeLength = source.size();
+  byteCode.pShaderBytecode = source.data();
 
   WrappedID3D12Shader *sh = WrappedID3D12Shader::AddShader(byteCode, m_pDevice, NULL);
 
-  SAFE_RELEASE(blob);
-
   *id = sh->GetResourceID();
 }
 
-void D3D12Replay::BuildTargetShader(std::string source, std::string entry,
-                                    const ShaderCompileFlags &compileFlags, ShaderStage type,
-                                    ResourceId *id, std::string *errors)
+void D3D12Replay::BuildTargetShader(ShaderEncoding sourceEncoding, bytebuf source,
+                                    std::string entry, const ShaderCompileFlags &compileFlags,
+                                    ShaderStage type, ResourceId *id, std::string *errors)
 {
   ShaderCompileFlags debugCompileFlags =
       DXBC::EncodeFlags(DXBC::DecodeFlags(compileFlags) | D3DCOMPILE_DEBUG);
 
-  BuildShader(source, entry, debugCompileFlags, type, id, errors);
+  BuildShader(sourceEncoding, source, entry, debugCompileFlags, type, id, errors);
 }
 
 void D3D12Replay::ReplaceResource(ResourceId from, ResourceId to)
@@ -3282,7 +3291,10 @@ void D3D12Replay::BuildCustomShader(std::string source, std::string entry,
                                     const ShaderCompileFlags &compileFlags, ShaderStage type,
                                     ResourceId *id, std::string *errors)
 {
-  BuildShader(source, entry, compileFlags, type, id, errors);
+  bytebuf buf;
+  buf.resize(source.size());
+  memcpy(buf.data(), source.c_str(), buf.size());
+  BuildShader(ShaderEncoding::HLSL, buf, entry, compileFlags, type, id, errors);
 }
 
 ResourceId D3D12Replay::ApplyCustomShader(ResourceId shader, ResourceId texid, uint32_t mip,

renderdoc/driver/d3d12/d3d12_replay.h

@@ -136,12 +136,13 @@ public:
   void GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip,
                       const GetTextureDataParams &params, bytebuf &data);
 
-  void BuildTargetShader(string source, string entry, const ShaderCompileFlags &compileFlags,
-                         ShaderStage type, ResourceId *id, string *errors);
   rdcarray<ShaderEncoding> GetTargetShaderEncodings()
   {
     return {ShaderEncoding::DXBC, ShaderEncoding::HLSL};
   }
+  void BuildTargetShader(ShaderEncoding sourceEncoding, bytebuf source, string entry,
+                         const ShaderCompileFlags &compileFlags, ShaderStage type, ResourceId *id,
+                         string *errors);
   void ReplaceResource(ResourceId from, ResourceId to);
   void RemoveReplacement(ResourceId id);
 
@@ -205,8 +206,9 @@ private:
 
   void CreateSOBuffers();
 
-  void BuildShader(std::string source, std::string entry, const ShaderCompileFlags &compileFlags,
-                   ShaderStage type, ResourceId *id, std::string *errors);
+  void BuildShader(ShaderEncoding sourceEncoding, bytebuf source, std::string entry,
+                   const ShaderCompileFlags &compileFlags, ShaderStage type, ResourceId *id,
+                   std::string *errors);
 
   bool RenderTextureInternal(D3D12_CPU_DESCRIPTOR_HANDLE rtv, TextureDisplay cfg,
                              TexDisplayFlags flags);

renderdoc/driver/gl/gl_replay.cpp

@@ -2736,8 +2736,9 @@ void GLReplay::FreeCustomShader(ResourceId id)
   m_pDriver->glDeleteProgram(m_pDriver->GetResourceManager()->GetCurrentResource(id).name);
 }
 
-void GLReplay::BuildTargetShader(string source, string entry, const ShaderCompileFlags &compileFlags,
-                                 ShaderStage type, ResourceId *id, string *errors)
+void GLReplay::BuildTargetShader(ShaderEncoding sourceEncoding, bytebuf source, string entry,
+                                 const ShaderCompileFlags &compileFlags, ShaderStage type,
+                                 ResourceId *id, string *errors)
 {
   if(id == NULL || errors == NULL)
   {
@@ -2768,7 +2769,8 @@ void GLReplay::BuildTargetShader(string source, string entry, const ShaderCompil
     }
   }
 
-  const char *src = source.c_str();
+  std::string glsl((char *)source.begin(), (char *)source.end());
+  const char *src = glsl.c_str();
   GLuint shader = drv.glCreateShader(shtype);
   drv.glShaderSource(shader, 1, &src, NULL);
   drv.glCompileShader(shader);

renderdoc/driver/gl/gl_replay.h

@@ -178,9 +178,10 @@ public:
 
   void RenderMesh(uint32_t eventId, const vector<MeshFormat> &secondaryDraws, const MeshDisplay &cfg);
 
-  void BuildTargetShader(string source, string entry, const ShaderCompileFlags &compileFlags,
-                         ShaderStage type, ResourceId *id, string *errors);
   rdcarray<ShaderEncoding> GetTargetShaderEncodings() { return {ShaderEncoding::GLSL}; }
+  void BuildTargetShader(ShaderEncoding sourceEncoding, bytebuf source, string entry,
+                         const ShaderCompileFlags &compileFlags, ShaderStage type, ResourceId *id,
+                         string *errors);
   void BuildCustomShader(string source, string entry, const ShaderCompileFlags &compileFlags,
                          ShaderStage type, ResourceId *id, string *errors);
   void FreeCustomShader(ResourceId id);

renderdoc/driver/vulkan/vk_replay.cpp

@@ -3117,39 +3117,48 @@ ResourceId VulkanReplay::ApplyCustomShader(ResourceId shader, ResourceId texid,
   return GetResID(GetDebugManager()->GetCustomTexture());
 }
 
-void VulkanReplay::BuildTargetShader(string source, string entry,
+void VulkanReplay::BuildTargetShader(ShaderEncoding sourceEncoding, bytebuf source, string entry,
                                      const ShaderCompileFlags &compileFlags, ShaderStage type,
                                      ResourceId *id, string *errors)
 {
-  SPIRVShaderStage stage = SPIRVShaderStage::Invalid;
-
-  switch(type)
-  {
-    case ShaderStage::Vertex: stage = SPIRVShaderStage::Vertex; break;
-    case ShaderStage::Hull: stage = SPIRVShaderStage::TessControl; break;
-    case ShaderStage::Domain: stage = SPIRVShaderStage::TessEvaluation; break;
-    case ShaderStage::Geometry: stage = SPIRVShaderStage::Geometry; break;
-    case ShaderStage::Pixel: stage = SPIRVShaderStage::Fragment; break;
-    case ShaderStage::Compute: stage = SPIRVShaderStage::Compute; break;
-    default:
-      RDCERR("Unexpected type in BuildShader!");
-      *id = ResourceId();
-      return;
-  }
-
-  vector<string> sources;
-  sources.push_back(source);
   vector<uint32_t> spirv;
 
-  SPIRVCompilationSettings settings(SPIRVSourceLanguage::VulkanGLSL, stage);
-
-  string output = CompileSPIRV(settings, sources, spirv);
-
-  if(spirv.empty())
+  if(sourceEncoding == ShaderEncoding::GLSL)
   {
-    *id = ResourceId();
-    *errors = output;
-    return;
+    SPIRVShaderStage stage = SPIRVShaderStage::Invalid;
+
+    switch(type)
+    {
+      case ShaderStage::Vertex: stage = SPIRVShaderStage::Vertex; break;
+      case ShaderStage::Hull: stage = SPIRVShaderStage::TessControl; break;
+      case ShaderStage::Domain: stage = SPIRVShaderStage::TessEvaluation; break;
+      case ShaderStage::Geometry: stage = SPIRVShaderStage::Geometry; break;
+      case ShaderStage::Pixel: stage = SPIRVShaderStage::Fragment; break;
+      case ShaderStage::Compute: stage = SPIRVShaderStage::Compute; break;
+      default:
+        RDCERR("Unexpected type in BuildShader!");
+        *id = ResourceId();
+        return;
+    }
+
+    vector<string> sources;
+    sources.push_back(std::string((char *)source.begin(), (char *)source.end()));
+
+    SPIRVCompilationSettings settings(SPIRVSourceLanguage::VulkanGLSL, stage);
+
+    string output = CompileSPIRV(settings, sources, spirv);
+
+    if(spirv.empty())
+    {
+      *id = ResourceId();
+      *errors = output;
+      return;
+    }
+  }
+  else
+  {
+    spirv.resize(source.size() / 4);
+    memcpy(&spirv[0], source.data(), source.size());
   }
 
   VkShaderModuleCreateInfo modinfo = {

renderdoc/driver/vulkan/vk_replay.h

@@ -265,12 +265,13 @@ public:
 
   void RenderMesh(uint32_t eventId, const vector<MeshFormat> &secondaryDraws, const MeshDisplay &cfg);
 
-  void BuildTargetShader(string source, string entry, const ShaderCompileFlags &compileFlags,
-                         ShaderStage type, ResourceId *id, string *errors);
   rdcarray<ShaderEncoding> GetTargetShaderEncodings()
   {
     return {ShaderEncoding::SPIRV, ShaderEncoding::GLSL};
   }
+  void BuildTargetShader(ShaderEncoding sourceEncoding, bytebuf source, string entry,
+                         const ShaderCompileFlags &compileFlags, ShaderStage type, ResourceId *id,
+                         string *errors);
   void BuildCustomShader(string source, string entry, const ShaderCompileFlags &compileFlags,
                          ShaderStage type, ResourceId *id, string *errors);
   void FreeCustomShader(ResourceId id);

renderdoc/replay/replay_controller.cpp

@@ -1777,8 +1777,16 @@ rdcarray<ShaderEncoding> ReplayController::GetTargetShaderEncodings()
 }
 
 rdcpair<ResourceId, rdcstr> ReplayController::BuildTargetShader(
-    const char *entry, const char *source, const ShaderCompileFlags &compileFlags, ShaderStage type)
+    const char *entry, ShaderEncoding sourceEncoding, bytebuf source,
+    const ShaderCompileFlags &compileFlags, ShaderStage type)
 {
+  rdcarray<ShaderEncoding> encodings = m_pDevice->GetTargetShaderEncodings();
+
+  if(encodings.indexOf(sourceEncoding) == -1)
+    return make_rdcpair<ResourceId, rdcstr>(
+        ResourceId(),
+        StringFormat::Fmt("Shader Encoding '%s' is not supported", ToStr(sourceEncoding).c_str()));
+
   ResourceId id;
   string errs;
 
@@ -1793,7 +1801,7 @@ rdcpair<ResourceId, rdcstr> ReplayController::BuildTargetShader(
     default: RDCERR("Unexpected type in BuildShader!"); return rdcpair<ResourceId, rdcstr>();
   }
 
-  m_pDevice->BuildTargetShader(source, entry, compileFlags, type, &id, &errs);
+  m_pDevice->BuildTargetShader(sourceEncoding, source, entry, compileFlags, type, &id, &errs);
 
   if(id != ResourceId())
     m_TargetResources.insert(id);

renderdoc/replay/replay_controller.h

@@ -150,8 +150,9 @@ public:
                                                 ShaderStage type);
   void FreeCustomShader(ResourceId id);
 
-  rdcpair<ResourceId, rdcstr> BuildTargetShader(const char *entry, const char *source,
   rdcarray<ShaderEncoding> GetTargetShaderEncodings();
+  rdcpair<ResourceId, rdcstr> BuildTargetShader(const char *entry, ShaderEncoding sourceEncoding,
+                                                bytebuf source,
                                                 const ShaderCompileFlags &compileFlags,
                                                 ShaderStage type);
   void ReplaceResource(ResourceId from, ResourceId to);

renderdoc/replay/replay_driver.h

@@ -137,8 +137,9 @@ public:
   virtual void GetTextureData(ResourceId tex, uint32_t arrayIdx, uint32_t mip,
                               const GetTextureDataParams &params, bytebuf &data) = 0;
 
-  virtual void BuildTargetShader(string source, string entry, const ShaderCompileFlags &compileFlags,
-                                 ShaderStage type, ResourceId *id, string *errors) = 0;
+  virtual void BuildTargetShader(ShaderEncoding sourceEncoding, bytebuf source, string entry,
+                                 const ShaderCompileFlags &compileFlags, ShaderStage type,
+                                 ResourceId *id, string *errors) = 0;
   virtual rdcarray<ShaderEncoding> GetTargetShaderEncodings() = 0;
   virtual void ReplaceResource(ResourceId from, ResourceId to) = 0;
   virtual void RemoveReplacement(ResourceId id) = 0;