Preserve stdout/stderr from external shader tools and display to user

qrenderdoc/Code/Interface/PersistantConfig.h

@@ -112,6 +112,18 @@ inline ShaderEncoding ToolOutput(KnownShaderTool tool)
   return ShaderEncoding::Unknown;
 }
 
+DOCUMENT(R"(Contains the output from invoking a :class:`ShaderProcessingTool`, including both the
+actual output data desired as well as any stdout/stderr messages.
+)");
+struct ShaderToolOutput
+{
+  DOCUMENT("The output log - containing the information about the tool run and any errors.");
+  rdcstr log;
+
+  DOCUMENT("The actual output data from the tool");
+  bytebuf result;
+};
+
 DOCUMENT(R"(Describes an external program that can be used to process shaders, typically either
 compiling from a high-level language to a binary format, or decompiling from the binary format to
 a high-level language or textual representation.
@@ -171,10 +183,11 @@ struct ShaderProcessingTool
 :param str args: arguments to pass to the tool. The default arguments can be obtained using
   :meth:`DefaultArguments` which can then be customised as desired. Passing an empty string uses the
   default arguments.
-:return: The disassembly, or an empty string if something went wrong.
-:rtype: ``str``
+:return: The result of running the tool.
+:rtype: ShaderToolOutput
 )");
-  rdcstr DisassembleShader(QWidget *window, const ShaderReflection *reflection, rdcstr args) const;
+  ShaderToolOutput DisassembleShader(QWidget *window, const ShaderReflection *reflection,
+                                     rdcstr args) const;
 
   DOCUMENT(R"(Runs this program to disassemble a given shader source.
 
@@ -185,11 +198,11 @@ struct ShaderProcessingTool
 :param str args: arguments to pass to the tool. The default arguments can be obtained using
   :meth:`DefaultArguments` which can then be customised as desired. Passing an empty string uses the
   default arguments.
-:return: The compiled shader code, or an empty buffer if something went wrong.
-:rtype: ``bytes``
+:return: The result of running the tool.
+:rtype: ShaderToolOutput
 )");
-  bytebuf CompileShader(QWidget *window, rdcstr source, rdcstr entryPoint, ShaderStage stage,
-                        rdcstr args) const;
+  ShaderToolOutput CompileShader(QWidget *window, rdcstr source, rdcstr entryPoint,
+                                 ShaderStage stage, rdcstr args) const;
 };
 
 DECLARE_REFLECTION_STRUCT(ShaderProcessingTool);

qrenderdoc/Code/Interface/ShaderProcessingTool.cpp

@@ -24,6 +24,7 @@
 
 #include <QApplication>
 #include <QFile>
+#include <QStandardPaths>
 #include "Code/QRDUtils.h"
 #include "QRDInterface.h"
 
@@ -42,14 +43,150 @@ std::string DoStringise(const KnownShaderTool &el)
   END_ENUM_STRINGISE();
 }
 
-rdcstr ShaderProcessingTool::DisassembleShader(QWidget *window, const ShaderReflection *shaderDetails,
-                                               rdcstr arguments) const
+static QString tmpPath(const QString &filename)
 {
-  if(executable.isEmpty())
-    return "";
+  return QDir(QDir::tempPath()).absoluteFilePath(filename);
+}
 
-  QString input_file = QDir(QDir::tempPath()).absoluteFilePath(lit("shader_input"));
-  QString output_file = QDir(QDir::tempPath()).absoluteFilePath(lit("shader_output"));
+static ShaderToolOutput RunTool(const ShaderProcessingTool &tool, QWidget *window,
+                                QString input_file, QString output_file, QStringList &argList)
+{
+  bool writesToFile = true;
+
+  if(input_file.isEmpty())
+    input_file = tmpPath(lit("shader_input"));
+
+  if(output_file.isEmpty())
+  {
+    output_file = tmpPath(lit("shader_output"));
+    writesToFile = false;
+  }
+
+  // ensure we don't have any leftover output files.
+  QFile::remove(output_file);
+
+  QString stdout_file = QDir(QDir::tempPath()).absoluteFilePath(lit("shader_stdout"));
+
+  ShaderToolOutput ret;
+
+  if(tool.executable.isEmpty())
+  {
+    ret.log = QApplication::translate("ShaderProcessingTool",
+                                      "ERROR: No Executable specified in tool '%1'")
+                  .arg(tool.name);
+
+    return ret;
+  }
+
+  QString path = tool.executable;
+
+  if(!QDir::isAbsolutePath(path))
+  {
+    path = QStandardPaths::findExecutable(path);
+
+    if(path.isEmpty())
+    {
+      ret.log = QApplication::translate("ShaderProcessingTool",
+                                        "ERROR: Couldn't find executable '%1' in path")
+                    .arg(tool.executable);
+
+      return ret;
+    }
+  }
+
+  QByteArray stdout_data;
+  QProcess process;
+
+  LambdaThread *thread = new LambdaThread([&]() {
+    if(!writesToFile)
+      process.setStandardOutputFile(output_file);
+    else
+      process.setStandardOutputFile(stdout_file);
+
+    // for now merge stdout/stderr together. Maybe we should separate these and somehow annotate
+    // them? Merging is difficult without messing up order, and some tools output non-errors to
+    // stderr
+    process.setStandardErrorFile(stdout_file);
+
+    process.start(tool.executable, argList);
+    process.waitForFinished();
+
+    {
+      QFile outputHandle(output_file);
+      if(outputHandle.open(QFile::ReadOnly))
+      {
+        ret.result = outputHandle.readAll();
+        outputHandle.close();
+      }
+    }
+
+    {
+      QFile stdoutHandle(stdout_file);
+      if(stdoutHandle.open(QFile::ReadOnly))
+      {
+        stdout_data = stdoutHandle.readAll();
+        stdoutHandle.close();
+      }
+    }
+
+    // The input files typically aren't large and we don't generate unique names so they won't be
+    // overwritten.
+    // Leaving them alone means the user can try to recreate the tool invocation themselves.
+    // QFile::remove(input_file);
+    QFile::remove(output_file);
+    QFile::remove(stdout_file);
+  });
+  thread->start();
+
+  ShowProgressDialog(window, QApplication::translate("ShaderProcessingTool",
+                                                     "Please wait - running external tool"),
+                     [thread]() { return !thread->isRunning(); });
+
+  thread->deleteLater();
+
+  QString processStatus;
+
+  if(process.exitStatus() == QProcess::CrashExit)
+  {
+    processStatus = QApplication::translate("ShaderProcessingTool", "Process crashed with code %1.")
+                        .arg(process.exitCode());
+  }
+  else
+  {
+    processStatus = QApplication::translate("ShaderProcessingTool", "Process exited with code %1.")
+                        .arg(process.exitCode());
+  }
+
+  ret.log = QApplication::translate("ShaderProcessingTool",
+                                    "Running \"%1\" %2\n"
+                                    "%3\n"
+                                    "%4\n"
+                                    "Output file is %5 bytes")
+                .arg(path)
+                .arg(argList.join(QLatin1Char(' ')))
+                .arg(QString::fromUtf8(stdout_data))
+                .arg(processStatus)
+                .arg(ret.result.count());
+
+  return ret;
+}
+
+ShaderToolOutput ShaderProcessingTool::DisassembleShader(QWidget *window,
+                                                         const ShaderReflection *shaderDetails,
+                                                         rdcstr arguments) const
+{
+  QStringList argList = ParseArgsList(arguments.isEmpty() ? DefaultArguments() : arguments);
+
+  QString input_file, output_file;
+
+  // replace arguments after expansion to avoid problems with quoting paths etc
+  for(QString &arg : argList)
+  {
+    if(arg == lit("{input_file}"))
+      arg = input_file = tmpPath(lit("shader_input"));
+    if(arg == lit("{output_file}"))
+      arg = output_file = tmpPath(lit("shader_output"));
+  }
 
   QFile binHandle(input_file);
   if(binHandle.open(QFile::WriteOnly | QIODevice::Truncate))
@@ -60,95 +197,42 @@ rdcstr ShaderProcessingTool::DisassembleShader(QWidget *window, const ShaderRefl
   }
   else
   {
-    RDDialog::critical(
-        window, QApplication::translate("ShaderProcessingTool", "Error writing temp file"),
-        QApplication::translate("ShaderProcessingTool", "Couldn't write temporary file %1.")
-            .arg(input_file));
-    return "";
+    ShaderToolOutput ret;
+
+    ret.log = QApplication::translate("ShaderProcessingTool",
+                                      "ERROR: Couldn't write input to temporary file '%1'")
+                  .arg(input_file);
+
+    return ret;
   }
 
-  QString programArguments = arguments;
-
-  if(programArguments.isEmpty())
-    programArguments = DefaultArguments();
-
-  if(!programArguments.contains(lit("{input_file}")))
-  {
-    RDDialog::critical(
-        window, QApplication::translate("ShaderProcessingTool", "Wrongly configured tool"),
-        QApplication::translate(
-            "ShaderProcessingTool",
-            "Please use {input_file} in the tool arguments to specify the input file."));
-    return "";
-  }
-
-  QString outputData;
-
-  QString expandedargs = programArguments;
-
-  bool writesToFile = expandedargs.contains(lit("{output_file}"));
-
-  expandedargs.replace(lit("{input_file}"), input_file);
-  expandedargs.replace(lit("{output_file}"), output_file);
-
-  QStringList argList = ParseArgsList(expandedargs);
-
-  LambdaThread *thread =
-      new LambdaThread([this, window, &outputData, argList, input_file, output_file, writesToFile]() {
-        QProcess process;
-        process.start(executable, argList);
-        process.waitForFinished();
-
-        if(process.exitStatus() != QProcess::NormalExit || process.exitCode() != 0)
-        {
-          if(window)
-          {
-            GUIInvoke::call(window, [window]() {
-              RDDialog::critical(
-                  window, QApplication::translate("ShaderProcessingTool", "Error running tool"),
-                  QApplication::translate(
-                      "ShaderProcessingTool",
-                      "There was an error invoking the external shader processing tool."));
-            });
-          }
-        }
-
-        if(writesToFile)
-        {
-          QFile outputHandle(output_file);
-          if(outputHandle.open(QFile::ReadOnly))
-          {
-            outputData = QString::fromUtf8(outputHandle.readAll());
-            outputHandle.close();
-          }
-        }
-        else
-        {
-          outputData = QString::fromUtf8(process.readAll());
-        }
-
-        QFile::remove(input_file);
-        QFile::remove(output_file);
-      });
-  thread->start();
-
-  ShowProgressDialog(window, QApplication::translate("ShaderProcessingTool",
-                                                     "Please wait - running external tool"),
-                     [thread]() { return !thread->isRunning(); });
-
-  thread->deleteLater();
-
-  return outputData;
+  return RunTool(*this, window, input_file, output_file, argList);
 }
 
-bytebuf ShaderProcessingTool::CompileShader(QWidget *window, rdcstr source, rdcstr entryPoint,
-                                            ShaderStage stage, rdcstr arguments) const
+ShaderToolOutput ShaderProcessingTool::CompileShader(QWidget *window, rdcstr source,
+                                                     rdcstr entryPoint, ShaderStage stage,
+                                                     rdcstr arguments) const
 {
-  if(executable.isEmpty())
-    return bytebuf();
+  QStringList argList = ParseArgsList(arguments.isEmpty() ? DefaultArguments() : arguments);
 
-  QString input_file = QDir(QDir::tempPath()).absoluteFilePath(lit("shader_input"));
-  QString output_file = QDir(QDir::tempPath()).absoluteFilePath(lit("shader_output"));
+  QString input_file, output_file;
+
+  const QString glsl_stage4[ENUM_ARRAY_SIZE(ShaderStage)] = {
+      lit("vert"), lit("tesc"), lit("tese"), lit("geom"), lit("frag"), lit("comp"),
+  };
+
+  // replace arguments after expansion to avoid problems with quoting paths etc
+  for(QString &arg : argList)
+  {
+    if(arg == lit("{input_file}"))
+      arg = input_file = tmpPath(lit("shader_input"));
+    if(arg == lit("{output_file}"))
+      arg = output_file = tmpPath(lit("shader_output"));
+    if(arg == lit("{entry_point}"))
+      arg = entryPoint;
+    if(arg == lit("{glsl_stage4}"))
+      arg = glsl_stage4[int(stage)];
+  }
 
   QFile binHandle(input_file);
   if(binHandle.open(QFile::WriteOnly | QIODevice::Truncate))
@@ -158,90 +242,14 @@ bytebuf ShaderProcessingTool::CompileShader(QWidget *window, rdcstr source, rdcs
   }
   else
   {
-    RDDialog::critical(
-        window, QApplication::translate("ShaderProcessingTool", "Error writing temp file"),
-        QApplication::translate("ShaderProcessingTool", "Couldn't write temporary file %1.")
-            .arg(input_file));
-    return bytebuf();
+    ShaderToolOutput ret;
+
+    ret.log = QApplication::translate("ShaderProcessingTool",
+                                      "ERROR: Couldn't write input to temporary file '%1'")
+                  .arg(input_file);
+
+    return ret;
   }
 
-  QString programArguments = arguments;
-
-  if(programArguments.isEmpty())
-    programArguments = DefaultArguments();
-
-  if(!programArguments.contains(lit("{input_file}")))
-  {
-    RDDialog::critical(
-        window, QApplication::translate("ShaderProcessingTool", "Wrongly configured tool"),
-        QApplication::translate(
-            "ShaderProcessingTool",
-            "Please use {input_file} in the tool arguments to specify the input file."));
-    return bytebuf();
-  }
-
-  bytebuf outputData;
-
-  QString expandedargs = programArguments;
-
-  bool writesToFile = expandedargs.contains(lit("{output_file}"));
-
-  expandedargs.replace(lit("{input_file}"), input_file);
-  expandedargs.replace(lit("{entry_point}"), entryPoint);
-  expandedargs.replace(lit("{output_file}"), output_file);
-
-  const QString glsl_stage4[ENUM_ARRAY_SIZE(ShaderStage)] = {
-      lit("vert"), lit("tesc"), lit("tese"), lit("geom"), lit("frag"), lit("comp"),
-  };
-
-  expandedargs.replace(lit("{glsl_stage4}"), glsl_stage4[int(stage)]);
-
-  QStringList argList = ParseArgsList(expandedargs);
-
-  LambdaThread *thread =
-      new LambdaThread([this, window, &outputData, argList, input_file, output_file, writesToFile]() {
-        QProcess process;
-        process.start(executable, argList);
-        process.waitForFinished();
-
-        if(process.exitStatus() != QProcess::NormalExit || process.exitCode() != 0)
-        {
-          if(window)
-          {
-            GUIInvoke::call(window, [window]() {
-              RDDialog::critical(
-                  window, QApplication::translate("ShaderProcessingTool", "Error running tool"),
-                  QApplication::translate(
-                      "ShaderProcessingTool",
-                      "There was an error invoking the external shader processing tool."));
-            });
-          }
-        }
-
-        if(writesToFile)
-        {
-          QFile outputHandle(output_file);
-          if(outputHandle.open(QFile::ReadOnly))
-          {
-            outputData = outputHandle.readAll();
-            outputHandle.close();
-          }
-        }
-        else
-        {
-          outputData = process.readAll();
-        }
-
-        QFile::remove(input_file);
-        QFile::remove(output_file);
-      });
-  thread->start();
-
-  ShowProgressDialog(window, QApplication::translate("ShaderProcessingTool",
-                                                     "Please wait - running external tool"),
-                     [thread]() { return !thread->isRunning(); });
-
-  thread->deleteLater();
-
-  return outputData;
+  return RunTool(*this, window, input_file, output_file, argList);
 }

qrenderdoc/Code/pyrenderdoc/qrenderdoc_stub.cpp

@@ -77,16 +77,18 @@ rdcstr ShaderProcessingTool::DefaultArguments() const
   return "";
 }
 
-rdcstr ShaderProcessingTool::DisassembleShader(QWidget *window, const ShaderReflection *shaderDetails,
-                                               rdcstr arguments) const
+ShaderToolOutput ShaderProcessingTool::DisassembleShader(QWidget *window,
+                                                         const ShaderReflection *shaderDetails,
+                                                         rdcstr arguments) const
 {
-  return "";
+  return {};
 }
 
-bytebuf ShaderProcessingTool::CompileShader(QWidget *window, rdcstr source, rdcstr entryPoint,
-                                            ShaderStage stage, rdcstr arguments) const
+ShaderToolOutput ShaderProcessingTool::CompileShader(QWidget *window, rdcstr source,
+                                                     rdcstr entryPoint, ShaderStage stage,
+                                                     rdcstr arguments) const
 {
-  return bytebuf();
+  return {};
 }
 
 ////////////////////////////////////////////////////////////////////////////////

qrenderdoc/Windows/PipelineState/PipelineStateViewer.cpp

@@ -707,62 +707,63 @@ void PipelineStateViewer::shaderEdit_clicked()
     menu->actions()[0]->trigger();
 }
 
-void PipelineStateViewer::EditShader(ResourceId id, ShaderStage shaderType, const rdcstr &entry,
-                                     ShaderCompileFlags compileFlags, ShaderEncoding encoding,
-                                     const rdcstrpairs &files)
+IShaderViewer *PipelineStateViewer::EditShader(ResourceId id, ShaderStage shaderType,
+                                               const rdcstr &entry, ShaderCompileFlags compileFlags,
+                                               ShaderEncoding encoding, const rdcstrpairs &files)
 {
-  IShaderViewer *sv = m_Ctx.EditShader(
-      false, shaderType, entry, files, encoding, compileFlags,
-      // save callback
-      [shaderType, id](ICaptureContext *ctx, IShaderViewer *viewer, ShaderEncoding shaderEncoding,
-                       ShaderCompileFlags flags, rdcstr entryFunc, bytebuf shaderBytes) {
+  auto saveCallback = [shaderType, id](ICaptureContext *ctx, IShaderViewer *viewer,
+                                       ShaderEncoding shaderEncoding, ShaderCompileFlags flags,
+                                       rdcstr entryFunc, bytebuf shaderBytes) {
+    if(shaderBytes.isEmpty())
+      return;
 
-        if(shaderBytes.isEmpty())
-          return;
+    ANALYTIC_SET(UIFeatures.ShaderEditing, true);
 
-        ANALYTIC_SET(UIFeatures.ShaderEditing, true);
+    // invoke off to the ReplayController to replace the capture's shader
+    // with our edited one
+    ctx->Replay().AsyncInvoke([ctx, entryFunc, shaderBytes, shaderEncoding, flags, shaderType, id,
+                               viewer](IReplayController *r) {
+      rdcstr errs;
 
-        // invoke off to the ReplayController to replace the capture's shader
-        // with our edited one
-        ctx->Replay().AsyncInvoke([ctx, entryFunc, shaderBytes, shaderEncoding, flags, shaderType,
-                                   id, viewer](IReplayController *r) {
-          rdcstr errs;
+      ResourceId from = id;
+      ResourceId to;
 
-          ResourceId from = id;
-          ResourceId to;
+      std::tie(to, errs) =
+          r->BuildTargetShader(entryFunc.c_str(), shaderEncoding, shaderBytes, flags, shaderType);
 
-          std::tie(to, errs) = r->BuildTargetShader(entryFunc.c_str(), shaderEncoding, shaderBytes,
-                                                    flags, shaderType);
+      GUIInvoke::call(viewer->Widget(), [viewer, errs]() { viewer->ShowErrors(errs); });
+      if(to == ResourceId())
+      {
+        r->RemoveReplacement(from);
+        GUIInvoke::call(viewer->Widget(), [ctx]() { ctx->RefreshStatus(); });
+      }
+      else
+      {
+        r->ReplaceResource(from, to);
+        GUIInvoke::call(viewer->Widget(), [ctx]() { ctx->RefreshStatus(); });
+      }
+    });
+  };
 
-          GUIInvoke::call(viewer->Widget(), [viewer, errs]() { viewer->ShowErrors(errs); });
-          if(to == ResourceId())
-          {
-            r->RemoveReplacement(from);
-            GUIInvoke::call(viewer->Widget(), [ctx]() { ctx->RefreshStatus(); });
-          }
-          else
-          {
-            r->ReplaceResource(from, to);
-            GUIInvoke::call(viewer->Widget(), [ctx]() { ctx->RefreshStatus(); });
-          }
-        });
-      },
+  auto closeCallback = [id](ICaptureContext *ctx) {
+    // remove the replacement on close (we could make this more sophisticated if there
+    // was a place to control replaced resources/shaders).
+    ctx->Replay().AsyncInvoke([ctx, id](IReplayController *r) {
+      r->RemoveReplacement(id);
+      GUIInvoke::call(ctx->GetMainWindow()->Widget(), [ctx] { ctx->RefreshStatus(); });
+    });
+  };
 
-      // Close Callback
-      [id](ICaptureContext *ctx) {
-        // remove the replacement on close (we could make this more sophisticated if there
-        // was a place to control replaced resources/shaders).
-        ctx->Replay().AsyncInvoke([ctx, id](IReplayController *r) {
-          r->RemoveReplacement(id);
-          GUIInvoke::call(ctx->GetMainWindow()->Widget(), [ctx] { ctx->RefreshStatus(); });
-        });
-      });
+  IShaderViewer *sv = m_Ctx.EditShader(false, shaderType, entry, files, encoding, compileFlags,
+                                       saveCallback, closeCallback);
 
   m_Ctx.AddDockWindow(sv->Widget(), DockReference::AddTo, this);
+
+  return sv;
 }
 
-void PipelineStateViewer::EditOriginalShaderSource(ResourceId id,
-                                                   const ShaderReflection *shaderDetails)
+IShaderViewer *PipelineStateViewer::EditOriginalShaderSource(ResourceId id,
+                                                             const ShaderReflection *shaderDetails)
 {
   QSet<uint> uniqueFiles;
   rdcstrpairs files;
@@ -783,23 +784,28 @@ void PipelineStateViewer::EditOriginalShaderSource(ResourceId id,
     files.push_back(make_rdcpair(s.filename, s.contents));
   }
 
-  EditShader(id, shaderDetails->stage, shaderDetails->entryPoint,
-             shaderDetails->debugInfo.compileFlags, shaderDetails->debugInfo.encoding, files);
+  return EditShader(id, shaderDetails->stage, shaderDetails->entryPoint,
+                    shaderDetails->debugInfo.compileFlags, shaderDetails->debugInfo.encoding, files);
 }
 
-void PipelineStateViewer::EditDecompiledSource(const ShaderProcessingTool &tool, ResourceId id,
-                                               const ShaderReflection *shaderDetails)
+IShaderViewer *PipelineStateViewer::EditDecompiledSource(const ShaderProcessingTool &tool,
+                                                         ResourceId id,
+                                                         const ShaderReflection *shaderDetails)
 {
-  QString source = tool.DisassembleShader(this, shaderDetails, "");
+  ShaderToolOutput out = tool.DisassembleShader(this, shaderDetails, "");
 
-  if(source.isEmpty())
-    return;
+  rdcstr source;
+  source.assign((const char *)out.result.data(), out.result.size());
 
   rdcstrpairs files;
   files.push_back(make_rdcpair<rdcstr, rdcstr>("decompiled", source));
 
-  EditShader(id, shaderDetails->stage, shaderDetails->entryPoint,
-             shaderDetails->debugInfo.compileFlags, tool.output, files);
+  IShaderViewer *sv = EditShader(id, shaderDetails->stage, shaderDetails->entryPoint,
+                                 shaderDetails->debugInfo.compileFlags, tool.output, files);
+
+  sv->ShowErrors(out.log);
+
+  return sv;
 }
 
 void PipelineStateViewer::SetupShaderEditButton(QToolButton *button, ResourceId pipelineId,

qrenderdoc/Windows/PipelineState/PipelineStateViewer.h

@@ -89,11 +89,12 @@ private:
   QMenu *editMenus[6] = {};
 
   QString GenerateHLSLStub(const ShaderReflection *shaderDetails, const QString &entryFunc);
-  void EditShader(ResourceId id, ShaderStage shaderType, const rdcstr &entry,
-                  ShaderCompileFlags compileFlags, ShaderEncoding encoding, const rdcstrpairs &files);
-  void EditOriginalShaderSource(ResourceId id, const ShaderReflection *shaderDetails);
-  void EditDecompiledSource(const ShaderProcessingTool &tool, ResourceId id,
-                            const ShaderReflection *shaderDetails);
+  IShaderViewer *EditShader(ResourceId id, ShaderStage shaderType, const rdcstr &entry,
+                            ShaderCompileFlags compileFlags, ShaderEncoding encoding,
+                            const rdcstrpairs &files);
+  IShaderViewer *EditOriginalShaderSource(ResourceId id, const ShaderReflection *shaderDetails);
+  IShaderViewer *EditDecompiledSource(const ShaderProcessingTool &tool, ResourceId id,
+                                      const ShaderReflection *shaderDetails);
 
   void MakeShaderVariablesHLSL(bool cbufferContents, const rdcarray<ShaderConstant> &vars,
                                QString &struct_contents, QString &struct_defs);

qrenderdoc/Windows/ShaderViewer.cpp

@@ -306,9 +306,9 @@ void ShaderViewer::editShader(bool customShader, ShaderStage stage, const QStrin
   if(!customShader)
   {
     ui->compilationGroup->setWindowTitle(tr("Compilation Settings"));
-    ui->docking->addToolWindow(
-        ui->compilationGroup,
-        ToolWindowManager::AreaReference(ToolWindowManager::AddTo, ui->docking->areaOf(m_Errors)));
+    ui->docking->addToolWindow(ui->compilationGroup,
+                               ToolWindowManager::AreaReference(
+                                   ToolWindowManager::LeftOf, ui->docking->areaOf(m_Errors), 0.5f));
     ui->docking->setToolWindowProperties(
         ui->compilationGroup,
         ToolWindowManager::HideCloseButton | ToolWindowManager::DisallowFloatWindow);
@@ -1193,10 +1193,17 @@ void ShaderViewer::disassemble_typeChanged(int index)
   {
     if(targetStr == targetName(disasm))
     {
-      QString result = disasm.DisassembleShader(this, m_ShaderDetails, "");
+      ShaderToolOutput out = disasm.DisassembleShader(this, m_ShaderDetails, "");
+
+      const char *text;
+
+      if(out.result.isEmpty())
+        text = out.log.c_str();
+      else
+        text = (const char *)out.result.data();
 
       m_DisassemblyView->setReadOnly(false);
-      m_DisassemblyView->setText(result.toUtf8().data());
+      m_DisassemblyView->setText(text);
       m_DisassemblyView->setReadOnly(true);
       m_DisassemblyView->emptyUndoBuffer();
       return;
@@ -3462,17 +3469,16 @@ void ShaderViewer::on_refresh_clicked()
       {
         if(QString(tool.name) == ui->compileTool->currentText())
         {
-          bytebuf result = tool.CompileShader(this, source, ui->entryFunc->text(), m_Stage,
-                                              ui->toolCommandLine->toPlainText());
+          ShaderToolOutput out = tool.CompileShader(this, source, ui->entryFunc->text(), m_Stage,
+                                                    ui->toolCommandLine->toPlainText());
 
-          if(result.isEmpty())
-          {
-            ShowErrors(tr("Error invoking '%1' to compile source").arg(tool.name));
+          ShowErrors(out.log);
+
+          if(out.result.isEmpty())
             return;
-          }
 
           encoding = tool.output;
-          shaderBytes = result;
+          shaderBytes = out.result;
           break;
         }
       }