Implement UI side of detecting vulkan layer problems and fixing as admin

* Stub functions for the actual detection/fixing part.

qrenderdoc/Code/QRDUtils.cpp

@@ -34,6 +34,7 @@
 #include <QMetaMethod>
 #include <QProcess>
 #include <QProgressDialog>
+#include <QStandardPaths>
 #include <QTreeWidget>
 #include <QtMath>
 
@@ -673,6 +674,138 @@ protected:
   static const int maxProgress = 1000;
 };
 
+#if defined(Q_OS_WIN32)
+#include <shellapi.h>
+#include <windows.h>
+#endif
+
+bool RunProcessAsAdmin(const QString &fullExecutablePath, const QStringList &params,
+                       std::function<void()> finishedCallback)
+{
+#if defined(Q_OS_WIN32)
+
+  std::wstring wideExe = fullExecutablePath.toStdWString();
+  std::wstring wideParams = params.join(QChar(' ')).toStdWString();
+
+  SHELLEXECUTEINFOW info = {};
+  info.cbSize = sizeof(info);
+  info.fMask = SEE_MASK_NOCLOSEPROCESS;
+  info.lpVerb = L"runas";
+  info.lpFile = wideExe.c_str();
+  info.lpParameters = wideParams.c_str();
+  info.nShow = SW_SHOWNORMAL;
+
+  ShellExecuteExW(&info);
+
+  if((uintptr_t)info.hInstApp > 32 && info.hProcess != NULL)
+  {
+    if(finishedCallback)
+    {
+      HANDLE h = info.hProcess;
+
+      // do the wait on another thread
+      LambdaThread *thread = new LambdaThread([h, finishedCallback]() {
+        WaitForSingleObject(h, 30000);
+        CloseHandle(h);
+        GUIInvoke::call(finishedCallback);
+      });
+      thread->selfDelete(true);
+      thread->start();
+    }
+    else
+    {
+      CloseHandle(info.hProcess);
+    }
+
+    return true;
+  }
+
+  return false;
+
+#else
+  // try to find a way to run the application elevated.
+  const QString graphicalSudo[] = {
+      "pkexec", "kdesudo", "gksudo", "beesu",
+  };
+
+  // if none of the graphical options, then look for sudo and either
+  const QString termEmulator[] = {
+      "x-terminal-emulator", "gnome-terminal", "knosole", "xterm",
+  };
+
+  for(const QString &sudo : graphicalSudo)
+  {
+    QString inPath = QStandardPaths::findExecutable(sudo);
+
+    // can't find in path
+    if(inPath.isEmpty())
+      continue;
+
+    QProcess *process = new QProcess;
+
+    QStringList sudoParams;
+    sudoParams << fullExecutablePath;
+    for(const QString &p : params)
+      sudoParams << p;
+
+    qInfo() << "Running" << sudo << "with params" << sudoParams;
+
+    // run with sudo
+    process->start(sudo, sudoParams);
+
+    // when the process exits, call the callback and delete
+    QObject::connect(process, OverloadedSlot<int>::of(&QProcess::finished),
+                     [process, finishedCallback](int exitCode) {
+                       process->deleteLater();
+                       GUIInvoke::call(finishedCallback);
+                     });
+
+    return true;
+  }
+
+  QString sudo = QStandardPaths::findExecutable("sudo");
+
+  if(sudo.isEmpty())
+  {
+    qCritical() << "Couldn't find graphical or terminal sudo program!\n"
+                << "Please run " << fullExecutablePath << "with args" << params << "manually.";
+    return false;
+  }
+
+  for(const QString &term : termEmulator)
+  {
+    QString inPath = QStandardPaths::findExecutable(term);
+
+    // can't find in path
+    if(inPath.isEmpty())
+      continue;
+
+    QProcess *process = new QProcess;
+
+    // run terminal sudo with emulator
+    QStringList termParams;
+    termParams << "-e"
+               << QString("bash -c 'sudo %1 %2'").arg(fullExecutablePath).arg(params.join(QChar(' ')));
+
+    process->start(term, termParams);
+
+    // when the process exits, call the callback and delete
+    QObject::connect(process, OverloadedSlot<int>::of(&QProcess::finished),
+                     [process, finishedCallback](int exitCode) {
+                       process->deleteLater();
+                       GUIInvoke::call(finishedCallback);
+                     });
+
+    return true;
+  }
+
+  qCritical() << "Couldn't find graphical or terminal emulator to launch sudo.\n"
+              << "Please run " << fullExecutablePath << "with args" << params << "manually.";
+
+  return false;
+#endif
+}
+
 void ShowProgressDialog(QWidget *window, const QString &labelText, ProgressFinishedMethod finished,
                         ProgressUpdateMethod update)
 {

qrenderdoc/Code/QRDUtils.h

@@ -687,6 +687,9 @@ class QProgressDialog;
 typedef std::function<float()> ProgressUpdateMethod;
 typedef std::function<bool()> ProgressFinishedMethod;
 
+bool RunProcessAsAdmin(const QString &fullExecutablePath, const QStringList &params,
+                       std::function<void()> finishedCallback = std::function<void()>());
+
 void ShowProgressDialog(QWidget *window, const QString &labelText, ProgressFinishedMethod finished,
                         ProgressUpdateMethod update = ProgressUpdateMethod());
 

qrenderdoc/Code/qrenderdoc.cpp

@@ -64,12 +64,24 @@ int main(int argc, char *argv[])
       temp = true;
   }
 
+  for(int i = 0; i < argc; i++)
+  {
+    if(!QString::compare(argv[i], "--install_vulkan_layer") && i + 1 < argc)
+    {
+      if(!QString::compare(argv[i + 1], "root"))
+        RENDERDOC_UpdateVulkanLayerRegistration(true);
+      else
+        RENDERDOC_UpdateVulkanLayerRegistration(false);
+      return 0;
+    }
+  }
+
   QString remoteHost = "";
   uint remoteIdent = 0;
 
   for(int i = 0; i + 1 < argc; i++)
   {
-    if(!QString::compare(argv[i], "--REMOTEACCESS", Qt::CaseInsensitive))
+    if(!QString::compare(argv[i], "--remoteaccess", Qt::CaseInsensitive))
     {
       QRegularExpression regexp("^([a-zA-Z0-9_-]+:)?([0-9]+)$");
 

qrenderdoc/Windows/Dialogs/CaptureDialog.cpp

@@ -157,8 +157,7 @@ CaptureDialog::CaptureDialog(CaptureContext &ctx, OnCaptureMethod captureCallbac
   // sort by PID by default
   ui->processList->sortByColumn(1, Qt::AscendingOrder);
 
-  // TODO Vulkan Layer
-  ui->vulkanLayerWarn->setVisible(false);
+  ui->vulkanLayerWarn->setVisible(RENDERDOC_NeedVulkanLayerRegistration(NULL, NULL, NULL));
 
   m_CaptureCallback = captureCallback;
   m_InjectCallback = injectCallback;
@@ -280,7 +279,133 @@ void CaptureDialog::on_exePath_textChanged(const QString &text)
 
 void CaptureDialog::on_vulkanLayerWarn_clicked()
 {
-  // TODO Vulkan Layer
+  QString caption = tr("Configure Vulkan layer settings in registry?");
+
+  uint32_t flags = 0;
+  rdctype::array<rdctype::str> myJSONs;
+  rdctype::array<rdctype::str> otherJSONs;
+
+  RENDERDOC_NeedVulkanLayerRegistration(&flags, &myJSONs, &otherJSONs);
+
+  const bool hasOtherJSON = (flags & eVulkan_OtherInstallsRegistered);
+  const bool thisRegistered = (flags & eVulkan_ThisInstallRegistered);
+  const bool needElevation = (flags & eVulkan_NeedElevation);
+  const bool couldElevate = (flags & eVulkan_CouldElevate);
+  const bool registerAll = (flags & eVulkan_RegisterAll);
+  const bool updateAllowed = (flags & eVulkan_UpdateAllowed);
+
+  QString msg =
+      tr("Vulkan capture happens through the API's layer mechanism. RenderDoc has detected that ");
+
+  if(hasOtherJSON)
+  {
+    if(otherJSONs.count > 1)
+      msg +=
+          tr("there are other RenderDoc builds registered already. They must be disabled so that "
+             "capture can happen without nasty clashes.");
+    else
+      msg +=
+          tr("there is another RenderDoc build registered already. It must be disabled so that "
+             "capture can happen without nasty clashes.");
+
+    if(!thisRegistered)
+      msg += tr(" Also ");
+  }
+
+  if(!thisRegistered)
+  {
+    msg +=
+        tr("the layer for this installation is not yet registered. This could be due to an "
+           "upgrade from a version that didn't support Vulkan, or if this version is just a loose "
+           "unzip/dev build.");
+  }
+
+  msg += tr("\n\nWould you like to proceed with the following changes?\n\n");
+
+  if(hasOtherJSON)
+  {
+    for(const rdctype::str &j : otherJSONs)
+      msg += (updateAllowed ? tr("Unregister/update: %1\n") : tr("Unregister: %1\n")).arg(ToQStr(j));
+
+    msg += "\n";
+  }
+
+  if(!thisRegistered)
+  {
+    if(registerAll)
+    {
+      for(const rdctype::str &j : myJSONs)
+        msg += (updateAllowed ? tr("Register/update: %1\n") : tr("Register: %1\n")).arg(ToQStr(j));
+    }
+    else
+    {
+      msg += updateAllowed ? tr("Register one of:\n") : tr("Register/update one of:\n");
+      for(const rdctype::str &j : myJSONs)
+        msg += tr("  -- %1\n").arg(ToQStr(j));
+    }
+
+    msg += "\n";
+  }
+
+  msg += tr("This is a one-off change, it won't be needed again unless the installation moves.");
+
+  QMessageBox::StandardButton install = RDDialog::question(this, caption, msg, RDDialog::YesNoCancel);
+
+  if(install == QMessageBox::Yes)
+  {
+    bool run = false;
+    bool admin = false;
+
+    // if we need to elevate, just try it.
+    if(needElevation)
+    {
+      admin = run = true;
+    }
+    // if we could elevate, ask the user
+    else if(couldElevate)
+    {
+      QMessageBox::StandardButton elevate = RDDialog::question(
+          this, tr("System layer install"),
+          tr("Do you want to elevate permissions to install the layer at a system level?"),
+          RDDialog::YesNoCancel);
+
+      if(elevate == QMessageBox::Yes)
+        admin = true;
+      else if(elevate == QMessageBox::No)
+        admin = false;
+
+      run = (elevate != QMessageBox::Cancel);
+    }
+    // otherwise run non-elevated
+    else
+    {
+      run = true;
+    }
+
+    if(run)
+    {
+      if(admin)
+      {
+        RunProcessAsAdmin(qApp->applicationFilePath(), QStringList() << "--install_vulkan_layer"
+                                                                     << "root",
+                          [this]() {
+                            // ui->vulkanLayerWarn->setVisible(RENDERDOC_NeedVulkanLayerRegistration(NULL,
+                            // NULL, NULL));
+                            ui->vulkanLayerWarn->setVisible(false);
+                          });
+        return;
+      }
+      else
+      {
+        QProcess process;
+        process.start(qApp->applicationFilePath(), QStringList() << "--install_vulkan_layer"
+                                                                 << "user");
+        process.waitForFinished(300);
+      }
+    }
+
+    ui->vulkanLayerWarn->setVisible(RENDERDOC_NeedVulkanLayerRegistration(NULL, NULL, NULL));
+  }
 }
 
 void CaptureDialog::on_processRefesh_clicked()

qrenderdoc/Windows/Dialogs/CaptureDialog.ui

@@ -274,48 +274,17 @@
        <height>36</height>
       </size>
      </property>
-     <property name="palette">
-      <palette>
-       <active>
-        <colorrole role="Button">
-         <brush brushstyle="SolidPattern">
-          <color alpha="255">
-           <red>255</red>
-           <green>255</green>
-           <blue>220</blue>
-          </color>
-         </brush>
-        </colorrole>
-       </active>
-       <inactive>
-        <colorrole role="Button">
-         <brush brushstyle="SolidPattern">
-          <color alpha="255">
-           <red>255</red>
-           <green>255</green>
-           <blue>220</blue>
-          </color>
-         </brush>
-        </colorrole>
-       </inactive>
-       <disabled>
-        <colorrole role="Button">
-         <brush brushstyle="SolidPattern">
-          <color alpha="255">
-           <red>255</red>
-           <green>255</green>
-           <blue>220</blue>
-          </color>
-         </brush>
-        </colorrole>
-       </disabled>
-      </palette>
-     </property>
      <property name="cursor">
       <cursorShape>PointingHandCursor</cursorShape>
      </property>
-     <property name="autoFillBackground">
-      <bool>true</bool>
+     <property name="styleSheet">
+      <string notr="true">QToolButton {
+background-color: #ffffdc;
+border: 1px solid black;
+}
+QToolButton:hover {
+background-color: #ddddbe;
+}</string>
      </property>
      <property name="text">
       <string>Warning: Vulkan capture is not configured.

renderdoc/api/replay/renderdoc_replay.h

@@ -642,6 +642,14 @@ RENDERDOC_ExecuteAndInject(const char *app, const char *workingDir, const char *
 extern "C" RENDERDOC_API uint32_t RENDERDOC_CC RENDERDOC_InjectIntoProcess(
     uint32_t pid, void *env, const char *logfile, const CaptureOptions *opts, bool32 waitForExit);
 
+//////////////////////////////////////////////////////////////////////////
+// Vulkan layer handling
+//////////////////////////////////////////////////////////////////////////
+
+extern "C" RENDERDOC_API bool RENDERDOC_CC RENDERDOC_NeedVulkanLayerRegistration(
+    uint32_t *flags, rdctype::array<rdctype::str> *myJSONs, rdctype::array<rdctype::str> *otherJSONs);
+extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_UpdateVulkanLayerRegistration(bool elevate);
+
 //////////////////////////////////////////////////////////////////////////
 // Miscellaneous!
 //////////////////////////////////////////////////////////////////////////

renderdoc/api/replay/replay_enums.h

@@ -601,3 +601,13 @@ enum LogMessageType
   eLogType_Fatal,
   eLogType_NumTypes,
 };
+
+enum VulkanFlags
+{
+  eVulkan_OtherInstallsRegistered = 0x1,
+  eVulkan_ThisInstallRegistered = 0x2,
+  eVulkan_NeedElevation = 0x4,
+  eVulkan_CouldElevate = 0x8,
+  eVulkan_RegisterAll = 0x10,
+  eVulkan_UpdateAllowed = 0x20,
+};

renderdoc/replay/entry_points.cpp

@@ -798,3 +798,16 @@ extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_StartAndroidRemoteServer()
   adbExecCommand(
       "shell am start -n org.renderdoc.renderdoccmd/.Loader -e renderdoccmd remoteserver");
 }
+
+extern "C" RENDERDOC_API bool RENDERDOC_CC RENDERDOC_NeedVulkanLayerRegistration(
+    uint32_t *flags, rdctype::array<rdctype::str> *myJSONs, rdctype::array<rdctype::str> *otherJSONs)
+{
+  // stub
+
+  return false;
+}
+
+extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_UpdateVulkanLayerRegistration(bool elevate)
+{
+  // stub
+}