From 045192bf9dd5d9f8a400515d68f6dfb010ff7a1b Mon Sep 17 00:00:00 2001
From: baldurk <baldurk@baldurk.org>
Date: Fri, 5 Oct 2018 18:14:40 +0100
Subject: [PATCH] Invoke renderdoccmd as root on linux to install vulkan layer,
 not Qt UI

* On linux sometimes you can't invoke a GUI application as root. To work around
  this, we use renderdoccmd to register the layer if it's available, or fail if
  it isn't.
---
 qrenderdoc/Windows/Dialogs/CaptureDialog.cpp | 35 ++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/qrenderdoc/Windows/Dialogs/CaptureDialog.cpp b/qrenderdoc/Windows/Dialogs/CaptureDialog.cpp
index 2b6aec8e6..8e1b55c5c 100644
--- a/qrenderdoc/Windows/Dialogs/CaptureDialog.cpp
+++ b/qrenderdoc/Windows/Dialogs/CaptureDialog.cpp
@@ -26,6 +26,7 @@
 #include <QMouseEvent>
 #include <QSortFilterProxyModel>
 #include <QStandardItemModel>
+#include <QStandardPaths>
 #include "3rdparty/flowlayout/FlowLayout.h"
 #include "3rdparty/toolwindowmanager/ToolWindowManager.h"
 #include "Code/QRDUtils.h"
@@ -401,9 +402,43 @@ void CaptureDialog::vulkanLayerWarn_mouseClick()
     {
       if(admin)
       {
+// linux sometimes can't run GUI apps as root, so we have to run renderdoccmd. Check that it's
+// installed, error if not, then invoke it.
+#if defined(Q_OS_LINUX)
+        QDir binDir = QFileInfo(qApp->applicationFilePath()).absoluteDir();
+
+        QString cmd = lit("renderdoccmd");
+
+        if(binDir.exists(cmd))
+        {
+          // it's next to our exe, run that
+          cmd = binDir.absoluteFilePath(cmd);
+        }
+        else
+        {
+          QString inPath = QStandardPaths::findExecutable(cmd);
+
+          if(inPath.isEmpty())
+          {
+            RDDialog::critical(
+                this, tr("Can't locate renderdoccmd"),
+                tr("On linux we must run renderdoccmd as root to register the layer, because "
+                   "graphical applications like qrenderdoc may fail to launch.\n\n"
+                   "renderdoccmd could not be located either next to this qrenderdoc executable or "
+                   "in PATH."));
+            return;
+          }
+
+          // it's in the path, we can continue
+        }
+
+        RunProcessAsAdmin(cmd, QStringList() << lit("vulkanregister") << lit("--system"), this,
+                          [this]() { ui->vulkanLayerWarn->setVisible(false); });
+#else
         RunProcessAsAdmin(qApp->applicationFilePath(),
                           QStringList() << lit("--install_vulkan_layer") << lit("root"), this,
                           [this]() { ui->vulkanLayerWarn->setVisible(false); });
+#endif
         return;
       }
       else