Implement global hook feature for Qt

qrenderdoc/Code/Interface/PersistantConfig.cpp

@@ -259,6 +259,9 @@ bool PersistantConfig::Load(const QString &filename)
 
 bool PersistantConfig::Save()
 {
+  if(m_Filename.isEmpty())
+    return true;
+
   // update serialize list
   RemoteHostList.clear();
   for(RemoteHost *host : RemoteHosts)
@@ -269,6 +272,11 @@ bool PersistantConfig::Save()
   return Serialize(m_Filename);
 }
 
+void PersistantConfig::Close()
+{
+  m_Filename = QString();
+}
+
 void PersistantConfig::SetupFormatting()
 {
   Formatter::setParams(*this);

qrenderdoc/Code/Interface/PersistantConfig.h

@@ -432,6 +432,15 @@ propagated and will not be forgotten in the case of crash or otherwise unexpecte
 )");
   bool Save();
 
+  DOCUMENT(R"(Closes the config file so that subsequent calls to Save() will not write to disk at
+the file the config was loaded from.
+
+This function is rarely directly used, except in the case where RenderDoc is relaunching itself and
+wants to avoid file locking conflicts between the closing instance saving, and the loading instance
+loading. It can explicitly save and close before relaunching.
+)");
+  void Close();
+
   DOCUMENT("Configures the :class:`Formatter` class with the settings from this config.");
   void SetupFormatting();
 

qrenderdoc/Code/QRDUtils.cpp

@@ -726,8 +726,55 @@ protected:
 #include <windows.h>
 
 #include <shellapi.h>
+#else
+#include <unistd.h>
 #endif
 
+typedef LSTATUS(APIENTRY *PFN_RegCreateKeyExA)(HKEY hKey, LPCSTR lpSubKey, DWORD Reserved,
+                                               LPSTR lpClass, DWORD dwOptions, REGSAM samDesired,
+                                               CONST LPSECURITY_ATTRIBUTES lpSecurityAttributes,
+                                               PHKEY phkResult, LPDWORD lpdwDisposition);
+
+typedef LSTATUS(APIENTRY *PFN_RegCloseKey)(HKEY hKey);
+
+bool IsRunningAsAdmin()
+{
+#if defined(Q_OS_WIN32)
+  // try to open HKLM\Software for write.
+  HKEY key = NULL;
+
+  // access dynamically to get around the pain of trying to link to extra window libs in qt
+  HMODULE mod = LoadLibraryA("advapi32.dll");
+
+  if(mod == NULL)
+    return false;
+
+  PFN_RegCreateKeyExA create = (PFN_RegCreateKeyExA)GetProcAddress(mod, "RegCreateKeyExA");
+  PFN_RegCloseKey close = (PFN_RegCloseKey)GetProcAddress(mod, "RegCloseKey");
+
+  LSTATUS ret = ERROR_PROC_NOT_FOUND;
+
+  if(create && close)
+  {
+    ret = create(HKEY_LOCAL_MACHINE, "SOFTWARE", 0, NULL, 0, KEY_READ | KEY_WRITE, NULL, &key, NULL);
+
+    if(key)
+      close(key);
+  }
+
+  FreeLibrary(mod);
+
+  return (ret == ERROR_SUCCESS);
+
+#else
+
+  // this isn't ideal, we should check something else since a user may have permissions to do what
+  // we want to do
+  return geteuid() == 0;
+
+#endif
+}
+
 bool RunProcessAsAdmin(const QString &fullExecutablePath, const QStringList &params,
                        std::function<void()> finishedCallback)
 {
@@ -991,6 +1038,12 @@ void ShowProgressDialog(QWidget *window, const QString &labelText, ProgressFinis
   progressTickerThread.wait();
 }
 
+void setEnabledMultiple(const QList<QWidget *> &widgets, bool enabled)
+{
+  for(QWidget *w : widgets)
+    w->setEnabled(enabled);
+}
+
 QString GetSystemUsername()
 {
   QProcessEnvironment env = QProcessEnvironment::systemEnvironment();

qrenderdoc/Code/QRDUtils.h

@@ -881,12 +881,15 @@ typedef std::function<float()> ProgressUpdateMethod;
 typedef std::function<bool()> ProgressFinishedMethod;
 
 QStringList ParseArgsList(const QString &args);
+bool IsRunningAsAdmin();
 bool RunProcessAsAdmin(const QString &fullExecutablePath, const QStringList &params,
                        std::function<void()> finishedCallback = std::function<void()>());
 
 void ShowProgressDialog(QWidget *window, const QString &labelText, ProgressFinishedMethod finished,
                         ProgressUpdateMethod update = ProgressUpdateMethod());
 
+void setEnabledMultiple(const QList<QWidget *> &widgets, bool enabled);
+
 QString GetSystemUsername();
 
 QColor contrastingColor(const QColor &col, const QColor &defaultCol);

qrenderdoc/Windows/Dialogs/CaptureDialog.cpp

@@ -402,6 +402,18 @@ void CaptureDialog::on_processRefesh_clicked()
   fillProcessList();
 }
 
+bool CaptureDialog::checkAllowClose()
+{
+  if(RENDERDOC_IsGlobalHookActive())
+  {
+    RDDialog::critical(this, tr("Global hook active"),
+                       tr("Cannot close this window while global hook is active."));
+    return false;
+  }
+
+  return true;
+}
+
 void CaptureDialog::on_exePathBrowse_clicked()
 {
   QString initDir;
@@ -487,12 +499,115 @@ void CaptureDialog::on_envVarEdit_clicked()
 
 void CaptureDialog::on_toggleGlobal_clicked()
 {
-  if(ui->toggleGlobal->isEnabled())
-  {
-    ui->toggleGlobal->setChecked(!ui->toggleGlobal->isChecked());
+  if(!ui->toggleGlobal->isEnabled())
+    return;
 
-    UpdateGlobalHook();
+  ui->toggleGlobal->setEnabled(false);
+
+  QList<QWidget *> enableDisableWidgets = {ui->exePath,       ui->exePathBrowse, ui->workDirPath,
+                                           ui->workDirBrowse, ui->cmdline,       ui->launch,
+                                           ui->saveSettings,  ui->loadSettings};
+
+  for(QWidget *o : ui->optionsGroup->findChildren<QWidget *>(QString(), Qt::FindDirectChildrenOnly))
+    if(o)
+      enableDisableWidgets << o;
+
+  for(QWidget *o : ui->actionGroup->findChildren<QWidget *>(QString(), Qt::FindDirectChildrenOnly))
+    if(o)
+      enableDisableWidgets << o;
+
+  if(ui->toggleGlobal->isChecked())
+  {
+    if(!IsRunningAsAdmin())
+    {
+      QMessageBox::StandardButton res = RDDialog::question(
+          this, tr("Restart as admin?"),
+          tr("RenderDoc needs to restart with administrator privileges. Restart?"),
+          RDDialog::YesNoCancel);
+
+      if(res == QMessageBox::Yes)
+      {
+        QString capfile = QDir::temp().absoluteFilePath(lit("global.cap"));
+
+        bool wasChecked = ui->AutoStart->isChecked();
+        ui->AutoStart->setChecked(false);
+
+        SaveSettings(capfile);
+
+        ui->AutoStart->setChecked(wasChecked);
+
+        // save the config here explicitly
+        m_Ctx.Config().Save();
+
+        bool success = RunProcessAsAdmin(qApp->applicationFilePath(), QStringList() << capfile);
+
+        if(success)
+        {
+          // close the config so that when we're shutting down we don't conflict with new process
+          // loading
+          m_Ctx.Config().Close();
+          m_Ctx.GetMainWindow()->Widget()->close();
+          return;
+        }
+
+        // don't restart if it failed for some reason (e.g. user clicked no to the elevation prompt)
+        ui->toggleGlobal->setChecked(false);
+        ui->toggleGlobal->setEnabled(true);
+        return;
+      }
+      else
+      {
+        ui->toggleGlobal->setChecked(false);
+        ui->toggleGlobal->setEnabled(true);
+        return;
+      }
+    }
+
+    setEnabledMultiple(enableDisableWidgets, false);
+
+    ui->toggleGlobal->setText(tr("Disable Global Hook"));
+
+    if(RENDERDOC_IsGlobalHookActive())
+      RENDERDOC_StopGlobalHook();
+
+    QString exe = ui->exePath->text();
+
+    QString logfile = m_Ctx.TempLogFilename(QFileInfo(exe).baseName());
+
+    bool success =
+        RENDERDOC_StartGlobalHook(exe.toUtf8().data(), logfile.toUtf8().data(), Settings().Options);
+
+    if(!success)
+    {
+      // tidy up and exit
+      RDDialog::critical(
+          this, tr("Couldn't start global hook"),
+          tr("Aborting. Couldn't start global hook. Check diagnostic log in help menu for more "
+             "information"));
+
+      setEnabledMultiple(enableDisableWidgets, true);
+
+      // won't recurse because it's not enabled yet
+      ui->toggleGlobal->setChecked(false);
+      ui->toggleGlobal->setText(tr("Enable Global Hook"));
+      ui->toggleGlobal->setEnabled(true);
+      return;
+    }
   }
+  else
+  {
+    // not checked
+    if(RENDERDOC_IsGlobalHookActive())
+      RENDERDOC_StopGlobalHook();
+
+    setEnabledMultiple(enableDisableWidgets, true);
+
+    ui->toggleGlobal->setText(tr("Enable Global Hook"));
+  }
+
+  ui->toggleGlobal->setEnabled(true);
+
+  UpdateGlobalHook();
 }
 
 void CaptureDialog::on_saveSettings_clicked()
@@ -679,7 +794,8 @@ void CaptureDialog::LoadSettings(QString filename)
 
 void CaptureDialog::UpdateGlobalHook()
 {
-  ui->globalGroup->setVisible(!IsInjectMode() && m_Ctx.Config().AllowGlobalHook);
+  ui->globalGroup->setVisible(!IsInjectMode() && m_Ctx.Config().AllowGlobalHook &&
+                              RENDERDOC_CanGlobalHook());
 
   if(ui->exePath->text().length() >= 4)
   {

qrenderdoc/Windows/Dialogs/CaptureDialog.h

@@ -72,6 +72,9 @@ public:
   void SaveSettings(QString filename) override;
   void UpdateGlobalHook() override;
 
+public slots:
+  bool checkAllowClose();
+
 private slots:
   // automatic slots
   void on_exePathBrowse_clicked();

qrenderdoc/Windows/Dialogs/SettingsDialog.cpp

@@ -89,6 +89,15 @@ SettingsDialog::SettingsDialog(ICaptureContext &ctx, QWidget *parent)
   ui->Formatter_NegExp->setValue(m_Ctx.Config().Formatter_NegExp);
   ui->Formatter_PosExp->setValue(m_Ctx.Config().Formatter_PosExp);
 
+  if(!RENDERDOC_CanGlobalHook())
+  {
+    ui->AllowGlobalHook->setEnabled(false);
+
+    QString disabledTooltip = tr("Global hooking is not supported on this platform");
+    ui->AllowGlobalHook->setToolTip(disabledTooltip);
+    ui->globalHookLabel->setToolTip(disabledTooltip);
+  }
+
   m_Init = false;
 
   QObject::connect(ui->Formatter_MinFigures, OverloadedSlot<int>::of(&QSpinBox::valueChanged), this,

qrenderdoc/Windows/Dialogs/SettingsDialog.ui

@@ -156,7 +156,7 @@ e.g. a value of 2 means 0 will display as 0.00, 0.5 as 0.50</string>
            </widget>
           </item>
           <item row="8" column="0">
-           <widget class="QLabel" name="label_7">
+           <widget class="QLabel" name="globalHookLabel">
             <property name="toolTip">
              <string>Enables functionality on the capture application window that will insert RenderDoc automatically
 into all new processes created - then inject into the target (matching) executable.

qrenderdoc/Windows/MainWindow.cpp

@@ -1587,6 +1587,14 @@ void MainWindow::closeEvent(QCloseEvent *event)
     }
   }
 
+  if(RENDERDOC_IsGlobalHookActive())
+  {
+    RDDialog::critical(this, tr("Global hook active"),
+                       tr("Cannot close RenderDoc while global hook is active."));
+    event->ignore();
+    return;
+  }
+
   if(!PromptCloseLog())
   {
     event->ignore();