Add option to launch/wait for debugger on python contexts

qrenderdoc/Code/Interface/PersistantConfig.h

@@ -373,6 +373,15 @@ DECLARE_REFLECTION_STRUCT(BugReport);
       ":type: bool");                                                                              \
   CONFIG_SETTING_VAL(public, bool, bool, Python_DebugEnabled, true)                                \
                                                                                                    \
+  DOCUMENT(                                                                                        \
+      "``True`` if a new instance of VS Code should be launched when debugging a python script "   \
+      "or UI extension.\n"                                                                         \
+      "\n"                                                                                         \
+      "Defaults to ``True``."                                                                      \
+      ""                                                                                           \
+      ":type: bool");                                                                              \
+  CONFIG_SETTING_VAL(public, bool, bool, Python_LaunchVSCode, true)                                \
+                                                                                                   \
   DOCUMENT(                                                                                        \
       "The path to an copy of the ``debugpy`` module which should be loaded for debugging."        \
       ""                                                                                           \

qrenderdoc/Code/Interface/QRDInterface.h

@@ -1248,6 +1248,13 @@ QWidget.
 )");
   virtual void RunScript() = 0;
 
+  DOCUMENT(R"(Debugs the current script in the python shell.
+
+This runs the script but waits for a debugger to connect first before beginning. The user
+will have the option to cancel, which aborts the run of the script.
+)");
+  virtual void DebugScript() = 0;
+
 protected:
   IPythonShell() = default;
   ~IPythonShell() = default;

qrenderdoc/Code/pyrenderdoc/PythonContext.cpp

@@ -91,6 +91,8 @@ extern "C" QWidget *UnwrapBareQWidget(PyObject *);
 
 extern "C" PyObject *GetCurrentGlobalHandle();
 
+QSemaphore debuggerWaitSemaphore;
+
 // little utility function to convert a PyObject * that we know is a string to a QString
 static inline QString ToQStr(PyObject *value)
 {
@@ -1405,7 +1407,7 @@ QString PythonContext::versionString()
   return QFormatStr("%1.%2.%3").arg(PY_MAJOR_VERSION).arg(PY_MINOR_VERSION).arg(PY_MICRO_VERSION);
 }
 
-void PythonContext::executeString(const QString &filename, const QString &source)
+void PythonContext::executeString(const QString &filename, const QString &source, bool debugging)
 {
   if(!initialised())
   {
@@ -1447,13 +1449,22 @@ void PythonContext::executeString(const QString &filename, const QString &source
       Py_CompileString(source.toUtf8().data(), tempFilename.toUtf8().data(),
                        source.count(QLatin1Char('\n')) == 0 ? Py_single_input : Py_file_input);
 
+  bool debugAttached = false;
+
+  if(debugging)
+    debugAttached = PythonContext::WaitForDebugger();
+
   bool caughtException = false;
   QString typeStr;
   QString valueStr;
   int finalLine = -1;
   QList<QString> frames;
 
-  if(compiled)
+  if(debugging && !debugAttached)
+  {
+    // don't do anything, we wanted to debug and the attaching was cancelled - don't execute
+  }
+  else if(compiled)
   {
     PrepareDebugTracing();
 
@@ -1522,7 +1533,7 @@ void PythonContext::executeString(const QString &filename, const QString &source
 
 void PythonContext::executeString(const QString &source)
 {
-  executeString(QString(), source);
+  executeString(QString(), source, false);
 }
 
 void PythonContext::executeFile(const QString &filename)
@@ -1540,7 +1551,7 @@ void PythonContext::executeFile(const QString &filename)
   {
     QByteArray py = f.readAll();
 
-    executeString(filename, QString::fromUtf8(py));
+    executeString(filename, QString::fromUtf8(py), false);
   }
   else
   {
@@ -2056,6 +2067,152 @@ PyObject *PythonContext::outstream_trace(PyObject *self, PyObject *args, PyObjec
   return self;
 }
 
+void PythonContext::PrepareDebuggerWait()
+{
+  if(!m_DebugPy)
+    return;
+
+  // reset the semaphore
+  while(debuggerWaitSemaphore.available())
+    debuggerWaitSemaphore.tryAcquire();
+
+  // set up one count in the semaphore
+  debuggerWaitSemaphore.release();
+}
+
+bool PythonContext::WaitForDebugger()
+{
+  if(!m_DebugPy)
+    return false;
+
+  PyGILState_STATE gil = PyGILState_Ensure();
+
+  // don't care about the return value
+  Py_XDECREF(PyObject_CallMethod(m_DebugPy, "wait_for_client", NULL));
+
+  // return whether a client connected
+
+  PyObject *is_connected = PyObject_CallMethod(m_DebugPy, "is_client_connected", NULL);
+  bool ret = (PyBool_Check(is_connected) && is_connected == Py_True);
+  Py_XDECREF(is_connected);
+
+  // indicate that the work has finished
+  debuggerWaitSemaphore.tryAcquire(1);
+
+  PyGILState_Release(gil);
+
+  return ret;
+}
+
+void PythonContext::LaunchDebugger(QWidget *window, PersistantConfig &config, QString context_location)
+{
+  if(!m_DebugPy)
+    return;
+
+  if(context_location.isEmpty())
+  {
+    for(QString path : QStandardPaths::standardLocations(QStandardPaths::AppDataLocation))
+    {
+      QDir tmpDir(path);
+      tmpDir.mkpath(lit("pytmp"));
+      tmpDir.cd(lit("pytmp"));
+      if(tmpDir.exists())
+      {
+        context_location = tmpDir.absolutePath();
+        break;
+      }
+    }
+  }
+
+  // don't overwrite an existing file, to allow user customisation
+  QDir context_dir(context_location);
+  if(!context_dir.exists(lit(".vscode/launch.json")))
+  {
+    context_dir.mkpath(lit(".vscode"));
+    QFile launch(context_dir.absoluteFilePath(lit(".vscode/launch.json")));
+    launch.open(QFile::Truncate | QFile::WriteOnly);
+    launch.write(R"(
+{
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python Debugger: Remote Attach",
+            "type": "debugpy",
+            "request": "attach",
+            "connect": { "host": "localhost", "port": 5678 }
+        }
+    ]
+}
+	)");
+    launch.close();
+
+    // write tasks to auto-attach. This will require user approval
+    QFile tasks(context_dir.absoluteFilePath(lit(".vscode/tasks.json")));
+    tasks.open(QFile::Truncate | QFile::WriteOnly);
+    tasks.write(R"(
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "attach on startup",
+            "command": "${command:workbench.action.debug.start}",
+            "runOptions": {
+                "runOn": "folderOpen"
+            }
+        }
+    ]
+}
+	)");
+    tasks.close();
+  }
+
+  GUIInvoke::defer(window, [window, &config, context_location]() {
+    // wait a short while before displaying the progress dialog in case a debugger is already connected
+    for(int i = 0; debuggerWaitSemaphore.available() == 1 && i < 40; i++)
+      QThread::msleep(5);
+
+    // if we should launch vs code and there's nothing connected, do that now
+    if(config.Python_LaunchVSCode)
+    {
+      PyGILState_STATE gil = PyGILState_Ensure();
+
+      PyObject *is_connected_ret = PyObject_CallMethod(m_DebugPy, "is_client_connected", NULL);
+      bool debugger_connected = (PyBool_Check(is_connected_ret) && is_connected_ret == Py_True);
+      Py_XDECREF(is_connected_ret);
+
+      PyGILState_Release(gil);
+
+      QString code_path = QStandardPaths::findExecutable(lit("code"));
+
+      if(!debugger_connected && !code_path.isEmpty())
+      {
+        QStringList args;
+        args << lit("-n");
+        if(!context_location.isEmpty())
+          args << context_location;
+
+        QProcess::startDetached(code_path, args);
+      }
+    }
+
+    ShowProgressDialog(
+        window, tr("Waiting for debugger to connect.\n\nListening on localhost:5678"),
+        []() { return debuggerWaitSemaphore.available() == 0; }, NULL,
+        []() {
+          PyGILState_STATE gil = PyGILState_Ensure();
+
+          PyObject *wait_for_client = PyObject_SafeGetAttrString(m_DebugPy, "wait_for_client");
+          if(wait_for_client)
+          {
+            Py_XDECREF(PyObject_CallMethod(wait_for_client, "cancel", NULL));
+            Py_XDECREF(wait_for_client);
+          }
+
+          PyGILState_Release(gil);
+        });
+  });
+}
+
 extern "C" PyThreadState *GetExecutingThreadState(PyObject *global_handle)
 {
   OutputRedirector *redirector = (OutputRedirector *)global_handle;

qrenderdoc/Code/pyrenderdoc/PythonContext.h

@@ -69,6 +69,10 @@ public:
 
   static void PrepareDebugTracing();
 
+  static void PrepareDebuggerWait();
+  static bool WaitForDebugger();
+  static void LaunchDebugger(QWidget *window, PersistantConfig &config, QString context_location);
+
   bool CheckInterfaces(rdcstr &log);
 
   QString versionString();
@@ -115,7 +119,8 @@ signals:
 
 public slots:
   void executeString(const QString &source);
-  void executeString(const QString &filename, const QString &source);
+  void executeString(const QString &filename, const QString &source, bool debugging);
+
   void executeFile(const QString &filename);
   void setGlobal(const char *varName, const char *typeName, void *object);
   void setPyGlobal(const char *varName, PyObject *object);

qrenderdoc/Windows/Dialogs/ExtensionManager.cpp

@@ -29,6 +29,7 @@
 #include <QRegularExpression>
 #include "Code/Interface/QRDInterface.h"
 #include "Code/Resources.h"
+#include "Code/pyrenderdoc/PythonContext.h"
 #include "Widgets/Extended/RDHeaderView.h"
 #include "Windows/MainWindow.h"
 #include "ui_ExtensionManager.h"
@@ -54,6 +55,7 @@ ExtensionManager::ExtensionManager(ICaptureContext &ctx)
   ui->author->setText(lit("---"));
   ui->URL->setText(lit("---"));
   ui->reload->setEnabled(false);
+  ui->debug->setEnabled(false);
   ui->alwaysLoad->setEnabled(false);
 
   QObject::connect(ui->buttonBox, &QDialogButtonBox::accepted, this, &QDialog::accept);
@@ -128,6 +130,38 @@ void ExtensionManager::on_reload_clicked()
   }
 }
 
+void ExtensionManager::on_debug_clicked()
+{
+  if(m_Extensions.empty())
+    return;
+
+  RDTreeWidgetItem *item = ui->extensions->currentItem();
+  if(!item)
+    return;
+
+  int idx = ui->extensions->indexOfTopLevelItem(item);
+
+  if(idx >= 0 && idx < m_Extensions.count())
+  {
+    const ExtensionMetadata &e = m_Extensions[idx];
+    if(!e.name.isEmpty())
+    {
+      PythonContext::PrepareDebuggerWait();
+
+      LambdaThread *thread = new LambdaThread([this]() {
+        PythonContext::WaitForDebugger();
+
+        GUIInvoke::call(this, [this]() { on_reload_clicked(); });
+      });
+
+      thread->selfDelete(true);
+      thread->start();
+
+      PythonContext::LaunchDebugger(this, m_Ctx.Config(), QFileInfo(e.filePath).absoluteFilePath());
+    }
+  }
+}
+
 void ExtensionManager::on_openLocation_clicked()
 {
   if(m_Extensions.empty())
@@ -239,6 +273,7 @@ void ExtensionManager::update_currentItem(RDTreeWidgetItem *item)
 
       bool loaded = item->checkState(2) == Qt::Checked;
       ui->reload->setEnabled(true);
+      ui->debug->setEnabled(true);
       ui->reload->setText(loaded ? tr("Reload") : tr("Load"));
       ui->alwaysLoad->setEnabled(loaded);
 

qrenderdoc/Windows/Dialogs/ExtensionManager.h

@@ -49,6 +49,7 @@ public:
 private slots:
   // automatic slots
   void on_reload_clicked();
+  void on_debug_clicked();
   void on_openLocation_clicked();
   void on_alwaysLoad_toggled(bool checked);
   void on_extensions_currentItemChanged(RDTreeWidgetItem *item, RDTreeWidgetItem *);

qrenderdoc/Windows/Dialogs/ExtensionManager.ui

@@ -7,7 +7,7 @@
     <x>0</x>
     <y>0</y>
     <width>556</width>
-    <height>544</height>
+    <height>572</height>
    </rect>
   </property>
   <property name="minimumSize">
@@ -191,10 +191,23 @@
          </spacer>
         </item>
         <item>
-         <layout class="QVBoxLayout" name="verticalLayout_2">
+         <layout class="QHBoxLayout" name="extensionButtonsLayout">
           <property name="leftMargin">
            <number>0</number>
           </property>
+          <item>
+           <spacer name="horizontalSpacer2">
+            <property name="orientation">
+             <enum>Qt::Horizontal</enum>
+            </property>
+            <property name="sizeHint" stdset="0">
+             <size>
+              <width>40</width>
+              <height>20</height>
+             </size>
+            </property>
+           </spacer>
+          </item>
           <item>
            <widget class="QPushButton" name="openLocation">
             <property name="text">
@@ -209,6 +222,13 @@
             </property>
            </widget>
           </item>
+          <item>
+           <widget class="QPushButton" name="debug">
+            <property name="text">
+             <string>Debug</string>
+            </property>
+           </widget>
+          </item>
           <item>
            <widget class="QCheckBox" name="alwaysLoad">
             <property name="text">

qrenderdoc/Windows/Dialogs/SettingsDialog.cpp

@@ -308,6 +308,7 @@ SettingsDialog::SettingsDialog(ICaptureContext &ctx, QWidget *parent)
   ui->EventBrowser_ColorEventRow->setChecked(m_Ctx.Config().EventBrowser_ColorEventRow);
 
   ui->Python_DebugEnabled->setChecked(m_Ctx.Config().Python_DebugEnabled);
+  ui->Python_LaunchVSCode->setChecked(m_Ctx.Config().Python_LaunchVSCode);
 
   ui->Python_DebugPyDir->setText(m_Ctx.Config().Python_DebugPyDir);
 
@@ -878,6 +879,13 @@ void SettingsDialog::on_Python_DebugEnabled_toggled(bool checked)
   m_Ctx.Config().Save();
 }
 
+void SettingsDialog::on_Python_LaunchVSCode_toggled(bool checked)
+{
+  m_Ctx.Config().Python_LaunchVSCode = ui->Python_LaunchVSCode->isChecked();
+
+  m_Ctx.Config().Save();
+}
+
 // texture viewer
 void SettingsDialog::on_TextureViewer_PerTexSettings_toggled(bool checked)
 {

qrenderdoc/Windows/Dialogs/SettingsDialog.h

@@ -86,6 +86,7 @@ private slots:
   void on_Python_DebugPyDirBrowse_clicked();
   void on_Python_DebugPyDir_textEdited(const QString &dir);
   void on_Python_DebugEnabled_toggled(bool checked);
+  void on_Python_LaunchVSCode_toggled(bool checked);
 
   // texture viewer
   void on_TextureViewer_PerTexSettings_toggled(bool checked);

qrenderdoc/Windows/Dialogs/SettingsDialog.ui

@@ -751,7 +751,26 @@ After interop is enabled you will need to reload any capture.</string>
             </property>
            </widget>
           </item>
-          <item row="4" column="0" colspan="2">
+          <item row="4" column="0">
+           <widget class="QLabel" name="Python_LaunchVSCodeLabel">
+            <property name="text">
+             <string>Launch VS Code for debugging</string>
+            </property>
+           </widget>
+          </item>
+          <item row="4" column="1">
+           <widget class="QCheckBox" name="Python_LaunchVSCode">
+            <property name="toolTip">
+             <string>When debugging a python script or UI extension, launch new instance of VS Code.
+
+If disabled, debugging will do nothing and wait for a debugger connection.</string>
+            </property>
+            <property name="text">
+             <string/>
+            </property>
+           </widget>
+          </item>
+          <item row="5" column="0" colspan="2">
            <spacer name="verticalSpacer_9">
             <property name="orientation">
              <enum>Qt::Vertical</enum>

qrenderdoc/Windows/PythonShell.cpp

@@ -1059,7 +1059,7 @@ rdcstr PythonShell::GetScriptText()
   return scriptEditor->getText(scriptEditor->textLength() + 1).data();
 }
 
-void PythonShell::RunScript()
+void PythonShell::runScript(bool debugging)
 {
   PythonContext *context = newContext();
 
@@ -1073,11 +1073,14 @@ void PythonShell::RunScript()
 
   enableButtons(false);
 
-  LambdaThread *thread = new LambdaThread([this, script, context]() {
+  if(debugging)
+    PythonContext::PrepareDebuggerWait();
+
+  LambdaThread *thread = new LambdaThread([this, debugging, script, context]() {
     PythonContext::AddDebuggableThread();
 
     scriptContext = context;
-    context->executeString(lit("script.py"), script);
+    context->executeString(lit("script.py"), script, debugging);
     scriptContext = NULL;
 
     GUIInvoke::call(this, [this, context]() {
@@ -1091,6 +1094,9 @@ void PythonShell::RunScript()
   thread->setName(lit("Python script"));
   thread->selfDelete(true);
   thread->start();
+
+  if(debugging)
+    PythonContext::LaunchDebugger(this, m_Ctx.Config(), QString());
 }
 
 void PythonShell::on_execute_clicked()
@@ -1200,6 +1206,11 @@ void PythonShell::on_runScript_clicked()
   RunScript();
 }
 
+void PythonShell::on_debugScript_clicked()
+{
+  DebugScript();
+}
+
 void PythonShell::on_abortRun_clicked()
 {
   if(scriptContext)
@@ -1528,6 +1539,12 @@ void PythonShell::enableButtons(bool enable)
   ui->saveScript->setEnabled(enable);
   ui->runScript->setEnabled(enable);
   ui->abortRun->setEnabled(!enable);
+  ui->debugScript->setEnabled(enable);
+
+  if(enable && !m_Ctx.Config().Python_DebugEnabled)
+  {
+    ui->debugScript->setEnabled(false);
+  }
 }
 
 void PythonShell::startAutocomplete()

qrenderdoc/Windows/PythonShell.h

@@ -55,7 +55,8 @@ public:
   void SetScriptText(rdcstr script) override;
   bool LoadScriptFromFilename(rdcstr filename) override;
   rdcstr GetScriptText() override;
-  void RunScript() override;
+  void RunScript() override { runScript(false); }
+  void DebugScript() override { runScript(true); }
 private slots:
   // automatic slots
   void on_execute_clicked();
@@ -64,6 +65,7 @@ private slots:
   void on_openScript_clicked();
   void on_saveScript_clicked();
   void on_runScript_clicked();
+  void on_debugScript_clicked();
   void on_abortRun_clicked();
 
   // manual slots
@@ -96,6 +98,8 @@ private:
   PythonContext *newImportedDummyContext();
   void setGlobals(PythonContext *ret);
 
+  void runScript(bool debugging);
+
   void startAutocomplete();
   void selectedHelp(QString word);
   void refreshCurrentHelp();

qrenderdoc/Windows/PythonShell.ui

@@ -236,6 +236,26 @@
             </property>
            </widget>
           </item>
+          <item>
+           <widget class="QToolButton" name="debugScript">
+            <property name="toolTip">
+             <string>Begin running the script in python</string>
+            </property>
+            <property name="text">
+             <string>Debug</string>
+            </property>
+            <property name="icon">
+             <iconset resource="../Resources/resources.qrc">
+              <normaloff>:/wrench.png</normaloff>:/wrench.png</iconset>
+            </property>
+            <property name="toolButtonStyle">
+             <enum>Qt::ToolButtonTextBesideIcon</enum>
+            </property>
+            <property name="autoRaise">
+             <bool>true</bool>
+            </property>
+           </widget>
+          </item>
           <item>
            <widget class="QToolButton" name="abortRun">
             <property name="toolTip">