Change python scripting 'debug' button to just attach, not run

docs/python_api/ide_integration.rst

@@ -26,7 +26,7 @@ To use it for debugging python code and with fully-featured autocomplete, here a
 #. Enable the :abbr:`Features → Tasks → Allow Automatic Tasks (task.allowAutomaticTasks)` (``@id:task.allowAutomaticTasks``) setting (optional).
 #. When actively debugging, enable ``Breakpoints →  User Uncaught Exceptions`` at the bottom of the ``Run and Debug`` sidebar.
 
-If you just installed the debugging extensions, you will have to restart the RenderDoc UI for them to be found. After this you can use the :guilabel:`Debug` buttons in the python scripting panel or the extension manager to debug python code, with full autocomplete in VS Code. In the VS Code settings JSON this looks like so:
+If you just installed the debugging extensions, you will have to restart the RenderDoc UI for them to be found. After this you can use the :guilabel:`Attach External Debugger` button in the python scripting panel to debug python code, with full autocomplete in VS Code. In the VS Code settings JSON this looks like so:
 
 .. highlight:: json
 .. code:: json
@@ -68,7 +68,7 @@ If you have just installed these extensions, you will need to restart RenderDoc
 
 Once ``debugpy`` has been loaded, the debugger is listening on the default local port ``5678``. Within VS Code or your IDE you can configure what may be called a 'remote attach' or 'debug server attach' connecting to ``localhost`` on port ``5678``.
 
-If RenderDoc has detected your installation of VS Code it also provides convenient ways to debug UI extensions and scripts. From the extension manager or in a python script you have written you can press the :guilabel:`Debug` button. This will automatically try to launch VS Code with the necessary environment to connect a debugger. If you enable the ``Allow Automatic Tasks`` option in VS Code's settings it will automatically connect to the debugger on startup, otherwise you will have to choose to start debugging in order to connect.
+If RenderDoc has detected your installation of VS Code it also provides convenient ways to debug UI extensions and scripts. In a python script you have written you can press the :guilabel:`Attach External Debugger` button. This will automatically try to launch VS Code with the necessary environment to connect a debugger. If you enable the ``Allow Automatic Tasks`` option in VS Code's settings it will automatically connect to the debugger on startup, otherwise you will have to choose to start debugging in order to connect.
 
 .. figure:: ../imgs/python/DebuggerAttached.png
 

qrenderdoc/Code/Interface/QRDInterface.h

@@ -1560,18 +1560,28 @@ created if the file fails to load
 )");
   virtual void RunScript() = 0;
 
-  DOCUMENT(R"(Debugs the current script in the python shell.
+  DOCUMENT(R"(Tries to launch a python debugger to attach to the current process, in the context
+of a given extension.
 
-This runs the script but waits for a debugger to connect first before beginning. The user
-will have the option to cancel, which aborts the run of the script.
+Returns immediately if a debugger is already connected, or is not supported.
+
+If the extension name is not a loaded extension, no debugger will be launched.
+
+.. note::
+  The extension name is only used if a new debugger instance is launched, it will target that
+  particular extension's source location. An attached python debugger can debug all python code
+  in the instance.
+
+:param str extensionName: The package name of the extension to use as context for launching a
+  debugger program.
 )");
-  virtual void DebugScript() = 0;
+  virtual void AttachDebugger(const rdcstr &extensionName) = 0;
 
   DOCUMENT(R"(Sets the filter on the output panel to only show output from the given extension.
 
 If the extension does not exist or is not loaded, no change will be made to the output filter.
 
-:param str extensionName: The name of the extension to show output from.
+:param str extensionName: The package name of the extension to show output from.
 )");
   virtual void SetExtensionOutputFilter(const rdcstr &extensionName) = 0;
 

qrenderdoc/Code/pyrenderdoc/PythonContext.cpp

@@ -1511,7 +1511,7 @@ QString PythonContext::GetTempFilename(QString filename)
   return QString();
 }
 
-void PythonContext::executeString(const QString &filename, const QString &source, bool debugging)
+void PythonContext::executeString(const QString &filename, const QString &source)
 {
   if(!initialised())
   {
@@ -1527,24 +1527,6 @@ void PythonContext::executeString(const QString &filename, const QString &source
   {
     tempFilename = lit("<interactive.py>");
   }
-  else if(!QFile::exists(filename))
-  {
-    for(QString path : QStandardPaths::standardLocations(QStandardPaths::AppDataLocation))
-    {
-      QDir tmpDir(path);
-      tmpDir.mkpath(lit("pytmp"));
-      tmpDir.cd(lit("pytmp"));
-      if(tmpDir.exists())
-      {
-        tempFilename = tmpDir.absoluteFilePath(filename);
-        QFile f(tempFilename);
-        f.open(QFile::Truncate | QFile::WriteOnly);
-        f.write(source.toUtf8().data());
-        f.close();
-        break;
-      }
-    }
-  }
 
   location.file = tempFilename;
   location.line = 1;
@@ -1557,20 +1539,13 @@ void PythonContext::executeString(const QString &filename, const QString &source
 
   bool debugAttached = false;
 
-  if(debugging)
-    debugAttached = PythonContext::WaitForDebugger();
-
   bool caughtException = false;
   QString typeStr;
   QString valueStr;
   int finalLine = -1;
   QList<QString> frames;
 
-  if(debugging && !debugAttached)
-  {
-    // don't do anything, we wanted to debug and the attaching was cancelled - don't execute
-  }
-  else if(compiled)
+  if(compiled)
   {
     PrepareDebugTracing();
 
@@ -1642,7 +1617,7 @@ void PythonContext::executeString(const QString &filename, const QString &source
 
 void PythonContext::executeString(const QString &source)
 {
-  executeString(QString(), source, false);
+  executeString(QString(), source);
 }
 
 void PythonContext::executeFile(const QString &filename)
@@ -1661,7 +1636,7 @@ void PythonContext::executeFile(const QString &filename)
   {
     QByteArray py = f.readAll();
 
-    executeString(filename, QString::fromUtf8(py), false);
+    executeString(filename, QString::fromUtf8(py));
   }
   else
   {

qrenderdoc/Code/pyrenderdoc/PythonContext.h

@@ -145,7 +145,7 @@ signals:
 
 public slots:
   void executeString(const QString &source);
-  void executeString(const QString &filename, const QString &source, bool debugging);
+  void executeString(const QString &filename, const QString &source);
 
   void executeFile(const QString &filename);
   void setGlobal(const char *varName, const char *typeName, void *object);

qrenderdoc/Code/pyrenderdoc/PythonInvokers.cpp

@@ -1263,7 +1263,10 @@ struct IPythonShellInvoker : UIThreadInvoker<IPythonShell>
   }
   rdcstr GetScriptText() { return InvokeRetFunction<rdcstr>(&IPythonShell::GetScriptText); }
   void RunScript() { return InvokeVoidFunction(&IPythonShell::RunScript); }
-  void DebugScript() { return InvokeVoidFunction(&IPythonShell::DebugScript); }
+  void AttachDebugger(const rdcstr &extensionName)
+  {
+    return InvokeVoidFunction(&IPythonShell::AttachDebugger, extensionName);
+  }
   void SetExtensionOutputFilter(const rdcstr &extensionName)
   {
     return InvokeVoidFunction(&IPythonShell::SetExtensionOutputFilter, extensionName);

qrenderdoc/Windows/PythonShell.cpp

@@ -351,7 +351,10 @@ PythonShell::PythonShell(ICaptureContext &ctx, QWidget *parent)
     bool hasDebugger = PythonContext::IsDebuggerConnected();
 
     if(m_DebuggerAttached != hasDebugger)
+    {
+      updateButtonStates();
       updateNonDebugWarning();
+    }
 
     m_DebuggerAttached = hasDebugger;
   });
@@ -484,13 +487,13 @@ PythonShell::PythonShell(ICaptureContext &ctx, QWidget *parent)
   m_Ctx.GetMainWindow()->RegisterShortcut("CTRL+S", this,
                                           [this](QWidget *) { this->on_saveScript_clicked(); });
 
+  updateButtonStates();
+
   // we defer debugging loading onto a thread so check after a delay
   QTimer::singleShot(1200, [this]() {
     if(!PythonContext::IsDebuggingEnabled())
     {
-      ui->debugScript->setEnabled(false);
-      ui->debugScript->setToolTip(
-          tr("Debugging not supported - check documentation for setup instructions"));
+      updateButtonStates();
     }
   });
 }
@@ -565,7 +568,8 @@ void PythonShell::editorTab_Changed(int index)
 
   ui->saveScript->setEnabled(editor && editor->filename() != QString());
 
-  enableButtons(ui->newScript->isEnabled());
+  updateButtonStates();
+
   updateNonDebugWarning();
 }
 
@@ -978,6 +982,13 @@ void PythonShell::updateNonDebugWarning()
       edit->setWarning(QString());
     }
   }
+
+  updateButtonStates();
+}
+
+void PythonShell::updateButtonStates()
+{
+  enableButtons(ui->newScript->isEnabled());
 }
 
 void PythonShell::addRecentFile(rdcstr filename)
@@ -1220,7 +1231,7 @@ void PythonShell::ShowHelp()
   ToolWindowManager::raiseToolWindow(ui->helpGroup);
 }
 
-void PythonShell::runScript(bool debugging)
+void PythonShell::RunScript()
 {
   EditorWrapper *editor = curEditor();
 
@@ -1244,9 +1255,6 @@ void PythonShell::runScript(bool debugging)
 
   enableButtons(false);
 
-  if(debugging)
-    PythonContext::PrepareDebuggerWait();
-
   // save any changes
   if(editor->isModified() && !editor->filename().isEmpty())
   {
@@ -1265,12 +1273,12 @@ void PythonShell::runScript(bool debugging)
 
   m_CurLineTimer->start();
 
-  LambdaThread *thread = new LambdaThread([this, debugging, script, context, editor]() {
+  LambdaThread *thread = new LambdaThread([this, script, context, editor]() {
     PythonContext::AddDebuggableThread();
 
     scriptContext = context;
     runningScriptEditor = editor->scintilla();
-    context->executeString(editor->filename(), script, debugging);
+    context->executeString(editor->filename(), script);
     scriptContext = NULL;
 
     GUIInvoke::call(this, [this, context]() {
@@ -1290,9 +1298,24 @@ void PythonShell::runScript(bool debugging)
   thread->setName(lit("Python script"));
   thread->selfDelete(true);
   thread->start();
+}
 
-  if(debugging)
-    PythonContext::LaunchDebugger(this, m_Ctx.Config(), QString());
+void PythonShell::AttachDebugger(const rdcstr &contextLocation)
+{
+  QString path = contextLocation;
+  for(const ExtensionMetadata &e : m_Ctx.Extensions().GetInstalledExtensions())
+  {
+    if(e.package == contextLocation)
+      path = e.filePath;
+  }
+
+  if(!QDir(path).exists())
+    return;
+
+  PythonContext::LaunchDebugger(this, m_Ctx.Config(), path);
+
+  updateButtonStates();
+  updateNonDebugWarning();
 }
 
 void PythonShell::on_findReplace_clicked()
@@ -1405,9 +1428,32 @@ void PythonShell::on_runScript_clicked()
   RunScript();
 }
 
-void PythonShell::on_debugScript_clicked()
+void PythonShell::on_debugAttach_clicked()
 {
-  DebugScript();
+  EditorWrapper *editor = curEditor();
+
+  if(!editor)
+    return;
+
+  QString filename = editor->filename();
+
+  if(editor->isUIExtension())
+  {
+    filename = QFileInfo(filename).absolutePath();
+
+    for(const ExtensionMetadata &e : m_Ctx.Extensions().GetInstalledExtensions())
+    {
+      if(filename.startsWith(QDir(e.filePath).absolutePath()))
+      {
+        AttachDebugger(e.package);
+        return;
+      }
+    }
+  }
+  else if(!filename.isEmpty())
+  {
+    AttachDebugger(QFileInfo(filename).absoluteDir().absolutePath());
+  }
 }
 
 void PythonShell::on_abortRun_clicked()
@@ -2268,12 +2314,23 @@ void PythonShell::enableButtons(bool enable)
   ui->saveScript->setEnabled(enable);
   ui->runScript->setEnabled(enable);
   ui->abortRun->setEnabled(!enable);
-  ui->debugScript->setEnabled(enable);
+  ui->debugAttach->setEnabled(enable && !m_DebuggerAttached && PythonContext::IsDebuggingEnabled());
+  ui->debugAttach->setToolTip(QString());
 
   EditorWrapper *editor = curEditor();
 
   ui->runScript->setToolTip(QString());
 
+  if(enable && m_DebuggerAttached)
+  {
+    ui->debugAttach->setToolTip(tr("Debugger is already attached"));
+  }
+  else if(enable && !PythonContext::IsDebuggingEnabled())
+  {
+    ui->debugAttach->setToolTip(
+        tr("Debugging not supported - check documentation for setup instructions"));
+  }
+
   if(editor)
   {
     if(editor->isUIExtension())
@@ -2283,9 +2340,10 @@ void PythonShell::enableButtons(bool enable)
     }
   }
 
-  if(enable && !m_Ctx.Config().Python_DebugEnabled)
+  if(editor == NULL || editor->filename().isEmpty())
   {
-    ui->debugScript->setEnabled(false);
+    ui->debugAttach->setEnabled(false);
+    ui->debugAttach->setToolTip(tr("Debugger requires a script saved to disk"));
   }
 }
 

qrenderdoc/Windows/PythonShell.h

@@ -106,8 +106,8 @@ public:
   bool LoadScriptFromFilename(rdcstr filename) override;
   void CreateNewScriptEditor(rdcstr name, rdcstr text) override;
   rdcstr GetScriptText() override;
-  void RunScript() override { runScript(false); }
-  void DebugScript() override { runScript(true); }
+  void RunScript() override;
+  void AttachDebugger(const rdcstr &extensionName) override;
 
   void SetExtensionOutputFilter(const rdcstr &extensionName) override;
   void SetScriptOutputFilter() override;
@@ -135,7 +135,7 @@ private slots:
   void on_saveAsScript_clicked();
 
   void on_runScript_clicked();
-  void on_debugScript_clicked();
+  void on_debugAttach_clicked();
   void on_abortRun_clicked();
   void on_outputContext_currentIndexChanged(int idx);
 
@@ -227,6 +227,7 @@ private:
 
   void updateEditorCloseButton();
   void updateNonDebugWarning();
+  void updateButtonStates();
 
   void addRecentFile(rdcstr filename);
   void updateRecentFiles(bool added);
@@ -240,8 +241,6 @@ private:
   PythonContext *newContext();
   void setGlobals(PythonContext *ret);
 
-  void runScript(bool debugging);
-
   void doAutocomplete(ScintillaEdit *editor);
   void doFunccomplete(ScintillaEdit *editor);
 

qrenderdoc/Windows/PythonShell.ui

@@ -305,12 +305,9 @@
      </widget>
     </item>
     <item>
-     <widget class="QToolButton" name="debugScript">
-      <property name="toolTip">
-       <string>Begin running the script in python</string>
-      </property>
+     <widget class="QToolButton" name="debugAttach">
       <property name="text">
-       <string>Debug</string>
+       <string>Attach External Debugger</string>
       </property>
       <property name="icon">
        <iconset resource="../Resources/resources.qrc">