Fix C++ invokes being responsible for destroying python callbacks

* If the last refcount on a python lambda/temp function is released when a
  wrapping std::function is destroyed in a C++ invoke, we can't destroy it
  safely. Instead we queue up that decref and process it the next chance we're
  able (which is either when the current execution finishes for a python shell
  execution, or on the next function call which handles extensions).

qrenderdoc/Code/pyrenderdoc/PythonContext.cpp

@@ -137,6 +137,10 @@ QMap<rdcstr, PyObject *> PythonContext::extensions;
 
 static PyObject *current_global_handle = NULL;
 
+static QMutex decrefQueueMutex;
+static QList<PyObject *> decrefQueue;
+extern "C" void ProcessDecRefQueue();
+
 void FetchException(QString &typeStr, QString &valueStr, int &finalLine, QList<QString> &frames)
 {
   PyObject *exObj = NULL, *valueObj = NULL, *tracebackObj = NULL;
@@ -845,6 +849,8 @@ void PythonContext::executeString(const QString &filename, const QString &source
 
     PyEval_SetTrace(NULL, NULL);
 
+    ProcessDecRefQueue();
+
     Py_XDECREF(thisobj);
     Py_XDECREF(traceContext);
   }
@@ -1327,6 +1333,26 @@ extern "C" void SetThreadBlocking(PyObject *global_handle, bool block)
     redirector->block = block;
 }
 
+extern "C" void QueueDecRef(PyObject *obj)
+{
+  QMutexLocker lock(&decrefQueueMutex);
+
+  decrefQueue.push_back(obj);
+}
+
+extern "C" void ProcessDecRefQueue()
+{
+  QMutexLocker lock(&decrefQueueMutex);
+
+  if(decrefQueue.isEmpty())
+    return;
+
+  for(PyObject *obj : decrefQueue)
+    Py_XDECREF(obj);
+
+  decrefQueue.clear();
+}
+
 extern "C" QWidget *QWidgetFromPy(PyObject *widget)
 {
   return PythonContext::QWidgetFromPy(widget);

qrenderdoc/Code/pyrenderdoc/function_conversion.h

@@ -33,6 +33,8 @@ extern "C" PyObject *GetCurrentGlobalHandle();
 extern "C" void HandleException(PyObject *global_handle);
 extern "C" bool IsThreadBlocking(PyObject *global_handle);
 extern "C" void SetThreadBlocking(PyObject *global_handle, bool block);
+extern "C" void QueueDecRef(PyObject *obj);
+extern "C" void ProcessDecRefQueue();
 
 struct ExceptionData
 {
@@ -184,17 +186,15 @@ struct PyObjectRefCounter
   }
   ~PyObjectRefCounter()
   {
-// in non-release, check that we're currently executing if we're about to delete the object.
-#if !defined(RELEASE)
-    if(obj->ob_refcnt == 1 && PyGILState_Check() == 0)
-    {
-      RENDERDOC_LogMessage(LogType::Error, "QTRD", __FILE__, __LINE__,
-                           "Deleting PyObjectRefCounter without python executing on this thread");
-      // return and leak the object rather than crashing
-      return;
-    }
-#endif
-    Py_DECREF(obj);
+    // it may not be safe at the point this is destroyed to decref the object. For example if a
+    // python lambda is passed into a C++ invoke function, we will be holding the only reference to
+    // that lambda here when the async invoke completes and destroyed the std::function wrapping it.
+    // Without python executing we can't decref it to 0. Instead we queue the decref, and it will be
+    // done as soon as safely possible.
+    if(PyGILState_Check() == 0)
+      QueueDecRef(obj);
+    else
+      Py_DECREF(obj);
   }
   PyObject *obj;
 };
@@ -247,6 +247,8 @@ struct varfunc
       return rettype();
     }
 
+    ProcessDecRefQueue();
+
     PyObject *result = PyObject_Call(func, args, 0);
 
     if(result == NULL)
@@ -309,6 +311,9 @@ funcType ConvertFunc(const char *funcname, PyObject *func, ExceptionHandler exHa
   if(!global_internal_handle)
     global_internal_handle = GetCurrentGlobalHandle();
 
+  // process any dangling functions that may need to be cleared up
+  ProcessDecRefQueue();
+
   // create a copy that will keep the function object alive as long as the lambda is
   PyObjectRefCounter funcptr(func);
 

qrenderdoc/Code/pyrenderdoc/pyrenderdoc_stub.cpp

@@ -64,4 +64,12 @@ extern "C" void SetThreadBlocking(PyObject *global_handle, bool block)
 {
 }
 
+extern "C" void QueueDecRef(PyObject *obj)
+{
+}
+
+extern "C" void ProcessDecRefQueue()
+{
+}
+
 REPLAY_PROGRAM_MARKER()