renderdoc/qrenderdoc/Code/pyrenderdoc/PythonContext.cpp

/******************************************************************************
 * The MIT License (MIT)
 *
 * Copyright (c) 2017 Baldur Karlsson
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 ******************************************************************************/

// must be included first
#include <Python.h>
#include <frameobject.h>

#if PYSIDE2_ENABLED
// PySide Qt integration, must be included before Qt headers
// warning C4522: 'Shiboken::AutoDecRef': multiple assignment operators specified
#pragma warning(disable : 4522)
#include <pyside.h>
#include <pyside2_qtwidgets_python.h>
#include <shiboken.h>

PyTypeObject **SbkPySide2_QtCoreTypes = NULL;
PyTypeObject **SbkPySide2_QtGuiTypes = NULL;
PyTypeObject **SbkPySide2_QtWidgetsTypes = NULL;
#endif

#include <QApplication>
#include <QDebug>
#include <QFile>
#include <QThread>
#include "PythonContext.h"
#include "renderdoc_replay.h"

// defined in SWIG-generated renderdoc_python.cpp
extern "C" PyObject *PyInit__renderdoc(void);
extern "C" PyObject *PassObjectToPython(const char *type, void *obj);

#ifdef WIN32

// on Win32 the renderdoc.py is compiled in as a windows resource. Extract and return
#include <windows.h>
#include "Resources/resource.h"

QByteArray GetWrapperModule()
{
  HRSRC res = FindResource(NULL, MAKEINTRESOURCE(renderdoc_py_module), MAKEINTRESOURCE(TYPE_EMBED));
  HGLOBAL data = LoadResource(NULL, res);

  if(!data)
    return QByteArray();

  DWORD resSize = SizeofResource(NULL, res);
  const char *resData = (const char *)LockResource(data);

  return QByteArray(resData, (int)resSize);
}

#else

// Otherwise it's compiled in via include-bin which converts to a .c with extern array
extern unsigned char renderdoc_py[];
extern unsigned int renderdoc_py_len;

QByteArray GetWrapperModule()
{
  return QByteArray((const char *)renderdoc_py, (int)renderdoc_py_len);
}

#endif

// little utility function to convert a PyObject * that we know is a string to a QString
static inline QString ToQStr(PyObject *value)
{
  if(value)
  {
    PyObject *repr = PyObject_Str(value);
    if(repr == NULL)
      return "";

    PyObject *decoded = PyUnicode_AsUTF8String(repr);
    if(decoded == NULL)
      return "";

    QString ret = PyBytes_AsString(decoded);

    Py_DecRef(decoded);
    Py_DecRef(repr);

    return ret;
  }

  return "";
}

static wchar_t program_name[] = L"qrenderdoc";

struct OutputRedirector
{
  PyObject_HEAD;
  union
  {
    // we union with a uint64_t to ensure it's always a ulonglong even on 32-bit
    uint64_t dummy;
    PythonContext *context;
  };
  int isStdError;
};

static PyTypeObject OutputRedirectorType = {PyVarObject_HEAD_INIT(NULL, 0)};

static PyMethodDef OutputRedirector_methods[] = {
    {"write", NULL, METH_VARARGS, "Writes to the output window"},
    {"flush", NULL, METH_NOARGS, "Does nothing - only provided for compatibility"},
    {NULL}};

PyObject *PythonContext::main_dict = NULL;

void FetchException(QString &typeStr, QString &valueStr, QList<QString> frames)
{
  PyObject *exObj = NULL, *valueObj = NULL, *tracebackObj = NULL;

  PyErr_Fetch(&exObj, &valueObj, &tracebackObj);

  PyErr_NormalizeException(&exObj, &valueObj, &tracebackObj);

  if(exObj && PyType_Check(exObj))
  {
    PyTypeObject *type = (PyTypeObject *)exObj;

    typeStr = QString::fromUtf8(type->tp_name);
  }
  else
  {
    typeStr = "";
  }

  if(valueObj)
    valueStr = ToQStr(valueObj);

  if(tracebackObj)
  {
    PyObject *tracebackModule = PyImport_ImportModule("traceback");

    if(tracebackModule)
    {
      PyObject *func = PyObject_GetAttrString(tracebackModule, "format_tb");

      if(func && PyCallable_Check(func))
      {
        PyObject *args = Py_BuildValue("(N)", tracebackObj);
        PyObject *formattedTB = PyObject_CallObject(func, args);

        if(formattedTB)
        {
          Py_ssize_t size = PyList_Size(formattedTB);
          for(Py_ssize_t i = 0; i < size; i++)
          {
            PyObject *el = PyList_GetItem(formattedTB, i);

            frames << ToQStr(el);
          }

          Py_DecRef(formattedTB);
        }

        Py_DecRef(args);
      }
    }
  }

  Py_DecRef(exObj);
  Py_DecRef(valueObj);
  Py_DecRef(tracebackObj);
}

void PythonContext::GlobalInit()
{
  // must happen on the UI thread
  if(qApp->thread() != QThread::currentThread())
  {
    qFatal("PythonContext::GlobalInit MUST be called from the UI thread");
    return;
  }

  PyImport_AppendInittab("_renderdoc", &PyInit__renderdoc);

  Py_SetProgramName(program_name);

  Py_Initialize();

  PyEval_InitThreads();

  QByteArray module_src = GetWrapperModule();

  if(module_src.isEmpty())
  {
    qCritical() << "renderdoc.py wrapper is corrupt/empty. Check build configuration to ensure "
                   "SWIG compiled properly with python support.";
    return;
  }

  PyObject *renderdoc_py_compiled =
      Py_CompileString(module_src.data(), "renderdoc.py", Py_file_input);

  if(!renderdoc_py_compiled)
  {
    qCritical() << "Failed to compile renderdoc.py wrapper, python will not be available";
    return;
  }

  OutputRedirectorType.tp_name = "renderdoc_output_redirector";
  OutputRedirectorType.tp_basicsize = sizeof(OutputRedirector);
  OutputRedirectorType.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_FINALIZE;
  OutputRedirectorType.tp_doc =
      "Output redirector, to be able to catch output to stdout and stderr";
  OutputRedirectorType.tp_new = PyType_GenericNew;
  OutputRedirectorType.tp_dealloc = &PythonContext::outstream_del;
  OutputRedirectorType.tp_methods = OutputRedirector_methods;

  OutputRedirector_methods[0].ml_meth = &PythonContext::outstream_write;
  OutputRedirector_methods[1].ml_meth = &PythonContext::outstream_flush;

  PyObject *main_module = PyImport_AddModule("__main__");

  PyObject *rdoc_module = PyImport_ExecCodeModule("renderdoc", renderdoc_py_compiled);

  Py_XDECREF(renderdoc_py_compiled);

  PyModule_AddObject(main_module, "renderdoc", rdoc_module);

  main_dict = PyModule_GetDict(main_module);

  // replace sys.stdout and sys.stderr with our own objects. These have a 'this' pointer of NULL,
  // which then indicates they need to forward to a global object

  // import sys
  PyDict_SetItemString(main_dict, "sys", PyImport_ImportModule("sys"));

  // sysobj = sys
  PyObject *sysobj = PyDict_GetItemString(main_dict, "sys");

  // sysobj.stdout = renderdoc_output_redirector()
  // sysobj.stderr = renderdoc_output_redirector()
  if(PyType_Ready(&OutputRedirectorType) >= 0)
  {
    PyObject *redirector = PyObject_CallFunction((PyObject *)&OutputRedirectorType, "");
    PyObject_SetAttrString(sysobj, "stdout", redirector);

    OutputRedirector *output = (OutputRedirector *)redirector;
    output->isStdError = 0;
    output->context = NULL;

    redirector = PyObject_CallFunction((PyObject *)&OutputRedirectorType, "");
    PyObject_SetAttrString(sysobj, "stderr", redirector);

    output = (OutputRedirector *)redirector;
    output->isStdError = 1;
    output->context = NULL;
  }

// if we need to append to sys.path to locate PySide2, do that now
#if defined(PYSIDE2_SYS_PATH)
  {
    PyObject *syspath = PyObject_GetAttrString(sysobj, "path");

#ifndef STRINGIZE
#define STRINGIZE2(a) #a
#define STRINGIZE(a) STRINGIZE2(a)
#endif

    PyObject *str = PyUnicode_FromString(STRINGIZE(PYSIDE2_SYS_PATH));

    PyList_Append(syspath, str);

    Py_DecRef(str);
    Py_DecRef(syspath);
  }
#endif

// set up PySide
#if PYSIDE2_ENABLED
  {
    Shiboken::AutoDecRef core(Shiboken::Module::import("PySide2.QtCore"));
    if(!core.isNull())
      SbkPySide2_QtCoreTypes = Shiboken::Module::getTypes(core);
    else
      qCritical() << "Failed to load PySide2.QtCore";

    Shiboken::AutoDecRef gui(Shiboken::Module::import("PySide2.QtGui"));
    if(!gui.isNull())
      SbkPySide2_QtGuiTypes = Shiboken::Module::getTypes(gui);
    else
      qCritical() << "Failed to load PySide2.QtGui";

    Shiboken::AutoDecRef widgets(Shiboken::Module::import("PySide2.QtWidgets"));
    if(!widgets.isNull())
      SbkPySide2_QtWidgetsTypes = Shiboken::Module::getTypes(widgets);
    else
      qCritical() << "Failed to load PySide2.QtWidgets";
  }
#endif

  // release GIL so that python work can now happen on any thread
  PyEval_SaveThread();
}

bool PythonContext::initialised()
{
  return main_dict != NULL;
}

PythonContext::PythonContext(QObject *parent) : QObject(parent)
{
  if(!initialised())
    return;

  // acquire the GIL and make sure this thread is init'd
  PyGILState_STATE gil = PyGILState_Ensure();

  // clone our own local context
  context_namespace = PyDict_Copy(main_dict);

  QString typeStr;
  QString valueStr = "";
  QList<QString> frames;

  // set global output that point to this context. It is responsible for deleting the context when
  // it goes out of scope
  PyObject *redirector = PyObject_CallFunction((PyObject *)&OutputRedirectorType, "");
  if(redirector)
  {
    PyDict_SetItemString(context_namespace, "_renderdoc_internal", redirector);

    OutputRedirector *output = (OutputRedirector *)redirector;
    output->context = this;
    Py_DECREF(redirector);
  }

  // release the GIL again
  PyGILState_Release(gil);
}

void PythonContext::Finish()
{
  PyGILState_STATE gil = PyGILState_Ensure();

  // release our external handle to globals. It'll now only be ref'd from inside
  Py_XDECREF(context_namespace);

  PyGILState_Release(gil);
}

void PythonContext::GlobalShutdown()
{
  // must happen on the UI thread
  if(qApp->thread() != QThread::currentThread())
  {
    qFatal("PythonContext::GlobalShutdown MUST be called from the UI thread");
    return;
  }

  // acquire the GIL, so we can shut down
  PyGILState_Ensure();

  Py_Finalize();
}

void PythonContext::executeString(const QString &filename, const QString &source)
{
  if(!initialised())
  {
    emit exception(
        "SystemError",
        "Python integration failed to initialise, see diagnostic log for more information.", {});
    return;
  }

  location.file = filename;
  location.line = 1;

  PyGILState_STATE gil = PyGILState_Ensure();

  PyObject *compiled =
      Py_CompileString(source.toUtf8().data(), filename.toUtf8().data(), Py_file_input);

  PyObject *ret = NULL;

  if(compiled)
  {
    PyObject *traceContext = PyDict_New();

    uintptr_t thisint = (uintptr_t) this;
    uint64_t thisuint64 = (uint64_t)thisint;
    PyObject *thisobj = PyLong_FromUnsignedLongLong(thisuint64);

    PyDict_SetItemString(traceContext, "thisobj", thisobj);
    PyDict_SetItemString(traceContext, "compiled", compiled);

    PyEval_SetTrace(&PythonContext::traceEvent, traceContext);

    m_State = PyGILState_GetThisThreadState();

    ret = PyEval_EvalCode(compiled, context_namespace, context_namespace);

    m_State = NULL;

    Py_XDECREF(thisobj);
    Py_XDECREF(traceContext);
  }

  Py_DecRef(compiled);

  QString typeStr;
  QString valueStr = "";
  QList<QString> frames;
  bool caughtException = (ret == NULL);

  if(caughtException)
    FetchException(typeStr, valueStr, frames);

  Py_XDECREF(ret);

  PyGILState_Release(gil);

  if(caughtException)
    emit exception(typeStr, valueStr, frames);
}

void PythonContext::executeString(const QString &source)
{
  executeString("<interactive.py>", source);
}

void PythonContext::executeFile(const QString &filename)
{
  QFile f(filename);

  if(!f.exists())
  {
    emit exception("FileNotFoundError", QString("No such file or directory: %1").arg(filename), {});
    return;
  }

  if(f.open(QIODevice::ReadOnly | QIODevice::Text))
  {
    QByteArray py = f.readAll();

    executeString(filename, QString::fromUtf8(py));
  }
  else
  {
    emit exception("IOError", QString("%1: %2").arg(f.errorString()).arg(filename), {});
  }
}

void PythonContext::setGlobal(const char *varName, const char *typeName, void *object)
{
  if(!initialised())
  {
    emit exception(
        "SystemError",
        "Python integration failed to initialise, see diagnostic log for more information.", {});
    return;
  }

  PyGILState_STATE gil = PyGILState_Ensure();

  PyObject *obj = PassObjectToPython(typeName, object);

  int ret = -1;

  if(obj)
    ret = PyDict_SetItemString(context_namespace, varName, obj);

  PyGILState_Release(gil);

  if(ret != 0)
  {
    emit exception("RuntimeError",
                   QString("Failed to set variable '%1' of type '%2'").arg(varName).arg(typeName),
                   {});
    return;
  }

  setPyGlobal(varName, obj);
}

template <>
void PythonContext::setGlobal(const char *varName, PyObject *object)
{
  setPyGlobal(varName, object);
}

template <>
void PythonContext::setGlobal(const char *varName, QObject *object)
{
  setQtGlobal(varName, object);
}

template <>
void PythonContext::setGlobal(const char *varName, QWidget *object)
{
  setQtGlobal(varName, object);
}

void PythonContext::setQtGlobal_internal(const char *varName, const char *typeName, QObject *object)
{
#if PYSIDE2_ENABLED
  if(!initialised())
  {
    emit exception(
        "SystemError",
        "Python integration failed to initialise, see diagnostic log for more information.", {});
    return;
  }

  if(!SbkPySide2_QtCoreTypes || !SbkPySide2_QtGuiTypes || !SbkPySide2_QtWidgetsTypes)
  {
    emit exception("SystemError",
                   "PySide Qt library didn't load, see diagnostic log for more information.", {});
    return;
  }

  PyObject *obj =
      Shiboken::Object::newObject(reinterpret_cast<SbkObjectType *>(Shiboken::SbkType<QObject>()),
                                  object, false, false, typeName);

  if(!obj)
  {
    emit exception("RuntimeError",
                   QString("Failed to set variable '%1' of type '%2'").arg(varName).arg(typeName),
                   {});
    return;
  }

  setPyGlobal(varName, obj);
#else
  emit exception("SystemError", "PySide2 was not enabled at build-time, cannot set Qt variable.", {});
#endif
}

void PythonContext::setPyGlobal(const char *varName, PyObject *obj)
{
  if(!initialised())
  {
    emit exception(
        "SystemError",
        "Python integration failed to initialise, see diagnostic log for more information.", {});
    return;
  }

  int ret = -1;

  PyGILState_STATE gil = PyGILState_Ensure();

  if(obj)
    ret = PyDict_SetItemString(context_namespace, varName, obj);

  PyGILState_Release(gil);

  if(ret == 0)
    return;

  emit exception("RuntimeError", QString("Failed to set variable '%1'").arg(varName), {});
}

void PythonContext::outstream_del(PyObject *self)
{
  OutputRedirector *redirector = (OutputRedirector *)self;

  if(redirector)
  {
    PythonContext *context = redirector->context;

    delete context;
  }
}

PyObject *PythonContext::outstream_write(PyObject *self, PyObject *args)
{
  const char *text = NULL;

  if(!PyArg_ParseTuple(args, "z:write", &text))
    return NULL;

  OutputRedirector *redirector = (OutputRedirector *)self;

  if(redirector)
  {
    PythonContext *context = redirector->context;
    // most likely this is NULL because the sys.stdout override is static and shared amongst
    // contexts. So look up the global variable that stores the context
    if(context == NULL)
    {
      PyObject *globals = PyEval_GetGlobals();
      if(globals)
      {
        OutputRedirector *global =
            (OutputRedirector *)PyDict_GetItemString(globals, "_renderdoc_internal");
        if(global)
          context = global->context;
      }
    }

    if(context)
    {
      emit context->textOutput(redirector->isStdError ? true : false, QString::fromUtf8(text));
    }
  }

  Py_RETURN_NONE;
}

PyObject *PythonContext::outstream_flush(PyObject *self, PyObject *args)
{
  Py_RETURN_NONE;
}

int PythonContext::traceEvent(PyObject *obj, PyFrameObject *frame, int what, PyObject *arg)
{
  PyObject *compiled = PyDict_GetItemString(obj, "compiled");
  if(compiled == (PyObject *)frame->f_code && what == PyTrace_LINE)
  {
    PyObject *thisobj = PyDict_GetItemString(obj, "thisobj");

    uint64_t thisuint64 = PyLong_AsUnsignedLongLong(thisobj);
    uintptr_t thisint = (uintptr_t)thisuint64;
    PythonContext *context = (PythonContext *)thisint;

    emit context->traceLine(context->location.file, context->location.line);
  }

  return 0;
}

extern "C" PyThreadState *GetExecutingThreadState(PyObject *global_handle)
{
  OutputRedirector *redirector = (OutputRedirector *)global_handle;
  if(redirector->context)
    return redirector->context->GetExecutingThreadState();

  return NULL;
}

extern "C" void HandleException(PyObject *global_handle)
{
  QString typeStr;
  QString valueStr = "";
  QList<QString> frames;

  FetchException(typeStr, valueStr, frames);

  OutputRedirector *redirector = (OutputRedirector *)global_handle;
  if(redirector->context)
    emit redirector->context->exception(typeStr, valueStr, frames);
}