From 9424b8a3fe9566ec88ed09766c04ab77d8dbd8e4 Mon Sep 17 00:00:00 2001 From: baldurk Date: Wed, 27 Jan 2021 14:44:25 +0000 Subject: [PATCH] Add documentation on setting up python dev environment --- .../pycharm_helpers/module_redeclarator.patch | 451 ++++++++++++++++++ docs/python_api/dev_environment.rst | 49 ++ docs/python_api/index.rst | 1 + 3 files changed, 501 insertions(+) create mode 100644 docs/pycharm_helpers/module_redeclarator.patch create mode 100644 docs/python_api/dev_environment.rst diff --git a/docs/pycharm_helpers/module_redeclarator.patch b/docs/pycharm_helpers/module_redeclarator.patch new file mode 100644 index 000000000..62882f022 --- /dev/null +++ b/docs/pycharm_helpers/module_redeclarator.patch @@ -0,0 +1,451 @@ +--- "/c/Program Files/JetBrains/PyCharm Community Edition 2020.3.2/plugins/python-ce/helpers/generator3/module_redeclarator.py" 2020-12-30 17:08:24.000000000 +0000 ++++ pycharm_helpers/plugins/python-ce/helpers/generator3/module_redeclarator.py 2021-01-27 14:13:18.876000000 +0000 +@@ -2,6 +2,15 @@ + from generator3.util_methods import * + from generator3.util_methods import get_relative_path_by_qname + from generator3.docstring_parsing import * ++import re ++import enum ++ ++# Add patterns for identifying declarations we care about ++RTYPE_PATTERN = re.compile(r"[:@]rtype:\s*(.*)") ++TYPE_PATTERN = re.compile(r"[:@]type\s*:\s*(.*)") ++PARAM_PATTERN = re.compile(r"[:@]param\s+([^:]*)\s+([^: ]*):") ++NAMESPACE_PATTERN = re.compile(r"([^[\]]+\.)*([a-zA-Z][a-zA-Z0-9_]+)") ++DEFAULTS = re.compile(r"(=.*)") + + + class emptylistdict(dict): +@@ -79,6 +88,8 @@ + # we write things into buffers out-of-order + self.header_buf = Buf(self) + self.imports_buf = Buf(self) ++ # each class gets its own list of dependency imports, and so do functions ++ self.func_imports_buf = Buf(self) + self.functions_buf = Buf(self) + self.classes_buf = Buf(self) + self.classes_buffs = list() +@@ -94,6 +105,8 @@ + self.doing_builtins = doing_builtins + self.ret_type_cache = {} + self.used_imports = emptylistdict() # qual_mod_name -> [imported_names,..]: actually used imported names ++ # if we use Typing hints, import some things for it ++ self.used_typing = False + + def _initializeQApp4(self): + try: # QtGui should be imported _before_ QtCore package. +@@ -129,19 +142,23 @@ + if self.split_modules: + last_pkg_dir = build_pkg_structure(self.cache_dir, self.qname) + with fopen(os.path.join(last_pkg_dir, "__init__.py"), "w") as init: +- for buf in (self.header_buf, self.imports_buf, self.functions_buf, self.classes_buf): ++ for buf in (self.header_buf, self.imports_buf, self.classes_buf): + buf.flush(init) + + data = "" +- for buf in self.classes_buffs: ++ for (buf,imports) in self.classes_buffs: + with fopen(os.path.join(last_pkg_dir, buf.name) + '.py', "w") as dummy: + self.header_buf.flush(dummy) + self.imports_buf.flush(dummy) ++ imports.flush(dummy) + buf.flush(dummy) + data += self.create_local_import(buf.name) + + init.write(data) +- self.footer_buf.flush(init) ++ ++ # Write out functions last so they can reference the classes imported above ++ for buf in (self.func_imports_buf, self.functions_buf, self.footer_buf): ++ buf.flush(init) + else: + last_pkg_dir = build_pkg_structure(self.cache_dir, '.'.join(qname_parts[:-1])) + # In some rare cases submodules of a binary might have been generated earlier than the module +@@ -155,13 +172,16 @@ + else: + skeleton_path = os.path.join(last_pkg_dir, qname_parts[-1] + '.py') + with fopen(skeleton_path, "w") as mod: +- for buf in (self.header_buf, self.imports_buf, self.functions_buf, self.classes_buf): ++ for buf in (self.header_buf, self.imports_buf, self.classes_buf): + buf.flush(mod) + +- for buf in self.classes_buffs: ++ for (buf,imports) in self.classes_buffs: ++ imports.flush(mod) + buf.flush(mod) + +- self.footer_buf.flush(mod) ++ # Write out functions last so they can reference the classes imported above ++ for buf in (self.func_imports_buf, self.functions_buf, self.footer_buf): ++ buf.flush(mod) + + # Some builtin classes effectively change __init__ signature without overriding it. + # This callable serves as a placeholder to be replaced via REDEFINED_BUILTIN_SIGS +@@ -177,6 +197,50 @@ + data += "." + data += name + " import " + name + "\n" + return data ++ ++ # Parse a typing declaration and add the actual types references ++ def add_import_types(self, import_types, type_decl): ++ if '[' not in type_decl: ++ import_types.add(type_decl) ++ return ++ ++ match = re.match('^[tT]uple\[(.*)\]$', type_decl) ++ if match: ++ for i in match.group(1).split(','): ++ self.add_import_types(import_types, i.strip()) ++ return ++ ++ match = re.match('^[lL]ist\[(.*)\]$', type_decl) ++ if match: ++ self.add_import_types(import_types, match.group(1).strip()) ++ return ++ ++ match = re.match('^[cC]allable\[\[(.*)\]\s*,\s*(.*)\]$', type_decl) ++ if match: ++ for i in match.group(1).split(','): ++ self.add_import_types(import_types, i.strip()) ++ self.add_import_types(import_types, match.group(2).strip()) ++ return ++ ++ # For a given type that's referenced, figure out where to import it from and add ++ # to the list ++ def process_import_type(self, used_imports, p_modname, classname, import_type): ++ parent = None ++ ++ if import_type == '...': ++ return ++ ++ if import_type in dir(sys.modules[p_modname]): ++ if import_type != classname: ++ parent = '.' ++ child = import_type ++ elif '.' in import_type: ++ imp_split = import_type.split('.') ++ parent = '.'.join(imp_split) ++ child = imp_split[-1] ++ ++ if parent is not None and child not in used_imports[parent]: ++ used_imports[parent].append(child) + + def find_imported_name(self, item): + """ +@@ -433,7 +497,43 @@ + if first_param: + seq.insert(0, first_param) + seq = make_names_unique(seq) +- return (seq, ret_type, doc_node) ++ ++ import_types = set() ++ ret_hint = None ++ ++ # Try to use :rtype: to add explicit type annotations to return type, since PyCharm ++ # doesn't parse rtype properly (at least with split modules) ++ if ret_type is None and ':rtype:' in signature_string: ++ result = RTYPE_PATTERN.search(signature_string) ++ if result is not None: ++ type_decl = result.group(1).strip() ++ if type_decl != class_name: ++ self.add_import_types(import_types, type_decl) ++ ret_hint = NAMESPACE_PATTERN.sub(r'\2', type_decl) ++ else: ++ ret_hint = "'" + type_decl + "'" ++ self.used_typing = True ++ ++ # Also use :param: to add necessary imports and fix up the parameters. ++ # PyCharm supports parsing :param: but the skeletons need the right imports ++ # and it fails for namespaced parameters (since the 'raw' parameter type ++ # foo.bar refers to a generated submodule called bar with the skeleton bar ++ # inside. ++ for p in PARAM_PATTERN.findall(signature_string): ++ type_decl = p[0] ++ if type_decl != class_name: ++ self.add_import_types(import_types, type_decl) ++ try: ++ defaults_match = [DEFAULTS.search(s) for s in seq] ++ idx = [DEFAULTS.sub('', s) for s in seq].index(p[1]) ++ seq[idx] = '{}: {}'.format(p[1], NAMESPACE_PATTERN.sub(r'\2', p[0])) ++ if defaults_match[idx]: ++ seq[idx] += defaults_match[idx].group(1) ++ except ValueError: ++ note("Warning: Unrecognised parameter {} in parameter list {}".format(p, seq)) ++ pass ++ ++ return (seq, ret_type, doc_node, list(import_types), ret_hint) + + def parse_func_doc(self, func_doc, func_id, func_name, class_name, deco=None, sip_generated=False): + """ +@@ -453,18 +553,23 @@ + overloads.append(part[i + len(signature):]) + if len(overloads) > 1: + docstring_results = [self.restore_by_docstring(overload, class_name, deco) for overload in overloads] ++ import_types = [] + ret_types = [] + for result in docstring_results: + rt = result[1] + if rt and rt not in ret_types: + ret_types.append(rt) ++ imps = result[3] ++ for imp in imps: ++ if imp and imp not in import_types: ++ import_types.append(imp) + if ret_types: + ret_literal = " or ".join(ret_types) + else: + ret_literal = None + param_lists = [result[0] for result in docstring_results] + spec = build_signature(func_name, restore_parameters_for_overloads(param_lists)) +- return (spec, ret_literal, "restored from __doc__ with multiple overloads") ++ return (spec, ret_literal, "restored from __doc__ with multiple overloads", import_types) + + # find the first thing to look like a definition + prefix_re = re.compile(r"\s*(?:(\w+)[ \t]+)?" + func_id + r"\s*\(") # "foo(..." or "int foo(..." +@@ -472,18 +577,21 @@ + # parse the part that looks right + if match: + ret_hint = match.group(1) +- params, ret_literal, doc_note = self.restore_by_docstring(func_doc[match.end():], class_name, deco, ret_hint) ++ params, ret, doc_note, import_types, ret_hint = self.restore_by_docstring(func_doc[match.end():], class_name, deco, ret_hint) + spec = func_name + flatten(params) +- return (spec, ret_literal, doc_note) ++ # if we got a type hint, put it on the function declaration ++ if ret_hint: ++ spec = spec + ' -> ' + ret_hint ++ return (spec, ret, doc_note, import_types) + else: +- return (None, None, None) ++ return (None, None, None, []) + + + def is_predefined_builtin(self, module_name, class_name, func_name): + return self.doing_builtins and module_name == BUILTIN_MOD_NAME and ( + class_name, func_name) in PREDEFINED_BUILTIN_SIGS + +- def redo_function(self, out, p_func, p_name, indent, p_class=None, p_modname=None, classname=None, seen=None): ++ def redo_function(self, out, p_func, p_name, indent, p_class=None, p_modname=None, classname=None, seen=None, used_imports=None): + """ + Restore function argument list as best we can. + @param out output function of a Buf +@@ -531,6 +639,8 @@ + deco = "classmethod" + elif type(p_func).__name__.startswith('staticmethod'): + deco = "staticmethod" ++ elif str(descriptor).startswith('= 30 + for item_name in [cls_item[0] for cls_item in cls_list]: + buf = ClassBuf(item_name, self) +- self.classes_buffs.append(buf) ++ imports = ClassBuf(item_name + '_imports', self) ++ self.classes_buffs.append((buf,imports)) + out = buf.out + if item_name in omitted_names: + out(0, "# definition of ", item_name, " omitted") + continue + item = classes[item_name] +- self.redo_class(out, item, item_name, 0, p_modname=p_name, seen=seen_classes, inspect_dir=inspect_dir) ++ used_imports = emptylistdict() ++ self.redo_class(out, item, item_name, 0, p_modname=p_name, seen=seen_classes, inspect_dir=inspect_dir, used_imports=used_imports) ++ # if we don't have split modules we can't import dependencies, but we also ++ # have an ordering constraint - classes need to be declared after any classes ++ # they reference in type hints. This is broken either way though even with the ++ # return literals ++ if not self.split_modules: ++ func_used_imports['.'] = [] ++ self.output_import_froms(imports.out, used_imports) + self._defined[item_name] = True + out(0, "") # empty line after each item + +@@ -1083,19 +1255,38 @@ + for value in values_to_add: + self.footer_buf.out(0, value) + # imports: last, because previous parts could alter used_imports or hidden_imports +- self.output_import_froms() ++ ++ out = self.imports_buf.out ++ self.output_import_froms(out, self.used_imports) ++ ++ if self.hidden_imports: ++ self.add_import_header_if_needed() ++ for mod_name in sorted_no_case(self.hidden_imports.keys()): ++ out(0, 'import ', mod_name, ' as ', self.hidden_imports[mod_name]) ++ out(0, "") # empty line after group ++ ++ if self.used_typing: ++ self.add_import_header_if_needed() ++ out(0, 'from typing import List, Tuple, Callable, Any') ++ out(0, "") # empty line after group ++ + if self.imports_buf.isEmpty(): +- self.imports_buf.out(0, "# no imports") +- self.imports_buf.out(0, "") # empty line after imports ++ out(0, "# no imports") ++ out(0, "") # empty line after imports + +- def output_import_froms(self): ++ def output_import_froms(self, out, imports_list): + """Mention all imported names known within the module, wrapping as per PEP.""" +- out = self.imports_buf.out +- if self.used_imports: ++ if imports_list: + self.add_import_header_if_needed() +- for mod_name in sorted_no_case(self.used_imports.keys()): +- import_names = self.used_imports[mod_name] +- if import_names: ++ for mod_name in sorted_no_case(imports_list.keys()): ++ import_names = imports_list[mod_name] ++ if mod_name == '.': ++ # if this is a local import, we need to treat it specially to import ++ # the class inside the referenced module ++ for n in import_names: ++ out(0, "from .%s import %s" % (n, n)) # empty line after group ++ out(0, "") # empty line after group ++ elif import_names: + self._defined[mod_name] = True + right_pos = 0 # tracks width of list to fold it at right margin + import_heading = "from % s import (" % mod_name +@@ -1127,10 +1318,4 @@ + + out(0, "") # empty line after group + +- if self.hidden_imports: +- self.add_import_header_if_needed() +- for mod_name in sorted_no_case(self.hidden_imports.keys()): +- out(0, 'import ', mod_name, ' as ', self.hidden_imports[mod_name]) +- out(0, "") # empty line after group +- + diff --git a/docs/python_api/dev_environment.rst b/docs/python_api/dev_environment.rst new file mode 100644 index 000000000..5f20efae8 --- /dev/null +++ b/docs/python_api/dev_environment.rst @@ -0,0 +1,49 @@ +Python development environment +============================== + +This document outlines how to set up a development environment with suitable IDE type information for the RenderDoc modules. It's entirely optional, and you can write your python code in whichever way makes most sense for you. We will be configuring the PyCharm IDE to have autocomplete for all RenderDoc types. + +Building RenderDoc +------------------ + +The first step is to build RenderDoc from source to ensure you have a matching python module and RenderDoc build. Due to binary incompatibilities the python module can't be widely distributed since it is linked against a specific python minor version. + +Build instructions for your platform are available `on github `_. Take note of the python version that you build against, since this will need to be used later. + +.. note:: + On windows by default RenderDoc builds against python 3.6 which is what it's distributed with. + + This can be overridden by setting the environment variable ``RENDERDOC_PYTHON_PREFIX32`` and/or ``RENDERDOC_PYTHON_PREFIX64`` to point to a python installation. + + RenderDoc requires pythonXY.lib, include files such as include/Python.h, as well as a .zip of the standard library. If you installed python with an installer you have the first two, and can generate the third by zipping the contents of the Lib folder. If you downloaded the embeddable zip you will only have the library zip, you need to obtain the include files and library separately. + +Once you have compiled RenderDoc copy the python module into the same folder as the main renderdoc library. On windows this means copying out of the ``pymodules`` subdirectory, on linux this will likely be the case already. We do this to keep things simple, so the python module can load the library without needing to change ``PATH``. + +Installing PyCharm +------------------ + +Now install PyCharm, for this document we will install 2020.3.2. Any version is fine, though newer versions may require modification to work properly. + +Before you run PyCharm, we will replace one file in it to generate better type information for RenderDoc. In the RenderDoc repository there's a `pycharm_helpers folder `_. Copying the content of the plugins folder over the folder in your PyCharm installation will update the file that is customised. You can back it up beforehand at this path: ``plugins/python-ce/helpers/generator3/module_redeclarator.py``. + +If you're using a different version of PyCharm you can try to apply the patch also available in that folder. + +Configuring python module +------------------------- + +You can now launch PyCharm and open or create the python project where you'll be writing code. Now we'll configure the python interpreter. This must match the python version that you built against above - the same major and minor version, and the same bitness (32-bit to 32-bit or 64-bit to 64-bit). + +Go into :guilabel:`File` enter :guilabel:`Settings`. On the left you can go into the project and to :guilabel:`Python Interpreter`. + +In the first entry click the gear next to :guilabel:`Python Interpreter` and choose :guilabel:`Add` if the interpreter you want isn't available. You can now configure a System Interpreter with the correct version as above. + +Once you've chosen the correct interpreter we'll also tell it where to find the RenderDoc libraries since they won't be in the default python path. Go back to the gear wheel but this time select :guilabel:`Show All`. With your chosen interpreter selected click on the tree icon at the bottom labeled ``Show paths for the selected interpreter`` and add the directory where you have the ``renderdoc`` and ``qrenderdoc`` modules, as well as the ``renderdoc`` library. + +If everything went well, PyCharm should load that interpreter for the project and discover the renderdoc python modules. It will then generate stubs for them with correct typing information so you can benefit from proper autocomplete while writing python code. + +Troubleshooting +--------------- + +If you get an error about "No module named 'renderdoc'" then something has gone wrong with how the interpreter finds and loads the python module. Ensure you have the right path specified and that the interpreter is the correctly matching version for the python module you compiled. + +To regenerate the generated python stubs delete your ``python_stubs`` folder in the JetBrains local cache. On windows this is in ``%LOCALAPPDATA%/JetBrains``. diff --git a/docs/python_api/index.rst b/docs/python_api/index.rst index 913e4ab60..58ba4f7a6 100644 --- a/docs/python_api/index.rst +++ b/docs/python_api/index.rst @@ -27,6 +27,7 @@ Each example has a simple motivating goal and shows how to achieve it using the .. toctree:: examples/renderdoc_intro examples/qrenderdoc_intro + dev_environment examples/basics examples/renderdoc/index examples/qrenderdoc/index