From dd3e7ab121bf156140d4afa2e7ab357ae099692f Mon Sep 17 00:00:00 2001 From: baldurk Date: Sat, 5 Dec 2020 13:30:04 +0000 Subject: [PATCH] Add a modified and improved version of PyCharm's skeleton generator --- docs/pycharm_helpers/LICENSE.txt | 202 +++ docs/pycharm_helpers/README.md | 7 + .../helpers/generator3/module_redeclarator.py | 1318 +++++++++++++++++ 3 files changed, 1527 insertions(+) create mode 100644 docs/pycharm_helpers/LICENSE.txt create mode 100644 docs/pycharm_helpers/README.md create mode 100644 docs/pycharm_helpers/plugins/python-ce/helpers/generator3/module_redeclarator.py diff --git a/docs/pycharm_helpers/LICENSE.txt b/docs/pycharm_helpers/LICENSE.txt new file mode 100644 index 000000000..345fb7797 --- /dev/null +++ b/docs/pycharm_helpers/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2000-2018 JetBrains s.r.o. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/docs/pycharm_helpers/README.md b/docs/pycharm_helpers/README.md new file mode 100644 index 000000000..bb6709d9d --- /dev/null +++ b/docs/pycharm_helpers/README.md @@ -0,0 +1,7 @@ +# PyCharm helpers + +PyCharm does a fairly good job of code completion etc with RenderDoc's python modules, but when generating reflection for the modules it doesn't quite get everything - e.g. some parameter types and especially return types aren't properly parsed and don't get detected as the right types. + +The information is in the docstrings in standard format but the skeleton generator PyCharm uses has some issues. + +The file in this directory tree can be copied over the corresponding file in the PyCharm distribution to generate better skeletons. This is written against PyCharm Community Edition 2020.3, though most likely the patched file can be applied over other versions successfully. diff --git a/docs/pycharm_helpers/plugins/python-ce/helpers/generator3/module_redeclarator.py b/docs/pycharm_helpers/plugins/python-ce/helpers/generator3/module_redeclarator.py new file mode 100644 index 000000000..342c5c036 --- /dev/null +++ b/docs/pycharm_helpers/plugins/python-ce/helpers/generator3/module_redeclarator.py @@ -0,0 +1,1318 @@ +from generator3.core import OriginType +from generator3.util_methods import * +from generator3.util_methods import get_relative_path_by_qname +from generator3.docstring_parsing import * +import re +import enum + +# Add patterns for identifying declarations we care about +RTYPE_PATTERN = re.compile(r"[:@]rtype:\s*(.*)") +TYPE_PATTERN = re.compile(r"[:@]type\s*:\s*(.*)") +PARAM_PATTERN = re.compile(r"[:@]param\s+([^:]*)\s+([^: ]*):") +NAMESPACE_PATTERN = re.compile(r"([^[\]]+\.)*([a-zA-Z][a-zA-Z0-9_]+)") +DEFAULTS = re.compile(r"(=.*)") + + +class emptylistdict(dict): + """defaultdict not available before 2.5; simplest reimplementation using [] as default""" + + def __getitem__(self, item): + if item in self: + return dict.__getitem__(self, item) + else: + it = [] + self.__setitem__(item, it) + return it + + +class Buf(object): + """Buffers data in a list, can write to a file. Indentation is provided externally.""" + + def __init__(self, indenter): + self.data = [] + self.indenter = indenter + + def put(self, data): + if data: + self.data.append(ensureUnicode(data)) + + def out(self, indent, *what): + """Output the arguments, indenting as needed, and adding an eol""" + self.put(self.indenter.indent(indent)) + for item in what: + self.put(item) + self.put("\n") + + def flush_bytes(self, outfile): + for data in self.data: + outfile.write(data.encode(OUT_ENCODING, "replace")) + + def flush_str(self, outfile): + for data in self.data: + outfile.write(data) + + if version[0] < 3: + flush = flush_bytes + else: + flush = flush_str + + def isEmpty(self): + return len(self.data) == 0 + + +class ClassBuf(Buf): + def __init__(self, name, indenter): + super(ClassBuf, self).__init__(indenter) + self.name = name + + +#noinspection PyBroadException +class ModuleRedeclarator(object): + def __init__(self, module, mod_qname, mod_filename, cache_dir, indent_size=4, doing_builtins=False): + """ + @param module: module object + @param mod_qname: module qualified name + @param mod_filename: filename of binary module (the .dll or .so). Can be None for modules + that don't have corresponding binary files (e.g. builtins) + @param cache_dir: per-binary cache directory where the generated stub will be stored. + Normally, it's "/python_stubs/cache//". + @param indent_size: amount of space characters per indent + """ + import generator3.core + self.test_mode = generator3.core.is_test_mode() + self.gen_version = generator3.core.version() + self.module = module + self.qname = mod_qname + self.cache_dir = cache_dir + self.mod_filename = mod_filename + # we write things into buffers out-of-order + self.header_buf = Buf(self) + self.imports_buf = Buf(self) + # each class gets its own list of dependency imports, and so do functions + self.func_imports_buf = Buf(self) + self.functions_buf = Buf(self) + self.classes_buf = Buf(self) + self.classes_buffs = list() + self.footer_buf = Buf(self) + self.indent_size = indent_size + self._indent_step = " " * self.indent_size + self.split_modules = False + # + self.imported_modules = {"": the_builtins} # explicit module imports: {"name": module} + self.hidden_imports = {} # {'real_mod_name': 'alias'}; we alias names with "__" since we don't want them exported + # ^ used for things that we don't re-export but need to import, e.g. certain base classes in gnome. + self._defined = {} # stores True for every name defined so far, to break circular refs in values + self.doing_builtins = doing_builtins + self.ret_type_cache = {} + self.used_imports = emptylistdict() # qual_mod_name -> [imported_names,..]: actually used imported names + # if we use Typing hints, import some things for it + self.used_typing = False + + def _initializeQApp4(self): + try: # QtGui should be imported _before_ QtCore package. + # This is done for the QWidget references from QtCore (such as QSignalMapper). Known bug in PyQt 4.7+ + # Causes "TypeError: C++ type 'QWidget*' is not supported as a native Qt signal type" + import PyQt4.QtGui + except ImportError: + pass + + # manually instantiate and keep reference to singleton QCoreApplication (we don't want it to be deleted during the introspection) + # use QCoreApplication instead of QApplication to avoid blinking app in Dock on Mac OS + try: + from PyQt4.QtCore import QCoreApplication + self.app = QCoreApplication([]) + return + except ImportError: + pass + + def _initializeQApp5(self): + try: + from PyQt5.QtCore import QCoreApplication + self.app = QCoreApplication([]) + return + except ImportError: + pass + + def indent(self, level): + """Return indentation whitespace for given level.""" + return self._indent_step * level + + def flush(self): + qname_parts = self.qname.split('.') + if self.split_modules: + last_pkg_dir = build_pkg_structure(self.cache_dir, self.qname) + with fopen(os.path.join(last_pkg_dir, "__init__.py"), "w") as init: + for buf in (self.header_buf, self.imports_buf, self.classes_buf): + buf.flush(init) + + data = "" + for (buf,imports) in self.classes_buffs: + with fopen(os.path.join(last_pkg_dir, buf.name) + '.py', "w") as dummy: + self.header_buf.flush(dummy) + self.imports_buf.flush(dummy) + imports.flush(dummy) + buf.flush(dummy) + data += self.create_local_import(buf.name) + + init.write(data) + + # Write out functions last so they can reference the classes imported above + for buf in (self.func_imports_buf, self.functions_buf, self.footer_buf): + buf.flush(init) + else: + last_pkg_dir = build_pkg_structure(self.cache_dir, '.'.join(qname_parts[:-1])) + # In some rare cases submodules of a binary might have been generated earlier than the module + # for the binary itself. For instance, it happens for "pyexpat" built-in module which + # submodules "pyexpat.errors" and "pyexpat.model" are processed together with "_elementtree" + # and "pickle" before "pyexpat" and thus empty pyexpat/__init__.py for them should be replaced + # with the skeleton for the main module itself later on. + existing_pkg_init = os.path.join(last_pkg_dir, qname_parts[-1], '__init__.py') + if os.path.exists(existing_pkg_init): + skeleton_path = existing_pkg_init + else: + skeleton_path = os.path.join(last_pkg_dir, qname_parts[-1] + '.py') + with fopen(skeleton_path, "w") as mod: + for buf in (self.header_buf, self.imports_buf, self.classes_buf): + buf.flush(mod) + + for (buf,imports) in self.classes_buffs: + imports.flush(mod) + buf.flush(mod) + + # Write out functions last so they can reference the classes imported above + for buf in (self.func_imports_buf, self.functions_buf, self.footer_buf): + buf.flush(mod) + + # Some builtin classes effectively change __init__ signature without overriding it. + # This callable serves as a placeholder to be replaced via REDEFINED_BUILTIN_SIGS + def fake_builtin_init(self): + pass # just a callable, sig doesn't matter + + fake_builtin_init.__doc__ = object.__init__.__doc__ # this forces class's doc to be used instead + + def create_local_import(self, name): + if len(name.split(".")) > 1: return "" + data = "from " + if version[0] >= 3: + data += "." + data += name + " import " + name + "\n" + return data + + # Parse a typing declaration and add the actual types references + def add_import_types(self, import_types, type_decl): + if '[' not in type_decl: + import_types.add(type_decl) + return + + match = re.match('^[tT]uple\[(.*)\]$', type_decl) + if match: + for i in match.group(1).split(','): + self.add_import_types(import_types, i.strip()) + return + + match = re.match('^[lL]ist\[(.*)\]$', type_decl) + if match: + self.add_import_types(import_types, match.group(1).strip()) + return + + match = re.match('^[cC]allable\[\[(.*)\]\s*,\s*(.*)\]$', type_decl) + if match: + for i in match.group(1).split(','): + self.add_import_types(import_types, i.strip()) + self.add_import_types(import_types, match.group(2).strip()) + return + + # For a given type that's referenced, figure out where to import it from and add + # to the list + def process_import_type(self, used_imports, p_modname, classname, import_type): + parent = None + + if import_type in dir(sys.modules[p_modname]): + if import_type != classname: + parent = '.' + child = import_type + elif '.' in import_type: + imp_split = import_type.split('.') + parent = '.'.join(imp_split) + child = imp_split[-1] + + if parent is not None and child not in used_imports[parent]: + used_imports[parent].append(child) + + def find_imported_name(self, item): + """ + Finds out how the item is represented in imported modules. + @param item what to check + @return qualified name (like "sys.stdin") or None + """ + # TODO: return a pair, not a glued string + if not isinstance(item, SIMPLEST_TYPES): + for mname in self.imported_modules: + m = self.imported_modules[mname] + for inner_name in m.__dict__: + suspect = getattr(m, inner_name) + if suspect is item: + if mname: + mname += "." + elif self.module is the_builtins: # don't short-circuit builtins + return None + return mname + inner_name + return None + + _initializers = ( + (dict, "{}"), + (tuple, "()"), + (list, "[]"), + ) + + def invent_initializer(self, a_type): + """ + Returns an innocuous initializer expression for a_type, or "None" + """ + for initializer_type, r in self._initializers: + if initializer_type == a_type: + return r + # NOTE: here we could handle things like defaultdict, sets, etc if we wanted + return "None" + + def fmt_value(self, out, p_value, indent, prefix="", postfix="", as_name=None, seen_values=None): + """ + Formats and outputs value (it occupies an entire line or several lines). + @param out function that does output (a Buf.out) + @param p_value the value. + @param indent indent level. + @param prefix text to print before the value + @param postfix text to print after the value + @param as_name hints which name are we trying to print; helps with circular refs. + @param seen_values a list of keys we've seen if we're processing a dict + """ + SELF_VALUE = "" + ERR_VALUE = "" + if isinstance(p_value, SIMPLEST_TYPES): + out(indent, prefix, reliable_repr(p_value), postfix) + else: + if sys.platform == "cli": + imported_name = None + else: + imported_name = self.find_imported_name(p_value) + if imported_name: + out(indent, prefix, imported_name, postfix) + # TODO: kind of self.used_imports[imported_name].append(p_value) but split imported_name + # else we could potentially return smth we did not otherwise import. but not likely. + else: + if isinstance(p_value, (list, tuple)): + if not seen_values: + seen_values = [p_value] + if len(p_value) == 0: + out(indent, prefix, repr(p_value), postfix) + else: + if isinstance(p_value, list): + lpar, rpar = "[", "]" + else: + lpar, rpar = "(", ")" + out(indent, prefix, lpar) + for value in p_value: + if value in seen_values: + value = SELF_VALUE + elif not isinstance(value, SIMPLEST_TYPES): + seen_values.append(value) + self.fmt_value(out, value, indent + 1, postfix=",", seen_values=seen_values) + out(indent, rpar, postfix) + elif isinstance(p_value, dict): + if len(p_value) == 0: + out(indent, prefix, repr(p_value), postfix) + else: + if not seen_values: + seen_values = [p_value] + out(indent, prefix, "{") + keys = list(p_value.keys()) + try: + keys.sort() + except TypeError: + pass # unsortable keys happen, e,g, in py3k _ctypes + for k in keys: + value = p_value[k] + + try: + is_seen = value in seen_values + except: + is_seen = False + value = ERR_VALUE + + if is_seen: + value = SELF_VALUE + elif not isinstance(value, SIMPLEST_TYPES): + seen_values.append(value) + if isinstance(k, SIMPLEST_TYPES): + self.fmt_value(out, value, indent + 1, prefix=repr(k) + ": ", postfix=",", + seen_values=seen_values) + else: + # both key and value need fancy formatting + self.fmt_value(out, k, indent + 1, postfix=": ", seen_values=seen_values) + self.fmt_value(out, value, indent + 2, seen_values=seen_values) + out(indent + 1, ",") + out(indent, "}", postfix) + else: # something else, maybe representable + # look up this value in the module. + if sys.platform == "cli": + out(indent, prefix, "None", postfix) + return + found_name = "" + for inner_name in self.module.__dict__: + if self.module.__dict__[inner_name] is p_value: + found_name = inner_name + break + if self._defined.get(found_name, False): + out(indent, prefix, found_name, postfix) + elif hasattr(self, "app"): + return + else: + # a forward / circular declaration happens + notice = "" + try: + representation = repr(p_value) + except Exception: + import traceback + traceback.print_exc(file=sys.stderr) + return + if not self.test_mode: + real_value = cleanup(representation) + else: + # Don't rely on repr() output in tests, as it may contain memory layout dependent id + real_value = '' + if found_name: + if found_name == as_name: + notice = " # (!) real value is %r" % real_value + real_value = "None" + else: + notice = " # (!) forward: %s, real value is %r" % (found_name, real_value) + if SANE_REPR_RE.match(real_value) and is_valid_expr(real_value): + out(indent, prefix, real_value, postfix, notice) + else: + if not found_name: + notice = " # (!) real value is %r" % real_value + out(indent, prefix, "None", postfix, notice) + + def get_ret_type(self, attr): + """ + Returns a return type string as given by T_RETURN in tokens, or None + """ + if attr: + ret_type = RET_TYPE.get(attr, None) + if ret_type: + return ret_type + thing = getattr(self.module, attr, None) + if thing: + if not isinstance(thing, type) and is_callable(thing): # a function + return None # TODO: maybe divinate a return type; see pygame.mixer.Channel + return attr + # adds no noticeable slowdown, I did measure. dch. + for im_name, im_module in self.imported_modules.items(): + cache_key = (im_name, attr) + cached = self.ret_type_cache.get(cache_key, None) + if cached: + return cached + ret_type = getattr(im_module, attr, None) + if ret_type: + if isinstance(ret_type, type): + # detect a constructor + constr_args = detect_constructor(ret_type) + if constr_args is None: + constr_args = "*(), **{}" # a silly catch-all constructor + reference = "%s(%s)" % (attr, constr_args) + elif is_callable(ret_type): # a function, classes are ruled out above + return None + else: + reference = attr + if im_name: + result = "%s.%s" % (im_name, reference) + else: # built-in + result = reference + self.ret_type_cache[cache_key] = result + return result + # TODO: handle things like "[a, b,..] and (foo,..)" + return None + + + SIG_DOC_NOTE = "restored from __doc__" + SIG_DOC_UNRELIABLY = "NOTE: unreliably restored from __doc__ " + + def restore_by_docstring(self, signature_string, class_name, deco=None, ret_hint=None): + """ + @param signature_string: parameter list extracted from the doc string. + @param class_name: name of the containing class, or None + @param deco: decorator to use + @param ret_hint: return type hint, if available + @return (reconstructed_spec, return_type, note) or (None, _, _) if failed. + """ + action("restoring func %r of class %r", signature_string, class_name) + # parse + parsing_failed = False + ret_type = None + try: + # strict parsing + tokens = paramSeqAndRest.parseString(signature_string, True) + ret_name = None + if tokens: + ret_t = tokens[-1] + if ret_t[0] is T_RETURN: + ret_name = ret_t[1] + ret_type = self.get_ret_type(ret_name) or self.get_ret_type(ret_hint) + except ParseException: + # it did not parse completely; scavenge what we can + parsing_failed = True + tokens = [] + try: + # most unrestrictive parsing + tokens = paramSeq.parseString(signature_string, False) + except ParseException: + pass + # + seq = transform_seq(tokens) + + # add safe defaults for unparsed + if parsing_failed: + doc_node = self.SIG_DOC_UNRELIABLY + starred = None + double_starred = None + for one in seq: + if type(one) is str: + if one.startswith("**"): + double_starred = one + elif one.startswith("*"): + starred = one + if not starred: + seq.append("*args") + if not double_starred: + seq.append("**kwargs") + else: + doc_node = self.SIG_DOC_NOTE + + # add 'self' if needed YYY + if class_name and (not seq or seq[0] != 'self'): + first_param = propose_first_param(deco) + if first_param: + seq.insert(0, first_param) + seq = make_names_unique(seq) + + import_types = set() + ret_hint = None + + # Try to use :rtype: to add explicit type annotations to return type, since PyCharm + # doesn't parse rtype properly (at least with split modules) + if ret_type is None and ':rtype:' in signature_string: + result = RTYPE_PATTERN.search(signature_string) + if result is not None: + type_decl = result.group(1).strip() + if type_decl != class_name: + self.add_import_types(import_types, type_decl) + ret_hint = NAMESPACE_PATTERN.sub(r'\2', type_decl) + else: + ret_hint = "'" + type_decl + "'" + self.used_typing = True + + # Also use :param: to add necessary imports and fix up the parameters. + # PyCharm supports parsing :param: but the skeletons need the right imports + # and it fails for namespaced parameters (since the 'raw' parameter type + # foo.bar refers to a generated submodule called bar with the skeleton bar + # inside. + for p in PARAM_PATTERN.findall(signature_string): + type_decl = p[0] + if type_decl != class_name: + self.add_import_types(import_types, type_decl) + try: + defaults_match = [DEFAULTS.search(s) for s in seq] + idx = [DEFAULTS.sub('', s) for s in seq].index(p[1]) + seq[idx] = '{}: {}'.format(p[1], NAMESPACE_PATTERN.sub(r'\2', p[0])) + if defaults_match[idx]: + seq[idx] += defaults_match[idx].group(1) + except ValueError: + note("Warning: Unrecognised parameter {} in parameter list {}".format(p, seq)) + pass + + return (seq, ret_type, doc_node, list(import_types), ret_hint) + + def parse_func_doc(self, func_doc, func_id, func_name, class_name, deco=None, sip_generated=False): + """ + @param func_doc: __doc__ of the function. + @param func_id: name to look for as identifier of the function in docstring + @param func_name: name of the function. + @param class_name: name of the containing class, or None + @param deco: decorator to use + @return (reconstructed_spec, return_literal, note) or (None, _, _) if failed. + """ + if sip_generated: + overloads = [] + for part in func_doc.split('\n'): + signature = func_id + '(' + i = part.find(signature) + if i >= 0: + overloads.append(part[i + len(signature):]) + if len(overloads) > 1: + docstring_results = [self.restore_by_docstring(overload, class_name, deco) for overload in overloads] + import_types = [] + ret_types = [] + for result in docstring_results: + rt = result[1] + if rt and rt not in ret_types: + ret_types.append(rt) + imps = result[3] + for imp in imps: + if imp and imp not in import_types: + import_types.append(imp) + if ret_types: + ret_literal = " or ".join(ret_types) + else: + ret_literal = None + param_lists = [result[0] for result in docstring_results] + spec = build_signature(func_name, restore_parameters_for_overloads(param_lists)) + return (spec, ret_literal, "restored from __doc__ with multiple overloads", import_types) + + # find the first thing to look like a definition + prefix_re = re.compile(r"\s*(?:(\w+)[ \t]+)?" + func_id + r"\s*\(") # "foo(..." or "int foo(..." + match = prefix_re.search(func_doc) # Note: this and previous line may consume up to 35% of time + # parse the part that looks right + if match: + ret_hint = match.group(1) + params, ret, doc_note, import_types, ret_hint = self.restore_by_docstring(func_doc[match.end():], class_name, deco, ret_hint) + spec = func_name + flatten(params) + # if we got a type hint, put it on the function declaration + if ret_hint: + spec = spec + ' -> ' + ret_hint + return (spec, ret, doc_note, import_types) + else: + return (None, None, None, []) + + + def is_predefined_builtin(self, module_name, class_name, func_name): + return self.doing_builtins and module_name == BUILTIN_MOD_NAME and ( + class_name, func_name) in PREDEFINED_BUILTIN_SIGS + + def redo_function(self, out, p_func, p_name, indent, p_class=None, p_modname=None, classname=None, seen=None, used_imports=None): + """ + Restore function argument list as best we can. + @param out output function of a Buf + @param p_func function or method object + @param p_name function name as known to owner + @param indent indentation level + @param p_class the class that contains this function as a method + @param p_modname module name + @param seen {id(func): name} map of functions already seen in the same namespace; + id() because *some* functions are unhashable (eg _elementtree.Comment in py2.7) + """ + action("redoing func %r of class %r", p_name, p_class) + if seen is not None: + other_func = seen.get(id(p_func), None) + if other_func and getattr(other_func, "__doc__", None) is getattr(p_func, "__doc__", None): + # _bisect.bisect == _bisect.bisect_right in py31, but docs differ + out(indent, p_name, " = ", seen[id(p_func)]) + out(indent, "") + return + else: + seen[id(p_func)] = p_name + # real work + if classname is None: + classname = p_class and p_class.__name__ or None + if p_class and hasattr(p_class, '__mro__'): + sip_generated = [base_t for base_t in p_class.__mro__ if 'sip.simplewrapper' in str(base_t)] + else: + sip_generated = False + deco = None + deco_comment = "" + mod_class_method_tuple = (p_modname, classname, p_name) + ret_literal = None + is_init = False + # any decorators? + action("redoing decos of func %r of class %r", p_name, p_class) + if self.doing_builtins and p_modname == BUILTIN_MOD_NAME: + deco = KNOWN_DECORATORS.get((classname, p_name), None) + if deco: + deco_comment = " # known case" + elif p_class and p_name in p_class.__dict__: + # detect native methods declared with METH_CLASS flag + descriptor = p_class.__dict__[p_name] + if p_name != "__new__" and type(descriptor).__name__.startswith('classmethod'): + # 'classmethod_descriptor' in Python 2.x and 3.x, 'classmethod' in Jython + deco = "classmethod" + elif type(p_func).__name__.startswith('staticmethod'): + deco = "staticmethod" + elif str(descriptor).startswith(' __foo_bar + bases_list.append(mangled_qualifier + "." + base_name) + self.hidden_imports[qual_module_name] = mangled_qualifier + else: + bases_list.append(base_name) + base_def = "(" + ", ".join(bases_list) + ")" + + if self.split_modules: + for base in bases_list: + local_import = self.create_local_import(base) + if local_import: + out(indent, local_import) + out(indent, "class ", p_name, base_def, ":", + skipped_bases and " # skipped bases: " + ", ".join(skipped_bases) or "") + out_doc_attr(out, p_class, indent + 1) + # inner parts + methods = {} + properties = {} + others = {} + we_are_the_base_class = p_modname == BUILTIN_MOD_NAME and p_name == "object" + field_source = {} + try: + if hasattr(p_class, "__dict__") and not inspect_dir: + field_source = p_class.__dict__ + field_keys = field_source.keys() # Jython 2.5.1 _codecs fail here + else: + field_keys = dir(p_class) # this includes unwanted inherited methods, but no dict + inheritance is rare + except: + field_keys = () + for item_name in field_keys: + item_qname = p_modname + '.' + p_name + '.' + item_name + if item_name in ("__doc__", "__module__"): + if we_are_the_base_class: + item = "" # must be declared in base types + else: + continue # in all other cases must be skipped + elif keyword.iskeyword(item_name): # for example, PyQt4 contains definitions of methods named 'exec' + continue + elif item_qname in CLASS_ATTR_BLACKLIST: + note('skipping blacklisted attribute ' + item_qname) + item = field_source.get(item_name) + else: + try: + item = getattr(p_class, item_name) # let getters do the magic + except AttributeError: + item = field_source.get(item_name) # have it raw + if item is None: + continue + except Exception: + continue + + # Don't generate skeleton for internal enum properties + if isinstance(p_class, enum.EnumMeta) and (is_callable(item) or item_name[0] == '_'): + continue + + if is_callable(item) and not isinstance(item, type): + methods[item_name] = item + elif is_property(item): + properties[item_name] = item + else: + others[item_name] = item + # + if we_are_the_base_class: + others["__dict__"] = {} # force-feed it, for __dict__ does not contain a reference to itself :) + # add fake __init__s to have the right sig + if p_class in FAKE_BUILTIN_INITS: + methods["__init__"] = self.fake_builtin_init + note("Faking init of %s", p_name) + elif '__init__' not in methods: + init_method = getattr(p_class, '__init__', None) + if init_method: + methods['__init__'] = init_method + + # + seen_funcs = {} + for item_name in sorted_no_case(methods.keys()): + item = methods[item_name] + try: + self.redo_function(out, item, item_name, indent + 1, p_class, p_modname, classname=p_name, seen=seen_funcs, used_imports=used_imports) + except: + handle_error_func(item_name, out) + # + known_props = KNOWN_PROPS.get(p_modname, {}) + a_setter = "lambda self, v: None" + a_deleter = "lambda self: None" + for item_name in sorted_no_case(properties.keys()): + item = properties[item_name] + prop_docstring = getattr(item, '__doc__', None) + prop_key = (p_name, item_name) + if prop_key in known_props: + prop_descr = known_props.get(prop_key, None) + if prop_descr is None: + continue # explicitly omitted + acc_line, getter_and_type = prop_descr + if getter_and_type: + getter, prop_type = getter_and_type + else: + getter, prop_type = None, None + out(indent + 1, item_name, + " = property(", format_accessors(acc_line, getter, a_setter, a_deleter), ")" + ) + if prop_type: + if prop_docstring: + out(indent + 1, '"""', prop_docstring) + out(0, "") + out(indent + 1, ':type: ', prop_type) + out(indent + 1, '"""') + else: + out(indent + 1, '""":type: ', prop_type, '"""') + out(0, "") + else: + # for properties with docstrings put them inside the getter so that PyCharm + # displays them + if prop_docstring: + ret = '' + param = '' + + # Additionally if we see :type: in the docstring, add type hints + result = TYPE_PATTERN.search(prop_docstring) + if result is not None: + type_decl = result.group(1).strip() + import_types = set() + if type_decl == p_name or type_decl == 'List[{}]'.format(p_name): + type_decl = "'" + type_decl + "'" + else: + self.add_import_types(import_types, type_decl) + for imp in import_types: + self.process_import_type(used_imports, p_modname, p_name, imp) + type_decl = type_decl.replace('...', '__ellipses__').split('.')[-1].replace('__ellipses__', '...') + ret = ' -> {}'.format(type_decl) + param = ': {}'.format(type_decl) + + out(indent + 1, "@property") + out(indent + 1, "def {}(self){}:".format(item_name, ret)) + out(indent + 2, '"""', prop_docstring, '"""') + out(indent + 2, 'pass') + out(0, "") + out(indent + 1, "@{}.setter".format(item_name)) + out(indent + 1, "def {}(self, value{}):".format(item_name, param)) + out(indent + 2, 'pass') + out(0, "") + + self.used_typing = True + continue + + out(indent + 1, item_name, " = property(lambda self: object(), lambda self, v: None, lambda self: None) # default") + out(0, "") + if properties: + out(0, "") # empty line after the block + # + for item_name in sorted_no_case(others.keys()): + item = others[item_name] + self.fmt_value(out, item, indent + 1, prefix=item_name + " = ") + if p_name == "object": + out(indent + 1, "__module__ = ''") + if others: + out(0, "") # empty line after the block + # + if not methods and not properties and not others: + out(indent + 1, "pass") + + def redo_simple_header(self, p_name): + """Puts boilerplate code on the top""" + out = self.header_buf.out # 1st class methods rule :) + out(0, "# encoding: %s" % OUT_ENCODING) # line 1 + # NOTE: maybe encoding should be selectable + if hasattr(self.module, "__name__"): + self_name = self.module.__name__ + if self_name != p_name: + mod_name = " calls itself " + self_name + else: + mod_name = "" + else: + mod_name = " does not know its name" + out(0, "# module ", p_name, mod_name) # line 2 + + origin_type = OriginType.FILE + import generator3.core + if generator3.core.is_pregeneration_mode(): + origin = origin_type = OriginType.PREGENERATED + elif self.mod_filename: + origin = self.mod_filename + elif p_name in sys.builtin_module_names: + origin = origin_type = OriginType.BUILTIN + else: + try: + origin = getattr(self.module, "__file__") + except AttributeError: + origin = origin_type = OriginType.BUILTIN + + if self.test_mode and origin_type == OriginType.FILE: + origin = get_relative_path_by_qname(origin, self.qname) + + + out(0, "# from %s" % origin) # line 3 + out(0, "# by generator %s" % self.gen_version) # line 4 + if p_name == BUILTIN_MOD_NAME and version[0] == 2 and version[1] >= 6: + out(0, "from __future__ import print_function") + out_doc_attr(out, self.module, 0) + + def redo_imports(self): + module_type = type(sys) + for item_name in self.module.__dict__.keys(): + try: + item = self.module.__dict__[item_name] + except: + continue + if type(item) is module_type: # not isinstance, py2.7 + PyQt4.QtCore on windows have a bug here + self.imported_modules[item_name] = item + self.add_import_header_if_needed() + ref_notice = getattr(item, "__file__", str(item)) if not self.test_mode else '' + if hasattr(item, "__name__"): + self.imports_buf.out(0, "import ", item.__name__, " as ", item_name, " # ", ref_notice) + else: + self.imports_buf.out(0, item_name, " = None # ??? name unknown; ", ref_notice) + + def add_import_header_if_needed(self): + if self.imports_buf.isEmpty(): + self.imports_buf.out(0, "") + self.imports_buf.out(0, "# imports") + + def redo(self, p_name, inspect_dir): + """ + Restores module declarations. + Intended for built-in modules and thus does not handle import statements. + @param p_name name of module + """ + action("redoing header of module %r %r", p_name, str(self.module)) + + if "pyqt4" in p_name.lower(): # qt4 specific patch + self._initializeQApp4() + elif "pyqt5" in p_name.lower(): # qt5 specific patch + self._initializeQApp5() + + self.redo_simple_header(p_name) + + # find whatever other self.imported_modules the module knows; effectively these are imports + action("redoing imports of module %r %r", p_name, str(self.module)) + try: + self.redo_imports() + except: + pass + + action("redoing innards of module %r %r", p_name, str(self.module)) + + module_type = type(sys) + # group what we have into buckets + vars_simple = {} + vars_complex = {} + funcs = {} + classes = {} + module_dict = self.module.__dict__ + if inspect_dir: + module_dict = dir(self.module) + for item_name in module_dict: + note("looking at %s", item_name) + if item_name in ( + "__dict__", "__doc__", "__module__", "__file__", "__name__", "__builtins__", "__package__"): + continue # handled otherwise + if self.test_mode and item_name in ('__loader__', '__spec__', '__cached__'): + continue + try: + item = getattr(self.module, item_name) # let getters do the magic + except AttributeError: + if not item_name in self.module.__dict__: continue + item = self.module.__dict__[item_name] # have it raw + # check if it has percolated from an imported module + except NotImplementedError: + if not item_name in self.module.__dict__: continue + item = self.module.__dict__[item_name] # have it raw + + # unless we're adamantly positive that the name was imported, we assume it is defined here + mod_name = None # module from which p_name might have been imported + # IronPython has non-trivial reexports in System module, but not in others: + skip_modname = sys.platform == "cli" and p_name != "System" + surely_not_imported_mods = KNOWN_FAKE_REEXPORTERS.get(p_name, ()) + ## can't figure weirdness in some modules, assume no reexports: + #skip_modname = skip_modname or p_name in self.KNOWN_FAKE_REEXPORTERS + if not skip_modname: + try: + mod_name = getattr(item, '__module__', None) + except: + pass + # we assume that module foo.bar never imports foo; foo may import foo.bar. (see pygame and pygame.rect) + maybe_import_mod_name = mod_name if isinstance(mod_name, type(p_name)) else '' + import_is_from_top = len(p_name) > len(maybe_import_mod_name) and p_name.startswith(maybe_import_mod_name) + note("mod_name = %s, prospective = %s, from top = %s", mod_name, maybe_import_mod_name, import_is_from_top) + want_to_import = False + if (mod_name + and mod_name != BUILTIN_MOD_NAME + and mod_name != p_name + and mod_name not in surely_not_imported_mods + and not import_is_from_top + ): + # import looks valid, but maybe it's a .py file? we're certain not to import from .py + # e.g. this rules out _collections import collections and builtins import site. + try: + imported = __import__(mod_name) # ok to repeat, Python caches for us + if imported: + qualifiers = mod_name.split(".")[1:] + for qual in qualifiers: + imported = getattr(imported, qual, None) + if not imported: + break + imported_path = (getattr(imported, '__file__', False) or "").lower() + want_to_import = not (imported_path.endswith('.py') or imported_path.endswith('.pyc')) + imported_name = getattr(imported, "__name__", None) + if imported_name == p_name: + want_to_import = False + note("path of %r is %r, want? %s", mod_name, imported_path, want_to_import) + except ImportError: + want_to_import = False + # NOTE: if we fail to import, we define 'imported' names here lest we lose them at all + if want_to_import: + import_list = self.used_imports[mod_name] + if item_name not in import_list: + import_list.append(item_name) + if not want_to_import: + if isinstance(item, type) or type(item).__name__ == 'classobj': + classes[item_name] = item + elif is_callable(item): # some classes are callable, check them before functions + funcs[item_name] = item + elif isinstance(item, module_type): + continue # self.imported_modules handled above already + else: + if isinstance(item, SIMPLEST_TYPES): + vars_simple[item_name] = item + else: + vars_complex[item_name] = item + + # sort and output every bucket + action("outputting innards of module %r %r", p_name, str(self.module)) + # + omitted_names = OMIT_NAME_IN_MODULE.get(p_name, []) + if vars_simple: + out = self.functions_buf.out + prefix = "" # try to group variables by common prefix + PREFIX_LEN = 2 # default prefix length if we can't guess better + out(0, "# Variables with simple values") + for item_name in sorted_no_case(vars_simple.keys()): + if item_name in omitted_names: + out(0, "# definition of " + item_name + " omitted") + continue + item = vars_simple[item_name] + # track the prefix + if len(item_name) >= PREFIX_LEN: + prefix_pos = string.rfind(item_name, "_") # most prefixes end in an underscore + if prefix_pos < 1: + prefix_pos = PREFIX_LEN + beg = item_name[0:prefix_pos] + if prefix != beg: + out(0, "") # space out from other prefix + prefix = beg + else: + prefix = "" + # output + replacement = REPLACE_MODULE_VALUES.get((p_name, item_name), None) + if replacement is not None: + out(0, item_name, " = ", replacement, " # real value of type ", str(type(item)), " replaced") + elif is_skipped_in_module(p_name, item_name): + t_item = type(item) + out(0, item_name, " = ", self.invent_initializer(t_item), " # real value of type ", str(t_item), + " skipped") + else: + self.fmt_value(out, item, 0, prefix=item_name + " = ") + self._defined[item_name] = True + out(0, "") # empty line after vars + # + if funcs: + out = self.functions_buf.out + out(0, "# functions") + out(0, "") + seen_funcs = {} + func_used_imports = emptylistdict() + for item_name in sorted_no_case(funcs.keys()): + if item_name in omitted_names: + out(0, "# definition of ", item_name, " omitted") + continue + item = funcs[item_name] + try: + self.redo_function(out, item, item_name, 0, p_modname=p_name, seen=seen_funcs, used_imports=func_used_imports) + except: + handle_error_func(item_name, out) + # don't import anything from . for functions, functions are emitted in __init__ + # and all local classes are imported by the time they're defined + func_used_imports['.'] = [] + self.output_import_froms(self.func_imports_buf.out, func_used_imports) + else: + self.functions_buf.out(0, "# no functions") + # + if classes: + self.classes_buf.out(0, "# classes") + self.classes_buf.out(0, "") + seen_classes = {} + # sort classes so that inheritance order is preserved + cls_list = [] # items are (class_name, mro_tuple) + for cls_name in sorted_no_case(classes.keys()): + cls = classes[cls_name] + ins_index = len(cls_list) + for i in range(ins_index): + maybe_child_bases = cls_list[i][1] + if cls in maybe_child_bases: + ins_index = i # we could not go farther than current ins_index + break # ...and need not go fartehr than first known child + cls_list.insert(ins_index, (cls_name, get_mro(cls))) + self.split_modules = self.mod_filename and len(cls_list) >= 30 + for item_name in [cls_item[0] for cls_item in cls_list]: + buf = ClassBuf(item_name, self) + imports = ClassBuf(item_name + '_imports', self) + self.classes_buffs.append((buf,imports)) + out = buf.out + if item_name in omitted_names: + out(0, "# definition of ", item_name, " omitted") + continue + item = classes[item_name] + used_imports = emptylistdict() + self.redo_class(out, item, item_name, 0, p_modname=p_name, seen=seen_classes, inspect_dir=inspect_dir, used_imports=used_imports) + # if we don't have split modules we can't import dependencies, but we also + # have an ordering constraint - classes need to be declared after any classes + # they reference in type hints. This is broken either way though even with the + # return literals + if not self.split_modules: + func_used_imports['.'] = [] + self.output_import_froms(imports.out, used_imports) + self._defined[item_name] = True + out(0, "") # empty line after each item + + if self.doing_builtins and p_name == BUILTIN_MOD_NAME and version[0] < 3: + # classobj still supported + txt = classobj_txt + self.classes_buf.out(0, txt) + + if self.doing_builtins and p_name == BUILTIN_MOD_NAME: + txt = create_generator() + self.classes_buf.out(0, txt) + txt = create_async_generator() + self.classes_buf.out(0, txt) + txt = create_function() + self.classes_buf.out(0, txt) + txt = create_method() + self.classes_buf.out(0, txt) + txt = create_coroutine() + self.classes_buf.out(0, txt) + + # Fake + if version[0] >= 3 or (version[0] == 2 and version[1] >= 6): + namedtuple_text = create_named_tuple() + self.classes_buf.out(0, namedtuple_text) + + else: + self.classes_buf.out(0, "# no classes") + # + if vars_complex: + out = self.footer_buf.out + out(0, "# variables with complex values") + out(0, "") + for item_name in sorted_no_case(vars_complex.keys()): + if item_name in omitted_names: + out(0, "# definition of " + item_name + " omitted") + continue + item = vars_complex[item_name] + if str(type(item)) == "": + continue # this is an IronPython submodule, we mustn't generate a reference for it in the base module + replacement = REPLACE_MODULE_VALUES.get((p_name, item_name), None) + if replacement is not None: + out(0, item_name + " = " + replacement + " # real value of type " + str(type(item)) + " replaced") + elif is_skipped_in_module(p_name, item_name): + t_item = type(item) + out(0, item_name + " = " + self.invent_initializer(t_item) + " # real value of type " + str( + t_item) + " skipped") + else: + self.fmt_value(out, item, 0, prefix=item_name + " = ", as_name=item_name) + self._defined[item_name] = True + out(0, "") # empty line after each item + values_to_add = ADD_VALUE_IN_MODULE.get(p_name, None) + if values_to_add: + self.footer_buf.out(0, "# intermittent names") + for value in values_to_add: + self.footer_buf.out(0, value) + # imports: last, because previous parts could alter used_imports or hidden_imports + + out = self.imports_buf.out + self.output_import_froms(out, self.used_imports) + + if self.hidden_imports: + self.add_import_header_if_needed() + for mod_name in sorted_no_case(self.hidden_imports.keys()): + out(0, 'import ', mod_name, ' as ', self.hidden_imports[mod_name]) + out(0, "") # empty line after group + + if self.used_typing: + self.add_import_header_if_needed() + out(0, 'from typing import List, Tuple, Callable, Any') + out(0, "") # empty line after group + + if self.imports_buf.isEmpty(): + out(0, "# no imports") + out(0, "") # empty line after imports + + def output_import_froms(self, out, imports_list): + """Mention all imported names known within the module, wrapping as per PEP.""" + if imports_list: + self.add_import_header_if_needed() + for mod_name in sorted_no_case(imports_list.keys()): + import_names = imports_list[mod_name] + if mod_name == '.': + # if this is a local import, we need to treat it specially to import + # the class inside the referenced module + for n in import_names: + out(0, "from .%s import %s" % (n, n)) # empty line after group + out(0, "") # empty line after group + elif import_names: + self._defined[mod_name] = True + right_pos = 0 # tracks width of list to fold it at right margin + import_heading = "from % s import (" % mod_name + right_pos += len(import_heading) + names_pack = [import_heading] + indent_level = 0 + import_names = list(import_names) + import_names.sort() + for n in import_names: + self._defined[n] = True + len_n = len(n) + if right_pos + len_n >= 78: + out(indent_level, *names_pack) + names_pack = [n, ", "] + if indent_level == 0: + indent_level = 1 # all but first line is indented + right_pos = self.indent_size + len_n + 2 + else: + names_pack.append(n) + names_pack.append(", ") + right_pos += (len_n + 2) + # last line is... + if indent_level == 0: # one line + names_pack[0] = names_pack[0][:-1] # cut off lpar + names_pack[-1] = "" # cut last comma + else: # last line of multiline + names_pack[-1] = ")" # last comma -> rpar + out(indent_level, *names_pack) + + out(0, "") # empty line after group + +