From 62b2e2226d1c81c42e4aaf7292975fa1e3050da3 Mon Sep 17 00:00:00 2001 From: baldurk Date: Thu, 13 Jun 2024 13:32:42 +0100 Subject: [PATCH] Commit modified version of intellij-community stubgen & wrapper script --- docs/credits_acknowledgements.rst | 4 + docs/regenerate_stubs.py | 36 + docs/stubs_generation/LICENSE.txt | 202 + docs/stubs_generation/README.md | 7 + .../helpers/generator3/__init__.py | 5 + .../helpers/generator3/__main__.py | 216 + .../helpers/generator3/_vendor/__init__.py | 0 .../generator3/_vendor/pyparsing_py2.py | 3749 +++++++++++++++++ .../generator3/_vendor/pyparsing_py3.py | 3586 ++++++++++++++++ .../helpers/generator3/clr_tools.py | 63 + .../helpers/generator3/constants.py | 722 ++++ .../helpers/generator3/core.py | 651 +++ .../helpers/generator3/docstring_parsing.py | 210 + .../helpers/generator3/extra.py | 165 + .../helpers/generator3/module_redeclarator.py | 1353 ++++++ .../helpers/generator3/required_gen_version | 50 + .../helpers/generator3/util_methods.py | 938 +++++ .../helpers/generator3/version.txt | 1 + util/installer/LICENSE.rtf | 1 + 19 files changed, 11959 insertions(+) create mode 100644 docs/regenerate_stubs.py create mode 100644 docs/stubs_generation/LICENSE.txt create mode 100644 docs/stubs_generation/README.md create mode 100644 docs/stubs_generation/helpers/generator3/__init__.py create mode 100644 docs/stubs_generation/helpers/generator3/__main__.py create mode 100644 docs/stubs_generation/helpers/generator3/_vendor/__init__.py create mode 100644 docs/stubs_generation/helpers/generator3/_vendor/pyparsing_py2.py create mode 100644 docs/stubs_generation/helpers/generator3/_vendor/pyparsing_py3.py create mode 100644 docs/stubs_generation/helpers/generator3/clr_tools.py create mode 100644 docs/stubs_generation/helpers/generator3/constants.py create mode 100644 docs/stubs_generation/helpers/generator3/core.py create mode 100644 docs/stubs_generation/helpers/generator3/docstring_parsing.py create mode 100644 docs/stubs_generation/helpers/generator3/extra.py create mode 100644 docs/stubs_generation/helpers/generator3/module_redeclarator.py create mode 100644 docs/stubs_generation/helpers/generator3/required_gen_version create mode 100644 docs/stubs_generation/helpers/generator3/util_methods.py create mode 100644 docs/stubs_generation/helpers/generator3/version.txt diff --git a/docs/credits_acknowledgements.rst b/docs/credits_acknowledgements.rst index 50435ff3d..116a1f807 100644 --- a/docs/credits_acknowledgements.rst +++ b/docs/credits_acknowledgements.rst @@ -140,6 +140,10 @@ The following libraries and components are incorporated into RenderDoc, listed h Used to simplify compatibility with a broad range of Python versions. +* `intellij-community `_ - Copyright Contributors to the intellij-community project. Distributed under the Apache License. + + Used to generate Python stubs for binary modules. + Thanks ------ diff --git a/docs/regenerate_stubs.py b/docs/regenerate_stubs.py new file mode 100644 index 000000000..cab1b846e --- /dev/null +++ b/docs/regenerate_stubs.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 + +import os +import sys +import struct + +if len(sys.argv) != 2: + print(f"Usage: {sys.argv[0]} [path/to/stubs/folder]") + sys.exit(1) + +# path to module libraries for windows +if struct.calcsize("P") == 8: + binpath = '../x64/' +else: + binpath = '../Win32/' + +# Prioritise release over development builds +sys.path.insert(0, os.path.abspath(binpath + 'Development/pymodules')) +sys.path.insert(0, os.path.abspath(binpath + 'Release/pymodules')) + +# Add the build paths to PATH so renderdoc.dll can be located +os.environ["PATH"] += os.pathsep + os.path.abspath(binpath + 'Development/') +os.environ["PATH"] += os.pathsep + os.path.abspath(binpath + 'Release/') + +if sys.platform == 'win32' and sys.version_info[1] >= 8: + os.add_dll_directory(binpath + 'Release/') + os.add_dll_directory(binpath + 'Development/') + +# path to module libraries for linux +sys.path.insert(0, os.path.abspath('../build/lib')) + +from stubs_generation.helpers import generator3 + +if __name__ == '__main__': + generator3.main(['renderdoc', '-d', sys.argv[1]]) + generator3.main(['qrenderdoc', '-d', sys.argv[1]]) diff --git a/docs/stubs_generation/LICENSE.txt b/docs/stubs_generation/LICENSE.txt new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/docs/stubs_generation/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/docs/stubs_generation/README.md b/docs/stubs_generation/README.md new file mode 100644 index 000000000..0038b3278 --- /dev/null +++ b/docs/stubs_generation/README.md @@ -0,0 +1,7 @@ +## Python stubs generation + +This folder contains a slightly modified version of [intellij-community](https://github.com/JetBrains/intellij-community/tree/master/python/helpers/generator3)'s stub generator for python binary modules, to parse RST documentation and generate typed helpers. + +It has had some modifications to work better within RenderDoc's desired environment rather than an IDE, as well as producing more desirable output. + +It remains licensed under the Apache license to the original contributors. diff --git a/docs/stubs_generation/helpers/generator3/__init__.py b/docs/stubs_generation/helpers/generator3/__init__.py new file mode 100644 index 000000000..52599806d --- /dev/null +++ b/docs/stubs_generation/helpers/generator3/__init__.py @@ -0,0 +1,5 @@ +# make this usable as module +from .__main__ import main +from .__main__ import _bootstrap_sys_path + +_bootstrap_sys_path() \ No newline at end of file diff --git a/docs/stubs_generation/helpers/generator3/__main__.py b/docs/stubs_generation/helpers/generator3/__main__.py new file mode 100644 index 000000000..a9edd2228 --- /dev/null +++ b/docs/stubs_generation/helpers/generator3/__main__.py @@ -0,0 +1,216 @@ +import argparse +import atexit +import json +import logging +import os +import sys + +_containing_dir = os.path.dirname(os.path.abspath(__file__)) +_helpers_dir = os.path.dirname(_containing_dir) + + +def _cleanup_sys_path(): + return [root for root in sys.path + if os.path.normpath(root) not in (_containing_dir, _helpers_dir)] + + +def _bootstrap_sys_path(): + sys.path.insert(0, _helpers_dir) + + +def _setup_logging(): + from generator3.util_methods import configure_logging + configure_logging(logging.DEBUG) + + +def _enable_segfault_tracebacks(): + try: + import faulthandler + + faulthandler.enable() + except ImportError: + pass + + +def _configure_multiprocessing(): + required_start_method = os.environ.get('GENERATOR3_MULTIPROCESSING_START_METHOD') + if required_start_method: + import multiprocessing + # Available only since Python 3.4 + multiprocessing.set_start_method(required_start_method) + + +def parse_args(gen_version, args_list): + parser = argparse.ArgumentParser( + prog='generator3', + description='Generates interface skeletons (binary stubs) for binary and ' + 'built-in Python modules.' + ) + parser.add_argument( + '-d', metavar='PATH', dest='output_dir', + help='Output dir, must be writable. If not given, current dir is used.' + ) + # TODO using os.pathsep might cause problems with remote interpreters when host and + # target OS don't match + parser.add_argument( + '-s', metavar='PATH_LIST', dest='roots', + type=(lambda s: s.split(os.pathsep)), default=[], + help='List of root directories to scan for binaries separated with `os.pathsep`' + ' character. These directories will be added in `sys.path`.' + ) + parser.add_argument( + '--name-pattern', metavar='PATTERN', + help='Shell-like glob pattern restricting generation only to modules with ' + 'matching qualified names, e.g, "_ast" or "numpy.*".' + ) + parser.add_argument( + '--builtins-only', action='store_true', + help='Limit generation only to the modules in `sys.builtin_module_names`.' + ) + parser.add_argument( + '--state-file', metavar='PATH', + type=argparse.FileType('rb'), + help='Path to the input ".state.json" file. If "-", the file is passed via ' + 'stdin. The resulting ".state.json" will be generated automatically in ' + 'the skeletons directory.' + ) + parser.add_argument( + '--init-state-file', action='store_true', + help='Generate a new ".state.json" file in the skeletons directory.' + ) + + # Common flags + # TODO evaluate these flags, some of them seem redundant now with proper logging + parser.add_argument( + '-q', dest='quiet', action='store_true', + help='Be quiet, do not print anything on stdout. Errors still go to stderr.' + ) + parser.add_argument( + '-v', dest='verbose', action='store_true', + help='Be verbose, print lots of debug output to stderr.' + ) + + parser.add_argument('-V', action='version', version=gen_version) + + extra_modes = parser.add_argument_group('extra modes') + extra_modes.add_argument( + '-S', dest='list_sources_mode', action='store_true', + help='Lists all python sources found in `sys.path` and directories specified ' + 'with -s.' + ) + extra_modes.add_argument( + '-z', dest='zip_sources_archive', metavar='ARCHIVE', + help='Zip files to specified archive. Accepts files to be archived from stdin ' + 'in format: .' + ) + extra_modes.add_argument( + '-u', dest='zip_roots_archive', metavar='ARCHIVE', + help='Zip all source files from `sys.path` and provided roots in the specified ' + 'archive.' + ) + + clr_specific = parser.add_argument_group('CLR specific options') + clr_specific.add_argument( + '-c', dest='clr_assemblies', metavar='MODULES', + type=(lambda s: s.split(';')), default=[], + help='Semicolon separated list of CLR assemblies to be imported.' + ) + clr_specific.add_argument( + '-p', dest='run_clr_profiler', action='store_true', help='Run CLR profiler.' + ) + + parser.add_argument( + "mod_name", nargs='?', default=None, + help='Qualified name of a single module to analyze.' + ) + parser.add_argument( + "mod_path", nargs='?', default=None, + help='Path to the specified module if it\'s not builtin.' + ) + return parser.parse_args(args_list) + + +def main(args_list): + import generator3.core + import generator3.extra + from generator3.clr_tools import get_namespace_by_name + from generator3.constants import Timer + from generator3.core import version, GenerationStatus, SkeletonGenerator + from generator3.util_methods import set_verbose, say, note, print_profile + + args = parse_args(version(), args_list) + + generator3.core.quiet = args.quiet + set_verbose(args.verbose) + + if args.roots: + for p in args.roots: + if p and p not in sys.path: + # we need this to make things in additional dirs importable + sys.path.append(p) + note("Altered sys.path: %r", sys.path) + + if args.state_file: + # We can't completely shut off stdin in case Docker-based interpreter to use + # json.load() and have to retreat to reading the content line-wise + if args.state_file.name == '': + state_json = json.loads(sys.stdin.readline()) # utf-8 by default + else: + with args.state_file as f: + state_json = json.loads(f.read().decode(encoding='utf-8')) + else: + state_json = None + + target_roots = _cleanup_sys_path() + + if args.list_sources_mode: + say(version()) + generator3.extra.list_sources(target_roots) + sys.exit(0) + + if args.zip_sources_archive: + generator3.extra.zip_sources(args.zip_sources_archive) + sys.exit(0) + + if args.zip_roots_archive: + generator3.extra.zip_stdlib(target_roots, args.zip_roots_archive) + sys.exit(0) + + generator = SkeletonGenerator( + output_dir=args.output_dir or '.', # implement documented default to current directory + roots=target_roots, + state_json=state_json, + write_state_json=bool(args.init_state_file or args.state_file) + ) + + timer = Timer() + if not args.mod_name: + generator.discover_and_process_all_modules(name_pattern=args.name_pattern, + builtins_only=args.builtins_only) + sys.exit(0) + + if sys.platform == 'cli': + # noinspection PyUnresolvedReferences + import clr + + for ref in args.clr_assemblies: + clr.AddReferenceByPartialName(ref) + + if args.run_clr_profiler: + atexit.register(print_profile) + + # We take module name from import statement + args.mod_name = get_namespace_by_name(args.mod_name) + + if generator.process_module(args.mod_name, args.mod_path) == GenerationStatus.FAILED: + sys.exit(1) + + say("Generation completed in %d ms", timer.elapsed()) + + +if __name__ == "__main__": + _bootstrap_sys_path() + _setup_logging() + _enable_segfault_tracebacks() + _configure_multiprocessing() + main(sys.argv[1:]) diff --git a/docs/stubs_generation/helpers/generator3/_vendor/__init__.py b/docs/stubs_generation/helpers/generator3/_vendor/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/docs/stubs_generation/helpers/generator3/_vendor/pyparsing_py2.py b/docs/stubs_generation/helpers/generator3/_vendor/pyparsing_py2.py new file mode 100644 index 000000000..b1372e15f --- /dev/null +++ b/docs/stubs_generation/helpers/generator3/_vendor/pyparsing_py2.py @@ -0,0 +1,3749 @@ +# module pyparsing.py +# +# Copyright (c) 2003-2011 Paul T. McGuire +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +#from __future__ import generators + +__doc__ = \ +""" +pyparsing module - Classes and methods to define and execute parsing grammars + +The pyparsing module is an alternative approach to creating and executing simple grammars, +vs. the traditional lex/yacc approach, or the use of regular expressions. With pyparsing, you +don't need to learn a new syntax for defining grammars or matching expressions - the parsing module +provides a library of classes that you use to construct the grammar directly in Python. + +Here is a program to parse "Hello, World!" (or any greeting of the form C{", !"}):: + + from pyparsing import Word, alphas + + # define grammar of a greeting + greet = Word( alphas ) + "," + Word( alphas ) + "!" + + hello = "Hello, World!" + print hello, "->", greet.parseString( hello ) + +The program outputs the following:: + + Hello, World! -> ['Hello', ',', 'World', '!'] + +The Python representation of the grammar is quite readable, owing to the self-explanatory +class names, and the use of '+', '|' and '^' operators. + +The parsed results returned from C{parseString()} can be accessed as a nested list, a dictionary, or an +object with named attributes. + +The pyparsing module handles some of the problems that are typically vexing when writing text parsers: + - extra or missing whitespace (the above program will also handle "Hello,World!", "Hello , World !", etc.) + - quoted strings + - embedded comments +""" + +__version__ = "1.5.6" +__versionTime__ = "26 June 2011 10:53" +__author__ = "Paul McGuire " + +import string +from weakref import ref as wkref +import copy +import sys +import warnings +import re +import sre_constants +#~ sys.stderr.write( "testing pyparsing module, version %s, %s\n" % (__version__,__versionTime__ ) ) + +__all__ = [ +'And', 'CaselessKeyword', 'CaselessLiteral', 'CharsNotIn', 'Combine', 'Dict', 'Each', 'Empty', +'FollowedBy', 'Forward', 'GoToColumn', 'Group', 'Keyword', 'LineEnd', 'LineStart', 'Literal', +'MatchFirst', 'NoMatch', 'NotAny', 'OneOrMore', 'OnlyOnce', 'Optional', 'Or', +'ParseBaseException', 'ParseElementEnhance', 'ParseException', 'ParseExpression', 'ParseFatalException', +'ParseResults', 'ParseSyntaxException', 'ParserElement', 'QuotedString', 'RecursiveGrammarException', +'Regex', 'SkipTo', 'StringEnd', 'StringStart', 'Suppress', 'Token', 'TokenConverter', 'Upcase', +'White', 'Word', 'WordEnd', 'WordStart', 'ZeroOrMore', +'alphanums', 'alphas', 'alphas8bit', 'anyCloseTag', 'anyOpenTag', 'cStyleComment', 'col', +'commaSeparatedList', 'commonHTMLEntity', 'countedArray', 'cppStyleComment', 'dblQuotedString', +'dblSlashComment', 'delimitedList', 'dictOf', 'downcaseTokens', 'empty', 'getTokensEndLoc', 'hexnums', +'htmlComment', 'javaStyleComment', 'keepOriginalText', 'line', 'lineEnd', 'lineStart', 'lineno', +'makeHTMLTags', 'makeXMLTags', 'matchOnlyAtCol', 'matchPreviousExpr', 'matchPreviousLiteral', +'nestedExpr', 'nullDebugAction', 'nums', 'oneOf', 'opAssoc', 'operatorPrecedence', 'printables', +'punc8bit', 'pythonStyleComment', 'quotedString', 'removeQuotes', 'replaceHTMLEntity', +'replaceWith', 'restOfLine', 'sglQuotedString', 'srange', 'stringEnd', +'stringStart', 'traceParseAction', 'unicodeString', 'upcaseTokens', 'withAttribute', +'indentedBlock', 'originalTextFor', +] + +""" +Detect if we are running version 3.X and make appropriate changes +Robert A. Clark +""" +_PY3K = sys.version_info[0] > 2 +if _PY3K: + _MAX_INT = sys.maxsize + basestring = str + unichr = chr + _ustr = str + alphas = string.ascii_lowercase + string.ascii_uppercase +else: + _MAX_INT = sys.maxint + range = xrange + set = lambda s : dict( [(c,0) for c in s] ) + alphas = string.lowercase + string.uppercase + + def _ustr(obj): + """Drop-in replacement for str(obj) that tries to be Unicode friendly. It first tries + str(obj). If that fails with a UnicodeEncodeError, then it tries unicode(obj). It + then < returns the unicode object | encodes it with the default encoding | ... >. + """ + if isinstance(obj,unicode): + return obj + + try: + # If this works, then _ustr(obj) has the same behaviour as str(obj), so + # it won't break any existing code. + return str(obj) + + except UnicodeEncodeError: + # The Python docs (http://docs.python.org/ref/customization.html#l2h-182) + # state that "The return value must be a string object". However, does a + # unicode object (being a subclass of basestring) count as a "string + # object"? + # If so, then return a unicode object: + return unicode(obj) + # Else encode it... but how? There are many choices... :) + # Replace unprintables with escape codes? + #return unicode(obj).encode(sys.getdefaultencoding(), 'backslashreplace_errors') + # Replace unprintables with question marks? + #return unicode(obj).encode(sys.getdefaultencoding(), 'replace') + # ... + + alphas = string.lowercase + string.uppercase + +# build list of single arg builtins, tolerant of Python version, that can be used as parse actions +singleArgBuiltins = [] +import __builtin__ +for fname in "sum len enumerate sorted reversed list tuple set any all".split(): + try: + singleArgBuiltins.append(getattr(__builtin__,fname)) + except AttributeError: + continue + +def _xml_escape(data): + """Escape &, <, >, ", ', etc. in a string of data.""" + + # ampersand must be replaced first + from_symbols = '&><"\'' + to_symbols = ['&'+s+';' for s in "amp gt lt quot apos".split()] + for from_,to_ in zip(from_symbols, to_symbols): + data = data.replace(from_, to_) + return data + +class _Constants(object): + pass + +nums = string.digits +hexnums = nums + "ABCDEFabcdef" +alphanums = alphas + nums +_bslash = chr(92) +printables = "".join( [ c for c in string.printable if c not in string.whitespace ] ) + +class ParseBaseException(Exception): + """base exception class for all parsing runtime exceptions""" + # Performance tuning: we construct a *lot* of these, so keep this + # constructor as small and fast as possible + def __init__( self, pstr, loc=0, msg=None, elem=None ): + self.loc = loc + if msg is None: + self.msg = pstr + self.pstr = "" + else: + self.msg = msg + self.pstr = pstr + self.parserElement = elem + + def __getattr__( self, aname ): + """supported attributes by name are: + - lineno - returns the line number of the exception text + - col - returns the column number of the exception text + - line - returns the line containing the exception text + """ + if( aname == "lineno" ): + return lineno( self.loc, self.pstr ) + elif( aname in ("col", "column") ): + return col( self.loc, self.pstr ) + elif( aname == "line" ): + return line( self.loc, self.pstr ) + else: + raise AttributeError(aname) + + def __str__( self ): + return "%s (at char %d), (line:%d, col:%d)" % \ + ( self.msg, self.loc, self.lineno, self.column ) + def __repr__( self ): + return _ustr(self) + def markInputline( self, markerString = ">!<" ): + """Extracts the exception line from the input string, and marks + the location of the exception with a special symbol. + """ + line_str = self.line + line_column = self.column - 1 + if markerString: + line_str = "".join( [line_str[:line_column], + markerString, line_str[line_column:]]) + return line_str.strip() + def __dir__(self): + return "loc msg pstr parserElement lineno col line " \ + "markInputLine __str__ __repr__".split() + +class ParseException(ParseBaseException): + """exception thrown when parse expressions don't match class; + supported attributes by name are: + - lineno - returns the line number of the exception text + - col - returns the column number of the exception text + - line - returns the line containing the exception text + """ + pass + +class ParseFatalException(ParseBaseException): + """user-throwable exception thrown when inconsistent parse content + is found; stops all parsing immediately""" + pass + +class ParseSyntaxException(ParseFatalException): + """just like C{ParseFatalException}, but thrown internally when an + C{ErrorStop} ('-' operator) indicates that parsing is to stop immediately because + an unbacktrackable syntax error has been found""" + def __init__(self, pe): + super(ParseSyntaxException, self).__init__( + pe.pstr, pe.loc, pe.msg, pe.parserElement) + +#~ class ReparseException(ParseBaseException): + #~ """Experimental class - parse actions can raise this exception to cause + #~ pyparsing to reparse the input string: + #~ - with a modified input string, and/or + #~ - with a modified start location + #~ Set the values of the ReparseException in the constructor, and raise the + #~ exception in a parse action to cause pyparsing to use the new string/location. + #~ Setting the values as None causes no change to be made. + #~ """ + #~ def __init_( self, newstring, restartLoc ): + #~ self.newParseText = newstring + #~ self.reparseLoc = restartLoc + +class RecursiveGrammarException(Exception): + """exception thrown by C{validate()} if the grammar could be improperly recursive""" + def __init__( self, parseElementList ): + self.parseElementTrace = parseElementList + + def __str__( self ): + return "RecursiveGrammarException: %s" % self.parseElementTrace + +class _ParseResultsWithOffset(object): + def __init__(self,p1,p2): + self.tup = (p1,p2) + def __getitem__(self,i): + return self.tup[i] + def __repr__(self): + return repr(self.tup) + def setOffset(self,i): + self.tup = (self.tup[0],i) + +class ParseResults(object): + """Structured parse results, to provide multiple means of access to the parsed data: + - as a list (C{len(results)}) + - by list index (C{results[0], results[1]}, etc.) + - by attribute (C{results.}) + """ + #~ __slots__ = ( "__toklist", "__tokdict", "__doinit", "__name", "__parent", "__accumNames", "__weakref__" ) + def __new__(cls, toklist, name=None, asList=True, modal=True ): + if isinstance(toklist, cls): + return toklist + retobj = object.__new__(cls) + retobj.__doinit = True + return retobj + + # Performance tuning: we construct a *lot* of these, so keep this + # constructor as small and fast as possible + def __init__( self, toklist, name=None, asList=True, modal=True, isinstance=isinstance ): + if self.__doinit: + self.__doinit = False + self.__name = None + self.__parent = None + self.__accumNames = {} + if isinstance(toklist, list): + self.__toklist = toklist[:] + else: + self.__toklist = [toklist] + self.__tokdict = dict() + + if name is not None and name: + if not modal: + self.__accumNames[name] = 0 + if isinstance(name,int): + name = _ustr(name) # will always return a str, but use _ustr for consistency + self.__name = name + if not toklist in (None,'',[]): + if isinstance(toklist,basestring): + toklist = [ toklist ] + if asList: + if isinstance(toklist,ParseResults): + self[name] = _ParseResultsWithOffset(toklist.copy(),0) + else: + self[name] = _ParseResultsWithOffset(ParseResults(toklist[0]),0) + self[name].__name = name + else: + try: + self[name] = toklist[0] + except (KeyError,TypeError,IndexError): + self[name] = toklist + + def __getitem__( self, i ): + if isinstance( i, (int,slice) ): + return self.__toklist[i] + else: + if i not in self.__accumNames: + return self.__tokdict[i][-1][0] + else: + return ParseResults([ v[0] for v in self.__tokdict[i] ]) + + def __setitem__( self, k, v, isinstance=isinstance ): + if isinstance(v,_ParseResultsWithOffset): + self.__tokdict[k] = self.__tokdict.get(k,list()) + [v] + sub = v[0] + elif isinstance(k,int): + self.__toklist[k] = v + sub = v + else: + self.__tokdict[k] = self.__tokdict.get(k,list()) + [_ParseResultsWithOffset(v,0)] + sub = v + if isinstance(sub,ParseResults): + sub.__parent = wkref(self) + + def __delitem__( self, i ): + if isinstance(i,(int,slice)): + mylen = len( self.__toklist ) + del self.__toklist[i] + + # convert int to slice + if isinstance(i, int): + if i < 0: + i += mylen + i = slice(i, i+1) + # get removed indices + removed = list(range(*i.indices(mylen))) + removed.reverse() + # fixup indices in token dictionary + for name in self.__tokdict: + occurrences = self.__tokdict[name] + for j in removed: + for k, (value, position) in enumerate(occurrences): + occurrences[k] = _ParseResultsWithOffset(value, position - (position > j)) + else: + del self.__tokdict[i] + + def __contains__( self, k ): + return k in self.__tokdict + + def __len__( self ): return len( self.__toklist ) + def __bool__(self): return len( self.__toklist ) > 0 + __nonzero__ = __bool__ + def __iter__( self ): return iter( self.__toklist ) + def __reversed__( self ): return iter( self.__toklist[::-1] ) + def keys( self ): + """Returns all named result keys.""" + return self.__tokdict.keys() + + def pop( self, index=-1 ): + """Removes and returns item at specified index (default=last). + Will work with either numeric indices or dict-key indicies.""" + ret = self[index] + del self[index] + return ret + + def get(self, key, defaultValue=None): + """Returns named result matching the given key, or if there is no + such name, then returns the given C{defaultValue} or C{None} if no + C{defaultValue} is specified.""" + if key in self: + return self[key] + else: + return defaultValue + + def insert( self, index, insStr ): + """Inserts new element at location index in the list of parsed tokens.""" + self.__toklist.insert(index, insStr) + # fixup indices in token dictionary + for name in self.__tokdict: + occurrences = self.__tokdict[name] + for k, (value, position) in enumerate(occurrences): + occurrences[k] = _ParseResultsWithOffset(value, position + (position > index)) + + def items( self ): + """Returns all named result keys and values as a list of tuples.""" + return [(k,self[k]) for k in self.__tokdict] + + def values( self ): + """Returns all named result values.""" + return [ v[-1][0] for v in self.__tokdict.values() ] + + def __getattr__( self, name ): + if True: #name not in self.__slots__: + if name in self.__tokdict: + if name not in self.__accumNames: + return self.__tokdict[name][-1][0] + else: + return ParseResults([ v[0] for v in self.__tokdict[name] ]) + else: + return "" + return None + + def __add__( self, other ): + ret = self.copy() + ret += other + return ret + + def __iadd__( self, other ): + if other.__tokdict: + offset = len(self.__toklist) + addoffset = ( lambda a: (a<0 and offset) or (a+offset) ) + otheritems = other.__tokdict.items() + otherdictitems = [(k, _ParseResultsWithOffset(v[0],addoffset(v[1])) ) + for (k,vlist) in otheritems for v in vlist] + for k,v in otherdictitems: + self[k] = v + if isinstance(v[0],ParseResults): + v[0].__parent = wkref(self) + + self.__toklist += other.__toklist + self.__accumNames.update( other.__accumNames ) + return self + + def __radd__(self, other): + if isinstance(other,int) and other == 0: + return self.copy() + + def __repr__( self ): + return "(%s, %s)" % ( repr( self.__toklist ), repr( self.__tokdict ) ) + + def __str__( self ): + out = "[" + sep = "" + for i in self.__toklist: + if isinstance(i, ParseResults): + out += sep + _ustr(i) + else: + out += sep + repr(i) + sep = ", " + out += "]" + return out + + def _asStringList( self, sep='' ): + out = [] + for item in self.__toklist: + if out and sep: + out.append(sep) + if isinstance( item, ParseResults ): + out += item._asStringList() + else: + out.append( _ustr(item) ) + return out + + def asList( self ): + """Returns the parse results as a nested list of matching tokens, all converted to strings.""" + out = [] + for res in self.__toklist: + if isinstance(res,ParseResults): + out.append( res.asList() ) + else: + out.append( res ) + return out + + def asDict( self ): + """Returns the named parse results as dictionary.""" + return dict( self.items() ) + + def copy( self ): + """Returns a new copy of a C{ParseResults} object.""" + ret = ParseResults( self.__toklist ) + ret.__tokdict = self.__tokdict.copy() + ret.__parent = self.__parent + ret.__accumNames.update( self.__accumNames ) + ret.__name = self.__name + return ret + + def asXML( self, doctag=None, namedItemsOnly=False, indent="", formatted=True ): + """Returns the parse results as XML. Tags are created for tokens and lists that have defined results names.""" + nl = "\n" + out = [] + namedItems = dict( [ (v[1],k) for (k,vlist) in self.__tokdict.items() + for v in vlist ] ) + nextLevelIndent = indent + " " + + # collapse out indents if formatting is not desired + if not formatted: + indent = "" + nextLevelIndent = "" + nl = "" + + selfTag = None + if doctag is not None: + selfTag = doctag + else: + if self.__name: + selfTag = self.__name + + if not selfTag: + if namedItemsOnly: + return "" + else: + selfTag = "ITEM" + + out += [ nl, indent, "<", selfTag, ">" ] + + worklist = self.__toklist + for i,res in enumerate(worklist): + if isinstance(res,ParseResults): + if i in namedItems: + out += [ res.asXML(namedItems[i], + namedItemsOnly and doctag is None, + nextLevelIndent, + formatted)] + else: + out += [ res.asXML(None, + namedItemsOnly and doctag is None, + nextLevelIndent, + formatted)] + else: + # individual token, see if there is a name for it + resTag = None + if i in namedItems: + resTag = namedItems[i] + if not resTag: + if namedItemsOnly: + continue + else: + resTag = "ITEM" + xmlBodyText = _xml_escape(_ustr(res)) + out += [ nl, nextLevelIndent, "<", resTag, ">", + xmlBodyText, + "" ] + + out += [ nl, indent, "" ] + return "".join(out) + + def __lookup(self,sub): + for k,vlist in self.__tokdict.items(): + for v,loc in vlist: + if sub is v: + return k + return None + + def getName(self): + """Returns the results name for this token expression.""" + if self.__name: + return self.__name + elif self.__parent: + par = self.__parent() + if par: + return par.__lookup(self) + else: + return None + elif (len(self) == 1 and + len(self.__tokdict) == 1 and + self.__tokdict.values()[0][0][1] in (0,-1)): + return self.__tokdict.keys()[0] + else: + return None + + def dump(self,indent='',depth=0): + """Diagnostic method for listing out the contents of a C{ParseResults}. + Accepts an optional C{indent} argument so that this string can be embedded + in a nested display of other data.""" + out = [] + out.append( indent+_ustr(self.asList()) ) + keys = self.items() + keys.sort() + for k,v in keys: + if out: + out.append('\n') + out.append( "%s%s- %s: " % (indent,(' '*depth), k) ) + if isinstance(v,ParseResults): + if v.keys(): + out.append( v.dump(indent,depth+1) ) + else: + out.append(_ustr(v)) + else: + out.append(_ustr(v)) + return "".join(out) + + # add support for pickle protocol + def __getstate__(self): + return ( self.__toklist, + ( self.__tokdict.copy(), + self.__parent is not None and self.__parent() or None, + self.__accumNames, + self.__name ) ) + + def __setstate__(self,state): + self.__toklist = state[0] + (self.__tokdict, + par, + inAccumNames, + self.__name) = state[1] + self.__accumNames = {} + self.__accumNames.update(inAccumNames) + if par is not None: + self.__parent = wkref(par) + else: + self.__parent = None + + def __dir__(self): + return dir(super(ParseResults,self)) + self.keys() + +def col (loc,strg): + """Returns current column within a string, counting newlines as line separators. + The first column is number 1. + + Note: the default parsing behavior is to expand tabs in the input string + before starting the parsing process. See L{I{ParserElement.parseString}} for more information + on parsing strings containing s, and suggested methods to maintain a + consistent view of the parsed string, the parse location, and line and column + positions within the parsed string. + """ + return (loc} for more information + on parsing strings containing s, and suggested methods to maintain a + consistent view of the parsed string, the parse location, and line and column + positions within the parsed string. + """ + return strg.count("\n",0,loc) + 1 + +def line( loc, strg ): + """Returns the line of text containing loc within a string, counting newlines as line separators. + """ + lastCR = strg.rfind("\n", 0, loc) + nextCR = strg.find("\n", loc) + if nextCR >= 0: + return strg[lastCR+1:nextCR] + else: + return strg[lastCR+1:] + +def _defaultStartDebugAction( instring, loc, expr ): + print ("Match " + _ustr(expr) + " at loc " + _ustr(loc) + "(%d,%d)" % ( lineno(loc,instring), col(loc,instring) )) + +def _defaultSuccessDebugAction( instring, startloc, endloc, expr, toks ): + print ("Matched " + _ustr(expr) + " -> " + str(toks.asList())) + +def _defaultExceptionDebugAction( instring, loc, expr, exc ): + print ("Exception raised:" + _ustr(exc)) + +def nullDebugAction(*args): + """'Do-nothing' debug action, to suppress debugging output during parsing.""" + pass + +'decorator to trim function calls to match the arity of the target' +if not _PY3K: + def _trim_arity(func, maxargs=2): + limit = [0] + def wrapper(*args): + while 1: + try: + return func(*args[limit[0]:]) + except TypeError: + if limit[0] <= maxargs: + limit[0] += 1 + continue + raise + return wrapper +else: + def _trim_arity(func, maxargs=2): + limit = maxargs + def wrapper(*args): + #~ nonlocal limit + while 1: + try: + return func(*args[limit:]) + except TypeError: + if limit: + limit -= 1 + continue + raise + return wrapper + +class ParserElement(object): + """Abstract base level parser element class.""" + DEFAULT_WHITE_CHARS = " \n\t\r" + verbose_stacktrace = False + + def setDefaultWhitespaceChars( chars ): + """Overrides the default whitespace chars + """ + ParserElement.DEFAULT_WHITE_CHARS = chars + setDefaultWhitespaceChars = staticmethod(setDefaultWhitespaceChars) + + def __init__( self, savelist=False ): + self.parseAction = list() + self.failAction = None + #~ self.name = "" # don't define self.name, let subclasses try/except upcall + self.strRepr = None + self.resultsName = None + self.saveAsList = savelist + self.skipWhitespace = True + self.whiteChars = ParserElement.DEFAULT_WHITE_CHARS + self.copyDefaultWhiteChars = True + self.mayReturnEmpty = False # used when checking for left-recursion + self.keepTabs = False + self.ignoreExprs = list() + self.debug = False + self.streamlined = False + self.mayIndexError = True # used to optimize exception handling for subclasses that don't advance parse index + self.errmsg = "" + self.modalResults = True # used to mark results names as modal (report only last) or cumulative (list all) + self.debugActions = ( None, None, None ) #custom debug actions + self.re = None + self.callPreparse = True # used to avoid redundant calls to preParse + self.callDuringTry = False + + def copy( self ): + """Make a copy of this C{ParserElement}. Useful for defining different parse actions + for the same parsing pattern, using copies of the original parse element.""" + cpy = copy.copy( self ) + cpy.parseAction = self.parseAction[:] + cpy.ignoreExprs = self.ignoreExprs[:] + if self.copyDefaultWhiteChars: + cpy.whiteChars = ParserElement.DEFAULT_WHITE_CHARS + return cpy + + def setName( self, name ): + """Define name for this expression, for use in debugging.""" + self.name = name + self.errmsg = "Expected " + self.name + if hasattr(self,"exception"): + self.exception.msg = self.errmsg + return self + + def setResultsName( self, name, listAllMatches=False ): + """Define name for referencing matching tokens as a nested attribute + of the returned parse results. + NOTE: this returns a *copy* of the original C{ParserElement} object; + this is so that the client can define a basic element, such as an + integer, and reference it in multiple places with different names. + + You can also set results names using the abbreviated syntax, + C{expr("name")} in place of C{expr.setResultsName("name")} - + see L{I{__call__}<__call__>}. + """ + newself = self.copy() + if name.endswith("*"): + name = name[:-1] + listAllMatches=True + newself.resultsName = name + newself.modalResults = not listAllMatches + return newself + + def setBreak(self,breakFlag = True): + """Method to invoke the Python pdb debugger when this element is + about to be parsed. Set C{breakFlag} to True to enable, False to + disable. + """ + if breakFlag: + _parseMethod = self._parse + def breaker(instring, loc, doActions=True, callPreParse=True): + import pdb + pdb.set_trace() + return _parseMethod( instring, loc, doActions, callPreParse ) + breaker._originalParseMethod = _parseMethod + self._parse = breaker + else: + if hasattr(self._parse,"_originalParseMethod"): + self._parse = self._parse._originalParseMethod + return self + + def setParseAction( self, *fns, **kwargs ): + """Define action to perform when successfully matching parse element definition. + Parse action fn is a callable method with 0-3 arguments, called as C{fn(s,loc,toks)}, + C{fn(loc,toks)}, C{fn(toks)}, or just C{fn()}, where: + - s = the original string being parsed (see note below) + - loc = the location of the matching substring + - toks = a list of the matched tokens, packaged as a ParseResults object + If the functions in fns modify the tokens, they can return them as the return + value from fn, and the modified list of tokens will replace the original. + Otherwise, fn does not need to return any value. + + Note: the default parsing behavior is to expand tabs in the input string + before starting the parsing process. See L{I{parseString}} for more information + on parsing strings containing s, and suggested methods to maintain a + consistent view of the parsed string, the parse location, and line and column + positions within the parsed string. + """ + self.parseAction = list(map(_trim_arity, list(fns))) + self.callDuringTry = ("callDuringTry" in kwargs and kwargs["callDuringTry"]) + return self + + def addParseAction( self, *fns, **kwargs ): + """Add parse action to expression's list of parse actions. See L{I{setParseAction}}.""" + self.parseAction += list(map(_trim_arity, list(fns))) + self.callDuringTry = self.callDuringTry or ("callDuringTry" in kwargs and kwargs["callDuringTry"]) + return self + + def setFailAction( self, fn ): + """Define action to perform if parsing fails at this expression. + Fail acton fn is a callable function that takes the arguments + C{fn(s,loc,expr,err)} where: + - s = string being parsed + - loc = location where expression match was attempted and failed + - expr = the parse expression that failed + - err = the exception thrown + The function returns no value. It may throw C{ParseFatalException} + if it is desired to stop parsing immediately.""" + self.failAction = fn + return self + + def _skipIgnorables( self, instring, loc ): + exprsFound = True + while exprsFound: + exprsFound = False + for e in self.ignoreExprs: + try: + while 1: + loc,dummy = e._parse( instring, loc ) + exprsFound = True + except ParseException: + pass + return loc + + def preParse( self, instring, loc ): + if self.ignoreExprs: + loc = self._skipIgnorables( instring, loc ) + + if self.skipWhitespace: + wt = self.whiteChars + instrlen = len(instring) + while loc < instrlen and instring[loc] in wt: + loc += 1 + + return loc + + def parseImpl( self, instring, loc, doActions=True ): + return loc, [] + + def postParse( self, instring, loc, tokenlist ): + return tokenlist + + #~ @profile + def _parseNoCache( self, instring, loc, doActions=True, callPreParse=True ): + debugging = ( self.debug ) #and doActions ) + + if debugging or self.failAction: + #~ print ("Match",self,"at loc",loc,"(%d,%d)" % ( lineno(loc,instring), col(loc,instring) )) + if (self.debugActions[0] ): + self.debugActions[0]( instring, loc, self ) + if callPreParse and self.callPreparse: + preloc = self.preParse( instring, loc ) + else: + preloc = loc + tokensStart = preloc + try: + try: + loc,tokens = self.parseImpl( instring, preloc, doActions ) + except IndexError: + raise ParseException( instring, len(instring), self.errmsg, self ) + except ParseBaseException: + #~ print ("Exception raised:", err) + err = None + if self.debugActions[2]: + err = sys.exc_info()[1] + self.debugActions[2]( instring, tokensStart, self, err ) + if self.failAction: + if err is None: + err = sys.exc_info()[1] + self.failAction( instring, tokensStart, self, err ) + raise + else: + if callPreParse and self.callPreparse: + preloc = self.preParse( instring, loc ) + else: + preloc = loc + tokensStart = preloc + if self.mayIndexError or loc >= len(instring): + try: + loc,tokens = self.parseImpl( instring, preloc, doActions ) + except IndexError: + raise ParseException( instring, len(instring), self.errmsg, self ) + else: + loc,tokens = self.parseImpl( instring, preloc, doActions ) + + tokens = self.postParse( instring, loc, tokens ) + + retTokens = ParseResults( tokens, self.resultsName, asList=self.saveAsList, modal=self.modalResults ) + if self.parseAction and (doActions or self.callDuringTry): + if debugging: + try: + for fn in self.parseAction: + tokens = fn( instring, tokensStart, retTokens ) + if tokens is not None: + retTokens = ParseResults( tokens, + self.resultsName, + asList=self.saveAsList and isinstance(tokens,(ParseResults,list)), + modal=self.modalResults ) + except ParseBaseException: + #~ print "Exception raised in user parse action:", err + if (self.debugActions[2] ): + err = sys.exc_info()[1] + self.debugActions[2]( instring, tokensStart, self, err ) + raise + else: + for fn in self.parseAction: + tokens = fn( instring, tokensStart, retTokens ) + if tokens is not None: + retTokens = ParseResults( tokens, + self.resultsName, + asList=self.saveAsList and isinstance(tokens,(ParseResults,list)), + modal=self.modalResults ) + + if debugging: + #~ print ("Matched",self,"->",retTokens.asList()) + if (self.debugActions[1] ): + self.debugActions[1]( instring, tokensStart, loc, self, retTokens ) + + return loc, retTokens + + def tryParse( self, instring, loc ): + try: + return self._parse( instring, loc, doActions=False )[0] + except ParseFatalException: + raise ParseException( instring, loc, self.errmsg, self) + + # this method gets repeatedly called during backtracking with the same arguments - + # we can cache these arguments and save ourselves the trouble of re-parsing the contained expression + def _parseCache( self, instring, loc, doActions=True, callPreParse=True ): + lookup = (self,instring,loc,callPreParse,doActions) + if lookup in ParserElement._exprArgCache: + value = ParserElement._exprArgCache[ lookup ] + if isinstance(value, Exception): + raise value + return (value[0],value[1].copy()) + else: + try: + value = self._parseNoCache( instring, loc, doActions, callPreParse ) + ParserElement._exprArgCache[ lookup ] = (value[0],value[1].copy()) + return value + except ParseBaseException: + pe = sys.exc_info()[1] + ParserElement._exprArgCache[ lookup ] = pe + raise + + _parse = _parseNoCache + + # argument cache for optimizing repeated calls when backtracking through recursive expressions + _exprArgCache = {} + def resetCache(): + ParserElement._exprArgCache.clear() + resetCache = staticmethod(resetCache) + + _packratEnabled = False + def enablePackrat(): + """Enables "packrat" parsing, which adds memoizing to the parsing logic. + Repeated parse attempts at the same string location (which happens + often in many complex grammars) can immediately return a cached value, + instead of re-executing parsing/validating code. Memoizing is done of + both valid results and parsing exceptions. + + This speedup may break existing programs that use parse actions that + have side-effects. For this reason, packrat parsing is disabled when + you first import pyparsing. To activate the packrat feature, your + program must call the class method C{ParserElement.enablePackrat()}. If + your program uses C{psyco} to "compile as you go", you must call + C{enablePackrat} before calling C{psyco.full()}. If you do not do this, + Python will crash. For best results, call C{enablePackrat()} immediately + after importing pyparsing. + """ + if not ParserElement._packratEnabled: + ParserElement._packratEnabled = True + ParserElement._parse = ParserElement._parseCache + enablePackrat = staticmethod(enablePackrat) + + def parseString( self, instring, parseAll=False ): + """Execute the parse expression with the given string. + This is the main interface to the client code, once the complete + expression has been built. + + If you want the grammar to require that the entire input string be + successfully parsed, then set C{parseAll} to True (equivalent to ending + the grammar with C{StringEnd()}). + + Note: C{parseString} implicitly calls C{expandtabs()} on the input string, + in order to report proper column numbers in parse actions. + If the input string contains tabs and + the grammar uses parse actions that use the C{loc} argument to index into the + string being parsed, you can ensure you have a consistent view of the input + string by: + - calling C{parseWithTabs} on your grammar before calling C{parseString} + (see L{I{parseWithTabs}}) + - define your parse action using the full C{(s,loc,toks)} signature, and + reference the input string using the parse action's C{s} argument + - explictly expand the tabs in your input string before calling + C{parseString} + """ + ParserElement.resetCache() + if not self.streamlined: + self.streamline() + #~ self.saveAsList = True + for e in self.ignoreExprs: + e.streamline() + if not self.keepTabs: + instring = instring.expandtabs() + try: + loc, tokens = self._parse( instring, 0 ) + if parseAll: + loc = self.preParse( instring, loc ) + se = Empty() + StringEnd() + se._parse( instring, loc ) + except ParseBaseException: + if ParserElement.verbose_stacktrace: + raise + else: + # catch and re-raise exception from here, clears out pyparsing internal stack trace + exc = sys.exc_info()[1] + raise exc + else: + return tokens + + def scanString( self, instring, maxMatches=_MAX_INT, overlap=False ): + """Scan the input string for expression matches. Each match will return the + matching tokens, start location, and end location. May be called with optional + C{maxMatches} argument, to clip scanning after 'n' matches are found. If + C{overlap} is specified, then overlapping matches will be reported. + + Note that the start and end locations are reported relative to the string + being parsed. See L{I{parseString}} for more information on parsing + strings with embedded tabs.""" + if not self.streamlined: + self.streamline() + for e in self.ignoreExprs: + e.streamline() + + if not self.keepTabs: + instring = _ustr(instring).expandtabs() + instrlen = len(instring) + loc = 0 + preparseFn = self.preParse + parseFn = self._parse + ParserElement.resetCache() + matches = 0 + try: + while loc <= instrlen and matches < maxMatches: + try: + preloc = preparseFn( instring, loc ) + nextLoc,tokens = parseFn( instring, preloc, callPreParse=False ) + except ParseException: + loc = preloc+1 + else: + if nextLoc > loc: + matches += 1 + yield tokens, preloc, nextLoc + if overlap: + nextloc = preparseFn( instring, loc ) + if nextloc > loc: + loc = nextLoc + else: + loc += 1 + else: + loc = nextLoc + else: + loc = preloc+1 + except ParseBaseException: + if ParserElement.verbose_stacktrace: + raise + else: + # catch and re-raise exception from here, clears out pyparsing internal stack trace + exc = sys.exc_info()[1] + raise exc + + def transformString( self, instring ): + """Extension to C{scanString}, to modify matching text with modified tokens that may + be returned from a parse action. To use C{transformString}, define a grammar and + attach a parse action to it that modifies the returned token list. + Invoking C{transformString()} on a target string will then scan for matches, + and replace the matched text patterns according to the logic in the parse + action. C{transformString()} returns the resulting transformed string.""" + out = [] + lastE = 0 + # force preservation of s, to minimize unwanted transformation of string, and to + # keep string locs straight between transformString and scanString + self.keepTabs = True + try: + for t,s,e in self.scanString( instring ): + out.append( instring[lastE:s] ) + if t: + if isinstance(t,ParseResults): + out += t.asList() + elif isinstance(t,list): + out += t + else: + out.append(t) + lastE = e + out.append(instring[lastE:]) + out = [o for o in out if o] + return "".join(map(_ustr,_flatten(out))) + except ParseBaseException: + if ParserElement.verbose_stacktrace: + raise + else: + # catch and re-raise exception from here, clears out pyparsing internal stack trace + exc = sys.exc_info()[1] + raise exc + + def searchString( self, instring, maxMatches=_MAX_INT ): + """Another extension to C{scanString}, simplifying the access to the tokens found + to match the given parse expression. May be called with optional + C{maxMatches} argument, to clip searching after 'n' matches are found. + """ + try: + return ParseResults([ t for t,s,e in self.scanString( instring, maxMatches ) ]) + except ParseBaseException: + if ParserElement.verbose_stacktrace: + raise + else: + # catch and re-raise exception from here, clears out pyparsing internal stack trace + exc = sys.exc_info()[1] + raise exc + + def __add__(self, other ): + """Implementation of + operator - returns And""" + if isinstance( other, basestring ): + other = Literal( other ) + if not isinstance( other, ParserElement ): + warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, stacklevel=2) + return None + return And( [ self, other ] ) + + def __radd__(self, other ): + """Implementation of + operator when left operand is not a C{ParserElement}""" + if isinstance( other, basestring ): + other = Literal( other ) + if not isinstance( other, ParserElement ): + warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, stacklevel=2) + return None + return other + self + + def __sub__(self, other): + """Implementation of - operator, returns C{And} with error stop""" + if isinstance( other, basestring ): + other = Literal( other ) + if not isinstance( other, ParserElement ): + warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, stacklevel=2) + return None + return And( [ self, And._ErrorStop(), other ] ) + + def __rsub__(self, other ): + """Implementation of - operator when left operand is not a C{ParserElement}""" + if isinstance( other, basestring ): + other = Literal( other ) + if not isinstance( other, ParserElement ): + warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, stacklevel=2) + return None + return other - self + + def __mul__(self,other): + """Implementation of * operator, allows use of C{expr * 3} in place of + C{expr + expr + expr}. Expressions may also me multiplied by a 2-integer + tuple, similar to C{{min,max}} multipliers in regular expressions. Tuples + may also include C{None} as in: + - C{expr*(n,None)} or C{expr*(n,)} is equivalent + to C{expr*n + ZeroOrMore(expr)} + (read as "at least n instances of C{expr}") + - C{expr*(None,n)} is equivalent to C{expr*(0,n)} + (read as "0 to n instances of C{expr}") + - C{expr*(None,None)} is equivalent to C{ZeroOrMore(expr)} + - C{expr*(1,None)} is equivalent to C{OneOrMore(expr)} + + Note that C{expr*(None,n)} does not raise an exception if + more than n exprs exist in the input stream; that is, + C{expr*(None,n)} does not enforce a maximum number of expr + occurrences. If this behavior is desired, then write + C{expr*(None,n) + ~expr} + + """ + if isinstance(other,int): + minElements, optElements = other,0 + elif isinstance(other,tuple): + other = (other + (None, None))[:2] + if other[0] is None: + other = (0, other[1]) + if isinstance(other[0],int) and other[1] is None: + if other[0] == 0: + return ZeroOrMore(self) + if other[0] == 1: + return OneOrMore(self) + else: + return self*other[0] + ZeroOrMore(self) + elif isinstance(other[0],int) and isinstance(other[1],int): + minElements, optElements = other + optElements -= minElements + else: + raise TypeError("cannot multiply 'ParserElement' and ('%s','%s') objects", type(other[0]),type(other[1])) + else: + raise TypeError("cannot multiply 'ParserElement' and '%s' objects", type(other)) + + if minElements < 0: + raise ValueError("cannot multiply ParserElement by negative value") + if optElements < 0: + raise ValueError("second tuple value must be greater or equal to first tuple value") + if minElements == optElements == 0: + raise ValueError("cannot multiply ParserElement by 0 or (0,0)") + + if (optElements): + def makeOptionalList(n): + if n>1: + return Optional(self + makeOptionalList(n-1)) + else: + return Optional(self) + if minElements: + if minElements == 1: + ret = self + makeOptionalList(optElements) + else: + ret = And([self]*minElements) + makeOptionalList(optElements) + else: + ret = makeOptionalList(optElements) + else: + if minElements == 1: + ret = self + else: + ret = And([self]*minElements) + return ret + + def __rmul__(self, other): + return self.__mul__(other) + + def __or__(self, other ): + """Implementation of | operator - returns C{MatchFirst}""" + if isinstance( other, basestring ): + other = Literal( other ) + if not isinstance( other, ParserElement ): + warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, stacklevel=2) + return None + return MatchFirst( [ self, other ] ) + + def __ror__(self, other ): + """Implementation of | operator when left operand is not a C{ParserElement}""" + if isinstance( other, basestring ): + other = Literal( other ) + if not isinstance( other, ParserElement ): + warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, stacklevel=2) + return None + return other | self + + def __xor__(self, other ): + """Implementation of ^ operator - returns C{Or}""" + if isinstance( other, basestring ): + other = Literal( other ) + if not isinstance( other, ParserElement ): + warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, stacklevel=2) + return None + return Or( [ self, other ] ) + + def __rxor__(self, other ): + """Implementation of ^ operator when left operand is not a C{ParserElement}""" + if isinstance( other, basestring ): + other = Literal( other ) + if not isinstance( other, ParserElement ): + warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, stacklevel=2) + return None + return other ^ self + + def __and__(self, other ): + """Implementation of & operator - returns C{Each}""" + if isinstance( other, basestring ): + other = Literal( other ) + if not isinstance( other, ParserElement ): + warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, stacklevel=2) + return None + return Each( [ self, other ] ) + + def __rand__(self, other ): + """Implementation of & operator when left operand is not a C{ParserElement}""" + if isinstance( other, basestring ): + other = Literal( other ) + if not isinstance( other, ParserElement ): + warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, stacklevel=2) + return None + return other & self + + def __invert__( self ): + """Implementation of ~ operator - returns C{NotAny}""" + return NotAny( self ) + + def __call__(self, name): + """Shortcut for C{setResultsName}, with C{listAllMatches=default}:: + userdata = Word(alphas).setResultsName("name") + Word(nums+"-").setResultsName("socsecno") + could be written as:: + userdata = Word(alphas)("name") + Word(nums+"-")("socsecno") + + If C{name} is given with a trailing C{'*'} character, then C{listAllMatches} will be + passed as C{True}. + """ + return self.setResultsName(name) + + def suppress( self ): + """Suppresses the output of this C{ParserElement}; useful to keep punctuation from + cluttering up returned output. + """ + return Suppress( self ) + + def leaveWhitespace( self ): + """Disables the skipping of whitespace before matching the characters in the + C{ParserElement}'s defined pattern. This is normally only used internally by + the pyparsing module, but may be needed in some whitespace-sensitive grammars. + """ + self.skipWhitespace = False + return self + + def setWhitespaceChars( self, chars ): + """Overrides the default whitespace chars + """ + self.skipWhitespace = True + self.whiteChars = chars + self.copyDefaultWhiteChars = False + return self + + def parseWithTabs( self ): + """Overrides default behavior to expand C{}s to spaces before parsing the input string. + Must be called before C{parseString} when the input grammar contains elements that + match C{} characters.""" + self.keepTabs = True + return self + + def ignore( self, other ): + """Define expression to be ignored (e.g., comments) while doing pattern + matching; may be called repeatedly, to define multiple comment or other + ignorable patterns. + """ + if isinstance( other, Suppress ): + if other not in self.ignoreExprs: + self.ignoreExprs.append( other.copy() ) + else: + self.ignoreExprs.append( Suppress( other.copy() ) ) + return self + + def setDebugActions( self, startAction, successAction, exceptionAction ): + """Enable display of debugging messages while doing pattern matching.""" + self.debugActions = (startAction or _defaultStartDebugAction, + successAction or _defaultSuccessDebugAction, + exceptionAction or _defaultExceptionDebugAction) + self.debug = True + return self + + def setDebug( self, flag=True ): + """Enable display of debugging messages while doing pattern matching. + Set C{flag} to True to enable, False to disable.""" + if flag: + self.setDebugActions( _defaultStartDebugAction, _defaultSuccessDebugAction, _defaultExceptionDebugAction ) + else: + self.debug = False + return self + + def __str__( self ): + return self.name + + def __repr__( self ): + return _ustr(self) + + def streamline( self ): + self.streamlined = True + self.strRepr = None + return self + + def checkRecursion( self, parseElementList ): + pass + + def validate( self, validateTrace=[] ): + """Check defined expressions for valid structure, check for infinite recursive definitions.""" + self.checkRecursion( [] ) + + def parseFile( self, file_or_filename, parseAll=False ): + """Execute the parse expression on the given file or filename. + If a filename is specified (instead of a file object), + the entire file is opened, read, and closed before parsing. + """ + try: + file_contents = file_or_filename.read() + except AttributeError: + f = open(file_or_filename, "rb") + file_contents = f.read() + f.close() + try: + return self.parseString(file_contents, parseAll) + except ParseBaseException: + # catch and re-raise exception from here, clears out pyparsing internal stack trace + exc = sys.exc_info()[1] + raise exc + + def getException(self): + return ParseException("",0,self.errmsg,self) + + def __getattr__(self,aname): + if aname == "myException": + self.myException = ret = self.getException(); + return ret; + else: + raise AttributeError("no such attribute " + aname) + + def __eq__(self,other): + if isinstance(other, ParserElement): + return self is other or self.__dict__ == other.__dict__ + elif isinstance(other, basestring): + try: + self.parseString(_ustr(other), parseAll=True) + return True + except ParseBaseException: + return False + else: + return super(ParserElement,self)==other + + def __ne__(self,other): + return not (self == other) + + def __hash__(self): + return hash(id(self)) + + def __req__(self,other): + return self == other + + def __rne__(self,other): + return not (self == other) + + +class Token(ParserElement): + """Abstract C{ParserElement} subclass, for defining atomic matching patterns.""" + def __init__( self ): + super(Token,self).__init__( savelist=False ) + + def setName(self, name): + s = super(Token,self).setName(name) + self.errmsg = "Expected " + self.name + return s + + +class Empty(Token): + """An empty token, will always match.""" + def __init__( self ): + super(Empty,self).__init__() + self.name = "Empty" + self.mayReturnEmpty = True + self.mayIndexError = False + + +class NoMatch(Token): + """A token that will never match.""" + def __init__( self ): + super(NoMatch,self).__init__() + self.name = "NoMatch" + self.mayReturnEmpty = True + self.mayIndexError = False + self.errmsg = "Unmatchable token" + + def parseImpl( self, instring, loc, doActions=True ): + exc = self.myException + exc.loc = loc + exc.pstr = instring + raise exc + + +class Literal(Token): + """Token to exactly match a specified string.""" + def __init__( self, matchString ): + super(Literal,self).__init__() + self.match = matchString + self.matchLen = len(matchString) + try: + self.firstMatchChar = matchString[0] + except IndexError: + warnings.warn("null string passed to Literal; use Empty() instead", + SyntaxWarning, stacklevel=2) + self.__class__ = Empty + self.name = '"%s"' % _ustr(self.match) + self.errmsg = "Expected " + self.name + self.mayReturnEmpty = False + self.mayIndexError = False + + # Performance tuning: this routine gets called a *lot* + # if this is a single character match string and the first character matches, + # short-circuit as quickly as possible, and avoid calling startswith + #~ @profile + def parseImpl( self, instring, loc, doActions=True ): + if (instring[loc] == self.firstMatchChar and + (self.matchLen==1 or instring.startswith(self.match,loc)) ): + return loc+self.matchLen, self.match + #~ raise ParseException( instring, loc, self.errmsg ) + exc = self.myException + exc.loc = loc + exc.pstr = instring + raise exc +_L = Literal + +class Keyword(Token): + """Token to exactly match a specified string as a keyword, that is, it must be + immediately followed by a non-keyword character. Compare with C{Literal}:: + Literal("if") will match the leading C{'if'} in C{'ifAndOnlyIf'}. + Keyword("if") will not; it will only match the leading C{'if'} in C{'if x=1'}, or C{'if(y==2)'} + Accepts two optional constructor arguments in addition to the keyword string: + C{identChars} is a string of characters that would be valid identifier characters, + defaulting to all alphanumerics + "_" and "$"; C{caseless} allows case-insensitive + matching, default is C{False}. + """ + DEFAULT_KEYWORD_CHARS = alphanums+"_$" + + def __init__( self, matchString, identChars=DEFAULT_KEYWORD_CHARS, caseless=False ): + super(Keyword,self).__init__() + self.match = matchString + self.matchLen = len(matchString) + try: + self.firstMatchChar = matchString[0] + except IndexError: + warnings.warn("null string passed to Keyword; use Empty() instead", + SyntaxWarning, stacklevel=2) + self.name = '"%s"' % self.match + self.errmsg = "Expected " + self.name + self.mayReturnEmpty = False + self.mayIndexError = False + self.caseless = caseless + if caseless: + self.caselessmatch = matchString.upper() + identChars = identChars.upper() + self.identChars = set(identChars) + + def parseImpl( self, instring, loc, doActions=True ): + if self.caseless: + if ( (instring[ loc:loc+self.matchLen ].upper() == self.caselessmatch) and + (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen].upper() not in self.identChars) and + (loc == 0 or instring[loc-1].upper() not in self.identChars) ): + return loc+self.matchLen, self.match + else: + if (instring[loc] == self.firstMatchChar and + (self.matchLen==1 or instring.startswith(self.match,loc)) and + (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen] not in self.identChars) and + (loc == 0 or instring[loc-1] not in self.identChars) ): + return loc+self.matchLen, self.match + #~ raise ParseException( instring, loc, self.errmsg ) + exc = self.myException + exc.loc = loc + exc.pstr = instring + raise exc + + def copy(self): + c = super(Keyword,self).copy() + c.identChars = Keyword.DEFAULT_KEYWORD_CHARS + return c + + def setDefaultKeywordChars( chars ): + """Overrides the default Keyword chars + """ + Keyword.DEFAULT_KEYWORD_CHARS = chars + setDefaultKeywordChars = staticmethod(setDefaultKeywordChars) + +class CaselessLiteral(Literal): + """Token to match a specified string, ignoring case of letters. + Note: the matched results will always be in the case of the given + match string, NOT the case of the input text. + """ + def __init__( self, matchString ): + super(CaselessLiteral,self).__init__( matchString.upper() ) + # Preserve the defining literal. + self.returnString = matchString + self.name = "'%s'" % self.returnString + self.errmsg = "Expected " + self.name + + def parseImpl( self, instring, loc, doActions=True ): + if instring[ loc:loc+self.matchLen ].upper() == self.match: + return loc+self.matchLen, self.returnString + #~ raise ParseException( instring, loc, self.errmsg ) + exc = self.myException + exc.loc = loc + exc.pstr = instring + raise exc + +class CaselessKeyword(Keyword): + def __init__( self, matchString, identChars=Keyword.DEFAULT_KEYWORD_CHARS ): + super(CaselessKeyword,self).__init__( matchString, identChars, caseless=True ) + + def parseImpl( self, instring, loc, doActions=True ): + if ( (instring[ loc:loc+self.matchLen ].upper() == self.caselessmatch) and + (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen].upper() not in self.identChars) ): + return loc+self.matchLen, self.match + #~ raise ParseException( instring, loc, self.errmsg ) + exc = self.myException + exc.loc = loc + exc.pstr = instring + raise exc + +class Word(Token): + """Token for matching words composed of allowed character sets. + Defined with string containing all allowed initial characters, + an optional string containing allowed body characters (if omitted, + defaults to the initial character set), and an optional minimum, + maximum, and/or exact length. The default value for C{min} is 1 (a + minimum value < 1 is not valid); the default values for C{max} and C{exact} + are 0, meaning no maximum or exact length restriction. An optional + C{exclude} parameter can list characters that might be found in + the input C{bodyChars} string; useful to define a word of all printables + except for one or two characters, for instance. + """ + def __init__( self, initChars, bodyChars=None, min=1, max=0, exact=0, asKeyword=False, excludeChars=None ): + super(Word,self).__init__() + if excludeChars: + initChars = ''.join([c for c in initChars if c not in excludeChars]) + if bodyChars: + bodyChars = ''.join([c for c in bodyChars if c not in excludeChars]) + self.initCharsOrig = initChars + self.initChars = set(initChars) + if bodyChars : + self.bodyCharsOrig = bodyChars + self.bodyChars = set(bodyChars) + else: + self.bodyCharsOrig = initChars + self.bodyChars = set(initChars) + + self.maxSpecified = max > 0 + + if min < 1: + raise ValueError("cannot specify a minimum length < 1; use Optional(Word()) if zero-length word is permitted") + + self.minLen = min + + if max > 0: + self.maxLen = max + else: + self.maxLen = _MAX_INT + + if exact > 0: + self.maxLen = exact + self.minLen = exact + + self.name = _ustr(self) + self.errmsg = "Expected " + self.name + self.mayIndexError = False + self.asKeyword = asKeyword + + if ' ' not in self.initCharsOrig+self.bodyCharsOrig and (min==1 and max==0 and exact==0): + if self.bodyCharsOrig == self.initCharsOrig: + self.reString = "[%s]+" % _escapeRegexRangeChars(self.initCharsOrig) + elif len(self.bodyCharsOrig) == 1: + self.reString = "%s[%s]*" % \ + (re.escape(self.initCharsOrig), + _escapeRegexRangeChars(self.bodyCharsOrig),) + else: + self.reString = "[%s][%s]*" % \ + (_escapeRegexRangeChars(self.initCharsOrig), + _escapeRegexRangeChars(self.bodyCharsOrig),) + if self.asKeyword: + self.reString = r"\b"+self.reString+r"\b" + try: + self.re = re.compile( self.reString ) + except: + self.re = None + + def parseImpl( self, instring, loc, doActions=True ): + if self.re: + result = self.re.match(instring,loc) + if not result: + exc = self.myException + exc.loc = loc + exc.pstr = instring + raise exc + + loc = result.end() + return loc, result.group() + + if not(instring[ loc ] in self.initChars): + #~ raise ParseException( instring, loc, self.errmsg ) + exc = self.myException + exc.loc = loc + exc.pstr = instring + raise exc + start = loc + loc += 1 + instrlen = len(instring) + bodychars = self.bodyChars + maxloc = start + self.maxLen + maxloc = min( maxloc, instrlen ) + while loc < maxloc and instring[loc] in bodychars: + loc += 1 + + throwException = False + if loc - start < self.minLen: + throwException = True + if self.maxSpecified and loc < instrlen and instring[loc] in bodychars: + throwException = True + if self.asKeyword: + if (start>0 and instring[start-1] in bodychars) or (loc4: + return s[:4]+"..." + else: + return s + + if ( self.initCharsOrig != self.bodyCharsOrig ): + self.strRepr = "W:(%s,%s)" % ( charsAsStr(self.initCharsOrig), charsAsStr(self.bodyCharsOrig) ) + else: + self.strRepr = "W:(%s)" % charsAsStr(self.initCharsOrig) + + return self.strRepr + + +class Regex(Token): + """Token for matching strings that match a given regular expression. + Defined with string specifying the regular expression in a form recognized by the inbuilt Python re module. + """ + compiledREtype = type(re.compile("[A-Z]")) + def __init__( self, pattern, flags=0): + """The parameters C{pattern} and C{flags} are passed to the C{re.compile()} function as-is. See the Python C{re} module for an explanation of the acceptable patterns and flags.""" + super(Regex,self).__init__() + + if isinstance(pattern, basestring): + if len(pattern) == 0: + warnings.warn("null string passed to Regex; use Empty() instead", + SyntaxWarning, stacklevel=2) + + self.pattern = pattern + self.flags = flags + + try: + self.re = re.compile(self.pattern, self.flags) + self.reString = self.pattern + except sre_constants.error: + warnings.warn("invalid pattern (%s) passed to Regex" % pattern, + SyntaxWarning, stacklevel=2) + raise + + elif isinstance(pattern, Regex.compiledREtype): + self.re = pattern + self.pattern = \ + self.reString = str(pattern) + self.flags = flags + + else: + raise ValueError("Regex may only be constructed with a string or a compiled RE object") + + self.name = _ustr(self) + self.errmsg = "Expected " + self.name + self.mayIndexError = False + self.mayReturnEmpty = True + + def parseImpl( self, instring, loc, doActions=True ): + result = self.re.match(instring,loc) + if not result: + exc = self.myException + exc.loc = loc + exc.pstr = instring + raise exc + + loc = result.end() + d = result.groupdict() + ret = ParseResults(result.group()) + if d: + for k in d: + ret[k] = d[k] + return loc,ret + + def __str__( self ): + try: + return super(Regex,self).__str__() + except: + pass + + if self.strRepr is None: + self.strRepr = "Re:(%s)" % repr(self.pattern) + + return self.strRepr + + +class QuotedString(Token): + """Token for matching strings that are delimited by quoting characters. + """ + def __init__( self, quoteChar, escChar=None, escQuote=None, multiline=False, unquoteResults=True, endQuoteChar=None): + """ + Defined with the following parameters: + - quoteChar - string of one or more characters defining the quote delimiting string + - escChar - character to escape quotes, typically backslash (default=None) + - escQuote - special quote sequence to escape an embedded quote string (such as SQL's "" to escape an embedded ") (default=None) + - multiline - boolean indicating whether quotes can span multiple lines (default=False) + - unquoteResults - boolean indicating whether the matched text should be unquoted (default=True) + - endQuoteChar - string of one or more characters defining the end of the quote delimited string (default=None => same as quoteChar) + """ + super(QuotedString,self).__init__() + + # remove white space from quote chars - wont work anyway + quoteChar = quoteChar.strip() + if len(quoteChar) == 0: + warnings.warn("quoteChar cannot be the empty string",SyntaxWarning,stacklevel=2) + raise SyntaxError() + + if endQuoteChar is None: + endQuoteChar = quoteChar + else: + endQuoteChar = endQuoteChar.strip() + if len(endQuoteChar) == 0: + warnings.warn("endQuoteChar cannot be the empty string",SyntaxWarning,stacklevel=2) + raise SyntaxError() + + self.quoteChar = quoteChar + self.quoteCharLen = len(quoteChar) + self.firstQuoteChar = quoteChar[0] + self.endQuoteChar = endQuoteChar + self.endQuoteCharLen = len(endQuoteChar) + self.escChar = escChar + self.escQuote = escQuote + self.unquoteResults = unquoteResults + + if multiline: + self.flags = re.MULTILINE | re.DOTALL + self.pattern = r'%s(?:[^%s%s]' % \ + ( re.escape(self.quoteChar), + _escapeRegexRangeChars(self.endQuoteChar[0]), + (escChar is not None and _escapeRegexRangeChars(escChar) or '') ) + else: + self.flags = 0 + self.pattern = r'%s(?:[^%s\n\r%s]' % \ + ( re.escape(self.quoteChar), + _escapeRegexRangeChars(self.endQuoteChar[0]), + (escChar is not None and _escapeRegexRangeChars(escChar) or '') ) + if len(self.endQuoteChar) > 1: + self.pattern += ( + '|(?:' + ')|(?:'.join(["%s[^%s]" % (re.escape(self.endQuoteChar[:i]), + _escapeRegexRangeChars(self.endQuoteChar[i])) + for i in range(len(self.endQuoteChar)-1,0,-1)]) + ')' + ) + if escQuote: + self.pattern += (r'|(?:%s)' % re.escape(escQuote)) + if escChar: + self.pattern += (r'|(?:%s.)' % re.escape(escChar)) + charset = ''.join(set(self.quoteChar[0]+self.endQuoteChar[0])).replace('^',r'\^').replace('-',r'\-') + self.escCharReplacePattern = re.escape(self.escChar)+("([%s])" % charset) + self.pattern += (r')*%s' % re.escape(self.endQuoteChar)) + + try: + self.re = re.compile(self.pattern, self.flags) + self.reString = self.pattern + except sre_constants.error: + warnings.warn("invalid pattern (%s) passed to Regex" % self.pattern, + SyntaxWarning, stacklevel=2) + raise + + self.name = _ustr(self) + self.errmsg = "Expected " + self.name + self.mayIndexError = False + self.mayReturnEmpty = True + + def parseImpl( self, instring, loc, doActions=True ): + result = instring[loc] == self.firstQuoteChar and self.re.match(instring,loc) or None + if not result: + exc = self.myException + exc.loc = loc + exc.pstr = instring + raise exc + + loc = result.end() + ret = result.group() + + if self.unquoteResults: + + # strip off quotes + ret = ret[self.quoteCharLen:-self.endQuoteCharLen] + + if isinstance(ret,basestring): + # replace escaped characters + if self.escChar: + ret = re.sub(self.escCharReplacePattern,"\g<1>",ret) + + # replace escaped quotes + if self.escQuote: + ret = ret.replace(self.escQuote, self.endQuoteChar) + + return loc, ret + + def __str__( self ): + try: + return super(QuotedString,self).__str__() + except: + pass + + if self.strRepr is None: + self.strRepr = "quoted string, starting with %s ending with %s" % (self.quoteChar, self.endQuoteChar) + + return self.strRepr + + +class CharsNotIn(Token): + """Token for matching words composed of characters *not* in a given set. + Defined with string containing all disallowed characters, and an optional + minimum, maximum, and/or exact length. The default value for C{min} is 1 (a + minimum value < 1 is not valid); the default values for C{max} and C{exact} + are 0, meaning no maximum or exact length restriction. + """ + def __init__( self, notChars, min=1, max=0, exact=0 ): + super(CharsNotIn,self).__init__() + self.skipWhitespace = False + self.notChars = notChars + + if min < 1: + raise ValueError("cannot specify a minimum length < 1; use Optional(CharsNotIn()) if zero-length char group is permitted") + + self.minLen = min + + if max > 0: + self.maxLen = max + else: + self.maxLen = _MAX_INT + + if exact > 0: + self.maxLen = exact + self.minLen = exact + + self.name = _ustr(self) + self.errmsg = "Expected " + self.name + self.mayReturnEmpty = ( self.minLen == 0 ) + self.mayIndexError = False + + def parseImpl( self, instring, loc, doActions=True ): + if instring[loc] in self.notChars: + #~ raise ParseException( instring, loc, self.errmsg ) + exc = self.myException + exc.loc = loc + exc.pstr = instring + raise exc + + start = loc + loc += 1 + notchars = self.notChars + maxlen = min( start+self.maxLen, len(instring) ) + while loc < maxlen and \ + (instring[loc] not in notchars): + loc += 1 + + if loc - start < self.minLen: + #~ raise ParseException( instring, loc, self.errmsg ) + exc = self.myException + exc.loc = loc + exc.pstr = instring + raise exc + + return loc, instring[start:loc] + + def __str__( self ): + try: + return super(CharsNotIn, self).__str__() + except: + pass + + if self.strRepr is None: + if len(self.notChars) > 4: + self.strRepr = "!W:(%s...)" % self.notChars[:4] + else: + self.strRepr = "!W:(%s)" % self.notChars + + return self.strRepr + +class White(Token): + """Special matching class for matching whitespace. Normally, whitespace is ignored + by pyparsing grammars. This class is included when some whitespace structures + are significant. Define with a string containing the whitespace characters to be + matched; default is C{" \\t\\r\\n"}. Also takes optional C{min}, C{max}, and C{exact} arguments, + as defined for the C{Word} class.""" + whiteStrs = { + " " : "", + "\t": "", + "\n": "", + "\r": "", + "\f": "", + } + def __init__(self, ws=" \t\r\n", min=1, max=0, exact=0): + super(White,self).__init__() + self.matchWhite = ws + self.setWhitespaceChars( "".join([c for c in self.whiteChars if c not in self.matchWhite]) ) + #~ self.leaveWhitespace() + self.name = ("".join([White.whiteStrs[c] for c in self.matchWhite])) + self.mayReturnEmpty = True + self.errmsg = "Expected " + self.name + + self.minLen = min + + if max > 0: + self.maxLen = max + else: + self.maxLen = _MAX_INT + + if exact > 0: + self.maxLen = exact + self.minLen = exact + + def parseImpl( self, instring, loc, doActions=True ): + if not(instring[ loc ] in self.matchWhite): + #~ raise ParseException( instring, loc, self.errmsg ) + exc = self.myException + exc.loc = loc + exc.pstr = instring + raise exc + start = loc + loc += 1 + maxloc = start + self.maxLen + maxloc = min( maxloc, len(instring) ) + while loc < maxloc and instring[loc] in self.matchWhite: + loc += 1 + + if loc - start < self.minLen: + #~ raise ParseException( instring, loc, self.errmsg ) + exc = self.myException + exc.loc = loc + exc.pstr = instring + raise exc + + return loc, instring[start:loc] + + +class _PositionToken(Token): + def __init__( self ): + super(_PositionToken,self).__init__() + self.name=self.__class__.__name__ + self.mayReturnEmpty = True + self.mayIndexError = False + +class GoToColumn(_PositionToken): + """Token to advance to a specific column of input text; useful for tabular report scraping.""" + def __init__( self, colno ): + super(GoToColumn,self).__init__() + self.col = colno + + def preParse( self, instring, loc ): + if col(loc,instring) != self.col: + instrlen = len(instring) + if self.ignoreExprs: + loc = self._skipIgnorables( instring, loc ) + while loc < instrlen and instring[loc].isspace() and col( loc, instring ) != self.col : + loc += 1 + return loc + + def parseImpl( self, instring, loc, doActions=True ): + thiscol = col( loc, instring ) + if thiscol > self.col: + raise ParseException( instring, loc, "Text not in expected column", self ) + newloc = loc + self.col - thiscol + ret = instring[ loc: newloc ] + return newloc, ret + +class LineStart(_PositionToken): + """Matches if current position is at the beginning of a line within the parse string""" + def __init__( self ): + super(LineStart,self).__init__() + self.setWhitespaceChars( ParserElement.DEFAULT_WHITE_CHARS.replace("\n","") ) + self.errmsg = "Expected start of line" + + def preParse( self, instring, loc ): + preloc = super(LineStart,self).preParse(instring,loc) + if instring[preloc] == "\n": + loc += 1 + return loc + + def parseImpl( self, instring, loc, doActions=True ): + if not( loc==0 or + (loc == self.preParse( instring, 0 )) or + (instring[loc-1] == "\n") ): #col(loc, instring) != 1: + #~ raise ParseException( instring, loc, "Expected start of line" ) + exc = self.myException + exc.loc = loc + exc.pstr = instring + raise exc + return loc, [] + +class LineEnd(_PositionToken): + """Matches if current position is at the end of a line within the parse string""" + def __init__( self ): + super(LineEnd,self).__init__() + self.setWhitespaceChars( ParserElement.DEFAULT_WHITE_CHARS.replace("\n","") ) + self.errmsg = "Expected end of line" + + def parseImpl( self, instring, loc, doActions=True ): + if loc len(instring): + return loc, [] + else: + exc = self.myException + exc.loc = loc + exc.pstr = instring + raise exc + +class WordStart(_PositionToken): + """Matches if the current position is at the beginning of a Word, and + is not preceded by any character in a given set of C{wordChars} + (default=C{printables}). To emulate the C{\b} behavior of regular expressions, + use C{WordStart(alphanums)}. C{WordStart} will also match at the beginning of + the string being parsed, or at the beginning of a line. + """ + def __init__(self, wordChars = printables): + super(WordStart,self).__init__() + self.wordChars = set(wordChars) + self.errmsg = "Not at the start of a word" + + def parseImpl(self, instring, loc, doActions=True ): + if loc != 0: + if (instring[loc-1] in self.wordChars or + instring[loc] not in self.wordChars): + exc = self.myException + exc.loc = loc + exc.pstr = instring + raise exc + return loc, [] + +class WordEnd(_PositionToken): + """Matches if the current position is at the end of a Word, and + is not followed by any character in a given set of C{wordChars} + (default=C{printables}). To emulate the C{\b} behavior of regular expressions, + use C{WordEnd(alphanums)}. C{WordEnd} will also match at the end of + the string being parsed, or at the end of a line. + """ + def __init__(self, wordChars = printables): + super(WordEnd,self).__init__() + self.wordChars = set(wordChars) + self.skipWhitespace = False + self.errmsg = "Not at the end of a word" + + def parseImpl(self, instring, loc, doActions=True ): + instrlen = len(instring) + if instrlen>0 and loc maxExcLoc: + maxException = err + maxExcLoc = err.loc + except IndexError: + if len(instring) > maxExcLoc: + maxException = ParseException(instring,len(instring),e.errmsg,self) + maxExcLoc = len(instring) + else: + if loc2 > maxMatchLoc: + maxMatchLoc = loc2 + maxMatchExp = e + + if maxMatchLoc < 0: + if maxException is not None: + raise maxException + else: + raise ParseException(instring, loc, "no defined alternatives to match", self) + + return maxMatchExp._parse( instring, loc, doActions ) + + def __ixor__(self, other ): + if isinstance( other, basestring ): + other = Literal( other ) + return self.append( other ) #Or( [ self, other ] ) + + def __str__( self ): + if hasattr(self,"name"): + return self.name + + if self.strRepr is None: + self.strRepr = "{" + " ^ ".join( [ _ustr(e) for e in self.exprs ] ) + "}" + + return self.strRepr + + def checkRecursion( self, parseElementList ): + subRecCheckList = parseElementList[:] + [ self ] + for e in self.exprs: + e.checkRecursion( subRecCheckList ) + + +class MatchFirst(ParseExpression): + """Requires that at least one C{ParseExpression} is found. + If two expressions match, the first one listed is the one that will match. + May be constructed using the C{'|'} operator. + """ + def __init__( self, exprs, savelist = False ): + super(MatchFirst,self).__init__(exprs, savelist) + if exprs: + self.mayReturnEmpty = False + for e in self.exprs: + if e.mayReturnEmpty: + self.mayReturnEmpty = True + break + else: + self.mayReturnEmpty = True + + def parseImpl( self, instring, loc, doActions=True ): + maxExcLoc = -1 + maxException = None + for e in self.exprs: + try: + ret = e._parse( instring, loc, doActions ) + return ret + except ParseException, err: + if err.loc > maxExcLoc: + maxException = err + maxExcLoc = err.loc + except IndexError: + if len(instring) > maxExcLoc: + maxException = ParseException(instring,len(instring),e.errmsg,self) + maxExcLoc = len(instring) + + # only got here if no expression matched, raise exception for match that made it the furthest + else: + if maxException is not None: + raise maxException + else: + raise ParseException(instring, loc, "no defined alternatives to match", self) + + def __ior__(self, other ): + if isinstance( other, basestring ): + other = Literal( other ) + return self.append( other ) #MatchFirst( [ self, other ] ) + + def __str__( self ): + if hasattr(self,"name"): + return self.name + + if self.strRepr is None: + self.strRepr = "{" + " | ".join( [ _ustr(e) for e in self.exprs ] ) + "}" + + return self.strRepr + + def checkRecursion( self, parseElementList ): + subRecCheckList = parseElementList[:] + [ self ] + for e in self.exprs: + e.checkRecursion( subRecCheckList ) + + +class Each(ParseExpression): + """Requires all given C{ParseExpression}s to be found, but in any order. + Expressions may be separated by whitespace. + May be constructed using the C{'&'} operator. + """ + def __init__( self, exprs, savelist = True ): + super(Each,self).__init__(exprs, savelist) + self.mayReturnEmpty = True + for e in self.exprs: + if not e.mayReturnEmpty: + self.mayReturnEmpty = False + break + self.skipWhitespace = True + self.initExprGroups = True + + def parseImpl( self, instring, loc, doActions=True ): + if self.initExprGroups: + opt1 = [ e.expr for e in self.exprs if isinstance(e,Optional) ] + opt2 = [ e for e in self.exprs if e.mayReturnEmpty and e not in opt1 ] + self.optionals = opt1 + opt2 + self.multioptionals = [ e.expr for e in self.exprs if isinstance(e,ZeroOrMore) ] + self.multirequired = [ e.expr for e in self.exprs if isinstance(e,OneOrMore) ] + self.required = [ e for e in self.exprs if not isinstance(e,(Optional,ZeroOrMore,OneOrMore)) ] + self.required += self.multirequired + self.initExprGroups = False + tmpLoc = loc + tmpReqd = self.required[:] + tmpOpt = self.optionals[:] + matchOrder = [] + + keepMatching = True + while keepMatching: + tmpExprs = tmpReqd + tmpOpt + self.multioptionals + self.multirequired + failed = [] + for e in tmpExprs: + try: + tmpLoc = e.tryParse( instring, tmpLoc ) + except ParseException: + failed.append(e) + else: + matchOrder.append(e) + if e in tmpReqd: + tmpReqd.remove(e) + elif e in tmpOpt: + tmpOpt.remove(e) + if len(failed) == len(tmpExprs): + keepMatching = False + + if tmpReqd: + missing = ", ".join( [ _ustr(e) for e in tmpReqd ] ) + raise ParseException(instring,loc,"Missing one or more required elements (%s)" % missing ) + + # add any unmatched Optionals, in case they have default values defined + matchOrder += [e for e in self.exprs if isinstance(e,Optional) and e.expr in tmpOpt] + + resultlist = [] + for e in matchOrder: + loc,results = e._parse(instring,loc,doActions) + resultlist.append(results) + + finalResults = ParseResults([]) + for r in resultlist: + dups = {} + for k in r.keys(): + if k in finalResults.keys(): + tmp = ParseResults(finalResults[k]) + tmp += ParseResults(r[k]) + dups[k] = tmp + finalResults += ParseResults(r) + for k,v in dups.items(): + finalResults[k] = v + return loc, finalResults + + def __str__( self ): + if hasattr(self,"name"): + return self.name + + if self.strRepr is None: + self.strRepr = "{" + " & ".join( [ _ustr(e) for e in self.exprs ] ) + "}" + + return self.strRepr + + def checkRecursion( self, parseElementList ): + subRecCheckList = parseElementList[:] + [ self ] + for e in self.exprs: + e.checkRecursion( subRecCheckList ) + + +class ParseElementEnhance(ParserElement): + """Abstract subclass of C{ParserElement}, for combining and post-processing parsed tokens.""" + def __init__( self, expr, savelist=False ): + super(ParseElementEnhance,self).__init__(savelist) + if isinstance( expr, basestring ): + expr = Literal(expr) + self.expr = expr + self.strRepr = None + if expr is not None: + self.mayIndexError = expr.mayIndexError + self.mayReturnEmpty = expr.mayReturnEmpty + self.setWhitespaceChars( expr.whiteChars ) + self.skipWhitespace = expr.skipWhitespace + self.saveAsList = expr.saveAsList + self.callPreparse = expr.callPreparse + self.ignoreExprs.extend(expr.ignoreExprs) + + def parseImpl( self, instring, loc, doActions=True ): + if self.expr is not None: + return self.expr._parse( instring, loc, doActions, callPreParse=False ) + else: + raise ParseException("",loc,self.errmsg,self) + + def leaveWhitespace( self ): + self.skipWhitespace = False + self.expr = self.expr.copy() + if self.expr is not None: + self.expr.leaveWhitespace() + return self + + def ignore( self, other ): + if isinstance( other, Suppress ): + if other not in self.ignoreExprs: + super( ParseElementEnhance, self).ignore( other ) + if self.expr is not None: + self.expr.ignore( self.ignoreExprs[-1] ) + else: + super( ParseElementEnhance, self).ignore( other ) + if self.expr is not None: + self.expr.ignore( self.ignoreExprs[-1] ) + return self + + def streamline( self ): + super(ParseElementEnhance,self).streamline() + if self.expr is not None: + self.expr.streamline() + return self + + def checkRecursion( self, parseElementList ): + if self in parseElementList: + raise RecursiveGrammarException( parseElementList+[self] ) + subRecCheckList = parseElementList[:] + [ self ] + if self.expr is not None: + self.expr.checkRecursion( subRecCheckList ) + + def validate( self, validateTrace=[] ): + tmp = validateTrace[:]+[self] + if self.expr is not None: + self.expr.validate(tmp) + self.checkRecursion( [] ) + + def __str__( self ): + try: + return super(ParseElementEnhance,self).__str__() + except: + pass + + if self.strRepr is None and self.expr is not None: + self.strRepr = "%s:(%s)" % ( self.__class__.__name__, _ustr(self.expr) ) + return self.strRepr + + +class FollowedBy(ParseElementEnhance): + """Lookahead matching of the given parse expression. C{FollowedBy} + does *not* advance the parsing position within the input string, it only + verifies that the specified parse expression matches at the current + position. C{FollowedBy} always returns a null token list.""" + def __init__( self, expr ): + super(FollowedBy,self).__init__(expr) + self.mayReturnEmpty = True + + def parseImpl( self, instring, loc, doActions=True ): + self.expr.tryParse( instring, loc ) + return loc, [] + + +class NotAny(ParseElementEnhance): + """Lookahead to disallow matching with the given parse expression. C{NotAny} + does *not* advance the parsing position within the input string, it only + verifies that the specified parse expression does *not* match at the current + position. Also, C{NotAny} does *not* skip over leading whitespace. C{NotAny} + always returns a null token list. May be constructed using the '~' operator.""" + def __init__( self, expr ): + super(NotAny,self).__init__(expr) + #~ self.leaveWhitespace() + self.skipWhitespace = False # do NOT use self.leaveWhitespace(), don't want to propagate to exprs + self.mayReturnEmpty = True + self.errmsg = "Found unwanted token, "+_ustr(self.expr) + + def parseImpl( self, instring, loc, doActions=True ): + try: + self.expr.tryParse( instring, loc ) + except (ParseException,IndexError): + pass + else: + #~ raise ParseException(instring, loc, self.errmsg ) + exc = self.myException + exc.loc = loc + exc.pstr = instring + raise exc + return loc, [] + + def __str__( self ): + if hasattr(self,"name"): + return self.name + + if self.strRepr is None: + self.strRepr = "~{" + _ustr(self.expr) + "}" + + return self.strRepr + + +class ZeroOrMore(ParseElementEnhance): + """Optional repetition of zero or more of the given expression.""" + def __init__( self, expr ): + super(ZeroOrMore,self).__init__(expr) + self.mayReturnEmpty = True + + def parseImpl( self, instring, loc, doActions=True ): + tokens = [] + try: + loc, tokens = self.expr._parse( instring, loc, doActions, callPreParse=False ) + hasIgnoreExprs = ( len(self.ignoreExprs) > 0 ) + while 1: + if hasIgnoreExprs: + preloc = self._skipIgnorables( instring, loc ) + else: + preloc = loc + loc, tmptokens = self.expr._parse( instring, preloc, doActions ) + if tmptokens or tmptokens.keys(): + tokens += tmptokens + except (ParseException,IndexError): + pass + + return loc, tokens + + def __str__( self ): + if hasattr(self,"name"): + return self.name + + if self.strRepr is None: + self.strRepr = "[" + _ustr(self.expr) + "]..." + + return self.strRepr + + def setResultsName( self, name, listAllMatches=False ): + ret = super(ZeroOrMore,self).setResultsName(name,listAllMatches) + ret.saveAsList = True + return ret + + +class OneOrMore(ParseElementEnhance): + """Repetition of one or more of the given expression.""" + def parseImpl( self, instring, loc, doActions=True ): + # must be at least one + loc, tokens = self.expr._parse( instring, loc, doActions, callPreParse=False ) + try: + hasIgnoreExprs = ( len(self.ignoreExprs) > 0 ) + while 1: + if hasIgnoreExprs: + preloc = self._skipIgnorables( instring, loc ) + else: + preloc = loc + loc, tmptokens = self.expr._parse( instring, preloc, doActions ) + if tmptokens or tmptokens.keys(): + tokens += tmptokens + except (ParseException,IndexError): + pass + + return loc, tokens + + def __str__( self ): + if hasattr(self,"name"): + return self.name + + if self.strRepr is None: + self.strRepr = "{" + _ustr(self.expr) + "}..." + + return self.strRepr + + def setResultsName( self, name, listAllMatches=False ): + ret = super(OneOrMore,self).setResultsName(name,listAllMatches) + ret.saveAsList = True + return ret + +class _NullToken(object): + def __bool__(self): + return False + __nonzero__ = __bool__ + def __str__(self): + return "" + +_optionalNotMatched = _NullToken() +class Optional(ParseElementEnhance): + """Optional matching of the given expression. + A default return string can also be specified, if the optional expression + is not found. + """ + def __init__( self, exprs, default=_optionalNotMatched ): + super(Optional,self).__init__( exprs, savelist=False ) + self.defaultValue = default + self.mayReturnEmpty = True + + def parseImpl( self, instring, loc, doActions=True ): + try: + loc, tokens = self.expr._parse( instring, loc, doActions, callPreParse=False ) + except (ParseException,IndexError): + if self.defaultValue is not _optionalNotMatched: + if self.expr.resultsName: + tokens = ParseResults([ self.defaultValue ]) + tokens[self.expr.resultsName] = self.defaultValue + else: + tokens = [ self.defaultValue ] + else: + tokens = [] + return loc, tokens + + def __str__( self ): + if hasattr(self,"name"): + return self.name + + if self.strRepr is None: + self.strRepr = "[" + _ustr(self.expr) + "]" + + return self.strRepr + + +class SkipTo(ParseElementEnhance): + """Token for skipping over all undefined text until the matched expression is found. + If C{include} is set to true, the matched expression is also parsed (the skipped text + and matched expression are returned as a 2-element list). The C{ignore} + argument is used to define grammars (typically quoted strings and comments) that + might contain false matches. + """ + def __init__( self, other, include=False, ignore=None, failOn=None ): + super( SkipTo, self ).__init__( other ) + self.ignoreExpr = ignore + self.mayReturnEmpty = True + self.mayIndexError = False + self.includeMatch = include + self.asList = False + if failOn is not None and isinstance(failOn, basestring): + self.failOn = Literal(failOn) + else: + self.failOn = failOn + self.errmsg = "No match found for "+_ustr(self.expr) + + def parseImpl( self, instring, loc, doActions=True ): + startLoc = loc + instrlen = len(instring) + expr = self.expr + failParse = False + while loc <= instrlen: + try: + if self.failOn: + try: + self.failOn.tryParse(instring, loc) + except ParseBaseException: + pass + else: + failParse = True + raise ParseException(instring, loc, "Found expression " + str(self.failOn)) + failParse = False + if self.ignoreExpr is not None: + while 1: + try: + loc = self.ignoreExpr.tryParse(instring,loc) + # print "found ignoreExpr, advance to", loc + except ParseBaseException: + break + expr._parse( instring, loc, doActions=False, callPreParse=False ) + skipText = instring[startLoc:loc] + if self.includeMatch: + loc,mat = expr._parse(instring,loc,doActions,callPreParse=False) + if mat: + skipRes = ParseResults( skipText ) + skipRes += mat + return loc, [ skipRes ] + else: + return loc, [ skipText ] + else: + return loc, [ skipText ] + except (ParseException,IndexError): + if failParse: + raise + else: + loc += 1 + exc = self.myException + exc.loc = loc + exc.pstr = instring + raise exc + +class Forward(ParseElementEnhance): + """Forward declaration of an expression to be defined later - + used for recursive grammars, such as algebraic infix notation. + When the expression is known, it is assigned to the C{Forward} variable using the '<<' operator. + + Note: take care when assigning to C{Forward} not to overlook precedence of operators. + Specifically, '|' has a lower precedence than '<<', so that:: + fwdExpr << a | b | c + will actually be evaluated as:: + (fwdExpr << a) | b | c + thereby leaving b and c out as parseable alternatives. It is recommended that you + explicitly group the values inserted into the C{Forward}:: + fwdExpr << (a | b | c) + """ + def __init__( self, other=None ): + super(Forward,self).__init__( other, savelist=False ) + + def __lshift__( self, other ): + if isinstance( other, basestring ): + other = Literal(other) + self.expr = other + self.mayReturnEmpty = other.mayReturnEmpty + self.strRepr = None + self.mayIndexError = self.expr.mayIndexError + self.mayReturnEmpty = self.expr.mayReturnEmpty + self.setWhitespaceChars( self.expr.whiteChars ) + self.skipWhitespace = self.expr.skipWhitespace + self.saveAsList = self.expr.saveAsList + self.ignoreExprs.extend(self.expr.ignoreExprs) + return None + + def leaveWhitespace( self ): + self.skipWhitespace = False + return self + + def streamline( self ): + if not self.streamlined: + self.streamlined = True + if self.expr is not None: + self.expr.streamline() + return self + + def validate( self, validateTrace=[] ): + if self not in validateTrace: + tmp = validateTrace[:]+[self] + if self.expr is not None: + self.expr.validate(tmp) + self.checkRecursion([]) + + def __str__( self ): + if hasattr(self,"name"): + return self.name + + self._revertClass = self.__class__ + self.__class__ = _ForwardNoRecurse + try: + if self.expr is not None: + retString = _ustr(self.expr) + else: + retString = "None" + finally: + self.__class__ = self._revertClass + return self.__class__.__name__ + ": " + retString + + def copy(self): + if self.expr is not None: + return super(Forward,self).copy() + else: + ret = Forward() + ret << self + return ret + +class _ForwardNoRecurse(Forward): + def __str__( self ): + return "..." + +class TokenConverter(ParseElementEnhance): + """Abstract subclass of C{ParseExpression}, for converting parsed results.""" + def __init__( self, expr, savelist=False ): + super(TokenConverter,self).__init__( expr )#, savelist ) + self.saveAsList = False + +class Upcase(TokenConverter): + """Converter to upper case all matching tokens.""" + def __init__(self, *args): + super(Upcase,self).__init__(*args) + warnings.warn("Upcase class is deprecated, use upcaseTokens parse action instead", + DeprecationWarning,stacklevel=2) + + def postParse( self, instring, loc, tokenlist ): + return list(map( string.upper, tokenlist )) + + +class Combine(TokenConverter): + """Converter to concatenate all matching tokens to a single string. + By default, the matching patterns must also be contiguous in the input string; + this can be disabled by specifying C{'adjacent=False'} in the constructor. + """ + def __init__( self, expr, joinString="", adjacent=True ): + super(Combine,self).__init__( expr ) + # suppress whitespace-stripping in contained parse expressions, but re-enable it on the Combine itself + if adjacent: + self.leaveWhitespace() + self.adjacent = adjacent + self.skipWhitespace = True + self.joinString = joinString + self.callPreparse = True + + def ignore( self, other ): + if self.adjacent: + ParserElement.ignore(self, other) + else: + super( Combine, self).ignore( other ) + return self + + def postParse( self, instring, loc, tokenlist ): + retToks = tokenlist.copy() + del retToks[:] + retToks += ParseResults([ "".join(tokenlist._asStringList(self.joinString)) ], modal=self.modalResults) + + if self.resultsName and len(retToks.keys())>0: + return [ retToks ] + else: + return retToks + +class Group(TokenConverter): + """Converter to return the matched tokens as a list - useful for returning tokens of C{ZeroOrMore} and C{OneOrMore} expressions.""" + def __init__( self, expr ): + super(Group,self).__init__( expr ) + self.saveAsList = True + + def postParse( self, instring, loc, tokenlist ): + return [ tokenlist ] + +class Dict(TokenConverter): + """Converter to return a repetitive expression as a list, but also as a dictionary. + Each element can also be referenced using the first token in the expression as its key. + Useful for tabular report scraping when the first column can be used as a item key. + """ + def __init__( self, exprs ): + super(Dict,self).__init__( exprs ) + self.saveAsList = True + + def postParse( self, instring, loc, tokenlist ): + for i,tok in enumerate(tokenlist): + if len(tok) == 0: + continue + ikey = tok[0] + if isinstance(ikey,int): + ikey = _ustr(tok[0]).strip() + if len(tok)==1: + tokenlist[ikey] = _ParseResultsWithOffset("",i) + elif len(tok)==2 and not isinstance(tok[1],ParseResults): + tokenlist[ikey] = _ParseResultsWithOffset(tok[1],i) + else: + dictvalue = tok.copy() #ParseResults(i) + del dictvalue[0] + if len(dictvalue)!= 1 or (isinstance(dictvalue,ParseResults) and dictvalue.keys()): + tokenlist[ikey] = _ParseResultsWithOffset(dictvalue,i) + else: + tokenlist[ikey] = _ParseResultsWithOffset(dictvalue[0],i) + + if self.resultsName: + return [ tokenlist ] + else: + return tokenlist + + +class Suppress(TokenConverter): + """Converter for ignoring the results of a parsed expression.""" + def postParse( self, instring, loc, tokenlist ): + return [] + + def suppress( self ): + return self + + +class OnlyOnce(object): + """Wrapper for parse actions, to ensure they are only called once.""" + def __init__(self, methodCall): + self.callable = _trim_arity(methodCall) + self.called = False + def __call__(self,s,l,t): + if not self.called: + results = self.callable(s,l,t) + self.called = True + return results + raise ParseException(s,l,"") + def reset(self): + self.called = False + +def traceParseAction(f): + """Decorator for debugging parse actions.""" + f = _trim_arity(f) + def z(*paArgs): + thisFunc = f.func_name + s,l,t = paArgs[-3:] + if len(paArgs)>3: + thisFunc = paArgs[0].__class__.__name__ + '.' + thisFunc + sys.stderr.write( ">>entering %s(line: '%s', %d, %s)\n" % (thisFunc,line(l,s),l,t) ) + try: + ret = f(*paArgs) + except Exception: + exc = sys.exc_info()[1] + sys.stderr.write( "<", "|".join( [ _escapeRegexChars(sym) for sym in symbols] )) + try: + if len(symbols)==len("".join(symbols)): + return Regex( "[%s]" % "".join( [ _escapeRegexRangeChars(sym) for sym in symbols] ) ) + else: + return Regex( "|".join( [ re.escape(sym) for sym in symbols] ) ) + except: + warnings.warn("Exception creating Regex for oneOf, building MatchFirst", + SyntaxWarning, stacklevel=2) + + + # last resort, just use MatchFirst + return MatchFirst( [ parseElementClass(sym) for sym in symbols ] ) + +def dictOf( key, value ): + """Helper to easily and clearly define a dictionary by specifying the respective patterns + for the key and value. Takes care of defining the C{Dict}, C{ZeroOrMore}, and C{Group} tokens + in the proper order. The key pattern can include delimiting markers or punctuation, + as long as they are suppressed, thereby leaving the significant key text. The value + pattern can include named results, so that the C{Dict} results can include named token + fields. + """ + return Dict( ZeroOrMore( Group ( key + value ) ) ) + +def originalTextFor(expr, asString=True): + """Helper to return the original, untokenized text for a given expression. Useful to + restore the parsed fields of an HTML start tag into the raw tag text itself, or to + revert separate tokens with intervening whitespace back to the original matching + input text. Simpler to use than the parse action C{L{keepOriginalText}}, and does not + require the inspect module to chase up the call stack. By default, returns a + string containing the original parsed text. + + If the optional C{asString} argument is passed as C{False}, then the return value is a + C{ParseResults} containing any results names that were originally matched, and a + single token containing the original matched text from the input string. So if + the expression passed to C{L{originalTextFor}} contains expressions with defined + results names, you must set C{asString} to C{False} if you want to preserve those + results name values.""" + locMarker = Empty().setParseAction(lambda s,loc,t: loc) + endlocMarker = locMarker.copy() + endlocMarker.callPreparse = False + matchExpr = locMarker("_original_start") + expr + endlocMarker("_original_end") + if asString: + extractText = lambda s,l,t: s[t._original_start:t._original_end] + else: + def extractText(s,l,t): + del t[:] + t.insert(0, s[t._original_start:t._original_end]) + del t["_original_start"] + del t["_original_end"] + matchExpr.setParseAction(extractText) + return matchExpr + +def ungroup(expr): + """Helper to undo pyparsing's default grouping of And expressions, even + if all but one are non-empty.""" + return TokenConverter(expr).setParseAction(lambda t:t[0]) + +# convenience constants for positional expressions +empty = Empty().setName("empty") +lineStart = LineStart().setName("lineStart") +lineEnd = LineEnd().setName("lineEnd") +stringStart = StringStart().setName("stringStart") +stringEnd = StringEnd().setName("stringEnd") + +_escapedPunc = Word( _bslash, r"\[]-*.$+^?()~ ", exact=2 ).setParseAction(lambda s,l,t:t[0][1]) +_printables_less_backslash = "".join([ c for c in printables if c not in r"\]" ]) +_escapedHexChar = Regex(r"\\0?[xX][0-9a-fA-F]+").setParseAction(lambda s,l,t:unichr(int(t[0][1:],16))) +_escapedOctChar = Regex(r"\\0[0-7]+").setParseAction(lambda s,l,t:unichr(int(t[0][1:],8))) +_singleChar = _escapedPunc | _escapedHexChar | _escapedOctChar | Word(_printables_less_backslash,exact=1) +_charRange = Group(_singleChar + Suppress("-") + _singleChar) +_reBracketExpr = Literal("[") + Optional("^").setResultsName("negate") + Group( OneOrMore( _charRange | _singleChar ) ).setResultsName("body") + "]" + +_expanded = lambda p: (isinstance(p,ParseResults) and ''.join([ unichr(c) for c in range(ord(p[0]),ord(p[1])+1) ]) or p) + +def srange(s): + r"""Helper to easily define string ranges for use in Word construction. Borrows + syntax from regexp '[]' string range definitions:: + srange("[0-9]") -> "0123456789" + srange("[a-z]") -> "abcdefghijklmnopqrstuvwxyz" + srange("[a-z$_]") -> "abcdefghijklmnopqrstuvwxyz$_" + The input string must be enclosed in []'s, and the returned string is the expanded + character set joined into a single string. + The values enclosed in the []'s may be:: + a single character + an escaped character with a leading backslash (such as \- or \]) + an escaped hex character with a leading '\x' (\x21, which is a '!' character) + (\0x## is also supported for backwards compatibility) + an escaped octal character with a leading '\0' (\041, which is a '!' character) + a range of any of the above, separated by a dash ('a-z', etc.) + any combination of the above ('aeiouy', 'a-zA-Z0-9_$', etc.) + """ + try: + return "".join([_expanded(part) for part in _reBracketExpr.parseString(s).body]) + except: + return "" + +def matchOnlyAtCol(n): + """Helper method for defining parse actions that require matching at a specific + column in the input text. + """ + def verifyCol(strg,locn,toks): + if col(locn,strg) != n: + raise ParseException(strg,locn,"matched token not at column %d" % n) + return verifyCol + +def replaceWith(replStr): + """Helper method for common parse actions that simply return a literal value. Especially + useful when used with C{transformString()}. + """ + def _replFunc(*args): + return [replStr] + return _replFunc + +def removeQuotes(s,l,t): + """Helper parse action for removing quotation marks from parsed quoted strings. + To use, add this parse action to quoted string using:: + quotedString.setParseAction( removeQuotes ) + """ + return t[0][1:-1] + +def upcaseTokens(s,l,t): + """Helper parse action to convert tokens to upper case.""" + return [ tt.upper() for tt in map(_ustr,t) ] + +def downcaseTokens(s,l,t): + """Helper parse action to convert tokens to lower case.""" + return [ tt.lower() for tt in map(_ustr,t) ] + +def keepOriginalText(s,startLoc,t): + """DEPRECATED - use new helper method C{originalTextFor}. + Helper parse action to preserve original parsed text, + overriding any nested parse actions.""" + try: + endloc = getTokensEndLoc() + except ParseException: + raise ParseFatalException("incorrect usage of keepOriginalText - may only be called as a parse action") + del t[:] + t += ParseResults(s[startLoc:endloc]) + return t + +def getTokensEndLoc(): + """Method to be called from within a parse action to determine the end + location of the parsed tokens.""" + import inspect + fstack = inspect.stack() + try: + # search up the stack (through intervening argument normalizers) for correct calling routine + for f in fstack[2:]: + if f[3] == "_parseNoCache": + endloc = f[0].f_locals["loc"] + return endloc + else: + raise ParseFatalException("incorrect usage of getTokensEndLoc - may only be called from within a parse action") + finally: + del fstack + +def _makeTags(tagStr, xml): + """Internal helper to construct opening and closing tag expressions, given a tag name""" + if isinstance(tagStr,basestring): + resname = tagStr + tagStr = Keyword(tagStr, caseless=not xml) + else: + resname = tagStr.name + + tagAttrName = Word(alphas,alphanums+"_-:") + if (xml): + tagAttrValue = dblQuotedString.copy().setParseAction( removeQuotes ) + openTag = Suppress("<") + tagStr("tag") + \ + Dict(ZeroOrMore(Group( tagAttrName + Suppress("=") + tagAttrValue ))) + \ + Optional("/",default=[False]).setResultsName("empty").setParseAction(lambda s,l,t:t[0]=='/') + Suppress(">") + else: + printablesLessRAbrack = "".join( [ c for c in printables if c not in ">" ] ) + tagAttrValue = quotedString.copy().setParseAction( removeQuotes ) | Word(printablesLessRAbrack) + openTag = Suppress("<") + tagStr("tag") + \ + Dict(ZeroOrMore(Group( tagAttrName.setParseAction(downcaseTokens) + \ + Optional( Suppress("=") + tagAttrValue ) ))) + \ + Optional("/",default=[False]).setResultsName("empty").setParseAction(lambda s,l,t:t[0]=='/') + Suppress(">") + closeTag = Combine(_L("") + + openTag = openTag.setResultsName("start"+"".join(resname.replace(":"," ").title().split())).setName("<%s>" % tagStr) + closeTag = closeTag.setResultsName("end"+"".join(resname.replace(":"," ").title().split())).setName("" % tagStr) + openTag.tag = resname + closeTag.tag = resname + return openTag, closeTag + +def makeHTMLTags(tagStr): + """Helper to construct opening and closing tag expressions for HTML, given a tag name""" + return _makeTags( tagStr, False ) + +def makeXMLTags(tagStr): + """Helper to construct opening and closing tag expressions for XML, given a tag name""" + return _makeTags( tagStr, True ) + +def withAttribute(*args,**attrDict): + """Helper to create a validating parse action to be used with start tags created + with C{makeXMLTags} or C{makeHTMLTags}. Use C{withAttribute} to qualify a starting tag + with a required attribute value, to avoid false matches on common tags such as + C{} or C{
}. + + Call C{withAttribute} with a series of attribute names and values. Specify the list + of filter attributes names and values as: + - keyword arguments, as in C{(align="right")}, or + - as an explicit dict with C{**} operator, when an attribute name is also a Python + reserved word, as in C{**{"class":"Customer", "align":"right"}} + - a list of name-value tuples, as in ( ("ns1:class", "Customer"), ("ns2:align","right") ) + For attribute names with a namespace prefix, you must use the second form. Attribute + names are matched insensitive to upper/lower case. + + To verify that the attribute exists, but without specifying a value, pass + C{withAttribute.ANY_VALUE} as the value. + """ + if args: + attrs = args[:] + else: + attrs = attrDict.items() + attrs = [(k,v) for k,v in attrs] + def pa(s,l,tokens): + for attrName,attrValue in attrs: + if attrName not in tokens: + raise ParseException(s,l,"no matching attribute " + attrName) + if attrValue != withAttribute.ANY_VALUE and tokens[attrName] != attrValue: + raise ParseException(s,l,"attribute '%s' has value '%s', must be '%s'" % + (attrName, tokens[attrName], attrValue)) + return pa +withAttribute.ANY_VALUE = object() + +opAssoc = _Constants() +opAssoc.LEFT = object() +opAssoc.RIGHT = object() + +def operatorPrecedence( baseExpr, opList ): + """Helper method for constructing grammars of expressions made up of + operators working in a precedence hierarchy. Operators may be unary or + binary, left- or right-associative. Parse actions can also be attached + to operator expressions. + + Parameters: + - baseExpr - expression representing the most basic element for the nested + - opList - list of tuples, one for each operator precedence level in the + expression grammar; each tuple is of the form + (opExpr, numTerms, rightLeftAssoc, parseAction), where: + - opExpr is the pyparsing expression for the operator; + may also be a string, which will be converted to a Literal; + if numTerms is 3, opExpr is a tuple of two expressions, for the + two operators separating the 3 terms + - numTerms is the number of terms for this operator (must + be 1, 2, or 3) + - rightLeftAssoc is the indicator whether the operator is + right or left associative, using the pyparsing-defined + constants opAssoc.RIGHT and opAssoc.LEFT. + - parseAction is the parse action to be associated with + expressions matching this operator expression (the + parse action tuple member may be omitted) + """ + ret = Forward() + lastExpr = baseExpr | ( Suppress('(') + ret + Suppress(')') ) + for i,operDef in enumerate(opList): + opExpr,arity,rightLeftAssoc,pa = (operDef + (None,))[:4] + if arity == 3: + if opExpr is None or len(opExpr) != 2: + raise ValueError("if numterms=3, opExpr must be a tuple or list of two expressions") + opExpr1, opExpr2 = opExpr + thisExpr = Forward()#.setName("expr%d" % i) + if rightLeftAssoc == opAssoc.LEFT: + if arity == 1: + matchExpr = FollowedBy(lastExpr + opExpr) + Group( lastExpr + OneOrMore( opExpr ) ) + elif arity == 2: + if opExpr is not None: + matchExpr = FollowedBy(lastExpr + opExpr + lastExpr) + Group( lastExpr + OneOrMore( opExpr + lastExpr ) ) + else: + matchExpr = FollowedBy(lastExpr+lastExpr) + Group( lastExpr + OneOrMore(lastExpr) ) + elif arity == 3: + matchExpr = FollowedBy(lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr) + \ + Group( lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr ) + else: + raise ValueError("operator must be unary (1), binary (2), or ternary (3)") + elif rightLeftAssoc == opAssoc.RIGHT: + if arity == 1: + # try to avoid LR with this extra test + if not isinstance(opExpr, Optional): + opExpr = Optional(opExpr) + matchExpr = FollowedBy(opExpr.expr + thisExpr) + Group( opExpr + thisExpr ) + elif arity == 2: + if opExpr is not None: + matchExpr = FollowedBy(lastExpr + opExpr + thisExpr) + Group( lastExpr + OneOrMore( opExpr + thisExpr ) ) + else: + matchExpr = FollowedBy(lastExpr + thisExpr) + Group( lastExpr + OneOrMore( thisExpr ) ) + elif arity == 3: + matchExpr = FollowedBy(lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr) + \ + Group( lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr ) + else: + raise ValueError("operator must be unary (1), binary (2), or ternary (3)") + else: + raise ValueError("operator must indicate right or left associativity") + if pa: + matchExpr.setParseAction( pa ) + thisExpr << ( matchExpr | lastExpr ) + lastExpr = thisExpr + ret << lastExpr + return ret + +dblQuotedString = Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\x[0-9a-fA-F]+)|(?:\\.))*"').setName("string enclosed in double quotes") +sglQuotedString = Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\x[0-9a-fA-F]+)|(?:\\.))*'").setName("string enclosed in single quotes") +quotedString = Regex(r'''(?:"(?:[^"\n\r\\]|(?:"")|(?:\\x[0-9a-fA-F]+)|(?:\\.))*")|(?:'(?:[^'\n\r\\]|(?:'')|(?:\\x[0-9a-fA-F]+)|(?:\\.))*')''').setName("quotedString using single or double quotes") +unicodeString = Combine(_L('u') + quotedString.copy()) + +def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString.copy()): + """Helper method for defining nested lists enclosed in opening and closing + delimiters ("(" and ")" are the default). + + Parameters: + - opener - opening character for a nested list (default="("); can also be a pyparsing expression + - closer - closing character for a nested list (default=")"); can also be a pyparsing expression + - content - expression for items within the nested lists (default=None) + - ignoreExpr - expression for ignoring opening and closing delimiters (default=quotedString) + + If an expression is not provided for the content argument, the nested + expression will capture all whitespace-delimited content between delimiters + as a list of separate values. + + Use the C{ignoreExpr} argument to define expressions that may contain + opening or closing characters that should not be treated as opening + or closing characters for nesting, such as quotedString or a comment + expression. Specify multiple expressions using an C{L{Or}} or C{L{MatchFirst}}. + The default is L{quotedString}, but if no expressions are to be ignored, + then pass C{None} for this argument. + """ + if opener == closer: + raise ValueError("opening and closing strings cannot be the same") + if content is None: + if isinstance(opener,basestring) and isinstance(closer,basestring): + if len(opener) == 1 and len(closer)==1: + if ignoreExpr is not None: + content = (Combine(OneOrMore(~ignoreExpr + + CharsNotIn(opener+closer+ParserElement.DEFAULT_WHITE_CHARS,exact=1)) + ).setParseAction(lambda t:t[0].strip())) + else: + content = (empty.copy()+CharsNotIn(opener+closer+ParserElement.DEFAULT_WHITE_CHARS + ).setParseAction(lambda t:t[0].strip())) + else: + if ignoreExpr is not None: + content = (Combine(OneOrMore(~ignoreExpr + + ~Literal(opener) + ~Literal(closer) + + CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS,exact=1)) + ).setParseAction(lambda t:t[0].strip())) + else: + content = (Combine(OneOrMore(~Literal(opener) + ~Literal(closer) + + CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS,exact=1)) + ).setParseAction(lambda t:t[0].strip())) + else: + raise ValueError("opening and closing arguments must be strings if no content expression is given") + ret = Forward() + if ignoreExpr is not None: + ret << Group( Suppress(opener) + ZeroOrMore( ignoreExpr | ret | content ) + Suppress(closer) ) + else: + ret << Group( Suppress(opener) + ZeroOrMore( ret | content ) + Suppress(closer) ) + return ret + +def indentedBlock(blockStatementExpr, indentStack, indent=True): + """Helper method for defining space-delimited indentation blocks, such as + those used to define block statements in Python source code. + + Parameters: + - blockStatementExpr - expression defining syntax of statement that + is repeated within the indented block + - indentStack - list created by caller to manage indentation stack + (multiple statementWithIndentedBlock expressions within a single grammar + should share a common indentStack) + - indent - boolean indicating whether block must be indented beyond the + the current level; set to False for block of left-most statements + (default=True) + + A valid block must contain at least one C{blockStatement}. + """ + def checkPeerIndent(s,l,t): + if l >= len(s): return + curCol = col(l,s) + if curCol != indentStack[-1]: + if curCol > indentStack[-1]: + raise ParseFatalException(s,l,"illegal nesting") + raise ParseException(s,l,"not a peer entry") + + def checkSubIndent(s,l,t): + curCol = col(l,s) + if curCol > indentStack[-1]: + indentStack.append( curCol ) + else: + raise ParseException(s,l,"not a subentry") + + def checkUnindent(s,l,t): + if l >= len(s): return + curCol = col(l,s) + if not(indentStack and curCol < indentStack[-1] and curCol <= indentStack[-2]): + raise ParseException(s,l,"not an unindent") + indentStack.pop() + + NL = OneOrMore(LineEnd().setWhitespaceChars("\t ").suppress()) + INDENT = Empty() + Empty().setParseAction(checkSubIndent) + PEER = Empty().setParseAction(checkPeerIndent) + UNDENT = Empty().setParseAction(checkUnindent) + if indent: + smExpr = Group( Optional(NL) + + #~ FollowedBy(blockStatementExpr) + + INDENT + (OneOrMore( PEER + Group(blockStatementExpr) + Optional(NL) )) + UNDENT) + else: + smExpr = Group( Optional(NL) + + (OneOrMore( PEER + Group(blockStatementExpr) + Optional(NL) )) ) + blockStatementExpr.ignore(_bslash + LineEnd()) + return smExpr + +alphas8bit = srange(r"[\0xc0-\0xd6\0xd8-\0xf6\0xf8-\0xff]") +punc8bit = srange(r"[\0xa1-\0xbf\0xd7\0xf7]") + +anyOpenTag,anyCloseTag = makeHTMLTags(Word(alphas,alphanums+"_:")) +commonHTMLEntity = Combine(_L("&") + oneOf("gt lt amp nbsp quot").setResultsName("entity") +";").streamline() +_htmlEntityMap = dict(zip("gt lt amp nbsp quot".split(),'><& "')) +replaceHTMLEntity = lambda t : t.entity in _htmlEntityMap and _htmlEntityMap[t.entity] or None + +# it's easy to get these comment structures wrong - they're very common, so may as well make them available +cStyleComment = Regex(r"/\*(?:[^*]*\*+)+?/").setName("C style comment") + +htmlComment = Regex(r"") +restOfLine = Regex(r".*").leaveWhitespace() +dblSlashComment = Regex(r"\/\/(\\\n|.)*").setName("// comment") +cppStyleComment = Regex(r"/(?:\*(?:[^*]*\*+)+?/|/[^\n]*(?:\n[^\n]*)*?(?:(?" + str(tokenlist)) + print ("tokens = " + str(tokens)) + print ("tokens.columns = " + str(tokens.columns)) + print ("tokens.tables = " + str(tokens.tables)) + print (tokens.asXML("SQL",True)) + except ParseBaseException: + err = sys.exc_info()[1] + print (teststring + "->") + print (err.line) + print (" "*(err.column-1) + "^") + print (err) + print() + + selectToken = CaselessLiteral( "select" ) + fromToken = CaselessLiteral( "from" ) + + ident = Word( alphas, alphanums + "_$" ) + columnName = delimitedList( ident, ".", combine=True ).setParseAction( upcaseTokens ) + columnNameList = Group( delimitedList( columnName ) )#.setName("columns") + tableName = delimitedList( ident, ".", combine=True ).setParseAction( upcaseTokens ) + tableNameList = Group( delimitedList( tableName ) )#.setName("tables") + simpleSQL = ( selectToken + \ + ( '*' | columnNameList ).setResultsName( "columns" ) + \ + fromToken + \ + tableNameList.setResultsName( "tables" ) ) + + test( "SELECT * from XYZZY, ABC" ) + test( "select * from SYS.XYZZY" ) + test( "Select A from Sys.dual" ) + test( "Select AA,BB,CC from Sys.dual" ) + test( "Select A, B, C from Sys.dual" ) + test( "Select A, B, C from Sys.dual" ) + test( "Xelect A, B, C from Sys.dual" ) + test( "Select A, B, C frox Sys.dual" ) + test( "Select" ) + test( "Select ^^^ frox Sys.dual" ) + test( "Select A, B, C from Sys.dual, Table2 " ) diff --git a/docs/stubs_generation/helpers/generator3/_vendor/pyparsing_py3.py b/docs/stubs_generation/helpers/generator3/_vendor/pyparsing_py3.py new file mode 100644 index 000000000..13f07a54f --- /dev/null +++ b/docs/stubs_generation/helpers/generator3/_vendor/pyparsing_py3.py @@ -0,0 +1,3586 @@ +# module pyparsing.py +# +# Copyright (c) 2003-2011 Paul T. McGuire +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +#from __future__ import generators + +__doc__ = \ +""" +pyparsing module - Classes and methods to define and execute parsing grammars + +The pyparsing module is an alternative approach to creating and executing simple grammars, +vs. the traditional lex/yacc approach, or the use of regular expressions. With pyparsing, you +don't need to learn a new syntax for defining grammars or matching expressions - the parsing module +provides a library of classes that you use to construct the grammar directly in Python. + +Here is a program to parse "Hello, World!" (or any greeting of the form C{", !"}):: + + from pyparsing import Word, alphas + + # define grammar of a greeting + greet = Word( alphas ) + "," + Word( alphas ) + "!" + + hello = "Hello, World!" + print hello, "->", greet.parseString( hello ) + +The program outputs the following:: + + Hello, World! -> ['Hello', ',', 'World', '!'] + +The Python representation of the grammar is quite readable, owing to the self-explanatory +class names, and the use of '+', '|' and '^' operators. + +The parsed results returned from C{parseString()} can be accessed as a nested list, a dictionary, or an +object with named attributes. + +The pyparsing module handles some of the problems that are typically vexing when writing text parsers: + - extra or missing whitespace (the above program will also handle "Hello,World!", "Hello , World !", etc.) + - quoted strings + - embedded comments +""" + +__version__ = "1.5.6" +__versionTime__ = "26 June 2011 10:53" +__author__ = "Paul McGuire " + +import string +from weakref import ref as wkref +import copy +import sys +import warnings +import re +import sre_constants +import collections.abc +#~ sys.stderr.write( "testing pyparsing module, version %s, %s\n" % (__version__,__versionTime__ ) ) + +__all__ = [ +'And', 'CaselessKeyword', 'CaselessLiteral', 'CharsNotIn', 'Combine', 'Dict', 'Each', 'Empty', +'FollowedBy', 'Forward', 'GoToColumn', 'Group', 'Keyword', 'LineEnd', 'LineStart', 'Literal', +'MatchFirst', 'NoMatch', 'NotAny', 'OneOrMore', 'OnlyOnce', 'Optional', 'Or', +'ParseBaseException', 'ParseElementEnhance', 'ParseException', 'ParseExpression', 'ParseFatalException', +'ParseResults', 'ParseSyntaxException', 'ParserElement', 'QuotedString', 'RecursiveGrammarException', +'Regex', 'SkipTo', 'StringEnd', 'StringStart', 'Suppress', 'Token', 'TokenConverter', 'Upcase', +'White', 'Word', 'WordEnd', 'WordStart', 'ZeroOrMore', +'alphanums', 'alphas', 'alphas8bit', 'anyCloseTag', 'anyOpenTag', 'cStyleComment', 'col', +'commaSeparatedList', 'commonHTMLEntity', 'countedArray', 'cppStyleComment', 'dblQuotedString', +'dblSlashComment', 'delimitedList', 'dictOf', 'downcaseTokens', 'empty', 'getTokensEndLoc', 'hexnums', +'htmlComment', 'javaStyleComment', 'keepOriginalText', 'line', 'lineEnd', 'lineStart', 'lineno', +'makeHTMLTags', 'makeXMLTags', 'matchOnlyAtCol', 'matchPreviousExpr', 'matchPreviousLiteral', +'nestedExpr', 'nullDebugAction', 'nums', 'oneOf', 'opAssoc', 'operatorPrecedence', 'printables', +'punc8bit', 'pythonStyleComment', 'quotedString', 'removeQuotes', 'replaceHTMLEntity', +'replaceWith', 'restOfLine', 'sglQuotedString', 'srange', 'stringEnd', +'stringStart', 'traceParseAction', 'unicodeString', 'upcaseTokens', 'withAttribute', +'indentedBlock', 'originalTextFor', +] + +_MAX_INT = sys.maxsize +basestring = str +unichr = chr +_ustr = str +alphas = string.ascii_lowercase + string.ascii_uppercase + +# build list of single arg builtins, that can be used as parse actions +singleArgBuiltins = [sum, len, enumerate, sorted, reversed, list, tuple, set, any, all] + +def _xml_escape(data): + """Escape &, <, >, ", ', etc. in a string of data.""" + + # ampersand must be replaced first + from_symbols = '&><"\'' + to_symbols = ['&'+s+';' for s in "amp gt lt quot apos".split()] + for from_,to_ in zip(from_symbols, to_symbols): + data = data.replace(from_, to_) + return data + +class _Constants(object): + pass + +nums = string.digits +hexnums = nums + "ABCDEFabcdef" +alphanums = alphas + nums +_bslash = chr(92) +printables = "".join( [ c for c in string.printable if c not in string.whitespace ] ) + +class ParseBaseException(Exception): + """base exception class for all parsing runtime exceptions""" + # Performance tuning: we construct a *lot* of these, so keep this + # constructor as small and fast as possible + def __init__( self, pstr, loc=0, msg=None, elem=None ): + self.loc = loc + if msg is None: + self.msg = pstr + self.pstr = "" + else: + self.msg = msg + self.pstr = pstr + self.parserElement = elem + + def __getattr__( self, aname ): + """supported attributes by name are: + - lineno - returns the line number of the exception text + - col - returns the column number of the exception text + - line - returns the line containing the exception text + """ + if( aname == "lineno" ): + return lineno( self.loc, self.pstr ) + elif( aname in ("col", "column") ): + return col( self.loc, self.pstr ) + elif( aname == "line" ): + return line( self.loc, self.pstr ) + else: + raise AttributeError(aname) + + def __str__( self ): + return "%s (at char %d), (line:%d, col:%d)" % \ + ( self.msg, self.loc, self.lineno, self.column ) + def __repr__( self ): + return _ustr(self) + def markInputline( self, markerString = ">!<" ): + """Extracts the exception line from the input string, and marks + the location of the exception with a special symbol. + """ + line_str = self.line + line_column = self.column - 1 + if markerString: + line_str = "".join( [line_str[:line_column], + markerString, line_str[line_column:]]) + return line_str.strip() + def __dir__(self): + return "loc msg pstr parserElement lineno col line " \ + "markInputLine __str__ __repr__".split() + +class ParseException(ParseBaseException): + """exception thrown when parse expressions don't match class; + supported attributes by name are: + - lineno - returns the line number of the exception text + - col - returns the column number of the exception text + - line - returns the line containing the exception text + """ + pass + +class ParseFatalException(ParseBaseException): + """user-throwable exception thrown when inconsistent parse content + is found; stops all parsing immediately""" + pass + +class ParseSyntaxException(ParseFatalException): + """just like C{ParseFatalException}, but thrown internally when an + C{ErrorStop} ('-' operator) indicates that parsing is to stop immediately because + an unbacktrackable syntax error has been found""" + def __init__(self, pe): + super(ParseSyntaxException, self).__init__( + pe.pstr, pe.loc, pe.msg, pe.parserElement) + +#~ class ReparseException(ParseBaseException): + #~ """Experimental class - parse actions can raise this exception to cause + #~ pyparsing to reparse the input string: + #~ - with a modified input string, and/or + #~ - with a modified start location + #~ Set the values of the ReparseException in the constructor, and raise the + #~ exception in a parse action to cause pyparsing to use the new string/location. + #~ Setting the values as None causes no change to be made. + #~ """ + #~ def __init_( self, newstring, restartLoc ): + #~ self.newParseText = newstring + #~ self.reparseLoc = restartLoc + +class RecursiveGrammarException(Exception): + """exception thrown by C{validate()} if the grammar could be improperly recursive""" + def __init__( self, parseElementList ): + self.parseElementTrace = parseElementList + + def __str__( self ): + return "RecursiveGrammarException: %s" % self.parseElementTrace + +class _ParseResultsWithOffset(object): + def __init__(self,p1,p2): + self.tup = (p1,p2) + def __getitem__(self,i): + return self.tup[i] + def __repr__(self): + return repr(self.tup) + def setOffset(self,i): + self.tup = (self.tup[0],i) + +class ParseResults(object): + """Structured parse results, to provide multiple means of access to the parsed data: + - as a list (C{len(results)}) + - by list index (C{results[0], results[1]}, etc.) + - by attribute (C{results.}) + """ + #~ __slots__ = ( "__toklist", "__tokdict", "__doinit", "__name", "__parent", "__accumNames", "__weakref__" ) + def __new__(cls, toklist, name=None, asList=True, modal=True ): + if isinstance(toklist, cls): + return toklist + retobj = object.__new__(cls) + retobj.__doinit = True + return retobj + + # Performance tuning: we construct a *lot* of these, so keep this + # constructor as small and fast as possible + def __init__( self, toklist, name=None, asList=True, modal=True, isinstance=isinstance ): + if self.__doinit: + self.__doinit = False + self.__name = None + self.__parent = None + self.__accumNames = {} + if isinstance(toklist, list): + self.__toklist = toklist[:] + else: + self.__toklist = [toklist] + self.__tokdict = dict() + + if name is not None and name: + if not modal: + self.__accumNames[name] = 0 + if isinstance(name,int): + name = _ustr(name) # will always return a str, but use _ustr for consistency + self.__name = name + if not toklist in (None,'',[]): + if isinstance(toklist,basestring): + toklist = [ toklist ] + if asList: + if isinstance(toklist,ParseResults): + self[name] = _ParseResultsWithOffset(toklist.copy(),0) + else: + self[name] = _ParseResultsWithOffset(ParseResults(toklist[0]),0) + self[name].__name = name + else: + try: + self[name] = toklist[0] + except (KeyError,TypeError,IndexError): + self[name] = toklist + + def __getitem__( self, i ): + if isinstance( i, (int,slice) ): + return self.__toklist[i] + else: + if i not in self.__accumNames: + return self.__tokdict[i][-1][0] + else: + return ParseResults([ v[0] for v in self.__tokdict[i] ]) + + def __setitem__( self, k, v, isinstance=isinstance ): + if isinstance(v,_ParseResultsWithOffset): + self.__tokdict[k] = self.__tokdict.get(k,list()) + [v] + sub = v[0] + elif isinstance(k,int): + self.__toklist[k] = v + sub = v + else: + self.__tokdict[k] = self.__tokdict.get(k,list()) + [_ParseResultsWithOffset(v,0)] + sub = v + if isinstance(sub,ParseResults): + sub.__parent = wkref(self) + + def __delitem__( self, i ): + if isinstance(i,(int,slice)): + mylen = len( self.__toklist ) + del self.__toklist[i] + + # convert int to slice + if isinstance(i, int): + if i < 0: + i += mylen + i = slice(i, i+1) + # get removed indices + removed = list(range(*i.indices(mylen))) + removed.reverse() + # fixup indices in token dictionary + for name in self.__tokdict: + occurrences = self.__tokdict[name] + for j in removed: + for k, (value, position) in enumerate(occurrences): + occurrences[k] = _ParseResultsWithOffset(value, position - (position > j)) + else: + del self.__tokdict[i] + + def __contains__( self, k ): + return k in self.__tokdict + + def __len__( self ): return len( self.__toklist ) + def __bool__(self): return len( self.__toklist ) > 0 + __nonzero__ = __bool__ + def __iter__( self ): return iter( self.__toklist ) + def __reversed__( self ): return iter( self.__toklist[::-1] ) + def keys( self ): + """Returns all named result keys.""" + return self.__tokdict.keys() + + def pop( self, index=-1 ): + """Removes and returns item at specified index (default=last). + Will work with either numeric indices or dict-key indicies.""" + ret = self[index] + del self[index] + return ret + + def get(self, key, defaultValue=None): + """Returns named result matching the given key, or if there is no + such name, then returns the given C{defaultValue} or C{None} if no + C{defaultValue} is specified.""" + if key in self: + return self[key] + else: + return defaultValue + + def insert( self, index, insStr ): + """Inserts new element at location index in the list of parsed tokens.""" + self.__toklist.insert(index, insStr) + # fixup indices in token dictionary + for name in self.__tokdict: + occurrences = self.__tokdict[name] + for k, (value, position) in enumerate(occurrences): + occurrences[k] = _ParseResultsWithOffset(value, position + (position > index)) + + def items( self ): + """Returns all named result keys and values as a list of tuples.""" + return [(k,self[k]) for k in self.__tokdict] + + def values( self ): + """Returns all named result values.""" + return [ v[-1][0] for v in self.__tokdict.values() ] + + def __getattr__( self, name ): + if True: #name not in self.__slots__: + if name in self.__tokdict: + if name not in self.__accumNames: + return self.__tokdict[name][-1][0] + else: + return ParseResults([ v[0] for v in self.__tokdict[name] ]) + else: + return "" + return None + + def __add__( self, other ): + ret = self.copy() + ret += other + return ret + + def __iadd__( self, other ): + if other.__tokdict: + offset = len(self.__toklist) + addoffset = ( lambda a: (a<0 and offset) or (a+offset) ) + otheritems = other.__tokdict.items() + otherdictitems = [(k, _ParseResultsWithOffset(v[0],addoffset(v[1])) ) + for (k,vlist) in otheritems for v in vlist] + for k,v in otherdictitems: + self[k] = v + if isinstance(v[0],ParseResults): + v[0].__parent = wkref(self) + + self.__toklist += other.__toklist + self.__accumNames.update( other.__accumNames ) + return self + + def __radd__(self, other): + if isinstance(other,int) and other == 0: + return self.copy() + + def __repr__( self ): + return "(%s, %s)" % ( repr( self.__toklist ), repr( self.__tokdict ) ) + + def __str__( self ): + out = "[" + sep = "" + for i in self.__toklist: + if isinstance(i, ParseResults): + out += sep + _ustr(i) + else: + out += sep + repr(i) + sep = ", " + out += "]" + return out + + def _asStringList( self, sep='' ): + out = [] + for item in self.__toklist: + if out and sep: + out.append(sep) + if isinstance( item, ParseResults ): + out += item._asStringList() + else: + out.append( _ustr(item) ) + return out + + def asList( self ): + """Returns the parse results as a nested list of matching tokens, all converted to strings.""" + out = [] + for res in self.__toklist: + if isinstance(res,ParseResults): + out.append( res.asList() ) + else: + out.append( res ) + return out + + def asDict( self ): + """Returns the named parse results as dictionary.""" + return dict( self.items() ) + + def copy( self ): + """Returns a new copy of a C{ParseResults} object.""" + ret = ParseResults( self.__toklist ) + ret.__tokdict = self.__tokdict.copy() + ret.__parent = self.__parent + ret.__accumNames.update( self.__accumNames ) + ret.__name = self.__name + return ret + + def asXML( self, doctag=None, namedItemsOnly=False, indent="", formatted=True ): + """Returns the parse results as XML. Tags are created for tokens and lists that have defined results names.""" + nl = "\n" + out = [] + namedItems = dict( [ (v[1],k) for (k,vlist) in self.__tokdict.items() + for v in vlist ] ) + nextLevelIndent = indent + " " + + # collapse out indents if formatting is not desired + if not formatted: + indent = "" + nextLevelIndent = "" + nl = "" + + selfTag = None + if doctag is not None: + selfTag = doctag + else: + if self.__name: + selfTag = self.__name + + if not selfTag: + if namedItemsOnly: + return "" + else: + selfTag = "ITEM" + + out += [ nl, indent, "<", selfTag, ">" ] + + worklist = self.__toklist + for i,res in enumerate(worklist): + if isinstance(res,ParseResults): + if i in namedItems: + out += [ res.asXML(namedItems[i], + namedItemsOnly and doctag is None, + nextLevelIndent, + formatted)] + else: + out += [ res.asXML(None, + namedItemsOnly and doctag is None, + nextLevelIndent, + formatted)] + else: + # individual token, see if there is a name for it + resTag = None + if i in namedItems: + resTag = namedItems[i] + if not resTag: + if namedItemsOnly: + continue + else: + resTag = "ITEM" + xmlBodyText = _xml_escape(_ustr(res)) + out += [ nl, nextLevelIndent, "<", resTag, ">", + xmlBodyText, + "" ] + + out += [ nl, indent, "" ] + return "".join(out) + + def __lookup(self,sub): + for k,vlist in self.__tokdict.items(): + for v,loc in vlist: + if sub is v: + return k + return None + + def getName(self): + """Returns the results name for this token expression.""" + if self.__name: + return self.__name + elif self.__parent: + par = self.__parent() + if par: + return par.__lookup(self) + else: + return None + elif (len(self) == 1 and + len(self.__tokdict) == 1 and + self.__tokdict.values()[0][0][1] in (0,-1)): + return self.__tokdict.keys()[0] + else: + return None + + def dump(self,indent='',depth=0): + """Diagnostic method for listing out the contents of a C{ParseResults}. + Accepts an optional C{indent} argument so that this string can be embedded + in a nested display of other data.""" + out = [] + out.append( indent+_ustr(self.asList()) ) + keys = self.items() + keys.sort() + for k,v in keys: + if out: + out.append('\n') + out.append( "%s%s- %s: " % (indent,(' '*depth), k) ) + if isinstance(v,ParseResults): + if v.keys(): + out.append( v.dump(indent,depth+1) ) + else: + out.append(_ustr(v)) + else: + out.append(_ustr(v)) + return "".join(out) + + # add support for pickle protocol + def __getstate__(self): + return ( self.__toklist, + ( self.__tokdict.copy(), + self.__parent is not None and self.__parent() or None, + self.__accumNames, + self.__name ) ) + + def __setstate__(self,state): + self.__toklist = state[0] + (self.__tokdict, + par, + inAccumNames, + self.__name) = state[1] + self.__accumNames = {} + self.__accumNames.update(inAccumNames) + if par is not None: + self.__parent = wkref(par) + else: + self.__parent = None + + def __dir__(self): + return dir(super(ParseResults,self)) + self.keys() + +collections.abc.MutableMapping.register(ParseResults) + +def col (loc,strg): + """Returns current column within a string, counting newlines as line separators. + The first column is number 1. + + Note: the default parsing behavior is to expand tabs in the input string + before starting the parsing process. See L{I{ParserElement.parseString}} for more information + on parsing strings containing s, and suggested methods to maintain a + consistent view of the parsed string, the parse location, and line and column + positions within the parsed string. + """ + return (loc} for more information + on parsing strings containing s, and suggested methods to maintain a + consistent view of the parsed string, the parse location, and line and column + positions within the parsed string. + """ + return strg.count("\n",0,loc) + 1 + +def line( loc, strg ): + """Returns the line of text containing loc within a string, counting newlines as line separators. + """ + lastCR = strg.rfind("\n", 0, loc) + nextCR = strg.find("\n", loc) + if nextCR >= 0: + return strg[lastCR+1:nextCR] + else: + return strg[lastCR+1:] + +def _defaultStartDebugAction( instring, loc, expr ): + print ("Match " + _ustr(expr) + " at loc " + _ustr(loc) + "(%d,%d)" % ( lineno(loc,instring), col(loc,instring) )) + +def _defaultSuccessDebugAction( instring, startloc, endloc, expr, toks ): + print ("Matched " + _ustr(expr) + " -> " + str(toks.asList())) + +def _defaultExceptionDebugAction( instring, loc, expr, exc ): + print ("Exception raised:" + _ustr(exc)) + +def nullDebugAction(*args): + """'Do-nothing' debug action, to suppress debugging output during parsing.""" + pass + +'decorator to trim function calls to match the arity of the target' +def _trim_arity(func, maxargs=2): + limit = maxargs + def wrapper(*args): + nonlocal limit + while 1: + try: + return func(*args[limit:]) + except TypeError: + if limit: + limit -= 1 + continue + raise + return wrapper + +class ParserElement(object): + """Abstract base level parser element class.""" + DEFAULT_WHITE_CHARS = " \n\t\r" + verbose_stacktrace = False + + def setDefaultWhitespaceChars( chars ): + """Overrides the default whitespace chars + """ + ParserElement.DEFAULT_WHITE_CHARS = chars + setDefaultWhitespaceChars = staticmethod(setDefaultWhitespaceChars) + + def __init__( self, savelist=False ): + self.parseAction = list() + self.failAction = None + #~ self.name = "" # don't define self.name, let subclasses try/except upcall + self.strRepr = None + self.resultsName = None + self.saveAsList = savelist + self.skipWhitespace = True + self.whiteChars = ParserElement.DEFAULT_WHITE_CHARS + self.copyDefaultWhiteChars = True + self.mayReturnEmpty = False # used when checking for left-recursion + self.keepTabs = False + self.ignoreExprs = list() + self.debug = False + self.streamlined = False + self.mayIndexError = True # used to optimize exception handling for subclasses that don't advance parse index + self.errmsg = "" + self.modalResults = True # used to mark results names as modal (report only last) or cumulative (list all) + self.debugActions = ( None, None, None ) #custom debug actions + self.re = None + self.callPreparse = True # used to avoid redundant calls to preParse + self.callDuringTry = False + + def copy( self ): + """Make a copy of this C{ParserElement}. Useful for defining different parse actions + for the same parsing pattern, using copies of the original parse element.""" + cpy = copy.copy( self ) + cpy.parseAction = self.parseAction[:] + cpy.ignoreExprs = self.ignoreExprs[:] + if self.copyDefaultWhiteChars: + cpy.whiteChars = ParserElement.DEFAULT_WHITE_CHARS + return cpy + + def setName( self, name ): + """Define name for this expression, for use in debugging.""" + self.name = name + self.errmsg = "Expected " + self.name + if hasattr(self,"exception"): + self.exception.msg = self.errmsg + return self + + def setResultsName( self, name, listAllMatches=False ): + """Define name for referencing matching tokens as a nested attribute + of the returned parse results. + NOTE: this returns a *copy* of the original C{ParserElement} object; + this is so that the client can define a basic element, such as an + integer, and reference it in multiple places with different names. + + You can also set results names using the abbreviated syntax, + C{expr("name")} in place of C{expr.setResultsName("name")} - + see L{I{__call__}<__call__>}. + """ + newself = self.copy() + if name.endswith("*"): + name = name[:-1] + listAllMatches=True + newself.resultsName = name + newself.modalResults = not listAllMatches + return newself + + def setBreak(self,breakFlag = True): + """Method to invoke the Python pdb debugger when this element is + about to be parsed. Set C{breakFlag} to True to enable, False to + disable. + """ + if breakFlag: + _parseMethod = self._parse + def breaker(instring, loc, doActions=True, callPreParse=True): + import pdb + pdb.set_trace() + return _parseMethod( instring, loc, doActions, callPreParse ) + breaker._originalParseMethod = _parseMethod + self._parse = breaker + else: + if hasattr(self._parse,"_originalParseMethod"): + self._parse = self._parse._originalParseMethod + return self + + def setParseAction( self, *fns, **kwargs ): + """Define action to perform when successfully matching parse element definition. + Parse action fn is a callable method with 0-3 arguments, called as C{fn(s,loc,toks)}, + C{fn(loc,toks)}, C{fn(toks)}, or just C{fn()}, where: + - s = the original string being parsed (see note below) + - loc = the location of the matching substring + - toks = a list of the matched tokens, packaged as a ParseResults object + If the functions in fns modify the tokens, they can return them as the return + value from fn, and the modified list of tokens will replace the original. + Otherwise, fn does not need to return any value. + + Note: the default parsing behavior is to expand tabs in the input string + before starting the parsing process. See L{I{parseString}} for more information + on parsing strings containing s, and suggested methods to maintain a + consistent view of the parsed string, the parse location, and line and column + positions within the parsed string. + """ + self.parseAction = list(map(_trim_arity, list(fns))) + self.callDuringTry = ("callDuringTry" in kwargs and kwargs["callDuringTry"]) + return self + + def addParseAction( self, *fns, **kwargs ): + """Add parse action to expression's list of parse actions. See L{I{setParseAction}}.""" + self.parseAction += list(map(_trim_arity, list(fns))) + self.callDuringTry = self.callDuringTry or ("callDuringTry" in kwargs and kwargs["callDuringTry"]) + return self + + def setFailAction( self, fn ): + """Define action to perform if parsing fails at this expression. + Fail acton fn is a callable function that takes the arguments + C{fn(s,loc,expr,err)} where: + - s = string being parsed + - loc = location where expression match was attempted and failed + - expr = the parse expression that failed + - err = the exception thrown + The function returns no value. It may throw C{ParseFatalException} + if it is desired to stop parsing immediately.""" + self.failAction = fn + return self + + def _skipIgnorables( self, instring, loc ): + exprsFound = True + while exprsFound: + exprsFound = False + for e in self.ignoreExprs: + try: + while 1: + loc,dummy = e._parse( instring, loc ) + exprsFound = True + except ParseException: + pass + return loc + + def preParse( self, instring, loc ): + if self.ignoreExprs: + loc = self._skipIgnorables( instring, loc ) + + if self.skipWhitespace: + wt = self.whiteChars + instrlen = len(instring) + while loc < instrlen and instring[loc] in wt: + loc += 1 + + return loc + + def parseImpl( self, instring, loc, doActions=True ): + return loc, [] + + def postParse( self, instring, loc, tokenlist ): + return tokenlist + + #~ @profile + def _parseNoCache( self, instring, loc, doActions=True, callPreParse=True ): + debugging = ( self.debug ) #and doActions ) + + if debugging or self.failAction: + #~ print ("Match",self,"at loc",loc,"(%d,%d)" % ( lineno(loc,instring), col(loc,instring) )) + if (self.debugActions[0] ): + self.debugActions[0]( instring, loc, self ) + if callPreParse and self.callPreparse: + preloc = self.preParse( instring, loc ) + else: + preloc = loc + tokensStart = preloc + try: + try: + loc,tokens = self.parseImpl( instring, preloc, doActions ) + except IndexError: + raise ParseException( instring, len(instring), self.errmsg, self ) + except ParseBaseException as err: + #~ print ("Exception raised:", err) + if self.debugActions[2]: + self.debugActions[2]( instring, tokensStart, self, err ) + if self.failAction: + self.failAction( instring, tokensStart, self, err ) + raise + else: + if callPreParse and self.callPreparse: + preloc = self.preParse( instring, loc ) + else: + preloc = loc + tokensStart = preloc + if self.mayIndexError or loc >= len(instring): + try: + loc,tokens = self.parseImpl( instring, preloc, doActions ) + except IndexError: + raise ParseException( instring, len(instring), self.errmsg, self ) + else: + loc,tokens = self.parseImpl( instring, preloc, doActions ) + + tokens = self.postParse( instring, loc, tokens ) + + retTokens = ParseResults( tokens, self.resultsName, asList=self.saveAsList, modal=self.modalResults ) + if self.parseAction and (doActions or self.callDuringTry): + if debugging: + try: + for fn in self.parseAction: + tokens = fn( instring, tokensStart, retTokens ) + if tokens is not None: + retTokens = ParseResults( tokens, + self.resultsName, + asList=self.saveAsList and isinstance(tokens,(ParseResults,list)), + modal=self.modalResults ) + except ParseBaseException as err: + #~ print "Exception raised in user parse action:", err + if (self.debugActions[2] ): + self.debugActions[2]( instring, tokensStart, self, err ) + raise + else: + for fn in self.parseAction: + tokens = fn( instring, tokensStart, retTokens ) + if tokens is not None: + retTokens = ParseResults( tokens, + self.resultsName, + asList=self.saveAsList and isinstance(tokens,(ParseResults,list)), + modal=self.modalResults ) + + if debugging: + #~ print ("Matched",self,"->",retTokens.asList()) + if (self.debugActions[1] ): + self.debugActions[1]( instring, tokensStart, loc, self, retTokens ) + + return loc, retTokens + + def tryParse( self, instring, loc ): + try: + return self._parse( instring, loc, doActions=False )[0] + except ParseFatalException: + raise ParseException( instring, loc, self.errmsg, self) + + # this method gets repeatedly called during backtracking with the same arguments - + # we can cache these arguments and save ourselves the trouble of re-parsing the contained expression + def _parseCache( self, instring, loc, doActions=True, callPreParse=True ): + lookup = (self,instring,loc,callPreParse,doActions) + if lookup in ParserElement._exprArgCache: + value = ParserElement._exprArgCache[ lookup ] + if isinstance(value, Exception): + raise value + return (value[0],value[1].copy()) + else: + try: + value = self._parseNoCache( instring, loc, doActions, callPreParse ) + ParserElement._exprArgCache[ lookup ] = (value[0],value[1].copy()) + return value + except ParseBaseException as pe: + exc.__traceback__ = None + ParserElement._exprArgCache[ lookup ] = pe + raise + + _parse = _parseNoCache + + # argument cache for optimizing repeated calls when backtracking through recursive expressions + _exprArgCache = {} + def resetCache(): + ParserElement._exprArgCache.clear() + resetCache = staticmethod(resetCache) + + _packratEnabled = False + def enablePackrat(): + """Enables "packrat" parsing, which adds memoizing to the parsing logic. + Repeated parse attempts at the same string location (which happens + often in many complex grammars) can immediately return a cached value, + instead of re-executing parsing/validating code. Memoizing is done of + both valid results and parsing exceptions. + + This speedup may break existing programs that use parse actions that + have side-effects. For this reason, packrat parsing is disabled when + you first import pyparsing. To activate the packrat feature, your + program must call the class method C{ParserElement.enablePackrat()}. If + your program uses C{psyco} to "compile as you go", you must call + C{enablePackrat} before calling C{psyco.full()}. If you do not do this, + Python will crash. For best results, call C{enablePackrat()} immediately + after importing pyparsing. + """ + if not ParserElement._packratEnabled: + ParserElement._packratEnabled = True + ParserElement._parse = ParserElement._parseCache + enablePackrat = staticmethod(enablePackrat) + + def parseString( self, instring, parseAll=False ): + """Execute the parse expression with the given string. + This is the main interface to the client code, once the complete + expression has been built. + + If you want the grammar to require that the entire input string be + successfully parsed, then set C{parseAll} to True (equivalent to ending + the grammar with C{StringEnd()}). + + Note: C{parseString} implicitly calls C{expandtabs()} on the input string, + in order to report proper column numbers in parse actions. + If the input string contains tabs and + the grammar uses parse actions that use the C{loc} argument to index into the + string being parsed, you can ensure you have a consistent view of the input + string by: + - calling C{parseWithTabs} on your grammar before calling C{parseString} + (see L{I{parseWithTabs}}) + - define your parse action using the full C{(s,loc,toks)} signature, and + reference the input string using the parse action's C{s} argument + - explictly expand the tabs in your input string before calling + C{parseString} + """ + ParserElement.resetCache() + if not self.streamlined: + self.streamline() + #~ self.saveAsList = True + for e in self.ignoreExprs: + e.streamline() + if not self.keepTabs: + instring = instring.expandtabs() + try: + loc, tokens = self._parse( instring, 0 ) + if parseAll: + loc = self.preParse( instring, loc ) + se = Empty() + StringEnd() + se._parse( instring, loc ) + except ParseBaseException as exc: + if ParserElement.verbose_stacktrace: + raise + else: + # catch and re-raise exception from here, clears out pyparsing internal stack trace + raise exc + else: + return tokens + + def scanString( self, instring, maxMatches=_MAX_INT, overlap=False ): + """Scan the input string for expression matches. Each match will return the + matching tokens, start location, and end location. May be called with optional + C{maxMatches} argument, to clip scanning after 'n' matches are found. If + C{overlap} is specified, then overlapping matches will be reported. + + Note that the start and end locations are reported relative to the string + being parsed. See L{I{parseString}} for more information on parsing + strings with embedded tabs.""" + if not self.streamlined: + self.streamline() + for e in self.ignoreExprs: + e.streamline() + + if not self.keepTabs: + instring = _ustr(instring).expandtabs() + instrlen = len(instring) + loc = 0 + preparseFn = self.preParse + parseFn = self._parse + ParserElement.resetCache() + matches = 0 + try: + while loc <= instrlen and matches < maxMatches: + try: + preloc = preparseFn( instring, loc ) + nextLoc,tokens = parseFn( instring, preloc, callPreParse=False ) + except ParseException: + loc = preloc+1 + else: + if nextLoc > loc: + matches += 1 + yield tokens, preloc, nextLoc + if overlap: + nextloc = preparseFn( instring, loc ) + if nextloc > loc: + loc = nextLoc + else: + loc += 1 + else: + loc = nextLoc + else: + loc = preloc+1 + except ParseBaseException as exc: + if ParserElement.verbose_stacktrace: + raise + else: + # catch and re-raise exception from here, clears out pyparsing internal stack trace + raise exc + + def transformString( self, instring ): + """Extension to C{scanString}, to modify matching text with modified tokens that may + be returned from a parse action. To use C{transformString}, define a grammar and + attach a parse action to it that modifies the returned token list. + Invoking C{transformString()} on a target string will then scan for matches, + and replace the matched text patterns according to the logic in the parse + action. C{transformString()} returns the resulting transformed string.""" + out = [] + lastE = 0 + # force preservation of s, to minimize unwanted transformation of string, and to + # keep string locs straight between transformString and scanString + self.keepTabs = True + try: + for t,s,e in self.scanString( instring ): + out.append( instring[lastE:s] ) + if t: + if isinstance(t,ParseResults): + out += t.asList() + elif isinstance(t,list): + out += t + else: + out.append(t) + lastE = e + out.append(instring[lastE:]) + out = [o for o in out if o] + return "".join(map(_ustr,_flatten(out))) + except ParseBaseException as exc: + if ParserElement.verbose_stacktrace: + raise + else: + # catch and re-raise exception from here, clears out pyparsing internal stack trace + raise exc + + def searchString( self, instring, maxMatches=_MAX_INT ): + """Another extension to C{scanString}, simplifying the access to the tokens found + to match the given parse expression. May be called with optional + C{maxMatches} argument, to clip searching after 'n' matches are found. + """ + try: + return ParseResults([ t for t,s,e in self.scanString( instring, maxMatches ) ]) + except ParseBaseException as exc: + if ParserElement.verbose_stacktrace: + raise + else: + # catch and re-raise exception from here, clears out pyparsing internal stack trace + raise exc + + def __add__(self, other ): + """Implementation of + operator - returns And""" + if isinstance( other, basestring ): + other = Literal( other ) + if not isinstance( other, ParserElement ): + warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, stacklevel=2) + return None + return And( [ self, other ] ) + + def __radd__(self, other ): + """Implementation of + operator when left operand is not a C{ParserElement}""" + if isinstance( other, basestring ): + other = Literal( other ) + if not isinstance( other, ParserElement ): + warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, stacklevel=2) + return None + return other + self + + def __sub__(self, other): + """Implementation of - operator, returns C{And} with error stop""" + if isinstance( other, basestring ): + other = Literal( other ) + if not isinstance( other, ParserElement ): + warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, stacklevel=2) + return None + return And( [ self, And._ErrorStop(), other ] ) + + def __rsub__(self, other ): + """Implementation of - operator when left operand is not a C{ParserElement}""" + if isinstance( other, basestring ): + other = Literal( other ) + if not isinstance( other, ParserElement ): + warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, stacklevel=2) + return None + return other - self + + def __mul__(self,other): + """Implementation of * operator, allows use of C{expr * 3} in place of + C{expr + expr + expr}. Expressions may also me multiplied by a 2-integer + tuple, similar to C{{min,max}} multipliers in regular expressions. Tuples + may also include C{None} as in: + - C{expr*(n,None)} or C{expr*(n,)} is equivalent + to C{expr*n + ZeroOrMore(expr)} + (read as "at least n instances of C{expr}") + - C{expr*(None,n)} is equivalent to C{expr*(0,n)} + (read as "0 to n instances of C{expr}") + - C{expr*(None,None)} is equivalent to C{ZeroOrMore(expr)} + - C{expr*(1,None)} is equivalent to C{OneOrMore(expr)} + + Note that C{expr*(None,n)} does not raise an exception if + more than n exprs exist in the input stream; that is, + C{expr*(None,n)} does not enforce a maximum number of expr + occurrences. If this behavior is desired, then write + C{expr*(None,n) + ~expr} + + """ + if isinstance(other,int): + minElements, optElements = other,0 + elif isinstance(other,tuple): + other = (other + (None, None))[:2] + if other[0] is None: + other = (0, other[1]) + if isinstance(other[0],int) and other[1] is None: + if other[0] == 0: + return ZeroOrMore(self) + if other[0] == 1: + return OneOrMore(self) + else: + return self*other[0] + ZeroOrMore(self) + elif isinstance(other[0],int) and isinstance(other[1],int): + minElements, optElements = other + optElements -= minElements + else: + raise TypeError("cannot multiply 'ParserElement' and ('%s','%s') objects", type(other[0]),type(other[1])) + else: + raise TypeError("cannot multiply 'ParserElement' and '%s' objects", type(other)) + + if minElements < 0: + raise ValueError("cannot multiply ParserElement by negative value") + if optElements < 0: + raise ValueError("second tuple value must be greater or equal to first tuple value") + if minElements == optElements == 0: + raise ValueError("cannot multiply ParserElement by 0 or (0,0)") + + if (optElements): + def makeOptionalList(n): + if n>1: + return Optional(self + makeOptionalList(n-1)) + else: + return Optional(self) + if minElements: + if minElements == 1: + ret = self + makeOptionalList(optElements) + else: + ret = And([self]*minElements) + makeOptionalList(optElements) + else: + ret = makeOptionalList(optElements) + else: + if minElements == 1: + ret = self + else: + ret = And([self]*minElements) + return ret + + def __rmul__(self, other): + return self.__mul__(other) + + def __or__(self, other ): + """Implementation of | operator - returns C{MatchFirst}""" + if isinstance( other, basestring ): + other = Literal( other ) + if not isinstance( other, ParserElement ): + warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, stacklevel=2) + return None + return MatchFirst( [ self, other ] ) + + def __ror__(self, other ): + """Implementation of | operator when left operand is not a C{ParserElement}""" + if isinstance( other, basestring ): + other = Literal( other ) + if not isinstance( other, ParserElement ): + warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, stacklevel=2) + return None + return other | self + + def __xor__(self, other ): + """Implementation of ^ operator - returns C{Or}""" + if isinstance( other, basestring ): + other = Literal( other ) + if not isinstance( other, ParserElement ): + warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, stacklevel=2) + return None + return Or( [ self, other ] ) + + def __rxor__(self, other ): + """Implementation of ^ operator when left operand is not a C{ParserElement}""" + if isinstance( other, basestring ): + other = Literal( other ) + if not isinstance( other, ParserElement ): + warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, stacklevel=2) + return None + return other ^ self + + def __and__(self, other ): + """Implementation of & operator - returns C{Each}""" + if isinstance( other, basestring ): + other = Literal( other ) + if not isinstance( other, ParserElement ): + warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, stacklevel=2) + return None + return Each( [ self, other ] ) + + def __rand__(self, other ): + """Implementation of & operator when left operand is not a C{ParserElement}""" + if isinstance( other, basestring ): + other = Literal( other ) + if not isinstance( other, ParserElement ): + warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, stacklevel=2) + return None + return other & self + + def __invert__( self ): + """Implementation of ~ operator - returns C{NotAny}""" + return NotAny( self ) + + def __call__(self, name): + """Shortcut for C{setResultsName}, with C{listAllMatches=default}:: + userdata = Word(alphas).setResultsName("name") + Word(nums+"-").setResultsName("socsecno") + could be written as:: + userdata = Word(alphas)("name") + Word(nums+"-")("socsecno") + + If C{name} is given with a trailing C{'*'} character, then C{listAllMatches} will be + passed as C{True}. + """ + return self.setResultsName(name) + + def suppress( self ): + """Suppresses the output of this C{ParserElement}; useful to keep punctuation from + cluttering up returned output. + """ + return Suppress( self ) + + def leaveWhitespace( self ): + """Disables the skipping of whitespace before matching the characters in the + C{ParserElement}'s defined pattern. This is normally only used internally by + the pyparsing module, but may be needed in some whitespace-sensitive grammars. + """ + self.skipWhitespace = False + return self + + def setWhitespaceChars( self, chars ): + """Overrides the default whitespace chars + """ + self.skipWhitespace = True + self.whiteChars = chars + self.copyDefaultWhiteChars = False + return self + + def parseWithTabs( self ): + """Overrides default behavior to expand C{}s to spaces before parsing the input string. + Must be called before C{parseString} when the input grammar contains elements that + match C{} characters.""" + self.keepTabs = True + return self + + def ignore( self, other ): + """Define expression to be ignored (e.g., comments) while doing pattern + matching; may be called repeatedly, to define multiple comment or other + ignorable patterns. + """ + if isinstance( other, Suppress ): + if other not in self.ignoreExprs: + self.ignoreExprs.append( other.copy() ) + else: + self.ignoreExprs.append( Suppress( other.copy() ) ) + return self + + def setDebugActions( self, startAction, successAction, exceptionAction ): + """Enable display of debugging messages while doing pattern matching.""" + self.debugActions = (startAction or _defaultStartDebugAction, + successAction or _defaultSuccessDebugAction, + exceptionAction or _defaultExceptionDebugAction) + self.debug = True + return self + + def setDebug( self, flag=True ): + """Enable display of debugging messages while doing pattern matching. + Set C{flag} to True to enable, False to disable.""" + if flag: + self.setDebugActions( _defaultStartDebugAction, _defaultSuccessDebugAction, _defaultExceptionDebugAction ) + else: + self.debug = False + return self + + def __str__( self ): + return self.name + + def __repr__( self ): + return _ustr(self) + + def streamline( self ): + self.streamlined = True + self.strRepr = None + return self + + def checkRecursion( self, parseElementList ): + pass + + def validate( self, validateTrace=[] ): + """Check defined expressions for valid structure, check for infinite recursive definitions.""" + self.checkRecursion( [] ) + + def parseFile( self, file_or_filename, parseAll=False ): + """Execute the parse expression on the given file or filename. + If a filename is specified (instead of a file object), + the entire file is opened, read, and closed before parsing. + """ + try: + file_contents = file_or_filename.read() + except AttributeError: + f = open(file_or_filename, "rb") + file_contents = f.read() + f.close() + try: + return self.parseString(file_contents, parseAll) + except ParseBaseException as exc: + if ParserElement.verbose_stacktrace: + raise + else: + # catch and re-raise exception from here, clears out pyparsing internal stack trace + raise exc + + def __eq__(self,other): + if isinstance(other, ParserElement): + return self is other or self.__dict__ == other.__dict__ + elif isinstance(other, basestring): + try: + self.parseString(_ustr(other), parseAll=True) + return True + except ParseBaseException: + return False + else: + return super(ParserElement,self)==other + + def __ne__(self,other): + return not (self == other) + + def __hash__(self): + return hash(id(self)) + + def __req__(self,other): + return self == other + + def __rne__(self,other): + return not (self == other) + + +class Token(ParserElement): + """Abstract C{ParserElement} subclass, for defining atomic matching patterns.""" + def __init__( self ): + super(Token,self).__init__( savelist=False ) + + def setName(self, name): + s = super(Token,self).setName(name) + self.errmsg = "Expected " + self.name + return s + + +class Empty(Token): + """An empty token, will always match.""" + def __init__( self ): + super(Empty,self).__init__() + self.name = "Empty" + self.mayReturnEmpty = True + self.mayIndexError = False + + +class NoMatch(Token): + """A token that will never match.""" + def __init__( self ): + super(NoMatch,self).__init__() + self.name = "NoMatch" + self.mayReturnEmpty = True + self.mayIndexError = False + self.errmsg = "Unmatchable token" + + def parseImpl( self, instring, loc, doActions=True ): + raise ParseException(instring, loc, self.errmsg, self) + + +class Literal(Token): + """Token to exactly match a specified string.""" + def __init__( self, matchString ): + super(Literal,self).__init__() + self.match = matchString + self.matchLen = len(matchString) + try: + self.firstMatchChar = matchString[0] + except IndexError: + warnings.warn("null string passed to Literal; use Empty() instead", + SyntaxWarning, stacklevel=2) + self.__class__ = Empty + self.name = '"%s"' % _ustr(self.match) + self.errmsg = "Expected " + self.name + self.mayReturnEmpty = False + self.mayIndexError = False + + # Performance tuning: this routine gets called a *lot* + # if this is a single character match string and the first character matches, + # short-circuit as quickly as possible, and avoid calling startswith + #~ @profile + def parseImpl( self, instring, loc, doActions=True ): + if (instring[loc] == self.firstMatchChar and + (self.matchLen==1 or instring.startswith(self.match,loc)) ): + return loc+self.matchLen, self.match + raise ParseException(instring, loc, self.errmsg, self) +_L = Literal + +class Keyword(Token): + """Token to exactly match a specified string as a keyword, that is, it must be + immediately followed by a non-keyword character. Compare with C{Literal}:: + Literal("if") will match the leading C{'if'} in C{'ifAndOnlyIf'}. + Keyword("if") will not; it will only match the leading C{'if'} in C{'if x=1'}, or C{'if(y==2)'} + Accepts two optional constructor arguments in addition to the keyword string: + C{identChars} is a string of characters that would be valid identifier characters, + defaulting to all alphanumerics + "_" and "$"; C{caseless} allows case-insensitive + matching, default is C{False}. + """ + DEFAULT_KEYWORD_CHARS = alphanums+"_$" + + def __init__( self, matchString, identChars=DEFAULT_KEYWORD_CHARS, caseless=False ): + super(Keyword,self).__init__() + self.match = matchString + self.matchLen = len(matchString) + try: + self.firstMatchChar = matchString[0] + except IndexError: + warnings.warn("null string passed to Keyword; use Empty() instead", + SyntaxWarning, stacklevel=2) + self.name = '"%s"' % self.match + self.errmsg = "Expected " + self.name + self.mayReturnEmpty = False + self.mayIndexError = False + self.caseless = caseless + if caseless: + self.caselessmatch = matchString.upper() + identChars = identChars.upper() + self.identChars = set(identChars) + + def parseImpl( self, instring, loc, doActions=True ): + if self.caseless: + if ( (instring[ loc:loc+self.matchLen ].upper() == self.caselessmatch) and + (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen].upper() not in self.identChars) and + (loc == 0 or instring[loc-1].upper() not in self.identChars) ): + return loc+self.matchLen, self.match + else: + if (instring[loc] == self.firstMatchChar and + (self.matchLen==1 or instring.startswith(self.match,loc)) and + (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen] not in self.identChars) and + (loc == 0 or instring[loc-1] not in self.identChars) ): + return loc+self.matchLen, self.match + raise ParseException(instring, loc, self.errmsg, self) + + def copy(self): + c = super(Keyword,self).copy() + c.identChars = Keyword.DEFAULT_KEYWORD_CHARS + return c + + def setDefaultKeywordChars( chars ): + """Overrides the default Keyword chars + """ + Keyword.DEFAULT_KEYWORD_CHARS = chars + setDefaultKeywordChars = staticmethod(setDefaultKeywordChars) + +class CaselessLiteral(Literal): + """Token to match a specified string, ignoring case of letters. + Note: the matched results will always be in the case of the given + match string, NOT the case of the input text. + """ + def __init__( self, matchString ): + super(CaselessLiteral,self).__init__( matchString.upper() ) + # Preserve the defining literal. + self.returnString = matchString + self.name = "'%s'" % self.returnString + self.errmsg = "Expected " + self.name + + def parseImpl( self, instring, loc, doActions=True ): + if instring[ loc:loc+self.matchLen ].upper() == self.match: + return loc+self.matchLen, self.returnString + raise ParseException(instring, loc, self.errmsg, self) + +class CaselessKeyword(Keyword): + def __init__( self, matchString, identChars=Keyword.DEFAULT_KEYWORD_CHARS ): + super(CaselessKeyword,self).__init__( matchString, identChars, caseless=True ) + + def parseImpl( self, instring, loc, doActions=True ): + if ( (instring[ loc:loc+self.matchLen ].upper() == self.caselessmatch) and + (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen].upper() not in self.identChars) ): + return loc+self.matchLen, self.match + raise ParseException(instring, loc, self.errmsg, self) + +class Word(Token): + """Token for matching words composed of allowed character sets. + Defined with string containing all allowed initial characters, + an optional string containing allowed body characters (if omitted, + defaults to the initial character set), and an optional minimum, + maximum, and/or exact length. The default value for C{min} is 1 (a + minimum value < 1 is not valid); the default values for C{max} and C{exact} + are 0, meaning no maximum or exact length restriction. An optional + C{exclude} parameter can list characters that might be found in + the input C{bodyChars} string; useful to define a word of all printables + except for one or two characters, for instance. + """ + def __init__( self, initChars, bodyChars=None, min=1, max=0, exact=0, asKeyword=False, excludeChars=None ): + super(Word,self).__init__() + if excludeChars: + initChars = ''.join([c for c in initChars if c not in excludeChars]) + if bodyChars: + bodyChars = ''.join([c for c in bodyChars if c not in excludeChars]) + self.initCharsOrig = initChars + self.initChars = set(initChars) + if bodyChars : + self.bodyCharsOrig = bodyChars + self.bodyChars = set(bodyChars) + else: + self.bodyCharsOrig = initChars + self.bodyChars = set(initChars) + + self.maxSpecified = max > 0 + + if min < 1: + raise ValueError("cannot specify a minimum length < 1; use Optional(Word()) if zero-length word is permitted") + + self.minLen = min + + if max > 0: + self.maxLen = max + else: + self.maxLen = _MAX_INT + + if exact > 0: + self.maxLen = exact + self.minLen = exact + + self.name = _ustr(self) + self.errmsg = "Expected " + self.name + self.mayIndexError = False + self.asKeyword = asKeyword + + if ' ' not in self.initCharsOrig+self.bodyCharsOrig and (min==1 and max==0 and exact==0): + if self.bodyCharsOrig == self.initCharsOrig: + self.reString = "[%s]+" % _escapeRegexRangeChars(self.initCharsOrig) + elif len(self.bodyCharsOrig) == 1: + self.reString = "%s[%s]*" % \ + (re.escape(self.initCharsOrig), + _escapeRegexRangeChars(self.bodyCharsOrig),) + else: + self.reString = "[%s][%s]*" % \ + (_escapeRegexRangeChars(self.initCharsOrig), + _escapeRegexRangeChars(self.bodyCharsOrig),) + if self.asKeyword: + self.reString = r"\b"+self.reString+r"\b" + try: + self.re = re.compile( self.reString ) + except: + self.re = None + + def parseImpl( self, instring, loc, doActions=True ): + if self.re: + result = self.re.match(instring,loc) + if not result: + raise ParseException(instring, loc, self.errmsg, self) + + loc = result.end() + return loc, result.group() + + if not(instring[ loc ] in self.initChars): + raise ParseException(instring, loc, self.errmsg, self) + + start = loc + loc += 1 + instrlen = len(instring) + bodychars = self.bodyChars + maxloc = start + self.maxLen + maxloc = min( maxloc, instrlen ) + while loc < maxloc and instring[loc] in bodychars: + loc += 1 + + throwException = False + if loc - start < self.minLen: + throwException = True + if self.maxSpecified and loc < instrlen and instring[loc] in bodychars: + throwException = True + if self.asKeyword: + if (start>0 and instring[start-1] in bodychars) or (loc4: + return s[:4]+"..." + else: + return s + + if ( self.initCharsOrig != self.bodyCharsOrig ): + self.strRepr = "W:(%s,%s)" % ( charsAsStr(self.initCharsOrig), charsAsStr(self.bodyCharsOrig) ) + else: + self.strRepr = "W:(%s)" % charsAsStr(self.initCharsOrig) + + return self.strRepr + + +class Regex(Token): + """Token for matching strings that match a given regular expression. + Defined with string specifying the regular expression in a form recognized by the inbuilt Python re module. + """ + compiledREtype = type(re.compile("[A-Z]")) + def __init__( self, pattern, flags=0): + """The parameters C{pattern} and C{flags} are passed to the C{re.compile()} function as-is. See the Python C{re} module for an explanation of the acceptable patterns and flags.""" + super(Regex,self).__init__() + + if isinstance(pattern, basestring): + if len(pattern) == 0: + warnings.warn("null string passed to Regex; use Empty() instead", + SyntaxWarning, stacklevel=2) + + self.pattern = pattern + self.flags = flags + + try: + self.re = re.compile(self.pattern, self.flags) + self.reString = self.pattern + except sre_constants.error: + warnings.warn("invalid pattern (%s) passed to Regex" % pattern, + SyntaxWarning, stacklevel=2) + raise + + elif isinstance(pattern, Regex.compiledREtype): + self.re = pattern + self.pattern = \ + self.reString = str(pattern) + self.flags = flags + + else: + raise ValueError("Regex may only be constructed with a string or a compiled RE object") + + self.name = _ustr(self) + self.errmsg = "Expected " + self.name + self.mayIndexError = False + self.mayReturnEmpty = True + + def parseImpl( self, instring, loc, doActions=True ): + result = self.re.match(instring,loc) + if not result: + raise ParseException(instring, loc, self.errmsg, self) + + loc = result.end() + d = result.groupdict() + ret = ParseResults(result.group()) + if d: + for k in d: + ret[k] = d[k] + return loc,ret + + def __str__( self ): + try: + return super(Regex,self).__str__() + except: + pass + + if self.strRepr is None: + self.strRepr = "Re:(%s)" % repr(self.pattern) + + return self.strRepr + + +class QuotedString(Token): + """Token for matching strings that are delimited by quoting characters. + """ + def __init__( self, quoteChar, escChar=None, escQuote=None, multiline=False, unquoteResults=True, endQuoteChar=None): + """ + Defined with the following parameters: + - quoteChar - string of one or more characters defining the quote delimiting string + - escChar - character to escape quotes, typically backslash (default=None) + - escQuote - special quote sequence to escape an embedded quote string (such as SQL's "" to escape an embedded ") (default=None) + - multiline - boolean indicating whether quotes can span multiple lines (default=False) + - unquoteResults - boolean indicating whether the matched text should be unquoted (default=True) + - endQuoteChar - string of one or more characters defining the end of the quote delimited string (default=None => same as quoteChar) + """ + super(QuotedString,self).__init__() + + # remove white space from quote chars - wont work anyway + quoteChar = quoteChar.strip() + if len(quoteChar) == 0: + warnings.warn("quoteChar cannot be the empty string",SyntaxWarning,stacklevel=2) + raise SyntaxError() + + if endQuoteChar is None: + endQuoteChar = quoteChar + else: + endQuoteChar = endQuoteChar.strip() + if len(endQuoteChar) == 0: + warnings.warn("endQuoteChar cannot be the empty string",SyntaxWarning,stacklevel=2) + raise SyntaxError() + + self.quoteChar = quoteChar + self.quoteCharLen = len(quoteChar) + self.firstQuoteChar = quoteChar[0] + self.endQuoteChar = endQuoteChar + self.endQuoteCharLen = len(endQuoteChar) + self.escChar = escChar + self.escQuote = escQuote + self.unquoteResults = unquoteResults + + if multiline: + self.flags = re.MULTILINE | re.DOTALL + self.pattern = r'%s(?:[^%s%s]' % \ + ( re.escape(self.quoteChar), + _escapeRegexRangeChars(self.endQuoteChar[0]), + (escChar is not None and _escapeRegexRangeChars(escChar) or '') ) + else: + self.flags = 0 + self.pattern = r'%s(?:[^%s\n\r%s]' % \ + ( re.escape(self.quoteChar), + _escapeRegexRangeChars(self.endQuoteChar[0]), + (escChar is not None and _escapeRegexRangeChars(escChar) or '') ) + if len(self.endQuoteChar) > 1: + self.pattern += ( + '|(?:' + ')|(?:'.join(["%s[^%s]" % (re.escape(self.endQuoteChar[:i]), + _escapeRegexRangeChars(self.endQuoteChar[i])) + for i in range(len(self.endQuoteChar)-1,0,-1)]) + ')' + ) + if escQuote: + self.pattern += (r'|(?:%s)' % re.escape(escQuote)) + if escChar: + self.pattern += (r'|(?:%s.)' % re.escape(escChar)) + charset = ''.join(set(self.quoteChar[0]+self.endQuoteChar[0])).replace('^',r'\^').replace('-',r'\-') + self.escCharReplacePattern = re.escape(self.escChar)+("([%s])" % charset) + self.pattern += (r')*%s' % re.escape(self.endQuoteChar)) + + try: + self.re = re.compile(self.pattern, self.flags) + self.reString = self.pattern + except sre_constants.error: + warnings.warn("invalid pattern (%s) passed to Regex" % self.pattern, + SyntaxWarning, stacklevel=2) + raise + + self.name = _ustr(self) + self.errmsg = "Expected " + self.name + self.mayIndexError = False + self.mayReturnEmpty = True + + def parseImpl( self, instring, loc, doActions=True ): + result = instring[loc] == self.firstQuoteChar and self.re.match(instring,loc) or None + if not result: + raise ParseException(instring, loc, self.errmsg, self) + + loc = result.end() + ret = result.group() + + if self.unquoteResults: + + # strip off quotes + ret = ret[self.quoteCharLen:-self.endQuoteCharLen] + + if isinstance(ret,basestring): + # replace escaped characters + if self.escChar: + ret = re.sub(self.escCharReplacePattern, r"\g<1>", ret) + + # replace escaped quotes + if self.escQuote: + ret = ret.replace(self.escQuote, self.endQuoteChar) + + return loc, ret + + def __str__( self ): + try: + return super(QuotedString,self).__str__() + except: + pass + + if self.strRepr is None: + self.strRepr = "quoted string, starting with %s ending with %s" % (self.quoteChar, self.endQuoteChar) + + return self.strRepr + + +class CharsNotIn(Token): + """Token for matching words composed of characters *not* in a given set. + Defined with string containing all disallowed characters, and an optional + minimum, maximum, and/or exact length. The default value for C{min} is 1 (a + minimum value < 1 is not valid); the default values for C{max} and C{exact} + are 0, meaning no maximum or exact length restriction. + """ + def __init__( self, notChars, min=1, max=0, exact=0 ): + super(CharsNotIn,self).__init__() + self.skipWhitespace = False + self.notChars = notChars + + if min < 1: + raise ValueError("cannot specify a minimum length < 1; use Optional(CharsNotIn()) if zero-length char group is permitted") + + self.minLen = min + + if max > 0: + self.maxLen = max + else: + self.maxLen = _MAX_INT + + if exact > 0: + self.maxLen = exact + self.minLen = exact + + self.name = _ustr(self) + self.errmsg = "Expected " + self.name + self.mayReturnEmpty = ( self.minLen == 0 ) + self.mayIndexError = False + + def parseImpl( self, instring, loc, doActions=True ): + if instring[loc] in self.notChars: + raise ParseException(instring, loc, self.errmsg, self) + + start = loc + loc += 1 + notchars = self.notChars + maxlen = min( start+self.maxLen, len(instring) ) + while loc < maxlen and \ + (instring[loc] not in notchars): + loc += 1 + + if loc - start < self.minLen: + raise ParseException(instring, loc, self.errmsg, self) + + return loc, instring[start:loc] + + def __str__( self ): + try: + return super(CharsNotIn, self).__str__() + except: + pass + + if self.strRepr is None: + if len(self.notChars) > 4: + self.strRepr = "!W:(%s...)" % self.notChars[:4] + else: + self.strRepr = "!W:(%s)" % self.notChars + + return self.strRepr + +class White(Token): + """Special matching class for matching whitespace. Normally, whitespace is ignored + by pyparsing grammars. This class is included when some whitespace structures + are significant. Define with a string containing the whitespace characters to be + matched; default is C{" \\t\\r\\n"}. Also takes optional C{min}, C{max}, and C{exact} arguments, + as defined for the C{Word} class.""" + whiteStrs = { + " " : "", + "\t": "", + "\n": "", + "\r": "", + "\f": "", + } + def __init__(self, ws=" \t\r\n", min=1, max=0, exact=0): + super(White,self).__init__() + self.matchWhite = ws + self.setWhitespaceChars( "".join([c for c in self.whiteChars if c not in self.matchWhite]) ) + #~ self.leaveWhitespace() + self.name = ("".join([White.whiteStrs[c] for c in self.matchWhite])) + self.mayReturnEmpty = True + self.errmsg = "Expected " + self.name + + self.minLen = min + + if max > 0: + self.maxLen = max + else: + self.maxLen = _MAX_INT + + if exact > 0: + self.maxLen = exact + self.minLen = exact + + def parseImpl( self, instring, loc, doActions=True ): + if not(instring[ loc ] in self.matchWhite): + raise ParseException(instring, loc, self.errmsg, self) + start = loc + loc += 1 + maxloc = start + self.maxLen + maxloc = min( maxloc, len(instring) ) + while loc < maxloc and instring[loc] in self.matchWhite: + loc += 1 + + if loc - start < self.minLen: + raise ParseException(instring, loc, self.errmsg, self) + + return loc, instring[start:loc] + + +class _PositionToken(Token): + def __init__( self ): + super(_PositionToken,self).__init__() + self.name=self.__class__.__name__ + self.mayReturnEmpty = True + self.mayIndexError = False + +class GoToColumn(_PositionToken): + """Token to advance to a specific column of input text; useful for tabular report scraping.""" + def __init__( self, colno ): + super(GoToColumn,self).__init__() + self.col = colno + + def preParse( self, instring, loc ): + if col(loc,instring) != self.col: + instrlen = len(instring) + if self.ignoreExprs: + loc = self._skipIgnorables( instring, loc ) + while loc < instrlen and instring[loc].isspace() and col( loc, instring ) != self.col : + loc += 1 + return loc + + def parseImpl( self, instring, loc, doActions=True ): + thiscol = col( loc, instring ) + if thiscol > self.col: + raise ParseException( instring, loc, "Text not in expected column", self ) + newloc = loc + self.col - thiscol + ret = instring[ loc: newloc ] + return newloc, ret + +class LineStart(_PositionToken): + """Matches if current position is at the beginning of a line within the parse string""" + def __init__( self ): + super(LineStart,self).__init__() + self.setWhitespaceChars( ParserElement.DEFAULT_WHITE_CHARS.replace("\n","") ) + self.errmsg = "Expected start of line" + + def preParse( self, instring, loc ): + preloc = super(LineStart,self).preParse(instring,loc) + if instring[preloc] == "\n": + loc += 1 + return loc + + def parseImpl( self, instring, loc, doActions=True ): + if not( loc==0 or + (loc == self.preParse( instring, 0 )) or + (instring[loc-1] == "\n") ): #col(loc, instring) != 1: + raise ParseException(instring, loc, self.errmsg, self) + return loc, [] + +class LineEnd(_PositionToken): + """Matches if current position is at the end of a line within the parse string""" + def __init__( self ): + super(LineEnd,self).__init__() + self.setWhitespaceChars( ParserElement.DEFAULT_WHITE_CHARS.replace("\n","") ) + self.errmsg = "Expected end of line" + + def parseImpl( self, instring, loc, doActions=True ): + if loc len(instring): + return loc, [] + else: + raise ParseException(instring, loc, self.errmsg, self) + +class WordStart(_PositionToken): + """Matches if the current position is at the beginning of a Word, and + is not preceded by any character in a given set of C{wordChars} + (default=C{printables}). To emulate the C{\b} behavior of regular expressions, + use C{WordStart(alphanums)}. C{WordStart} will also match at the beginning of + the string being parsed, or at the beginning of a line. + """ + def __init__(self, wordChars = printables): + super(WordStart,self).__init__() + self.wordChars = set(wordChars) + self.errmsg = "Not at the start of a word" + + def parseImpl(self, instring, loc, doActions=True ): + if loc != 0: + if (instring[loc-1] in self.wordChars or + instring[loc] not in self.wordChars): + raise ParseException(instring, loc, self.errmsg, self) + return loc, [] + +class WordEnd(_PositionToken): + """Matches if the current position is at the end of a Word, and + is not followed by any character in a given set of C{wordChars} + (default=C{printables}). To emulate the C{\b} behavior of regular expressions, + use C{WordEnd(alphanums)}. C{WordEnd} will also match at the end of + the string being parsed, or at the end of a line. + """ + def __init__(self, wordChars = printables): + super(WordEnd,self).__init__() + self.wordChars = set(wordChars) + self.skipWhitespace = False + self.errmsg = "Not at the end of a word" + + def parseImpl(self, instring, loc, doActions=True ): + instrlen = len(instring) + if instrlen>0 and loc maxExcLoc: + maxException = err + maxExcLoc = err.loc + except IndexError: + if len(instring) > maxExcLoc: + maxException = ParseException(instring,len(instring),e.errmsg,self) + maxExcLoc = len(instring) + else: + if loc2 > maxMatchLoc: + maxMatchLoc = loc2 + maxMatchExp = e + + if maxMatchLoc < 0: + if maxException is not None: + raise maxException + else: + raise ParseException(instring, loc, "no defined alternatives to match", self) + + return maxMatchExp._parse( instring, loc, doActions ) + + def __ixor__(self, other ): + if isinstance( other, basestring ): + other = Literal( other ) + return self.append( other ) #Or( [ self, other ] ) + + def __str__( self ): + if hasattr(self,"name"): + return self.name + + if self.strRepr is None: + self.strRepr = "{" + " ^ ".join( [ _ustr(e) for e in self.exprs ] ) + "}" + + return self.strRepr + + def checkRecursion( self, parseElementList ): + subRecCheckList = parseElementList[:] + [ self ] + for e in self.exprs: + e.checkRecursion( subRecCheckList ) + + +class MatchFirst(ParseExpression): + """Requires that at least one C{ParseExpression} is found. + If two expressions match, the first one listed is the one that will match. + May be constructed using the C{'|'} operator. + """ + def __init__( self, exprs, savelist = False ): + super(MatchFirst,self).__init__(exprs, savelist) + if exprs: + self.mayReturnEmpty = False + for e in self.exprs: + if e.mayReturnEmpty: + self.mayReturnEmpty = True + break + else: + self.mayReturnEmpty = True + + def parseImpl( self, instring, loc, doActions=True ): + maxExcLoc = -1 + maxException = None + for e in self.exprs: + try: + ret = e._parse( instring, loc, doActions ) + return ret + except ParseException as err: + if err.loc > maxExcLoc: + maxException = err + maxExcLoc = err.loc + except IndexError: + if len(instring) > maxExcLoc: + maxException = ParseException(instring,len(instring),e.errmsg,self) + maxExcLoc = len(instring) + + # only got here if no expression matched, raise exception for match that made it the furthest + else: + if maxException is not None: + raise maxException + else: + raise ParseException(instring, loc, "no defined alternatives to match", self) + + def __ior__(self, other ): + if isinstance( other, basestring ): + other = Literal( other ) + return self.append( other ) #MatchFirst( [ self, other ] ) + + def __str__( self ): + if hasattr(self,"name"): + return self.name + + if self.strRepr is None: + self.strRepr = "{" + " | ".join( [ _ustr(e) for e in self.exprs ] ) + "}" + + return self.strRepr + + def checkRecursion( self, parseElementList ): + subRecCheckList = parseElementList[:] + [ self ] + for e in self.exprs: + e.checkRecursion( subRecCheckList ) + + +class Each(ParseExpression): + """Requires all given C{ParseExpression}s to be found, but in any order. + Expressions may be separated by whitespace. + May be constructed using the C{'&'} operator. + """ + def __init__( self, exprs, savelist = True ): + super(Each,self).__init__(exprs, savelist) + self.mayReturnEmpty = True + for e in self.exprs: + if not e.mayReturnEmpty: + self.mayReturnEmpty = False + break + self.skipWhitespace = True + self.initExprGroups = True + + def parseImpl( self, instring, loc, doActions=True ): + if self.initExprGroups: + opt1 = [ e.expr for e in self.exprs if isinstance(e,Optional) ] + opt2 = [ e for e in self.exprs if e.mayReturnEmpty and e not in opt1 ] + self.optionals = opt1 + opt2 + self.multioptionals = [ e.expr for e in self.exprs if isinstance(e,ZeroOrMore) ] + self.multirequired = [ e.expr for e in self.exprs if isinstance(e,OneOrMore) ] + self.required = [ e for e in self.exprs if not isinstance(e,(Optional,ZeroOrMore,OneOrMore)) ] + self.required += self.multirequired + self.initExprGroups = False + tmpLoc = loc + tmpReqd = self.required[:] + tmpOpt = self.optionals[:] + matchOrder = [] + + keepMatching = True + while keepMatching: + tmpExprs = tmpReqd + tmpOpt + self.multioptionals + self.multirequired + failed = [] + for e in tmpExprs: + try: + tmpLoc = e.tryParse( instring, tmpLoc ) + except ParseException: + failed.append(e) + else: + matchOrder.append(e) + if e in tmpReqd: + tmpReqd.remove(e) + elif e in tmpOpt: + tmpOpt.remove(e) + if len(failed) == len(tmpExprs): + keepMatching = False + + if tmpReqd: + missing = ", ".join( [ _ustr(e) for e in tmpReqd ] ) + raise ParseException(instring,loc,"Missing one or more required elements (%s)" % missing ) + + # add any unmatched Optionals, in case they have default values defined + matchOrder += list(e for e in self.exprs if isinstance(e,Optional) and e.expr in tmpOpt) + + resultlist = [] + for e in matchOrder: + loc,results = e._parse(instring,loc,doActions) + resultlist.append(results) + + finalResults = ParseResults([]) + for r in resultlist: + dups = {} + for k in r.keys(): + if k in finalResults.keys(): + tmp = ParseResults(finalResults[k]) + tmp += ParseResults(r[k]) + dups[k] = tmp + finalResults += ParseResults(r) + for k,v in dups.items(): + finalResults[k] = v + return loc, finalResults + + def __str__( self ): + if hasattr(self,"name"): + return self.name + + if self.strRepr is None: + self.strRepr = "{" + " & ".join( [ _ustr(e) for e in self.exprs ] ) + "}" + + return self.strRepr + + def checkRecursion( self, parseElementList ): + subRecCheckList = parseElementList[:] + [ self ] + for e in self.exprs: + e.checkRecursion( subRecCheckList ) + + +class ParseElementEnhance(ParserElement): + """Abstract subclass of C{ParserElement}, for combining and post-processing parsed tokens.""" + def __init__( self, expr, savelist=False ): + super(ParseElementEnhance,self).__init__(savelist) + if isinstance( expr, basestring ): + expr = Literal(expr) + self.expr = expr + self.strRepr = None + if expr is not None: + self.mayIndexError = expr.mayIndexError + self.mayReturnEmpty = expr.mayReturnEmpty + self.setWhitespaceChars( expr.whiteChars ) + self.skipWhitespace = expr.skipWhitespace + self.saveAsList = expr.saveAsList + self.callPreparse = expr.callPreparse + self.ignoreExprs.extend(expr.ignoreExprs) + + def parseImpl( self, instring, loc, doActions=True ): + if self.expr is not None: + return self.expr._parse( instring, loc, doActions, callPreParse=False ) + else: + raise ParseException("",loc,self.errmsg,self) + + def leaveWhitespace( self ): + self.skipWhitespace = False + self.expr = self.expr.copy() + if self.expr is not None: + self.expr.leaveWhitespace() + return self + + def ignore( self, other ): + if isinstance( other, Suppress ): + if other not in self.ignoreExprs: + super( ParseElementEnhance, self).ignore( other ) + if self.expr is not None: + self.expr.ignore( self.ignoreExprs[-1] ) + else: + super( ParseElementEnhance, self).ignore( other ) + if self.expr is not None: + self.expr.ignore( self.ignoreExprs[-1] ) + return self + + def streamline( self ): + super(ParseElementEnhance,self).streamline() + if self.expr is not None: + self.expr.streamline() + return self + + def checkRecursion( self, parseElementList ): + if self in parseElementList: + raise RecursiveGrammarException( parseElementList+[self] ) + subRecCheckList = parseElementList[:] + [ self ] + if self.expr is not None: + self.expr.checkRecursion( subRecCheckList ) + + def validate( self, validateTrace=[] ): + tmp = validateTrace[:]+[self] + if self.expr is not None: + self.expr.validate(tmp) + self.checkRecursion( [] ) + + def __str__( self ): + try: + return super(ParseElementEnhance,self).__str__() + except: + pass + + if self.strRepr is None and self.expr is not None: + self.strRepr = "%s:(%s)" % ( self.__class__.__name__, _ustr(self.expr) ) + return self.strRepr + + +class FollowedBy(ParseElementEnhance): + """Lookahead matching of the given parse expression. C{FollowedBy} + does *not* advance the parsing position within the input string, it only + verifies that the specified parse expression matches at the current + position. C{FollowedBy} always returns a null token list.""" + def __init__( self, expr ): + super(FollowedBy,self).__init__(expr) + self.mayReturnEmpty = True + + def parseImpl( self, instring, loc, doActions=True ): + self.expr.tryParse( instring, loc ) + return loc, [] + + +class NotAny(ParseElementEnhance): + """Lookahead to disallow matching with the given parse expression. C{NotAny} + does *not* advance the parsing position within the input string, it only + verifies that the specified parse expression does *not* match at the current + position. Also, C{NotAny} does *not* skip over leading whitespace. C{NotAny} + always returns a null token list. May be constructed using the '~' operator.""" + def __init__( self, expr ): + super(NotAny,self).__init__(expr) + #~ self.leaveWhitespace() + self.skipWhitespace = False # do NOT use self.leaveWhitespace(), don't want to propagate to exprs + self.mayReturnEmpty = True + self.errmsg = "Found unwanted token, "+_ustr(self.expr) + + def parseImpl( self, instring, loc, doActions=True ): + try: + self.expr.tryParse( instring, loc ) + except (ParseException,IndexError): + pass + else: + raise ParseException(instring, loc, self.errmsg, self) + return loc, [] + + def __str__( self ): + if hasattr(self,"name"): + return self.name + + if self.strRepr is None: + self.strRepr = "~{" + _ustr(self.expr) + "}" + + return self.strRepr + + +class ZeroOrMore(ParseElementEnhance): + """Optional repetition of zero or more of the given expression.""" + def __init__( self, expr ): + super(ZeroOrMore,self).__init__(expr) + self.mayReturnEmpty = True + + def parseImpl( self, instring, loc, doActions=True ): + tokens = [] + try: + loc, tokens = self.expr._parse( instring, loc, doActions, callPreParse=False ) + hasIgnoreExprs = ( len(self.ignoreExprs) > 0 ) + while 1: + if hasIgnoreExprs: + preloc = self._skipIgnorables( instring, loc ) + else: + preloc = loc + loc, tmptokens = self.expr._parse( instring, preloc, doActions ) + if tmptokens or tmptokens.keys(): + tokens += tmptokens + except (ParseException,IndexError): + pass + + return loc, tokens + + def __str__( self ): + if hasattr(self,"name"): + return self.name + + if self.strRepr is None: + self.strRepr = "[" + _ustr(self.expr) + "]..." + + return self.strRepr + + def setResultsName( self, name, listAllMatches=False ): + ret = super(ZeroOrMore,self).setResultsName(name,listAllMatches) + ret.saveAsList = True + return ret + + +class OneOrMore(ParseElementEnhance): + """Repetition of one or more of the given expression.""" + def parseImpl( self, instring, loc, doActions=True ): + # must be at least one + loc, tokens = self.expr._parse( instring, loc, doActions, callPreParse=False ) + try: + hasIgnoreExprs = ( len(self.ignoreExprs) > 0 ) + while 1: + if hasIgnoreExprs: + preloc = self._skipIgnorables( instring, loc ) + else: + preloc = loc + loc, tmptokens = self.expr._parse( instring, preloc, doActions ) + if tmptokens or tmptokens.keys(): + tokens += tmptokens + except (ParseException,IndexError): + pass + + return loc, tokens + + def __str__( self ): + if hasattr(self,"name"): + return self.name + + if self.strRepr is None: + self.strRepr = "{" + _ustr(self.expr) + "}..." + + return self.strRepr + + def setResultsName( self, name, listAllMatches=False ): + ret = super(OneOrMore,self).setResultsName(name,listAllMatches) + ret.saveAsList = True + return ret + +class _NullToken(object): + def __bool__(self): + return False + __nonzero__ = __bool__ + def __str__(self): + return "" + +_optionalNotMatched = _NullToken() +class Optional(ParseElementEnhance): + """Optional matching of the given expression. + A default return string can also be specified, if the optional expression + is not found. + """ + def __init__( self, exprs, default=_optionalNotMatched ): + super(Optional,self).__init__( exprs, savelist=False ) + self.defaultValue = default + self.mayReturnEmpty = True + + def parseImpl( self, instring, loc, doActions=True ): + try: + loc, tokens = self.expr._parse( instring, loc, doActions, callPreParse=False ) + except (ParseException,IndexError): + if self.defaultValue is not _optionalNotMatched: + if self.expr.resultsName: + tokens = ParseResults([ self.defaultValue ]) + tokens[self.expr.resultsName] = self.defaultValue + else: + tokens = [ self.defaultValue ] + else: + tokens = [] + return loc, tokens + + def __str__( self ): + if hasattr(self,"name"): + return self.name + + if self.strRepr is None: + self.strRepr = "[" + _ustr(self.expr) + "]" + + return self.strRepr + + +class SkipTo(ParseElementEnhance): + """Token for skipping over all undefined text until the matched expression is found. + If C{include} is set to true, the matched expression is also parsed (the skipped text + and matched expression are returned as a 2-element list). The C{ignore} + argument is used to define grammars (typically quoted strings and comments) that + might contain false matches. + """ + def __init__( self, other, include=False, ignore=None, failOn=None ): + super( SkipTo, self ).__init__( other ) + self.ignoreExpr = ignore + self.mayReturnEmpty = True + self.mayIndexError = False + self.includeMatch = include + self.asList = False + if failOn is not None and isinstance(failOn, basestring): + self.failOn = Literal(failOn) + else: + self.failOn = failOn + self.errmsg = "No match found for "+_ustr(self.expr) + + def parseImpl( self, instring, loc, doActions=True ): + startLoc = loc + instrlen = len(instring) + expr = self.expr + failParse = False + while loc <= instrlen: + try: + if self.failOn: + try: + self.failOn.tryParse(instring, loc) + except ParseBaseException: + pass + else: + failParse = True + raise ParseException(instring, loc, "Found expression " + str(self.failOn)) + failParse = False + if self.ignoreExpr is not None: + while 1: + try: + loc = self.ignoreExpr.tryParse(instring,loc) + # print("found ignoreExpr, advance to", loc) + except ParseBaseException: + break + expr._parse( instring, loc, doActions=False, callPreParse=False ) + skipText = instring[startLoc:loc] + if self.includeMatch: + loc,mat = expr._parse(instring,loc,doActions,callPreParse=False) + if mat: + skipRes = ParseResults( skipText ) + skipRes += mat + return loc, [ skipRes ] + else: + return loc, [ skipText ] + else: + return loc, [ skipText ] + except (ParseException,IndexError): + if failParse: + raise + else: + loc += 1 + raise ParseException(instring, loc, self.errmsg, self) + +class Forward(ParseElementEnhance): + """Forward declaration of an expression to be defined later - + used for recursive grammars, such as algebraic infix notation. + When the expression is known, it is assigned to the C{Forward} variable using the '<<' operator. + + Note: take care when assigning to C{Forward} not to overlook precedence of operators. + Specifically, '|' has a lower precedence than '<<', so that:: + fwdExpr << a | b | c + will actually be evaluated as:: + (fwdExpr << a) | b | c + thereby leaving b and c out as parseable alternatives. It is recommended that you + explicitly group the values inserted into the C{Forward}:: + fwdExpr << (a | b | c) + """ + def __init__( self, other=None ): + super(Forward,self).__init__( other, savelist=False ) + + def __lshift__( self, other ): + if isinstance( other, basestring ): + other = Literal(other) + self.expr = other + self.mayReturnEmpty = other.mayReturnEmpty + self.strRepr = None + self.mayIndexError = self.expr.mayIndexError + self.mayReturnEmpty = self.expr.mayReturnEmpty + self.setWhitespaceChars( self.expr.whiteChars ) + self.skipWhitespace = self.expr.skipWhitespace + self.saveAsList = self.expr.saveAsList + self.ignoreExprs.extend(self.expr.ignoreExprs) + return None + + def leaveWhitespace( self ): + self.skipWhitespace = False + return self + + def streamline( self ): + if not self.streamlined: + self.streamlined = True + if self.expr is not None: + self.expr.streamline() + return self + + def validate( self, validateTrace=[] ): + if self not in validateTrace: + tmp = validateTrace[:]+[self] + if self.expr is not None: + self.expr.validate(tmp) + self.checkRecursion([]) + + def __str__( self ): + if hasattr(self,"name"): + return self.name + + self._revertClass = self.__class__ + self.__class__ = _ForwardNoRecurse + try: + if self.expr is not None: + retString = _ustr(self.expr) + else: + retString = "None" + finally: + self.__class__ = self._revertClass + return self.__class__.__name__ + ": " + retString + + def copy(self): + if self.expr is not None: + return super(Forward,self).copy() + else: + ret = Forward() + ret << self + return ret + +class _ForwardNoRecurse(Forward): + def __str__( self ): + return "..." + +class TokenConverter(ParseElementEnhance): + """Abstract subclass of C{ParseExpression}, for converting parsed results.""" + def __init__( self, expr, savelist=False ): + super(TokenConverter,self).__init__( expr )#, savelist ) + self.saveAsList = False + +class Upcase(TokenConverter): + """Converter to upper case all matching tokens.""" + def __init__(self, *args): + super(Upcase,self).__init__(*args) + warnings.warn("Upcase class is deprecated, use upcaseTokens parse action instead", + DeprecationWarning,stacklevel=2) + + def postParse( self, instring, loc, tokenlist ): + return list(map( string.upper, tokenlist )) + + +class Combine(TokenConverter): + """Converter to concatenate all matching tokens to a single string. + By default, the matching patterns must also be contiguous in the input string; + this can be disabled by specifying C{'adjacent=False'} in the constructor. + """ + def __init__( self, expr, joinString="", adjacent=True ): + super(Combine,self).__init__( expr ) + # suppress whitespace-stripping in contained parse expressions, but re-enable it on the Combine itself + if adjacent: + self.leaveWhitespace() + self.adjacent = adjacent + self.skipWhitespace = True + self.joinString = joinString + self.callPreparse = True + + def ignore( self, other ): + if self.adjacent: + ParserElement.ignore(self, other) + else: + super( Combine, self).ignore( other ) + return self + + def postParse( self, instring, loc, tokenlist ): + retToks = tokenlist.copy() + del retToks[:] + retToks += ParseResults([ "".join(tokenlist._asStringList(self.joinString)) ], modal=self.modalResults) + + if self.resultsName and len(retToks.keys())>0: + return [ retToks ] + else: + return retToks + +class Group(TokenConverter): + """Converter to return the matched tokens as a list - useful for returning tokens of C{ZeroOrMore} and C{OneOrMore} expressions.""" + def __init__( self, expr ): + super(Group,self).__init__( expr ) + self.saveAsList = True + + def postParse( self, instring, loc, tokenlist ): + return [ tokenlist ] + +class Dict(TokenConverter): + """Converter to return a repetitive expression as a list, but also as a dictionary. + Each element can also be referenced using the first token in the expression as its key. + Useful for tabular report scraping when the first column can be used as a item key. + """ + def __init__( self, exprs ): + super(Dict,self).__init__( exprs ) + self.saveAsList = True + + def postParse( self, instring, loc, tokenlist ): + for i,tok in enumerate(tokenlist): + if len(tok) == 0: + continue + ikey = tok[0] + if isinstance(ikey,int): + ikey = _ustr(tok[0]).strip() + if len(tok)==1: + tokenlist[ikey] = _ParseResultsWithOffset("",i) + elif len(tok)==2 and not isinstance(tok[1],ParseResults): + tokenlist[ikey] = _ParseResultsWithOffset(tok[1],i) + else: + dictvalue = tok.copy() #ParseResults(i) + del dictvalue[0] + if len(dictvalue)!= 1 or (isinstance(dictvalue,ParseResults) and dictvalue.keys()): + tokenlist[ikey] = _ParseResultsWithOffset(dictvalue,i) + else: + tokenlist[ikey] = _ParseResultsWithOffset(dictvalue[0],i) + + if self.resultsName: + return [ tokenlist ] + else: + return tokenlist + + +class Suppress(TokenConverter): + """Converter for ignoring the results of a parsed expression.""" + def postParse( self, instring, loc, tokenlist ): + return [] + + def suppress( self ): + return self + + +class OnlyOnce(object): + """Wrapper for parse actions, to ensure they are only called once.""" + def __init__(self, methodCall): + self.callable = _trim_arity(methodCall) + self.called = False + def __call__(self,s,l,t): + if not self.called: + results = self.callable(s,l,t) + self.called = True + return results + raise ParseException(s,l,"") + def reset(self): + self.called = False + +def traceParseAction(f): + """Decorator for debugging parse actions.""" + f = _trim_arity(f) + def z(*paArgs): + thisFunc = f.func_name + s,l,t = paArgs[-3:] + if len(paArgs)>3: + thisFunc = paArgs[0].__class__.__name__ + '.' + thisFunc + sys.stderr.write( ">>entering %s(line: '%s', %d, %s)\n" % (thisFunc,line(l,s),l,t) ) + try: + ret = f(*paArgs) + except Exception as exc: + sys.stderr.write( "<", "|".join( [ _escapeRegexChars(sym) for sym in symbols] )) + try: + if len(symbols)==len("".join(symbols)): + return Regex( "[%s]" % "".join( [ _escapeRegexRangeChars(sym) for sym in symbols] ) ) + else: + return Regex( "|".join( [ re.escape(sym) for sym in symbols] ) ) + except: + warnings.warn("Exception creating Regex for oneOf, building MatchFirst", + SyntaxWarning, stacklevel=2) + + + # last resort, just use MatchFirst + return MatchFirst( [ parseElementClass(sym) for sym in symbols ] ) + +def dictOf( key, value ): + """Helper to easily and clearly define a dictionary by specifying the respective patterns + for the key and value. Takes care of defining the C{Dict}, C{ZeroOrMore}, and C{Group} tokens + in the proper order. The key pattern can include delimiting markers or punctuation, + as long as they are suppressed, thereby leaving the significant key text. The value + pattern can include named results, so that the C{Dict} results can include named token + fields. + """ + return Dict( ZeroOrMore( Group ( key + value ) ) ) + +def originalTextFor(expr, asString=True): + """Helper to return the original, untokenized text for a given expression. Useful to + restore the parsed fields of an HTML start tag into the raw tag text itself, or to + revert separate tokens with intervening whitespace back to the original matching + input text. Simpler to use than the parse action C{L{keepOriginalText}}, and does not + require the inspect module to chase up the call stack. By default, returns a + string containing the original parsed text. + + If the optional C{asString} argument is passed as C{False}, then the return value is a + C{ParseResults} containing any results names that were originally matched, and a + single token containing the original matched text from the input string. So if + the expression passed to C{L{originalTextFor}} contains expressions with defined + results names, you must set C{asString} to C{False} if you want to preserve those + results name values.""" + locMarker = Empty().setParseAction(lambda s,loc,t: loc) + endlocMarker = locMarker.copy() + endlocMarker.callPreparse = False + matchExpr = locMarker("_original_start") + expr + endlocMarker("_original_end") + if asString: + extractText = lambda s,l,t: s[t._original_start:t._original_end] + else: + def extractText(s,l,t): + del t[:] + t.insert(0, s[t._original_start:t._original_end]) + del t["_original_start"] + del t["_original_end"] + matchExpr.setParseAction(extractText) + return matchExpr + +def ungroup(expr): + """Helper to undo pyparsing's default grouping of And expressions, even + if all but one are non-empty.""" + return TokenConverter(expr).setParseAction(lambda t:t[0]) + +# convenience constants for positional expressions +empty = Empty().setName("empty") +lineStart = LineStart().setName("lineStart") +lineEnd = LineEnd().setName("lineEnd") +stringStart = StringStart().setName("stringStart") +stringEnd = StringEnd().setName("stringEnd") + +_escapedPunc = Word( _bslash, r"\[]-*.$+^?()~ ", exact=2 ).setParseAction(lambda s,l,t:t[0][1]) +_printables_less_backslash = "".join([ c for c in printables if c not in r"\]" ]) +_escapedHexChar = Regex(r"\\0?[xX][0-9a-fA-F]+").setParseAction(lambda s,l,t:unichr(int(t[0][1:],16))) +_escapedOctChar = Regex(r"\\0[0-7]+").setParseAction(lambda s,l,t:unichr(int(t[0][1:],8))) +_singleChar = _escapedPunc | _escapedHexChar | _escapedOctChar | Word(_printables_less_backslash,exact=1) +_charRange = Group(_singleChar + Suppress("-") + _singleChar) +_reBracketExpr = Literal("[") + Optional("^").setResultsName("negate") + Group( OneOrMore( _charRange | _singleChar ) ).setResultsName("body") + "]" + +_expanded = lambda p: (isinstance(p,ParseResults) and ''.join([ unichr(c) for c in range(ord(p[0]),ord(p[1])+1) ]) or p) + +def srange(s): + r"""Helper to easily define string ranges for use in Word construction. Borrows + syntax from regexp '[]' string range definitions:: + srange("[0-9]") -> "0123456789" + srange("[a-z]") -> "abcdefghijklmnopqrstuvwxyz" + srange("[a-z$_]") -> "abcdefghijklmnopqrstuvwxyz$_" + The input string must be enclosed in []'s, and the returned string is the expanded + character set joined into a single string. + The values enclosed in the []'s may be:: + a single character + an escaped character with a leading backslash (such as \- or \]) + an escaped hex character with a leading '\x' (\x21, which is a '!' character) + (\0x## is also supported for backwards compatibility) + an escaped octal character with a leading '\0' (\041, which is a '!' character) + a range of any of the above, separated by a dash ('a-z', etc.) + any combination of the above ('aeiouy', 'a-zA-Z0-9_$', etc.) + """ + try: + return "".join([_expanded(part) for part in _reBracketExpr.parseString(s).body]) + except: + return "" + +def matchOnlyAtCol(n): + """Helper method for defining parse actions that require matching at a specific + column in the input text. + """ + def verifyCol(strg,locn,toks): + if col(locn,strg) != n: + raise ParseException(strg,locn,"matched token not at column %d" % n) + return verifyCol + +def replaceWith(replStr): + """Helper method for common parse actions that simply return a literal value. Especially + useful when used with C{transformString()}. + """ + def _replFunc(*args): + return [replStr] + return _replFunc + +def removeQuotes(s,l,t): + """Helper parse action for removing quotation marks from parsed quoted strings. + To use, add this parse action to quoted string using:: + quotedString.setParseAction( removeQuotes ) + """ + return t[0][1:-1] + +def upcaseTokens(s,l,t): + """Helper parse action to convert tokens to upper case.""" + return [ tt.upper() for tt in map(_ustr,t) ] + +def downcaseTokens(s,l,t): + """Helper parse action to convert tokens to lower case.""" + return [ tt.lower() for tt in map(_ustr,t) ] + +def keepOriginalText(s,startLoc,t): + """DEPRECATED - use new helper method C{originalTextFor}. + Helper parse action to preserve original parsed text, + overriding any nested parse actions.""" + try: + endloc = getTokensEndLoc() + except ParseException: + raise ParseFatalException("incorrect usage of keepOriginalText - may only be called as a parse action") + del t[:] + t += ParseResults(s[startLoc:endloc]) + return t + +def getTokensEndLoc(): + """Method to be called from within a parse action to determine the end + location of the parsed tokens.""" + import inspect + fstack = inspect.stack() + try: + # search up the stack (through intervening argument normalizers) for correct calling routine + for f in fstack[2:]: + if f[3] == "_parseNoCache": + endloc = f[0].f_locals["loc"] + return endloc + else: + raise ParseFatalException("incorrect usage of getTokensEndLoc - may only be called from within a parse action") + finally: + del fstack + +def _makeTags(tagStr, xml): + """Internal helper to construct opening and closing tag expressions, given a tag name""" + if isinstance(tagStr,basestring): + resname = tagStr + tagStr = Keyword(tagStr, caseless=not xml) + else: + resname = tagStr.name + + tagAttrName = Word(alphas,alphanums+"_-:") + if (xml): + tagAttrValue = dblQuotedString.copy().setParseAction( removeQuotes ) + openTag = Suppress("<") + tagStr("tag") + \ + Dict(ZeroOrMore(Group( tagAttrName + Suppress("=") + tagAttrValue ))) + \ + Optional("/",default=[False]).setResultsName("empty").setParseAction(lambda s,l,t:t[0]=='/') + Suppress(">") + else: + printablesLessRAbrack = "".join( [ c for c in printables if c not in ">" ] ) + tagAttrValue = quotedString.copy().setParseAction( removeQuotes ) | Word(printablesLessRAbrack) + openTag = Suppress("<") + tagStr("tag") + \ + Dict(ZeroOrMore(Group( tagAttrName.setParseAction(downcaseTokens) + \ + Optional( Suppress("=") + tagAttrValue ) ))) + \ + Optional("/",default=[False]).setResultsName("empty").setParseAction(lambda s,l,t:t[0]=='/') + Suppress(">") + closeTag = Combine(_L("") + + openTag = openTag.setResultsName("start"+"".join(resname.replace(":"," ").title().split())).setName("<%s>" % tagStr) + closeTag = closeTag.setResultsName("end"+"".join(resname.replace(":"," ").title().split())).setName("" % tagStr) + openTag.tag = resname + closeTag.tag = resname + return openTag, closeTag + +def makeHTMLTags(tagStr): + """Helper to construct opening and closing tag expressions for HTML, given a tag name""" + return _makeTags( tagStr, False ) + +def makeXMLTags(tagStr): + """Helper to construct opening and closing tag expressions for XML, given a tag name""" + return _makeTags( tagStr, True ) + +def withAttribute(*args,**attrDict): + """Helper to create a validating parse action to be used with start tags created + with C{makeXMLTags} or C{makeHTMLTags}. Use C{withAttribute} to qualify a starting tag + with a required attribute value, to avoid false matches on common tags such as + C{} or C{
}. + + Call C{withAttribute} with a series of attribute names and values. Specify the list + of filter attributes names and values as: + - keyword arguments, as in C{(align="right")}, or + - as an explicit dict with C{**} operator, when an attribute name is also a Python + reserved word, as in C{**{"class":"Customer", "align":"right"}} + - a list of name-value tuples, as in ( ("ns1:class", "Customer"), ("ns2:align","right") ) + For attribute names with a namespace prefix, you must use the second form. Attribute + names are matched insensitive to upper/lower case. + + To verify that the attribute exists, but without specifying a value, pass + C{withAttribute.ANY_VALUE} as the value. + """ + if args: + attrs = args[:] + else: + attrs = attrDict.items() + attrs = [(k,v) for k,v in attrs] + def pa(s,l,tokens): + for attrName,attrValue in attrs: + if attrName not in tokens: + raise ParseException(s,l,"no matching attribute " + attrName) + if attrValue != withAttribute.ANY_VALUE and tokens[attrName] != attrValue: + raise ParseException(s,l,"attribute '%s' has value '%s', must be '%s'" % + (attrName, tokens[attrName], attrValue)) + return pa +withAttribute.ANY_VALUE = object() + +opAssoc = _Constants() +opAssoc.LEFT = object() +opAssoc.RIGHT = object() + +def operatorPrecedence( baseExpr, opList ): + """Helper method for constructing grammars of expressions made up of + operators working in a precedence hierarchy. Operators may be unary or + binary, left- or right-associative. Parse actions can also be attached + to operator expressions. + + Parameters: + - baseExpr - expression representing the most basic element for the nested + - opList - list of tuples, one for each operator precedence level in the + expression grammar; each tuple is of the form + (opExpr, numTerms, rightLeftAssoc, parseAction), where: + - opExpr is the pyparsing expression for the operator; + may also be a string, which will be converted to a Literal; + if numTerms is 3, opExpr is a tuple of two expressions, for the + two operators separating the 3 terms + - numTerms is the number of terms for this operator (must + be 1, 2, or 3) + - rightLeftAssoc is the indicator whether the operator is + right or left associative, using the pyparsing-defined + constants opAssoc.RIGHT and opAssoc.LEFT. + - parseAction is the parse action to be associated with + expressions matching this operator expression (the + parse action tuple member may be omitted) + """ + ret = Forward() + lastExpr = baseExpr | ( Suppress('(') + ret + Suppress(')') ) + for i,operDef in enumerate(opList): + opExpr,arity,rightLeftAssoc,pa = (operDef + (None,))[:4] + if arity == 3: + if opExpr is None or len(opExpr) != 2: + raise ValueError("if numterms=3, opExpr must be a tuple or list of two expressions") + opExpr1, opExpr2 = opExpr + thisExpr = Forward()#.setName("expr%d" % i) + if rightLeftAssoc == opAssoc.LEFT: + if arity == 1: + matchExpr = FollowedBy(lastExpr + opExpr) + Group( lastExpr + OneOrMore( opExpr ) ) + elif arity == 2: + if opExpr is not None: + matchExpr = FollowedBy(lastExpr + opExpr + lastExpr) + Group( lastExpr + OneOrMore( opExpr + lastExpr ) ) + else: + matchExpr = FollowedBy(lastExpr+lastExpr) + Group( lastExpr + OneOrMore(lastExpr) ) + elif arity == 3: + matchExpr = FollowedBy(lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr) + \ + Group( lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr ) + else: + raise ValueError("operator must be unary (1), binary (2), or ternary (3)") + elif rightLeftAssoc == opAssoc.RIGHT: + if arity == 1: + # try to avoid LR with this extra test + if not isinstance(opExpr, Optional): + opExpr = Optional(opExpr) + matchExpr = FollowedBy(opExpr.expr + thisExpr) + Group( opExpr + thisExpr ) + elif arity == 2: + if opExpr is not None: + matchExpr = FollowedBy(lastExpr + opExpr + thisExpr) + Group( lastExpr + OneOrMore( opExpr + thisExpr ) ) + else: + matchExpr = FollowedBy(lastExpr + thisExpr) + Group( lastExpr + OneOrMore( thisExpr ) ) + elif arity == 3: + matchExpr = FollowedBy(lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr) + \ + Group( lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr ) + else: + raise ValueError("operator must be unary (1), binary (2), or ternary (3)") + else: + raise ValueError("operator must indicate right or left associativity") + if pa: + matchExpr.setParseAction( pa ) + thisExpr << ( matchExpr | lastExpr ) + lastExpr = thisExpr + ret << lastExpr + return ret + +dblQuotedString = Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\x[0-9a-fA-F]+)|(?:\\.))*"').setName("string enclosed in double quotes") +sglQuotedString = Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\x[0-9a-fA-F]+)|(?:\\.))*'").setName("string enclosed in single quotes") +quotedString = Regex(r'''(?:"(?:[^"\n\r\\]|(?:"")|(?:\\x[0-9a-fA-F]+)|(?:\\.))*")|(?:'(?:[^'\n\r\\]|(?:'')|(?:\\x[0-9a-fA-F]+)|(?:\\.))*')''').setName("quotedString using single or double quotes") +unicodeString = Combine(_L('u') + quotedString.copy()) + +def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString.copy()): + """Helper method for defining nested lists enclosed in opening and closing + delimiters ("(" and ")" are the default). + + Parameters: + - opener - opening character for a nested list (default="("); can also be a pyparsing expression + - closer - closing character for a nested list (default=")"); can also be a pyparsing expression + - content - expression for items within the nested lists (default=None) + - ignoreExpr - expression for ignoring opening and closing delimiters (default=quotedString) + + If an expression is not provided for the content argument, the nested + expression will capture all whitespace-delimited content between delimiters + as a list of separate values. + + Use the C{ignoreExpr} argument to define expressions that may contain + opening or closing characters that should not be treated as opening + or closing characters for nesting, such as quotedString or a comment + expression. Specify multiple expressions using an C{L{Or}} or C{L{MatchFirst}}. + The default is L{quotedString}, but if no expressions are to be ignored, + then pass C{None} for this argument. + """ + if opener == closer: + raise ValueError("opening and closing strings cannot be the same") + if content is None: + if isinstance(opener,basestring) and isinstance(closer,basestring): + if len(opener) == 1 and len(closer)==1: + if ignoreExpr is not None: + content = (Combine(OneOrMore(~ignoreExpr + + CharsNotIn(opener+closer+ParserElement.DEFAULT_WHITE_CHARS,exact=1)) + ).setParseAction(lambda t:t[0].strip())) + else: + content = (empty.copy()+CharsNotIn(opener+closer+ParserElement.DEFAULT_WHITE_CHARS + ).setParseAction(lambda t:t[0].strip())) + else: + if ignoreExpr is not None: + content = (Combine(OneOrMore(~ignoreExpr + + ~Literal(opener) + ~Literal(closer) + + CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS,exact=1)) + ).setParseAction(lambda t:t[0].strip())) + else: + content = (Combine(OneOrMore(~Literal(opener) + ~Literal(closer) + + CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS,exact=1)) + ).setParseAction(lambda t:t[0].strip())) + else: + raise ValueError("opening and closing arguments must be strings if no content expression is given") + ret = Forward() + if ignoreExpr is not None: + ret << Group( Suppress(opener) + ZeroOrMore( ignoreExpr | ret | content ) + Suppress(closer) ) + else: + ret << Group( Suppress(opener) + ZeroOrMore( ret | content ) + Suppress(closer) ) + return ret + +def indentedBlock(blockStatementExpr, indentStack, indent=True): + """Helper method for defining space-delimited indentation blocks, such as + those used to define block statements in Python source code. + + Parameters: + - blockStatementExpr - expression defining syntax of statement that + is repeated within the indented block + - indentStack - list created by caller to manage indentation stack + (multiple statementWithIndentedBlock expressions within a single grammar + should share a common indentStack) + - indent - boolean indicating whether block must be indented beyond the + the current level; set to False for block of left-most statements + (default=True) + + A valid block must contain at least one C{blockStatement}. + """ + def checkPeerIndent(s,l,t): + if l >= len(s): return + curCol = col(l,s) + if curCol != indentStack[-1]: + if curCol > indentStack[-1]: + raise ParseFatalException(s,l,"illegal nesting") + raise ParseException(s,l,"not a peer entry") + + def checkSubIndent(s,l,t): + curCol = col(l,s) + if curCol > indentStack[-1]: + indentStack.append( curCol ) + else: + raise ParseException(s,l,"not a subentry") + + def checkUnindent(s,l,t): + if l >= len(s): return + curCol = col(l,s) + if not(indentStack and curCol < indentStack[-1] and curCol <= indentStack[-2]): + raise ParseException(s,l,"not an unindent") + indentStack.pop() + + NL = OneOrMore(LineEnd().setWhitespaceChars("\t ").suppress()) + INDENT = Empty() + Empty().setParseAction(checkSubIndent) + PEER = Empty().setParseAction(checkPeerIndent) + UNDENT = Empty().setParseAction(checkUnindent) + if indent: + smExpr = Group( Optional(NL) + + #~ FollowedBy(blockStatementExpr) + + INDENT + (OneOrMore( PEER + Group(blockStatementExpr) + Optional(NL) )) + UNDENT) + else: + smExpr = Group( Optional(NL) + + (OneOrMore( PEER + Group(blockStatementExpr) + Optional(NL) )) ) + blockStatementExpr.ignore(_bslash + LineEnd()) + return smExpr + +alphas8bit = srange(r"[\0xc0-\0xd6\0xd8-\0xf6\0xf8-\0xff]") +punc8bit = srange(r"[\0xa1-\0xbf\0xd7\0xf7]") + +anyOpenTag,anyCloseTag = makeHTMLTags(Word(alphas,alphanums+"_:")) +commonHTMLEntity = Combine(_L("&") + oneOf("gt lt amp nbsp quot").setResultsName("entity") +";").streamline() +_htmlEntityMap = dict(zip("gt lt amp nbsp quot".split(),'><& "')) +replaceHTMLEntity = lambda t : t.entity in _htmlEntityMap and _htmlEntityMap[t.entity] or None + +# it's easy to get these comment structures wrong - they're very common, so may as well make them available +cStyleComment = Regex(r"/\*(?:[^*]*\*+)+?/").setName("C style comment") + +htmlComment = Regex(r"") +restOfLine = Regex(r".*").leaveWhitespace() +dblSlashComment = Regex(r"\/\/(\\\n|.)*").setName("// comment") +cppStyleComment = Regex(r"/(?:\*(?:[^*]*\*+)+?/|/[^\n]*(?:\n[^\n]*)*?(?:(?" + str(tokenlist)) + print ("tokens = " + str(tokens)) + print ("tokens.columns = " + str(tokens.columns)) + print ("tokens.tables = " + str(tokens.tables)) + print (tokens.asXML("SQL",True)) + except ParseBaseException as err: + print (teststring + "->") + print (err.line) + print (" "*(err.column-1) + "^") + print (err) + print() + + selectToken = CaselessLiteral( "select" ) + fromToken = CaselessLiteral( "from" ) + + ident = Word( alphas, alphanums + "_$" ) + columnName = delimitedList( ident, ".", combine=True ).setParseAction( upcaseTokens ) + columnNameList = Group( delimitedList( columnName ) )#.setName("columns") + tableName = delimitedList( ident, ".", combine=True ).setParseAction( upcaseTokens ) + tableNameList = Group( delimitedList( tableName ) )#.setName("tables") + simpleSQL = ( selectToken + \ + ( '*' | columnNameList ).setResultsName( "columns" ) + \ + fromToken + \ + tableNameList.setResultsName( "tables" ) ) + + test( "SELECT * from XYZZY, ABC" ) + test( "select * from SYS.XYZZY" ) + test( "Select A from Sys.dual" ) + test( "Select AA,BB,CC from Sys.dual" ) + test( "Select A, B, C from Sys.dual" ) + test( "Select A, B, C from Sys.dual" ) + test( "Xelect A, B, C from Sys.dual" ) + test( "Select A, B, C frox Sys.dual" ) + test( "Select" ) + test( "Select ^^^ frox Sys.dual" ) + test( "Select A, B, C from Sys.dual, Table2 " ) diff --git a/docs/stubs_generation/helpers/generator3/clr_tools.py b/docs/stubs_generation/helpers/generator3/clr_tools.py new file mode 100644 index 000000000..f4c3cfbf9 --- /dev/null +++ b/docs/stubs_generation/helpers/generator3/clr_tools.py @@ -0,0 +1,63 @@ +# coding=utf-8 +""" +.NET (CLR) specific functions +""" +__author__ = 'Ilya.Kazakevich' + + +def get_namespace_by_name(object_name): + """ + Gets namespace for full object name. Sometimes last element of name is module while it may be class. + For System.Console returns System, for System.Web returns System.Web. + Be sure all required assemblies are loaded (i.e. clr.AddRef.. is called) + :param object_name: name to parse + :return: namespace + """ + (imported_object, object_name) = _import_first(object_name) + parts = object_name.partition(".") + first_part = parts[0] + remain_part = parts[2] + + while remain_part and type(_get_attr_by_name(imported_object, remain_part)) is type: # While we are in class + remain_part = remain_part.rpartition(".")[0] + + if remain_part: + return first_part + "." + remain_part + else: + return first_part + + +def _import_first(object_name): + """ + Some times we can not import module directly. For example, Some.Class.InnerClass could not be imported: you need to import "Some.Class" + or even "Some" instead. This function tries to find part of name that could be loaded + + :param object_name: name in dotted notation like "Some.Function.Here" + :return: (imported_object, object_name): tuple with object and its name + """ + while object_name: + try: + return (__import__(object_name, globals=[], locals=[], fromlist=[]), object_name) + except ImportError: + object_name = object_name.rpartition(".")[0] # Remove rightest part + raise Exception("No module name found in name " + object_name) + + +def _get_attr_by_name(obj, name): + """ + Accepts chain of attributes in dot notation like "some.property.name" and gets them on object + :param obj: object to introspec + :param name: attribute name + :return attribute + + >>> str(_get_attr_by_name("A", "__class__.__class__")) + "" + + >>> str(_get_attr_by_name("A", "__class__.__len__.__class__")) + "" + """ + result = obj + parts = name.split('.') + for part in parts: + result = getattr(result, part) + return result diff --git a/docs/stubs_generation/helpers/generator3/constants.py b/docs/stubs_generation/helpers/generator3/constants.py new file mode 100644 index 000000000..190c0699b --- /dev/null +++ b/docs/stubs_generation/helpers/generator3/constants.py @@ -0,0 +1,722 @@ +import os +import re +import string +import sys +import time +import types + +OUT_ENCODING = 'utf-8' + +version = ( + (sys.hexversion & (0xff << 24)) >> 24, + (sys.hexversion & (0xff << 16)) >> 16 +) + +if version[0] >= 3: + #noinspection PyUnresolvedReferences + import builtins as the_builtins + + string = "".__class__ + + STR_TYPES = (getattr(the_builtins, "bytes"), str) + + NUM_TYPES = (int, float) + SIMPLEST_TYPES = NUM_TYPES + STR_TYPES + (None.__class__,) + EASY_TYPES = NUM_TYPES + STR_TYPES + (None.__class__, dict, tuple, list) + + def the_exec(source, context): + exec (source, context) + + + # noinspection PyUnresolvedReferences + from inspect import getfullargspec + +else: # < 3.0 + import __builtin__ as the_builtins + + STR_TYPES = (getattr(the_builtins, "unicode"), str) + + NUM_TYPES = (int, long, float) + SIMPLEST_TYPES = NUM_TYPES + STR_TYPES + (types.NoneType,) + EASY_TYPES = NUM_TYPES + STR_TYPES + (types.NoneType, dict, tuple, list) + + def the_exec(source, context): + #noinspection PyRedundantParentheses + exec (source) in context + + def getfullargspec(func): + import inspect + return inspect.getargspec(func) + ([], None, {}) + +if version[0] == 2 and version[1] < 4: + HAS_DECORATORS = False + + def lstrip(s, prefix): + i = 0 + while s[i] == prefix: + i += 1 + return s[i:] + +else: + HAS_DECORATORS = True + lstrip = string.lstrip + +# return type inference helper table +INT_LIT = '0' +FLOAT_LIT = '0.0' +DICT_LIT = '{}' +LIST_LIT = '[]' +TUPLE_LIT = '()' +BOOL_LIT = 'False' +RET_TYPE = {# {'type_name': 'value_string'} lookup table + # chaining + "self": "self", + "self.": "self", + # int + "int": INT_LIT, + "Int": INT_LIT, + "integer": INT_LIT, + "Integer": INT_LIT, + "short": INT_LIT, + "long": INT_LIT, + "number": INT_LIT, + "Number": INT_LIT, + # float + "float": FLOAT_LIT, + "Float": FLOAT_LIT, + "double": FLOAT_LIT, + "Double": FLOAT_LIT, + "floating": FLOAT_LIT, + # boolean + "bool": BOOL_LIT, + "boolean": BOOL_LIT, + "Bool": BOOL_LIT, + "Boolean": BOOL_LIT, + "True": BOOL_LIT, + "true": BOOL_LIT, + "False": BOOL_LIT, + "false": BOOL_LIT, + # list + 'list': LIST_LIT, + 'List': LIST_LIT, + '[]': LIST_LIT, + # tuple + "tuple": TUPLE_LIT, + "sequence": TUPLE_LIT, + "Sequence": TUPLE_LIT, + # dict + "dict": DICT_LIT, + "Dict": DICT_LIT, + "dictionary": DICT_LIT, + "Dictionary": DICT_LIT, + "map": DICT_LIT, + "Map": DICT_LIT, + "hashtable": DICT_LIT, + "Hashtable": DICT_LIT, + "{}": DICT_LIT, + # "objects" + "object": "object()", +} +if version[0] < 3: + UNICODE_LIT = 'u""' + BYTES_LIT = '""' + RET_TYPE.update({ + 'string': BYTES_LIT, + 'String': BYTES_LIT, + 'str': BYTES_LIT, + 'Str': BYTES_LIT, + 'character': BYTES_LIT, + 'char': BYTES_LIT, + 'unicode': UNICODE_LIT, + 'Unicode': UNICODE_LIT, + 'bytes': BYTES_LIT, + 'byte': BYTES_LIT, + 'Bytes': BYTES_LIT, + 'Byte': BYTES_LIT, + }) + DEFAULT_STR_LIT = BYTES_LIT + # also, files: + RET_TYPE.update({ + 'file': "file('/dev/null')", + }) + + def ensureUnicode(data): + if type(data) == str: + return data.decode(OUT_ENCODING, 'replace') + return unicode(data) +else: + UNICODE_LIT = '""' + BYTES_LIT = 'b""' + RET_TYPE.update({ + 'string': UNICODE_LIT, + 'String': UNICODE_LIT, + 'str': UNICODE_LIT, + 'Str': UNICODE_LIT, + 'character': UNICODE_LIT, + 'char': UNICODE_LIT, + 'unicode': UNICODE_LIT, + 'Unicode': UNICODE_LIT, + 'bytes': BYTES_LIT, + 'byte': BYTES_LIT, + 'Bytes': BYTES_LIT, + 'Byte': BYTES_LIT, + }) + DEFAULT_STR_LIT = UNICODE_LIT + # also, files: we can't provide an easy expression on py3k + RET_TYPE.update({ + 'file': None, + }) + + def ensureUnicode(data): + if type(data) == bytes: + return data.decode(OUT_ENCODING, 'replace') + return str(data) + +if version[0] > 2: + import io # in 3.0 + + + def fopen(name, mode): + kwargs = {} + if 'b' not in mode: + kwargs['encoding'] = OUT_ENCODING + return io.open(name, mode, **kwargs) +else: + fopen = open + +if sys.platform == 'cli': + #noinspection PyUnresolvedReferences + from System import DateTime + + class Timer(object): + def __init__(self): + self.started = DateTime.Now + + def elapsed(self): + return (DateTime.Now - self.started).TotalMilliseconds +else: + class Timer(object): + def __init__(self): + self.started = time.time() + + def elapsed(self): + return int((time.time() - self.started) * 1000) + +IS_JAVA = hasattr(os, "java") + +BUILTIN_MOD_NAME = the_builtins.__name__ + +IDENT_PATTERN = r"[A-Za-z_][0-9A-Za-z_]*" # re pattern for identifier +STR_CHAR_PATTERN = r"[0-9A-Za-z_.,\+\-&\*% ]" + +DOC_FUNC_RE = re.compile(r"(?:.*\.)?(\w+)\(([^\)]*)\).*") # $1 = function name, $2 = arglist + +SANE_REPR_RE = re.compile(IDENT_PATTERN + r"(?:\(.*\))?") # identifier with possible (...), go catches + +STARS_IDENT_RE = re.compile(r"(\*?\*?" + IDENT_PATTERN + ")") # $1 = identifier, maybe with a * or ** + +IDENT_EQ_RE = re.compile("(" + IDENT_PATTERN + r"\s*=)") # $1 = identifier with a following '=' + +SIMPLE_VALUE_RE = re.compile( + r"(\([+-]?[0-9](?:\s*,\s*[+-]?[0-9])*\))|" + # a numeric tuple, e.g. in pygame + r"([+-]?[0-9]+\.?[0-9]*(?:[Ee]?[+-]?[0-9]+\.?[0-9]*)?)|" + # number + r"('" + STR_CHAR_PATTERN + "*')|" + # single-quoted string + r'("' + STR_CHAR_PATTERN + '*")|' + # double-quoted string + r"(\[\])|" + + r"(\{\})|" + + r"(\(\))|" + + r"(True|False|None)" +) # $? = sane default value + +if version[0] < 3: + _PYTHON2_IDENT_RE = re.compile(IDENT_PATTERN + "$") + + is_identifier = _PYTHON2_IDENT_RE.match +else: + is_identifier = str.isidentifier + +# Some values are known to be of no use in source and needs to be suppressed. +# Dict is keyed by module names, with "*" meaning "any module"; +# values are lists of names of members whose value must be pruned. +SKIP_VALUE_IN_MODULE = { + "sys": ( + "modules", "path_importer_cache", "argv", "builtins", + "last_traceback", "last_type", "last_value", "builtin_module_names", + ), + "posix": ( + "environ", + ), + "nt": ( + "environ", + ), + "zipimport": ( + "_zip_directory_cache", + ), + "*": (BUILTIN_MOD_NAME,) +} +# {"module": ("name",..)}: omit the names from the skeleton at all. +OMIT_NAME_IN_MODULE = {} + +if version[0] >= 3: + v = OMIT_NAME_IN_MODULE.get(BUILTIN_MOD_NAME, []) + ["True", "False", "None", "__debug__"] + OMIT_NAME_IN_MODULE[BUILTIN_MOD_NAME] = v + +if IS_JAVA and version > (2, 4): # in 2.5.1 things are way weird! + OMIT_NAME_IN_MODULE['_codecs'] = ['EncodingMap'] + OMIT_NAME_IN_MODULE['_hashlib'] = ['Hash'] + +ADD_VALUE_IN_MODULE = { + "sys": ("exc_value = Exception()", "exc_traceback=None"), # only present after an exception in current thread +} + +# Some values are special and are better represented by hand-crafted constructs. +# Dict is keyed by (module name, member name) and value is the replacement. +REPLACE_MODULE_VALUES = { + ("numpy.core.multiarray", "typeinfo"): "{}", + ("psycopg2._psycopg", "string_types"): "{}", # badly mangled __eq__ breaks fmtValue + ("PyQt5.QtWidgets", "qApp") : "QApplication()", # instead of None +} +if version[0] <= 2: + REPLACE_MODULE_VALUES[(BUILTIN_MOD_NAME, "None")] = "object()" + for std_file in ("stdin", "stdout", "stderr"): + REPLACE_MODULE_VALUES[("sys", std_file)] = "open('')" # + +# Some functions and methods of some builtin classes have special signatures. +# {("class", "method"): ("signature_string")} +PREDEFINED_BUILTIN_SIGS = { #TODO: user-skeleton + ("type", "__init__"): "(cls, what, bases=None, dict=None)", # two sigs squeezed into one + ("object", "__init__"): "(self)", + ("object", "__new__"): "(cls, *more)", # only for the sake of parameter names readability + ("object", "__subclasshook__"): "(cls, subclass)", # trusting PY-1818 on sig + ("int", "__init__"): "(self, x, base=10)", # overrides a fake + ("list", "__init__"): "(self, seq=())", + ("tuple", "__init__"): "(self, seq=())", # overrides a fake + ("set", "__init__"): "(self, seq=())", + ("dict", "__init__"): "(self, seq=None, **kwargs)", + ("property", "__init__"): "(self, fget=None, fset=None, fdel=None, doc=None)", + # TODO: infer, doc comments have it + ("dict", "update"): "(self, E=None, **F)", # docstring nearly lies + (None, "zip"): "(seq1, seq2, *more_seqs)", + (None, "range"): "(start=None, stop=None, step=None)", # suboptimal: allows empty arglist + (None, "filter"): "(function_or_none, sequence)", + (None, "iter"): "(source, sentinel=None)", + (None, "getattr"): "(object, name, default=None)", + ('frozenset', "__init__"): "(self, seq=())", + ("bytearray", "__init__"): "(self, source=None, encoding=None, errors='strict')", +} + +if version[0] < 3: + PREDEFINED_BUILTIN_SIGS[ + ("unicode", "__init__")] = "(self, string=u'', encoding=None, errors='strict')" # overrides a fake + PREDEFINED_BUILTIN_SIGS[("super", "__init__")] = "(self, type1, type2=None)" + PREDEFINED_BUILTIN_SIGS[ + (None, "min")] = "(*args, **kwargs)" # too permissive, but py2.x won't allow a better sig + PREDEFINED_BUILTIN_SIGS[(None, "max")] = "(*args, **kwargs)" + PREDEFINED_BUILTIN_SIGS[("str", "__init__")] = "(self, string='')" # overrides a fake + PREDEFINED_BUILTIN_SIGS[(None, "print")] = "(*args, **kwargs)" # can't do better in 2.x +else: + PREDEFINED_BUILTIN_SIGS[("super", "__init__")] = "(self, type1=None, type2=None)" + PREDEFINED_BUILTIN_SIGS[(None, "min")] = "(*args, key=None)" + PREDEFINED_BUILTIN_SIGS[(None, "max")] = "(*args, key=None)" + PREDEFINED_BUILTIN_SIGS[ + (None, "open")] = "(file, mode='r', buffering=None, encoding=None, errors=None, newline=None, closefd=True)" + PREDEFINED_BUILTIN_SIGS[ + ("str", "__init__")] = "(self, value='', encoding=None, errors='strict')" # overrides a fake + PREDEFINED_BUILTIN_SIGS[("str", "format")] = "(self, *args, **kwargs)" + PREDEFINED_BUILTIN_SIGS[ + ("bytes", "__init__")] = "(self, value=b'', encoding=None, errors='strict')" # overrides a fake + PREDEFINED_BUILTIN_SIGS[("bytes", "format")] = "(self, *args, **kwargs)" + PREDEFINED_BUILTIN_SIGS[(None, "print")] = "(self, *args, sep=' ', end='\\n', file=None)" # proper signature + +if (2, 6) <= version < (3, 0): + PREDEFINED_BUILTIN_SIGS[("unicode", "format")] = "(self, *args, **kwargs)" + PREDEFINED_BUILTIN_SIGS[("str", "format")] = "(self, *args, **kwargs)" + +if version == (2, 5): + PREDEFINED_BUILTIN_SIGS[("unicode", "splitlines")] = "(keepends=None)" # a typo in docstring there + +if version >= (2, 7): + PREDEFINED_BUILTIN_SIGS[ + ("enumerate", "__init__")] = "(self, iterable, start=0)" # dosctring omits this completely. + +if version < (3, 3): + datetime_mod = "datetime" +else: + datetime_mod = "_datetime" + + +# NOTE: per-module signature data may be lazily imported +# keyed by (module_name, class_name, method_name). PREDEFINED_BUILTIN_SIGS might be a layer of it. +# value is ("signature", "return_literal") +PREDEFINED_MOD_CLASS_SIGS = { #TODO: user-skeleton + (BUILTIN_MOD_NAME, None, 'divmod'): ("(x, y)", "(0, 0)"), + + ("binascii", None, "hexlify"): ("(data)", BYTES_LIT), + ("binascii", None, "unhexlify"): ("(hexstr)", BYTES_LIT), + + ("time", None, "ctime"): ("(seconds=None)", DEFAULT_STR_LIT), + + ("_struct", None, "pack"): ("(fmt, *args)", BYTES_LIT), + ("_struct", None, "pack_into"): ("(fmt, buffer, offset, *args)", None), + ("_struct", None, "unpack"): ("(fmt, string)", None), + ("_struct", None, "unpack_from"): ("(fmt, buffer, offset=0)", None), + ("_struct", None, "calcsize"): ("(fmt)", INT_LIT), + ("_struct", "Struct", "__init__"): ("(self, fmt)", None), + ("_struct", "Struct", "pack"): ("(self, *args)", BYTES_LIT), + ("_struct", "Struct", "pack_into"): ("(self, buffer, offset, *args)", None), + ("_struct", "Struct", "unpack"): ("(self, string)", None), + ("_struct", "Struct", "unpack_from"): ("(self, buffer, offset=0)", None), + + (datetime_mod, "date", "__new__"): ("(cls, year=None, month=None, day=None)", None), + (datetime_mod, "date", "fromordinal"): ("(cls, ordinal)", "date(1,1,1)"), + (datetime_mod, "date", "fromtimestamp"): ("(cls, timestamp)", "date(1,1,1)"), + (datetime_mod, "date", "isocalendar"): ("(self)", "(1, 1, 1)"), + (datetime_mod, "date", "isoformat"): ("(self)", DEFAULT_STR_LIT), + (datetime_mod, "date", "isoweekday"): ("(self)", INT_LIT), + (datetime_mod, "date", "replace"): ("(self, year=None, month=None, day=None)", "date(1,1,1)"), + (datetime_mod, "date", "strftime"): ("(self, format)", DEFAULT_STR_LIT), + (datetime_mod, "date", "timetuple"): ("(self)", "(0, 0, 0, 0, 0, 0, 0, 0, 0)"), + (datetime_mod, "date", "today"): ("(self)", "date(1, 1, 1)"), + (datetime_mod, "date", "toordinal"): ("(self)", INT_LIT), + (datetime_mod, "date", "weekday"): ("(self)", INT_LIT), + (datetime_mod, "timedelta", "__new__" + ): ( + "(cls, days=None, seconds=None, microseconds=None, milliseconds=None, minutes=None, hours=None, weeks=None)", + None), + (datetime_mod, "datetime", "__new__" + ): ( + "(cls, year=None, month=None, day=None, hour=None, minute=None, second=None, microsecond=None, tzinfo=None)", + None), + (datetime_mod, "datetime", "astimezone"): ("(self, tz)", "datetime(1, 1, 1)"), + (datetime_mod, "datetime", "combine"): ("(cls, date, time)", "datetime(1, 1, 1)"), + (datetime_mod, "datetime", "date"): ("(self)", "datetime(1, 1, 1)"), + (datetime_mod, "datetime", "fromtimestamp"): ("(cls, timestamp, tz=None)", "datetime(1, 1, 1)"), + (datetime_mod, "datetime", "isoformat"): ("(self, sep='T')", DEFAULT_STR_LIT), + (datetime_mod, "datetime", "now"): ("(cls, tz=None)", "datetime(1, 1, 1)"), + (datetime_mod, "datetime", "strptime"): ("(cls, date_string, format)", DEFAULT_STR_LIT), + (datetime_mod, "datetime", "replace" ): + ( + "(self, year=None, month=None, day=None, hour=None, minute=None, second=None, microsecond=None, tzinfo=None)", + "datetime(1, 1, 1)"), + (datetime_mod, "datetime", "time"): ("(self)", "time(0, 0)"), + (datetime_mod, "datetime", "timetuple"): ("(self)", "(0, 0, 0, 0, 0, 0, 0, 0, 0)"), + (datetime_mod, "datetime", "timetz"): ("(self)", "time(0, 0)"), + (datetime_mod, "datetime", "utcfromtimestamp"): ("(self, timestamp)", "datetime(1, 1, 1)"), + (datetime_mod, "datetime", "utcnow"): ("(cls)", "datetime(1, 1, 1)"), + (datetime_mod, "datetime", "utctimetuple"): ("(self)", "(0, 0, 0, 0, 0, 0, 0, 0, 0)"), + (datetime_mod, "time", "__new__"): ( + "(cls, hour=None, minute=None, second=None, microsecond=None, tzinfo=None)", None), + (datetime_mod, "time", "isoformat"): ("(self)", DEFAULT_STR_LIT), + (datetime_mod, "time", "replace"): ( + "(self, hour=None, minute=None, second=None, microsecond=None, tzinfo=None)", "time(0, 0)"), + (datetime_mod, "time", "strftime"): ("(self, format)", DEFAULT_STR_LIT), + (datetime_mod, "tzinfo", "dst"): ("(self, date_time)", INT_LIT), + (datetime_mod, "tzinfo", "fromutc"): ("(self, date_time)", "datetime(1, 1, 1)"), + (datetime_mod, "tzinfo", "tzname"): ("(self, date_time)", DEFAULT_STR_LIT), + (datetime_mod, "tzinfo", "utcoffset"): ("(self, date_time)", INT_LIT), + + ("_io", None, "open"): ("(name, mode=None, buffering=None)", "file('/dev/null')"), + ("_io", "FileIO", "read"): ("(self, size=-1)", DEFAULT_STR_LIT), + ("_fileio", "_FileIO", "read"): ("(self, size=-1)", DEFAULT_STR_LIT), + + ("thread", None, "start_new"): ("(function, args, kwargs=None)", INT_LIT), + ("_thread", None, "start_new"): ("(function, args, kwargs=None)", INT_LIT), + + ("itertools", "groupby", "__init__"): ("(self, iterable, key=None)", None), + ("itertools", None, "groupby"): ("(iterable, key=None)", LIST_LIT), + + ("cStringIO", "OutputType", "seek"): ("(self, position, mode=0)", None), + ("cStringIO", "InputType", "seek"): ("(self, position, mode=0)", None), + + # NOTE: here we stand on shaky ground providing sigs for 3rd-party modules, though well-known + ("numpy.core.multiarray", "ndarray", "__array__"): ("(self, dtype=None)", None), + ("numpy.core.multiarray", None, "arange"): ("(start=None, stop=None, step=None, dtype=None)", None), + # same as range() + ("numpy.core.multiarray", None, "set_numeric_ops"): ("(**ops)", None), + ("numpy.random.mtrand", None, "rand"): ("(*dn)", None), + ("numpy.random.mtrand", None, "randn"): ("(*dn)", None), + ("numpy.core.multiarray", "ndarray", "reshape"): ("(self, shape, *shapes, order='C')", None), + ("numpy.core.multiarray", "ndarray", "resize"): ("(self, *new_shape, refcheck=True)", None), +} + +bin_collections_names = ['collections', '_collections'] + +for name in bin_collections_names: + PREDEFINED_MOD_CLASS_SIGS[(name, "deque", "__init__")] = ("(self, iterable=(), maxlen=None)", None) + PREDEFINED_MOD_CLASS_SIGS[(name, "defaultdict", "__init__")] = ("(self, default_factory=None, **kwargs)", None) + +if version[0] < 3: + PREDEFINED_MOD_CLASS_SIGS[("exceptions", "BaseException", "__unicode__")] = ("(self)", UNICODE_LIT) + PREDEFINED_MOD_CLASS_SIGS[("itertools", "product", "__init__")] = ("(self, *iterables, **kwargs)", LIST_LIT) +else: + PREDEFINED_MOD_CLASS_SIGS[("itertools", "product", "__init__")] = ("(self, *iterables, repeat=1)", LIST_LIT) + +if version[0] < 3: + PREDEFINED_MOD_CLASS_SIGS[("PyQt4.QtCore", None, "pyqtSlot")] = ( + "(*types, **keywords)", None) # doc assumes py3k syntax + +# known properties of modules +# {{"module": {"class", "property" : ("letters", ("getter", "type"))}}, +# where letters is any set of r,w,d (read, write, del) and "getter" is a source of typed getter. +# if value is None, the property should be omitted. +# read-only properties that return an object are not listed. +G_OBJECT = ("lambda self: object()", None) +G_TYPE = ("lambda self: type(object)", "type") +G_DICT = ("lambda self: {}", "dict") +G_STR = ("lambda self: ''", "string") +G_TUPLE = ("lambda self: tuple()", "tuple") +G_FLOAT = ("lambda self: 0.0", "float") +G_INT = ("lambda self: 0", "int") +G_BOOL = ("lambda self: True", "bool") + +KNOWN_PROPS = { + BUILTIN_MOD_NAME: { + ("object", '__class__'): ('r', G_TYPE), + ('complex', 'real'): ('r', G_FLOAT), + ('complex', 'imag'): ('r', G_FLOAT), + ("file", 'softspace'): ('r', G_BOOL), + ("file", 'name'): ('r', G_STR), + ("file", 'encoding'): ('r', G_STR), + ("file", 'mode'): ('r', G_STR), + ("file", 'closed'): ('r', G_BOOL), + ("file", 'newlines'): ('r', G_STR), + ("slice", 'start'): ('r', G_INT), + ("slice", 'step'): ('r', G_INT), + ("slice", 'stop'): ('r', G_INT), + ("super", '__thisclass__'): ('r', G_TYPE), + ("super", '__self__'): ('r', G_TYPE), + ("super", '__self_class__'): ('r', G_TYPE), + ("type", '__basicsize__'): ('r', G_INT), + ("type", '__itemsize__'): ('r', G_INT), + ("type", '__base__'): ('r', G_TYPE), + ("type", '__flags__'): ('r', G_INT), + ("type", '__mro__'): ('r', G_TUPLE), + ("type", '__bases__'): ('r', G_TUPLE), + ("type", '__dictoffset__'): ('r', G_INT), + ("type", '__dict__'): ('r', G_DICT), + ("type", '__name__'): ('r', G_STR), + ("type", '__weakrefoffset__'): ('r', G_INT), + }, + "exceptions": { + ("BaseException", '__dict__'): ('r', G_DICT), + ("BaseException", 'message'): ('rwd', G_STR), + ("BaseException", 'args'): ('r', G_TUPLE), + ("EnvironmentError", 'errno'): ('rwd', G_INT), + ("EnvironmentError", 'message'): ('rwd', G_STR), + ("EnvironmentError", 'strerror'): ('rwd', G_INT), + ("EnvironmentError", 'filename'): ('rwd', G_STR), + ("SyntaxError", 'text'): ('rwd', G_STR), + ("SyntaxError", 'print_file_and_line'): ('rwd', G_BOOL), + ("SyntaxError", 'filename'): ('rwd', G_STR), + ("SyntaxError", 'lineno'): ('rwd', G_INT), + ("SyntaxError", 'offset'): ('rwd', G_INT), + ("SyntaxError", 'msg'): ('rwd', G_STR), + ("SyntaxError", 'message'): ('rwd', G_STR), + ("SystemExit", 'message'): ('rwd', G_STR), + ("SystemExit", 'code'): ('rwd', G_OBJECT), + ("UnicodeDecodeError", '__basicsize__'): None, + ("UnicodeDecodeError", '__itemsize__'): None, + ("UnicodeDecodeError", '__base__'): None, + ("UnicodeDecodeError", '__flags__'): ('rwd', G_INT), + ("UnicodeDecodeError", '__mro__'): None, + ("UnicodeDecodeError", '__bases__'): None, + ("UnicodeDecodeError", '__dictoffset__'): None, + ("UnicodeDecodeError", '__dict__'): None, + ("UnicodeDecodeError", '__name__'): None, + ("UnicodeDecodeError", '__weakrefoffset__'): None, + ("UnicodeEncodeError", 'end'): ('rwd', G_INT), + ("UnicodeEncodeError", 'encoding'): ('rwd', G_STR), + ("UnicodeEncodeError", 'object'): ('rwd', G_OBJECT), + ("UnicodeEncodeError", 'start'): ('rwd', G_INT), + ("UnicodeEncodeError", 'reason'): ('rwd', G_STR), + ("UnicodeEncodeError", 'message'): ('rwd', G_STR), + ("UnicodeTranslateError", 'end'): ('rwd', G_INT), + ("UnicodeTranslateError", 'encoding'): ('rwd', G_STR), + ("UnicodeTranslateError", 'object'): ('rwd', G_OBJECT), + ("UnicodeTranslateError", 'start'): ('rwd', G_INT), + ("UnicodeTranslateError", 'reason'): ('rwd', G_STR), + ("UnicodeTranslateError", 'message'): ('rwd', G_STR), + }, + '_ast': { + ("AST", '__dict__'): ('rd', G_DICT), + }, + 'posix': { + ("statvfs_result", 'f_flag'): ('r', G_INT), + ("statvfs_result", 'f_bavail'): ('r', G_INT), + ("statvfs_result", 'f_favail'): ('r', G_INT), + ("statvfs_result", 'f_files'): ('r', G_INT), + ("statvfs_result", 'f_frsize'): ('r', G_INT), + ("statvfs_result", 'f_blocks'): ('r', G_INT), + ("statvfs_result", 'f_ffree'): ('r', G_INT), + ("statvfs_result", 'f_bfree'): ('r', G_INT), + ("statvfs_result", 'f_namemax'): ('r', G_INT), + ("statvfs_result", 'f_bsize'): ('r', G_INT), + + ("stat_result", 'st_ctime'): ('r', G_INT), + ("stat_result", 'st_rdev'): ('r', G_INT), + ("stat_result", 'st_mtime'): ('r', G_INT), + ("stat_result", 'st_blocks'): ('r', G_INT), + ("stat_result", 'st_gid'): ('r', G_INT), + ("stat_result", 'st_nlink'): ('r', G_INT), + ("stat_result", 'st_ino'): ('r', G_INT), + ("stat_result", 'st_blksize'): ('r', G_INT), + ("stat_result", 'st_dev'): ('r', G_INT), + ("stat_result", 'st_size'): ('r', G_INT), + ("stat_result", 'st_mode'): ('r', G_INT), + ("stat_result", 'st_uid'): ('r', G_INT), + ("stat_result", 'st_atime'): ('r', G_INT), + }, + "pwd": { + ("struct_pwent", 'pw_dir'): ('r', G_STR), + ("struct_pwent", 'pw_gid'): ('r', G_INT), + ("struct_pwent", 'pw_passwd'): ('r', G_STR), + ("struct_pwent", 'pw_gecos'): ('r', G_STR), + ("struct_pwent", 'pw_shell'): ('r', G_STR), + ("struct_pwent", 'pw_name'): ('r', G_STR), + ("struct_pwent", 'pw_uid'): ('r', G_INT), + + ("struct_passwd", 'pw_dir'): ('r', G_STR), + ("struct_passwd", 'pw_gid'): ('r', G_INT), + ("struct_passwd", 'pw_passwd'): ('r', G_STR), + ("struct_passwd", 'pw_gecos'): ('r', G_STR), + ("struct_passwd", 'pw_shell'): ('r', G_STR), + ("struct_passwd", 'pw_name'): ('r', G_STR), + ("struct_passwd", 'pw_uid'): ('r', G_INT), + }, + "thread": { + ("_local", '__dict__'): None + }, + "xxsubtype": { + ("spamdict", 'state'): ('r', G_INT), + ("spamlist", 'state'): ('r', G_INT), + }, + "zipimport": { + ("zipimporter", 'prefix'): ('r', G_STR), + ("zipimporter", 'archive'): ('r', G_STR), + ("zipimporter", '_files'): ('r', G_DICT), + }, + "_struct": { + ("Struct", "size"): ('r', G_INT), + ("Struct", "format"): ('r', G_STR), + }, + datetime_mod: { + ("datetime", "hour"): ('r', G_INT), + ("datetime", "minute"): ('r', G_INT), + ("datetime", "second"): ('r', G_INT), + ("datetime", "microsecond"): ('r', G_INT), + ("date", "day"): ('r', G_INT), + ("date", "month"): ('r', G_INT), + ("date", "year"): ('r', G_INT), + ("time", "hour"): ('r', G_INT), + ("time", "minute"): ('r', G_INT), + ("time", "second"): ('r', G_INT), + ("time", "microsecond"): ('r', G_INT), + ("timedelta", "days"): ('r', G_INT), + ("timedelta", "seconds"): ('r', G_INT), + ("timedelta", "microseconds"): ('r', G_INT), + }, +} + +# Sometimes module X defines item foo but foo.__module__ == 'Y' instead of 'X'; +# module Y just re-exports foo, and foo fakes being defined in Y. +# We list all such Ys keyed by X, all fully-qualified names: +# {"real_definer_module": ("fake_reexporter_module",..)} +KNOWN_FAKE_REEXPORTERS = { + "_collections": ('collections',), + "_functools": ('functools',), + "_socket": ('socket',), # .error, etc + "pyexpat": ('xml.parsers.expat',), + "_bsddb": ('bsddb.db',), + "pysqlite2._sqlite": ('pysqlite2.dbapi2',), # errors + "numpy.core.multiarray": ('numpy', 'numpy.core'), + "numpy.core._dotblas": ('numpy', 'numpy.core'), + "numpy.core.umath": ('numpy', 'numpy.core'), + "gtk._gtk": ('gtk', 'gtk.gdk',), + "gobject._gobject": ('gobject',), + "gnomecanvas": ("gnome.canvas",), +} + +KNOWN_FAKE_BASES = [] +# list of classes that pretend to be base classes but are mere wrappers, and their defining modules +# [(class, module),...] -- real objects, not names +#noinspection PyBroadException +try: + #noinspection PyUnresolvedReferences + import sip as sip_module # Qt specifically likes it + + if hasattr(sip_module, 'wrapper'): + KNOWN_FAKE_BASES.append((sip_module.wrapper, sip_module)) + if hasattr(sip_module, 'simplewrapper'): + KNOWN_FAKE_BASES.append((sip_module.simplewrapper, sip_module)) + del sip_module +except: + pass + +# This is a list of builtin classes to use fake init +FAKE_BUILTIN_INITS = (tuple, type, int, str) +if version[0] < 3: + FAKE_BUILTIN_INITS = FAKE_BUILTIN_INITS + (getattr(the_builtins, "unicode"),) +else: + FAKE_BUILTIN_INITS = FAKE_BUILTIN_INITS + (getattr(the_builtins, "str"), getattr(the_builtins, "bytes")) + +# Some builtin methods are decorated, but this is hard to detect. +# {("class_name", "method_name"): "decorator"} +KNOWN_DECORATORS = { + ("dict", "fromkeys"): "staticmethod", + ("object", "__subclasshook__"): "classmethod", + ("bytearray", "fromhex"): "classmethod", + ("bytes", "fromhex"): "classmethod", + ("bytearray", "maketrans"): "staticmethod", + ("bytes", "maketrans"): "staticmethod", + ("int", "from_bytes"): "classmethod", + ("float", "fromhex"): "staticmethod", +} + +classobj_txt = ( #TODO: user-skeleton +"class ___Classobj:" "\n" +" '''A mock class representing the old style class base.'''" "\n" +" __module__ = ''" "\n" +" __class__ = None" "\n" +"\n" +" def __init__(self):" "\n" +" pass" "\n" +" __dict__ = {}" "\n" +" __doc__ = ''" "\n" +) + +MAC_STDLIB_PATTERN = re.compile("/System/Library/Frameworks/Python\\.framework/Versions/(.+)/lib/python\\1/(.+)") +MAC_SKIP_MODULES = ["test", "ctypes/test", "distutils/tests", "email/test", + "importlib/test", "json/tests", "lib2to3/tests", + "bsddb/test", + "sqlite3/test", "tkinter/test", "idlelib", "antigravity"] + +POSIX_SKIP_MODULES = ["vtemodule", "PAMmodule", "_snackmodule", "/quodlibet/_mmkeys"] + +BIN_MODULE_FNAME_PAT = re.compile(r'([a-zA-Z_][0-9a-zA-Z_]*)\.(?:pyc|pyo|(?:(?:[a-zA-Z_0-9\-]+\.)?(?:so|pyd)))$') +# possible binary module filename: letter, alphanum architecture per PEP-3149 +TYPELIB_MODULE_FNAME_PAT = re.compile("([a-zA-Z_]+[0-9a-zA-Z]*)[0-9a-zA-Z-.]*\\.typelib") + +MODULES_INSPECT_DIR = ['gi.repository'] +TENSORFLOW_CONTRIB_OPS_MODULE_PATTERN = re.compile(r'tensorflow\.contrib\.(?:.+)\.(?:python\.ops\.|_dataset_ops$)') + +CLASS_ATTR_BLACKLIST = [ + 'google.protobuf.pyext._message.Message._extensions_by_name', + 'google.protobuf.pyext._message.Message._extensions_by_number', + 'panda3d.core.ExecutionEnvironment.environment_variables', +] + +SKELETON_HEADER_VERSION_LINE = re.compile(r'# by generator (?P\d+\.\d+)') +SKELETON_HEADER_ORIGIN_LINE = re.compile(r'# from (?P.*)') +REQUIRED_GEN_VERSION_LINE = re.compile(r'(?P\S+)\s+(?P\d+\.\d+)') +# "mod_path" and "mod_mtime" markers are used in tests +BLACKLIST_VERSION_LINE = re.compile(r'(?P{mod_path}|[^=]+) = (?P\d+\.\d+) (?P{mod_mtime}|\d+)') + +ENV_TEST_MODE_FLAG = 'GENERATOR3_TEST_MODE' +ENV_PREGENERATION_MODE_FLAG = "IS_PREGENERATED_SKELETONS" +ENV_VERSION = 'GENERATOR3_VERSION' +ENV_REQUIRED_GEN_VERSION_FILE = 'GENERATOR3_REQUIRED_GEN_VERSION_FILE' + +FAILED_VERSION_STAMP_PREFIX = '.failed__' + +CACHE_DIR_NAME = 'cache' +STATE_FILE_NAME = '.state.json' diff --git a/docs/stubs_generation/helpers/generator3/core.py b/docs/stubs_generation/helpers/generator3/core.py new file mode 100644 index 000000000..bbd2324d5 --- /dev/null +++ b/docs/stubs_generation/helpers/generator3/core.py @@ -0,0 +1,651 @@ +# encoding: utf-8 +import collections +import fnmatch +import json +import logging +from copy import deepcopy + +from generator3.util_methods import * + +# We need such conditional import always disabled at runtime in order to use +# "typing" without the need to actually bundle the module with PyCharm. +# It's similar to what Mypy recommends with its "MYPY" flag for compatibility +# with Python 3.5.1 (https://mypy.readthedocs.io/en/latest/common_issues.html#import-cycles). +TYPE_CHECKING = False +if TYPE_CHECKING: + from typing import List, Dict, Any, NewType, Tuple, Optional, TextIO + + SkeletonStatusId = NewType('SkeletonStatusId', str) + GenerationStatusId = NewType('GenerationStatusId', str) + GeneratorVersion = Tuple[int, int] + +# TODO: Move all CLR-specific functions to clr_tools +quiet = False +_parent_dir = os.path.dirname(os.path.abspath(__file__)) + + +# TODO move to property of Generator3 as soon as tests finished +@cached +def version(): + env_version = os.environ.get(ENV_VERSION) + if env_version: + return env_version + + with fopen(os.path.join(_parent_dir, 'version.txt'), 'r') as f: + return f.read().strip() + + +# TODO move to property of Generator3 as soon as tests finished +@cached +def required_gen_version_file_path(): + return os.environ.get(ENV_REQUIRED_GEN_VERSION_FILE, os.path.join(_parent_dir, 'required_gen_version')) + + +@cached +def is_test_mode(): + return ENV_TEST_MODE_FLAG in os.environ + + +@cached +def is_pregeneration_mode(): + return ENV_PREGENERATION_MODE_FLAG in os.environ + + +# find_binaries functionality +def cut_binary_lib_suffix(path, f): + """ + @param path where f lives + @param f file name of a possible binary lib file (no path) + @return f without a binary suffix (that is, an importable name) if path+f is indeed a binary lib, or None. + Note: if for .pyc or .pyo file a .py is found, None is returned. + """ + if not f.endswith((".pyc", ".typelib", ".pyo", ".so", ".pyd")): + return None + ret = None + match = BIN_MODULE_FNAME_PAT.match(f) + if match: + ret = match.group(1) + modlen = len('module') + retlen = len(ret) + if ret.endswith('module') and retlen > modlen and f.endswith('.so'): # what for? + ret = ret[:(retlen - modlen)] + if f.endswith('.pyc') or f.endswith('.pyo'): + fullname = os.path.join(path, f[:-1]) # check for __pycache__ is made outside + if os.path.exists(fullname): + ret = None + pat_match = TYPELIB_MODULE_FNAME_PAT.match(f) + if pat_match: + ret = "gi.repository." + pat_match.group(1) + return ret + + +def is_posix_skipped_module(path, f): + if os.name == 'posix': + name = os.path.join(path, f) + for mod in POSIX_SKIP_MODULES: + if name.endswith(mod): + return True + return False + + +def is_mac_skipped_module(path, f): + fullname = os.path.join(path, f) + m = MAC_STDLIB_PATTERN.match(fullname) + if not m: return 0 + relpath = m.group(2) + for module in MAC_SKIP_MODULES: + if relpath.startswith(module): return 1 + return 0 + + +def is_tensorflow_contrib_ops_module(qname): + # These modules cannot be imported directly. Instead tensorflow uses special + # tensorflow.contrib.util.loader.load_op_library() to load them and create + # Python modules at runtime. Their names in sys.modules are then md5 sums + # of the list of exported Python definitions. + return TENSORFLOW_CONTRIB_OPS_MODULE_PATTERN.match(qname) + + +def is_skipped_module(path, f, qname): + return (is_mac_skipped_module(path, f) or + is_posix_skipped_module(path, f[:f.rindex('.')]) or + 'pynestkernel' in f or + is_tensorflow_contrib_ops_module(qname)) + + +def is_module(d, root): + return (os.path.exists(os.path.join(root, d, "__init__.py")) or + os.path.exists(os.path.join(root, d, "__init__.pyc")) or + os.path.exists(os.path.join(root, d, "__init__.pyi")) or + os.path.exists(os.path.join(root, d, "__init__.pyo")) or + is_valid_implicit_namespace_package_name(d)) + + +def walk_python_path(path): + for root, dirs, files in os.walk(path): + if root.endswith('__pycache__'): + continue + dirs_copy = list(dirs) + for d in dirs_copy: + if d.endswith('__pycache__') or not is_module(d, root): + dirs.remove(d) + # some files show up but are actually non-existent symlinks + yield root, [f for f in files if os.path.exists(os.path.join(root, f))] + + +def file_modification_timestamp(path): + return int(os.stat(path).st_mtime) + + +def build_cache_dir_path(subdir, mod_qname, mod_path): + return os.path.join(subdir, module_hash(mod_qname, mod_path)) + + +def module_hash(mod_qname, mod_path): + # Hash the content of a physical module + if mod_path: + hash_ = physical_module_hash(mod_path) + else: + hash_ = builtin_module_hash() + # Use shorter hashes in test data as it might affect developers on Windows + if is_test_mode(): + return hash_[:10] + return hash_ + + +def builtin_module_hash(): + return sha256_digest(sys.version.encode(encoding='utf-8')) + + +def physical_module_hash(mod_path): + with fopen(mod_path, 'rb') as f: + return sha256_digest(f) + + +def version_to_tuple(version): + # type: (str) -> GeneratorVersion + # noinspection PyTypeChecker + return tuple(map(int, version.split('.'))) + + +class OriginType(object): + FILE = 'FILE' + BUILTIN = '(built-in)' + PREGENERATED = '(pre-generated)' + + +class SkeletonStatus(object): + UP_TO_DATE = 'UP_TO_DATE' # type: SkeletonStatusId + """ + Skeleton is up-to-date and doesn't need to be regenerated. + """ + FAILING = 'FAILING' # type: SkeletonStatusId + """ + Skeleton generation is known to fail for this module. + """ + OUTDATED = 'OUTDATED' # type: SkeletonStatusId + """ + Skeleton needs to be regenerated. + """ + + +def skeleton_status(base_dir, mod_qname, mod_path, sdk_skeleton_state=None): + # Force regeneration every time + return SkeletonStatus.OUTDATED + + +def read_used_generator_version_from_skeleton_header(base_dir, mod_qname): + # type: (str, str) -> Optional[GeneratorVersion] + for path in skeleton_path_candidates(base_dir, mod_qname, init_for_pkg=True): + with ignored_os_errors(errno.ENOENT): + with fopen(path, 'r') as f: + return read_generator_version_from_header(f) + return None + + +def read_generator_version_from_header(skeleton_file): + # type: (TextIO) -> Optional[GeneratorVersion] + for line in skeleton_file: + if not line.startswith('#'): + break + + m = SKELETON_HEADER_VERSION_LINE.match(line) + if m: + return version_to_tuple(m.group('version')) + return None + + +def skeleton_path_candidates(base_dir, mod_qname, init_for_pkg=False): + base_path = os.path.join(base_dir, *mod_qname.split('.')) + if init_for_pkg: + yield os.path.join(base_path, '__init__.py') + else: + yield base_path + yield base_path + '.py' + + +def read_failed_version_from_stamp(base_dir, mod_qname): + # type: (str, str) -> Optional[GeneratorVersion] + with ignored_os_errors(errno.ENOENT): + with fopen(os.path.join(base_dir, FAILED_VERSION_STAMP_PREFIX + mod_qname), 'r') as f: + return version_to_tuple(f.read().strip()) + # noinspection PyUnreachableCode + return None + + +def read_failed_version_and_mtime_from_legacy_blacklist(sdk_skeletons_dir, mod_path): + # type: (str, str) -> Optional[Tuple[GeneratorVersion, int]] + blacklist = read_legacy_blacklist_file(sdk_skeletons_dir, mod_path) + return blacklist.get(mod_path) + + +def read_legacy_blacklist_file(sdk_skeletons_dir, mod_path): + # type: (str, str) -> Dict[str, Tuple[GeneratorVersion, int]] + results = {} + with ignored_os_errors(errno.ENOENT): + with fopen(os.path.join(sdk_skeletons_dir, '.blacklist'), 'r') as f: + for line in f: + if not line or line.startswith('#'): + continue + + m = BLACKLIST_VERSION_LINE.match(line) + if m: + bin_path = m.group('path') + bin_mtime = m.group('mtime') + if is_test_mode() and bin_path == '{mod_path}': + bin_path = mod_path + if is_test_mode() and bin_mtime == '{mod_mtime}': + bin_mtime = file_modification_timestamp(mod_path) + else: + # On Java side modification time stored in milliseconds. + # Python API uses seconds for resolution in os.stat results. + bin_mtime = int(m.group('mtime')) / 1000 + results[bin_path] = (version_to_tuple(m.group('version')), bin_mtime) + return results + + +def read_required_version(mod_qname): + # type: (str) -> Optional[GeneratorVersion] + mod_id = '(built-in)' if mod_qname in sys.builtin_module_names else mod_qname + versions = read_required_gen_version_file() + # TODO use glob patterns here + return versions.get(mod_id, versions.get('(default)')) + + +def read_required_gen_version_file(): + # type: () -> Dict[str, GeneratorVersion] + result = {} + with fopen(required_gen_version_file_path(), 'r') as f: + for line in f: + if not line or line.startswith('#'): + continue + m = REQUIRED_GEN_VERSION_LINE.match(line) + if m: + result[m.group('name')] = version_to_tuple(m.group('version')) + + return result + + +class GenerationStatus(object): + FAILED = 'FAILED' # type: GenerationStatusId + """ + Either generation of a skeleton was attempted and failed or cache markers and/or .blacklist indicate that + it was impossible to generate it for the current version of the generator last time. + """ + + GENERATED = 'GENERATED' # type: GenerationStatusId + """ + Skeleton was successfully generated anew and copied both to the cache and a per-sdk skeletons directory. + """ + + COPIED = 'COPIED' # type: GenerationStatusId + """ + Skeleton was successfully copied from the cache to a per-sdk skeletons directory. + """ + + UP_TO_DATE = 'UP_TO_DATE' # type: GenerationStatusId + """ + Existing skeleton is up to date and, therefore, wasn't touched. + """ + + +def get_module_origin(mod_path, mod_qname): + if mod_qname in sys.builtin_module_names: + return OriginType.BUILTIN + + # Unless it's a builtin module all bundled skeletons should have + # file system independent "(pre-generated)" marker in their header + if is_pregeneration_mode(): + return OriginType.PREGENERATED + + if not mod_path: + return None + + if is_test_mode(): + return get_portable_test_module_path(mod_path, mod_qname) + return mod_path + + +def create_failed_version_stamp(base_dir, mod_qname): + failed_version_stamp = os.path.join(base_dir, FAILED_VERSION_STAMP_PREFIX + mod_qname) + with fopen(failed_version_stamp, 'w') as f: + f.write(version()) + return failed_version_stamp + + +def delete_failed_version_stamp(base_dir, mod_qname): + delete(os.path.join(base_dir, FAILED_VERSION_STAMP_PREFIX + mod_qname)) + + +BinaryModule = collections.namedtuple('BinaryModule', ['qname', 'path']) + + +def progress(text=None, fraction=None, minor=False): + data = {} + + if text is not None: + data['text'] = text + data['minor'] = minor + + if fraction is not None: + data['fraction'] = round(fraction, 2) + + control_message('progress', data) + + +def control_message(msg_type, data): + data['type'] = msg_type + say(json.dumps(data)) + + +def trace(msg, *args, **kwargs): + logging.log(logging.getLevelName('TRACE'), msg, *args, **kwargs) + + +class SkeletonGenerator(object): + def __init__(self, + output_dir, # type: str + roots=None, # type: List[str] + state_json=None, # type: Dict[str, Any] + write_state_json=False, + ): + self.output_dir = output_dir.rstrip(os.path.sep) + # TODO make cache directory configurable via CLI + self.cache_dir = os.path.join(os.path.dirname(self.output_dir), CACHE_DIR_NAME) + self.roots = roots + self.in_state_json = state_json + self.out_state_json = {'sdk_skeletons': {}} + self.write_state_json = write_state_json + + def discover_and_process_all_modules(self, name_pattern=None, builtins_only=False): + if name_pattern is None: + name_pattern = '*' + + all_modules = sorted(self.collect_builtin_modules(), key=(lambda b: b.qname)) + + if not builtins_only: + progress("Discovering binary modules...") + all_modules.extend(sorted(self.discover_binary_modules(), key=(lambda b: b.qname))) + + matching_modules = [m for m in all_modules if fnmatch.fnmatchcase(m.qname, name_pattern)] + + progress("Updating skeletons...") + for i, mod in enumerate(matching_modules): + progress(text=mod.qname, fraction=float(i) / len(matching_modules), minor=True) + self.process_module(mod.qname, mod.path) + progress(fraction=1.0) + + if self.write_state_json: + mkdir(self.output_dir) + state_json_path = os.path.join(self.output_dir, STATE_FILE_NAME) + logging.info('Writing skeletons state to %r', state_json_path) + with fopen(state_json_path, 'w') as f: + json.dump(self.out_state_json, f, sort_keys=True) + + @staticmethod + def collect_builtin_modules(): + # type: () -> List[BinaryModule] + names = list(sys.builtin_module_names) + if BUILTIN_MOD_NAME not in names: + names.append(BUILTIN_MOD_NAME) + if '__main__' in names: + names.remove('__main__') + return [BinaryModule(name, None) for name in names] + + def discover_binary_modules(self): + # type: () -> List[BinaryModule] + """ + Finds binaries in the given list of paths. + Understands nested paths, as sys.paths have it (both "a/b" and "a/b/c"). + Tries to be case-insensitive, but case-preserving. + """ + SEP = os.path.sep + res = {} # {name.upper(): (name, full_path)} # b/c windows is case-oblivious + if not self.roots: + return [] + # TODO Move to future InterpreterHandler + if IS_JAVA: # jython can't have binary modules + return [] + paths = sorted_no_case(self.roots) + for path in paths: + for root, files in walk_python_path(path): + cutpoint = path.rfind(SEP) + if cutpoint > 0: + preprefix = path[(cutpoint + len(SEP)):] + '.' + else: + preprefix = '' + prefix = root[(len(path) + len(SEP)):].replace(SEP, '.') + if prefix: + prefix += '.' + binaries = ((f, cut_binary_lib_suffix(root, f)) for f in files) + binaries = [(f, name) for (f, name) in binaries if name] + if binaries: + trace("root: %s path: %s prefix: %s preprefix: %s", root, path, prefix, preprefix) + for f, name in binaries: + the_name = prefix + name + if is_skipped_module(root, f, the_name): + trace('skipping module %s', the_name) + continue + trace("cutout: %s", name) + if preprefix: + trace("prefixes: %s %s", prefix, preprefix) + pre_name = (preprefix + prefix + name).upper() + if pre_name in res: + res.pop(pre_name) # there might be a dupe, if paths got both a/b and a/b/c + trace("done with %s", name) + file_path = os.path.join(root, f) + + res[the_name.upper()] = BinaryModule(the_name, file_path) + return list(res.values()) + + def process_module(self, mod_name, mod_path=None): + # type: (str, str) -> GenerationStatusId + if self.in_state_json: + existing_skeleton_meta = self.in_state_json['sdk_skeletons'].get(mod_name, {}) + sdk_skeleton_state = self.out_state_json['sdk_skeletons'][mod_name] = deepcopy(existing_skeleton_meta) + else: + sdk_skeleton_state = self.out_state_json['sdk_skeletons'][mod_name] = {} + + status = self.reuse_or_generate_skeleton(mod_name, mod_path, sdk_skeleton_state) + control_message('generation_result', { + 'module_name': mod_name, + 'module_origin': get_module_origin(mod_path, mod_name), + 'generation_status': status + }) + if mod_path: + sdk_skeleton_state['bin_mtime'] = file_modification_timestamp(mod_path) + + # If we skipped generation for already failing module, we can safely set + # the current generator version in ".state.json" as skipping means that this + # version is not greater (i.e. we don't need to distinguish between "skipped as failing" + # and "failed during generation"). + if status not in (GenerationStatus.UP_TO_DATE, GenerationStatus.COPIED): + # TODO don't update state_json inplace + sdk_skeleton_state['gen_version'] = version() + + sdk_skeleton_state['status'] = status + + if is_test_mode(): + sdk_skeleton_state.pop('bin_mtime', None) + return status + + def reuse_or_generate_skeleton(self, mod_name, mod_path, mod_state_json): + # type: (str, str, Dict[str, Any]) -> GenerationStatusId + if not quiet: + logging.info('%s (%r)', mod_name, mod_path or 'built-in') + action("doing nothing") + + try: + sdk_skeleton_status = skeleton_status(self.output_dir, mod_name, mod_path, mod_state_json) + if sdk_skeleton_status == SkeletonStatus.UP_TO_DATE: + return GenerationStatus.UP_TO_DATE + elif sdk_skeleton_status == SkeletonStatus.FAILING: + return GenerationStatus.FAILED + + # At this point we will either generate skeleton anew all take it from the cache. + # In either case state.json is supposed to be populated by this results. + if mod_state_json: + mod_state_json.clear() + + mod_cache_dir = build_cache_dir_path(self.cache_dir, mod_name, mod_path) + cached_skeleton_status = skeleton_status(mod_cache_dir, mod_name, mod_path, mod_state_json) + if cached_skeleton_status == SkeletonStatus.OUTDATED: + return execute_in_subprocess_synchronously(name='Skeleton Generator Worker', + func=generate_skeleton, + args=(mod_name, + mod_path, + mod_cache_dir, + self.output_dir), + kwargs={}, + failure_result=GenerationStatus.FAILED) + elif cached_skeleton_status == SkeletonStatus.FAILING: + logging.info('Cache entry for %s at %r indicates failed generation', mod_name, mod_cache_dir) + return GenerationStatus.FAILED + else: + # Copy entire skeletons directory if nothing needs to be updated + logging.info('Copying cached stubs for %s from %r to %r', mod_name, mod_cache_dir, self.output_dir) + copy_skeletons(mod_cache_dir, self.output_dir, get_module_origin(mod_path, mod_name)) + return GenerationStatus.COPIED + except: + exctype, value = sys.exc_info()[:2] + msg = "Failed to process %r while %s: %s" + args = mod_name, CURRENT_ACTION, str(value) + report(msg, *args) + if sys.platform == 'cli': + import traceback + traceback.print_exc(file=sys.stderr) + raise + + +@contextmanager +def imported_names_collected(): + imported_names = set() + + class MyFinder(object): + # noinspection PyMethodMayBeStatic + def find_module(self, fullname, path=None): + imported_names.add(fullname) + return None + + my_finder = MyFinder() + sys.meta_path.insert(0, my_finder) + try: + yield imported_names + finally: + sys.meta_path.remove(my_finder) + + +def generate_skeleton(name, mod_file_name, mod_cache_dir, output_dir): + # type: (str, str, str, str) -> GenerationStatusId + + logging.info('Updating cache for %s at %r', name, mod_cache_dir) + doing_builtins = mod_file_name is None + # All builtin modules go into the same directory + if not doing_builtins: + delete(mod_cache_dir) + + # delete output path so it can be regenerated + delete(os.path.join(output_dir, name)) + # we don't use the cache dir + #mkdir(mod_cache_dir) + + #create_failed_version_stamp(mod_cache_dir, name) + + action("importing") + old_modules = list(sys.modules.keys()) + with imported_names_collected() as imported_module_names: + __import__(name) # sys.modules will fill up with what we want + + redo_module(name, mod_file_name, mod_cache_dir, output_dir) + # The C library may have called Py_InitModule() multiple times to define several modules (gtk._gtk and gtk.gdk); + # restore all of them + path = name.split(".") + redo_imports = not ".".join(path[:-1]) in MODULES_INSPECT_DIR + if redo_imports: + initial_module_set = set(sys.modules) + for m in list(sys.modules): + if not m.startswith(name): + continue + # Python 2 puts dummy None entries in sys.modules for imports of + # top-level modules made from inside packages unless absolute + # imports are explicitly enabled. + # See https://www.python.org/dev/peps/pep-0328/#relative-imports-and-indirection-entries-in-sys-modules + if not sys.modules[m] or m.startswith("generator3"): + continue + action("looking at possible submodule %r", m) + if m == name or m in old_modules or m in sys.builtin_module_names: + continue + # Synthetic module, not explicitly imported + if m not in imported_module_names and not hasattr(sys.modules[m], '__file__'): + if not quiet: + logging.info('Processing submodule %s of %s', m, name) + action("opening %r", mod_cache_dir) + try: + redo_module(m, mod_file_name, cache_dir=mod_cache_dir, output_dir=output_dir) + extra_modules = set(sys.modules) - initial_module_set + if extra_modules: + report('Introspecting submodule %r of %r led to extra content of sys.modules: %s', + m, name, ', '.join(extra_modules)) + finally: + action("closing %r", mod_cache_dir) + return GenerationStatus.GENERATED + + +def redo_module(module_name, module_file_name, cache_dir, output_dir): + # type: (str, str, str, str) -> None + # gobject does 'del _gobject' in its __init__.py, so the chained attribute lookup code + # fails to find 'gobject._gobject'. thus we need to pull the module directly out of + # sys.modules + mod = sys.modules.get(module_name) + mod_path = module_name.split('.') + if not mod and sys.platform == 'cli': + # "import System.Collections" in IronPython 2.7 doesn't actually put System.Collections in sys.modules + # instead, sys.modules['System'] get set to a Microsoft.Scripting.Actions.NamespaceTracker and Collections can be + # accessed as its attribute + mod = sys.modules[mod_path[0]] + for component in mod_path[1:]: + try: + mod = getattr(mod, component) + except AttributeError: + mod = None + report("Failed to find CLR module " + module_name) + break + if mod: + action("restoring") + from generator3.module_redeclarator import ModuleRedeclarator + # Generate output directly in output folder, don't create cache folder + r = ModuleRedeclarator(mod, module_name, module_file_name, cache_dir=output_dir, + doing_builtins=(module_file_name is None)) + create_failed_version_stamp(output_dir, module_name) + r.redo(module_name, ".".join(mod_path[:-1]) in MODULES_INSPECT_DIR) + action("flushing") + r.flush() + delete_failed_version_stamp(output_dir, module_name) + # Incrementally copy whatever we managed to successfully generate so far + #copy_skeletons(cache_dir, output_dir, get_module_origin(module_file_name, module_name)) + else: + report("Failed to find imported module in sys.modules " + module_name) + + + + diff --git a/docs/stubs_generation/helpers/generator3/docstring_parsing.py b/docs/stubs_generation/helpers/generator3/docstring_parsing.py new file mode 100644 index 000000000..ebbddb555 --- /dev/null +++ b/docs/stubs_generation/helpers/generator3/docstring_parsing.py @@ -0,0 +1,210 @@ +import re +import sys + +from generator3.constants import STR_TYPES +from generator3.util_methods import sanitize_value + +# only support Python 3 +# noinspection PyUnresolvedReferences +from generator3._vendor.pyparsing_py3 import * + +# grammar to parse parameter lists + +# // snatched from parsePythonValue.py, from pyparsing samples, copyright 2006 by Paul McGuire but under BSD license. +# we don't suppress lots of punctuation because we want it back when we reconstruct the lists + +lparen, rparen, lbrack, rbrack, lbrace, rbrace, colon = map(Literal, "()[]{}:") + +integer = Combine(Optional(oneOf("+ -")) + Word(nums)).setName("integer") +real = Combine(Optional(oneOf("+ -")) + Word(nums) + "." + + Optional(Word(nums)) + + Optional(oneOf("e E") + Optional(oneOf("+ -")) + Word(nums))).setName("real") +tupleStr = Forward() +listStr = Forward() +dictStr = Forward() + +boolLiteral = oneOf("True False") +noneLiteral = Literal("None") + +listItem = real | integer | quotedString | unicodeString | boolLiteral | noneLiteral | \ + Group(listStr) | tupleStr | dictStr + +tupleStr << (Suppress("(") + Optional(delimitedList(listItem)) + + Optional(Literal(",")) + Suppress(")")).setResultsName("tuple") + +listStr << (lbrack + Optional(delimitedList(listItem) + + Optional(Literal(","))) + rbrack).setResultsName("list") + +dictEntry = Group(listItem + colon + listItem) +dictStr << (lbrace + Optional(delimitedList(dictEntry) + Optional(Literal(","))) + rbrace).setResultsName("dict") +# \\ end of the snatched part + +# our output format is s-expressions: +# (simple name optional_value) is name or name=value +# (nested (simple ...) (simple ...)) is (name, name,...) +# (opt ...) is [, ...] or suchlike. + +T_SIMPLE = 'Simple' +T_NESTED = 'Nested' +T_OPTIONAL = 'Opt' +T_RETURN = "Ret" + +TRIPLE_DOT = '...' + +COMMA = Suppress(",") +APOS = Suppress("'") +QUOTE = Suppress('"') +SP = Suppress(Optional(White())) + +ident = Word(alphas + "_", alphanums + "_-.").setName("ident") # we accept things like "foo-or-bar" +decorated_ident = ident + Optional(Suppress(SP + Literal(":") + SP + ident)) # accept "foo: bar", ignore "bar" +spaced_ident = Combine( + decorated_ident + ZeroOrMore(Literal(' ') + decorated_ident)) # we accept 'list or tuple' or 'C struct' + +# allow quoted names, because __setattr__, etc docs use it +paramname = spaced_ident | \ + APOS + spaced_ident + APOS | \ + QUOTE + spaced_ident + QUOTE + +parenthesized_tuple = (Literal("(") + Optional(delimitedList(listItem, combine=True)) + + Optional(Literal(",")) + Literal(")")).setResultsName("(tuple)") + +initializer = (SP + Suppress("=") + SP + Combine(parenthesized_tuple | listItem | ident)).setName( + "=init") # accept foo=defaultfoo + +param = Group(Empty().setParseAction(replaceWith(T_SIMPLE)) + Combine(Optional(oneOf("* **")) + paramname) + Optional( + initializer)) + +ellipsis = Group( + Empty().setParseAction(replaceWith(T_SIMPLE)) + \ + (Literal("..") + + ZeroOrMore(Literal('.'))).setParseAction(replaceWith(TRIPLE_DOT)) # we want to accept both 'foo,..' and 'foo, ...' +) + +paramSlot = Forward() + +simpleParamSeq = ZeroOrMore(paramSlot + COMMA) + Optional(paramSlot + Optional(COMMA)) +nestedParamSeq = Group( + Suppress('(').setParseAction(replaceWith(T_NESTED)) + \ + simpleParamSeq + Optional(ellipsis + Optional(COMMA) + Optional(simpleParamSeq)) + \ + Suppress(')') +) # we accept "(a1, ... an)" + +paramSlot << (param | nestedParamSeq) + +optionalPart = Forward() + +paramSeq = simpleParamSeq + Optional(optionalPart) # this is our approximate target + +optionalPart << ( + Group( + Suppress('[').setParseAction(replaceWith(T_OPTIONAL)) + Optional(COMMA) + + paramSeq + Optional(ellipsis) + + Suppress(']') + ) + | ellipsis +) + +return_type = Group( + Empty().setParseAction(replaceWith(T_RETURN)) + + Suppress(SP + (Literal("->") | (Literal(":") + SP + Literal("return"))) + SP) + + ident +) + +# this is our ideal target, with balancing paren and a multiline rest of doc. +paramSeqAndRest = paramSeq + Suppress(')') + Optional(return_type) + Suppress(Optional(Regex(r"(?s).*"))) + + +def transform_seq(results, toplevel=True): + """Transforms a tree of ParseResults into a param spec string.""" + is_clr = sys.platform == "cli" + ret = [] # add here token to join + for token in results: + token_type = token[0] + if token_type is T_SIMPLE: + token_name = token[1] + if len(token) == 3: # name with value + if toplevel: + ret.append(sanitize_ident(token_name, is_clr) + "=" + sanitize_value(token[2])) + else: + # smth like "a, (b1=1, b2=2)", make it "a, p_b" + return ["p_" + results[0][1]] # NOTE: for each item of tuple, return the same name of its 1st item. + elif token_name == TRIPLE_DOT: + if toplevel and not has_item_starting_with(ret, "*"): + ret.append("*more") + else: + # we're in a "foo, (bar1, bar2, ...)"; make it "foo, bar_tuple" + return extract_alpha_prefix(results[0][1]) + "_tuple" + else: # just name + ret.append(sanitize_ident(token_name, is_clr)) + elif token_type is T_NESTED: + inner = transform_seq(token[1:], False) + if len(inner) != 1: + ret.append(inner) + else: + ret.append(inner[0]) # [foo] -> foo + elif token_type is T_OPTIONAL: + ret.extend(transform_optional_seq(token)) + elif token_type is T_RETURN: + pass # this is handled elsewhere + else: + raise Exception("This cannot be a token type: " + repr(token_type)) + return ret + + +def transform_optional_seq(results): + """ + Produces a string that describes the optional part of parameters. + @param results must start from T_OPTIONAL. + """ + assert results[0] is T_OPTIONAL, "transform_optional_seq expects a T_OPTIONAL node, sees " + \ + repr(results[0]) + is_clr = sys.platform == "cli" + ret = [] + for token in results[1:]: + token_type = token[0] + if token_type is T_SIMPLE: + token_name = token[1] + if len(token) == 3: # name with value; little sense, but can happen in a deeply nested optional + ret.append(sanitize_ident(token_name, is_clr) + "=" + sanitize_value(token[2])) + elif token_name == '...': + # we're in a "foo, [bar, ...]"; make it "foo, *bar" + return ["*" + extract_alpha_prefix( + results[1][1])] # we must return a seq; [1] is first simple, [1][1] is its name + else: # just name + ret.append(sanitize_ident(token_name, is_clr) + "=None") + elif token_type is T_OPTIONAL: + ret.extend(transform_optional_seq(token)) + # maybe handle T_NESTED if such cases ever occur in real life + # it can't be nested in a sane case, really + return ret + + +def has_item_starting_with(p_seq, p_start): + for item in p_seq: + if isinstance(item, STR_TYPES) and item.startswith(p_start): + return True + return False + + +def sanitize_ident(x, is_clr=False): + """Takes an identifier and returns it sanitized""" + if x in ("class", "object", "def", "list", "tuple", "int", "float", "str", "unicode" "None"): + return "p_" + x + else: + if is_clr: + # it tends to have names like "int x", turn it to just x + xs = x.split(" ") + if len(xs) == 2: + return sanitize_ident(xs[1]) + return x.replace("-", "_").replace(" ", "_").replace(".", "_") # for things like "list-or-tuple" or "list or tuple" + + +def extract_alpha_prefix(p_string, default_prefix="some"): + """Returns 'foo' for things like 'foo1' or 'foo2'; if prefix cannot be found, the default is returned""" + match = NUM_IDENT_PATTERN.match(p_string) + prefix = match and match.groups()[match.lastindex - 1] or None + return prefix or default_prefix + + +NUM_IDENT_PATTERN = re.compile("([A-Za-z_]+)[0-9]?[A-Za-z_]*") # 'foo_123' -> $1 = 'foo_' diff --git a/docs/stubs_generation/helpers/generator3/extra.py b/docs/stubs_generation/helpers/generator3/extra.py new file mode 100644 index 000000000..134b58ff3 --- /dev/null +++ b/docs/stubs_generation/helpers/generator3/extra.py @@ -0,0 +1,165 @@ +import os +import re +import sys +import zipfile + +from generator3.core import walk_python_path +from generator3.util_methods import is_text_file, say, report + + +def is_source_file(path): + # Skip directories, character and block special devices, named pipes + # Do not skip regular files and symbolic links to regular files + if not os.path.isfile(path): + return False + + # Want to see that files regardless of their encoding. + if path.endswith(('-nspkg.pth', '.html', '.pxd', '.py', '.pyi', '.pyx')): + return True + has_bad_extension = path.endswith(( + # plotlywidget/static/index.js.map is 8.7 MiB. + # Many map files from notebook are near 2 MiB. + '.js.map', + + # uvloop/loop.c contains 6.4 MiB of code. + # Some header files from tensorflow has size more than 1 MiB. + '.h', '.c', + + # Test data of pycrypto, many files are near 1 MiB. + '.rsp', + + # No need to read these files even if they are small. + '.dll', '.pyc', '.pyd', '.pyo', '.so', + )) + if has_bad_extension: + return False + return is_text_file(path) + + +def list_sources(paths): + # noinspection PyBroadException + try: + for path in paths: + path = os.path.normpath(path) + + if path.endswith('.egg') and os.path.isfile(path): + say("%s\t%s\t%d", path, path, os.path.getsize(path)) + + for root, files in walk_python_path(path): + for name in files: + file_path = os.path.join(root, name) + if is_source_file(file_path): + say("%s\t%s\t%d", os.path.normpath(file_path), path, os.path.getsize(file_path)) + say('END') + sys.stdout.flush() + except: + import traceback + + traceback.print_exc() + sys.exit(1) + + +def zip_sources(zip_path): + if not os.path.exists(zip_path): + os.makedirs(zip_path) + + zip_filename = os.path.normpath(os.path.sep.join([zip_path, "skeletons.zip"])) + + try: + zip = zipfile.ZipFile(zip_filename, 'w', zipfile.ZIP_DEFLATED) + except: + zip = zipfile.ZipFile(zip_filename, 'w') + + try: + try: + while True: + line = sys.stdin.readline() + + if not line: + # TextIOWrapper.readline returns an empty string if EOF is hit immediately. + break + + line = line.strip() + + if line == '-': + break + + if line: + # This line will break the split: + # /.../dist-packages/setuptools/script template (dev).py setuptools/script template (dev).py + split_items = line.split() + if len(split_items) > 2: + # Currently it doesn't work for remote files like + # /System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python/setuptools/script (dev).tmpl + # TODO handle paths containing whitespaces more robustly + match_two_files = re.match(r'^(.+\.py)\s+(.+\.py)$', line) + if not match_two_files: + report("Error(zip_sources): invalid line '%s'" % line) + continue + split_items = match_two_files.group(1, 2) + (path, arcpath) = split_items + + # An attempt to recursively pack an archive leads to unlimited explosion of its size + if os.path.samefile(path, zip_filename): + continue + + zip.write(path, arcpath) + say('OK: ' + zip_filename) + sys.stdout.flush() + except: + import traceback + + traceback.print_exc() + say('Error creating archive.') + + sys.exit(1) + finally: + zip.close() + + +def add_to_zip(zip, paths): + # noinspection PyBroadException + try: + for path in paths: + print("Walking root %s" % path) + + path = os.path.normpath(path) + + if path.endswith('.egg') and os.path.isfile(path): + pass # TODO: handle eggs + + for root, files in walk_python_path(path): + for name in files: + file_path = os.path.join(root, name) + arcpath = os.path.relpath(file_path, path) + + zip.write(file_path, os.path.join(str(hash(path)), arcpath)) + except: + import traceback + + traceback.print_exc() + sys.exit(1) + + +def zip_stdlib(roots, zip_path): + if not os.path.exists(zip_path): + os.makedirs(zip_path) + + import platform + + zip_filename = os.path.normpath(os.path.sep.join([zip_path, "%s-%s-stdlib-%s.zip" % ( + 'Anaconda' if sys.version.find('Anaconda') != -1 else 'Python', + '.'.join(map(str, sys.version_info)), + platform.platform())])) + + print("Adding file to %s" % zip_filename) + + try: + zip = zipfile.ZipFile(zip_filename, 'w', zipfile.ZIP_DEFLATED) + except: + zip = zipfile.ZipFile(zip_filename, 'w') + + try: + add_to_zip(zip, roots) + finally: + zip.close() \ No newline at end of file diff --git a/docs/stubs_generation/helpers/generator3/module_redeclarator.py b/docs/stubs_generation/helpers/generator3/module_redeclarator.py new file mode 100644 index 000000000..96c684684 --- /dev/null +++ b/docs/stubs_generation/helpers/generator3/module_redeclarator.py @@ -0,0 +1,1353 @@ +from generator3.core import OriginType +from generator3.util_methods import * +from generator3.util_methods import get_portable_test_module_path +from generator3.docstring_parsing import * +import re +import enum + +# Add patterns for identifying declarations we care about +RTYPE_PATTERN = re.compile(r"[:@]rtype:\s*(.*)") +TYPE_PATTERN = re.compile(r"[:@]type\s*:\s*(.*)") +DATA_PATTERN = re.compile(r".. data::\s*(.*)") +PARAM_PATTERN = re.compile(r"[:@]param\s+([^:]*)\s+([^: ]*):") +NAMESPACE_PATTERN = re.compile(r"([^[\]]+\.)*([a-zA-Z][a-zA-Z0-9_]+)") +DEFAULTS = re.compile(r"(=.*)") + + +class emptylistdict(dict): + """defaultdict not available before 2.5; simplest reimplementation using [] as default""" + + def __getitem__(self, item): + if item in self: + return dict.__getitem__(self, item) + else: + it = [] + self.__setitem__(item, it) + return it + + +class Buf(object): + """Buffers data in a list, can write to a file. Indentation is provided externally.""" + + def __init__(self, indenter): + self.data = [] + self.indenter = indenter + + def put(self, data): + if data: + self.data.append(ensureUnicode(data)) + + def out(self, indent, *what): + """Output the arguments, indenting as needed, and adding an eol""" + self.put(self.indenter.indent(indent)) + for item in what: + self.put(item) + self.put("\n") + + def flush_bytes(self, outfile): + for data in self.data: + outfile.write(data.encode(OUT_ENCODING, "replace")) + + def flush_str(self, outfile): + for data in self.data: + outfile.write(data) + + if version[0] < 3: + flush = flush_bytes + else: + flush = flush_str + + def isEmpty(self): + return len(self.data) == 0 + + +class ClassBuf(Buf): + def __init__(self, name, indenter): + super(ClassBuf, self).__init__(indenter) + self.name = name + + +#noinspection PyBroadException +class ModuleRedeclarator(object): + def __init__(self, module, mod_qname, mod_filename, cache_dir, indent_size=4, doing_builtins=False): + """ + @param module: module object + @param mod_qname: module qualified name + @param mod_filename: filename of binary module (the .dll or .so). Can be None for modules + that don't have corresponding binary files (e.g. builtins) + @param cache_dir: per-binary cache directory where the generated stub will be stored. + Normally, it's "/python_stubs/cache//". + @param indent_size: amount of space characters per indent + """ + import generator3.core + self.test_mode = generator3.core.is_test_mode() + self.gen_version = generator3.core.version() + self.module = module + self.qname = mod_qname + self.cache_dir = cache_dir + self.mod_filename = mod_filename + # we write things into buffers out-of-order + self.header_buf = Buf(self) + self.imports_buf = Buf(self) + # each class gets its own list of dependency imports, and so do functions + self.func_imports_buf = Buf(self) + self.functions_buf = Buf(self) + self.classes_buf = Buf(self) + self.classes_buffs = list() + self.footer_buf = Buf(self) + self.indent_size = indent_size + self._indent_step = " " * self.indent_size + self.split_modules = False + # + self.imported_modules = {"": the_builtins} # explicit module imports: {"name": module} + self.hidden_imports = {} # {'real_mod_name': 'alias'}; we alias names with "__" since we don't want them exported + # ^ used for things that we don't re-export but need to import, e.g. certain base classes in gnome. + self._defined = {} # stores True for every name defined so far, to break circular refs in values + self.doing_builtins = doing_builtins + self.ret_type_cache = {} + self.used_imports = emptylistdict() # qual_mod_name -> [imported_names,..]: actually used imported names + # if we use Typing hints, import some things for it + self.used_typing = False + + def _initializeQApp4(self): + try: # QtGui should be imported _before_ QtCore package. + # This is done for the QWidget references from QtCore (such as QSignalMapper). Known bug in PyQt 4.7+ + # Causes "TypeError: C++ type 'QWidget*' is not supported as a native Qt signal type" + import PyQt4.QtGui + except ImportError: + pass + + # manually instantiate and keep reference to singleton QCoreApplication (we don't want it to be deleted during the introspection) + # use QCoreApplication instead of QApplication to avoid blinking app in Dock on Mac OS + try: + from PyQt4.QtCore import QCoreApplication + self.app = QCoreApplication([]) + return + except ImportError: + pass + + def _initializeQApp5(self): + try: + from PyQt5.QtCore import QCoreApplication + self.app = QCoreApplication([]) + return + except ImportError: + pass + + def indent(self, level): + """Return indentation whitespace for given level.""" + return self._indent_step * level + + def flush(self): + qname_parts = self.qname.split('.') + if self.split_modules: + last_pkg_dir = build_pkg_structure(self.cache_dir, self.qname) + with fopen(os.path.join(last_pkg_dir, "__init__.py"), "w") as init: + for buf in (self.header_buf, self.imports_buf, self.classes_buf): + buf.flush(init) + + data = "" + for (buf,imports) in self.classes_buffs: + with fopen(os.path.join(last_pkg_dir, buf.name) + '.py', "w") as dummy: + self.header_buf.flush(dummy) + self.imports_buf.flush(dummy) + imports.flush(dummy) + buf.flush(dummy) + data += self.create_local_import(buf.name) + + init.write(data) + + # Write out functions last so they can reference the classes imported above + for buf in (self.func_imports_buf, self.functions_buf, self.footer_buf): + buf.flush(init) + else: + last_pkg_dir = build_pkg_structure(self.cache_dir, '.'.join(qname_parts[:-1])) + # In some rare cases submodules of a binary might have been generated earlier than the module + # for the binary itself. For instance, it happens for "pyexpat" built-in module which + # submodules "pyexpat.errors" and "pyexpat.model" are processed together with "_elementtree" + # and "pickle" before "pyexpat" and thus empty pyexpat/__init__.py for them should be replaced + # with the skeleton for the main module itself later on. + existing_pkg_init = os.path.join(last_pkg_dir, qname_parts[-1], '__init__.py') + if os.path.exists(existing_pkg_init): + skeleton_path = existing_pkg_init + else: + skeleton_path = os.path.join(last_pkg_dir, qname_parts[-1] + '.py') + with fopen(skeleton_path, "w") as mod: + for buf in (self.header_buf, self.imports_buf, self.classes_buf): + buf.flush(mod) + + for (buf,imports) in self.classes_buffs: + imports.flush(mod) + buf.flush(mod) + + # Write out functions last so they can reference the classes imported above + for buf in (self.func_imports_buf, self.functions_buf, self.footer_buf): + buf.flush(mod) + + # Some builtin classes effectively change __init__ signature without overriding it. + # This callable serves as a placeholder to be replaced via REDEFINED_BUILTIN_SIGS + def fake_builtin_init(self): + pass # just a callable, sig doesn't matter + + fake_builtin_init.__doc__ = object.__init__.__doc__ # this forces class's doc to be used instead + + def create_local_import(self, name): + if len(name.split(".")) > 1: return "" + data = "from " + if version[0] >= 3: + data += "." + data += name + " import " + name + "\n" + return data + + # Parse a typing declaration and add the actual types references + def add_import_types(self, import_types, type_decl): + if '[' not in type_decl: + import_types.add(type_decl) + return + + match = re.match('^[tT]uple\[(.*)\]$', type_decl) + if match: + for i in match.group(1).split(','): + self.add_import_types(import_types, i.strip()) + return + + match = re.match('^[lL]ist\[(.*)\]$', type_decl) + if match: + self.add_import_types(import_types, match.group(1).strip()) + return + + match = re.match('^[cC]allable\[\[(.*)\]\s*,\s*(.*)\]$', type_decl) + if match: + for i in match.group(1).split(','): + self.add_import_types(import_types, i.strip()) + self.add_import_types(import_types, match.group(2).strip()) + return + + # For a given type that's referenced, figure out where to import it from and add + # to the list + def process_import_type(self, used_imports, p_modname, classname, import_type): + parent = None + + if import_type == '...': + return + + if import_type in dir(sys.modules[p_modname]): + if import_type != classname: + parent = '.' + child = import_type + elif '.' in import_type: + imp_split = import_type.split('.') + parent = '.'.join(imp_split) + child = imp_split[-1] + + if parent is not None and child not in used_imports[parent]: + used_imports[parent].append(child) + + def find_imported_name(self, item): + """ + Finds out how the item is represented in imported modules. + @param item what to check + @return qualified name (like "sys.stdin") or None + """ + # TODO: return a pair, not a glued string + if not isinstance(item, SIMPLEST_TYPES): + for mname in self.imported_modules: + m = self.imported_modules[mname] + for inner_name in m.__dict__: + suspect = getattr(m, inner_name) + if suspect is item: + if mname: + mname += "." + elif self.module is the_builtins: # don't short-circuit builtins + return None + return mname + inner_name + return None + + _initializers = ( + (dict, "{}"), + (tuple, "()"), + (list, "[]"), + ) + + def invent_initializer(self, a_type): + """ + Returns an innocuous initializer expression for a_type, or "None" + """ + for initializer_type, r in self._initializers: + if initializer_type == a_type: + return r + # NOTE: here we could handle things like defaultdict, sets, etc if we wanted + return "None" + + def fmt_value(self, out, p_value, indent, prefix="", postfix="", as_name=None, seen_values=None): + """ + Formats and outputs value (it occupies an entire line or several lines). + @param out function that does output (a Buf.out) + @param p_value the value. + @param indent indent level. + @param prefix text to print before the value + @param postfix text to print after the value + @param as_name hints which name are we trying to print; helps with circular refs. + @param seen_values a list of keys we've seen if we're processing a dict + """ + SELF_VALUE = "" + ERR_VALUE = "" + if isinstance(p_value, SIMPLEST_TYPES): + out(indent, prefix, reliable_repr(p_value), postfix) + else: + if sys.platform == "cli": + imported_name = None + else: + imported_name = self.find_imported_name(p_value) + if imported_name: + out(indent, prefix, imported_name, postfix) + # TODO: kind of self.used_imports[imported_name].append(p_value) but split imported_name + # else we could potentially return smth we did not otherwise import. but not likely. + else: + if isinstance(p_value, (list, tuple)): + if not seen_values: + seen_values = [p_value] + if len(p_value) == 0: + out(indent, prefix, repr(p_value), postfix) + else: + if isinstance(p_value, list): + lpar, rpar = "[", "]" + else: + lpar, rpar = "(", ")" + out(indent, prefix, lpar) + for value in p_value: + if value in seen_values: + value = SELF_VALUE + elif not isinstance(value, SIMPLEST_TYPES): + seen_values.append(value) + self.fmt_value(out, value, indent + 1, postfix=",", seen_values=seen_values) + out(indent, rpar, postfix) + elif isinstance(p_value, dict): + if len(p_value) == 0: + out(indent, prefix, repr(p_value), postfix) + else: + if not seen_values: + seen_values = [p_value] + out(indent, prefix, "{") + keys = list(p_value.keys()) + try: + keys.sort() + except TypeError: + pass # unsortable keys happen, e,g, in py3k _ctypes + for k in keys: + value = p_value[k] + + try: + is_seen = value in seen_values + except: + is_seen = False + value = ERR_VALUE + + if is_seen: + value = SELF_VALUE + elif not isinstance(value, SIMPLEST_TYPES): + seen_values.append(value) + if isinstance(k, SIMPLEST_TYPES): + self.fmt_value(out, value, indent + 1, prefix=repr(k) + ": ", postfix=",", + seen_values=seen_values) + else: + # both key and value need fancy formatting + self.fmt_value(out, k, indent + 1, postfix=": ", seen_values=seen_values) + self.fmt_value(out, value, indent + 2, seen_values=seen_values) + out(indent + 1, ",") + out(indent, "}", postfix) + else: # something else, maybe representable + # look up this value in the module. + if sys.platform == "cli": + out(indent, prefix, "None", postfix) + return + found_name = "" + for inner_name in self.module.__dict__: + if self.module.__dict__[inner_name] is p_value: + found_name = inner_name + break + if self._defined.get(found_name, False): + out(indent, prefix, found_name, postfix) + elif hasattr(self, "app"): + return + else: + # a forward / circular declaration happens + notice = "" + try: + representation = repr(p_value) + except Exception: + import traceback + traceback.print_exc(file=sys.stderr) + return + if not self.test_mode: + real_value = cleanup(representation) + else: + # Don't rely on repr() output in tests, as it may contain memory layout dependent id + real_value = '' + if found_name: + if found_name == as_name: + notice = " # (!) real value is %r" % real_value + real_value = "None" + else: + notice = " # (!) forward: %s, real value is %r" % (found_name, real_value) + if SANE_REPR_RE.match(real_value) and is_valid_expr(real_value): + out(indent, prefix, real_value, postfix, notice) + else: + if not found_name: + notice = " # (!) real value is %r" % real_value + out(indent, prefix, "None", postfix, notice) + + def get_ret_type(self, attr): + """ + Returns a return type string as given by T_RETURN in tokens, or None + """ + if attr: + ret_type = RET_TYPE.get(attr, None) + if ret_type: + return ret_type + thing = getattr(self.module, attr, None) + if thing: + if not isinstance(thing, type) and is_callable(thing): # a function + return None # TODO: maybe divinate a return type; see pygame.mixer.Channel + return attr + # adds no noticeable slowdown, I did measure. dch. + for im_name, im_module in self.imported_modules.items(): + cache_key = (im_name, attr) + cached = self.ret_type_cache.get(cache_key, None) + if cached: + return cached + ret_type = getattr(im_module, attr, None) + if ret_type: + if isinstance(ret_type, type): + # detect a constructor + constr_args = detect_constructor(ret_type) + if constr_args is None: + constr_args = "*(), **{}" # a silly catch-all constructor + reference = "%s(%s)" % (attr, constr_args) + elif is_callable(ret_type): # a function, classes are ruled out above + return None + else: + reference = attr + if im_name: + result = "%s.%s" % (im_name, reference) + else: # built-in + result = reference + self.ret_type_cache[cache_key] = result + return result + # TODO: handle things like "[a, b,..] and (foo,..)" + return None + + + SIG_DOC_NOTE = "restored from __doc__" + SIG_DOC_UNRELIABLY = "NOTE: unreliably restored from __doc__ " + + def restore_by_docstring(self, signature_string, class_name, deco=None, ret_hint=None): + """ + @param signature_string: parameter list extracted from the doc string. + @param class_name: name of the containing class, or None + @param deco: decorator to use + @param ret_hint: return type hint, if available + @return (reconstructed_spec, return_type, note) or (None, _, _) if failed. + """ + action("restoring func %r of class %r", signature_string, class_name) + # parse + parsing_failed = False + ret_type = None + try: + # strict parsing + tokens = paramSeqAndRest.parseString(signature_string, True) + ret_name = None + if tokens: + ret_t = tokens[-1] + if ret_t[0] is T_RETURN: + ret_name = ret_t[1] + ret_type = self.get_ret_type(ret_name) or self.get_ret_type(ret_hint) + except ParseException: + # it did not parse completely; scavenge what we can + parsing_failed = True + tokens = [] + try: + # most unrestrictive parsing + tokens = paramSeq.parseString(signature_string, False) + except ParseException: + pass + # + seq = transform_seq(tokens) + + # add safe defaults for unparsed + if parsing_failed: + doc_node = self.SIG_DOC_UNRELIABLY + starred = None + double_starred = None + for one in seq: + if type(one) is str: + if one.startswith("**"): + double_starred = one + elif one.startswith("*"): + starred = one + if not starred: + seq.append("*args") + if not double_starred: + seq.append("**kwargs") + else: + doc_node = self.SIG_DOC_NOTE + + # add 'self' if needed YYY + if class_name and (not seq or seq[0] != 'self'): + first_param = propose_first_param(deco) + if first_param: + seq.insert(0, first_param) + seq = make_names_unique(seq) + + import_types = set() + ret_hint = None + + # Try to use :rtype: to add explicit type annotations to return type, since PyCharm + # doesn't parse rtype properly (at least with split modules) + if ret_type is None and ':rtype:' in signature_string: + result = RTYPE_PATTERN.search(signature_string) + if result is not None: + type_decl = result.group(1).strip() + if type_decl != class_name: + self.add_import_types(import_types, type_decl) + ret_hint = NAMESPACE_PATTERN.sub(r'\2', type_decl) + else: + ret_hint = "'" + type_decl + "'" + self.used_typing = True + + # Also use :param: to add necessary imports and fix up the parameters. + # PyCharm supports parsing :param: but the skeletons need the right imports + # and it fails for namespaced parameters (since the 'raw' parameter type + # foo.bar refers to a generated submodule called bar with the skeleton bar + # inside. + for p in PARAM_PATTERN.findall(signature_string): + type_decl = p[0] + if type_decl != class_name: + self.add_import_types(import_types, type_decl) + try: + defaults_match = [DEFAULTS.search(s) for s in seq] + idx = [DEFAULTS.sub('', s) for s in seq].index(p[1]) + seq[idx] = '{}: {}'.format(p[1], NAMESPACE_PATTERN.sub(r'\2', p[0])) + if defaults_match[idx]: + seq[idx] += defaults_match[idx].group(1) + except ValueError: + note("Warning: Unrecognised parameter {} in parameter list {}".format(p, seq)) + pass + + return (seq, ret_type, doc_node, list(import_types), ret_hint) + + def parse_func_doc(self, func_doc, func_id, func_name, class_name, deco=None, sip_generated=False): + """ + @param func_doc: __doc__ of the function. + @param func_id: name to look for as identifier of the function in docstring + @param func_name: name of the function. + @param class_name: name of the containing class, or None + @param deco: decorator to use + @return (reconstructed_spec, return_literal, note) or (None, _, _) if failed. + """ + if sip_generated: + overloads = [] + for part in func_doc.split('\n'): + signature = func_id + '(' + i = part.find(signature) + if i >= 0: + overloads.append(part[i + len(signature):]) + if len(overloads) > 1: + docstring_results = [self.restore_by_docstring(overload, class_name, deco) for overload in overloads] + import_types = [] + ret_types = [] + for result in docstring_results: + rt = result[1] + if rt and rt not in ret_types: + ret_types.append(rt) + imps = result[3] + for imp in imps: + if imp and imp not in import_types: + import_types.append(imp) + if ret_types: + ret_literal = " or ".join(ret_types) + else: + ret_literal = None + param_lists = [result[0] for result in docstring_results] + spec = build_signature(func_name, restore_parameters_for_overloads(param_lists)) + return (spec, ret_literal, "restored from __doc__ with multiple overloads", import_types) + + # find the first thing to look like a definition + prefix_re = re.compile(r"\s*(?:(\w+)[ \t]+)?" + func_id + r"\s*\(") # "foo(..." or "int foo(..." + match = prefix_re.search(func_doc) # Note: this and previous line may consume up to 35% of time + # parse the part that looks right + if match: + ret_hint = match.group(1) + params, ret, doc_note, import_types, ret_hint = self.restore_by_docstring(func_doc[match.end():], class_name, deco, ret_hint) + spec = func_name + flatten(params) + # if we got a type hint, put it on the function declaration + if ret_hint: + spec = spec + ' -> ' + ret_hint + return (spec, ret, doc_note, import_types) + else: + return (None, None, None, []) + + + def is_predefined_builtin(self, module_name, class_name, func_name): + return self.doing_builtins and module_name == BUILTIN_MOD_NAME and ( + class_name, func_name) in PREDEFINED_BUILTIN_SIGS + + def redo_function(self, out, p_func, p_name, indent, p_class=None, p_modname=None, classname=None, seen=None, used_imports=None): + """ + Restore function argument list as best we can. + @param out output function of a Buf + @param p_func function or method object + @param p_name function name as known to owner + @param indent indentation level + @param p_class the class that contains this function as a method + @param p_modname module name + @param seen {id(func): name} map of functions already seen in the same namespace; + id() because *some* functions are unhashable (eg _elementtree.Comment in py2.7) + """ + action("redoing func %r of class %r", p_name, p_class) + if seen is not None: + other_func = seen.get(id(p_func), None) + if other_func and getattr(other_func, "__doc__", None) is getattr(p_func, "__doc__", None): + # _bisect.bisect == _bisect.bisect_right in py31, but docs differ + out(indent, p_name, " = ", seen[id(p_func)]) + out(indent, "") + return + else: + seen[id(p_func)] = p_name + # real work + if classname is None: + classname = p_class and p_class.__name__ or None + if p_class and hasattr(p_class, '__mro__'): + sip_generated = [base_t for base_t in p_class.__mro__ if 'sip.simplewrapper' in str(base_t)] + else: + sip_generated = False + deco = None + deco_comment = "" + mod_class_method_tuple = (p_modname, classname, p_name) + ret_literal = None + is_init = False + # any decorators? + action("redoing decos of func %r of class %r", p_name, p_class) + if self.doing_builtins and p_modname == BUILTIN_MOD_NAME: + deco = KNOWN_DECORATORS.get((classname, p_name), None) + if deco: + deco_comment = " # known case" + elif p_class and p_name in p_class.__dict__: + # detect native methods declared with METH_CLASS flag + descriptor = p_class.__dict__[p_name] + if p_name != "__new__" and type(descriptor).__name__.startswith('classmethod'): + # 'classmethod_descriptor' in Python 2.x and 3.x, 'classmethod' in Jython + deco = "classmethod" + elif type(p_func).__name__.startswith('staticmethod'): + deco = "staticmethod" + elif str(descriptor).startswith(' __foo_bar + bases_list.append(mangled_qualifier + "." + base_name) + self.hidden_imports[qual_module_name] = mangled_qualifier + else: + bases_list.append(base_name) + base_def = "(" + ", ".join(bases_list) + ")" + + if self.split_modules: + for base in bases_list: + local_import = self.create_local_import(base) + if local_import: + out(indent, local_import) + out(indent, "class ", p_name, base_def, ":", + skipped_bases and " # skipped bases: " + ", ".join(skipped_bases) or "") + out_doc_attr(out, p_class, indent + 1) + # inner parts + methods = {} + properties = {} + others = {} + we_are_the_base_class = p_modname == BUILTIN_MOD_NAME and p_name == "object" + field_source = {} + try: + if hasattr(p_class, "__dict__") and not inspect_dir: + field_source = p_class.__dict__ + field_keys = field_source.keys() # Jython 2.5.1 _codecs fail here + else: + field_keys = dir(p_class) # this includes unwanted inherited methods, but no dict + inheritance is rare + except: + field_keys = () + for item_name in field_keys: + item_qname = p_modname + '.' + p_name + '.' + item_name + if item_name in ("__doc__", "__module__"): + if we_are_the_base_class: + item = "" # must be declared in base types + else: + continue # in all other cases must be skipped + elif keyword.iskeyword(item_name): # for example, PyQt4 contains definitions of methods named 'exec' + continue + elif item_qname in CLASS_ATTR_BLACKLIST: + note('skipping blacklisted attribute ' + item_qname) + item = field_source.get(item_name) + else: + try: + item = getattr(p_class, item_name) # let getters do the magic + except AttributeError: + item = field_source.get(item_name) # have it raw + if item is None: + continue + except Exception: + continue + + # Don't generate skeleton for internal enum properties + if isinstance(p_class, enum.EnumMeta) and (is_callable(item) or item_name[0] == '_'): + continue + + if is_callable(item) and not isinstance(item, type): + methods[item_name] = item + elif is_property(item): + properties[item_name] = item + else: + others[item_name] = item + # + if we_are_the_base_class: + others["__dict__"] = {} # force-feed it, for __dict__ does not contain a reference to itself :) + # add fake __init__s to have the right sig + if p_class in FAKE_BUILTIN_INITS: + methods["__init__"] = self.fake_builtin_init + note("Faking init of %s", p_name) + elif '__init__' not in methods: + init_method = getattr(p_class, '__init__', None) + if init_method: + methods['__init__'] = init_method + + # + seen_funcs = {} + for item_name in sorted_no_case(methods.keys()): + item = methods[item_name] + try: + self.redo_function(out, item, item_name, indent + 1, p_class, p_modname, classname=p_name, seen=seen_funcs, used_imports=used_imports) + except: + handle_error_func(item_name, out) + # + known_props = KNOWN_PROPS.get(p_modname, {}) + a_setter = "lambda self, v: None" + a_deleter = "lambda self: None" + for item_name in sorted_no_case(properties.keys()): + item = properties[item_name] + prop_docstring = getattr(item, '__doc__', None) + prop_key = (p_name, item_name) + if prop_key in known_props: + prop_descr = known_props.get(prop_key, None) + if prop_descr is None: + continue # explicitly omitted + acc_line, getter_and_type = prop_descr + if getter_and_type: + getter, prop_type = getter_and_type + else: + getter, prop_type = None, None + out(indent + 1, item_name, + " = property(", format_accessors(acc_line, getter, a_setter, a_deleter), ")" + ) + if prop_type: + if prop_docstring: + out(indent + 1, '"""', prop_docstring) + out(0, "") + out(indent + 1, ':type: ', prop_type) + out(indent + 1, '"""') + else: + out(indent + 1, '""":type: ', prop_type, '"""') + out(0, "") + else: + # for properties with docstrings put them inside the getter so that PyCharm + # displays them + if prop_docstring: + ret = '' + param = '' + + # Additionally if we see :type: in the docstring, add type hints + result = TYPE_PATTERN.search(prop_docstring) + if result is not None: + type_decl = result.group(1).strip() + import_types = set() + if type_decl == p_name or type_decl == 'List[{}]'.format(p_name): + type_decl = "'" + type_decl + "'" + else: + self.add_import_types(import_types, type_decl) + for imp in import_types: + self.process_import_type(used_imports, p_modname, p_name, imp) + type_decl = type_decl.replace('...', '__ellipses__').split('.')[-1].replace('__ellipses__', '...') + ret = ' -> {}'.format(type_decl) + param = ': {}'.format(type_decl) + + out(indent + 1, "@property") + out(indent + 1, "def {}(self){}:".format(item_name, ret)) + out(indent + 2, '"""', prop_docstring, '"""') + out(indent + 2, 'pass') + out(0, "") + out(indent + 1, "@{}.setter".format(item_name)) + out(indent + 1, "def {}(self, value{}):".format(item_name, param)) + out(indent + 2, 'pass') + out(0, "") + + self.used_typing = True + continue + + out(indent + 1, item_name, " = property(lambda self: object(), lambda self, v: None, lambda self: None) # default") + out(0, "") + if properties: + out(0, "") # empty line after the block + # + class_doc = getattr(p_class, "__doc__", None) + for item_name in sorted_no_case(others.keys()): + item = others[item_name] + self.fmt_value(out, item, indent + 1, prefix=item_name + " = ") + # search for enum values documentation in class docstring, except for placeholder first/count values + if isinstance(p_class, enum.EnumMeta) and item_name not in ['First', 'Count']: + offs = 0 + while True: + result = DATA_PATTERN.search(class_doc, offs) + # stop if there are no more datas - we should have one for each, but don't fail + if result is None: + print(f"Couldn't find enum docstring for {item_name}") + break + + # check if this is the data declaration we're looking for + data_decl = result.group(1).strip() + if data_decl == item_name: + # it is! see if there's another data after + enum_doc_start = result.end(1) + next_result = DATA_PATTERN.search(class_doc, enum_doc_start) + # if there isn't, the docstring is the remainder of the class doc, + # otherwise the docstring stops at the next data + if next_result is None: + enum_doc = class_doc[enum_doc_start:] + else: + enum_doc = class_doc[enum_doc_start:next_result.start(0)] + + out_docstring(out, enum_doc.strip(), indent + 1) + out(0, "") # empty line after docstring + break + offs = result.start(1) + if p_name == "object": + out(indent + 1, "__module__ = ''") + if others: + out(0, "") # empty line after the block + # + if not methods and not properties and not others: + out(indent + 1, "pass") + + def redo_simple_header(self, p_name): + """Puts boilerplate code on the top""" + out = self.header_buf.out # 1st class methods rule :) + out(0, "# encoding: %s" % OUT_ENCODING) # line 1 + # NOTE: maybe encoding should be selectable + if hasattr(self.module, "__name__"): + self_name = self.module.__name__ + if self_name != p_name: + mod_name = " calls itself " + self_name + else: + mod_name = "" + else: + mod_name = " does not know its name" + out(0, "# module ", p_name, mod_name) # line 2 + + origin_type = OriginType.FILE + import generator3.core + if generator3.core.is_pregeneration_mode(): + origin = origin_type = OriginType.PREGENERATED + elif self.mod_filename: + origin = self.mod_filename + elif p_name in sys.builtin_module_names: + origin = origin_type = OriginType.BUILTIN + else: + try: + origin = getattr(self.module, "__file__") + except AttributeError: + origin = origin_type = OriginType.BUILTIN + + if self.test_mode and origin_type == OriginType.FILE: + origin = get_portable_test_module_path(origin, self.qname) + + + out(0, "# from %s" % origin) # line 3 + out(0, "# by generator %s" % self.gen_version) # line 4 + if p_name == BUILTIN_MOD_NAME and version[0] == 2 and version[1] >= 6: + out(0, "from __future__ import print_function") + out_doc_attr(out, self.module, 0) + + def redo_imports(self): + module_type = type(sys) + for item_name in self.module.__dict__.keys(): + try: + item = self.module.__dict__[item_name] + except: + continue + if type(item) is module_type: # not isinstance, py2.7 + PyQt4.QtCore on windows have a bug here + self.imported_modules[item_name] = item + self.add_import_header_if_needed() + ref_notice = getattr(item, "__file__", str(item)) if not self.test_mode else '' + if hasattr(item, "__name__"): + self.imports_buf.out(0, "import ", item.__name__, " as ", item_name, " # ", ref_notice) + else: + self.imports_buf.out(0, item_name, " = None # ??? name unknown; ", ref_notice) + + def add_import_header_if_needed(self): + if self.imports_buf.isEmpty(): + self.imports_buf.out(0, "") + self.imports_buf.out(0, "# imports") + + def redo(self, p_name, inspect_dir): + """ + Restores module declarations. + Intended for built-in modules and thus does not handle import statements. + @param p_name name of module + """ + action("redoing header of module %r %r", p_name, str(self.module)) + + if "pyqt4" in p_name.lower(): # qt4 specific patch + self._initializeQApp4() + elif "pyqt5" in p_name.lower(): # qt5 specific patch + self._initializeQApp5() + + self.redo_simple_header(p_name) + + # find whatever other self.imported_modules the module knows; effectively these are imports + action("redoing imports of module %r %r", p_name, str(self.module)) + try: + self.redo_imports() + except: + pass + + action("redoing innards of module %r %r", p_name, str(self.module)) + + module_type = type(sys) + # group what we have into buckets + vars_simple = {} + vars_complex = {} + funcs = {} + classes = {} + module_dict = self.module.__dict__ + if inspect_dir: + module_dict = dir(self.module) + for item_name in module_dict: + note("looking at %s", item_name) + # Python/C API can declare a symbol with an arbitrary name + if not is_identifier(item_name): # noqa + continue + if item_name in ( + "__dict__", "__doc__", "__module__", "__file__", "__name__", "__builtins__", "__package__"): + continue # handled otherwise + if self.test_mode and item_name in ('__loader__', '__spec__', '__cached__'): + continue + try: + item = getattr(self.module, item_name) # let getters do the magic + except AttributeError: + if not item_name in self.module.__dict__: continue + item = self.module.__dict__[item_name] # have it raw + # check if it has percolated from an imported module + except NotImplementedError: + if not item_name in self.module.__dict__: continue + item = self.module.__dict__[item_name] # have it raw + + # unless we're adamantly positive that the name was imported, we assume it is defined here + mod_name = None # module from which p_name might have been imported + # IronPython has non-trivial reexports in System module, but not in others: + skip_modname = sys.platform == "cli" and p_name != "System" + surely_not_imported_mods = KNOWN_FAKE_REEXPORTERS.get(p_name, ()) + ## can't figure weirdness in some modules, assume no reexports: + #skip_modname = skip_modname or p_name in self.KNOWN_FAKE_REEXPORTERS + if not skip_modname: + try: + mod_name = getattr(item, '__module__', None) + except: + pass + # we assume that module foo.bar never imports foo; foo may import foo.bar. (see pygame and pygame.rect) + maybe_import_mod_name = mod_name if isinstance(mod_name, type(p_name)) else '' + import_is_from_top = len(p_name) > len(maybe_import_mod_name) and p_name.startswith(maybe_import_mod_name) + note("mod_name = %s, prospective = %s, from top = %s", mod_name, maybe_import_mod_name, import_is_from_top) + want_to_import = False + if (mod_name + and mod_name != BUILTIN_MOD_NAME + and mod_name != p_name + and mod_name not in surely_not_imported_mods + and not import_is_from_top + ): + # import looks valid, but maybe it's a .py file? we're certain not to import from .py + # e.g. this rules out _collections import collections and builtins import site. + try: + imported = __import__(mod_name) # ok to repeat, Python caches for us + if imported: + qualifiers = mod_name.split(".")[1:] + for qual in qualifiers: + imported = getattr(imported, qual, None) + if not imported: + break + imported_path = (getattr(imported, '__file__', False) or "").lower() + want_to_import = not (imported_path.endswith('.py') or imported_path.endswith('.pyc')) + imported_name = getattr(imported, "__name__", None) + if imported_name == p_name: + want_to_import = False + note("path of %r is %r, want? %s", mod_name, imported_path, want_to_import) + except ImportError: + want_to_import = False + # NOTE: if we fail to import, we define 'imported' names here lest we lose them at all + if want_to_import: + import_list = self.used_imports[mod_name] + if item_name not in import_list: + import_list.append(item_name) + if not want_to_import: + if isinstance(item, type) or type(item).__name__ == 'classobj': + classes[item_name] = item + elif is_callable(item): # some classes are callable, check them before functions + funcs[item_name] = item + elif isinstance(item, module_type): + continue # self.imported_modules handled above already + else: + if isinstance(item, SIMPLEST_TYPES): + vars_simple[item_name] = item + else: + vars_complex[item_name] = item + + # sort and output every bucket + action("outputting innards of module %r %r", p_name, str(self.module)) + # + omitted_names = OMIT_NAME_IN_MODULE.get(p_name, []) + if vars_simple: + out = self.functions_buf.out + prefix = "" # try to group variables by common prefix + PREFIX_LEN = 2 # default prefix length if we can't guess better + out(0, "# Variables with simple values") + for item_name in sorted_no_case(vars_simple.keys()): + if item_name in omitted_names: + out(0, "# definition of " + item_name + " omitted") + continue + item = vars_simple[item_name] + # track the prefix + if len(item_name) >= PREFIX_LEN: + prefix_pos = string.rfind(item_name, "_") # most prefixes end in an underscore + if prefix_pos < 1: + prefix_pos = PREFIX_LEN + beg = item_name[0:prefix_pos] + if prefix != beg: + out(0, "") # space out from other prefix + prefix = beg + else: + prefix = "" + # output + replacement = REPLACE_MODULE_VALUES.get((p_name, item_name), None) + if replacement is not None: + out(0, item_name, " = ", replacement, " # real value of type ", str(type(item)), " replaced") + elif is_skipped_in_module(p_name, item_name): + t_item = type(item) + out(0, item_name, " = ", self.invent_initializer(t_item), " # real value of type ", str(t_item), + " skipped") + else: + self.fmt_value(out, item, 0, prefix=item_name + " = ") + self._defined[item_name] = True + out(0, "") # empty line after vars + # + if funcs: + out = self.functions_buf.out + out(0, "# functions") + out(0, "") + seen_funcs = {} + func_used_imports = emptylistdict() + for item_name in sorted_no_case(funcs.keys()): + if item_name in omitted_names: + out(0, "# definition of ", item_name, " omitted") + continue + item = funcs[item_name] + try: + self.redo_function(out, item, item_name, 0, p_modname=p_name, seen=seen_funcs, used_imports=func_used_imports) + except: + handle_error_func(item_name, out) + # don't import anything from . for functions, functions are emitted in __init__ + # and all local classes are imported by the time they're defined + func_used_imports['.'] = [] + self.output_import_froms(self.func_imports_buf.out, func_used_imports) + else: + self.functions_buf.out(0, "# no functions") + # + if classes: + self.classes_buf.out(0, "# classes") + self.classes_buf.out(0, "") + seen_classes = {} + # sort classes so that inheritance order is preserved + cls_list = [] # items are (class_name, mro_tuple) + for cls_name in sorted_no_case(classes.keys()): + cls = classes[cls_name] + ins_index = len(cls_list) + for i in range(ins_index): + maybe_child_bases = cls_list[i][1] + if cls in maybe_child_bases: + ins_index = i # we could not go farther than current ins_index + break # ...and need not go fartehr than first known child + cls_list.insert(ins_index, (cls_name, get_mro(cls))) + self.split_modules = True # always output split modules + for item_name in [cls_item[0] for cls_item in cls_list]: + buf = ClassBuf(item_name, self) + imports = ClassBuf(item_name + '_imports', self) + self.classes_buffs.append((buf,imports)) + out = buf.out + if item_name in omitted_names: + out(0, "# definition of ", item_name, " omitted") + continue + item = classes[item_name] + used_imports = emptylistdict() + self.redo_class(out, item, item_name, 0, p_modname=p_name, seen=seen_classes, inspect_dir=inspect_dir, used_imports=used_imports) + # if we don't have split modules we can't import dependencies, but we also + # have an ordering constraint - classes need to be declared after any classes + # they reference in type hints. This is broken either way though even with the + # return literals + if not self.split_modules: + func_used_imports['.'] = [] + self.output_import_froms(imports.out, used_imports) + self._defined[item_name] = True + out(0, "") # empty line after each item + + if self.doing_builtins and p_name == BUILTIN_MOD_NAME and version[0] < 3: + # classobj still supported + txt = classobj_txt + self.classes_buf.out(0, txt) + + if self.doing_builtins and p_name == BUILTIN_MOD_NAME: + txt = create_generator() + self.classes_buf.out(0, txt) + txt = create_async_generator() + self.classes_buf.out(0, txt) + txt = create_function() + self.classes_buf.out(0, txt) + txt = create_method() + self.classes_buf.out(0, txt) + txt = create_coroutine() + self.classes_buf.out(0, txt) + + # Fake + if version[0] >= 3 or (version[0] == 2 and version[1] >= 6): + namedtuple_text = create_named_tuple() + self.classes_buf.out(0, namedtuple_text) + + else: + self.classes_buf.out(0, "# no classes") + # + if vars_complex: + out = self.footer_buf.out + out(0, "# variables with complex values") + out(0, "") + for item_name in sorted_no_case(vars_complex.keys()): + if item_name in omitted_names: + out(0, "# definition of " + item_name + " omitted") + continue + item = vars_complex[item_name] + if str(type(item)) == "": + continue # this is an IronPython submodule, we mustn't generate a reference for it in the base module + replacement = REPLACE_MODULE_VALUES.get((p_name, item_name), None) + if replacement is not None: + out(0, item_name + " = " + replacement + " # real value of type " + str(type(item)) + " replaced") + elif is_skipped_in_module(p_name, item_name): + t_item = type(item) + out(0, item_name + " = " + self.invent_initializer(t_item) + " # real value of type " + str( + t_item) + " skipped") + else: + self.fmt_value(out, item, 0, prefix=item_name + " = ", as_name=item_name) + self._defined[item_name] = True + out(0, "") # empty line after each item + values_to_add = ADD_VALUE_IN_MODULE.get(p_name, None) + if values_to_add: + self.footer_buf.out(0, "# intermittent names") + for value in values_to_add: + self.footer_buf.out(0, value) + # imports: last, because previous parts could alter used_imports or hidden_imports + + out = self.imports_buf.out + self.output_import_froms(out, self.used_imports) + + if self.hidden_imports: + self.add_import_header_if_needed() + for mod_name in sorted_no_case(self.hidden_imports.keys()): + out(0, 'import ', mod_name, ' as ', self.hidden_imports[mod_name]) + out(0, "") # empty line after group + + if self.used_typing: + self.add_import_header_if_needed() + out(0, 'from typing import List, Tuple, Callable, Any') + out(0, "") # empty line after group + + if self.imports_buf.isEmpty(): + out(0, "# no imports") + out(0, "") # empty line after imports + + def output_import_froms(self, out, imports_list): + """Mention all imported names known within the module, wrapping as per PEP.""" + if imports_list: + self.add_import_header_if_needed() + for mod_name in sorted_no_case(imports_list.keys()): + import_names = imports_list[mod_name] + if mod_name == '.': + # if this is a local import, we need to treat it specially to import + # the class inside the referenced module + for n in import_names: + out(0, "from .%s import %s" % (n, n)) # empty line after group + out(0, "") # empty line after group + elif import_names: + self._defined[mod_name] = True + right_pos = 0 # tracks width of list to fold it at right margin + import_heading = "from % s import (" % mod_name + right_pos += len(import_heading) + names_pack = [import_heading] + indent_level = 0 + import_names = list(import_names) + import_names.sort() + for n in import_names: + self._defined[n] = True + len_n = len(n) + if right_pos + len_n >= 78: + out(indent_level, *names_pack) + names_pack = [n, ", "] + if indent_level == 0: + indent_level = 1 # all but first line is indented + right_pos = self.indent_size + len_n + 2 + else: + names_pack.append(n) + names_pack.append(", ") + right_pos += (len_n + 2) + # last line is... + if indent_level == 0: # one line + names_pack[0] = names_pack[0][:-1] # cut off lpar + names_pack[-1] = "" # cut last comma + else: # last line of multiline + names_pack[-1] = ")" # last comma -> rpar + out(indent_level, *names_pack) + + out(0, "") # empty line after group + + diff --git a/docs/stubs_generation/helpers/generator3/required_gen_version b/docs/stubs_generation/helpers/generator3/required_gen_version new file mode 100644 index 000000000..5fdf44547 --- /dev/null +++ b/docs/stubs_generation/helpers/generator3/required_gen_version @@ -0,0 +1,50 @@ +# This file lists minimum generator versions required for known packages / files. +# hash marks start line comments. +# name is either a package name (as used in import) or a predefined name in parentheses. +# version is two decimal numbers divided by a dot. +# settings equally apply to all platforms (jython, cpython, ipy). + +(default) 1.127 # anything not explicitly marked + +(built-in) 1.145 # skeletons of all built-in modules are built together +# Note: modules like itertools, etc are "(built-in)" and are ignored if given separately + +_fileio 1.127 +_io 1.127 +sys 1.127 +thread 1.127 +_thread 1.127 +_struct 1.127 +datetime 1.127 +_collections 1.127 + +PyQt4.Qsci 1.127 +PyQt4.QtAssistant 1.127 +PyQt4.QtCore 1.127 +PyQt4.QtDesigner 1.127 +PyQt4.QtGui 1.127 +PyQt4.QtHelp 1.127 +PyQt4.QtNetwork 1.127 +PyQt4.QtScriptTools 1.127 +PyQt4.QtScript 1.127 +PyQt4.QtSvg 1.127 +PyQt4.QtTest 1.127 +PyQt4.Qt 1.127 +PyQt4.QtWebKit 1.127 +PyQt4.QtXmlPatterns 1.127 +PyQt4.QtXml 1.127 + +pygame.fastevent 1.127 +pygame.image 1.127 + +sip 1.127 + +pysqlite2._sqlite 1.127 +_bsddb 1.127 + +h5py.h5 1.127 +h5py.h5i 1.127 +h5py.h5g 1.127 + +numpy.random.mtrand 1.140 +numpy.core.multiarray 1.143 \ No newline at end of file diff --git a/docs/stubs_generation/helpers/generator3/util_methods.py b/docs/stubs_generation/helpers/generator3/util_methods.py new file mode 100644 index 000000000..5afbbccbe --- /dev/null +++ b/docs/stubs_generation/helpers/generator3/util_methods.py @@ -0,0 +1,938 @@ +import ast +import collections +import errno +import functools +import hashlib +import json +import keyword +import logging +import multiprocessing +import shutil +from contextlib import contextmanager + +from generator3.constants import * + +try: + import inspect +except ImportError: + inspect = None + +BIN_READ_BLOCK = 64 * 1024 + + +def create_named_tuple(): #TODO: user-skeleton + return """ +class __namedtuple(tuple): + '''A mock base class for named tuples.''' + + __slots__ = () + _fields = () + + def __new__(cls, *args, **kwargs): + 'Create a new instance of the named tuple.' + return tuple.__new__(cls, *args) + + @classmethod + def _make(cls, iterable, new=tuple.__new__, len=len): + 'Make a new named tuple object from a sequence or iterable.' + return new(cls, iterable) + + def __repr__(self): + return '' + + def _asdict(self): + 'Return a new dict which maps field types to their values.' + return {} + + def _replace(self, **kwargs): + 'Return a new named tuple object replacing specified fields with new values.' + return self + + def __getnewargs__(self): + return tuple(self) +""" + +def create_generator(): + # Fake + if version[0] < 3: + next_name = "next" + else: + next_name = "__next__" + txt = """ +class __generator(object): + '''A mock class representing the generator function type.''' + def __init__(self): + self.gi_code = None + self.gi_frame = None + self.gi_running = 0 + + def __iter__(self): + '''Defined to support iteration over container.''' + pass + + def %s(self): + '''Return the next item from the container.''' + pass +""" % (next_name,) + if version[0] >= 3 or (version[0] == 2 and version[1] >= 5): + txt += """ + def close(self): + '''Raises new GeneratorExit exception inside the generator to terminate the iteration.''' + pass + + def send(self, value): + '''Resumes the generator and "sends" a value that becomes the result of the current yield-expression.''' + pass + + def throw(self, type, value=None, traceback=None): + '''Used to raise an exception inside the generator.''' + pass +""" + return txt + +def create_async_generator(): + # Fake + txt = """ +class __asyncgenerator(object): + '''A mock class representing the async generator function type.''' + def __init__(self): + '''Create an async generator object.''' + self.__name__ = '' + self.__qualname__ = '' + self.ag_await = None + self.ag_frame = None + self.ag_running = False + self.ag_code = None + + def __aiter__(self): + '''Defined to support iteration over container.''' + pass + + def __anext__(self): + '''Returns an awaitable, that performs one asynchronous generator iteration when awaited.''' + pass + + def aclose(self): + '''Returns an awaitable, that throws a GeneratorExit exception into generator.''' + pass + + def asend(self, value): + '''Returns an awaitable, that pushes the value object in generator.''' + pass + + def athrow(self, type, value=None, traceback=None): + '''Returns an awaitable, that throws an exception into generator.''' + pass +""" + return txt + +def create_function(): + txt = """ +class __function(object): + '''A mock class representing function type.''' + + def __init__(self): + self.__name__ = '' + self.__doc__ = '' + self.__dict__ = '' + self.__module__ = '' +""" + if version[0] == 2: + txt += """ + self.func_defaults = {} + self.func_globals = {} + self.func_closure = None + self.func_code = None + self.func_name = '' + self.func_doc = '' + self.func_dict = '' +""" + if version[0] >= 3 or (version[0] == 2 and version[1] >= 6): + txt += """ + self.__defaults__ = {} + self.__globals__ = {} + self.__closure__ = None + self.__code__ = None + self.__name__ = '' +""" + if version[0] >= 3: + txt += """ + self.__annotations__ = {} + self.__kwdefaults__ = {} +""" + if version[0] >= 3 and version[1] >= 3: + txt += """ + self.__qualname__ = '' +""" + return txt + +def create_method(): + txt = """ +class __method(object): + '''A mock class representing method type.''' + + def __init__(self): +""" + if version[0] == 2: + txt += """ + self.im_class = None + self.im_self = None + self.im_func = None +""" + if version[0] >= 3 or (version[0] == 2 and version[1] >= 6): + txt += """ + self.__func__ = None + self.__self__ = None +""" + return txt + + +def create_coroutine(): + if version[0] == 3 and version[1] >= 5: + return """ +class __coroutine(object): + '''A mock class representing coroutine type.''' + + def __init__(self): + self.__name__ = '' + self.__qualname__ = '' + self.cr_await = None + self.cr_frame = None + self.cr_running = False + self.cr_code = None + + def __await__(self): + return [] + + def close(self): + pass + + def send(self, value): + pass + + def throw(self, type, value=None, traceback=None): + pass +""" + return "" + + +def _searchbases(cls, accum): + # logic copied from inspect.py + if cls not in accum: + accum.append(cls) + for x in cls.__bases__: + _searchbases(x, accum) + + +def get_mro(a_class): + # logic copied from inspect.py + """Returns a tuple of MRO classes.""" + if hasattr(a_class, "__mro__"): + return a_class.__mro__ + elif hasattr(a_class, "__bases__"): + bases = [] + _searchbases(a_class, bases) + return tuple(bases) + else: + return tuple() + + +def get_bases(a_class): # TODO: test for classes that don't fit this scheme + """Returns a sequence of class's bases.""" + if hasattr(a_class, "__bases__"): + return a_class.__bases__ + else: + return () + + +def is_callable(x): + return hasattr(x, '__call__') + + +def sorted_no_case(p_array): + """Sort an array case insensitively, returns a sorted copy""" + p_array = list(p_array) + p_array = sorted(p_array, key=lambda x: x.upper()) + return p_array + + +def cleanup(value): + result = [] + prev = i = 0 + length = len(value) + last_ascii = chr(127) + while i < length: + char = value[i] + replacement = None + if char == '\n': + replacement = '\\n' + elif char == '\r': + replacement = '\\r' + elif char < ' ' or char > last_ascii: + replacement = '?' # NOTE: such chars are rare; long swaths could be precessed differently + if replacement: + result.append(value[prev:i]) + result.append(replacement) + prev = i + 1 + i += 1 + result.append(value[prev:]) + return "".join(result) + + +def is_valid_expr(s): + try: + compile(s, '', 'eval', ast.PyCF_ONLY_AST) + except SyntaxError: + return False + return True + + +_prop_types = [type(property())] +#noinspection PyBroadException +try: + _prop_types.append(types.GetSetDescriptorType) +except: + pass + +#noinspection PyBroadException +try: + _prop_types.append(types.MemberDescriptorType) +except: + pass + +_prop_types = tuple(_prop_types) + + +def is_property(x): + return isinstance(x, _prop_types) + + +def reliable_repr(value): + # some subclasses of built-in types (see PyGtk) may provide invalid __repr__ implementations, + # so we need to sanitize the output + if type(bool) == type and isinstance(value, bool): + return repr(bool(value)) + for num_type in NUM_TYPES: + if isinstance(value, num_type): + return repr(num_type(value)) + return repr(value) + + +def sanitize_value(p_value): + """Returns p_value or its part if it represents a sane simple value, else returns 'None'""" + if isinstance(p_value, STR_TYPES): + match = SIMPLE_VALUE_RE.match(p_value) + if match: + return match.groups()[match.lastindex - 1] + else: + return 'None' + elif isinstance(p_value, NUM_TYPES): + return reliable_repr(p_value) + elif p_value is None: + return 'None' + else: + if hasattr(p_value, "__name__") and hasattr(p_value, "__module__") and p_value.__module__ == BUILTIN_MOD_NAME: + return p_value.__name__ # float -> "float" + else: + return repr(repr(p_value)) # function -> "", etc + + +def report(msg, *data): + """Say something at error level (stderr)""" + sys.stderr.write(msg % data) + sys.stderr.write("\n") + + +def say(msg, *data): + """Say something at info level (stdout)""" + sys.stdout.write(msg % data) + sys.stdout.write("\n") + sys.stdout.flush() + + +def flatten(seq): + """Transforms tree lists like ['a', ['b', 'c'], 'd'] to strings like '(a, (b, c), d)', enclosing each tree level in parens.""" + ret = [] + for one in seq: + if type(one) is list: + ret.append(flatten(one)) + else: + ret.append(one) + return "(" + ", ".join(ret) + ")" + + +def make_names_unique(seq, name_map=None): + """ + Returns a copy of tree list seq where all clashing names are modified by numeric suffixes: + ['a', 'b', 'a', 'b'] becomes ['a', 'b', 'a_1', 'b_1']. + Each repeating name has its own counter in the name_map. + """ + ret = [] + if not name_map: + name_map = {} + for one in seq: + if type(one) is list: + ret.append(make_names_unique(one, name_map)) + else: + if keyword.iskeyword(one): + one += "_" + one_key = lstrip(one, "*") # starred parameters are unique sans stars + if one_key in name_map: + old_one = one_key + one = one + "_" + str(name_map[old_one]) + name_map[old_one] += 1 + else: + name_map[one_key] = 1 + ret.append(one) + return ret + + +def out_docstring(out_func, docstring, indent): + if not isinstance(docstring, str): return + lines = docstring.strip().split("\n") + if lines: + if len(lines) == 1: + out_func(indent, '""" ' + lines[0] + ' """') + else: + out_func(indent, '"""') + for line in lines: + try: + out_func(indent, line) + except UnicodeEncodeError: + continue + out_func(indent, '"""') + +def out_doc_attr(out_func, p_object, indent, p_class=None): + the_doc = getattr(p_object, "__doc__", None) + if the_doc: + if p_class and the_doc == object.__init__.__doc__ and p_object is not object.__init__ and p_class.__doc__: + the_doc = str(p_class.__doc__) # replace stock init's doc with class's; make it a certain string. + the_doc += "\n# (copied from class doc)" + out_docstring(out_func, the_doc, indent) + else: + out_func(indent, "# no doc") + +def is_skipped_in_module(p_module, p_value): + """ + Returns True if p_value's value must be skipped for module p_module. + """ + skip_list = SKIP_VALUE_IN_MODULE.get(p_module, []) + if p_value in skip_list: + return True + skip_list = SKIP_VALUE_IN_MODULE.get("*", []) + if p_value in skip_list: + return True + return False + +def restore_predefined_builtin(class_name, func_name): + spec = func_name + PREDEFINED_BUILTIN_SIGS[(class_name, func_name)] + note = "known special case of " + (class_name and class_name + "." or "") + func_name + return (spec, note) + +def restore_by_inspect(p_func): + """ + Returns paramlist restored by inspect. + """ + args, varg, kwarg, defaults, kwonlyargs, kwonlydefaults, _ = getfullargspec(p_func) + spec = [] + if defaults: + dcnt = len(defaults) - 1 + else: + dcnt = -1 + args = args or [] + args.reverse() # backwards, for easier defaults handling + for arg in args: + if dcnt >= 0: + arg += "=" + sanitize_value(defaults[dcnt]) + dcnt -= 1 + spec.insert(0, arg) + if varg: + spec.append("*" + varg) + elif kwonlyargs: + spec.append("*") + + kwonlydefaults = kwonlydefaults or {} + for arg in kwonlyargs: + if arg in kwonlydefaults: + spec.append(arg + '=' + sanitize_value(kwonlydefaults[arg])) + else: + spec.append(arg) + + if kwarg: + spec.append("**" + kwarg) + return flatten(spec) + +def restore_parameters_for_overloads(parameter_lists): + param_index = 0 + star_args = False + optional = False + params = [] + while True: + parameter_lists_copy = [pl for pl in parameter_lists] + for pl in parameter_lists_copy: + if param_index >= len(pl): + parameter_lists.remove(pl) + optional = True + if not parameter_lists: + break + name = parameter_lists[0][param_index] + for pl in parameter_lists[1:]: + if pl[param_index] != name: + star_args = True + break + if star_args: break + if optional and not '=' in name: + params.append(name + '=None') + else: + params.append(name) + param_index += 1 + if star_args: + params.append("*__args") + return params + +def build_signature(p_name, params): + return p_name + '(' + ', '.join(params) + ')' + + +def propose_first_param(deco): + """@return: name of missing first paramater, considering a decorator""" + if deco is None: + return "self" + if deco == "classmethod": + return "cls" + # if deco == "staticmethod": + return None + +def qualifier_of(cls, qualifiers_to_skip): + m = getattr(cls, "__module__", None) + if m in qualifiers_to_skip: + return "" + return m + +def handle_error_func(item_name, out): + exctype, value = sys.exc_info()[:2] + msg = "Error generating skeleton for function %s: %s" + args = item_name, value + report(msg, *args) + out(0, "# " + msg % args) + out(0, "") + +def format_accessors(accessor_line, getter, setter, deleter): + """Nicely format accessors, like 'getter, fdel=deleter'""" + ret = [] + consecutive = True + for key, arg, par in (('r', 'fget', getter), ('w', 'fset', setter), ('d', 'fdel', deleter)): + if key in accessor_line: + if consecutive: + ret.append(par) + else: + ret.append(arg + "=" + par) + else: + consecutive = False + return ", ".join(ret) + + +def has_regular_python_ext(file_name): + """Does name end with .py?""" + return file_name.endswith(".py") + # Note that the standard library on MacOS X 10.6 is shipped only as .pyc files, so we need to + # have them processed by the generator in order to have any code insight for the standard library. + + +def detect_constructor(p_class): + # try to inspect the thing + constr = getattr(p_class, "__init__") + if constr and inspect and inspect.isfunction(constr): + args, _, _, _, kwonlyargs, _, _ = getfullargspec(constr) + return ", ".join(args + [a + '=' + a for a in kwonlyargs]) + else: + return None + +############## notes, actions ################################################################# +_is_verbose = False # controlled by -v + +CURRENT_ACTION = "nothing yet" + +def action(msg, *data): + global CURRENT_ACTION + CURRENT_ACTION = msg % data + note(msg, *data) + + +def set_verbose(verbose): + global _is_verbose + _is_verbose = verbose + + +def note(msg, *data): + """Say something at debug info level (stderr)""" + if _is_verbose: + sys.stderr.write(msg % data) + sys.stderr.write("\n") + + +############## plaform-specific methods ####################################################### +import sys +if sys.platform == 'cli': + #noinspection PyUnresolvedReferences + import clr + +# http://blogs.msdn.com/curth/archive/2009/03/29/an-ironpython-profiler.aspx +def print_profile(): + data = [] + data.extend(clr.GetProfilerData()) + data.sort(lambda x, y: -cmp(x.ExclusiveTime, y.ExclusiveTime)) + + for pd in data: + say('%s\t%d\t%d\t%d', pd.Name, pd.InclusiveTime, pd.ExclusiveTime, pd.Calls) + +def is_clr_type(clr_type): + if not clr_type: return False + try: + clr.GetClrType(clr_type) + return True + except TypeError: + return False + +def restore_clr(p_name, p_class): + """ + Restore the function signature by the CLR type signature + :return (is_static, spec, sig_note) + """ + clr_type = clr.GetClrType(p_class) + if p_name == '__new__': + methods = [c for c in clr_type.GetConstructors()] + if not methods: + return False, p_name + '(self, *args)', 'cannot find CLR constructor' # "self" is always first argument of any non-static method + else: + methods = [m for m in clr_type.GetMethods() if m.Name == p_name] + if not methods: + bases = p_class.__bases__ + if len(bases) == 1 and p_name in dir(bases[0]): + # skip inherited methods + return False, None, None + return False, p_name + '(self, *args)', 'cannot find CLR method' + # "self" is always first argument of any non-static method + + parameter_lists = [] + for m in methods: + parameter_lists.append([p.Name for p in m.GetParameters()]) + params = restore_parameters_for_overloads(parameter_lists) + is_static = False + if not methods[0].IsStatic: + params = ['self'] + params + else: + is_static = True + return is_static, build_signature(p_name, params), None + + +def build_pkg_structure(base_dir, qname): + if not qname: + return base_dir + + subdirname = base_dir + for part in qname.split("."): + subdirname = os.path.join(subdirname, part) + if not os.path.isdir(subdirname): + action("creating subdir %r", subdirname) + os.makedirs(subdirname) + init_py = os.path.join(subdirname, "__init__.py") + if os.path.isfile(subdirname + ".py"): + os.rename(subdirname + ".py", init_py) + elif not os.path.isfile(init_py): + fopen(init_py, "w").close() + + return subdirname + + +def is_valid_implicit_namespace_package_name(s): + """ + Checks whether provided string could represent implicit namespace package name. + :param s: string to check + :return: True if provided string could represent implicit namespace package name and False otherwise + """ + return isidentifier(s) and not keyword.iskeyword(s) + + +def isidentifier(s): + """ + Checks whether provided string complies Python identifier syntax requirements. + :param s: string to check + :return: True if provided string comply Python identifier syntax requirements and False otherwise + """ + if version[0] >= 3: + return s.isidentifier() + else: + # quick test on provided string to comply major Python identifier syntax requirements + return (s and + not s[:1].isdigit() and + "-" not in s and + " " not in s) + + +@contextmanager +def ignored_os_errors(*errno): + try: + yield + # Since Python 3.3 IOError and OSError were merged into OSError + except EnvironmentError as e: + if e.errno not in errno: + raise + + +def mkdir(path): + try: + os.makedirs(path) + except EnvironmentError as e: + if e.errno != errno.EEXIST or not os.path.isdir(path): + raise + + +def copy(src, dst, merge=False, pre_copy_hook=None, conflict_handler=None, post_copy_hook=None): + if pre_copy_hook is None: + def pre_copy_hook(p1, p2): + return True + + if conflict_handler is None: + def conflict_handler(p1, p2): + return False + + if post_copy_hook is None: + def post_copy_hook(p1, p2): + pass + + if not pre_copy_hook(src, dst): + return + + # Note about shutil.copy vs shutil.copy2. + # There is an open CPython bug which breaks copy2 on NFS when it tries to copy the xattr. + # https://bugs.python.org/issue24564 + # https://youtrack.jetbrains.com/issue/PY-37523 + # However, in all our use cases, we do not care about the xattr, + # so just always use shutil.copy to avoid this problem. + if os.path.isdir(src): + if not merge: + if version[0] >= 3: + shutil.copytree(src, dst, copy_function=shutil.copy) + else: + shutil.copytree(src, dst) + else: + mkdir(dst) + for child in os.listdir(src): + child_src = os.path.join(src, child) + child_dst = os.path.join(dst, child) + try: + copy(child_src, child_dst, merge=merge, + pre_copy_hook=pre_copy_hook, + conflict_handler=conflict_handler, + post_copy_hook=post_copy_hook) + except OSError as e: + if e.errno == errno.EEXIST and not (os.path.isdir(child_src) and os.path.isdir(child_dst)): + if conflict_handler(child_src, child_dst): + continue + raise + else: + mkdir(os.path.dirname(dst)) + shutil.copy(src, dst) + post_copy_hook(src, dst) + + +def copy_skeletons(src_dir, dst_dir, new_origin=None): + def overwrite(src, dst): + delete(dst) + copy(src, dst) + return True + + # Remove packages/modules with the same import name + def mod_pkg_cleanup(src, dst): + dst_dir = os.path.dirname(dst) + name, ext = os.path.splitext(os.path.basename(src)) + if ext == '.py': + delete(os.path.join(dst_dir, name)) + elif not ext: + delete(dst + '.py') + + def override_origin_stamp(src, dst): + _, ext = os.path.splitext(dst) + if ext == '.py' and new_origin: + with fopen(dst, 'r') as f: + lines = f.readlines() + for i, line in enumerate(lines): + if not line.startswith('#'): + return + + m = SKELETON_HEADER_ORIGIN_LINE.match(line) + if m: + break + else: + return + with fopen(dst, 'w') as f: + lines[i] = '# from ' + new_origin + '\n' + f.writelines(lines) + + def post_copy_hook(src, dst): + override_origin_stamp(src, dst) + mod_pkg_cleanup(src, dst) + + def ignore_failed_version_stamps(src, dst): + return not os.path.basename(src).startswith(FAILED_VERSION_STAMP_PREFIX) + + copy(src_dir, dst_dir, merge=True, + pre_copy_hook=ignore_failed_version_stamps, + conflict_handler=overwrite, + post_copy_hook=post_copy_hook) + + +def delete(path, content=False): + with ignored_os_errors(errno.ENOENT): + if os.path.isdir(path): + if not content: + shutil.rmtree(path) + else: + for child in os.listdir(path): + delete(child) + else: + os.remove(path) + + +def cached(func): + func._results = {} + unknown = object() + + # noinspection PyProtectedMember + @functools.wraps(func) + def wrapper(*args): + result = func._results.get(args, unknown) + if result is unknown: + result = func._results[args] = func(*args) + return result + + return wrapper + + +def sha256_digest(binary_or_file): + # "bytes" type is available in Python 2.7 + if isinstance(binary_or_file, bytes): + return hashlib.sha256(binary_or_file).hexdigest() + else: + acc = hashlib.sha256() + while True: + block = binary_or_file.read(BIN_READ_BLOCK) + if not block: + break + acc.update(block) + return acc.hexdigest() + + +def get_portable_test_module_path(abs_path, qname): + abs_path_components = os.path.normpath(abs_path).split(os.path.sep) + qname_components_count = len(qname.split('.')) + if os.path.splitext(abs_path_components[-1])[0] == '__init__': + rel_path_components_count = qname_components_count + 1 + else: + rel_path_components_count = qname_components_count + return '/'.join(abs_path_components[-rel_path_components_count:]) + + +def is_text_file(path): + """ + Verify that some path is a text file (not a binary file). + Ideally there should be usage of libmagic but it can be not + installed on a target machine. + + Actually this algorithm is inspired by function `file_encoding` + from libmagic. + """ + try: + with open(path, 'rb') as candidate_stream: + # Buffer size like in libmagic + buffer = candidate_stream.read(256 * 1024) + except EnvironmentError: + return False + + # Verify that it looks like ASCII, UTF-8 or UTF-16. + for encoding in 'utf-8', 'utf-16', 'utf-16-be', 'utf-16-le': + try: + buffer.decode(encoding) + except UnicodeDecodeError as err: + if err.args[0].endswith(('truncated data', 'unexpected end of data')): + return True + else: + return True + + # Verify that it looks like ISO-8859 or non-ISO extended ASCII. + return all(c not in _bytes_that_never_appears_in_text for c in buffer) + + +_bytes_that_never_appears_in_text = set(range(7)) | {11} | set(range(14, 27)) | set(range(28, 32)) | {127} + + +# This wrapper is intentionally made top-level: local functions can't be pickled. +def _multiprocessing_wrapper(data, func, *args, **kwargs): + configure_logging(data.root_logger_level) + data.result_conn.send(func(*args, **kwargs)) + + +_MainProcessData = collections.namedtuple('_MainProcessData', ['result_conn', 'root_logger_level']) + + +def execute_in_subprocess_synchronously(name, func, args, kwargs, failure_result=None): + import multiprocessing as mp + + extra_process_kwargs = {} + if sys.version_info[0] >= 3: + extra_process_kwargs['daemon'] = True + + # There is no need to use a full-blown queue for single producer/single consumer scenario. + # Also, Pipes don't suffer from issues such as https://bugs.python.org/issue35797. + # TODO experiment with a shared queue maintained by multiprocessing.Manager + # (it will require an additional service process) + recv_conn, send_conn = mp.Pipe(duplex=False) + data = _MainProcessData(result_conn=send_conn, + root_logger_level=logging.getLogger().level) + p = mp.Process(name=name, + target=_multiprocessing_wrapper, + args=(data, func) + args, + kwargs=kwargs, + **extra_process_kwargs) + p.start() + # This is actually against the multiprocessing guidelines + # https://docs.python.org/3/library/multiprocessing.html#programming-guidelines + # but allows us to fail-fast if the child process terminated abnormally with a segfault + # (otherwise we would have to wait by timeout on acquiring the result) and should work + # fine for small result values such as generation status. + p.join() + if recv_conn.poll(): + return recv_conn.recv() + else: + return failure_result + + +def configure_logging(root_level): + logging.addLevelName(logging.DEBUG - 1, 'TRACE') + + root = logging.getLogger() + root.setLevel(root_level) + + # In environments where fork is implemented entire logging configuration is already inherited by child processes. + # Configuring it twice will lead to duplicated records. + + # Reset logger similarly to how it's done in logging.config + for h in root.handlers[:]: + root.removeHandler(h) + + for f in root.filters[:]: + root.removeFilter(f) + + class JsonFormatter(logging.Formatter): + def format(self, record): + s = super(JsonFormatter, self).format(record) + return json.dumps({ + 'type': 'log', + 'level': record.levelname.lower(), + 'message': s + }) + + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(JsonFormatter()) + root.addHandler(handler) diff --git a/docs/stubs_generation/helpers/generator3/version.txt b/docs/stubs_generation/helpers/generator3/version.txt new file mode 100644 index 000000000..630a34a63 --- /dev/null +++ b/docs/stubs_generation/helpers/generator3/version.txt @@ -0,0 +1 @@ +1.147 \ No newline at end of file diff --git a/util/installer/LICENSE.rtf b/util/installer/LICENSE.rtf index b12419667..9e72d67da 100644 --- a/util/installer/LICENSE.rtf +++ b/util/installer/LICENSE.rtf @@ -44,5 +44,6 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI {\field{\*\fldinst{HYPERLINK "{\pntext\f1\'B7\tab}http://openwall.info/wiki/people/solar/software/public-domain-source-code/md5"}}{\fldrslt{\ul\cf1 http://openwall.info/wiki/people/solar/software/public-domain-source-code/md5}}}\f0\fs22\line md5 released to the Public Domain by Alexander Peslyak.\par {\field{\*\fldinst{HYPERLINK "{\pntext\f1\'B7\tab}https://developer.nvidia.com/nsight-perf-sdk"}}{\fldrslt{\ul\cf1 https://developer.nvidia.com/nsight-perf-sdk}}}\f0\fs22\line NVIDIA Nsight Perf SDK distributed under the NVIDIA Nsight Perf SDK license.\par {\field{\*\fldinst{HYPERLINK "{\pntext\f1\'B7\tab}https://github.com/python/pythoncapi-compat"}}{\fldrslt{\ul\cf1 https://github.com/python/pythoncapi-compat}}}\f0\fs22\line python2api-compat distributed under the BSD Zero Clause License. Copyright Contributors to the pythoncapi_compat project.\par +{\field{\*\fldinst{HYPERLINK "{\pntext\f1\'B7\tab}https://github.com/JetBrains/intellij-community"}}{\fldrslt{\ul\cf1 https://github.com/JetBrains/intellij-community}}}\f0\fs22\line intellij-community distributed under the Apache License. Copyright Contributors to the intellij-community project.\par }