mirror of
https://github.com/baldurk/renderdoc.git
synced 2026-08-09 00:01:07 +00:00
Commit modified version of intellij-community stubgen & wrapper script
This commit is contained in:
baldurk
@@ -140,6 +140,10 @@ The following libraries and components are incorporated into RenderDoc, listed h
|
||||
|
||||
Used to simplify compatibility with a broad range of Python versions.
|
||||
|
||||
* `intellij-community <https://github.com/JetBrains/intellij-community>`_ - Copyright Contributors to the intellij-community project. Distributed under the Apache License.
|
||||
|
||||
Used to generate Python stubs for binary modules.
|
||||
|
||||
Thanks
|
||||
------
|
||||
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import sys
|
||||
import struct
|
||||
|
||||
if len(sys.argv) != 2:
|
||||
print(f"Usage: {sys.argv[0]} [path/to/stubs/folder]")
|
||||
sys.exit(1)
|
||||
|
||||
# path to module libraries for windows
|
||||
if struct.calcsize("P") == 8:
|
||||
binpath = '../x64/'
|
||||
else:
|
||||
binpath = '../Win32/'
|
||||
|
||||
# Prioritise release over development builds
|
||||
sys.path.insert(0, os.path.abspath(binpath + 'Development/pymodules'))
|
||||
sys.path.insert(0, os.path.abspath(binpath + 'Release/pymodules'))
|
||||
|
||||
# Add the build paths to PATH so renderdoc.dll can be located
|
||||
os.environ["PATH"] += os.pathsep + os.path.abspath(binpath + 'Development/')
|
||||
os.environ["PATH"] += os.pathsep + os.path.abspath(binpath + 'Release/')
|
||||
|
||||
if sys.platform == 'win32' and sys.version_info[1] >= 8:
|
||||
os.add_dll_directory(binpath + 'Release/')
|
||||
os.add_dll_directory(binpath + 'Development/')
|
||||
|
||||
# path to module libraries for linux
|
||||
sys.path.insert(0, os.path.abspath('../build/lib'))
|
||||
|
||||
from stubs_generation.helpers import generator3
|
||||
|
||||
if __name__ == '__main__':
|
||||
generator3.main(['renderdoc', '-d', sys.argv[1]])
|
||||
generator3.main(['qrenderdoc', '-d', sys.argv[1]])
|
||||
@@ -0,0 +1,202 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
@@ -0,0 +1,7 @@
|
||||
## Python stubs generation
|
||||
|
||||
This folder contains a slightly modified version of [intellij-community](https://github.com/JetBrains/intellij-community/tree/master/python/helpers/generator3)'s stub generator for python binary modules, to parse RST documentation and generate typed helpers.
|
||||
|
||||
It has had some modifications to work better within RenderDoc's desired environment rather than an IDE, as well as producing more desirable output.
|
||||
|
||||
It remains licensed under the Apache license to the original contributors.
|
||||
@@ -0,0 +1,5 @@
|
||||
# make this usable as module
|
||||
from .__main__ import main
|
||||
from .__main__ import _bootstrap_sys_path
|
||||
|
||||
_bootstrap_sys_path()
|
||||
@@ -0,0 +1,216 @@
|
||||
import argparse
|
||||
import atexit
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
||||
_containing_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
_helpers_dir = os.path.dirname(_containing_dir)
|
||||
|
||||
|
||||
def _cleanup_sys_path():
|
||||
return [root for root in sys.path
|
||||
if os.path.normpath(root) not in (_containing_dir, _helpers_dir)]
|
||||
|
||||
|
||||
def _bootstrap_sys_path():
|
||||
sys.path.insert(0, _helpers_dir)
|
||||
|
||||
|
||||
def _setup_logging():
|
||||
from generator3.util_methods import configure_logging
|
||||
configure_logging(logging.DEBUG)
|
||||
|
||||
|
||||
def _enable_segfault_tracebacks():
|
||||
try:
|
||||
import faulthandler
|
||||
|
||||
faulthandler.enable()
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
def _configure_multiprocessing():
|
||||
required_start_method = os.environ.get('GENERATOR3_MULTIPROCESSING_START_METHOD')
|
||||
if required_start_method:
|
||||
import multiprocessing
|
||||
# Available only since Python 3.4
|
||||
multiprocessing.set_start_method(required_start_method)
|
||||
|
||||
|
||||
def parse_args(gen_version, args_list):
|
||||
parser = argparse.ArgumentParser(
|
||||
prog='generator3',
|
||||
description='Generates interface skeletons (binary stubs) for binary and '
|
||||
'built-in Python modules.'
|
||||
)
|
||||
parser.add_argument(
|
||||
'-d', metavar='PATH', dest='output_dir',
|
||||
help='Output dir, must be writable. If not given, current dir is used.'
|
||||
)
|
||||
# TODO using os.pathsep might cause problems with remote interpreters when host and
|
||||
# target OS don't match
|
||||
parser.add_argument(
|
||||
'-s', metavar='PATH_LIST', dest='roots',
|
||||
type=(lambda s: s.split(os.pathsep)), default=[],
|
||||
help='List of root directories to scan for binaries separated with `os.pathsep`'
|
||||
' character. These directories will be added in `sys.path`.'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--name-pattern', metavar='PATTERN',
|
||||
help='Shell-like glob pattern restricting generation only to modules with '
|
||||
'matching qualified names, e.g, "_ast" or "numpy.*".'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--builtins-only', action='store_true',
|
||||
help='Limit generation only to the modules in `sys.builtin_module_names`.'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--state-file', metavar='PATH',
|
||||
type=argparse.FileType('rb'),
|
||||
help='Path to the input ".state.json" file. If "-", the file is passed via '
|
||||
'stdin. The resulting ".state.json" will be generated automatically in '
|
||||
'the skeletons directory.'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--init-state-file', action='store_true',
|
||||
help='Generate a new ".state.json" file in the skeletons directory.'
|
||||
)
|
||||
|
||||
# Common flags
|
||||
# TODO evaluate these flags, some of them seem redundant now with proper logging
|
||||
parser.add_argument(
|
||||
'-q', dest='quiet', action='store_true',
|
||||
help='Be quiet, do not print anything on stdout. Errors still go to stderr.'
|
||||
)
|
||||
parser.add_argument(
|
||||
'-v', dest='verbose', action='store_true',
|
||||
help='Be verbose, print lots of debug output to stderr.'
|
||||
)
|
||||
|
||||
parser.add_argument('-V', action='version', version=gen_version)
|
||||
|
||||
extra_modes = parser.add_argument_group('extra modes')
|
||||
extra_modes.add_argument(
|
||||
'-S', dest='list_sources_mode', action='store_true',
|
||||
help='Lists all python sources found in `sys.path` and directories specified '
|
||||
'with -s.'
|
||||
)
|
||||
extra_modes.add_argument(
|
||||
'-z', dest='zip_sources_archive', metavar='ARCHIVE',
|
||||
help='Zip files to specified archive. Accepts files to be archived from stdin '
|
||||
'in format: <filepath> <name in archive>.'
|
||||
)
|
||||
extra_modes.add_argument(
|
||||
'-u', dest='zip_roots_archive', metavar='ARCHIVE',
|
||||
help='Zip all source files from `sys.path` and provided roots in the specified '
|
||||
'archive.'
|
||||
)
|
||||
|
||||
clr_specific = parser.add_argument_group('CLR specific options')
|
||||
clr_specific.add_argument(
|
||||
'-c', dest='clr_assemblies', metavar='MODULES',
|
||||
type=(lambda s: s.split(';')), default=[],
|
||||
help='Semicolon separated list of CLR assemblies to be imported.'
|
||||
)
|
||||
clr_specific.add_argument(
|
||||
'-p', dest='run_clr_profiler', action='store_true', help='Run CLR profiler.'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"mod_name", nargs='?', default=None,
|
||||
help='Qualified name of a single module to analyze.'
|
||||
)
|
||||
parser.add_argument(
|
||||
"mod_path", nargs='?', default=None,
|
||||
help='Path to the specified module if it\'s not builtin.'
|
||||
)
|
||||
return parser.parse_args(args_list)
|
||||
|
||||
|
||||
def main(args_list):
|
||||
import generator3.core
|
||||
import generator3.extra
|
||||
from generator3.clr_tools import get_namespace_by_name
|
||||
from generator3.constants import Timer
|
||||
from generator3.core import version, GenerationStatus, SkeletonGenerator
|
||||
from generator3.util_methods import set_verbose, say, note, print_profile
|
||||
|
||||
args = parse_args(version(), args_list)
|
||||
|
||||
generator3.core.quiet = args.quiet
|
||||
set_verbose(args.verbose)
|
||||
|
||||
if args.roots:
|
||||
for p in args.roots:
|
||||
if p and p not in sys.path:
|
||||
# we need this to make things in additional dirs importable
|
||||
sys.path.append(p)
|
||||
note("Altered sys.path: %r", sys.path)
|
||||
|
||||
if args.state_file:
|
||||
# We can't completely shut off stdin in case Docker-based interpreter to use
|
||||
# json.load() and have to retreat to reading the content line-wise
|
||||
if args.state_file.name == '<stdin>':
|
||||
state_json = json.loads(sys.stdin.readline()) # utf-8 by default
|
||||
else:
|
||||
with args.state_file as f:
|
||||
state_json = json.loads(f.read().decode(encoding='utf-8'))
|
||||
else:
|
||||
state_json = None
|
||||
|
||||
target_roots = _cleanup_sys_path()
|
||||
|
||||
if args.list_sources_mode:
|
||||
say(version())
|
||||
generator3.extra.list_sources(target_roots)
|
||||
sys.exit(0)
|
||||
|
||||
if args.zip_sources_archive:
|
||||
generator3.extra.zip_sources(args.zip_sources_archive)
|
||||
sys.exit(0)
|
||||
|
||||
if args.zip_roots_archive:
|
||||
generator3.extra.zip_stdlib(target_roots, args.zip_roots_archive)
|
||||
sys.exit(0)
|
||||
|
||||
generator = SkeletonGenerator(
|
||||
output_dir=args.output_dir or '.', # implement documented default to current directory
|
||||
roots=target_roots,
|
||||
state_json=state_json,
|
||||
write_state_json=bool(args.init_state_file or args.state_file)
|
||||
)
|
||||
|
||||
timer = Timer()
|
||||
if not args.mod_name:
|
||||
generator.discover_and_process_all_modules(name_pattern=args.name_pattern,
|
||||
builtins_only=args.builtins_only)
|
||||
sys.exit(0)
|
||||
|
||||
if sys.platform == 'cli':
|
||||
# noinspection PyUnresolvedReferences
|
||||
import clr
|
||||
|
||||
for ref in args.clr_assemblies:
|
||||
clr.AddReferenceByPartialName(ref)
|
||||
|
||||
if args.run_clr_profiler:
|
||||
atexit.register(print_profile)
|
||||
|
||||
# We take module name from import statement
|
||||
args.mod_name = get_namespace_by_name(args.mod_name)
|
||||
|
||||
if generator.process_module(args.mod_name, args.mod_path) == GenerationStatus.FAILED:
|
||||
sys.exit(1)
|
||||
|
||||
say("Generation completed in %d ms", timer.elapsed())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
_bootstrap_sys_path()
|
||||
_setup_logging()
|
||||
_enable_segfault_tracebacks()
|
||||
_configure_multiprocessing()
|
||||
main(sys.argv[1:])
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,63 @@
|
||||
# coding=utf-8
|
||||
"""
|
||||
.NET (CLR) specific functions
|
||||
"""
|
||||
__author__ = 'Ilya.Kazakevich'
|
||||
|
||||
|
||||
def get_namespace_by_name(object_name):
|
||||
"""
|
||||
Gets namespace for full object name. Sometimes last element of name is module while it may be class.
|
||||
For System.Console returns System, for System.Web returns System.Web.
|
||||
Be sure all required assemblies are loaded (i.e. clr.AddRef.. is called)
|
||||
:param object_name: name to parse
|
||||
:return: namespace
|
||||
"""
|
||||
(imported_object, object_name) = _import_first(object_name)
|
||||
parts = object_name.partition(".")
|
||||
first_part = parts[0]
|
||||
remain_part = parts[2]
|
||||
|
||||
while remain_part and type(_get_attr_by_name(imported_object, remain_part)) is type: # While we are in class
|
||||
remain_part = remain_part.rpartition(".")[0]
|
||||
|
||||
if remain_part:
|
||||
return first_part + "." + remain_part
|
||||
else:
|
||||
return first_part
|
||||
|
||||
|
||||
def _import_first(object_name):
|
||||
"""
|
||||
Some times we can not import module directly. For example, Some.Class.InnerClass could not be imported: you need to import "Some.Class"
|
||||
or even "Some" instead. This function tries to find part of name that could be loaded
|
||||
|
||||
:param object_name: name in dotted notation like "Some.Function.Here"
|
||||
:return: (imported_object, object_name): tuple with object and its name
|
||||
"""
|
||||
while object_name:
|
||||
try:
|
||||
return (__import__(object_name, globals=[], locals=[], fromlist=[]), object_name)
|
||||
except ImportError:
|
||||
object_name = object_name.rpartition(".")[0] # Remove rightest part
|
||||
raise Exception("No module name found in name " + object_name)
|
||||
|
||||
|
||||
def _get_attr_by_name(obj, name):
|
||||
"""
|
||||
Accepts chain of attributes in dot notation like "some.property.name" and gets them on object
|
||||
:param obj: object to introspec
|
||||
:param name: attribute name
|
||||
:return attribute
|
||||
|
||||
>>> str(_get_attr_by_name("A", "__class__.__class__"))
|
||||
"<type 'type'>"
|
||||
|
||||
>>> str(_get_attr_by_name("A", "__class__.__len__.__class__"))
|
||||
"<type 'method_descriptor'>"
|
||||
"""
|
||||
result = obj
|
||||
parts = name.split('.')
|
||||
for part in parts:
|
||||
result = getattr(result, part)
|
||||
return result
|
||||
@@ -0,0 +1,722 @@
|
||||
import os
|
||||
import re
|
||||
import string
|
||||
import sys
|
||||
import time
|
||||
import types
|
||||
|
||||
OUT_ENCODING = 'utf-8'
|
||||
|
||||
version = (
|
||||
(sys.hexversion & (0xff << 24)) >> 24,
|
||||
(sys.hexversion & (0xff << 16)) >> 16
|
||||
)
|
||||
|
||||
if version[0] >= 3:
|
||||
#noinspection PyUnresolvedReferences
|
||||
import builtins as the_builtins
|
||||
|
||||
string = "".__class__
|
||||
|
||||
STR_TYPES = (getattr(the_builtins, "bytes"), str)
|
||||
|
||||
NUM_TYPES = (int, float)
|
||||
SIMPLEST_TYPES = NUM_TYPES + STR_TYPES + (None.__class__,)
|
||||
EASY_TYPES = NUM_TYPES + STR_TYPES + (None.__class__, dict, tuple, list)
|
||||
|
||||
def the_exec(source, context):
|
||||
exec (source, context)
|
||||
|
||||
|
||||
# noinspection PyUnresolvedReferences
|
||||
from inspect import getfullargspec
|
||||
|
||||
else: # < 3.0
|
||||
import __builtin__ as the_builtins
|
||||
|
||||
STR_TYPES = (getattr(the_builtins, "unicode"), str)
|
||||
|
||||
NUM_TYPES = (int, long, float)
|
||||
SIMPLEST_TYPES = NUM_TYPES + STR_TYPES + (types.NoneType,)
|
||||
EASY_TYPES = NUM_TYPES + STR_TYPES + (types.NoneType, dict, tuple, list)
|
||||
|
||||
def the_exec(source, context):
|
||||
#noinspection PyRedundantParentheses
|
||||
exec (source) in context
|
||||
|
||||
def getfullargspec(func):
|
||||
import inspect
|
||||
return inspect.getargspec(func) + ([], None, {})
|
||||
|
||||
if version[0] == 2 and version[1] < 4:
|
||||
HAS_DECORATORS = False
|
||||
|
||||
def lstrip(s, prefix):
|
||||
i = 0
|
||||
while s[i] == prefix:
|
||||
i += 1
|
||||
return s[i:]
|
||||
|
||||
else:
|
||||
HAS_DECORATORS = True
|
||||
lstrip = string.lstrip
|
||||
|
||||
# return type inference helper table
|
||||
INT_LIT = '0'
|
||||
FLOAT_LIT = '0.0'
|
||||
DICT_LIT = '{}'
|
||||
LIST_LIT = '[]'
|
||||
TUPLE_LIT = '()'
|
||||
BOOL_LIT = 'False'
|
||||
RET_TYPE = {# {'type_name': 'value_string'} lookup table
|
||||
# chaining
|
||||
"self": "self",
|
||||
"self.": "self",
|
||||
# int
|
||||
"int": INT_LIT,
|
||||
"Int": INT_LIT,
|
||||
"integer": INT_LIT,
|
||||
"Integer": INT_LIT,
|
||||
"short": INT_LIT,
|
||||
"long": INT_LIT,
|
||||
"number": INT_LIT,
|
||||
"Number": INT_LIT,
|
||||
# float
|
||||
"float": FLOAT_LIT,
|
||||
"Float": FLOAT_LIT,
|
||||
"double": FLOAT_LIT,
|
||||
"Double": FLOAT_LIT,
|
||||
"floating": FLOAT_LIT,
|
||||
# boolean
|
||||
"bool": BOOL_LIT,
|
||||
"boolean": BOOL_LIT,
|
||||
"Bool": BOOL_LIT,
|
||||
"Boolean": BOOL_LIT,
|
||||
"True": BOOL_LIT,
|
||||
"true": BOOL_LIT,
|
||||
"False": BOOL_LIT,
|
||||
"false": BOOL_LIT,
|
||||
# list
|
||||
'list': LIST_LIT,
|
||||
'List': LIST_LIT,
|
||||
'[]': LIST_LIT,
|
||||
# tuple
|
||||
"tuple": TUPLE_LIT,
|
||||
"sequence": TUPLE_LIT,
|
||||
"Sequence": TUPLE_LIT,
|
||||
# dict
|
||||
"dict": DICT_LIT,
|
||||
"Dict": DICT_LIT,
|
||||
"dictionary": DICT_LIT,
|
||||
"Dictionary": DICT_LIT,
|
||||
"map": DICT_LIT,
|
||||
"Map": DICT_LIT,
|
||||
"hashtable": DICT_LIT,
|
||||
"Hashtable": DICT_LIT,
|
||||
"{}": DICT_LIT,
|
||||
# "objects"
|
||||
"object": "object()",
|
||||
}
|
||||
if version[0] < 3:
|
||||
UNICODE_LIT = 'u""'
|
||||
BYTES_LIT = '""'
|
||||
RET_TYPE.update({
|
||||
'string': BYTES_LIT,
|
||||
'String': BYTES_LIT,
|
||||
'str': BYTES_LIT,
|
||||
'Str': BYTES_LIT,
|
||||
'character': BYTES_LIT,
|
||||
'char': BYTES_LIT,
|
||||
'unicode': UNICODE_LIT,
|
||||
'Unicode': UNICODE_LIT,
|
||||
'bytes': BYTES_LIT,
|
||||
'byte': BYTES_LIT,
|
||||
'Bytes': BYTES_LIT,
|
||||
'Byte': BYTES_LIT,
|
||||
})
|
||||
DEFAULT_STR_LIT = BYTES_LIT
|
||||
# also, files:
|
||||
RET_TYPE.update({
|
||||
'file': "file('/dev/null')",
|
||||
})
|
||||
|
||||
def ensureUnicode(data):
|
||||
if type(data) == str:
|
||||
return data.decode(OUT_ENCODING, 'replace')
|
||||
return unicode(data)
|
||||
else:
|
||||
UNICODE_LIT = '""'
|
||||
BYTES_LIT = 'b""'
|
||||
RET_TYPE.update({
|
||||
'string': UNICODE_LIT,
|
||||
'String': UNICODE_LIT,
|
||||
'str': UNICODE_LIT,
|
||||
'Str': UNICODE_LIT,
|
||||
'character': UNICODE_LIT,
|
||||
'char': UNICODE_LIT,
|
||||
'unicode': UNICODE_LIT,
|
||||
'Unicode': UNICODE_LIT,
|
||||
'bytes': BYTES_LIT,
|
||||
'byte': BYTES_LIT,
|
||||
'Bytes': BYTES_LIT,
|
||||
'Byte': BYTES_LIT,
|
||||
})
|
||||
DEFAULT_STR_LIT = UNICODE_LIT
|
||||
# also, files: we can't provide an easy expression on py3k
|
||||
RET_TYPE.update({
|
||||
'file': None,
|
||||
})
|
||||
|
||||
def ensureUnicode(data):
|
||||
if type(data) == bytes:
|
||||
return data.decode(OUT_ENCODING, 'replace')
|
||||
return str(data)
|
||||
|
||||
if version[0] > 2:
|
||||
import io # in 3.0
|
||||
|
||||
|
||||
def fopen(name, mode):
|
||||
kwargs = {}
|
||||
if 'b' not in mode:
|
||||
kwargs['encoding'] = OUT_ENCODING
|
||||
return io.open(name, mode, **kwargs)
|
||||
else:
|
||||
fopen = open
|
||||
|
||||
if sys.platform == 'cli':
|
||||
#noinspection PyUnresolvedReferences
|
||||
from System import DateTime
|
||||
|
||||
class Timer(object):
|
||||
def __init__(self):
|
||||
self.started = DateTime.Now
|
||||
|
||||
def elapsed(self):
|
||||
return (DateTime.Now - self.started).TotalMilliseconds
|
||||
else:
|
||||
class Timer(object):
|
||||
def __init__(self):
|
||||
self.started = time.time()
|
||||
|
||||
def elapsed(self):
|
||||
return int((time.time() - self.started) * 1000)
|
||||
|
||||
IS_JAVA = hasattr(os, "java")
|
||||
|
||||
BUILTIN_MOD_NAME = the_builtins.__name__
|
||||
|
||||
IDENT_PATTERN = r"[A-Za-z_][0-9A-Za-z_]*" # re pattern for identifier
|
||||
STR_CHAR_PATTERN = r"[0-9A-Za-z_.,\+\-&\*% ]"
|
||||
|
||||
DOC_FUNC_RE = re.compile(r"(?:.*\.)?(\w+)\(([^\)]*)\).*") # $1 = function name, $2 = arglist
|
||||
|
||||
SANE_REPR_RE = re.compile(IDENT_PATTERN + r"(?:\(.*\))?") # identifier with possible (...), go catches
|
||||
|
||||
STARS_IDENT_RE = re.compile(r"(\*?\*?" + IDENT_PATTERN + ")") # $1 = identifier, maybe with a * or **
|
||||
|
||||
IDENT_EQ_RE = re.compile("(" + IDENT_PATTERN + r"\s*=)") # $1 = identifier with a following '='
|
||||
|
||||
SIMPLE_VALUE_RE = re.compile(
|
||||
r"(\([+-]?[0-9](?:\s*,\s*[+-]?[0-9])*\))|" + # a numeric tuple, e.g. in pygame
|
||||
r"([+-]?[0-9]+\.?[0-9]*(?:[Ee]?[+-]?[0-9]+\.?[0-9]*)?)|" + # number
|
||||
r"('" + STR_CHAR_PATTERN + "*')|" + # single-quoted string
|
||||
r'("' + STR_CHAR_PATTERN + '*")|' + # double-quoted string
|
||||
r"(\[\])|" +
|
||||
r"(\{\})|" +
|
||||
r"(\(\))|" +
|
||||
r"(True|False|None)"
|
||||
) # $? = sane default value
|
||||
|
||||
if version[0] < 3:
|
||||
_PYTHON2_IDENT_RE = re.compile(IDENT_PATTERN + "$")
|
||||
|
||||
is_identifier = _PYTHON2_IDENT_RE.match
|
||||
else:
|
||||
is_identifier = str.isidentifier
|
||||
|
||||
# Some values are known to be of no use in source and needs to be suppressed.
|
||||
# Dict is keyed by module names, with "*" meaning "any module";
|
||||
# values are lists of names of members whose value must be pruned.
|
||||
SKIP_VALUE_IN_MODULE = {
|
||||
"sys": (
|
||||
"modules", "path_importer_cache", "argv", "builtins",
|
||||
"last_traceback", "last_type", "last_value", "builtin_module_names",
|
||||
),
|
||||
"posix": (
|
||||
"environ",
|
||||
),
|
||||
"nt": (
|
||||
"environ",
|
||||
),
|
||||
"zipimport": (
|
||||
"_zip_directory_cache",
|
||||
),
|
||||
"*": (BUILTIN_MOD_NAME,)
|
||||
}
|
||||
# {"module": ("name",..)}: omit the names from the skeleton at all.
|
||||
OMIT_NAME_IN_MODULE = {}
|
||||
|
||||
if version[0] >= 3:
|
||||
v = OMIT_NAME_IN_MODULE.get(BUILTIN_MOD_NAME, []) + ["True", "False", "None", "__debug__"]
|
||||
OMIT_NAME_IN_MODULE[BUILTIN_MOD_NAME] = v
|
||||
|
||||
if IS_JAVA and version > (2, 4): # in 2.5.1 things are way weird!
|
||||
OMIT_NAME_IN_MODULE['_codecs'] = ['EncodingMap']
|
||||
OMIT_NAME_IN_MODULE['_hashlib'] = ['Hash']
|
||||
|
||||
ADD_VALUE_IN_MODULE = {
|
||||
"sys": ("exc_value = Exception()", "exc_traceback=None"), # only present after an exception in current thread
|
||||
}
|
||||
|
||||
# Some values are special and are better represented by hand-crafted constructs.
|
||||
# Dict is keyed by (module name, member name) and value is the replacement.
|
||||
REPLACE_MODULE_VALUES = {
|
||||
("numpy.core.multiarray", "typeinfo"): "{}",
|
||||
("psycopg2._psycopg", "string_types"): "{}", # badly mangled __eq__ breaks fmtValue
|
||||
("PyQt5.QtWidgets", "qApp") : "QApplication()", # instead of None
|
||||
}
|
||||
if version[0] <= 2:
|
||||
REPLACE_MODULE_VALUES[(BUILTIN_MOD_NAME, "None")] = "object()"
|
||||
for std_file in ("stdin", "stdout", "stderr"):
|
||||
REPLACE_MODULE_VALUES[("sys", std_file)] = "open('')" #
|
||||
|
||||
# Some functions and methods of some builtin classes have special signatures.
|
||||
# {("class", "method"): ("signature_string")}
|
||||
PREDEFINED_BUILTIN_SIGS = { #TODO: user-skeleton
|
||||
("type", "__init__"): "(cls, what, bases=None, dict=None)", # two sigs squeezed into one
|
||||
("object", "__init__"): "(self)",
|
||||
("object", "__new__"): "(cls, *more)", # only for the sake of parameter names readability
|
||||
("object", "__subclasshook__"): "(cls, subclass)", # trusting PY-1818 on sig
|
||||
("int", "__init__"): "(self, x, base=10)", # overrides a fake
|
||||
("list", "__init__"): "(self, seq=())",
|
||||
("tuple", "__init__"): "(self, seq=())", # overrides a fake
|
||||
("set", "__init__"): "(self, seq=())",
|
||||
("dict", "__init__"): "(self, seq=None, **kwargs)",
|
||||
("property", "__init__"): "(self, fget=None, fset=None, fdel=None, doc=None)",
|
||||
# TODO: infer, doc comments have it
|
||||
("dict", "update"): "(self, E=None, **F)", # docstring nearly lies
|
||||
(None, "zip"): "(seq1, seq2, *more_seqs)",
|
||||
(None, "range"): "(start=None, stop=None, step=None)", # suboptimal: allows empty arglist
|
||||
(None, "filter"): "(function_or_none, sequence)",
|
||||
(None, "iter"): "(source, sentinel=None)",
|
||||
(None, "getattr"): "(object, name, default=None)",
|
||||
('frozenset', "__init__"): "(self, seq=())",
|
||||
("bytearray", "__init__"): "(self, source=None, encoding=None, errors='strict')",
|
||||
}
|
||||
|
||||
if version[0] < 3:
|
||||
PREDEFINED_BUILTIN_SIGS[
|
||||
("unicode", "__init__")] = "(self, string=u'', encoding=None, errors='strict')" # overrides a fake
|
||||
PREDEFINED_BUILTIN_SIGS[("super", "__init__")] = "(self, type1, type2=None)"
|
||||
PREDEFINED_BUILTIN_SIGS[
|
||||
(None, "min")] = "(*args, **kwargs)" # too permissive, but py2.x won't allow a better sig
|
||||
PREDEFINED_BUILTIN_SIGS[(None, "max")] = "(*args, **kwargs)"
|
||||
PREDEFINED_BUILTIN_SIGS[("str", "__init__")] = "(self, string='')" # overrides a fake
|
||||
PREDEFINED_BUILTIN_SIGS[(None, "print")] = "(*args, **kwargs)" # can't do better in 2.x
|
||||
else:
|
||||
PREDEFINED_BUILTIN_SIGS[("super", "__init__")] = "(self, type1=None, type2=None)"
|
||||
PREDEFINED_BUILTIN_SIGS[(None, "min")] = "(*args, key=None)"
|
||||
PREDEFINED_BUILTIN_SIGS[(None, "max")] = "(*args, key=None)"
|
||||
PREDEFINED_BUILTIN_SIGS[
|
||||
(None, "open")] = "(file, mode='r', buffering=None, encoding=None, errors=None, newline=None, closefd=True)"
|
||||
PREDEFINED_BUILTIN_SIGS[
|
||||
("str", "__init__")] = "(self, value='', encoding=None, errors='strict')" # overrides a fake
|
||||
PREDEFINED_BUILTIN_SIGS[("str", "format")] = "(self, *args, **kwargs)"
|
||||
PREDEFINED_BUILTIN_SIGS[
|
||||
("bytes", "__init__")] = "(self, value=b'', encoding=None, errors='strict')" # overrides a fake
|
||||
PREDEFINED_BUILTIN_SIGS[("bytes", "format")] = "(self, *args, **kwargs)"
|
||||
PREDEFINED_BUILTIN_SIGS[(None, "print")] = "(self, *args, sep=' ', end='\\n', file=None)" # proper signature
|
||||
|
||||
if (2, 6) <= version < (3, 0):
|
||||
PREDEFINED_BUILTIN_SIGS[("unicode", "format")] = "(self, *args, **kwargs)"
|
||||
PREDEFINED_BUILTIN_SIGS[("str", "format")] = "(self, *args, **kwargs)"
|
||||
|
||||
if version == (2, 5):
|
||||
PREDEFINED_BUILTIN_SIGS[("unicode", "splitlines")] = "(keepends=None)" # a typo in docstring there
|
||||
|
||||
if version >= (2, 7):
|
||||
PREDEFINED_BUILTIN_SIGS[
|
||||
("enumerate", "__init__")] = "(self, iterable, start=0)" # dosctring omits this completely.
|
||||
|
||||
if version < (3, 3):
|
||||
datetime_mod = "datetime"
|
||||
else:
|
||||
datetime_mod = "_datetime"
|
||||
|
||||
|
||||
# NOTE: per-module signature data may be lazily imported
|
||||
# keyed by (module_name, class_name, method_name). PREDEFINED_BUILTIN_SIGS might be a layer of it.
|
||||
# value is ("signature", "return_literal")
|
||||
PREDEFINED_MOD_CLASS_SIGS = { #TODO: user-skeleton
|
||||
(BUILTIN_MOD_NAME, None, 'divmod'): ("(x, y)", "(0, 0)"),
|
||||
|
||||
("binascii", None, "hexlify"): ("(data)", BYTES_LIT),
|
||||
("binascii", None, "unhexlify"): ("(hexstr)", BYTES_LIT),
|
||||
|
||||
("time", None, "ctime"): ("(seconds=None)", DEFAULT_STR_LIT),
|
||||
|
||||
("_struct", None, "pack"): ("(fmt, *args)", BYTES_LIT),
|
||||
("_struct", None, "pack_into"): ("(fmt, buffer, offset, *args)", None),
|
||||
("_struct", None, "unpack"): ("(fmt, string)", None),
|
||||
("_struct", None, "unpack_from"): ("(fmt, buffer, offset=0)", None),
|
||||
("_struct", None, "calcsize"): ("(fmt)", INT_LIT),
|
||||
("_struct", "Struct", "__init__"): ("(self, fmt)", None),
|
||||
("_struct", "Struct", "pack"): ("(self, *args)", BYTES_LIT),
|
||||
("_struct", "Struct", "pack_into"): ("(self, buffer, offset, *args)", None),
|
||||
("_struct", "Struct", "unpack"): ("(self, string)", None),
|
||||
("_struct", "Struct", "unpack_from"): ("(self, buffer, offset=0)", None),
|
||||
|
||||
(datetime_mod, "date", "__new__"): ("(cls, year=None, month=None, day=None)", None),
|
||||
(datetime_mod, "date", "fromordinal"): ("(cls, ordinal)", "date(1,1,1)"),
|
||||
(datetime_mod, "date", "fromtimestamp"): ("(cls, timestamp)", "date(1,1,1)"),
|
||||
(datetime_mod, "date", "isocalendar"): ("(self)", "(1, 1, 1)"),
|
||||
(datetime_mod, "date", "isoformat"): ("(self)", DEFAULT_STR_LIT),
|
||||
(datetime_mod, "date", "isoweekday"): ("(self)", INT_LIT),
|
||||
(datetime_mod, "date", "replace"): ("(self, year=None, month=None, day=None)", "date(1,1,1)"),
|
||||
(datetime_mod, "date", "strftime"): ("(self, format)", DEFAULT_STR_LIT),
|
||||
(datetime_mod, "date", "timetuple"): ("(self)", "(0, 0, 0, 0, 0, 0, 0, 0, 0)"),
|
||||
(datetime_mod, "date", "today"): ("(self)", "date(1, 1, 1)"),
|
||||
(datetime_mod, "date", "toordinal"): ("(self)", INT_LIT),
|
||||
(datetime_mod, "date", "weekday"): ("(self)", INT_LIT),
|
||||
(datetime_mod, "timedelta", "__new__"
|
||||
): (
|
||||
"(cls, days=None, seconds=None, microseconds=None, milliseconds=None, minutes=None, hours=None, weeks=None)",
|
||||
None),
|
||||
(datetime_mod, "datetime", "__new__"
|
||||
): (
|
||||
"(cls, year=None, month=None, day=None, hour=None, minute=None, second=None, microsecond=None, tzinfo=None)",
|
||||
None),
|
||||
(datetime_mod, "datetime", "astimezone"): ("(self, tz)", "datetime(1, 1, 1)"),
|
||||
(datetime_mod, "datetime", "combine"): ("(cls, date, time)", "datetime(1, 1, 1)"),
|
||||
(datetime_mod, "datetime", "date"): ("(self)", "datetime(1, 1, 1)"),
|
||||
(datetime_mod, "datetime", "fromtimestamp"): ("(cls, timestamp, tz=None)", "datetime(1, 1, 1)"),
|
||||
(datetime_mod, "datetime", "isoformat"): ("(self, sep='T')", DEFAULT_STR_LIT),
|
||||
(datetime_mod, "datetime", "now"): ("(cls, tz=None)", "datetime(1, 1, 1)"),
|
||||
(datetime_mod, "datetime", "strptime"): ("(cls, date_string, format)", DEFAULT_STR_LIT),
|
||||
(datetime_mod, "datetime", "replace" ):
|
||||
(
|
||||
"(self, year=None, month=None, day=None, hour=None, minute=None, second=None, microsecond=None, tzinfo=None)",
|
||||
"datetime(1, 1, 1)"),
|
||||
(datetime_mod, "datetime", "time"): ("(self)", "time(0, 0)"),
|
||||
(datetime_mod, "datetime", "timetuple"): ("(self)", "(0, 0, 0, 0, 0, 0, 0, 0, 0)"),
|
||||
(datetime_mod, "datetime", "timetz"): ("(self)", "time(0, 0)"),
|
||||
(datetime_mod, "datetime", "utcfromtimestamp"): ("(self, timestamp)", "datetime(1, 1, 1)"),
|
||||
(datetime_mod, "datetime", "utcnow"): ("(cls)", "datetime(1, 1, 1)"),
|
||||
(datetime_mod, "datetime", "utctimetuple"): ("(self)", "(0, 0, 0, 0, 0, 0, 0, 0, 0)"),
|
||||
(datetime_mod, "time", "__new__"): (
|
||||
"(cls, hour=None, minute=None, second=None, microsecond=None, tzinfo=None)", None),
|
||||
(datetime_mod, "time", "isoformat"): ("(self)", DEFAULT_STR_LIT),
|
||||
(datetime_mod, "time", "replace"): (
|
||||
"(self, hour=None, minute=None, second=None, microsecond=None, tzinfo=None)", "time(0, 0)"),
|
||||
(datetime_mod, "time", "strftime"): ("(self, format)", DEFAULT_STR_LIT),
|
||||
(datetime_mod, "tzinfo", "dst"): ("(self, date_time)", INT_LIT),
|
||||
(datetime_mod, "tzinfo", "fromutc"): ("(self, date_time)", "datetime(1, 1, 1)"),
|
||||
(datetime_mod, "tzinfo", "tzname"): ("(self, date_time)", DEFAULT_STR_LIT),
|
||||
(datetime_mod, "tzinfo", "utcoffset"): ("(self, date_time)", INT_LIT),
|
||||
|
||||
("_io", None, "open"): ("(name, mode=None, buffering=None)", "file('/dev/null')"),
|
||||
("_io", "FileIO", "read"): ("(self, size=-1)", DEFAULT_STR_LIT),
|
||||
("_fileio", "_FileIO", "read"): ("(self, size=-1)", DEFAULT_STR_LIT),
|
||||
|
||||
("thread", None, "start_new"): ("(function, args, kwargs=None)", INT_LIT),
|
||||
("_thread", None, "start_new"): ("(function, args, kwargs=None)", INT_LIT),
|
||||
|
||||
("itertools", "groupby", "__init__"): ("(self, iterable, key=None)", None),
|
||||
("itertools", None, "groupby"): ("(iterable, key=None)", LIST_LIT),
|
||||
|
||||
("cStringIO", "OutputType", "seek"): ("(self, position, mode=0)", None),
|
||||
("cStringIO", "InputType", "seek"): ("(self, position, mode=0)", None),
|
||||
|
||||
# NOTE: here we stand on shaky ground providing sigs for 3rd-party modules, though well-known
|
||||
("numpy.core.multiarray", "ndarray", "__array__"): ("(self, dtype=None)", None),
|
||||
("numpy.core.multiarray", None, "arange"): ("(start=None, stop=None, step=None, dtype=None)", None),
|
||||
# same as range()
|
||||
("numpy.core.multiarray", None, "set_numeric_ops"): ("(**ops)", None),
|
||||
("numpy.random.mtrand", None, "rand"): ("(*dn)", None),
|
||||
("numpy.random.mtrand", None, "randn"): ("(*dn)", None),
|
||||
("numpy.core.multiarray", "ndarray", "reshape"): ("(self, shape, *shapes, order='C')", None),
|
||||
("numpy.core.multiarray", "ndarray", "resize"): ("(self, *new_shape, refcheck=True)", None),
|
||||
}
|
||||
|
||||
bin_collections_names = ['collections', '_collections']
|
||||
|
||||
for name in bin_collections_names:
|
||||
PREDEFINED_MOD_CLASS_SIGS[(name, "deque", "__init__")] = ("(self, iterable=(), maxlen=None)", None)
|
||||
PREDEFINED_MOD_CLASS_SIGS[(name, "defaultdict", "__init__")] = ("(self, default_factory=None, **kwargs)", None)
|
||||
|
||||
if version[0] < 3:
|
||||
PREDEFINED_MOD_CLASS_SIGS[("exceptions", "BaseException", "__unicode__")] = ("(self)", UNICODE_LIT)
|
||||
PREDEFINED_MOD_CLASS_SIGS[("itertools", "product", "__init__")] = ("(self, *iterables, **kwargs)", LIST_LIT)
|
||||
else:
|
||||
PREDEFINED_MOD_CLASS_SIGS[("itertools", "product", "__init__")] = ("(self, *iterables, repeat=1)", LIST_LIT)
|
||||
|
||||
if version[0] < 3:
|
||||
PREDEFINED_MOD_CLASS_SIGS[("PyQt4.QtCore", None, "pyqtSlot")] = (
|
||||
"(*types, **keywords)", None) # doc assumes py3k syntax
|
||||
|
||||
# known properties of modules
|
||||
# {{"module": {"class", "property" : ("letters", ("getter", "type"))}},
|
||||
# where letters is any set of r,w,d (read, write, del) and "getter" is a source of typed getter.
|
||||
# if value is None, the property should be omitted.
|
||||
# read-only properties that return an object are not listed.
|
||||
G_OBJECT = ("lambda self: object()", None)
|
||||
G_TYPE = ("lambda self: type(object)", "type")
|
||||
G_DICT = ("lambda self: {}", "dict")
|
||||
G_STR = ("lambda self: ''", "string")
|
||||
G_TUPLE = ("lambda self: tuple()", "tuple")
|
||||
G_FLOAT = ("lambda self: 0.0", "float")
|
||||
G_INT = ("lambda self: 0", "int")
|
||||
G_BOOL = ("lambda self: True", "bool")
|
||||
|
||||
KNOWN_PROPS = {
|
||||
BUILTIN_MOD_NAME: {
|
||||
("object", '__class__'): ('r', G_TYPE),
|
||||
('complex', 'real'): ('r', G_FLOAT),
|
||||
('complex', 'imag'): ('r', G_FLOAT),
|
||||
("file", 'softspace'): ('r', G_BOOL),
|
||||
("file", 'name'): ('r', G_STR),
|
||||
("file", 'encoding'): ('r', G_STR),
|
||||
("file", 'mode'): ('r', G_STR),
|
||||
("file", 'closed'): ('r', G_BOOL),
|
||||
("file", 'newlines'): ('r', G_STR),
|
||||
("slice", 'start'): ('r', G_INT),
|
||||
("slice", 'step'): ('r', G_INT),
|
||||
("slice", 'stop'): ('r', G_INT),
|
||||
("super", '__thisclass__'): ('r', G_TYPE),
|
||||
("super", '__self__'): ('r', G_TYPE),
|
||||
("super", '__self_class__'): ('r', G_TYPE),
|
||||
("type", '__basicsize__'): ('r', G_INT),
|
||||
("type", '__itemsize__'): ('r', G_INT),
|
||||
("type", '__base__'): ('r', G_TYPE),
|
||||
("type", '__flags__'): ('r', G_INT),
|
||||
("type", '__mro__'): ('r', G_TUPLE),
|
||||
("type", '__bases__'): ('r', G_TUPLE),
|
||||
("type", '__dictoffset__'): ('r', G_INT),
|
||||
("type", '__dict__'): ('r', G_DICT),
|
||||
("type", '__name__'): ('r', G_STR),
|
||||
("type", '__weakrefoffset__'): ('r', G_INT),
|
||||
},
|
||||
"exceptions": {
|
||||
("BaseException", '__dict__'): ('r', G_DICT),
|
||||
("BaseException", 'message'): ('rwd', G_STR),
|
||||
("BaseException", 'args'): ('r', G_TUPLE),
|
||||
("EnvironmentError", 'errno'): ('rwd', G_INT),
|
||||
("EnvironmentError", 'message'): ('rwd', G_STR),
|
||||
("EnvironmentError", 'strerror'): ('rwd', G_INT),
|
||||
("EnvironmentError", 'filename'): ('rwd', G_STR),
|
||||
("SyntaxError", 'text'): ('rwd', G_STR),
|
||||
("SyntaxError", 'print_file_and_line'): ('rwd', G_BOOL),
|
||||
("SyntaxError", 'filename'): ('rwd', G_STR),
|
||||
("SyntaxError", 'lineno'): ('rwd', G_INT),
|
||||
("SyntaxError", 'offset'): ('rwd', G_INT),
|
||||
("SyntaxError", 'msg'): ('rwd', G_STR),
|
||||
("SyntaxError", 'message'): ('rwd', G_STR),
|
||||
("SystemExit", 'message'): ('rwd', G_STR),
|
||||
("SystemExit", 'code'): ('rwd', G_OBJECT),
|
||||
("UnicodeDecodeError", '__basicsize__'): None,
|
||||
("UnicodeDecodeError", '__itemsize__'): None,
|
||||
("UnicodeDecodeError", '__base__'): None,
|
||||
("UnicodeDecodeError", '__flags__'): ('rwd', G_INT),
|
||||
("UnicodeDecodeError", '__mro__'): None,
|
||||
("UnicodeDecodeError", '__bases__'): None,
|
||||
("UnicodeDecodeError", '__dictoffset__'): None,
|
||||
("UnicodeDecodeError", '__dict__'): None,
|
||||
("UnicodeDecodeError", '__name__'): None,
|
||||
("UnicodeDecodeError", '__weakrefoffset__'): None,
|
||||
("UnicodeEncodeError", 'end'): ('rwd', G_INT),
|
||||
("UnicodeEncodeError", 'encoding'): ('rwd', G_STR),
|
||||
("UnicodeEncodeError", 'object'): ('rwd', G_OBJECT),
|
||||
("UnicodeEncodeError", 'start'): ('rwd', G_INT),
|
||||
("UnicodeEncodeError", 'reason'): ('rwd', G_STR),
|
||||
("UnicodeEncodeError", 'message'): ('rwd', G_STR),
|
||||
("UnicodeTranslateError", 'end'): ('rwd', G_INT),
|
||||
("UnicodeTranslateError", 'encoding'): ('rwd', G_STR),
|
||||
("UnicodeTranslateError", 'object'): ('rwd', G_OBJECT),
|
||||
("UnicodeTranslateError", 'start'): ('rwd', G_INT),
|
||||
("UnicodeTranslateError", 'reason'): ('rwd', G_STR),
|
||||
("UnicodeTranslateError", 'message'): ('rwd', G_STR),
|
||||
},
|
||||
'_ast': {
|
||||
("AST", '__dict__'): ('rd', G_DICT),
|
||||
},
|
||||
'posix': {
|
||||
("statvfs_result", 'f_flag'): ('r', G_INT),
|
||||
("statvfs_result", 'f_bavail'): ('r', G_INT),
|
||||
("statvfs_result", 'f_favail'): ('r', G_INT),
|
||||
("statvfs_result", 'f_files'): ('r', G_INT),
|
||||
("statvfs_result", 'f_frsize'): ('r', G_INT),
|
||||
("statvfs_result", 'f_blocks'): ('r', G_INT),
|
||||
("statvfs_result", 'f_ffree'): ('r', G_INT),
|
||||
("statvfs_result", 'f_bfree'): ('r', G_INT),
|
||||
("statvfs_result", 'f_namemax'): ('r', G_INT),
|
||||
("statvfs_result", 'f_bsize'): ('r', G_INT),
|
||||
|
||||
("stat_result", 'st_ctime'): ('r', G_INT),
|
||||
("stat_result", 'st_rdev'): ('r', G_INT),
|
||||
("stat_result", 'st_mtime'): ('r', G_INT),
|
||||
("stat_result", 'st_blocks'): ('r', G_INT),
|
||||
("stat_result", 'st_gid'): ('r', G_INT),
|
||||
("stat_result", 'st_nlink'): ('r', G_INT),
|
||||
("stat_result", 'st_ino'): ('r', G_INT),
|
||||
("stat_result", 'st_blksize'): ('r', G_INT),
|
||||
("stat_result", 'st_dev'): ('r', G_INT),
|
||||
("stat_result", 'st_size'): ('r', G_INT),
|
||||
("stat_result", 'st_mode'): ('r', G_INT),
|
||||
("stat_result", 'st_uid'): ('r', G_INT),
|
||||
("stat_result", 'st_atime'): ('r', G_INT),
|
||||
},
|
||||
"pwd": {
|
||||
("struct_pwent", 'pw_dir'): ('r', G_STR),
|
||||
("struct_pwent", 'pw_gid'): ('r', G_INT),
|
||||
("struct_pwent", 'pw_passwd'): ('r', G_STR),
|
||||
("struct_pwent", 'pw_gecos'): ('r', G_STR),
|
||||
("struct_pwent", 'pw_shell'): ('r', G_STR),
|
||||
("struct_pwent", 'pw_name'): ('r', G_STR),
|
||||
("struct_pwent", 'pw_uid'): ('r', G_INT),
|
||||
|
||||
("struct_passwd", 'pw_dir'): ('r', G_STR),
|
||||
("struct_passwd", 'pw_gid'): ('r', G_INT),
|
||||
("struct_passwd", 'pw_passwd'): ('r', G_STR),
|
||||
("struct_passwd", 'pw_gecos'): ('r', G_STR),
|
||||
("struct_passwd", 'pw_shell'): ('r', G_STR),
|
||||
("struct_passwd", 'pw_name'): ('r', G_STR),
|
||||
("struct_passwd", 'pw_uid'): ('r', G_INT),
|
||||
},
|
||||
"thread": {
|
||||
("_local", '__dict__'): None
|
||||
},
|
||||
"xxsubtype": {
|
||||
("spamdict", 'state'): ('r', G_INT),
|
||||
("spamlist", 'state'): ('r', G_INT),
|
||||
},
|
||||
"zipimport": {
|
||||
("zipimporter", 'prefix'): ('r', G_STR),
|
||||
("zipimporter", 'archive'): ('r', G_STR),
|
||||
("zipimporter", '_files'): ('r', G_DICT),
|
||||
},
|
||||
"_struct": {
|
||||
("Struct", "size"): ('r', G_INT),
|
||||
("Struct", "format"): ('r', G_STR),
|
||||
},
|
||||
datetime_mod: {
|
||||
("datetime", "hour"): ('r', G_INT),
|
||||
("datetime", "minute"): ('r', G_INT),
|
||||
("datetime", "second"): ('r', G_INT),
|
||||
("datetime", "microsecond"): ('r', G_INT),
|
||||
("date", "day"): ('r', G_INT),
|
||||
("date", "month"): ('r', G_INT),
|
||||
("date", "year"): ('r', G_INT),
|
||||
("time", "hour"): ('r', G_INT),
|
||||
("time", "minute"): ('r', G_INT),
|
||||
("time", "second"): ('r', G_INT),
|
||||
("time", "microsecond"): ('r', G_INT),
|
||||
("timedelta", "days"): ('r', G_INT),
|
||||
("timedelta", "seconds"): ('r', G_INT),
|
||||
("timedelta", "microseconds"): ('r', G_INT),
|
||||
},
|
||||
}
|
||||
|
||||
# Sometimes module X defines item foo but foo.__module__ == 'Y' instead of 'X';
|
||||
# module Y just re-exports foo, and foo fakes being defined in Y.
|
||||
# We list all such Ys keyed by X, all fully-qualified names:
|
||||
# {"real_definer_module": ("fake_reexporter_module",..)}
|
||||
KNOWN_FAKE_REEXPORTERS = {
|
||||
"_collections": ('collections',),
|
||||
"_functools": ('functools',),
|
||||
"_socket": ('socket',), # .error, etc
|
||||
"pyexpat": ('xml.parsers.expat',),
|
||||
"_bsddb": ('bsddb.db',),
|
||||
"pysqlite2._sqlite": ('pysqlite2.dbapi2',), # errors
|
||||
"numpy.core.multiarray": ('numpy', 'numpy.core'),
|
||||
"numpy.core._dotblas": ('numpy', 'numpy.core'),
|
||||
"numpy.core.umath": ('numpy', 'numpy.core'),
|
||||
"gtk._gtk": ('gtk', 'gtk.gdk',),
|
||||
"gobject._gobject": ('gobject',),
|
||||
"gnomecanvas": ("gnome.canvas",),
|
||||
}
|
||||
|
||||
KNOWN_FAKE_BASES = []
|
||||
# list of classes that pretend to be base classes but are mere wrappers, and their defining modules
|
||||
# [(class, module),...] -- real objects, not names
|
||||
#noinspection PyBroadException
|
||||
try:
|
||||
#noinspection PyUnresolvedReferences
|
||||
import sip as sip_module # Qt specifically likes it
|
||||
|
||||
if hasattr(sip_module, 'wrapper'):
|
||||
KNOWN_FAKE_BASES.append((sip_module.wrapper, sip_module))
|
||||
if hasattr(sip_module, 'simplewrapper'):
|
||||
KNOWN_FAKE_BASES.append((sip_module.simplewrapper, sip_module))
|
||||
del sip_module
|
||||
except:
|
||||
pass
|
||||
|
||||
# This is a list of builtin classes to use fake init
|
||||
FAKE_BUILTIN_INITS = (tuple, type, int, str)
|
||||
if version[0] < 3:
|
||||
FAKE_BUILTIN_INITS = FAKE_BUILTIN_INITS + (getattr(the_builtins, "unicode"),)
|
||||
else:
|
||||
FAKE_BUILTIN_INITS = FAKE_BUILTIN_INITS + (getattr(the_builtins, "str"), getattr(the_builtins, "bytes"))
|
||||
|
||||
# Some builtin methods are decorated, but this is hard to detect.
|
||||
# {("class_name", "method_name"): "decorator"}
|
||||
KNOWN_DECORATORS = {
|
||||
("dict", "fromkeys"): "staticmethod",
|
||||
("object", "__subclasshook__"): "classmethod",
|
||||
("bytearray", "fromhex"): "classmethod",
|
||||
("bytes", "fromhex"): "classmethod",
|
||||
("bytearray", "maketrans"): "staticmethod",
|
||||
("bytes", "maketrans"): "staticmethod",
|
||||
("int", "from_bytes"): "classmethod",
|
||||
("float", "fromhex"): "staticmethod",
|
||||
}
|
||||
|
||||
classobj_txt = ( #TODO: user-skeleton
|
||||
"class ___Classobj:" "\n"
|
||||
" '''A mock class representing the old style class base.'''" "\n"
|
||||
" __module__ = ''" "\n"
|
||||
" __class__ = None" "\n"
|
||||
"\n"
|
||||
" def __init__(self):" "\n"
|
||||
" pass" "\n"
|
||||
" __dict__ = {}" "\n"
|
||||
" __doc__ = ''" "\n"
|
||||
)
|
||||
|
||||
MAC_STDLIB_PATTERN = re.compile("/System/Library/Frameworks/Python\\.framework/Versions/(.+)/lib/python\\1/(.+)")
|
||||
MAC_SKIP_MODULES = ["test", "ctypes/test", "distutils/tests", "email/test",
|
||||
"importlib/test", "json/tests", "lib2to3/tests",
|
||||
"bsddb/test",
|
||||
"sqlite3/test", "tkinter/test", "idlelib", "antigravity"]
|
||||
|
||||
POSIX_SKIP_MODULES = ["vtemodule", "PAMmodule", "_snackmodule", "/quodlibet/_mmkeys"]
|
||||
|
||||
BIN_MODULE_FNAME_PAT = re.compile(r'([a-zA-Z_][0-9a-zA-Z_]*)\.(?:pyc|pyo|(?:(?:[a-zA-Z_0-9\-]+\.)?(?:so|pyd)))$')
|
||||
# possible binary module filename: letter, alphanum architecture per PEP-3149
|
||||
TYPELIB_MODULE_FNAME_PAT = re.compile("([a-zA-Z_]+[0-9a-zA-Z]*)[0-9a-zA-Z-.]*\\.typelib")
|
||||
|
||||
MODULES_INSPECT_DIR = ['gi.repository']
|
||||
TENSORFLOW_CONTRIB_OPS_MODULE_PATTERN = re.compile(r'tensorflow\.contrib\.(?:.+)\.(?:python\.ops\.|_dataset_ops$)')
|
||||
|
||||
CLASS_ATTR_BLACKLIST = [
|
||||
'google.protobuf.pyext._message.Message._extensions_by_name',
|
||||
'google.protobuf.pyext._message.Message._extensions_by_number',
|
||||
'panda3d.core.ExecutionEnvironment.environment_variables',
|
||||
]
|
||||
|
||||
SKELETON_HEADER_VERSION_LINE = re.compile(r'# by generator (?P<version>\d+\.\d+)')
|
||||
SKELETON_HEADER_ORIGIN_LINE = re.compile(r'# from (?P<path>.*)')
|
||||
REQUIRED_GEN_VERSION_LINE = re.compile(r'(?P<name>\S+)\s+(?P<version>\d+\.\d+)')
|
||||
# "mod_path" and "mod_mtime" markers are used in tests
|
||||
BLACKLIST_VERSION_LINE = re.compile(r'(?P<path>{mod_path}|[^=]+) = (?P<version>\d+\.\d+) (?P<mtime>{mod_mtime}|\d+)')
|
||||
|
||||
ENV_TEST_MODE_FLAG = 'GENERATOR3_TEST_MODE'
|
||||
ENV_PREGENERATION_MODE_FLAG = "IS_PREGENERATED_SKELETONS"
|
||||
ENV_VERSION = 'GENERATOR3_VERSION'
|
||||
ENV_REQUIRED_GEN_VERSION_FILE = 'GENERATOR3_REQUIRED_GEN_VERSION_FILE'
|
||||
|
||||
FAILED_VERSION_STAMP_PREFIX = '.failed__'
|
||||
|
||||
CACHE_DIR_NAME = 'cache'
|
||||
STATE_FILE_NAME = '.state.json'
|
||||
@@ -0,0 +1,651 @@
|
||||
# encoding: utf-8
|
||||
import collections
|
||||
import fnmatch
|
||||
import json
|
||||
import logging
|
||||
from copy import deepcopy
|
||||
|
||||
from generator3.util_methods import *
|
||||
|
||||
# We need such conditional import always disabled at runtime in order to use
|
||||
# "typing" without the need to actually bundle the module with PyCharm.
|
||||
# It's similar to what Mypy recommends with its "MYPY" flag for compatibility
|
||||
# with Python 3.5.1 (https://mypy.readthedocs.io/en/latest/common_issues.html#import-cycles).
|
||||
TYPE_CHECKING = False
|
||||
if TYPE_CHECKING:
|
||||
from typing import List, Dict, Any, NewType, Tuple, Optional, TextIO
|
||||
|
||||
SkeletonStatusId = NewType('SkeletonStatusId', str)
|
||||
GenerationStatusId = NewType('GenerationStatusId', str)
|
||||
GeneratorVersion = Tuple[int, int]
|
||||
|
||||
# TODO: Move all CLR-specific functions to clr_tools
|
||||
quiet = False
|
||||
_parent_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
|
||||
# TODO move to property of Generator3 as soon as tests finished
|
||||
@cached
|
||||
def version():
|
||||
env_version = os.environ.get(ENV_VERSION)
|
||||
if env_version:
|
||||
return env_version
|
||||
|
||||
with fopen(os.path.join(_parent_dir, 'version.txt'), 'r') as f:
|
||||
return f.read().strip()
|
||||
|
||||
|
||||
# TODO move to property of Generator3 as soon as tests finished
|
||||
@cached
|
||||
def required_gen_version_file_path():
|
||||
return os.environ.get(ENV_REQUIRED_GEN_VERSION_FILE, os.path.join(_parent_dir, 'required_gen_version'))
|
||||
|
||||
|
||||
@cached
|
||||
def is_test_mode():
|
||||
return ENV_TEST_MODE_FLAG in os.environ
|
||||
|
||||
|
||||
@cached
|
||||
def is_pregeneration_mode():
|
||||
return ENV_PREGENERATION_MODE_FLAG in os.environ
|
||||
|
||||
|
||||
# find_binaries functionality
|
||||
def cut_binary_lib_suffix(path, f):
|
||||
"""
|
||||
@param path where f lives
|
||||
@param f file name of a possible binary lib file (no path)
|
||||
@return f without a binary suffix (that is, an importable name) if path+f is indeed a binary lib, or None.
|
||||
Note: if for .pyc or .pyo file a .py is found, None is returned.
|
||||
"""
|
||||
if not f.endswith((".pyc", ".typelib", ".pyo", ".so", ".pyd")):
|
||||
return None
|
||||
ret = None
|
||||
match = BIN_MODULE_FNAME_PAT.match(f)
|
||||
if match:
|
||||
ret = match.group(1)
|
||||
modlen = len('module')
|
||||
retlen = len(ret)
|
||||
if ret.endswith('module') and retlen > modlen and f.endswith('.so'): # what for?
|
||||
ret = ret[:(retlen - modlen)]
|
||||
if f.endswith('.pyc') or f.endswith('.pyo'):
|
||||
fullname = os.path.join(path, f[:-1]) # check for __pycache__ is made outside
|
||||
if os.path.exists(fullname):
|
||||
ret = None
|
||||
pat_match = TYPELIB_MODULE_FNAME_PAT.match(f)
|
||||
if pat_match:
|
||||
ret = "gi.repository." + pat_match.group(1)
|
||||
return ret
|
||||
|
||||
|
||||
def is_posix_skipped_module(path, f):
|
||||
if os.name == 'posix':
|
||||
name = os.path.join(path, f)
|
||||
for mod in POSIX_SKIP_MODULES:
|
||||
if name.endswith(mod):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_mac_skipped_module(path, f):
|
||||
fullname = os.path.join(path, f)
|
||||
m = MAC_STDLIB_PATTERN.match(fullname)
|
||||
if not m: return 0
|
||||
relpath = m.group(2)
|
||||
for module in MAC_SKIP_MODULES:
|
||||
if relpath.startswith(module): return 1
|
||||
return 0
|
||||
|
||||
|
||||
def is_tensorflow_contrib_ops_module(qname):
|
||||
# These modules cannot be imported directly. Instead tensorflow uses special
|
||||
# tensorflow.contrib.util.loader.load_op_library() to load them and create
|
||||
# Python modules at runtime. Their names in sys.modules are then md5 sums
|
||||
# of the list of exported Python definitions.
|
||||
return TENSORFLOW_CONTRIB_OPS_MODULE_PATTERN.match(qname)
|
||||
|
||||
|
||||
def is_skipped_module(path, f, qname):
|
||||
return (is_mac_skipped_module(path, f) or
|
||||
is_posix_skipped_module(path, f[:f.rindex('.')]) or
|
||||
'pynestkernel' in f or
|
||||
is_tensorflow_contrib_ops_module(qname))
|
||||
|
||||
|
||||
def is_module(d, root):
|
||||
return (os.path.exists(os.path.join(root, d, "__init__.py")) or
|
||||
os.path.exists(os.path.join(root, d, "__init__.pyc")) or
|
||||
os.path.exists(os.path.join(root, d, "__init__.pyi")) or
|
||||
os.path.exists(os.path.join(root, d, "__init__.pyo")) or
|
||||
is_valid_implicit_namespace_package_name(d))
|
||||
|
||||
|
||||
def walk_python_path(path):
|
||||
for root, dirs, files in os.walk(path):
|
||||
if root.endswith('__pycache__'):
|
||||
continue
|
||||
dirs_copy = list(dirs)
|
||||
for d in dirs_copy:
|
||||
if d.endswith('__pycache__') or not is_module(d, root):
|
||||
dirs.remove(d)
|
||||
# some files show up but are actually non-existent symlinks
|
||||
yield root, [f for f in files if os.path.exists(os.path.join(root, f))]
|
||||
|
||||
|
||||
def file_modification_timestamp(path):
|
||||
return int(os.stat(path).st_mtime)
|
||||
|
||||
|
||||
def build_cache_dir_path(subdir, mod_qname, mod_path):
|
||||
return os.path.join(subdir, module_hash(mod_qname, mod_path))
|
||||
|
||||
|
||||
def module_hash(mod_qname, mod_path):
|
||||
# Hash the content of a physical module
|
||||
if mod_path:
|
||||
hash_ = physical_module_hash(mod_path)
|
||||
else:
|
||||
hash_ = builtin_module_hash()
|
||||
# Use shorter hashes in test data as it might affect developers on Windows
|
||||
if is_test_mode():
|
||||
return hash_[:10]
|
||||
return hash_
|
||||
|
||||
|
||||
def builtin_module_hash():
|
||||
return sha256_digest(sys.version.encode(encoding='utf-8'))
|
||||
|
||||
|
||||
def physical_module_hash(mod_path):
|
||||
with fopen(mod_path, 'rb') as f:
|
||||
return sha256_digest(f)
|
||||
|
||||
|
||||
def version_to_tuple(version):
|
||||
# type: (str) -> GeneratorVersion
|
||||
# noinspection PyTypeChecker
|
||||
return tuple(map(int, version.split('.')))
|
||||
|
||||
|
||||
class OriginType(object):
|
||||
FILE = 'FILE'
|
||||
BUILTIN = '(built-in)'
|
||||
PREGENERATED = '(pre-generated)'
|
||||
|
||||
|
||||
class SkeletonStatus(object):
|
||||
UP_TO_DATE = 'UP_TO_DATE' # type: SkeletonStatusId
|
||||
"""
|
||||
Skeleton is up-to-date and doesn't need to be regenerated.
|
||||
"""
|
||||
FAILING = 'FAILING' # type: SkeletonStatusId
|
||||
"""
|
||||
Skeleton generation is known to fail for this module.
|
||||
"""
|
||||
OUTDATED = 'OUTDATED' # type: SkeletonStatusId
|
||||
"""
|
||||
Skeleton needs to be regenerated.
|
||||
"""
|
||||
|
||||
|
||||
def skeleton_status(base_dir, mod_qname, mod_path, sdk_skeleton_state=None):
|
||||
# Force regeneration every time
|
||||
return SkeletonStatus.OUTDATED
|
||||
|
||||
|
||||
def read_used_generator_version_from_skeleton_header(base_dir, mod_qname):
|
||||
# type: (str, str) -> Optional[GeneratorVersion]
|
||||
for path in skeleton_path_candidates(base_dir, mod_qname, init_for_pkg=True):
|
||||
with ignored_os_errors(errno.ENOENT):
|
||||
with fopen(path, 'r') as f:
|
||||
return read_generator_version_from_header(f)
|
||||
return None
|
||||
|
||||
|
||||
def read_generator_version_from_header(skeleton_file):
|
||||
# type: (TextIO) -> Optional[GeneratorVersion]
|
||||
for line in skeleton_file:
|
||||
if not line.startswith('#'):
|
||||
break
|
||||
|
||||
m = SKELETON_HEADER_VERSION_LINE.match(line)
|
||||
if m:
|
||||
return version_to_tuple(m.group('version'))
|
||||
return None
|
||||
|
||||
|
||||
def skeleton_path_candidates(base_dir, mod_qname, init_for_pkg=False):
|
||||
base_path = os.path.join(base_dir, *mod_qname.split('.'))
|
||||
if init_for_pkg:
|
||||
yield os.path.join(base_path, '__init__.py')
|
||||
else:
|
||||
yield base_path
|
||||
yield base_path + '.py'
|
||||
|
||||
|
||||
def read_failed_version_from_stamp(base_dir, mod_qname):
|
||||
# type: (str, str) -> Optional[GeneratorVersion]
|
||||
with ignored_os_errors(errno.ENOENT):
|
||||
with fopen(os.path.join(base_dir, FAILED_VERSION_STAMP_PREFIX + mod_qname), 'r') as f:
|
||||
return version_to_tuple(f.read().strip())
|
||||
# noinspection PyUnreachableCode
|
||||
return None
|
||||
|
||||
|
||||
def read_failed_version_and_mtime_from_legacy_blacklist(sdk_skeletons_dir, mod_path):
|
||||
# type: (str, str) -> Optional[Tuple[GeneratorVersion, int]]
|
||||
blacklist = read_legacy_blacklist_file(sdk_skeletons_dir, mod_path)
|
||||
return blacklist.get(mod_path)
|
||||
|
||||
|
||||
def read_legacy_blacklist_file(sdk_skeletons_dir, mod_path):
|
||||
# type: (str, str) -> Dict[str, Tuple[GeneratorVersion, int]]
|
||||
results = {}
|
||||
with ignored_os_errors(errno.ENOENT):
|
||||
with fopen(os.path.join(sdk_skeletons_dir, '.blacklist'), 'r') as f:
|
||||
for line in f:
|
||||
if not line or line.startswith('#'):
|
||||
continue
|
||||
|
||||
m = BLACKLIST_VERSION_LINE.match(line)
|
||||
if m:
|
||||
bin_path = m.group('path')
|
||||
bin_mtime = m.group('mtime')
|
||||
if is_test_mode() and bin_path == '{mod_path}':
|
||||
bin_path = mod_path
|
||||
if is_test_mode() and bin_mtime == '{mod_mtime}':
|
||||
bin_mtime = file_modification_timestamp(mod_path)
|
||||
else:
|
||||
# On Java side modification time stored in milliseconds.
|
||||
# Python API uses seconds for resolution in os.stat results.
|
||||
bin_mtime = int(m.group('mtime')) / 1000
|
||||
results[bin_path] = (version_to_tuple(m.group('version')), bin_mtime)
|
||||
return results
|
||||
|
||||
|
||||
def read_required_version(mod_qname):
|
||||
# type: (str) -> Optional[GeneratorVersion]
|
||||
mod_id = '(built-in)' if mod_qname in sys.builtin_module_names else mod_qname
|
||||
versions = read_required_gen_version_file()
|
||||
# TODO use glob patterns here
|
||||
return versions.get(mod_id, versions.get('(default)'))
|
||||
|
||||
|
||||
def read_required_gen_version_file():
|
||||
# type: () -> Dict[str, GeneratorVersion]
|
||||
result = {}
|
||||
with fopen(required_gen_version_file_path(), 'r') as f:
|
||||
for line in f:
|
||||
if not line or line.startswith('#'):
|
||||
continue
|
||||
m = REQUIRED_GEN_VERSION_LINE.match(line)
|
||||
if m:
|
||||
result[m.group('name')] = version_to_tuple(m.group('version'))
|
||||
|
||||
return result
|
||||
|
||||
|
||||
class GenerationStatus(object):
|
||||
FAILED = 'FAILED' # type: GenerationStatusId
|
||||
"""
|
||||
Either generation of a skeleton was attempted and failed or cache markers and/or .blacklist indicate that
|
||||
it was impossible to generate it for the current version of the generator last time.
|
||||
"""
|
||||
|
||||
GENERATED = 'GENERATED' # type: GenerationStatusId
|
||||
"""
|
||||
Skeleton was successfully generated anew and copied both to the cache and a per-sdk skeletons directory.
|
||||
"""
|
||||
|
||||
COPIED = 'COPIED' # type: GenerationStatusId
|
||||
"""
|
||||
Skeleton was successfully copied from the cache to a per-sdk skeletons directory.
|
||||
"""
|
||||
|
||||
UP_TO_DATE = 'UP_TO_DATE' # type: GenerationStatusId
|
||||
"""
|
||||
Existing skeleton is up to date and, therefore, wasn't touched.
|
||||
"""
|
||||
|
||||
|
||||
def get_module_origin(mod_path, mod_qname):
|
||||
if mod_qname in sys.builtin_module_names:
|
||||
return OriginType.BUILTIN
|
||||
|
||||
# Unless it's a builtin module all bundled skeletons should have
|
||||
# file system independent "(pre-generated)" marker in their header
|
||||
if is_pregeneration_mode():
|
||||
return OriginType.PREGENERATED
|
||||
|
||||
if not mod_path:
|
||||
return None
|
||||
|
||||
if is_test_mode():
|
||||
return get_portable_test_module_path(mod_path, mod_qname)
|
||||
return mod_path
|
||||
|
||||
|
||||
def create_failed_version_stamp(base_dir, mod_qname):
|
||||
failed_version_stamp = os.path.join(base_dir, FAILED_VERSION_STAMP_PREFIX + mod_qname)
|
||||
with fopen(failed_version_stamp, 'w') as f:
|
||||
f.write(version())
|
||||
return failed_version_stamp
|
||||
|
||||
|
||||
def delete_failed_version_stamp(base_dir, mod_qname):
|
||||
delete(os.path.join(base_dir, FAILED_VERSION_STAMP_PREFIX + mod_qname))
|
||||
|
||||
|
||||
BinaryModule = collections.namedtuple('BinaryModule', ['qname', 'path'])
|
||||
|
||||
|
||||
def progress(text=None, fraction=None, minor=False):
|
||||
data = {}
|
||||
|
||||
if text is not None:
|
||||
data['text'] = text
|
||||
data['minor'] = minor
|
||||
|
||||
if fraction is not None:
|
||||
data['fraction'] = round(fraction, 2)
|
||||
|
||||
control_message('progress', data)
|
||||
|
||||
|
||||
def control_message(msg_type, data):
|
||||
data['type'] = msg_type
|
||||
say(json.dumps(data))
|
||||
|
||||
|
||||
def trace(msg, *args, **kwargs):
|
||||
logging.log(logging.getLevelName('TRACE'), msg, *args, **kwargs)
|
||||
|
||||
|
||||
class SkeletonGenerator(object):
|
||||
def __init__(self,
|
||||
output_dir, # type: str
|
||||
roots=None, # type: List[str]
|
||||
state_json=None, # type: Dict[str, Any]
|
||||
write_state_json=False,
|
||||
):
|
||||
self.output_dir = output_dir.rstrip(os.path.sep)
|
||||
# TODO make cache directory configurable via CLI
|
||||
self.cache_dir = os.path.join(os.path.dirname(self.output_dir), CACHE_DIR_NAME)
|
||||
self.roots = roots
|
||||
self.in_state_json = state_json
|
||||
self.out_state_json = {'sdk_skeletons': {}}
|
||||
self.write_state_json = write_state_json
|
||||
|
||||
def discover_and_process_all_modules(self, name_pattern=None, builtins_only=False):
|
||||
if name_pattern is None:
|
||||
name_pattern = '*'
|
||||
|
||||
all_modules = sorted(self.collect_builtin_modules(), key=(lambda b: b.qname))
|
||||
|
||||
if not builtins_only:
|
||||
progress("Discovering binary modules...")
|
||||
all_modules.extend(sorted(self.discover_binary_modules(), key=(lambda b: b.qname)))
|
||||
|
||||
matching_modules = [m for m in all_modules if fnmatch.fnmatchcase(m.qname, name_pattern)]
|
||||
|
||||
progress("Updating skeletons...")
|
||||
for i, mod in enumerate(matching_modules):
|
||||
progress(text=mod.qname, fraction=float(i) / len(matching_modules), minor=True)
|
||||
self.process_module(mod.qname, mod.path)
|
||||
progress(fraction=1.0)
|
||||
|
||||
if self.write_state_json:
|
||||
mkdir(self.output_dir)
|
||||
state_json_path = os.path.join(self.output_dir, STATE_FILE_NAME)
|
||||
logging.info('Writing skeletons state to %r', state_json_path)
|
||||
with fopen(state_json_path, 'w') as f:
|
||||
json.dump(self.out_state_json, f, sort_keys=True)
|
||||
|
||||
@staticmethod
|
||||
def collect_builtin_modules():
|
||||
# type: () -> List[BinaryModule]
|
||||
names = list(sys.builtin_module_names)
|
||||
if BUILTIN_MOD_NAME not in names:
|
||||
names.append(BUILTIN_MOD_NAME)
|
||||
if '__main__' in names:
|
||||
names.remove('__main__')
|
||||
return [BinaryModule(name, None) for name in names]
|
||||
|
||||
def discover_binary_modules(self):
|
||||
# type: () -> List[BinaryModule]
|
||||
"""
|
||||
Finds binaries in the given list of paths.
|
||||
Understands nested paths, as sys.paths have it (both "a/b" and "a/b/c").
|
||||
Tries to be case-insensitive, but case-preserving.
|
||||
"""
|
||||
SEP = os.path.sep
|
||||
res = {} # {name.upper(): (name, full_path)} # b/c windows is case-oblivious
|
||||
if not self.roots:
|
||||
return []
|
||||
# TODO Move to future InterpreterHandler
|
||||
if IS_JAVA: # jython can't have binary modules
|
||||
return []
|
||||
paths = sorted_no_case(self.roots)
|
||||
for path in paths:
|
||||
for root, files in walk_python_path(path):
|
||||
cutpoint = path.rfind(SEP)
|
||||
if cutpoint > 0:
|
||||
preprefix = path[(cutpoint + len(SEP)):] + '.'
|
||||
else:
|
||||
preprefix = ''
|
||||
prefix = root[(len(path) + len(SEP)):].replace(SEP, '.')
|
||||
if prefix:
|
||||
prefix += '.'
|
||||
binaries = ((f, cut_binary_lib_suffix(root, f)) for f in files)
|
||||
binaries = [(f, name) for (f, name) in binaries if name]
|
||||
if binaries:
|
||||
trace("root: %s path: %s prefix: %s preprefix: %s", root, path, prefix, preprefix)
|
||||
for f, name in binaries:
|
||||
the_name = prefix + name
|
||||
if is_skipped_module(root, f, the_name):
|
||||
trace('skipping module %s', the_name)
|
||||
continue
|
||||
trace("cutout: %s", name)
|
||||
if preprefix:
|
||||
trace("prefixes: %s %s", prefix, preprefix)
|
||||
pre_name = (preprefix + prefix + name).upper()
|
||||
if pre_name in res:
|
||||
res.pop(pre_name) # there might be a dupe, if paths got both a/b and a/b/c
|
||||
trace("done with %s", name)
|
||||
file_path = os.path.join(root, f)
|
||||
|
||||
res[the_name.upper()] = BinaryModule(the_name, file_path)
|
||||
return list(res.values())
|
||||
|
||||
def process_module(self, mod_name, mod_path=None):
|
||||
# type: (str, str) -> GenerationStatusId
|
||||
if self.in_state_json:
|
||||
existing_skeleton_meta = self.in_state_json['sdk_skeletons'].get(mod_name, {})
|
||||
sdk_skeleton_state = self.out_state_json['sdk_skeletons'][mod_name] = deepcopy(existing_skeleton_meta)
|
||||
else:
|
||||
sdk_skeleton_state = self.out_state_json['sdk_skeletons'][mod_name] = {}
|
||||
|
||||
status = self.reuse_or_generate_skeleton(mod_name, mod_path, sdk_skeleton_state)
|
||||
control_message('generation_result', {
|
||||
'module_name': mod_name,
|
||||
'module_origin': get_module_origin(mod_path, mod_name),
|
||||
'generation_status': status
|
||||
})
|
||||
if mod_path:
|
||||
sdk_skeleton_state['bin_mtime'] = file_modification_timestamp(mod_path)
|
||||
|
||||
# If we skipped generation for already failing module, we can safely set
|
||||
# the current generator version in ".state.json" as skipping means that this
|
||||
# version is not greater (i.e. we don't need to distinguish between "skipped as failing"
|
||||
# and "failed during generation").
|
||||
if status not in (GenerationStatus.UP_TO_DATE, GenerationStatus.COPIED):
|
||||
# TODO don't update state_json inplace
|
||||
sdk_skeleton_state['gen_version'] = version()
|
||||
|
||||
sdk_skeleton_state['status'] = status
|
||||
|
||||
if is_test_mode():
|
||||
sdk_skeleton_state.pop('bin_mtime', None)
|
||||
return status
|
||||
|
||||
def reuse_or_generate_skeleton(self, mod_name, mod_path, mod_state_json):
|
||||
# type: (str, str, Dict[str, Any]) -> GenerationStatusId
|
||||
if not quiet:
|
||||
logging.info('%s (%r)', mod_name, mod_path or 'built-in')
|
||||
action("doing nothing")
|
||||
|
||||
try:
|
||||
sdk_skeleton_status = skeleton_status(self.output_dir, mod_name, mod_path, mod_state_json)
|
||||
if sdk_skeleton_status == SkeletonStatus.UP_TO_DATE:
|
||||
return GenerationStatus.UP_TO_DATE
|
||||
elif sdk_skeleton_status == SkeletonStatus.FAILING:
|
||||
return GenerationStatus.FAILED
|
||||
|
||||
# At this point we will either generate skeleton anew all take it from the cache.
|
||||
# In either case state.json is supposed to be populated by this results.
|
||||
if mod_state_json:
|
||||
mod_state_json.clear()
|
||||
|
||||
mod_cache_dir = build_cache_dir_path(self.cache_dir, mod_name, mod_path)
|
||||
cached_skeleton_status = skeleton_status(mod_cache_dir, mod_name, mod_path, mod_state_json)
|
||||
if cached_skeleton_status == SkeletonStatus.OUTDATED:
|
||||
return execute_in_subprocess_synchronously(name='Skeleton Generator Worker',
|
||||
func=generate_skeleton,
|
||||
args=(mod_name,
|
||||
mod_path,
|
||||
mod_cache_dir,
|
||||
self.output_dir),
|
||||
kwargs={},
|
||||
failure_result=GenerationStatus.FAILED)
|
||||
elif cached_skeleton_status == SkeletonStatus.FAILING:
|
||||
logging.info('Cache entry for %s at %r indicates failed generation', mod_name, mod_cache_dir)
|
||||
return GenerationStatus.FAILED
|
||||
else:
|
||||
# Copy entire skeletons directory if nothing needs to be updated
|
||||
logging.info('Copying cached stubs for %s from %r to %r', mod_name, mod_cache_dir, self.output_dir)
|
||||
copy_skeletons(mod_cache_dir, self.output_dir, get_module_origin(mod_path, mod_name))
|
||||
return GenerationStatus.COPIED
|
||||
except:
|
||||
exctype, value = sys.exc_info()[:2]
|
||||
msg = "Failed to process %r while %s: %s"
|
||||
args = mod_name, CURRENT_ACTION, str(value)
|
||||
report(msg, *args)
|
||||
if sys.platform == 'cli':
|
||||
import traceback
|
||||
traceback.print_exc(file=sys.stderr)
|
||||
raise
|
||||
|
||||
|
||||
@contextmanager
|
||||
def imported_names_collected():
|
||||
imported_names = set()
|
||||
|
||||
class MyFinder(object):
|
||||
# noinspection PyMethodMayBeStatic
|
||||
def find_module(self, fullname, path=None):
|
||||
imported_names.add(fullname)
|
||||
return None
|
||||
|
||||
my_finder = MyFinder()
|
||||
sys.meta_path.insert(0, my_finder)
|
||||
try:
|
||||
yield imported_names
|
||||
finally:
|
||||
sys.meta_path.remove(my_finder)
|
||||
|
||||
|
||||
def generate_skeleton(name, mod_file_name, mod_cache_dir, output_dir):
|
||||
# type: (str, str, str, str) -> GenerationStatusId
|
||||
|
||||
logging.info('Updating cache for %s at %r', name, mod_cache_dir)
|
||||
doing_builtins = mod_file_name is None
|
||||
# All builtin modules go into the same directory
|
||||
if not doing_builtins:
|
||||
delete(mod_cache_dir)
|
||||
|
||||
# delete output path so it can be regenerated
|
||||
delete(os.path.join(output_dir, name))
|
||||
# we don't use the cache dir
|
||||
#mkdir(mod_cache_dir)
|
||||
|
||||
#create_failed_version_stamp(mod_cache_dir, name)
|
||||
|
||||
action("importing")
|
||||
old_modules = list(sys.modules.keys())
|
||||
with imported_names_collected() as imported_module_names:
|
||||
__import__(name) # sys.modules will fill up with what we want
|
||||
|
||||
redo_module(name, mod_file_name, mod_cache_dir, output_dir)
|
||||
# The C library may have called Py_InitModule() multiple times to define several modules (gtk._gtk and gtk.gdk);
|
||||
# restore all of them
|
||||
path = name.split(".")
|
||||
redo_imports = not ".".join(path[:-1]) in MODULES_INSPECT_DIR
|
||||
if redo_imports:
|
||||
initial_module_set = set(sys.modules)
|
||||
for m in list(sys.modules):
|
||||
if not m.startswith(name):
|
||||
continue
|
||||
# Python 2 puts dummy None entries in sys.modules for imports of
|
||||
# top-level modules made from inside packages unless absolute
|
||||
# imports are explicitly enabled.
|
||||
# See https://www.python.org/dev/peps/pep-0328/#relative-imports-and-indirection-entries-in-sys-modules
|
||||
if not sys.modules[m] or m.startswith("generator3"):
|
||||
continue
|
||||
action("looking at possible submodule %r", m)
|
||||
if m == name or m in old_modules or m in sys.builtin_module_names:
|
||||
continue
|
||||
# Synthetic module, not explicitly imported
|
||||
if m not in imported_module_names and not hasattr(sys.modules[m], '__file__'):
|
||||
if not quiet:
|
||||
logging.info('Processing submodule %s of %s', m, name)
|
||||
action("opening %r", mod_cache_dir)
|
||||
try:
|
||||
redo_module(m, mod_file_name, cache_dir=mod_cache_dir, output_dir=output_dir)
|
||||
extra_modules = set(sys.modules) - initial_module_set
|
||||
if extra_modules:
|
||||
report('Introspecting submodule %r of %r led to extra content of sys.modules: %s',
|
||||
m, name, ', '.join(extra_modules))
|
||||
finally:
|
||||
action("closing %r", mod_cache_dir)
|
||||
return GenerationStatus.GENERATED
|
||||
|
||||
|
||||
def redo_module(module_name, module_file_name, cache_dir, output_dir):
|
||||
# type: (str, str, str, str) -> None
|
||||
# gobject does 'del _gobject' in its __init__.py, so the chained attribute lookup code
|
||||
# fails to find 'gobject._gobject'. thus we need to pull the module directly out of
|
||||
# sys.modules
|
||||
mod = sys.modules.get(module_name)
|
||||
mod_path = module_name.split('.')
|
||||
if not mod and sys.platform == 'cli':
|
||||
# "import System.Collections" in IronPython 2.7 doesn't actually put System.Collections in sys.modules
|
||||
# instead, sys.modules['System'] get set to a Microsoft.Scripting.Actions.NamespaceTracker and Collections can be
|
||||
# accessed as its attribute
|
||||
mod = sys.modules[mod_path[0]]
|
||||
for component in mod_path[1:]:
|
||||
try:
|
||||
mod = getattr(mod, component)
|
||||
except AttributeError:
|
||||
mod = None
|
||||
report("Failed to find CLR module " + module_name)
|
||||
break
|
||||
if mod:
|
||||
action("restoring")
|
||||
from generator3.module_redeclarator import ModuleRedeclarator
|
||||
# Generate output directly in output folder, don't create cache folder
|
||||
r = ModuleRedeclarator(mod, module_name, module_file_name, cache_dir=output_dir,
|
||||
doing_builtins=(module_file_name is None))
|
||||
create_failed_version_stamp(output_dir, module_name)
|
||||
r.redo(module_name, ".".join(mod_path[:-1]) in MODULES_INSPECT_DIR)
|
||||
action("flushing")
|
||||
r.flush()
|
||||
delete_failed_version_stamp(output_dir, module_name)
|
||||
# Incrementally copy whatever we managed to successfully generate so far
|
||||
#copy_skeletons(cache_dir, output_dir, get_module_origin(module_file_name, module_name))
|
||||
else:
|
||||
report("Failed to find imported module in sys.modules " + module_name)
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
import re
|
||||
import sys
|
||||
|
||||
from generator3.constants import STR_TYPES
|
||||
from generator3.util_methods import sanitize_value
|
||||
|
||||
# only support Python 3
|
||||
# noinspection PyUnresolvedReferences
|
||||
from generator3._vendor.pyparsing_py3 import *
|
||||
|
||||
# grammar to parse parameter lists
|
||||
|
||||
# // snatched from parsePythonValue.py, from pyparsing samples, copyright 2006 by Paul McGuire but under BSD license.
|
||||
# we don't suppress lots of punctuation because we want it back when we reconstruct the lists
|
||||
|
||||
lparen, rparen, lbrack, rbrack, lbrace, rbrace, colon = map(Literal, "()[]{}:")
|
||||
|
||||
integer = Combine(Optional(oneOf("+ -")) + Word(nums)).setName("integer")
|
||||
real = Combine(Optional(oneOf("+ -")) + Word(nums) + "." +
|
||||
Optional(Word(nums)) +
|
||||
Optional(oneOf("e E") + Optional(oneOf("+ -")) + Word(nums))).setName("real")
|
||||
tupleStr = Forward()
|
||||
listStr = Forward()
|
||||
dictStr = Forward()
|
||||
|
||||
boolLiteral = oneOf("True False")
|
||||
noneLiteral = Literal("None")
|
||||
|
||||
listItem = real | integer | quotedString | unicodeString | boolLiteral | noneLiteral | \
|
||||
Group(listStr) | tupleStr | dictStr
|
||||
|
||||
tupleStr << (Suppress("(") + Optional(delimitedList(listItem)) +
|
||||
Optional(Literal(",")) + Suppress(")")).setResultsName("tuple")
|
||||
|
||||
listStr << (lbrack + Optional(delimitedList(listItem) +
|
||||
Optional(Literal(","))) + rbrack).setResultsName("list")
|
||||
|
||||
dictEntry = Group(listItem + colon + listItem)
|
||||
dictStr << (lbrace + Optional(delimitedList(dictEntry) + Optional(Literal(","))) + rbrace).setResultsName("dict")
|
||||
# \\ end of the snatched part
|
||||
|
||||
# our output format is s-expressions:
|
||||
# (simple name optional_value) is name or name=value
|
||||
# (nested (simple ...) (simple ...)) is (name, name,...)
|
||||
# (opt ...) is [, ...] or suchlike.
|
||||
|
||||
T_SIMPLE = 'Simple'
|
||||
T_NESTED = 'Nested'
|
||||
T_OPTIONAL = 'Opt'
|
||||
T_RETURN = "Ret"
|
||||
|
||||
TRIPLE_DOT = '...'
|
||||
|
||||
COMMA = Suppress(",")
|
||||
APOS = Suppress("'")
|
||||
QUOTE = Suppress('"')
|
||||
SP = Suppress(Optional(White()))
|
||||
|
||||
ident = Word(alphas + "_", alphanums + "_-.").setName("ident") # we accept things like "foo-or-bar"
|
||||
decorated_ident = ident + Optional(Suppress(SP + Literal(":") + SP + ident)) # accept "foo: bar", ignore "bar"
|
||||
spaced_ident = Combine(
|
||||
decorated_ident + ZeroOrMore(Literal(' ') + decorated_ident)) # we accept 'list or tuple' or 'C struct'
|
||||
|
||||
# allow quoted names, because __setattr__, etc docs use it
|
||||
paramname = spaced_ident | \
|
||||
APOS + spaced_ident + APOS | \
|
||||
QUOTE + spaced_ident + QUOTE
|
||||
|
||||
parenthesized_tuple = (Literal("(") + Optional(delimitedList(listItem, combine=True)) +
|
||||
Optional(Literal(",")) + Literal(")")).setResultsName("(tuple)")
|
||||
|
||||
initializer = (SP + Suppress("=") + SP + Combine(parenthesized_tuple | listItem | ident)).setName(
|
||||
"=init") # accept foo=defaultfoo
|
||||
|
||||
param = Group(Empty().setParseAction(replaceWith(T_SIMPLE)) + Combine(Optional(oneOf("* **")) + paramname) + Optional(
|
||||
initializer))
|
||||
|
||||
ellipsis = Group(
|
||||
Empty().setParseAction(replaceWith(T_SIMPLE)) + \
|
||||
(Literal("..") +
|
||||
ZeroOrMore(Literal('.'))).setParseAction(replaceWith(TRIPLE_DOT)) # we want to accept both 'foo,..' and 'foo, ...'
|
||||
)
|
||||
|
||||
paramSlot = Forward()
|
||||
|
||||
simpleParamSeq = ZeroOrMore(paramSlot + COMMA) + Optional(paramSlot + Optional(COMMA))
|
||||
nestedParamSeq = Group(
|
||||
Suppress('(').setParseAction(replaceWith(T_NESTED)) + \
|
||||
simpleParamSeq + Optional(ellipsis + Optional(COMMA) + Optional(simpleParamSeq)) + \
|
||||
Suppress(')')
|
||||
) # we accept "(a1, ... an)"
|
||||
|
||||
paramSlot << (param | nestedParamSeq)
|
||||
|
||||
optionalPart = Forward()
|
||||
|
||||
paramSeq = simpleParamSeq + Optional(optionalPart) # this is our approximate target
|
||||
|
||||
optionalPart << (
|
||||
Group(
|
||||
Suppress('[').setParseAction(replaceWith(T_OPTIONAL)) + Optional(COMMA) +
|
||||
paramSeq + Optional(ellipsis) +
|
||||
Suppress(']')
|
||||
)
|
||||
| ellipsis
|
||||
)
|
||||
|
||||
return_type = Group(
|
||||
Empty().setParseAction(replaceWith(T_RETURN)) +
|
||||
Suppress(SP + (Literal("->") | (Literal(":") + SP + Literal("return"))) + SP) +
|
||||
ident
|
||||
)
|
||||
|
||||
# this is our ideal target, with balancing paren and a multiline rest of doc.
|
||||
paramSeqAndRest = paramSeq + Suppress(')') + Optional(return_type) + Suppress(Optional(Regex(r"(?s).*")))
|
||||
|
||||
|
||||
def transform_seq(results, toplevel=True):
|
||||
"""Transforms a tree of ParseResults into a param spec string."""
|
||||
is_clr = sys.platform == "cli"
|
||||
ret = [] # add here token to join
|
||||
for token in results:
|
||||
token_type = token[0]
|
||||
if token_type is T_SIMPLE:
|
||||
token_name = token[1]
|
||||
if len(token) == 3: # name with value
|
||||
if toplevel:
|
||||
ret.append(sanitize_ident(token_name, is_clr) + "=" + sanitize_value(token[2]))
|
||||
else:
|
||||
# smth like "a, (b1=1, b2=2)", make it "a, p_b"
|
||||
return ["p_" + results[0][1]] # NOTE: for each item of tuple, return the same name of its 1st item.
|
||||
elif token_name == TRIPLE_DOT:
|
||||
if toplevel and not has_item_starting_with(ret, "*"):
|
||||
ret.append("*more")
|
||||
else:
|
||||
# we're in a "foo, (bar1, bar2, ...)"; make it "foo, bar_tuple"
|
||||
return extract_alpha_prefix(results[0][1]) + "_tuple"
|
||||
else: # just name
|
||||
ret.append(sanitize_ident(token_name, is_clr))
|
||||
elif token_type is T_NESTED:
|
||||
inner = transform_seq(token[1:], False)
|
||||
if len(inner) != 1:
|
||||
ret.append(inner)
|
||||
else:
|
||||
ret.append(inner[0]) # [foo] -> foo
|
||||
elif token_type is T_OPTIONAL:
|
||||
ret.extend(transform_optional_seq(token))
|
||||
elif token_type is T_RETURN:
|
||||
pass # this is handled elsewhere
|
||||
else:
|
||||
raise Exception("This cannot be a token type: " + repr(token_type))
|
||||
return ret
|
||||
|
||||
|
||||
def transform_optional_seq(results):
|
||||
"""
|
||||
Produces a string that describes the optional part of parameters.
|
||||
@param results must start from T_OPTIONAL.
|
||||
"""
|
||||
assert results[0] is T_OPTIONAL, "transform_optional_seq expects a T_OPTIONAL node, sees " + \
|
||||
repr(results[0])
|
||||
is_clr = sys.platform == "cli"
|
||||
ret = []
|
||||
for token in results[1:]:
|
||||
token_type = token[0]
|
||||
if token_type is T_SIMPLE:
|
||||
token_name = token[1]
|
||||
if len(token) == 3: # name with value; little sense, but can happen in a deeply nested optional
|
||||
ret.append(sanitize_ident(token_name, is_clr) + "=" + sanitize_value(token[2]))
|
||||
elif token_name == '...':
|
||||
# we're in a "foo, [bar, ...]"; make it "foo, *bar"
|
||||
return ["*" + extract_alpha_prefix(
|
||||
results[1][1])] # we must return a seq; [1] is first simple, [1][1] is its name
|
||||
else: # just name
|
||||
ret.append(sanitize_ident(token_name, is_clr) + "=None")
|
||||
elif token_type is T_OPTIONAL:
|
||||
ret.extend(transform_optional_seq(token))
|
||||
# maybe handle T_NESTED if such cases ever occur in real life
|
||||
# it can't be nested in a sane case, really
|
||||
return ret
|
||||
|
||||
|
||||
def has_item_starting_with(p_seq, p_start):
|
||||
for item in p_seq:
|
||||
if isinstance(item, STR_TYPES) and item.startswith(p_start):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def sanitize_ident(x, is_clr=False):
|
||||
"""Takes an identifier and returns it sanitized"""
|
||||
if x in ("class", "object", "def", "list", "tuple", "int", "float", "str", "unicode" "None"):
|
||||
return "p_" + x
|
||||
else:
|
||||
if is_clr:
|
||||
# it tends to have names like "int x", turn it to just x
|
||||
xs = x.split(" ")
|
||||
if len(xs) == 2:
|
||||
return sanitize_ident(xs[1])
|
||||
return x.replace("-", "_").replace(" ", "_").replace(".", "_") # for things like "list-or-tuple" or "list or tuple"
|
||||
|
||||
|
||||
def extract_alpha_prefix(p_string, default_prefix="some"):
|
||||
"""Returns 'foo' for things like 'foo1' or 'foo2'; if prefix cannot be found, the default is returned"""
|
||||
match = NUM_IDENT_PATTERN.match(p_string)
|
||||
prefix = match and match.groups()[match.lastindex - 1] or None
|
||||
return prefix or default_prefix
|
||||
|
||||
|
||||
NUM_IDENT_PATTERN = re.compile("([A-Za-z_]+)[0-9]?[A-Za-z_]*") # 'foo_123' -> $1 = 'foo_'
|
||||
@@ -0,0 +1,165 @@
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import zipfile
|
||||
|
||||
from generator3.core import walk_python_path
|
||||
from generator3.util_methods import is_text_file, say, report
|
||||
|
||||
|
||||
def is_source_file(path):
|
||||
# Skip directories, character and block special devices, named pipes
|
||||
# Do not skip regular files and symbolic links to regular files
|
||||
if not os.path.isfile(path):
|
||||
return False
|
||||
|
||||
# Want to see that files regardless of their encoding.
|
||||
if path.endswith(('-nspkg.pth', '.html', '.pxd', '.py', '.pyi', '.pyx')):
|
||||
return True
|
||||
has_bad_extension = path.endswith((
|
||||
# plotlywidget/static/index.js.map is 8.7 MiB.
|
||||
# Many map files from notebook are near 2 MiB.
|
||||
'.js.map',
|
||||
|
||||
# uvloop/loop.c contains 6.4 MiB of code.
|
||||
# Some header files from tensorflow has size more than 1 MiB.
|
||||
'.h', '.c',
|
||||
|
||||
# Test data of pycrypto, many files are near 1 MiB.
|
||||
'.rsp',
|
||||
|
||||
# No need to read these files even if they are small.
|
||||
'.dll', '.pyc', '.pyd', '.pyo', '.so',
|
||||
))
|
||||
if has_bad_extension:
|
||||
return False
|
||||
return is_text_file(path)
|
||||
|
||||
|
||||
def list_sources(paths):
|
||||
# noinspection PyBroadException
|
||||
try:
|
||||
for path in paths:
|
||||
path = os.path.normpath(path)
|
||||
|
||||
if path.endswith('.egg') and os.path.isfile(path):
|
||||
say("%s\t%s\t%d", path, path, os.path.getsize(path))
|
||||
|
||||
for root, files in walk_python_path(path):
|
||||
for name in files:
|
||||
file_path = os.path.join(root, name)
|
||||
if is_source_file(file_path):
|
||||
say("%s\t%s\t%d", os.path.normpath(file_path), path, os.path.getsize(file_path))
|
||||
say('END')
|
||||
sys.stdout.flush()
|
||||
except:
|
||||
import traceback
|
||||
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def zip_sources(zip_path):
|
||||
if not os.path.exists(zip_path):
|
||||
os.makedirs(zip_path)
|
||||
|
||||
zip_filename = os.path.normpath(os.path.sep.join([zip_path, "skeletons.zip"]))
|
||||
|
||||
try:
|
||||
zip = zipfile.ZipFile(zip_filename, 'w', zipfile.ZIP_DEFLATED)
|
||||
except:
|
||||
zip = zipfile.ZipFile(zip_filename, 'w')
|
||||
|
||||
try:
|
||||
try:
|
||||
while True:
|
||||
line = sys.stdin.readline()
|
||||
|
||||
if not line:
|
||||
# TextIOWrapper.readline returns an empty string if EOF is hit immediately.
|
||||
break
|
||||
|
||||
line = line.strip()
|
||||
|
||||
if line == '-':
|
||||
break
|
||||
|
||||
if line:
|
||||
# This line will break the split:
|
||||
# /.../dist-packages/setuptools/script template (dev).py setuptools/script template (dev).py
|
||||
split_items = line.split()
|
||||
if len(split_items) > 2:
|
||||
# Currently it doesn't work for remote files like
|
||||
# /System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python/setuptools/script (dev).tmpl
|
||||
# TODO handle paths containing whitespaces more robustly
|
||||
match_two_files = re.match(r'^(.+\.py)\s+(.+\.py)$', line)
|
||||
if not match_two_files:
|
||||
report("Error(zip_sources): invalid line '%s'" % line)
|
||||
continue
|
||||
split_items = match_two_files.group(1, 2)
|
||||
(path, arcpath) = split_items
|
||||
|
||||
# An attempt to recursively pack an archive leads to unlimited explosion of its size
|
||||
if os.path.samefile(path, zip_filename):
|
||||
continue
|
||||
|
||||
zip.write(path, arcpath)
|
||||
say('OK: ' + zip_filename)
|
||||
sys.stdout.flush()
|
||||
except:
|
||||
import traceback
|
||||
|
||||
traceback.print_exc()
|
||||
say('Error creating archive.')
|
||||
|
||||
sys.exit(1)
|
||||
finally:
|
||||
zip.close()
|
||||
|
||||
|
||||
def add_to_zip(zip, paths):
|
||||
# noinspection PyBroadException
|
||||
try:
|
||||
for path in paths:
|
||||
print("Walking root %s" % path)
|
||||
|
||||
path = os.path.normpath(path)
|
||||
|
||||
if path.endswith('.egg') and os.path.isfile(path):
|
||||
pass # TODO: handle eggs
|
||||
|
||||
for root, files in walk_python_path(path):
|
||||
for name in files:
|
||||
file_path = os.path.join(root, name)
|
||||
arcpath = os.path.relpath(file_path, path)
|
||||
|
||||
zip.write(file_path, os.path.join(str(hash(path)), arcpath))
|
||||
except:
|
||||
import traceback
|
||||
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def zip_stdlib(roots, zip_path):
|
||||
if not os.path.exists(zip_path):
|
||||
os.makedirs(zip_path)
|
||||
|
||||
import platform
|
||||
|
||||
zip_filename = os.path.normpath(os.path.sep.join([zip_path, "%s-%s-stdlib-%s.zip" % (
|
||||
'Anaconda' if sys.version.find('Anaconda') != -1 else 'Python',
|
||||
'.'.join(map(str, sys.version_info)),
|
||||
platform.platform())]))
|
||||
|
||||
print("Adding file to %s" % zip_filename)
|
||||
|
||||
try:
|
||||
zip = zipfile.ZipFile(zip_filename, 'w', zipfile.ZIP_DEFLATED)
|
||||
except:
|
||||
zip = zipfile.ZipFile(zip_filename, 'w')
|
||||
|
||||
try:
|
||||
add_to_zip(zip, roots)
|
||||
finally:
|
||||
zip.close()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,50 @@
|
||||
# This file lists minimum generator versions required for known packages / files.
|
||||
# hash marks start line comments.
|
||||
# name is either a package name (as used in import) or a predefined name in parentheses.
|
||||
# version is two decimal numbers divided by a dot.
|
||||
# settings equally apply to all platforms (jython, cpython, ipy).
|
||||
|
||||
(default) 1.127 # anything not explicitly marked
|
||||
|
||||
(built-in) 1.145 # skeletons of all built-in modules are built together
|
||||
# Note: modules like itertools, etc are "(built-in)" and are ignored if given separately
|
||||
|
||||
_fileio 1.127
|
||||
_io 1.127
|
||||
sys 1.127
|
||||
thread 1.127
|
||||
_thread 1.127
|
||||
_struct 1.127
|
||||
datetime 1.127
|
||||
_collections 1.127
|
||||
|
||||
PyQt4.Qsci 1.127
|
||||
PyQt4.QtAssistant 1.127
|
||||
PyQt4.QtCore 1.127
|
||||
PyQt4.QtDesigner 1.127
|
||||
PyQt4.QtGui 1.127
|
||||
PyQt4.QtHelp 1.127
|
||||
PyQt4.QtNetwork 1.127
|
||||
PyQt4.QtScriptTools 1.127
|
||||
PyQt4.QtScript 1.127
|
||||
PyQt4.QtSvg 1.127
|
||||
PyQt4.QtTest 1.127
|
||||
PyQt4.Qt 1.127
|
||||
PyQt4.QtWebKit 1.127
|
||||
PyQt4.QtXmlPatterns 1.127
|
||||
PyQt4.QtXml 1.127
|
||||
|
||||
pygame.fastevent 1.127
|
||||
pygame.image 1.127
|
||||
|
||||
sip 1.127
|
||||
|
||||
pysqlite2._sqlite 1.127
|
||||
_bsddb 1.127
|
||||
|
||||
h5py.h5 1.127
|
||||
h5py.h5i 1.127
|
||||
h5py.h5g 1.127
|
||||
|
||||
numpy.random.mtrand 1.140
|
||||
numpy.core.multiarray 1.143
|
||||
@@ -0,0 +1,938 @@
|
||||
import ast
|
||||
import collections
|
||||
import errno
|
||||
import functools
|
||||
import hashlib
|
||||
import json
|
||||
import keyword
|
||||
import logging
|
||||
import multiprocessing
|
||||
import shutil
|
||||
from contextlib import contextmanager
|
||||
|
||||
from generator3.constants import *
|
||||
|
||||
try:
|
||||
import inspect
|
||||
except ImportError:
|
||||
inspect = None
|
||||
|
||||
BIN_READ_BLOCK = 64 * 1024
|
||||
|
||||
|
||||
def create_named_tuple(): #TODO: user-skeleton
|
||||
return """
|
||||
class __namedtuple(tuple):
|
||||
'''A mock base class for named tuples.'''
|
||||
|
||||
__slots__ = ()
|
||||
_fields = ()
|
||||
|
||||
def __new__(cls, *args, **kwargs):
|
||||
'Create a new instance of the named tuple.'
|
||||
return tuple.__new__(cls, *args)
|
||||
|
||||
@classmethod
|
||||
def _make(cls, iterable, new=tuple.__new__, len=len):
|
||||
'Make a new named tuple object from a sequence or iterable.'
|
||||
return new(cls, iterable)
|
||||
|
||||
def __repr__(self):
|
||||
return ''
|
||||
|
||||
def _asdict(self):
|
||||
'Return a new dict which maps field types to their values.'
|
||||
return {}
|
||||
|
||||
def _replace(self, **kwargs):
|
||||
'Return a new named tuple object replacing specified fields with new values.'
|
||||
return self
|
||||
|
||||
def __getnewargs__(self):
|
||||
return tuple(self)
|
||||
"""
|
||||
|
||||
def create_generator():
|
||||
# Fake <type 'generator'>
|
||||
if version[0] < 3:
|
||||
next_name = "next"
|
||||
else:
|
||||
next_name = "__next__"
|
||||
txt = """
|
||||
class __generator(object):
|
||||
'''A mock class representing the generator function type.'''
|
||||
def __init__(self):
|
||||
self.gi_code = None
|
||||
self.gi_frame = None
|
||||
self.gi_running = 0
|
||||
|
||||
def __iter__(self):
|
||||
'''Defined to support iteration over container.'''
|
||||
pass
|
||||
|
||||
def %s(self):
|
||||
'''Return the next item from the container.'''
|
||||
pass
|
||||
""" % (next_name,)
|
||||
if version[0] >= 3 or (version[0] == 2 and version[1] >= 5):
|
||||
txt += """
|
||||
def close(self):
|
||||
'''Raises new GeneratorExit exception inside the generator to terminate the iteration.'''
|
||||
pass
|
||||
|
||||
def send(self, value):
|
||||
'''Resumes the generator and "sends" a value that becomes the result of the current yield-expression.'''
|
||||
pass
|
||||
|
||||
def throw(self, type, value=None, traceback=None):
|
||||
'''Used to raise an exception inside the generator.'''
|
||||
pass
|
||||
"""
|
||||
return txt
|
||||
|
||||
def create_async_generator():
|
||||
# Fake <type 'asyncgenerator'>
|
||||
txt = """
|
||||
class __asyncgenerator(object):
|
||||
'''A mock class representing the async generator function type.'''
|
||||
def __init__(self):
|
||||
'''Create an async generator object.'''
|
||||
self.__name__ = ''
|
||||
self.__qualname__ = ''
|
||||
self.ag_await = None
|
||||
self.ag_frame = None
|
||||
self.ag_running = False
|
||||
self.ag_code = None
|
||||
|
||||
def __aiter__(self):
|
||||
'''Defined to support iteration over container.'''
|
||||
pass
|
||||
|
||||
def __anext__(self):
|
||||
'''Returns an awaitable, that performs one asynchronous generator iteration when awaited.'''
|
||||
pass
|
||||
|
||||
def aclose(self):
|
||||
'''Returns an awaitable, that throws a GeneratorExit exception into generator.'''
|
||||
pass
|
||||
|
||||
def asend(self, value):
|
||||
'''Returns an awaitable, that pushes the value object in generator.'''
|
||||
pass
|
||||
|
||||
def athrow(self, type, value=None, traceback=None):
|
||||
'''Returns an awaitable, that throws an exception into generator.'''
|
||||
pass
|
||||
"""
|
||||
return txt
|
||||
|
||||
def create_function():
|
||||
txt = """
|
||||
class __function(object):
|
||||
'''A mock class representing function type.'''
|
||||
|
||||
def __init__(self):
|
||||
self.__name__ = ''
|
||||
self.__doc__ = ''
|
||||
self.__dict__ = ''
|
||||
self.__module__ = ''
|
||||
"""
|
||||
if version[0] == 2:
|
||||
txt += """
|
||||
self.func_defaults = {}
|
||||
self.func_globals = {}
|
||||
self.func_closure = None
|
||||
self.func_code = None
|
||||
self.func_name = ''
|
||||
self.func_doc = ''
|
||||
self.func_dict = ''
|
||||
"""
|
||||
if version[0] >= 3 or (version[0] == 2 and version[1] >= 6):
|
||||
txt += """
|
||||
self.__defaults__ = {}
|
||||
self.__globals__ = {}
|
||||
self.__closure__ = None
|
||||
self.__code__ = None
|
||||
self.__name__ = ''
|
||||
"""
|
||||
if version[0] >= 3:
|
||||
txt += """
|
||||
self.__annotations__ = {}
|
||||
self.__kwdefaults__ = {}
|
||||
"""
|
||||
if version[0] >= 3 and version[1] >= 3:
|
||||
txt += """
|
||||
self.__qualname__ = ''
|
||||
"""
|
||||
return txt
|
||||
|
||||
def create_method():
|
||||
txt = """
|
||||
class __method(object):
|
||||
'''A mock class representing method type.'''
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
if version[0] == 2:
|
||||
txt += """
|
||||
self.im_class = None
|
||||
self.im_self = None
|
||||
self.im_func = None
|
||||
"""
|
||||
if version[0] >= 3 or (version[0] == 2 and version[1] >= 6):
|
||||
txt += """
|
||||
self.__func__ = None
|
||||
self.__self__ = None
|
||||
"""
|
||||
return txt
|
||||
|
||||
|
||||
def create_coroutine():
|
||||
if version[0] == 3 and version[1] >= 5:
|
||||
return """
|
||||
class __coroutine(object):
|
||||
'''A mock class representing coroutine type.'''
|
||||
|
||||
def __init__(self):
|
||||
self.__name__ = ''
|
||||
self.__qualname__ = ''
|
||||
self.cr_await = None
|
||||
self.cr_frame = None
|
||||
self.cr_running = False
|
||||
self.cr_code = None
|
||||
|
||||
def __await__(self):
|
||||
return []
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
def send(self, value):
|
||||
pass
|
||||
|
||||
def throw(self, type, value=None, traceback=None):
|
||||
pass
|
||||
"""
|
||||
return ""
|
||||
|
||||
|
||||
def _searchbases(cls, accum):
|
||||
# logic copied from inspect.py
|
||||
if cls not in accum:
|
||||
accum.append(cls)
|
||||
for x in cls.__bases__:
|
||||
_searchbases(x, accum)
|
||||
|
||||
|
||||
def get_mro(a_class):
|
||||
# logic copied from inspect.py
|
||||
"""Returns a tuple of MRO classes."""
|
||||
if hasattr(a_class, "__mro__"):
|
||||
return a_class.__mro__
|
||||
elif hasattr(a_class, "__bases__"):
|
||||
bases = []
|
||||
_searchbases(a_class, bases)
|
||||
return tuple(bases)
|
||||
else:
|
||||
return tuple()
|
||||
|
||||
|
||||
def get_bases(a_class): # TODO: test for classes that don't fit this scheme
|
||||
"""Returns a sequence of class's bases."""
|
||||
if hasattr(a_class, "__bases__"):
|
||||
return a_class.__bases__
|
||||
else:
|
||||
return ()
|
||||
|
||||
|
||||
def is_callable(x):
|
||||
return hasattr(x, '__call__')
|
||||
|
||||
|
||||
def sorted_no_case(p_array):
|
||||
"""Sort an array case insensitively, returns a sorted copy"""
|
||||
p_array = list(p_array)
|
||||
p_array = sorted(p_array, key=lambda x: x.upper())
|
||||
return p_array
|
||||
|
||||
|
||||
def cleanup(value):
|
||||
result = []
|
||||
prev = i = 0
|
||||
length = len(value)
|
||||
last_ascii = chr(127)
|
||||
while i < length:
|
||||
char = value[i]
|
||||
replacement = None
|
||||
if char == '\n':
|
||||
replacement = '\\n'
|
||||
elif char == '\r':
|
||||
replacement = '\\r'
|
||||
elif char < ' ' or char > last_ascii:
|
||||
replacement = '?' # NOTE: such chars are rare; long swaths could be precessed differently
|
||||
if replacement:
|
||||
result.append(value[prev:i])
|
||||
result.append(replacement)
|
||||
prev = i + 1
|
||||
i += 1
|
||||
result.append(value[prev:])
|
||||
return "".join(result)
|
||||
|
||||
|
||||
def is_valid_expr(s):
|
||||
try:
|
||||
compile(s, '<unknown>', 'eval', ast.PyCF_ONLY_AST)
|
||||
except SyntaxError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
_prop_types = [type(property())]
|
||||
#noinspection PyBroadException
|
||||
try:
|
||||
_prop_types.append(types.GetSetDescriptorType)
|
||||
except:
|
||||
pass
|
||||
|
||||
#noinspection PyBroadException
|
||||
try:
|
||||
_prop_types.append(types.MemberDescriptorType)
|
||||
except:
|
||||
pass
|
||||
|
||||
_prop_types = tuple(_prop_types)
|
||||
|
||||
|
||||
def is_property(x):
|
||||
return isinstance(x, _prop_types)
|
||||
|
||||
|
||||
def reliable_repr(value):
|
||||
# some subclasses of built-in types (see PyGtk) may provide invalid __repr__ implementations,
|
||||
# so we need to sanitize the output
|
||||
if type(bool) == type and isinstance(value, bool):
|
||||
return repr(bool(value))
|
||||
for num_type in NUM_TYPES:
|
||||
if isinstance(value, num_type):
|
||||
return repr(num_type(value))
|
||||
return repr(value)
|
||||
|
||||
|
||||
def sanitize_value(p_value):
|
||||
"""Returns p_value or its part if it represents a sane simple value, else returns 'None'"""
|
||||
if isinstance(p_value, STR_TYPES):
|
||||
match = SIMPLE_VALUE_RE.match(p_value)
|
||||
if match:
|
||||
return match.groups()[match.lastindex - 1]
|
||||
else:
|
||||
return 'None'
|
||||
elif isinstance(p_value, NUM_TYPES):
|
||||
return reliable_repr(p_value)
|
||||
elif p_value is None:
|
||||
return 'None'
|
||||
else:
|
||||
if hasattr(p_value, "__name__") and hasattr(p_value, "__module__") and p_value.__module__ == BUILTIN_MOD_NAME:
|
||||
return p_value.__name__ # float -> "float"
|
||||
else:
|
||||
return repr(repr(p_value)) # function -> "<function ...>", etc
|
||||
|
||||
|
||||
def report(msg, *data):
|
||||
"""Say something at error level (stderr)"""
|
||||
sys.stderr.write(msg % data)
|
||||
sys.stderr.write("\n")
|
||||
|
||||
|
||||
def say(msg, *data):
|
||||
"""Say something at info level (stdout)"""
|
||||
sys.stdout.write(msg % data)
|
||||
sys.stdout.write("\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def flatten(seq):
|
||||
"""Transforms tree lists like ['a', ['b', 'c'], 'd'] to strings like '(a, (b, c), d)', enclosing each tree level in parens."""
|
||||
ret = []
|
||||
for one in seq:
|
||||
if type(one) is list:
|
||||
ret.append(flatten(one))
|
||||
else:
|
||||
ret.append(one)
|
||||
return "(" + ", ".join(ret) + ")"
|
||||
|
||||
|
||||
def make_names_unique(seq, name_map=None):
|
||||
"""
|
||||
Returns a copy of tree list seq where all clashing names are modified by numeric suffixes:
|
||||
['a', 'b', 'a', 'b'] becomes ['a', 'b', 'a_1', 'b_1'].
|
||||
Each repeating name has its own counter in the name_map.
|
||||
"""
|
||||
ret = []
|
||||
if not name_map:
|
||||
name_map = {}
|
||||
for one in seq:
|
||||
if type(one) is list:
|
||||
ret.append(make_names_unique(one, name_map))
|
||||
else:
|
||||
if keyword.iskeyword(one):
|
||||
one += "_"
|
||||
one_key = lstrip(one, "*") # starred parameters are unique sans stars
|
||||
if one_key in name_map:
|
||||
old_one = one_key
|
||||
one = one + "_" + str(name_map[old_one])
|
||||
name_map[old_one] += 1
|
||||
else:
|
||||
name_map[one_key] = 1
|
||||
ret.append(one)
|
||||
return ret
|
||||
|
||||
|
||||
def out_docstring(out_func, docstring, indent):
|
||||
if not isinstance(docstring, str): return
|
||||
lines = docstring.strip().split("\n")
|
||||
if lines:
|
||||
if len(lines) == 1:
|
||||
out_func(indent, '""" ' + lines[0] + ' """')
|
||||
else:
|
||||
out_func(indent, '"""')
|
||||
for line in lines:
|
||||
try:
|
||||
out_func(indent, line)
|
||||
except UnicodeEncodeError:
|
||||
continue
|
||||
out_func(indent, '"""')
|
||||
|
||||
def out_doc_attr(out_func, p_object, indent, p_class=None):
|
||||
the_doc = getattr(p_object, "__doc__", None)
|
||||
if the_doc:
|
||||
if p_class and the_doc == object.__init__.__doc__ and p_object is not object.__init__ and p_class.__doc__:
|
||||
the_doc = str(p_class.__doc__) # replace stock init's doc with class's; make it a certain string.
|
||||
the_doc += "\n# (copied from class doc)"
|
||||
out_docstring(out_func, the_doc, indent)
|
||||
else:
|
||||
out_func(indent, "# no doc")
|
||||
|
||||
def is_skipped_in_module(p_module, p_value):
|
||||
"""
|
||||
Returns True if p_value's value must be skipped for module p_module.
|
||||
"""
|
||||
skip_list = SKIP_VALUE_IN_MODULE.get(p_module, [])
|
||||
if p_value in skip_list:
|
||||
return True
|
||||
skip_list = SKIP_VALUE_IN_MODULE.get("*", [])
|
||||
if p_value in skip_list:
|
||||
return True
|
||||
return False
|
||||
|
||||
def restore_predefined_builtin(class_name, func_name):
|
||||
spec = func_name + PREDEFINED_BUILTIN_SIGS[(class_name, func_name)]
|
||||
note = "known special case of " + (class_name and class_name + "." or "") + func_name
|
||||
return (spec, note)
|
||||
|
||||
def restore_by_inspect(p_func):
|
||||
"""
|
||||
Returns paramlist restored by inspect.
|
||||
"""
|
||||
args, varg, kwarg, defaults, kwonlyargs, kwonlydefaults, _ = getfullargspec(p_func)
|
||||
spec = []
|
||||
if defaults:
|
||||
dcnt = len(defaults) - 1
|
||||
else:
|
||||
dcnt = -1
|
||||
args = args or []
|
||||
args.reverse() # backwards, for easier defaults handling
|
||||
for arg in args:
|
||||
if dcnt >= 0:
|
||||
arg += "=" + sanitize_value(defaults[dcnt])
|
||||
dcnt -= 1
|
||||
spec.insert(0, arg)
|
||||
if varg:
|
||||
spec.append("*" + varg)
|
||||
elif kwonlyargs:
|
||||
spec.append("*")
|
||||
|
||||
kwonlydefaults = kwonlydefaults or {}
|
||||
for arg in kwonlyargs:
|
||||
if arg in kwonlydefaults:
|
||||
spec.append(arg + '=' + sanitize_value(kwonlydefaults[arg]))
|
||||
else:
|
||||
spec.append(arg)
|
||||
|
||||
if kwarg:
|
||||
spec.append("**" + kwarg)
|
||||
return flatten(spec)
|
||||
|
||||
def restore_parameters_for_overloads(parameter_lists):
|
||||
param_index = 0
|
||||
star_args = False
|
||||
optional = False
|
||||
params = []
|
||||
while True:
|
||||
parameter_lists_copy = [pl for pl in parameter_lists]
|
||||
for pl in parameter_lists_copy:
|
||||
if param_index >= len(pl):
|
||||
parameter_lists.remove(pl)
|
||||
optional = True
|
||||
if not parameter_lists:
|
||||
break
|
||||
name = parameter_lists[0][param_index]
|
||||
for pl in parameter_lists[1:]:
|
||||
if pl[param_index] != name:
|
||||
star_args = True
|
||||
break
|
||||
if star_args: break
|
||||
if optional and not '=' in name:
|
||||
params.append(name + '=None')
|
||||
else:
|
||||
params.append(name)
|
||||
param_index += 1
|
||||
if star_args:
|
||||
params.append("*__args")
|
||||
return params
|
||||
|
||||
def build_signature(p_name, params):
|
||||
return p_name + '(' + ', '.join(params) + ')'
|
||||
|
||||
|
||||
def propose_first_param(deco):
|
||||
"""@return: name of missing first paramater, considering a decorator"""
|
||||
if deco is None:
|
||||
return "self"
|
||||
if deco == "classmethod":
|
||||
return "cls"
|
||||
# if deco == "staticmethod":
|
||||
return None
|
||||
|
||||
def qualifier_of(cls, qualifiers_to_skip):
|
||||
m = getattr(cls, "__module__", None)
|
||||
if m in qualifiers_to_skip:
|
||||
return ""
|
||||
return m
|
||||
|
||||
def handle_error_func(item_name, out):
|
||||
exctype, value = sys.exc_info()[:2]
|
||||
msg = "Error generating skeleton for function %s: %s"
|
||||
args = item_name, value
|
||||
report(msg, *args)
|
||||
out(0, "# " + msg % args)
|
||||
out(0, "")
|
||||
|
||||
def format_accessors(accessor_line, getter, setter, deleter):
|
||||
"""Nicely format accessors, like 'getter, fdel=deleter'"""
|
||||
ret = []
|
||||
consecutive = True
|
||||
for key, arg, par in (('r', 'fget', getter), ('w', 'fset', setter), ('d', 'fdel', deleter)):
|
||||
if key in accessor_line:
|
||||
if consecutive:
|
||||
ret.append(par)
|
||||
else:
|
||||
ret.append(arg + "=" + par)
|
||||
else:
|
||||
consecutive = False
|
||||
return ", ".join(ret)
|
||||
|
||||
|
||||
def has_regular_python_ext(file_name):
|
||||
"""Does name end with .py?"""
|
||||
return file_name.endswith(".py")
|
||||
# Note that the standard library on MacOS X 10.6 is shipped only as .pyc files, so we need to
|
||||
# have them processed by the generator in order to have any code insight for the standard library.
|
||||
|
||||
|
||||
def detect_constructor(p_class):
|
||||
# try to inspect the thing
|
||||
constr = getattr(p_class, "__init__")
|
||||
if constr and inspect and inspect.isfunction(constr):
|
||||
args, _, _, _, kwonlyargs, _, _ = getfullargspec(constr)
|
||||
return ", ".join(args + [a + '=' + a for a in kwonlyargs])
|
||||
else:
|
||||
return None
|
||||
|
||||
############## notes, actions #################################################################
|
||||
_is_verbose = False # controlled by -v
|
||||
|
||||
CURRENT_ACTION = "nothing yet"
|
||||
|
||||
def action(msg, *data):
|
||||
global CURRENT_ACTION
|
||||
CURRENT_ACTION = msg % data
|
||||
note(msg, *data)
|
||||
|
||||
|
||||
def set_verbose(verbose):
|
||||
global _is_verbose
|
||||
_is_verbose = verbose
|
||||
|
||||
|
||||
def note(msg, *data):
|
||||
"""Say something at debug info level (stderr)"""
|
||||
if _is_verbose:
|
||||
sys.stderr.write(msg % data)
|
||||
sys.stderr.write("\n")
|
||||
|
||||
|
||||
############## plaform-specific methods #######################################################
|
||||
import sys
|
||||
if sys.platform == 'cli':
|
||||
#noinspection PyUnresolvedReferences
|
||||
import clr
|
||||
|
||||
# http://blogs.msdn.com/curth/archive/2009/03/29/an-ironpython-profiler.aspx
|
||||
def print_profile():
|
||||
data = []
|
||||
data.extend(clr.GetProfilerData())
|
||||
data.sort(lambda x, y: -cmp(x.ExclusiveTime, y.ExclusiveTime))
|
||||
|
||||
for pd in data:
|
||||
say('%s\t%d\t%d\t%d', pd.Name, pd.InclusiveTime, pd.ExclusiveTime, pd.Calls)
|
||||
|
||||
def is_clr_type(clr_type):
|
||||
if not clr_type: return False
|
||||
try:
|
||||
clr.GetClrType(clr_type)
|
||||
return True
|
||||
except TypeError:
|
||||
return False
|
||||
|
||||
def restore_clr(p_name, p_class):
|
||||
"""
|
||||
Restore the function signature by the CLR type signature
|
||||
:return (is_static, spec, sig_note)
|
||||
"""
|
||||
clr_type = clr.GetClrType(p_class)
|
||||
if p_name == '__new__':
|
||||
methods = [c for c in clr_type.GetConstructors()]
|
||||
if not methods:
|
||||
return False, p_name + '(self, *args)', 'cannot find CLR constructor' # "self" is always first argument of any non-static method
|
||||
else:
|
||||
methods = [m for m in clr_type.GetMethods() if m.Name == p_name]
|
||||
if not methods:
|
||||
bases = p_class.__bases__
|
||||
if len(bases) == 1 and p_name in dir(bases[0]):
|
||||
# skip inherited methods
|
||||
return False, None, None
|
||||
return False, p_name + '(self, *args)', 'cannot find CLR method'
|
||||
# "self" is always first argument of any non-static method
|
||||
|
||||
parameter_lists = []
|
||||
for m in methods:
|
||||
parameter_lists.append([p.Name for p in m.GetParameters()])
|
||||
params = restore_parameters_for_overloads(parameter_lists)
|
||||
is_static = False
|
||||
if not methods[0].IsStatic:
|
||||
params = ['self'] + params
|
||||
else:
|
||||
is_static = True
|
||||
return is_static, build_signature(p_name, params), None
|
||||
|
||||
|
||||
def build_pkg_structure(base_dir, qname):
|
||||
if not qname:
|
||||
return base_dir
|
||||
|
||||
subdirname = base_dir
|
||||
for part in qname.split("."):
|
||||
subdirname = os.path.join(subdirname, part)
|
||||
if not os.path.isdir(subdirname):
|
||||
action("creating subdir %r", subdirname)
|
||||
os.makedirs(subdirname)
|
||||
init_py = os.path.join(subdirname, "__init__.py")
|
||||
if os.path.isfile(subdirname + ".py"):
|
||||
os.rename(subdirname + ".py", init_py)
|
||||
elif not os.path.isfile(init_py):
|
||||
fopen(init_py, "w").close()
|
||||
|
||||
return subdirname
|
||||
|
||||
|
||||
def is_valid_implicit_namespace_package_name(s):
|
||||
"""
|
||||
Checks whether provided string could represent implicit namespace package name.
|
||||
:param s: string to check
|
||||
:return: True if provided string could represent implicit namespace package name and False otherwise
|
||||
"""
|
||||
return isidentifier(s) and not keyword.iskeyword(s)
|
||||
|
||||
|
||||
def isidentifier(s):
|
||||
"""
|
||||
Checks whether provided string complies Python identifier syntax requirements.
|
||||
:param s: string to check
|
||||
:return: True if provided string comply Python identifier syntax requirements and False otherwise
|
||||
"""
|
||||
if version[0] >= 3:
|
||||
return s.isidentifier()
|
||||
else:
|
||||
# quick test on provided string to comply major Python identifier syntax requirements
|
||||
return (s and
|
||||
not s[:1].isdigit() and
|
||||
"-" not in s and
|
||||
" " not in s)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def ignored_os_errors(*errno):
|
||||
try:
|
||||
yield
|
||||
# Since Python 3.3 IOError and OSError were merged into OSError
|
||||
except EnvironmentError as e:
|
||||
if e.errno not in errno:
|
||||
raise
|
||||
|
||||
|
||||
def mkdir(path):
|
||||
try:
|
||||
os.makedirs(path)
|
||||
except EnvironmentError as e:
|
||||
if e.errno != errno.EEXIST or not os.path.isdir(path):
|
||||
raise
|
||||
|
||||
|
||||
def copy(src, dst, merge=False, pre_copy_hook=None, conflict_handler=None, post_copy_hook=None):
|
||||
if pre_copy_hook is None:
|
||||
def pre_copy_hook(p1, p2):
|
||||
return True
|
||||
|
||||
if conflict_handler is None:
|
||||
def conflict_handler(p1, p2):
|
||||
return False
|
||||
|
||||
if post_copy_hook is None:
|
||||
def post_copy_hook(p1, p2):
|
||||
pass
|
||||
|
||||
if not pre_copy_hook(src, dst):
|
||||
return
|
||||
|
||||
# Note about shutil.copy vs shutil.copy2.
|
||||
# There is an open CPython bug which breaks copy2 on NFS when it tries to copy the xattr.
|
||||
# https://bugs.python.org/issue24564
|
||||
# https://youtrack.jetbrains.com/issue/PY-37523
|
||||
# However, in all our use cases, we do not care about the xattr,
|
||||
# so just always use shutil.copy to avoid this problem.
|
||||
if os.path.isdir(src):
|
||||
if not merge:
|
||||
if version[0] >= 3:
|
||||
shutil.copytree(src, dst, copy_function=shutil.copy)
|
||||
else:
|
||||
shutil.copytree(src, dst)
|
||||
else:
|
||||
mkdir(dst)
|
||||
for child in os.listdir(src):
|
||||
child_src = os.path.join(src, child)
|
||||
child_dst = os.path.join(dst, child)
|
||||
try:
|
||||
copy(child_src, child_dst, merge=merge,
|
||||
pre_copy_hook=pre_copy_hook,
|
||||
conflict_handler=conflict_handler,
|
||||
post_copy_hook=post_copy_hook)
|
||||
except OSError as e:
|
||||
if e.errno == errno.EEXIST and not (os.path.isdir(child_src) and os.path.isdir(child_dst)):
|
||||
if conflict_handler(child_src, child_dst):
|
||||
continue
|
||||
raise
|
||||
else:
|
||||
mkdir(os.path.dirname(dst))
|
||||
shutil.copy(src, dst)
|
||||
post_copy_hook(src, dst)
|
||||
|
||||
|
||||
def copy_skeletons(src_dir, dst_dir, new_origin=None):
|
||||
def overwrite(src, dst):
|
||||
delete(dst)
|
||||
copy(src, dst)
|
||||
return True
|
||||
|
||||
# Remove packages/modules with the same import name
|
||||
def mod_pkg_cleanup(src, dst):
|
||||
dst_dir = os.path.dirname(dst)
|
||||
name, ext = os.path.splitext(os.path.basename(src))
|
||||
if ext == '.py':
|
||||
delete(os.path.join(dst_dir, name))
|
||||
elif not ext:
|
||||
delete(dst + '.py')
|
||||
|
||||
def override_origin_stamp(src, dst):
|
||||
_, ext = os.path.splitext(dst)
|
||||
if ext == '.py' and new_origin:
|
||||
with fopen(dst, 'r') as f:
|
||||
lines = f.readlines()
|
||||
for i, line in enumerate(lines):
|
||||
if not line.startswith('#'):
|
||||
return
|
||||
|
||||
m = SKELETON_HEADER_ORIGIN_LINE.match(line)
|
||||
if m:
|
||||
break
|
||||
else:
|
||||
return
|
||||
with fopen(dst, 'w') as f:
|
||||
lines[i] = '# from ' + new_origin + '\n'
|
||||
f.writelines(lines)
|
||||
|
||||
def post_copy_hook(src, dst):
|
||||
override_origin_stamp(src, dst)
|
||||
mod_pkg_cleanup(src, dst)
|
||||
|
||||
def ignore_failed_version_stamps(src, dst):
|
||||
return not os.path.basename(src).startswith(FAILED_VERSION_STAMP_PREFIX)
|
||||
|
||||
copy(src_dir, dst_dir, merge=True,
|
||||
pre_copy_hook=ignore_failed_version_stamps,
|
||||
conflict_handler=overwrite,
|
||||
post_copy_hook=post_copy_hook)
|
||||
|
||||
|
||||
def delete(path, content=False):
|
||||
with ignored_os_errors(errno.ENOENT):
|
||||
if os.path.isdir(path):
|
||||
if not content:
|
||||
shutil.rmtree(path)
|
||||
else:
|
||||
for child in os.listdir(path):
|
||||
delete(child)
|
||||
else:
|
||||
os.remove(path)
|
||||
|
||||
|
||||
def cached(func):
|
||||
func._results = {}
|
||||
unknown = object()
|
||||
|
||||
# noinspection PyProtectedMember
|
||||
@functools.wraps(func)
|
||||
def wrapper(*args):
|
||||
result = func._results.get(args, unknown)
|
||||
if result is unknown:
|
||||
result = func._results[args] = func(*args)
|
||||
return result
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
def sha256_digest(binary_or_file):
|
||||
# "bytes" type is available in Python 2.7
|
||||
if isinstance(binary_or_file, bytes):
|
||||
return hashlib.sha256(binary_or_file).hexdigest()
|
||||
else:
|
||||
acc = hashlib.sha256()
|
||||
while True:
|
||||
block = binary_or_file.read(BIN_READ_BLOCK)
|
||||
if not block:
|
||||
break
|
||||
acc.update(block)
|
||||
return acc.hexdigest()
|
||||
|
||||
|
||||
def get_portable_test_module_path(abs_path, qname):
|
||||
abs_path_components = os.path.normpath(abs_path).split(os.path.sep)
|
||||
qname_components_count = len(qname.split('.'))
|
||||
if os.path.splitext(abs_path_components[-1])[0] == '__init__':
|
||||
rel_path_components_count = qname_components_count + 1
|
||||
else:
|
||||
rel_path_components_count = qname_components_count
|
||||
return '/'.join(abs_path_components[-rel_path_components_count:])
|
||||
|
||||
|
||||
def is_text_file(path):
|
||||
"""
|
||||
Verify that some path is a text file (not a binary file).
|
||||
Ideally there should be usage of libmagic but it can be not
|
||||
installed on a target machine.
|
||||
|
||||
Actually this algorithm is inspired by function `file_encoding`
|
||||
from libmagic.
|
||||
"""
|
||||
try:
|
||||
with open(path, 'rb') as candidate_stream:
|
||||
# Buffer size like in libmagic
|
||||
buffer = candidate_stream.read(256 * 1024)
|
||||
except EnvironmentError:
|
||||
return False
|
||||
|
||||
# Verify that it looks like ASCII, UTF-8 or UTF-16.
|
||||
for encoding in 'utf-8', 'utf-16', 'utf-16-be', 'utf-16-le':
|
||||
try:
|
||||
buffer.decode(encoding)
|
||||
except UnicodeDecodeError as err:
|
||||
if err.args[0].endswith(('truncated data', 'unexpected end of data')):
|
||||
return True
|
||||
else:
|
||||
return True
|
||||
|
||||
# Verify that it looks like ISO-8859 or non-ISO extended ASCII.
|
||||
return all(c not in _bytes_that_never_appears_in_text for c in buffer)
|
||||
|
||||
|
||||
_bytes_that_never_appears_in_text = set(range(7)) | {11} | set(range(14, 27)) | set(range(28, 32)) | {127}
|
||||
|
||||
|
||||
# This wrapper is intentionally made top-level: local functions can't be pickled.
|
||||
def _multiprocessing_wrapper(data, func, *args, **kwargs):
|
||||
configure_logging(data.root_logger_level)
|
||||
data.result_conn.send(func(*args, **kwargs))
|
||||
|
||||
|
||||
_MainProcessData = collections.namedtuple('_MainProcessData', ['result_conn', 'root_logger_level'])
|
||||
|
||||
|
||||
def execute_in_subprocess_synchronously(name, func, args, kwargs, failure_result=None):
|
||||
import multiprocessing as mp
|
||||
|
||||
extra_process_kwargs = {}
|
||||
if sys.version_info[0] >= 3:
|
||||
extra_process_kwargs['daemon'] = True
|
||||
|
||||
# There is no need to use a full-blown queue for single producer/single consumer scenario.
|
||||
# Also, Pipes don't suffer from issues such as https://bugs.python.org/issue35797.
|
||||
# TODO experiment with a shared queue maintained by multiprocessing.Manager
|
||||
# (it will require an additional service process)
|
||||
recv_conn, send_conn = mp.Pipe(duplex=False)
|
||||
data = _MainProcessData(result_conn=send_conn,
|
||||
root_logger_level=logging.getLogger().level)
|
||||
p = mp.Process(name=name,
|
||||
target=_multiprocessing_wrapper,
|
||||
args=(data, func) + args,
|
||||
kwargs=kwargs,
|
||||
**extra_process_kwargs)
|
||||
p.start()
|
||||
# This is actually against the multiprocessing guidelines
|
||||
# https://docs.python.org/3/library/multiprocessing.html#programming-guidelines
|
||||
# but allows us to fail-fast if the child process terminated abnormally with a segfault
|
||||
# (otherwise we would have to wait by timeout on acquiring the result) and should work
|
||||
# fine for small result values such as generation status.
|
||||
p.join()
|
||||
if recv_conn.poll():
|
||||
return recv_conn.recv()
|
||||
else:
|
||||
return failure_result
|
||||
|
||||
|
||||
def configure_logging(root_level):
|
||||
logging.addLevelName(logging.DEBUG - 1, 'TRACE')
|
||||
|
||||
root = logging.getLogger()
|
||||
root.setLevel(root_level)
|
||||
|
||||
# In environments where fork is implemented entire logging configuration is already inherited by child processes.
|
||||
# Configuring it twice will lead to duplicated records.
|
||||
|
||||
# Reset logger similarly to how it's done in logging.config
|
||||
for h in root.handlers[:]:
|
||||
root.removeHandler(h)
|
||||
|
||||
for f in root.filters[:]:
|
||||
root.removeFilter(f)
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
def format(self, record):
|
||||
s = super(JsonFormatter, self).format(record)
|
||||
return json.dumps({
|
||||
'type': 'log',
|
||||
'level': record.levelname.lower(),
|
||||
'message': s
|
||||
})
|
||||
|
||||
handler = logging.StreamHandler(sys.stdout)
|
||||
handler.setFormatter(JsonFormatter())
|
||||
root.addHandler(handler)
|
||||
@@ -0,0 +1 @@
|
||||
1.147
|
||||
@@ -44,5 +44,6 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
|
||||
{\field{\*\fldinst{HYPERLINK "{\pntext\f1\'B7\tab}http://openwall.info/wiki/people/solar/software/public-domain-source-code/md5"}}{\fldrslt{\ul\cf1 http://openwall.info/wiki/people/solar/software/public-domain-source-code/md5}}}\f0\fs22\line md5 released to the Public Domain by Alexander Peslyak.\par
|
||||
{\field{\*\fldinst{HYPERLINK "{\pntext\f1\'B7\tab}https://developer.nvidia.com/nsight-perf-sdk"}}{\fldrslt{\ul\cf1 https://developer.nvidia.com/nsight-perf-sdk}}}\f0\fs22\line NVIDIA Nsight Perf SDK distributed under the NVIDIA Nsight Perf SDK license.\par
|
||||
{\field{\*\fldinst{HYPERLINK "{\pntext\f1\'B7\tab}https://github.com/python/pythoncapi-compat"}}{\fldrslt{\ul\cf1 https://github.com/python/pythoncapi-compat}}}\f0\fs22\line python2api-compat distributed under the BSD Zero Clause License. Copyright Contributors to the pythoncapi_compat project.\par
|
||||
{\field{\*\fldinst{HYPERLINK "{\pntext\f1\'B7\tab}https://github.com/JetBrains/intellij-community"}}{\fldrslt{\ul\cf1 https://github.com/JetBrains/intellij-community}}}\f0\fs22\line intellij-community distributed under the Apache License. Copyright Contributors to the intellij-community project.\par
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user