From e57d9db9c4cc4196b8347ea3f0dab5ea7af76899 Mon Sep 17 00:00:00 2001 From: baldurk Date: Thu, 2 Apr 2026 13:52:10 +0100 Subject: [PATCH] Process scope declarations and identifiers in AST tree * We don't generate type information yet, _get_type is stubbed out. --- .../Code/pyrenderdoc/parse_reflection.py | 503 ++++++++++++++++++ 1 file changed, 503 insertions(+) diff --git a/qrenderdoc/Code/pyrenderdoc/parse_reflection.py b/qrenderdoc/Code/pyrenderdoc/parse_reflection.py index 5e89569b0..c3f9da707 100644 --- a/qrenderdoc/Code/pyrenderdoc/parse_reflection.py +++ b/qrenderdoc/Code/pyrenderdoc/parse_reflection.py @@ -3,6 +3,80 @@ import builtins from typing import List, Dict, Any, Tuple, Callable, TypeVar, Optional +# return whether an object is a specialisation of the given generic, +# e.g. _is_generic(Dict, Dict[str, int]) == True +def _is_generic(generic, obj): + if hasattr(obj, "__origin__") and hasattr(generic, "__origin__"): + if generic.__origin__ == obj.__origin__: + return True + + if hasattr(obj, "__origin__"): + return obj.__origin__ == generic + + return False + + +# return true for AST nodes that need their own scope - this is module level, then +# classes and functions which may be nested inside each other +def _is_scope_node(node): + return ( + isinstance(node, ast.Module) + or isinstance(node, ast.ClassDef) + or isinstance(node, ast.FunctionDef) + or isinstance(node, ast.AsyncFunctionDef) + ) + + +# get all the return statements immediately inside a function without going into nested +# functions +def _get_return_statements(node: ast.AST, root: bool = True): + if _is_scope_node(node) and not root: + return [] + + if isinstance(node, ast.Return): + return [node] + + ret = [] + + for recurse in ["body", "orelse", "finalbody"]: + if hasattr(node, recurse): + for n in getattr(node, recurse): + if not _is_scope_node(n): + ret += _get_return_statements(n, False) + + return ret + + +# Python 3.8+ is expected to have start and end lines. +# Before that, end was missing so we assume all statements are single +# line (or the last child). +def _get_linerange(node: ast.AST): + if not hasattr(node, "lineno"): + return (-1, -1) + + lineno = getattr(node, "lineno") + + if hasattr(node, "end_lineno"): + return (lineno, getattr(node, "end_lineno")) + + end_lineno = lineno + for recurse in ["body", "orelse", "finalbody"]: + if hasattr(node, recurse) and len(getattr(node, recurse)) > 0: + end_lineno = max(end_lineno, _get_linerange(getattr(node, recurse)[-1])[1]) + + return (lineno, end_lineno) + + +# true if this is a `self.foo` type attribute lookup, so we know +# to look up in the parent scope which is how we handle self +def _is_self_lookup(parsed: ast.AST): + return ( + isinstance(parsed, ast.Attribute) + and isinstance(parsed.value, ast.Name) + and parsed.value.id == "self" + ) + + # comment out all lines starting from a given point, # to try and make things compile. Stops when it hits # an indent that looks like the end of the statement @@ -113,6 +187,9 @@ class PyReflector: # the parsed module, or None if parsing completely failed self.module: Optional[ast.Module] + # for tests - a lookup to retrieve the actual typevar since they can't be compared by name + self.user_types: Dict[str, TypeVar] = {} + # the text that was actually parsed, including any truncation/commenting needed # to get it to compile self.parsed_text: str @@ -130,9 +207,23 @@ class PyReflector: # Mostly for internal use self.debug_types = debug_types + # the current scope for each line + self.scopes: List[Scope] = [] + + # a pending FIFO which we process classes and functions in. This is used + # so that if we want to infer a type we can steal it early and process functions + # out of normal declaration order. Can't resolve mutually-recursive functions + # that require type guessing but improves many common situations where a class + # method calls another that's declared later and doesn't have proper type + # annotations + self.pending: List[Tuple[Optional[Scope], ast.AST]] = [] + # try to parse the text self._parse_text(text) + # This checks module against None internally to help type checkers + self._process_module() + def _parse_text(self, text: str): # try a simple parse. If there are no syntax errors this will # succeed. @@ -222,3 +313,415 @@ class PyReflector: # get the source string for a given line def get_line_source(self, line: int): return self.parsed_text.splitlines()[line - 1] + + def _type_failure(self, err: str): + if self.debug_types: + return "@@" + err.replace("@", "") + "@@" + return Any + + def _get_type(self, scope: Scope, parsed: Optional[ast.AST]): + return None + + # we can try to guess function return values by looking at the types of + # the return statements. If they are all the same (ignoring possible none) + # then use that. If they're different we give up as we don't handle union/ + # varied types + def _guess_return_value(self, scope: Scope, node: ast.AST): + ret_types = [ + self._get_type(scope, r.value) for r in _get_return_statements(node) + ] + ret_types = list(set([r for r in ret_types if r is not None])) + + if len(ret_types) == 1 and not isinstance(ret_types[0], str): + return ret_types[0] + return type(None) + + # for a single statement, process the identifiers it creates and recurse + # as needed (but not into new scopes) + def _process_stmt(self, parent: Optional[Scope], parsed: ast.AST): + if parent is None: + raise ValueError("Expected parent for non-module") + + # for functions and classes we register their type as an identifier and update + # scopes, but push them onto the pending list and continue processing. + if isinstance(parsed, ast.ClassDef): + classscope = Scope() + classscope.name = f"class {parsed.name}" + classscope.parent = parent + classscope.parsed = parsed + classscope.type_obj = TypeVar(parsed.name) # type: ignore + self.user_types[parsed.name] = classscope.type_obj + classscope.is_class = True + + r = _get_linerange(parsed) + + for line in range(r[0], r[1] + 1): + self.scopes[line] = classscope + + id = Ident() + id.line = parsed.lineno + id.type_obj = classscope.type_obj + classscope.ident = id + parent.set_ident(parsed.name, id) + + self.pending.append((classscope, parsed)) + + return + elif isinstance(parsed, ast.FunctionDef) or isinstance( + parsed, ast.AsyncFunctionDef + ): + funcscope = Scope() + funcscope.name = f"function {parsed.name}" + funcscope.parent = parent + funcscope.parsed = parsed + funcscope.is_class = False + + r = _get_linerange(parsed) + + for line in range(r[0], r[1] + 1): + self.scopes[line] = funcscope + + id = Ident() + id.line = parsed.lineno + id.type_obj = self._get_type(parent, parsed) + # if there's no return annotation, we'll try to guess it + # later when this function gets processed + if parsed.returns is None: + id.lazy_node = parsed + else: + # if we know the return type already and this is a @property + # then pretend it is just a member of that type, not a function + if any( + [ + isinstance(a, ast.Name) and a.id == "property" + for a in parsed.decorator_list + ] + ): + id.type_obj = self._get_type(parent, parsed.returns) + + prop_setter = False + + # if this one is a property setter, @self.setter, then + # don't add it as an ident + if any( + [ + isinstance(a, ast.Attribute) + and a.attr == "setter" + and isinstance(a.value, ast.Name) + and a.value.id == parsed.name + for a in parsed.decorator_list + ] + ): + prop_setter = True + + funcscope.ident = id + if not prop_setter: + parent.set_ident(parsed.name, id) + + args = parsed.args + arg_list = [] + if "posonlyargs" in args._fields: + arg_list += args.posonlyargs + arg_list += args.args + + # resize up the defaults array to the right size, defaults are 'trailing' + # ie. if there are fewer defaults than arguments, the first ones (starting + # from position-only arguments) have defaults omitted + defaults = [None] * (len(arg_list) - len(args.defaults)) + args.defaults + + # len(kwonlyargs) == len(kw_defaults) because keyword defaults can come in + # any order + arg_list += args.kwonlyargs + defaults += args.kw_defaults + + for i, a in enumerate(arg_list): + id = Ident() + id.line = parsed.lineno + default_val = defaults[i] + if a.annotation is not None: + id.type_obj = self._get_type(parent, a.annotation) + elif default_val is not None: + id.type_obj = self._get_type(parent, default_val) + elif ( + a in parsed.args.args + and parsed.args.args.index(a) == 0 + and a.arg == "self" + and parent.parent is not None + ): + id.type_obj = parent.type_obj + else: + if self.debug_types: + id.type_obj = self._type_failure( + f"Unknown parameter type {a.arg} in {parsed.name}" + ) + id.type_obj = Any + funcscope.set_ident(a.arg, id) + + self.pending.append((funcscope, parsed)) + + return + + # for imports, try to import the module ourselves so that we can have proper types. + # this won't work well for relative imports or things that need a particular sys.path + # but will work for standard library modules + if isinstance(parsed, ast.Import): + for alias in parsed.names: + n = alias.asname + if n is None or n == "": + n = alias.name + + id = Ident() + id.line = parsed.lineno + try: + id.type_obj = __import__(alias.name, globals(), locals()) + except ImportError: + if self.debug_types: + print(f"Couldn't import {alias.name}") + id.type_obj = Any + parent.set_ident(n, id) + + if isinstance(parsed, ast.ImportFrom): + module = None + if parsed.module is not None: + try: + module = __import__( + parsed.module, + globals(), + locals(), + [a.name for a in parsed.names], + parsed.level, + ) + except ImportError: + if self.debug_types: + print(f"Couldn't import {parsed.module}") + module = None + for alias in parsed.names: + n = alias.asname + if n is None or n == "": + n = alias.name + + id = Ident() + id.line = parsed.lineno + if module is None: + try: + id.type_obj = __import__( + alias.name, globals(), locals(), [], parsed.level + ) + except ImportError: + if self.debug_types: + print(f"Couldn't import {alias.name}") + id.type_obj = Any + else: + if hasattr(module, alias.name): + id.type_obj = getattr(module, alias.name) + else: + id.type_obj = Any + parent.set_ident(n, id) + + # global/nonlocal we ignore for now, we assume the type won't + # change with any assignments there + + targets = [] + values = [] + unpacking = False + ident_col = None + + # AugAssign doesn't create a new object, so ignore it + + # for other things that create a new identifier register both the set of + # target names and the source values. For pure assignments note the + # column where the LHS ends so that we can identify both possibly different + # types of `foo` in the statement `foo = foo.bar` + + if isinstance(parsed, ast.For) or isinstance(parsed, ast.AsyncFor): + if isinstance(parsed.target, ast.Tuple) or isinstance( + parsed.target, ast.List + ): + targets = parsed.target.elts + else: + targets = [parsed.target] + values = [parsed.iter] * len(targets) + unpacking = True + + # unless this actually creates a new name we don't have to do anything + if isinstance(parsed, ast.With) or isinstance(parsed, ast.AsyncWith): + for item in parsed.items: + if item.optional_vars is not None: + if isinstance(item.optional_vars, ast.Name): + targets += [item.optional_vars] + values += [item.context_expr] + elif isinstance(item.optional_vars, ast.Tuple): + unpacking = True + targets += item.optional_vars.elts + values += [item.context_expr] * len(item.optional_vars.elts) + + # for annotated assignments, trust the annotation and don't try to evaluate the + # actual RHS + if isinstance(parsed, ast.AnnAssign): + targets += [parsed.target] + values += [parsed.annotation] + if parsed.value is not None: + ident_col = parsed.value.col_offset + + if isinstance(parsed, ast.Assign): + if len(parsed.targets) == 1 and ( + isinstance(parsed.targets[0], ast.Tuple) + or isinstance(parsed.targets[0], ast.List) + ): + targets = parsed.targets[0].elts + unpacking = True + else: + targets = parsed.targets + values = [parsed.value] * len(targets) + if parsed.value is not None: + ident_col = parsed.value.col_offset + + # starred has no effect for our purposes + for i in range(len(targets)): + t = targets[i] + while isinstance(t, ast.Starred): + t = t.value + targets[i] = t + + # we were in control of these arrays so they should be identically sized + if len(targets) != len(values): + raise RuntimeError("Didn't get equal number of targets and values") + + for i in range(len(targets)): + t = targets[i] + + ident_scope = parent + if isinstance(t, ast.Name): + name = t.id + + if isinstance(t.ctx, ast.Load): + raise ValueError("Didn't expect loading target") + elif _is_self_lookup(t) and parent.parent is not None: + # just to help the type checked, is_self_lookup already checked this + if isinstance(t, ast.Attribute): + name = t.attr + ident_scope = parent.parent + + # walk up to the class, in case of nested functions + while ( + ident_scope.type_obj is None and ident_scope.parent is not None + ): + ident_scope = ident_scope.parent + + if isinstance(t.ctx, ast.Load): + raise ValueError("Didn't expect loading target") + else: + raise RuntimeError("invalid") + else: + # otherwise do nothing, this is an assignment of a value and we don't + # track fully dynamic types and attributes + continue + + id = Ident() + id.line = _get_linerange(parsed)[0] + v = values[i] + id.type_obj = self._get_type(parent, v) + + if unpacking: + t: Any = id.type_obj + if _is_generic(Tuple, t): + # either out of bounds, or a `Tuple[...]`, either way call it Any + if i < len(t.__args__): + id.type_obj = t.__args__[i] + else: + id.type_obj = Any + elif _is_generic(List, t): + id.type_obj = t.__args__[0] + else: + id.type_obj = t + + if ident_col is not None: + id.col = ident_col + ident_scope.set_ident(name, id) + + # should only get here for things like loops, ifs, etc NOT for classes and functions + if _is_scope_node(parsed): + raise TypeError("Should not be recursing for scope node") + + for recurse in ["body", "orelse", "finalbody"]: + if recurse in parsed._fields: + for e in getattr(parsed, recurse): + self._process_stmt(parent, e) + + # function to process the n'th item in the pending list. Usually 0 to + # continue processing in declaration order but can be out-of-order if we + # want to crystallise a guessed return type for a function in order to + # get a better type at an earlier callsite + def _process_pending(self, idx: int): + scope, node = self.pending.pop(idx) + + # node is a module, class, or function. Process all the identifiers in it + # and add any nested classes or functions to the pending list + if _is_scope_node(node): + for st in getattr(node, "body"): + self._process_stmt(scope, st) + + # for functions that want guessed return types (lazy_node is not None) + # do that now + if ( + scope is not None + and scope.ident is not None + and scope.ident.lazy_node is not None + ): + if scope.ident.type_obj is not None: + args = scope.ident.type_obj.__args__[0:-1] + ret_type = self._guess_return_value(scope, scope.ident.lazy_node) + + # if this function was a property, don't make a callable just set + # the return type + if isinstance(scope.ident.lazy_node, ast.FunctionDef) and any( + [ + isinstance(a, ast.Name) and a.id == "property" + for a in scope.ident.lazy_node.decorator_list + ] + ): + scope.ident.type_obj = ret_type + elif len(args) == 1 and args[0] == ...: + scope.ident.type_obj = Callable[(..., ret_type)] + else: + scope.ident.type_obj = Callable[[a for a in args], ret_type] + scope.ident.lazy_node = None + pass + else: + raise TypeError("Unexpected type of object in pending list") + + def _process_module(self): + if self.module is not None: + + # start with just the module + modscope = Scope() + modscope.name = "module" + modscope.parsed = self.module + modscope.parent = None + modscope.is_class = False + + # set all globals, ignoring reserved ones with __ prefix - so this can be + # globals() without needing extra filtering + for k, v in self.starting_globals.items(): + if k.startswith("__"): + continue + id = Ident() + id.line = 0 + id.type_obj = v + modscope.set_ident(k, id) + + # modules don't have line ranges, so go to the last entry in the body + if len(self.module.body) > 0: + r = _get_linerange(self.module.body[-1]) + else: + self.scopes = [modscope] + return + + # start with every line pointing to the module scope + self.scopes = [modscope] * (r[1] + 1) + + for e in self.module.body: + self._process_stmt(modscope, e) + + while len(self.pending) > 0: + self._process_pending(0)