Files
rea/scripts/lib/module-boundaries.mjs
2c3dd946d5 feat(python): inspect CPython pyc headers and marshal code objects (#4826)
* feat(python): inspect CPython pyc headers and marshal code objects

Add inspect_python_pyc / inspect-python-pyc to read one explicit local
CPython 3.7–3.13 .pyc without executing bytecode or recovering source.

* chore(python): drop unused PythonPycHeader export

Knip reported the type as unused after inspect_python_pyc landed.

* fix(python): bound pyc expansion and decode marshal like CPython

- Count a nested code object reached through TYPE_REF once per copy the
  report carries, against max_objects and max_file_bytes, before counting
  or validating the tree
- Charge long digits to max_string_bytes, reject out-of-range and
  unnormalized digits, build the value in one pass, and report integers
  above 256 bits by bit length only
- Decode TYPE_UNICODE with surrogatepass semantics and reject other
  ill-formed UTF-8
- Map only 3.11 magics 3494-3495 to the py311 layout
- Report non-finite float and complex parts as inf, -inf or nan
- Allow an empty co_filename
- Require bytes for co_linetable and co_exceptiontable
- Bind cleanup-failure errors to their retained descriptor leases

* fix(python): list pyc code objects once instead of a recursive tree

Report each distinct code object once in code_objects, top-level first,
with nested_code as indexes. A child shared through TYPE_REF is no longer
copied per reference, and the MCP output schema is no longer recursive.
Names repeated through TYPE_REF still count against max_file_bytes for
each copy the report carries.

* fix(python): bound shared pyc constants and reject malformed tables

- Cache byte-string digests so a buffer shared through TYPE_REF is
  hashed once instead of on every reference.
- Charge every reported co_consts entry, including tables shared
  through TYPE_REF, against max_objects before summaries are built.
- Resolve the inspect-python-pyc path with absoluteCallerPath so
  link/../file.pyc reads the file the filesystem selects.
- Reject TYPE_NULL inside marshal sequences, as CPython does.
- Reject a co_localspluskinds length that does not match
  co_localsplusnames in the parser and the report schema.

* fix(python): bound shared pyc names and read CPython marshal graphs

Charge each reported code-constant name, keep integer bit lengths on the
decoded value, and list code objects nested inside constant containers.
Accept self-referential lists and dicts, marshal version 0 text floats,
empty code names, the original string byte count, and Python set and dict
lengths. Pass the CLI cancellation signal into command cleanup.

Co-authored-by: morluto <morluto@users.noreply.github.com>

* fix(python): bound pyc key expansion and shared nested code

Charge set and dictionary keys before a shared tuple doubles them, list each
nested code object once per owner, and accept code objects as CPython keys.

Co-authored-by: morluto <morluto@users.noreply.github.com>

* fix(python): correct pyc equality, counts, and nested walks

Summarize code objects only after the marshal graph is complete, and
collect nested code with an iterative walk that shares wrapper results
instead of copying every descendant list. Keep tuple boundaries and the
original string bytes in equality keys, and reject negative CPython counts.

Co-authored-by: morluto <morluto@users.noreply.github.com>

* fix(python): bound shared pyc names and keep the first dict key

Charge every copied name-table entry against the object budget before
building the report array, including empty strings that add no bytes.
When dictionary keys compare equal, keep the first key object and
replace only its value, matching CPython.

Co-authored-by: morluto <morluto@users.noreply.github.com>

* fix(python): walk each pyc dictionary child list once

Keep a container's child array on the nested-code frame so a wide
dictionary is not copied on every step of the synchronous marshal walk.

Co-authored-by: morluto <morluto@users.noreply.github.com>

* fix(python): reuse stored pyc dictionary children when combining

The nested walk keeps each container's child list on its frame.
Combining nested code rebuilt that list, and the 8,000-key regression
still accepted the old per-key copy. Use the stored list, and require
that dictionary to finish well under the copy's runtime while still
reporting a nested code object.

Co-authored-by: morluto <morluto@users.noreply.github.com>

* fix(python): keep distinct pyc code objects and reject corrupt ones

Encode each name table's length and entry boundaries in the equality
key so a different name list cannot share a memo key. Walk tuple and
frozenset keys on an explicit stack so a long TYPE_REF chain does not
overflow. Treat code objects CPython rejects for an odd co_code or an
undersized local table as malformed instead of complete coverage.

Co-authored-by: morluto <morluto@users.noreply.github.com>

---------

Co-authored-by: Matt Van Horn <455140+mvanhorn@users.noreply.github.com>
Co-authored-by: Cursor Agent <cursoragent@cursor.com>
Co-authored-by: morluto <morluto@users.noreply.github.com>
2026-10-22 19:00:08 +08:00

141 lines
4.2 KiB
JavaScript

import { parse } from "@babel/parser";
import { traverseFast } from "@babel/types";
import { isAbsolute, relative, resolve } from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
const PURE_LAYERS = new Set(["domain", "contracts"]);
const MIGRATED_PROVIDER_ROOTS = new Set([
"sqlite",
"evm",
"go",
"il2cpp",
"model",
"gpu",
"python",
"rust",
"wasm",
"captures",
"signals",
"android",
"firmware",
"ghidra",
"hopper",
"ida",
"inspector",
"javascript",
]);
const repositoryPath = (root, path) =>
relative(root, path).replaceAll("\\", "/");
const literalValue = (node) => {
if (node?.type === "StringLiteral") return node.value;
if (node?.type === "TemplateLiteral" && node.expressions.length === 0)
return node.quasis[0]?.value.cooked ?? undefined;
return undefined;
};
const importSpecifier = (node) => {
switch (node.type) {
case "ImportDeclaration":
case "ExportNamedDeclaration":
case "ExportAllDeclaration":
case "ImportExpression":
case "TSImportType":
return literalValue(node.source);
case "TSImportEqualsDeclaration":
return node.moduleReference.type === "TSExternalModuleReference"
? literalValue(node.moduleReference.expression)
: undefined;
default:
return undefined;
}
};
const sourceTarget = (file, specifier) => {
if (specifier.startsWith("file:")) return fileURLToPath(specifier);
if (isAbsolute(specifier)) return resolve(specifier);
return specifier.startsWith(".")
? fileURLToPath(new URL(specifier, pathToFileURL(file)))
: undefined;
};
const failedBoundary = (file, target) => {
if (!file.startsWith("src/") || !target.startsWith("src/")) return undefined;
const layer = file.split("/")[1];
const targetLayer = target.split("/")[1];
if (
file.startsWith("src/process/capture/") &&
(["application", "composition", "server", "cli", "main"].includes(
targetLayer,
) ||
/^src\/(?:cli|main)\./u.test(target))
)
return "process-capture";
if (
file.startsWith("src/application/javascript/") &&
/^src\/artifacts\/(?:Asar|Directory)ArtifactReader\.(?:js|ts)$/u.test(
target,
)
)
return "provider-construction";
if (
(file.startsWith("src/artifacts/javascript/") ||
file.startsWith("src/artifacts/apple/") ||
file.startsWith("src/artifacts/inventory/") ||
file.startsWith("src/artifacts/extraction/") ||
/^src\/artifacts\/ArtifactHash\.(?:js|ts)$/u.test(file)) &&
(["application", "composition", "server", "cli", "main"].includes(
targetLayer,
) ||
/^src\/(?:cli|main)\./u.test(target))
)
return "artifact-acquisition";
if (PURE_LAYERS.has(layer)) {
if (
(/^src\/[^/]+\//u.test(target) && !PURE_LAYERS.has(targetLayer)) ||
/^src\/(?:cli|main)\./u.test(target)
)
return "pure-layer";
}
if (layer === "application" && targetLayer === "composition")
return "application-composition";
if (
(layer === "application" || layer === "server") &&
(MIGRATED_PROVIDER_ROOTS.has(targetLayer) ||
/^src\/(?:artifacts\/ArtifactProvider|dotnet\/ManagedStaticProvider|native\/NativeMacOSProvider)\.[^/]+$/u.test(
target,
) ||
/^src\/browser\/[^/]*Provider\.[^/]+$/u.test(target))
)
return "provider-construction";
return undefined;
};
/** Check resolved source ownership, including type imports and reexports. */
export const inspectModuleBoundaries = (file, source, root) => {
const absoluteFile = resolve(root, file);
const importer = repositoryPath(root, absoluteFile);
const tree = parse(source, {
sourceType: "module",
plugins: ["typescript"],
});
const violations = [];
traverseFast(tree, (node) => {
const specifier = importSpecifier(node);
if (specifier === undefined) return;
const resolved = sourceTarget(absoluteFile, specifier);
if (resolved === undefined) return;
const target = repositoryPath(root, resolved);
const boundary = failedBoundary(importer, target);
if (boundary !== undefined)
violations.push({
file: importer,
line: node.loc?.start.line ?? 1,
specifier,
target,
boundary,
});
});
return violations;
};