diff --git a/docs/python-pyc.md b/docs/python-pyc.md index c2674a8cce..33942187fa 100644 --- a/docs/python-pyc.md +++ b/docs/python-pyc.md @@ -50,7 +50,9 @@ names, locals (or `localsplusnames` on 3.11+), const summaries, and bytecode length plus SHA-256. Code-object equality used by sets and dictionaries follows the selected layout: -3.7–3.10 ignore line tables, while 3.11+ include line and exception tables. The +3.7–3.10 derive `CO_NOFREE` from the free/cell tables for equality, while +3.11+ compare raw flags; raw `co_flags` remain in reported facts. The older +layouts ignore line tables, while 3.11+ include line and exception tables. The artifact SHA-256 identifies the exact selected `.pyc` bytes independently of this decoded object graph. Reference slots follow each tag's CPython behavior: singleton values and `TYPE_REF` do not reserve new slots even when their tag diff --git a/src/python/PythonPycParser.ts b/src/python/PythonPycParser.ts index 6698323ace..f27a9d1819 100644 --- a/src/python/PythonPycParser.ts +++ b/src/python/PythonPycParser.ts @@ -56,6 +56,8 @@ const TYPES_WITHOUT_REFERENCE_SLOTS = new Set([ /** CPython `CO_VARARGS` / `CO_VARKEYWORDS`. Each consumes one local name. */ const CO_VARARGS = 0x04; const CO_VARKEYWORDS = 0x08; +/** CPython 3.7–3.10 derives this bit from the free/cell variable tables. */ +const CO_NOFREE = 0x40; /** CPython `_PyLocals_Kind` bit counted by `_PyCode_Validate` as a local. */ const CO_FAST_LOCAL = 0x20; @@ -149,6 +151,7 @@ const containerChildren = (value: MarshalValue): readonly MarshalValue[] => { */ type CodeEquality = { readonly consts: readonly MarshalValue[]; + readonly effectiveFlags: number; readonly linetable: string; readonly exceptiontable: string; }; @@ -520,8 +523,15 @@ class MarshalReader { consts: [], nested: [], }; + const effectiveFlags = + this.layout === "py311" + ? flags + : (freevars?.length ?? 0) === 0 && (cellvars?.length ?? 0) === 0 + ? flags | CO_NOFREE + : flags & ~CO_NOFREE; this.#codeEquality.set(node, { consts, + effectiveFlags, linetable: `${linetable.length}:${sha256(linetable)}`, exceptiontable: `${exceptiontable.length}:${sha256(exceptiontable)}`, }); @@ -1176,7 +1186,7 @@ class MarshalReader { String(node.posonlyargcount ?? 0), String(node.kwonlyargcount), node.nlocals === null ? "" : String(node.nlocals), - String(node.flags), + String(material.effectiveFlags), String(node.firstlineno), `${node.bytecode.bytes}:${node.bytecode.sha256}`, this.#constKeyItems(material.consts), diff --git a/tests/acceptance/pythonPycCli.test.ts b/tests/acceptance/pythonPycCli.test.ts index 352d1a9fb8..ef55f2b98b 100644 --- a/tests/acceptance/pythonPycCli.test.ts +++ b/tests/acceptance/pythonPycCli.test.ts @@ -298,6 +298,116 @@ cliTest( }, ); +cliTest( + "uses constructor-normalized legacy flags for code keys and preserves raw flags", + async ({ cli }) => { + const root = await createTestTempDirectory( + "rea-python-pyc-effective-flags-", + ); + const cases = [ + { + name: "py37-empty-tables", + magic: 3394, + layout: "py37" as const, + freevars: [], + cellvars: [], + lengths: [1, 1], + }, + { + name: "py310-nonempty-tables", + magic: 3439, + layout: "py38" as const, + freevars: ["capture"], + cellvars: [], + lengths: [1, 1], + }, + { + name: "py311-raw-flags", + magic: 3531, + layout: "py311" as const, + freevars: [], + cellvars: [], + lengths: [2, 2], + }, + ]; + for (const testCase of cases) { + const code = (flags: number) => + marshalCode(testCase.layout, { + name: "f", + filename: "fixture.py", + firstlineno: 1, + flags, + freevars: testCase.freevars, + cellvars: testCase.cellvars, + }); + const first = code(0); + const second = code(0x40); + const frozenSet = Buffer.concat([ + Buffer.from([0x3e]), + Buffer.from([2, 0, 0, 0]), + first, + second, + ]); + const dict = Buffer.concat([ + Buffer.from([0x7b]), + first, + Buffer.from([0x4e]), + second, + Buffer.from([0x54, 0x30]), + ]); + const path = join(root, `${testCase.name}.pyc`); + await writeFile( + path, + Buffer.concat([ + pythonPycHeader(testCase.magic), + marshalCode(testCase.layout, { consts: [frozenSet, dict] }), + ]), + ); + const result = await cli.run({ + arguments: ["inspect-python-pyc", path, "--json"], + }); + expect(result.exitCode, result.stderr).toBe(0); + const evidence = toolContract("inspect_python_pyc").outputSchema.parse( + result.json, + ); + expect( + evidence.normalized_result.code_objects[0]?.consts.map((value) => + value.kind === "frozenset" || value.kind === "dict" + ? value.length + : null, + ), + ).toEqual(testCase.lengths); + expect( + evidence.normalized_result.code_objects + .filter((object) => object.name === "f") + .map((object) => object.flags), + ).toContain(0); + + const rawPath = join(root, `raw-${testCase.name}.pyc`); + await writeFile( + rawPath, + Buffer.concat([ + pythonPycHeader(testCase.magic), + marshalCode(testCase.layout, { + name: "f", + flags: 0x40, + freevars: testCase.freevars, + cellvars: testCase.cellvars, + }), + ]), + ); + const rawResult = await cli.run({ + arguments: ["inspect-python-pyc", rawPath, "--json"], + }); + expect(rawResult.exitCode, rawResult.stderr).toBe(0); + const rawEvidence = toolContract("inspect_python_pyc").outputSchema.parse( + rawResult.json, + ); + expect(rawEvidence.normalized_result.code_objects[0]?.flags).toBe(0x40); + } + }, +); + cliTest( "resolves a relative pyc path against the working directory", async ({ cli }) => {