diff --git a/src/application/javascript/JavaScriptArtifactAnalysis.ts b/src/application/javascript/JavaScriptArtifactAnalysis.ts index 72fb67edd3..fcba4cc165 100644 --- a/src/application/javascript/JavaScriptArtifactAnalysis.ts +++ b/src/application/javascript/JavaScriptArtifactAnalysis.ts @@ -2,7 +2,7 @@ import { createHash } from "node:crypto"; import { setImmediate } from "node:timers/promises"; import { resolveJavaScriptSourceMapReference } from "../../domain/javascript/javascriptSourceMapPaths.js"; -import { analyzeParsedJavaScriptStaticSource } from "../../domain/javascript/javascriptStaticAnalysis.js"; +import { analyzeParsedJavaScriptStaticSourceSteps } from "../../domain/javascript/javascriptStaticAnalysis.js"; import { analyzeParsedJavaScriptSemanticsSteps, classifyParsedJavaScriptOpenReceivers, @@ -190,7 +190,7 @@ function* analyzeArtifactFileSteps< } const openReceiverFacts = classifyParsedJavaScriptOpenReceivers(parsed); yield; - const analysis = analyzeParsedJavaScriptStaticSource( + const analysis = yield* analyzeParsedJavaScriptStaticSourceSteps( file.text.value, parsed, openReceiverFacts, diff --git a/src/domain/javascript/javascriptSemanticAnalysis.ts b/src/domain/javascript/javascriptSemanticAnalysis.ts index 50af0f2cbf..ff1dac49b9 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.ts @@ -33,7 +33,7 @@ import { } from "./javascriptSemanticState.js"; import { traverseJavaScriptAst } from "./javascriptSemanticTraversal.js"; import { semanticRequireOrigin } from "./javascriptSemanticRequireOrigin.js"; -import { collectSemanticMemberMutations } from "./javascriptSemanticMemberMutations.js"; +import { collectSemanticMemberMutationsSteps } from "./javascriptSemanticMemberMutations.js"; import { collectSemanticReturns, resolveSemanticModuleCallables, @@ -126,8 +126,7 @@ export function* analyzeParsedJavaScriptSemanticsSteps( const state = createState(file.program); collectDefinitions(file.program, state); yield; - collectSemanticMemberMutations(file.program, state); - yield; + yield* collectSemanticMemberMutationsSteps(file.program, state); traverseJavaScriptAst(file.program, { enter: (node) => collectSemanticModuleLink(node, state), }); diff --git a/src/domain/javascript/javascriptSemanticAnalysisSteps.test.ts b/src/domain/javascript/javascriptSemanticAnalysisSteps.test.ts index 9aaa71d935..7ae453914f 100644 --- a/src/domain/javascript/javascriptSemanticAnalysisSteps.test.ts +++ b/src/domain/javascript/javascriptSemanticAnalysisSteps.test.ts @@ -5,6 +5,10 @@ import { analyzeParsedJavaScriptSemanticsSteps, } from "./javascriptSemanticAnalysis.js"; import { parseJavaScriptSource } from "./javascriptSourceParser.js"; +import { + traverseJavaScriptAst, + traverseJavaScriptAstSteps, +} from "./javascriptSemanticTraversal.js"; const source = ` import { readFile } from "node:fs"; @@ -32,3 +36,26 @@ it("yields between semantic phases and returns the synchronous result (#1462)", if (reparsed === null) throw new Error("Expected parsed source"); expect(step.value).toEqual(analyzeParsedJavaScriptSemantics(reparsed)); }); + +it("traverses a large tree in steps with the synchronous visit order (#1462)", () => { + const parsed = parseJavaScriptSource( + Array.from( + { length: 400 }, + (_, index) => `const v${index} = [${index}];`, + ).join("\n"), + "large.js", + ); + if (parsed === null) throw new Error("Expected parsed source"); + const visits = (record: string[]) => ({ + enter: (node: { readonly type: string }) => record.push(`+${node.type}`), + exit: (node: { readonly type: string }) => record.push(`-${node.type}`), + }); + const synchronous: string[] = []; + traverseJavaScriptAst(parsed.program, visits(synchronous)); + const stepped: string[] = []; + const steps = traverseJavaScriptAstSteps(parsed.program, visits(stepped)); + let yields = 0; + while (steps.next().done !== true) yields += 1; + expect(yields).toBeGreaterThan(0); + expect(stepped).toEqual(synchronous); +}); diff --git a/src/domain/javascript/javascriptSemanticDerivedAnalysis.ts b/src/domain/javascript/javascriptSemanticDerivedAnalysis.ts index 545652a744..b30d368592 100644 --- a/src/domain/javascript/javascriptSemanticDerivedAnalysis.ts +++ b/src/domain/javascript/javascriptSemanticDerivedAnalysis.ts @@ -3,7 +3,7 @@ import type * as t from "@babel/types"; import { collectJavaScriptSemanticAsyncEffects } from "./javascriptSemanticAsyncEffects.js"; import { collectJavaScriptSemanticCalls } from "./javascriptSemanticCalls.js"; import { collectJavaScriptSemanticChildProcesses } from "./javascriptSemanticChildProcesses.js"; -import { collectJavaScriptSemanticFingerprints } from "./javascriptSemanticFingerprints.js"; +import { collectJavaScriptSemanticFingerprintsSteps } from "./javascriptSemanticFingerprints.js"; import { collectJavaScriptSemanticDataEffects } from "./javascriptSemanticDataEffects.js"; import type { JavaScriptSemanticCallable, @@ -84,19 +84,20 @@ export function* collectJavaScriptDerivedSemanticsSteps( callables, ); yield; - const functionFingerprints = collectJavaScriptSemanticFingerprints({ - state, - callables, - calls, - promises: promiseOperations, - events: asyncEffects.eventOperations, - timers: asyncEffects.timerOperations, - childProcessSpawns: childProcesses.childProcessSpawns, - childProcessInteractions: childProcesses.childProcessInteractions, - requestOperations: dataEffects.requestOperations, - resourceOperations, - parserPartial, - }); + const functionFingerprints = + yield* collectJavaScriptSemanticFingerprintsSteps({ + state, + callables, + calls, + promises: promiseOperations, + events: asyncEffects.eventOperations, + timers: asyncEffects.timerOperations, + childProcessSpawns: childProcesses.childProcessSpawns, + childProcessInteractions: childProcesses.childProcessInteractions, + requestOperations: dataEffects.requestOperations, + resourceOperations, + parserPartial, + }); return { ...calls, promiseOperations, diff --git a/src/domain/javascript/javascriptSemanticFingerprints.ts b/src/domain/javascript/javascriptSemanticFingerprints.ts index 3f73090296..3d4b828bea 100644 --- a/src/domain/javascript/javascriptSemanticFingerprints.ts +++ b/src/domain/javascript/javascriptSemanticFingerprints.ts @@ -17,7 +17,7 @@ import type { } from "./javascriptSemanticIr.js"; import { semanticCallableIdForNode } from "./javascriptSemanticProjection.js"; import type { JavaScriptSemanticAnalysisState } from "./javascriptSemanticState.js"; -import { traverseJavaScriptAst } from "./javascriptSemanticTraversal.js"; +import { traverseJavaScriptAstSteps } from "./javascriptSemanticTraversal.js"; interface SemanticFingerprintCollectionInput { readonly state: JavaScriptSemanticAnalysisState; @@ -33,10 +33,10 @@ interface SemanticFingerprintCollectionInput { readonly parserPartial: boolean; } -/** Build bounded rename- and formatting-resistant components per callable. */ -export const collectJavaScriptSemanticFingerprints = ( +/** Build bounded rename- and formatting-resistant components per callable, yielding between callables. */ +export function* collectJavaScriptSemanticFingerprintsSteps( input: SemanticFingerprintCollectionInput, -): JavaScriptSemanticFunctionFingerprint[] => { +): Generator { const { state, callables, @@ -67,11 +67,13 @@ export const collectJavaScriptSemanticFingerprints = ( ); const networkEffectsByOwner = groupByOwner(requestOperations, ownerId); const resourceEffectsByOwner = groupByOwner(resourceOperations, ownerId); - return callables.flatMap((callable) => { - if (callable.kind === "class") return []; + const fingerprints: JavaScriptSemanticFunctionFingerprint[] = []; + for (const callable of callables) { + if (callable.kind === "class") continue; const node = state.callableNodesById.get(callable.callableId); - if (node === undefined) return []; - const syntax = fingerprintSyntax(node, callable.callableId); + if (node === undefined) continue; + yield; + const syntax = yield* fingerprintSyntaxSteps(node, callable.callableId); const promiseEffects = promiseEffectsByOwner.get(callable.callableId) ?? []; const eventEffects = eventEffectsByOwner.get(callable.callableId) ?? []; const timerEffects = timerEffectsByOwner.get(callable.callableId) ?? []; @@ -80,50 +82,49 @@ export const collectJavaScriptSemanticFingerprints = ( const networkEffects = networkEffectsByOwner.get(callable.callableId) ?? []; const resourceEffects = resourceEffectsByOwner.get(callable.callableId) ?? []; - return [ - { - callableId: callable.callableId, - status: - parserPartial || callable.returnCoverage.status !== "complete" - ? "partial" - : "complete", - components: { - parameterArity: callableParameterArity(node), - normalizedAstSha256: digest(syntax.normalizedTokens), - controlFlowSha256: digest(syntax.controlFlowTokens), - relationShapeSha256: digest( - relationShape( - callable, - callSitesByCaller.get(callable.callableId) ?? [], - captureCountByCallable.get(callable.callableId) ?? 0, - promiseEffects, - ), + fingerprints.push({ + callableId: callable.callableId, + status: + parserPartial || callable.returnCoverage.status !== "complete" + ? "partial" + : "complete", + components: { + parameterArity: callableParameterArity(node), + normalizedAstSha256: digest(syntax.normalizedTokens), + controlFlowSha256: digest(syntax.controlFlowTokens), + relationShapeSha256: digest( + relationShape( + callable, + callSitesByCaller.get(callable.callableId) ?? [], + captureCountByCallable.get(callable.callableId) ?? 0, + promiseEffects, ), - literalSetSha256: digest( - [...new Set(syntax.literals)].sort(compareUnicodeCodePoints), - ), - effects: [ - ...(isAsyncCallable(node) ? (["async"] as const) : []), - ...(childProcessEffects.length > 0 - ? (["child-process"] as const) - : []), - ...(eventEffects.length > 0 ? (["event"] as const) : []), - ...(networkEffects.length > 0 ? (["network"] as const) : []), - ...(promiseEffects.length > 0 ? (["promise"] as const) : []), - ...(resourceEffects.length > 0 ? (["resource"] as const) : []), - ...(timerEffects.length > 0 ? (["timer"] as const) : []), - ], - }, - limitations: [ - "The fingerprint is a static candidate and does not prove behavioral equivalence.", - ...(parserPartial - ? ["Incomplete semantic recovery makes this fingerprint partial."] + ), + literalSetSha256: digest( + [...new Set(syntax.literals)].sort(compareUnicodeCodePoints), + ), + effects: [ + ...(isAsyncCallable(node) ? (["async"] as const) : []), + ...(childProcessEffects.length > 0 + ? (["child-process"] as const) : []), + ...(eventEffects.length > 0 ? (["event"] as const) : []), + ...(networkEffects.length > 0 ? (["network"] as const) : []), + ...(promiseEffects.length > 0 ? (["promise"] as const) : []), + ...(resourceEffects.length > 0 ? (["resource"] as const) : []), + ...(timerEffects.length > 0 ? (["timer"] as const) : []), ], }, - ]; - }); -}; + limitations: [ + "The fingerprint is a static candidate and does not prove behavioral equivalence.", + ...(parserPartial + ? ["Incomplete semantic recovery makes this fingerprint partial."] + : []), + ], + }); + } + return fingerprints; +} const ownerId = ( value: T, @@ -161,15 +162,15 @@ interface FingerprintSyntax { readonly literals: readonly string[]; } -const fingerprintSyntax = ( +function* fingerprintSyntaxSteps( root: t.Node, targetCallableId: string, -): FingerprintSyntax => { +): Generator { const normalizedTokens: string[] = []; const controlFlowTokens: string[] = []; const literals: string[] = []; const callableStack: string[] = []; - traverseJavaScriptAst(root, { + yield* traverseJavaScriptAstSteps(root, { enter: (node, parent) => { const callableId = semanticCallableIdForNode(node); if (callableId !== null) callableStack.push(callableId); @@ -187,7 +188,7 @@ const fingerprintSyntax = ( }, }); return { normalizedTokens, controlFlowTokens, literals }; -}; +} const normalizedNodeToken = (node: t.Node, parent: t.Node | null): string => { if (t.isIdentifier(node)) diff --git a/src/domain/javascript/javascriptSemanticMemberMutations.ts b/src/domain/javascript/javascriptSemanticMemberMutations.ts index 06a335bcac..12ceec71d0 100644 --- a/src/domain/javascript/javascriptSemanticMemberMutations.ts +++ b/src/domain/javascript/javascriptSemanticMemberMutations.ts @@ -25,7 +25,7 @@ import { type JavaScriptSemanticPropertyPath, } from "./javascriptSemanticPropertyPaths.js"; import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js"; -import { traverseJavaScriptAst } from "./javascriptSemanticTraversal.js"; +import { traverseJavaScriptAstSteps } from "./javascriptSemanticTraversal.js"; import { semanticStaticPropertyKey, unwrapJavaScriptExpression, @@ -34,13 +34,13 @@ import { type PropertyPath = JavaScriptSemanticPropertyPath; type ValueEffect = "write" | "escape"; -/** Preserve uncertainty from property writes and references exposed to calls. */ -export const collectSemanticMemberMutations = ( +/** Preserve uncertainty from property writes and references exposed to calls, yielding during traversal and fixpoint iteration. */ +export function* collectSemanticMemberMutationsSteps( program: t.Program, state: JavaScriptSemanticAnalysisState, -): void => { +): Generator { const parents = state.parentsByNode; - traverseJavaScriptAst(program, { + yield* traverseJavaScriptAstSteps(program, { enter: (node, parent) => { if (parent !== null) parents.set(node, parent); }, @@ -382,7 +382,7 @@ export const collectSemanticMemberMutations = ( ); } }; - traverseJavaScriptAst(program, { + yield* traverseJavaScriptAstSteps(program, { enter: (node, parent) => { for (const source of semanticIterationSources(node, parent)) deferIterable({ @@ -435,7 +435,8 @@ export const collectSemanticMemberMutations = ( const effectsBefore = recordedEffects.size; const iterablesBefore = iterableReferences.length; const iterationBefore: boolean = arrayIterationUnknown; - for (const reference of pendingReferences.splice(0)) { + for (const [index, reference] of pendingReferences.splice(0).entries()) { + if (index % 64 === 0) yield; const { initializer } = reference; if (referenceSourcesInactive(initializer, state)) pendingReferences.push(reference); @@ -480,7 +481,8 @@ export const collectSemanticMemberMutations = ( reference.originAt, ); } - for (const reference of [...iterableReferences]) { + for (const [index, reference] of [...iterableReferences].entries()) { + if (index % 64 === 0) yield; if (referenceSourcesInactive(reference, state)) continue; if (reference.iterationOnly) { const expanded = expandedIterationReferences(reference, state); @@ -518,7 +520,7 @@ export const collectSemanticMemberMutations = ( // Gathering effects can evaluate a primitive projection before its object // escapes. Final values must use the completed mutation state. clearSemanticPrimitiveBindingValues(state); -}; +} interface ReferencedValue { readonly node: t.Node; diff --git a/src/domain/javascript/javascriptSemanticTraversal.ts b/src/domain/javascript/javascriptSemanticTraversal.ts index 9b562a94c2..16960ba8fe 100644 --- a/src/domain/javascript/javascriptSemanticTraversal.ts +++ b/src/domain/javascript/javascriptSemanticTraversal.ts @@ -12,15 +12,28 @@ import * as t from "@babel/types"; */ export const traverseJavaScriptAst = ( root: t.Node, - visitor: { - readonly enter: ( - node: t.Node, - parent: t.Node | null, - readAncestors: () => readonly t.Node[], - ) => void; - readonly exit?: (node: t.Node, parent: t.Node | null) => void; - }, -): void => { + visitor: JavaScriptAstVisitor, +): void => completeSemanticSteps(traverseJavaScriptAstSteps(root, visitor)); + +/** Callbacks for one deterministic AST traversal. */ +export interface JavaScriptAstVisitor { + readonly enter: ( + node: t.Node, + parent: t.Node | null, + readAncestors: () => readonly t.Node[], + ) => void; + readonly exit?: (node: t.Node, parent: t.Node | null) => void; +} + +/** + * {@link traverseJavaScriptAst} with a yield every 1,024 visited nodes, so a + * caller can serve control messages during one large traversal. Visit order + * and callbacks are identical. + */ +export function* traverseJavaScriptAstSteps( + root: t.Node, + visitor: JavaScriptAstVisitor, +): Generator { const ancestors: t.Node[] = []; const enter = (node: t.Node, parent: t.Node | null): TraversalFrame => { visitor.enter(node, parent, () => [...ancestors]); @@ -28,12 +41,15 @@ export const traverseJavaScriptAst = ( return { node, parent, children: childNodes(node), nextIndex: 0 }; }; const pending = [enter(root, null)]; + let visited = 1; while (pending.length > 0) { const current = pending.at(-1); if (current === undefined) break; const child = current.children[current.nextIndex]; if (child !== undefined) { current.nextIndex += 1; + if (visited % 1024 === 0) yield; + visited += 1; pending.push(enter(child, current.node)); } else { visitor.exit?.(current.node, current.parent); @@ -41,6 +57,16 @@ export const traverseJavaScriptAst = ( pending.pop(); } } +} + +/** Run cooperative analysis steps to completion without yielding control. */ +export const completeSemanticSteps = ( + steps: Iterator, +): Value => { + for (;;) { + const step = steps.next(); + if (step.done === true) return step.value; + } }; interface TraversalFrame { diff --git a/src/domain/javascript/javascriptStaticAnalysis.ts b/src/domain/javascript/javascriptStaticAnalysis.ts index 524af811eb..264c94182c 100644 --- a/src/domain/javascript/javascriptStaticAnalysis.ts +++ b/src/domain/javascript/javascriptStaticAnalysis.ts @@ -1,6 +1,9 @@ import * as t from "@babel/types"; -import { traverseJavaScriptAst } from "./javascriptSemanticTraversal.js"; +import { + completeSemanticSteps, + traverseJavaScriptAstSteps, +} from "./javascriptSemanticTraversal.js"; import { inspectElectronStaticNode } from "./electronStaticAnalysis.js"; import { @@ -57,9 +60,19 @@ export const analyzeParsedJavaScriptStaticSource = ( source: string, file: ParsedJavaScriptSource, openReceiverFacts?: ReadonlyMap, -): JavaScriptStaticAnalysis => { +): JavaScriptStaticAnalysis => + completeSemanticSteps( + analyzeParsedJavaScriptStaticSourceSteps(source, file, openReceiverFacts), + ); + +/** Static analysis that yields during its source traversal; results are unchanged. */ +export function* analyzeParsedJavaScriptStaticSourceSteps( + source: string, + file: ParsedJavaScriptSource, + openReceiverFacts?: ReadonlyMap, +): Generator { const accumulator = createJavaScriptAnalysisAccumulator(source.length); - traverseStaticSource(file, { + yield* traverseStaticSourceSteps(file, { source, accumulator, ...(openReceiverFacts === undefined ? {} : { openReceiverFacts }), @@ -67,20 +80,20 @@ export const analyzeParsedJavaScriptStaticSource = ( }); addSourceMapDirectives(source, file.comments ?? [], accumulator); return finalizeStaticAnalysis(source, file, accumulator); -}; +} -const traverseStaticSource = ( +function* traverseStaticSourceSteps( file: ParsedJavaScriptSource, findings: JavaScriptFindingContext, -): void => { - traverseJavaScriptAst(file, { +): Generator { + yield* traverseJavaScriptAstSteps(file, { enter: (node) => { findings.accumulator.visitedNodes += 1; inspectNode(node, findings); return undefined; }, }); -}; +} const finalizeStaticAnalysis = ( source: string,