From 5d6eeb22fb3bc005163e3c680cf524ba7659b48a Mon Sep 17 00:00:00 2001 From: Kaoru0822 Date: Sat, 10 Oct 2026 05:27:41 -0700 Subject: [PATCH] fix(javascript): yield during the largest AST traversals (#1545) After #1542 the longest event-loop blocks on large bundles were single AST traversals inside one analyzer: member-mutation collection with its fixpoint, static source analysis, and function fingerprints, each about one second on Obsidian's 3.7 MB app.js. - traverseJavaScriptAstSteps yields every 1,024 visited nodes with the same visit order; traverseJavaScriptAst drains it, so every existing caller is unchanged. - Member mutations, static analysis and fingerprints run as steps. The member-mutation fixpoint yields every 64 references, and fingerprints yield between callables. - The now-unused synchronous member-mutation and fingerprint wrappers are removed. Graph IDs and complete result digests match main on three inputs, including obsidian.asar. Over two runs there, the longest block falls from about 920 ms to 740 ms, the second from 860 to 595 ms, and blocks over 250 ms from 27 to 18, with total time within 1.5%. Refs #1462 Co-authored-by: Claude Opus 5.5 --- .../javascript/JavaScriptArtifactAnalysis.ts | 4 +- .../javascript/javascriptSemanticAnalysis.ts | 5 +- .../javascriptSemanticAnalysisSteps.test.ts | 27 +++++ .../javascriptSemanticDerivedAnalysis.ts | 29 ++--- .../javascriptSemanticFingerprints.ts | 105 +++++++++--------- .../javascriptSemanticMemberMutations.ts | 20 ++-- .../javascript/javascriptSemanticTraversal.ts | 44 ++++++-- .../javascript/javascriptStaticAnalysis.ts | 29 +++-- 8 files changed, 166 insertions(+), 97 deletions(-) diff --git a/src/application/javascript/JavaScriptArtifactAnalysis.ts b/src/application/javascript/JavaScriptArtifactAnalysis.ts index 72fb67edd3..fcba4cc165 100644 --- a/src/application/javascript/JavaScriptArtifactAnalysis.ts +++ b/src/application/javascript/JavaScriptArtifactAnalysis.ts @@ -2,7 +2,7 @@ import { createHash } from "node:crypto"; import { setImmediate } from "node:timers/promises"; import { resolveJavaScriptSourceMapReference } from "../../domain/javascript/javascriptSourceMapPaths.js"; -import { analyzeParsedJavaScriptStaticSource } from "../../domain/javascript/javascriptStaticAnalysis.js"; +import { analyzeParsedJavaScriptStaticSourceSteps } from "../../domain/javascript/javascriptStaticAnalysis.js"; import { analyzeParsedJavaScriptSemanticsSteps, classifyParsedJavaScriptOpenReceivers, @@ -190,7 +190,7 @@ function* analyzeArtifactFileSteps< } const openReceiverFacts = classifyParsedJavaScriptOpenReceivers(parsed); yield; - const analysis = analyzeParsedJavaScriptStaticSource( + const analysis = yield* analyzeParsedJavaScriptStaticSourceSteps( file.text.value, parsed, openReceiverFacts, diff --git a/src/domain/javascript/javascriptSemanticAnalysis.ts b/src/domain/javascript/javascriptSemanticAnalysis.ts index 50af0f2cbf..ff1dac49b9 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.ts @@ -33,7 +33,7 @@ import { } from "./javascriptSemanticState.js"; import { traverseJavaScriptAst } from "./javascriptSemanticTraversal.js"; import { semanticRequireOrigin } from "./javascriptSemanticRequireOrigin.js"; -import { collectSemanticMemberMutations } from "./javascriptSemanticMemberMutations.js"; +import { collectSemanticMemberMutationsSteps } from "./javascriptSemanticMemberMutations.js"; import { collectSemanticReturns, resolveSemanticModuleCallables, @@ -126,8 +126,7 @@ export function* analyzeParsedJavaScriptSemanticsSteps( const state = createState(file.program); collectDefinitions(file.program, state); yield; - collectSemanticMemberMutations(file.program, state); - yield; + yield* collectSemanticMemberMutationsSteps(file.program, state); traverseJavaScriptAst(file.program, { enter: (node) => collectSemanticModuleLink(node, state), }); diff --git a/src/domain/javascript/javascriptSemanticAnalysisSteps.test.ts b/src/domain/javascript/javascriptSemanticAnalysisSteps.test.ts index 9aaa71d935..7ae453914f 100644 --- a/src/domain/javascript/javascriptSemanticAnalysisSteps.test.ts +++ b/src/domain/javascript/javascriptSemanticAnalysisSteps.test.ts @@ -5,6 +5,10 @@ import { analyzeParsedJavaScriptSemanticsSteps, } from "./javascriptSemanticAnalysis.js"; import { parseJavaScriptSource } from "./javascriptSourceParser.js"; +import { + traverseJavaScriptAst, + traverseJavaScriptAstSteps, +} from "./javascriptSemanticTraversal.js"; const source = ` import { readFile } from "node:fs"; @@ -32,3 +36,26 @@ it("yields between semantic phases and returns the synchronous result (#1462)", if (reparsed === null) throw new Error("Expected parsed source"); expect(step.value).toEqual(analyzeParsedJavaScriptSemantics(reparsed)); }); + +it("traverses a large tree in steps with the synchronous visit order (#1462)", () => { + const parsed = parseJavaScriptSource( + Array.from( + { length: 400 }, + (_, index) => `const v${index} = [${index}];`, + ).join("\n"), + "large.js", + ); + if (parsed === null) throw new Error("Expected parsed source"); + const visits = (record: string[]) => ({ + enter: (node: { readonly type: string }) => record.push(`+${node.type}`), + exit: (node: { readonly type: string }) => record.push(`-${node.type}`), + }); + const synchronous: string[] = []; + traverseJavaScriptAst(parsed.program, visits(synchronous)); + const stepped: string[] = []; + const steps = traverseJavaScriptAstSteps(parsed.program, visits(stepped)); + let yields = 0; + while (steps.next().done !== true) yields += 1; + expect(yields).toBeGreaterThan(0); + expect(stepped).toEqual(synchronous); +}); diff --git a/src/domain/javascript/javascriptSemanticDerivedAnalysis.ts b/src/domain/javascript/javascriptSemanticDerivedAnalysis.ts index 545652a744..b30d368592 100644 --- a/src/domain/javascript/javascriptSemanticDerivedAnalysis.ts +++ b/src/domain/javascript/javascriptSemanticDerivedAnalysis.ts @@ -3,7 +3,7 @@ import type * as t from "@babel/types"; import { collectJavaScriptSemanticAsyncEffects } from "./javascriptSemanticAsyncEffects.js"; import { collectJavaScriptSemanticCalls } from "./javascriptSemanticCalls.js"; import { collectJavaScriptSemanticChildProcesses } from "./javascriptSemanticChildProcesses.js"; -import { collectJavaScriptSemanticFingerprints } from "./javascriptSemanticFingerprints.js"; +import { collectJavaScriptSemanticFingerprintsSteps } from "./javascriptSemanticFingerprints.js"; import { collectJavaScriptSemanticDataEffects } from "./javascriptSemanticDataEffects.js"; import type { JavaScriptSemanticCallable, @@ -84,19 +84,20 @@ export function* collectJavaScriptDerivedSemanticsSteps( callables, ); yield; - const functionFingerprints = collectJavaScriptSemanticFingerprints({ - state, - callables, - calls, - promises: promiseOperations, - events: asyncEffects.eventOperations, - timers: asyncEffects.timerOperations, - childProcessSpawns: childProcesses.childProcessSpawns, - childProcessInteractions: childProcesses.childProcessInteractions, - requestOperations: dataEffects.requestOperations, - resourceOperations, - parserPartial, - }); + const functionFingerprints = + yield* collectJavaScriptSemanticFingerprintsSteps({ + state, + callables, + calls, + promises: promiseOperations, + events: asyncEffects.eventOperations, + timers: asyncEffects.timerOperations, + childProcessSpawns: childProcesses.childProcessSpawns, + childProcessInteractions: childProcesses.childProcessInteractions, + requestOperations: dataEffects.requestOperations, + resourceOperations, + parserPartial, + }); return { ...calls, promiseOperations, diff --git a/src/domain/javascript/javascriptSemanticFingerprints.ts b/src/domain/javascript/javascriptSemanticFingerprints.ts index 3f73090296..3d4b828bea 100644 --- a/src/domain/javascript/javascriptSemanticFingerprints.ts +++ b/src/domain/javascript/javascriptSemanticFingerprints.ts @@ -17,7 +17,7 @@ import type { } from "./javascriptSemanticIr.js"; import { semanticCallableIdForNode } from "./javascriptSemanticProjection.js"; import type { JavaScriptSemanticAnalysisState } from "./javascriptSemanticState.js"; -import { traverseJavaScriptAst } from "./javascriptSemanticTraversal.js"; +import { traverseJavaScriptAstSteps } from "./javascriptSemanticTraversal.js"; interface SemanticFingerprintCollectionInput { readonly state: JavaScriptSemanticAnalysisState; @@ -33,10 +33,10 @@ interface SemanticFingerprintCollectionInput { readonly parserPartial: boolean; } -/** Build bounded rename- and formatting-resistant components per callable. */ -export const collectJavaScriptSemanticFingerprints = ( +/** Build bounded rename- and formatting-resistant components per callable, yielding between callables. */ +export function* collectJavaScriptSemanticFingerprintsSteps( input: SemanticFingerprintCollectionInput, -): JavaScriptSemanticFunctionFingerprint[] => { +): Generator { const { state, callables, @@ -67,11 +67,13 @@ export const collectJavaScriptSemanticFingerprints = ( ); const networkEffectsByOwner = groupByOwner(requestOperations, ownerId); const resourceEffectsByOwner = groupByOwner(resourceOperations, ownerId); - return callables.flatMap((callable) => { - if (callable.kind === "class") return []; + const fingerprints: JavaScriptSemanticFunctionFingerprint[] = []; + for (const callable of callables) { + if (callable.kind === "class") continue; const node = state.callableNodesById.get(callable.callableId); - if (node === undefined) return []; - const syntax = fingerprintSyntax(node, callable.callableId); + if (node === undefined) continue; + yield; + const syntax = yield* fingerprintSyntaxSteps(node, callable.callableId); const promiseEffects = promiseEffectsByOwner.get(callable.callableId) ?? []; const eventEffects = eventEffectsByOwner.get(callable.callableId) ?? []; const timerEffects = timerEffectsByOwner.get(callable.callableId) ?? []; @@ -80,50 +82,49 @@ export const collectJavaScriptSemanticFingerprints = ( const networkEffects = networkEffectsByOwner.get(callable.callableId) ?? []; const resourceEffects = resourceEffectsByOwner.get(callable.callableId) ?? []; - return [ - { - callableId: callable.callableId, - status: - parserPartial || callable.returnCoverage.status !== "complete" - ? "partial" - : "complete", - components: { - parameterArity: callableParameterArity(node), - normalizedAstSha256: digest(syntax.normalizedTokens), - controlFlowSha256: digest(syntax.controlFlowTokens), - relationShapeSha256: digest( - relationShape( - callable, - callSitesByCaller.get(callable.callableId) ?? [], - captureCountByCallable.get(callable.callableId) ?? 0, - promiseEffects, - ), + fingerprints.push({ + callableId: callable.callableId, + status: + parserPartial || callable.returnCoverage.status !== "complete" + ? "partial" + : "complete", + components: { + parameterArity: callableParameterArity(node), + normalizedAstSha256: digest(syntax.normalizedTokens), + controlFlowSha256: digest(syntax.controlFlowTokens), + relationShapeSha256: digest( + relationShape( + callable, + callSitesByCaller.get(callable.callableId) ?? [], + captureCountByCallable.get(callable.callableId) ?? 0, + promiseEffects, ), - literalSetSha256: digest( - [...new Set(syntax.literals)].sort(compareUnicodeCodePoints), - ), - effects: [ - ...(isAsyncCallable(node) ? (["async"] as const) : []), - ...(childProcessEffects.length > 0 - ? (["child-process"] as const) - : []), - ...(eventEffects.length > 0 ? (["event"] as const) : []), - ...(networkEffects.length > 0 ? (["network"] as const) : []), - ...(promiseEffects.length > 0 ? (["promise"] as const) : []), - ...(resourceEffects.length > 0 ? (["resource"] as const) : []), - ...(timerEffects.length > 0 ? (["timer"] as const) : []), - ], - }, - limitations: [ - "The fingerprint is a static candidate and does not prove behavioral equivalence.", - ...(parserPartial - ? ["Incomplete semantic recovery makes this fingerprint partial."] + ), + literalSetSha256: digest( + [...new Set(syntax.literals)].sort(compareUnicodeCodePoints), + ), + effects: [ + ...(isAsyncCallable(node) ? (["async"] as const) : []), + ...(childProcessEffects.length > 0 + ? (["child-process"] as const) : []), + ...(eventEffects.length > 0 ? (["event"] as const) : []), + ...(networkEffects.length > 0 ? (["network"] as const) : []), + ...(promiseEffects.length > 0 ? (["promise"] as const) : []), + ...(resourceEffects.length > 0 ? (["resource"] as const) : []), + ...(timerEffects.length > 0 ? (["timer"] as const) : []), ], }, - ]; - }); -}; + limitations: [ + "The fingerprint is a static candidate and does not prove behavioral equivalence.", + ...(parserPartial + ? ["Incomplete semantic recovery makes this fingerprint partial."] + : []), + ], + }); + } + return fingerprints; +} const ownerId = ( value: T, @@ -161,15 +162,15 @@ interface FingerprintSyntax { readonly literals: readonly string[]; } -const fingerprintSyntax = ( +function* fingerprintSyntaxSteps( root: t.Node, targetCallableId: string, -): FingerprintSyntax => { +): Generator { const normalizedTokens: string[] = []; const controlFlowTokens: string[] = []; const literals: string[] = []; const callableStack: string[] = []; - traverseJavaScriptAst(root, { + yield* traverseJavaScriptAstSteps(root, { enter: (node, parent) => { const callableId = semanticCallableIdForNode(node); if (callableId !== null) callableStack.push(callableId); @@ -187,7 +188,7 @@ const fingerprintSyntax = ( }, }); return { normalizedTokens, controlFlowTokens, literals }; -}; +} const normalizedNodeToken = (node: t.Node, parent: t.Node | null): string => { if (t.isIdentifier(node)) diff --git a/src/domain/javascript/javascriptSemanticMemberMutations.ts b/src/domain/javascript/javascriptSemanticMemberMutations.ts index 06a335bcac..12ceec71d0 100644 --- a/src/domain/javascript/javascriptSemanticMemberMutations.ts +++ b/src/domain/javascript/javascriptSemanticMemberMutations.ts @@ -25,7 +25,7 @@ import { type JavaScriptSemanticPropertyPath, } from "./javascriptSemanticPropertyPaths.js"; import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js"; -import { traverseJavaScriptAst } from "./javascriptSemanticTraversal.js"; +import { traverseJavaScriptAstSteps } from "./javascriptSemanticTraversal.js"; import { semanticStaticPropertyKey, unwrapJavaScriptExpression, @@ -34,13 +34,13 @@ import { type PropertyPath = JavaScriptSemanticPropertyPath; type ValueEffect = "write" | "escape"; -/** Preserve uncertainty from property writes and references exposed to calls. */ -export const collectSemanticMemberMutations = ( +/** Preserve uncertainty from property writes and references exposed to calls, yielding during traversal and fixpoint iteration. */ +export function* collectSemanticMemberMutationsSteps( program: t.Program, state: JavaScriptSemanticAnalysisState, -): void => { +): Generator { const parents = state.parentsByNode; - traverseJavaScriptAst(program, { + yield* traverseJavaScriptAstSteps(program, { enter: (node, parent) => { if (parent !== null) parents.set(node, parent); }, @@ -382,7 +382,7 @@ export const collectSemanticMemberMutations = ( ); } }; - traverseJavaScriptAst(program, { + yield* traverseJavaScriptAstSteps(program, { enter: (node, parent) => { for (const source of semanticIterationSources(node, parent)) deferIterable({ @@ -435,7 +435,8 @@ export const collectSemanticMemberMutations = ( const effectsBefore = recordedEffects.size; const iterablesBefore = iterableReferences.length; const iterationBefore: boolean = arrayIterationUnknown; - for (const reference of pendingReferences.splice(0)) { + for (const [index, reference] of pendingReferences.splice(0).entries()) { + if (index % 64 === 0) yield; const { initializer } = reference; if (referenceSourcesInactive(initializer, state)) pendingReferences.push(reference); @@ -480,7 +481,8 @@ export const collectSemanticMemberMutations = ( reference.originAt, ); } - for (const reference of [...iterableReferences]) { + for (const [index, reference] of [...iterableReferences].entries()) { + if (index % 64 === 0) yield; if (referenceSourcesInactive(reference, state)) continue; if (reference.iterationOnly) { const expanded = expandedIterationReferences(reference, state); @@ -518,7 +520,7 @@ export const collectSemanticMemberMutations = ( // Gathering effects can evaluate a primitive projection before its object // escapes. Final values must use the completed mutation state. clearSemanticPrimitiveBindingValues(state); -}; +} interface ReferencedValue { readonly node: t.Node; diff --git a/src/domain/javascript/javascriptSemanticTraversal.ts b/src/domain/javascript/javascriptSemanticTraversal.ts index 9b562a94c2..16960ba8fe 100644 --- a/src/domain/javascript/javascriptSemanticTraversal.ts +++ b/src/domain/javascript/javascriptSemanticTraversal.ts @@ -12,15 +12,28 @@ import * as t from "@babel/types"; */ export const traverseJavaScriptAst = ( root: t.Node, - visitor: { - readonly enter: ( - node: t.Node, - parent: t.Node | null, - readAncestors: () => readonly t.Node[], - ) => void; - readonly exit?: (node: t.Node, parent: t.Node | null) => void; - }, -): void => { + visitor: JavaScriptAstVisitor, +): void => completeSemanticSteps(traverseJavaScriptAstSteps(root, visitor)); + +/** Callbacks for one deterministic AST traversal. */ +export interface JavaScriptAstVisitor { + readonly enter: ( + node: t.Node, + parent: t.Node | null, + readAncestors: () => readonly t.Node[], + ) => void; + readonly exit?: (node: t.Node, parent: t.Node | null) => void; +} + +/** + * {@link traverseJavaScriptAst} with a yield every 1,024 visited nodes, so a + * caller can serve control messages during one large traversal. Visit order + * and callbacks are identical. + */ +export function* traverseJavaScriptAstSteps( + root: t.Node, + visitor: JavaScriptAstVisitor, +): Generator { const ancestors: t.Node[] = []; const enter = (node: t.Node, parent: t.Node | null): TraversalFrame => { visitor.enter(node, parent, () => [...ancestors]); @@ -28,12 +41,15 @@ export const traverseJavaScriptAst = ( return { node, parent, children: childNodes(node), nextIndex: 0 }; }; const pending = [enter(root, null)]; + let visited = 1; while (pending.length > 0) { const current = pending.at(-1); if (current === undefined) break; const child = current.children[current.nextIndex]; if (child !== undefined) { current.nextIndex += 1; + if (visited % 1024 === 0) yield; + visited += 1; pending.push(enter(child, current.node)); } else { visitor.exit?.(current.node, current.parent); @@ -41,6 +57,16 @@ export const traverseJavaScriptAst = ( pending.pop(); } } +} + +/** Run cooperative analysis steps to completion without yielding control. */ +export const completeSemanticSteps = ( + steps: Iterator, +): Value => { + for (;;) { + const step = steps.next(); + if (step.done === true) return step.value; + } }; interface TraversalFrame { diff --git a/src/domain/javascript/javascriptStaticAnalysis.ts b/src/domain/javascript/javascriptStaticAnalysis.ts index 524af811eb..264c94182c 100644 --- a/src/domain/javascript/javascriptStaticAnalysis.ts +++ b/src/domain/javascript/javascriptStaticAnalysis.ts @@ -1,6 +1,9 @@ import * as t from "@babel/types"; -import { traverseJavaScriptAst } from "./javascriptSemanticTraversal.js"; +import { + completeSemanticSteps, + traverseJavaScriptAstSteps, +} from "./javascriptSemanticTraversal.js"; import { inspectElectronStaticNode } from "./electronStaticAnalysis.js"; import { @@ -57,9 +60,19 @@ export const analyzeParsedJavaScriptStaticSource = ( source: string, file: ParsedJavaScriptSource, openReceiverFacts?: ReadonlyMap, -): JavaScriptStaticAnalysis => { +): JavaScriptStaticAnalysis => + completeSemanticSteps( + analyzeParsedJavaScriptStaticSourceSteps(source, file, openReceiverFacts), + ); + +/** Static analysis that yields during its source traversal; results are unchanged. */ +export function* analyzeParsedJavaScriptStaticSourceSteps( + source: string, + file: ParsedJavaScriptSource, + openReceiverFacts?: ReadonlyMap, +): Generator { const accumulator = createJavaScriptAnalysisAccumulator(source.length); - traverseStaticSource(file, { + yield* traverseStaticSourceSteps(file, { source, accumulator, ...(openReceiverFacts === undefined ? {} : { openReceiverFacts }), @@ -67,20 +80,20 @@ export const analyzeParsedJavaScriptStaticSource = ( }); addSourceMapDirectives(source, file.comments ?? [], accumulator); return finalizeStaticAnalysis(source, file, accumulator); -}; +} -const traverseStaticSource = ( +function* traverseStaticSourceSteps( file: ParsedJavaScriptSource, findings: JavaScriptFindingContext, -): void => { - traverseJavaScriptAst(file, { +): Generator { + yield* traverseJavaScriptAstSteps(file, { enter: (node) => { findings.accumulator.visitedNodes += 1; inspectNode(node, findings); return undefined; }, }); -}; +} const finalizeStaticAnalysis = ( source: string,