From 77c8a1ba321c1a25bb3abef2e4733d13a3cfaaee Mon Sep 17 00:00:00 2001 From: Saim Shafique Date: Sat, 17 Oct 2026 12:33:41 +0500 Subject: [PATCH] fix(javascript): mark resource-skipped files parse_status skipped (#3852) * fix(javascript): mark resource-skipped files parse_status skipped Files above ~5 MB exhaust the worker heap during static analysis (or exceed the 8 MB static-checkpoint budget with a larger heap). They were inventoried with text included but never parsed, leaving parse_status null and no counter, so the summary read as 'nothing there' instead of 'not analyzed'. Report them as skipped: parse_status 'skipped' on the file observation and a skipped_javascript_files statistics counter. The analysis-failure reason keeps surfacing as a limitation entry. Fixes #3670 * fix(javascript): default skipped_javascript_files when reading old saved evidence Saved analysis evidence written before skipped_javascript_files existed failed validation against the strict reconstruction statistics schema, so users could not reuse old results for comparisons or runtime reconciliation. Accept the field's absence on read (defaulting to 0) while continuing to write it for new results. * fix(javascript): preserve unknown skipped count for legacy evidence The .default(0) gate fix fabricated a measured zero for saved pre-change evidence, so comparisons would receive a definitive "no files skipped" fact the producer never established. Use the codebase's exactOptional idiom instead: legacy reads accept absence and keep omission (unknown), while the producer still always writes the field (required by JavaScriptArtifactReconstructionResult). Tests: legacy payload now asserts the key stays absent/undefined; current-output test keeps asserting the producer-written counter. * fix(javascript): add required failure scope and status to graph fixture --------- Co-authored-by: Saim Shafique --- .../JavaScriptArtifactAnalysisTypes.ts | 16 ++ .../JavaScriptArtifactGraphStructure.test.ts | 148 ++++++++++++++++++ .../JavaScriptArtifactGraphStructure.ts | 6 +- .../JavaScriptArtifactReconstruction.ts | 9 +- .../javascriptExportShapeComparisonExample.ts | 1 + .../javascriptRuntimeReconciliationExample.ts | 1 + .../javascriptApplicationAnalysis.ts | 4 + ...javascriptApplicationAnalysisSteps.test.ts | 26 +++ 8 files changed, 209 insertions(+), 2 deletions(-) create mode 100644 src/application/javascript/JavaScriptArtifactGraphStructure.test.ts diff --git a/src/application/javascript/JavaScriptArtifactAnalysisTypes.ts b/src/application/javascript/JavaScriptArtifactAnalysisTypes.ts index ef174affb..500fb57cf 100644 --- a/src/application/javascript/JavaScriptArtifactAnalysisTypes.ts +++ b/src/application/javascript/JavaScriptArtifactAnalysisTypes.ts @@ -170,6 +170,22 @@ export interface AnalyzedJavaScriptArtifactFile< }; } +/** + * A JavaScript source selected for analysis whose worker produced no facts. + * The analysis failure carries the reason; the null parse status alone reads + * as "nothing there" instead of "not analyzed". + */ +export const isSkippedJavaScriptAnalysis = ( + entry: Pick< + AnalyzedJavaScriptArtifactFile, + "file" | "javascript" | "analysis_failure" + >, +): boolean => + entry.file.kind === "javascript" && + entry.file.text.included && + entry.javascript === null && + entry.analysis_failure !== undefined; + /** Complete static-analysis projection before graph construction. */ export interface JavaScriptArtifactAnalysis< SemanticIr extends JavaScriptModuleSemanticIr = JavaScriptSemanticIr, diff --git a/src/application/javascript/JavaScriptArtifactGraphStructure.test.ts b/src/application/javascript/JavaScriptArtifactGraphStructure.test.ts new file mode 100644 index 000000000..dbfc19d76 --- /dev/null +++ b/src/application/javascript/JavaScriptArtifactGraphStructure.test.ts @@ -0,0 +1,148 @@ +import { describe, expect, it } from "vitest"; + +import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySnapshot.js"; +import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js"; +import { analyzeJavaScriptStaticSource } from "../../domain/javascript/javascriptStaticAnalysis.js"; +import type { + AnalyzedJavaScriptArtifactFile, + JavaScriptModuleArtifactAnalysis, +} from "./JavaScriptArtifactAnalysisTypes.js"; +import { javascriptArtifactAnalysisScopes } from "./JavaScriptArtifactAnalysisTypes.js"; +import { JavaScriptArtifactGraphAccumulator } from "./JavaScriptArtifactGraphAccumulator.js"; +import type { JavaScriptArtifactGraphContext } from "./JavaScriptArtifactGraphContext.js"; +import { + addJavaScriptArtifactFiles, + createJavaScriptArtifactRootNode, +} from "./JavaScriptArtifactGraphStructure.js"; + +const SHA256 = "a".repeat(64); +const REASON = + "main.js: Resource constraint during analyze_javascript_application: Owned JavaScript worker exhausted its heap while analyzing main.js during static"; + +const skippedFile = (): JavaScriptArtifactFile => ({ + path: "main.js", + container_sha256: SHA256, + sha256: SHA256, + bytes: 5000107, + inventory_artifact_id: `art_${SHA256}`, + kind: "javascript", + unpacked: false, + text: { included: true, value: "const {ipcMain}=require('electron');\n" }, +}); + +const skippedEntry = ( + file: JavaScriptArtifactFile, +): AnalyzedJavaScriptArtifactFile => ({ + file, + javascript: null, + semantic: null, + analysis_failure: { + scope: javascriptArtifactAnalysisScopes["javascript"], + status: "interrupted", + reason: REASON, + error: { code: "resource_constraint" }, + limits: null, + }, +}); + +const analysisFor = ( + entries: readonly AnalyzedJavaScriptArtifactFile[], +): JavaScriptModuleArtifactAnalysis => ({ + files: entries, + packages: [], + json_modules: [], + html_scripts: [], + html_inline_scripts: [], + source_maps: [], + visited_ast_nodes: 0, + findings: 0, + modules: 0, + parse_failures: 0, + limitations: [], +}); + +const contextFor = ( + file: JavaScriptArtifactFile, + analysis: JavaScriptModuleArtifactAnalysis, +): JavaScriptArtifactGraphContext => { + const accumulator = new JavaScriptArtifactGraphAccumulator(); + const snapshot = { + manifest: { + root_artifact_id: `art_${SHA256}`, + root_sha256: SHA256, + root_format: "directory", + manifest_id: `agm_${SHA256}`, + graph_sha256: SHA256, + }, + nodes: [{ artifact_id: `art_${SHA256}`, sha256: SHA256, size: 1 }], + } as unknown as ArtifactInventorySnapshot; + const fileSet = { + files: [file], + unavailable_relevant_files: [], + containers: [], + text_bytes_read: file.bytes, + invalid_utf8_files: 0, + }; + return { + accumulator, + snapshot, + fileSet, + analysis, + directoryPaths: new Set(), + root: createJavaScriptArtifactRootNode(accumulator, snapshot), + filesByPath: new Map([[file.path, file]]), + unanalyzedFiles: undefined, + fileNodes: new Map(), + assetNodes: new Map(), + sourceModuleNodes: new Map(), + chunkNodes: new Map(), + moduleNodes: new Map(), + containerNodes: new Map(), + }; +}; + +const fileObservation = (context: JavaScriptArtifactGraphContext) => { + const node = context.accumulator + .nodes() + .find(({ kind }) => kind === "javascript-asset"); + if (node === undefined) throw new Error("Expected a javascript-asset node"); + const observation = node.observations[0]; + if (observation === undefined) throw new Error("Expected a file observation"); + return observation.properties; +}; + +describe("skipped JavaScript parse reporting", () => { + it("marks parse_status skipped for a JavaScript file whose worker analysis failed", () => { + const file = skippedFile(); + const context = contextFor(file, analysisFor([skippedEntry(file)])); + addJavaScriptArtifactFiles(context); + expect(fileObservation(context).parse_status).toBe("skipped"); + }); + + it("keeps parse_status null for a JavaScript file never attempted", () => { + const file = skippedFile(); + const entry: AnalyzedJavaScriptArtifactFile = { + file, + javascript: null, + semantic: null, + }; + const context = contextFor(file, analysisFor([entry])); + addJavaScriptArtifactFiles(context); + expect(fileObservation(context).parse_status).toBeNull(); + }); + + it("keeps the parsed status for an analyzed JavaScript file", () => { + const file = skippedFile(); + const javascript = analyzeJavaScriptStaticSource( + "const {ipcMain}=require('electron');\n", + ); + const entry: AnalyzedJavaScriptArtifactFile = { + file, + javascript, + semantic: null, + }; + const context = contextFor(file, analysisFor([entry])); + addJavaScriptArtifactFiles(context); + expect(fileObservation(context).parse_status).toBe(javascript.parse_status); + }); +}); diff --git a/src/application/javascript/JavaScriptArtifactGraphStructure.ts b/src/application/javascript/JavaScriptArtifactGraphStructure.ts index 43c00d93b..6ec343af1 100644 --- a/src/application/javascript/JavaScriptArtifactGraphStructure.ts +++ b/src/application/javascript/JavaScriptArtifactGraphStructure.ts @@ -15,6 +15,7 @@ import type { JavaScriptArtifactFile, } from "../../domain/javascript/javascriptArtifactFiles.js"; import type { JavaScriptArtifactGraphAccumulator } from "./JavaScriptArtifactGraphAccumulator.js"; +import { isSkippedJavaScriptAnalysis } from "./JavaScriptArtifactAnalysisTypes.js"; import { addArtifactContainsEdge, addUnavailableStaticParseScope, @@ -145,6 +146,7 @@ export const addJavaScriptArtifactFiles = ( file, analyzed.javascript, jsonValue, + isSkippedJavaScriptAnalysis(analyzed), ); if ( analyzed.application_projection_failure !== undefined && @@ -553,6 +555,7 @@ const createFileTarget = ( file: JavaScriptArtifactFile, javascript: JavaScriptModuleArtifactAnalysis["files"][number]["javascript"], json: JavaScriptModuleArtifactAnalysis["json_modules"][number] | undefined, + skipped: boolean, ): ApplicationNode => { const kind = artifactFileNodeKind(file.kind); return context.accumulator.addNode({ @@ -572,7 +575,8 @@ const createFileTarget = ( unpacked: file.unpacked, file_kind: file.kind, text_status: file.text.included ? "included" : file.text.reason, - parse_status: javascript?.parse_status ?? null, + parse_status: + javascript?.parse_status ?? (skipped ? "skipped" : null), json_parse_status: json?.status ?? null, json_top_level_keys: json?.top_level_keys ?? [], omitted_json_top_level_keys: json?.omitted_top_level_keys ?? 0, diff --git a/src/application/javascript/JavaScriptArtifactReconstruction.ts b/src/application/javascript/JavaScriptArtifactReconstruction.ts index cb7d66089..5f40f2928 100644 --- a/src/application/javascript/JavaScriptArtifactReconstruction.ts +++ b/src/application/javascript/JavaScriptArtifactReconstruction.ts @@ -20,7 +20,10 @@ import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySn import type { ArtifactInventoryPartialObservation } from "../../domain/artifactPartialObservation.js"; import type { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; import { analyzeAndAdoptJavaScriptArtifactFiles } from "./JavaScriptArtifactAnalysis.js"; -import { javascriptArtifactAnalysisScopes } from "./JavaScriptArtifactAnalysisTypes.js"; +import { + isSkippedJavaScriptAnalysis, + javascriptArtifactAnalysisScopes, +} from "./JavaScriptArtifactAnalysisTypes.js"; import { createJavaScriptSourceAnalysis } from "../../artifacts/javascript/JavaScriptSourceAnalysis.js"; import { JAVASCRIPT_ANALYSIS_RESOURCE_DEFAULTS } from "../../domain/javascript/javascriptAnalysisResourceControls.js"; import { createOwnedJavaScriptApplicationEvidenceCooperatively } from "./JavaScriptApplicationEvidence.js"; @@ -57,6 +60,7 @@ export interface JavaScriptArtifactReconstructionResult { readonly text_bytes_read: number; readonly invalid_utf8_files: number; readonly parsed_javascript_files: number; + readonly skipped_javascript_files: number; readonly visited_ast_nodes: number; readonly findings: number; readonly modules: number; @@ -275,6 +279,9 @@ export const reconstructJavaScriptArtifact = async ( parsed_javascript_files: analysis.files.filter( ({ javascript }) => javascript !== null, ).length, + skipped_javascript_files: analysis.files.filter( + isSkippedJavaScriptAnalysis, + ).length, visited_ast_nodes: analysis.visited_ast_nodes, findings: analysis.findings, modules: analysis.modules, diff --git a/src/contracts/javascript/javascriptExportShapeComparisonExample.ts b/src/contracts/javascript/javascriptExportShapeComparisonExample.ts index 65464336d..ddc6e47c3 100644 --- a/src/contracts/javascript/javascriptExportShapeComparisonExample.ts +++ b/src/contracts/javascript/javascriptExportShapeComparisonExample.ts @@ -71,6 +71,7 @@ const statistics = { text_bytes_read: 60, invalid_utf8_files: 0, parsed_javascript_files: 1, + skipped_javascript_files: 0, visited_ast_nodes: 12, findings: 1, modules: 0, diff --git a/src/contracts/javascript/javascriptRuntimeReconciliationExample.ts b/src/contracts/javascript/javascriptRuntimeReconciliationExample.ts index 4be73fab7..8a535c63f 100644 --- a/src/contracts/javascript/javascriptRuntimeReconciliationExample.ts +++ b/src/contracts/javascript/javascriptRuntimeReconciliationExample.ts @@ -166,6 +166,7 @@ export const JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE = createEvidence( text_bytes_read: script.bytes, invalid_utf8_files: 0, parsed_javascript_files: 1, + skipped_javascript_files: 0, visited_ast_nodes: 1, findings: 0, modules: 0, diff --git a/src/domain/javascript/javascriptApplicationAnalysis.ts b/src/domain/javascript/javascriptApplicationAnalysis.ts index 6387f8e90..7b1055b79 100644 --- a/src/domain/javascript/javascriptApplicationAnalysis.ts +++ b/src/domain/javascript/javascriptApplicationAnalysis.ts @@ -63,6 +63,10 @@ const reconstructionStatisticsSchema = z.strictObject({ text_bytes_read: countSchema, invalid_utf8_files: countSchema, parsed_javascript_files: countSchema, + // Old saved evidence predates this field: absence must stay unknown, not + // become a measured zero. The producer always writes it (required by the + // application-layer result type); readers treat a missing value as unknown. + skipped_javascript_files: countSchema.exactOptional(), visited_ast_nodes: countSchema, findings: countSchema, modules: countSchema, diff --git a/src/domain/javascript/javascriptApplicationAnalysisSteps.test.ts b/src/domain/javascript/javascriptApplicationAnalysisSteps.test.ts index a460de077..6356f92c0 100644 --- a/src/domain/javascript/javascriptApplicationAnalysisSteps.test.ts +++ b/src/domain/javascript/javascriptApplicationAnalysisSteps.test.ts @@ -324,3 +324,29 @@ describe("graphs transferred to their sealing steps", () => { ).toEqual(issuesFrom(() => createJavaScriptApplicationGraph(stale))); }); }); + +describe("skipped_javascript_files backward compatibility", () => { + it("preserves omission for saved evidence that predates the skipped counter", () => { + const legacy = structuredClone( + JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE.normalized_result, + ) as Record; + const statistics = legacy.statistics as Record; + delete statistics.skipped_javascript_files; + const result = javascriptApplicationAnalysisResultSchema.parse(legacy); + // A pre-change result can contain the very resource-skipped files this + // field reports, but it has no counter: unknown must not become a + // measured zero that comparisons would treat as "nothing skipped". + expect("skipped_javascript_files" in result.statistics).toBe(false); + expect(result.statistics.skipped_javascript_files).toBeUndefined(); + }); + + it("keeps the producer-written skipped count in current evidence", () => { + const current = structuredClone( + JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE.normalized_result, + ) as Record; + const statistics = current.statistics as Record; + statistics.skipped_javascript_files = 3; + const result = javascriptApplicationAnalysisResultSchema.parse(current); + expect(result.statistics.skipped_javascript_files).toBe(3); + }); +});