fix(javascript): isolate source analysis and retain bounded partial evidence (#1651)

* refactor(javascript): validate transferable file projections

* test(javascript): consume decoded module and static facts

* fix(javascript): isolate source analysis and retain bounded partial evidence

* test(javascript): verify cleanup retention and repeated source paths
This commit is contained in:
Ysopwnable authored and GitHub committed 2026-10-11 07:22:50 +08:00
1 parent 92cc0edd8e
commit 4999e4be2c
41 files changed
+3748 -176

No files matched your search

@@ -6,14 +6,9 @@ import {
analyzeJavaScriptApplicationInputSchema,
parseOwnedJavaScriptApplicationAnalysisSteps,
} from "../../domain/javascript/javascriptApplicationAnalysis.js";
import {
AnalysisAccessDeniedError,
AnalysisInputError,
AnalysisOutputError,
AnalysisUnsupportedTargetError,
} from "../../domain/analysisErrorCore.js";
import { AnalysisOutputError } from "../../domain/analysisErrorCore.js";
import { ArtifactOperationError } from "../../domain/artifactOperationError.js";
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
import { AnalysisError } from "../../domain/analysisErrorBase.js";
import type { Evidence } from "../../domain/evidence.js";
import { analysisInputErrorFromIssues } from "../../domain/inputIssueProjection.js";
import { err, ok, type Result } from "../../domain/result.js";
@@ -64,6 +59,12 @@ export const analyzeJavaScriptApplicationValidated = async (
input_path: input.input_path,
format: input.format,
integrity_policy: input.integrity_policy,
...(input.max_heap_mb === undefined
? {}
: { max_heap_mb: input.max_heap_mb }),
...(input.analysis_timeout_ms === undefined
? {}
: { analysis_timeout_ms: input.analysis_timeout_ms }),
},
scope,
options.signal,
@@ -110,13 +111,8 @@ export const analyzeJavaScriptApplicationValidated = async (
assertJavaScriptAnalysisActive(options.signal);
return ok(evidence);
} catch (cause: unknown) {
// Input selection failures are already typed by their cause.
if (
cause instanceof AnalysisInputError ||
cause instanceof AnalysisUnsupportedTargetError ||
cause instanceof AnalysisAccessDeniedError
)
return err(cause);
// Preserve execution constraints and their completed partial Evidence.
if (cause instanceof AnalysisError) return err(cause);
if (cause instanceof ArtifactReaderFailure)
return err(artifactFailureToOperationError(cause));
if (cause instanceof z.ZodError)
@@ -8,6 +8,7 @@ import {
classifyParsedJavaScriptOpenReceiversSteps,
} from "../../domain/javascript/javascriptSemanticAnalysis.js";
import type { JavaScriptSemanticIr } from "../../domain/javascript/javascriptSemanticIr.js";
import { projectJavaScriptModuleSemantics } from "../../domain/javascript/javascriptModuleSemanticIr.js";
import { parseJavaScriptSource } from "../../domain/javascript/javascriptSourceParser.js";
import { hasValidSourceMapContents } from "../../domain/sourceMapContents.js";
import { flattenSourceMapLeaves } from "../../domain/sourceMapEnvelope.js";
@@ -35,6 +36,15 @@ import type {
import { analyzeJavaScriptJsonModule } from "./JavaScriptJsonModules.js";
import { completeJavaScriptAnalysisSteps } from "./JavaScriptAnalysisControl.js";
import { htmlArtifactReferences } from "../../domain/javascript/htmlArtifactReferences.js";
import type { JavaScriptSourceAnalysisPort } from "../../domain/javascript/javascriptSourceAnalysis.js";
import type { JavaScriptSemanticGraphProjection } from "./JavaScriptSemanticGraphBuilder.js";
import { AnalysisError } from "../../domain/analysisErrorBase.js";
import { projectAnalysisError } from "../../domain/analysisErrorProjection.js";
import {
AnalysisCancelledError,
AnalysisResourceConstraintError,
AnalysisTimeoutError,
} from "../../domain/analysisErrorCore.js";
interface MutableArtifactAnalysis<
SemanticIr extends JavaScriptModuleSemanticIr,
@@ -95,27 +105,7 @@ export const analyzeAndProjectJavaScriptArtifactFiles = async (
): Generator<void, JavaScriptModuleSemanticIr> {
const projection = projectSemantics(file, ir);
if (projection !== undefined) while (projection.next().done !== true) yield;
const programScopes = ir.scopes.filter(({ kind }) => kind === "program");
const programScopeId = programScopes[0]?.scopeId;
const localNames = new Set(
ir.moduleLinks.map(({ localName }) => localName),
);
const callableIds = new Set(
ir.moduleLinks.map(({ callableId }) => callableId),
);
return {
scopes: programScopes,
bindings: ir.bindings.filter(
({ name, scopeId }) =>
scopeId === programScopeId && localNames.has(name),
),
callables: ir.callables.filter(({ callableId }) =>
callableIds.has(callableId),
),
moduleLinks: ir.moduleLinks,
coverage: ir.coverage,
limitations: ir.limitations,
};
return projectJavaScriptModuleSemantics(ir);
};
const context = { state, projectSemantics: projectFileSemantics };
for (const [index, file] of fileSet.files.entries()) {
@@ -130,6 +120,174 @@ export const analyzeAndProjectJavaScriptArtifactFiles = async (
return finalizeArtifactAnalysis(state);
};
/** Analyze sources behind a hard heap boundary, retaining collected facts on interruption. */
export const analyzeAndAdoptJavaScriptArtifactFiles = async (
fileSet: JavaScriptArtifactFileSet,
worker: Pick<JavaScriptSourceAnalysisPort, "analyze">,
projection: JavaScriptSemanticGraphProjection,
beforeFile: (
file: JavaScriptArtifactFile,
completed: number,
total: number,
) => Promise<void>,
signal?: AbortSignal,
): Promise<{
readonly analysis: JavaScriptModuleArtifactAnalysis;
readonly interruption: AnalysisError | null;
}> => {
const state = emptyArtifactAnalysis<JavaScriptModuleSemanticIr>();
let interruption: AnalysisError | null = null;
for (const [index, file] of fileSet.files.entries()) {
try {
await setImmediate();
await beforeFile(file, index, fileSet.files.length);
if (signal?.aborted === true)
throw new AnalysisCancelledError("analyze_javascript_application");
if (file.kind !== "javascript" || !file.text.included) {
const steps = analyzeArtifactFileSteps(file, {
state,
projectSemantics: function* (_file, ir) {
yield;
return projectJavaScriptModuleSemantics(ir);
},
});
await completeJavaScriptAnalysisSteps(steps, signal);
continue;
}
const result = await worker.analyze(
file,
projection.remainingFileNodeBudget(),
signal,
);
const facts = result.ok ? result.value : result.error;
const applicationProjectionFailure = facts.applicationProjectionFailure;
if (facts.projection !== null)
projection.adoptFileProjection(facts.projection);
const error = result.ok ? null : result.error.error;
const limits =
error instanceof AnalysisResourceConstraintError
? error.reportedLimits
: error instanceof AnalysisTimeoutError
? { worker_timeout_ms: error.timeoutMs }
: null;
const failure =
error === null
? undefined
: {
reason: `${file.path}: ${error.message}`,
error: projectAnalysisError(error),
limits,
};
state.files.push({
file,
javascript: facts.javascript,
semantic: facts.module === null ? null : { ir: facts.module },
...(failure === undefined ? {} : { analysis_failure: failure }),
...(applicationProjectionFailure === undefined
? {}
: {
application_projection_failure: {
reason: applicationProjectionFailure.message,
error: projectAnalysisError(applicationProjectionFailure),
limits: applicationProjectionFailure.reportedLimits,
},
}),
});
if (failure !== undefined && facts.projection === null)
projection.recordUnavailableFile(
file,
failure.reason,
failure.limits,
error instanceof AnalysisResourceConstraintError ||
error instanceof AnalysisTimeoutError
? "resource-limit"
: "incomplete-module",
);
if (facts.javascript !== null) {
state.visitedNodes += facts.javascript.visited_ast_nodes;
state.findings +=
findingCount(facts.javascript) +
(facts.module?.moduleLinks.length ?? 0);
state.modules += facts.javascript.bundler_registrations.reduce(
(count, item) => count + item.modules.length,
0,
);
if (facts.javascript.parse_status === "failed")
state.parseFailures += 1;
}
if (
error !== null &&
(!(error instanceof AnalysisResourceConstraintError) ||
error.reportedLimits?.parent_result_budget_exhausted === true)
) {
interruption = error;
break;
}
} catch (cause: unknown) {
if (signal?.aborted !== true && !(cause instanceof AnalysisError))
throw cause;
interruption =
signal?.aborted === true
? new AnalysisCancelledError("analyze_javascript_application")
: cause instanceof AnalysisError
? cause
: null;
if (interruption !== null) {
state.files.push({
file,
javascript: null,
semantic: null,
analysis_failure: {
reason: `${file.path}: ${interruption.message}`,
error: projectAnalysisError(interruption),
limits: null,
},
});
projection.recordUnavailableFile(
file,
interruption.message,
null,
"incomplete-module",
);
}
break;
}
}
if (interruption !== null) {
const analyzedPaths = new Set(state.files.map(({ file }) => file.path));
const failure = {
reason: `Source analysis was interrupted: ${interruption.message}`,
error: projectAnalysisError(interruption),
limits:
interruption instanceof AnalysisResourceConstraintError
? interruption.reportedLimits
: interruption instanceof AnalysisTimeoutError
? { worker_timeout_ms: interruption.timeoutMs }
: null,
};
for (const file of fileSet.files) {
if (analyzedPaths.has(file.path)) continue;
state.files.push({
file,
javascript: null,
semantic: null,
analysis_failure: failure,
});
if (file.kind === "javascript")
projection.recordUnavailableFile(
file,
failure.reason,
failure.limits,
interruption instanceof AnalysisResourceConstraintError ||
interruption instanceof AnalysisTimeoutError
? "resource-limit"
: "incomplete-module",
);
}
}
return { analysis: finalizeArtifactAnalysis(state), interruption };
};
const analyzeArtifactFiles = <SemanticIr extends JavaScriptModuleSemanticIr>(
fileSet: JavaScriptArtifactFileSet,
projectSemantics: ArtifactAnalysisContext<SemanticIr>["projectSemantics"],
@@ -161,6 +319,12 @@ const finalizeArtifactAnalysis = <
limitations: [
"JavaScript and HTML were parsed as inert text; bundle bootstrap code was never executed.",
"Static paths and relationships may remain unresolved when expressions are dynamic or obfuscated.",
...state.files.flatMap(
({ analysis_failure, application_projection_failure }) =>
[analysis_failure, application_projection_failure].flatMap(
(failure) => (failure === undefined ? [] : [failure.reason]),
),
),
],
};
};
@@ -4,7 +4,9 @@ import type {
JavaScriptStaticAnalysis,
} from "../../domain/javascript/javascriptStaticAnalysisTypes.js";
import type { JavaScriptSemanticIr } from "../../domain/javascript/javascriptSemanticIr.js";
import type { JavaScriptModuleSemanticIr } from "../../domain/javascript/javascriptModuleSemanticIr.js";
import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js";
import type { JsonValue } from "../../domain/jsonValue.js";
interface JavaScriptStructuredObservation {
readonly path: string;
@@ -89,15 +91,7 @@ export type JavaScriptJsonModuleObservation = JavaScriptStructuredObservation &
);
/** Module facts needed after file-local semantic projection has finished. */
export type JavaScriptModuleSemanticIr = Pick<
JavaScriptSemanticIr,
| "scopes"
| "bindings"
| "callables"
| "moduleLinks"
| "coverage"
| "limitations"
>;
export type { JavaScriptModuleSemanticIr } from "../../domain/javascript/javascriptModuleSemanticIr.js";
/** One relevant file plus optional AST-only JavaScript facts. */
export interface AnalyzedJavaScriptArtifactFile<
@@ -108,6 +102,18 @@ export interface AnalyzedJavaScriptArtifactFile<
readonly semantic: {
readonly ir: SemanticIr;
} | null;
/** Operational incompleteness, distinct from a malformed source file. */
readonly analysis_failure?: {
readonly reason: string;
readonly error: JsonValue;
readonly limits: Readonly<Record<string, JsonValue>> | null;
};
/** Facts retained inline when their expanded graph exceeds available memory. */
readonly application_projection_failure?: {
readonly reason: string;
readonly error: JsonValue;
readonly limits: Readonly<Record<string, JsonValue>> | null;
};
}
/** Complete static-analysis projection before graph construction. */
@@ -12,7 +12,7 @@ export class JavaScriptArtifactGraphAccumulator {
readonly #nodes = new Map<string, ApplicationNode>();
readonly #edges = new Map<string, ApplicationEdge>();
/** Create or merge one canonical node. */
/** Create or merge one canonical node, retaining a stable live construction reference. */
addNode<Input extends DisplayableNodeInput>(input: Input): ApplicationNode {
const created = createJavaScriptApplicationNode(displayableNode(input));
const existing = this.#nodes.get(created.node_id);
@@ -23,19 +23,23 @@ export class JavaScriptArtifactGraphAccumulator {
const bySemanticId = new Map(
[...existing.observations, ...created.observations].map((observation) => [
observation.observation_id,
observationInput(observation),
observation,
]),
);
const observations = [...bySemanticId.entries()]
.sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))
.map(([, observation]) => observation);
const merged = createJavaScriptApplicationNode({
kind: existing.kind,
identity: existing.identity,
observations,
});
this.#nodes.set(merged.node_id, merged);
return merged;
const observations = [...bySemanticId.values()].sort((left, right) =>
left.observation_id < right.observation_id
? -1
: left.observation_id > right.observation_id
? 1
: 0,
);
// Both factories already validated and identified these observations.
// Replacing the node retained every previous observation array through
// path/chunk/IPC indexes when multiple paths shared an entity identity.
existing.observations.length = 0;
for (const observation of observations)
existing.observations.push(observation);
return existing;
}
/** Create or replace one canonical edge by semantic identifier. */
@@ -56,17 +60,6 @@ export class JavaScriptArtifactGraphAccumulator {
}
}
const observationInput = (
observation: ApplicationNode["observations"][number],
) => {
const {
observation_id: _observationId,
identifier_strategy: _strategy,
...input
} = observation;
return input;
};
/** Node fields whose display form is normalized before validation. */
interface DisplayableNodeInput {
readonly observations: readonly { readonly label: string | null }[];
@@ -107,19 +107,33 @@ function* buildJavaScriptArtifactGraphInputSteps(
yield;
addJavaScriptArtifactFiles(context);
yield;
const packageRoots = addJavaScriptPackageNodes(context);
// Raw observations remain on the file nodes. Only admitted expansions feed
// the relationship builders, including their Electron and bundler callers.
const projectionContext = {
...context,
analysis: {
...analysis,
files: analysis.files.map((file) =>
file.application_projection_failure === undefined
? file
: { ...file, javascript: null, semantic: null },
),
},
};
const packageRoots = addJavaScriptPackageNodes(projectionContext);
yield;
addJavaScriptSourceModules(context);
addJavaScriptSourceModules(projectionContext);
yield;
const bundlerLimitations = addJavaScriptBundlerNodes(context);
const bundlerLimitations = addJavaScriptBundlerNodes(projectionContext);
const relationshipOmissions =
yield* addJavaScriptModuleRelationshipsSteps(context);
const findingLimitations = yield* addJavaScriptStaticFindingsSteps(context);
addElectronBoundaries(context);
yield* addJavaScriptModuleRelationshipsSteps(projectionContext);
const findingLimitations =
yield* addJavaScriptStaticFindingsSteps(projectionContext);
addElectronBoundaries(projectionContext);
yield;
addJavaScriptHtmlRoles(context);
addJavaScriptHtmlRoles(projectionContext);
yield;
addJavaScriptSourceMapOriginals(context);
addJavaScriptSourceMapOriginals(projectionContext);
yield;
const coverage = graphCoverage(context);
return {
@@ -153,14 +167,34 @@ const graphCoverage = (context: JavaScriptArtifactGraphContext) => {
javascript !== null && javascript.parse_status === "partial",
);
const unknownGap =
context.analysis.files.some(
({ analysis_failure, application_projection_failure }) =>
analysis_failure !== undefined ||
application_projection_failure !== undefined,
) ||
context.analysis.parse_failures > 0 ||
context.fileSet.invalid_utf8_files > 0 ||
sourceMapPolicyGap ||
malformedStructuredData ||
partialJavaScript;
if (resourceLimits.length > 0)
const executionLimits = context.analysis.files.flatMap(
({ analysis_failure, application_projection_failure }) =>
[analysis_failure, application_projection_failure].flatMap((failure) =>
failure === undefined
? []
: javaScriptAnalysisCoverageLimits(failure.limits),
),
);
if (resourceLimits.length > 0 || executionLimits.length > 0)
return partialApplicationCoverage(
semanticResourceLimitCoverage(resourceLimits),
[
...new Map(
[
...semanticResourceLimitCoverage(resourceLimits),
...executionLimits,
].map((limit) => [JSON.stringify(limit), limit]),
).values(),
],
null,
);
if (unknownGap) return partialApplicationCoverage([], null);
@@ -260,3 +294,4 @@ const graphLimitations = (
]),
];
};
import { javaScriptAnalysisCoverageLimits } from "../../domain/javascript/javascriptAnalysisResourceControls.js";
@@ -1,4 +1,5 @@
import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySnapshot.js";
import type { JsonValue } from "../../domain/jsonValue.js";
import { compositeKey } from "../../domain/unicodeCodePointOrder.js";
import {
completeApplicationCoverage,
@@ -99,6 +100,8 @@ export interface UnavailableParseInput {
readonly asset: ApplicationNode;
readonly operation: string;
readonly limitation: string;
readonly failure?: JsonValue;
readonly limits?: JavaScriptArtifactGraphCoverage["limits"];
}
/** Add a direct artifact-containment edge. */
@@ -182,12 +185,15 @@ export const addUnavailableStaticParseScope = (
path: input.file.path,
operation: input.operation,
limitation: input.limitation,
...(input.failure === undefined
? {}
: { analysis_failure: input.failure }),
},
evidence: unavailableAstEvidence({
sha256: input.file.sha256,
path: input.file.path,
operation: input.operation,
coverage: partialApplicationCoverage([], null),
coverage: partialApplicationCoverage(input.limits ?? [], null),
limitation: input.limitation,
}),
},
@@ -2,6 +2,9 @@ import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySn
import type { ApplicationNode } from "../../domain/javascript/javascriptApplicationGraphSchemas.js";
import { completeApplicationCoverage } from "../../domain/javascript/javascriptApplicationEvidenceSchemas.js";
import type { JavaScriptModuleArtifactAnalysis } from "./JavaScriptArtifactAnalysisTypes.js";
import { jsonValueSchema } from "../../domain/jsonValue.js";
import { freezeJsonSnapshot } from "../../domain/immutableJson.js";
import type { JavaScriptSourceRange } from "../../domain/javascript/javascriptStaticAnalysisTypes.js";
import type {
JavaScriptArtifactContainer,
JavaScriptArtifactFile,
@@ -12,12 +15,14 @@ import {
addUnavailableStaticParseScope,
artifactFileNodeKind,
artifactLocalIdentity,
javascriptAnalysisCoverage,
createElectronRoleNode,
linkElectronRoleToAsset,
type JavaScriptArtifactGraphContext,
} from "./JavaScriptArtifactGraphContext.js";
import {
artifactObservationEvidence,
astObservationEvidence,
staticInferenceEvidence,
} from "./JavaScriptArtifactGraphEvidence.js";
import { resolveArtifactPathByContext } from "./JavaScriptArtifactPathResolution.js";
@@ -127,6 +132,40 @@ export const addJavaScriptArtifactFiles = (
analyzed.javascript,
jsonValue,
);
if (
analyzed.application_projection_failure !== undefined &&
analyzed.javascript !== null
)
context.accumulator.addNode({
kind: target.kind,
identity: target.identity,
observations: [
{
label: file.path,
properties: {
static_analysis: freezeJsonSnapshot(
jsonValueSchema.parse(analyzed.javascript),
),
semantic_module_analysis:
analyzed.semantic === null
? null
: freezeJsonSnapshot(
jsonValueSchema.parse(analyzed.semantic.ir),
),
application_projection_failure:
analyzed.application_projection_failure.error,
},
evidence: astObservationEvidence({
sha256: file.sha256,
path: file.path,
operation: "retain-unexpanded-static-findings",
range: sourceFileRange(file),
coverage: javascriptAnalysisCoverage(analyzed.javascript),
limitations: [analyzed.application_projection_failure.reason],
}),
},
],
});
context.fileNodes.set(file.path, target);
if (file.kind === "javascript") context.assetNodes.set(file.path, target);
const entry = createAsarEntry(context, file);
@@ -159,7 +198,18 @@ export const addJavaScriptArtifactFiles = (
}),
});
}
if (
if (analyzed.analysis_failure !== undefined)
addUnavailableStaticParseScope(context, {
file,
asset: target,
operation: "analyze-javascript-source",
limitation: analyzed.analysis_failure.reason,
failure: analyzed.analysis_failure.error,
limits: javaScriptAnalysisCoverageLimits(
analyzed.analysis_failure.limits,
),
});
else if (
file.kind === "javascript" &&
(analyzed.javascript === null ||
analyzed.javascript.parse_status === "failed")
@@ -172,6 +222,17 @@ export const addJavaScriptArtifactFiles = (
? "JavaScript syntax could not be parsed."
: `JavaScript text was unavailable: ${file.text.reason}.`,
});
if (analyzed.application_projection_failure !== undefined)
addUnavailableStaticParseScope(context, {
file,
asset: target,
operation: "project-javascript-application",
limitation: analyzed.application_projection_failure.reason,
failure: analyzed.application_projection_failure.error,
limits: javaScriptAnalysisCoverageLimits(
analyzed.application_projection_failure.limits,
),
});
const packageValue = packagesByPath.get(file.path);
if (packageValue !== undefined && packageValue.status !== "included")
addUnavailableStaticParseScope(context, {
@@ -198,6 +259,23 @@ export const addJavaScriptArtifactFiles = (
}
};
const sourceFileRange = (
file: JavaScriptArtifactFile,
): JavaScriptSourceRange => {
if (!file.text.included)
throw new TypeError("Retained static facts require their source text");
let line = 1;
let lastLineStart = 0;
for (const match of file.text.value.matchAll(/\r\n|[\n\r\u2028\u2029]/gu)) {
line += 1;
lastLineStart = match.index + match[0].length;
}
return {
start: { line: 1, column: 0 },
end: { line, column: file.text.value.length - lastLineStart },
};
};
/** Project package metadata and its declared Electron roles. */
export const addJavaScriptPackageNodes = (
context: JavaScriptArtifactGraphContext,
@@ -407,3 +485,4 @@ const containerProperties = (container: JavaScriptArtifactContainer) => ({
bytes: container.bytes,
inventory_artifact_id: container.inventory_artifact_id,
});
import { javaScriptAnalysisCoverageLimits } from "../../domain/javascript/javascriptAnalysisResourceControls.js";
@@ -11,9 +11,15 @@ import { createJavaScriptArtifactReader as createReader } from "../../artifacts/
import type { JavaScriptApplicationGraph } from "../../domain/javascript/javascriptApplicationGraph.js";
import type { JavaScriptSemanticGraph } from "../../domain/javascript/javascriptSemanticGraph.js";
import type { ElectronBoundarySummary } from "../../domain/javascript/javascriptApplicationAnalysis.js";
import { parseOwnedJavaScriptApplicationAnalysisSteps } from "../../domain/javascript/javascriptApplicationAnalysis.js";
import { AnalysisError } from "../../domain/analysisErrorBase.js";
import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySnapshot.js";
import type { ArtifactInventoryPartialObservation } from "../../domain/artifactPartialObservation.js";
import { analyzeAndProjectJavaScriptArtifactFiles } from "./JavaScriptArtifactAnalysis.js";
import type { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
import { analyzeAndAdoptJavaScriptArtifactFiles } from "./JavaScriptArtifactAnalysis.js";
import { createJavaScriptSourceAnalysis } from "../../artifacts/javascript/JavaScriptSourceAnalysis.js";
import { JAVASCRIPT_ANALYSIS_RESOURCE_DEFAULTS } from "../../domain/javascript/javascriptAnalysisResourceControls.js";
import { createOwnedJavaScriptApplicationEvidenceCooperatively } from "./JavaScriptApplicationEvidence.js";
import { readJavaScriptArtifactFiles } from "../../artifacts/javascript/JavaScriptArtifactFiles.js";
import { buildImmutableJavaScriptArtifactGraphSteps } from "./JavaScriptArtifactGraphBuilder.js";
import {
@@ -21,7 +27,6 @@ import {
type JavaScriptArtifactReconstructionInput,
} from "./JavaScriptArtifactReconstructionInput.js";
import { scanCanonicalArtifactInventoryInScope } from "../../artifacts/inventory/scanCanonical.js";
import type { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
import { summarizeElectronBoundaries } from "./ElectronBoundaryAnalysis.js";
import { createJavaScriptSemanticGraphProjection } from "./JavaScriptSemanticGraphBuilder.js";
import type { ProgressReporter } from "../ProgressReporter.js";
@@ -63,12 +68,19 @@ export const reconstructJavaScriptArtifact = async (
progress?: ProgressReporter,
): Promise<JavaScriptArtifactReconstructionResult> =>
scope.run(async () => {
let effectiveSignal = signal;
let effectiveProgress = progress;
const reportPhase = async (
phase: string,
message: string,
): Promise<void> => {
await progress?.report({ phase, completed: 0, total: 1, message });
await checkpointJavaScriptAnalysis(signal);
await effectiveProgress?.report({
phase,
completed: 0,
total: 1,
message,
});
await checkpointJavaScriptAnalysis(effectiveSignal);
};
const input = javascriptArtifactReconstructionInputSchema.parse(rawInput);
abortIfNeeded(signal);
@@ -85,7 +97,13 @@ export const reconstructJavaScriptArtifact = async (
`Artifact inventory classified ${path} as ${snapshot.manifest.root_format}, not ${format}`,
);
const reader = createReader(path, format);
let outcome:
const readerOwner = {
kind: "reader" as const,
reader,
resource: `JavaScript artifact reader for ${path}`,
};
let readerOpen = true;
let reconstructionOutcome:
| {
readonly kind: "completed";
readonly result: JavaScriptArtifactReconstructionResult;
@@ -102,28 +120,75 @@ export const reconstructJavaScriptArtifact = async (
scope,
signal,
);
// Release acquisition before analysis consumes these owned file facts.
const earlyCleanup = await scope.release(readerOwner);
readerOpen = false;
if (earlyCleanup.kind === "failed")
throw ArtifactReaderFailure.withCleanup(
earlyCleanup.cause,
ArtifactReaderFailure.cleanupObservation(
earlyCleanup.cause,
readerOwner.resource,
),
{ kind: "artifact-inventory", inventory: snapshot },
);
await reportPhase(
"parse_javascript_sources",
`Parsing and projecting ${String(files.files.length)} application source files`,
);
const semanticProjection = createJavaScriptSemanticGraphProjection();
const analysis = await analyzeAndProjectJavaScriptArtifactFiles(
files,
semanticProjection.projectFileSteps,
async (file, completed, total) => {
abortIfNeeded(signal);
await progress?.report({
phase: "parse_javascript_source",
completed: 0,
total: 1,
message: `Parsing and projecting ${file.path} (${String(completed + 1)}/${String(total)})`,
});
abortIfNeeded(signal);
const worker = createJavaScriptSourceAnalysis(
{
heapMb:
input.max_heap_mb ?? JAVASCRIPT_ANALYSIS_RESOURCE_DEFAULTS.heapMb,
timeoutMs:
input.analysis_timeout_ms ??
JAVASCRIPT_ANALYSIS_RESOURCE_DEFAULTS.timeoutMs,
},
signal,
files.files.filter(
(file) => file.kind === "javascript" && file.text.included,
).length,
);
await checkpointJavaScriptAnalysis(signal);
abortIfNeeded(signal);
let outcome;
let cleanupFailure: AnalysisError | undefined;
try {
outcome = await analyzeAndAdoptJavaScriptArtifactFiles(
files,
worker,
semanticProjection,
async (file, completed, total) => {
abortIfNeeded(signal);
await progress?.report({
phase: "parse_javascript_source",
completed: 0,
total: 1,
message: `Parsing and projecting ${file.path} (${String(completed + 1)}/${String(total)})`,
});
abortIfNeeded(signal);
},
signal,
);
} finally {
try {
const cleanup = await scope.release({
kind: "javascript-source-analysis",
analysis: worker,
resource: `JavaScript source analyzer for ${path}`,
});
if (cleanup.kind === "failed") throw cleanup.cause;
} catch (cause: unknown) {
if (!(cause instanceof AnalysisError)) throw cause;
cleanupFailure = cause;
}
}
const analysis = outcome.analysis;
const interruption = cleanupFailure ?? outcome.interruption;
if (interruption !== null && interruption !== undefined) {
effectiveSignal = undefined;
effectiveProgress = undefined;
}
await checkpointJavaScriptAnalysis(effectiveSignal);
abortIfNeeded(effectiveSignal);
await reportPhase(
"build_javascript_application_graph",
"Constructing application and Electron boundary relationships",
@@ -139,7 +204,7 @@ export const reconstructJavaScriptArtifact = async (
);
const graph = await completeJavaScriptAnalysisSteps(
applicationGraphSteps,
signal,
effectiveSignal,
);
await reportPhase(
"build_javascript_semantic_graph",
@@ -155,63 +220,83 @@ export const reconstructJavaScriptArtifact = async (
);
const semanticGraph = await completeJavaScriptAnalysisSteps(
semanticGraphSteps,
signal,
effectiveSignal,
);
await checkpointJavaScriptAnalysis(signal);
outcome = {
kind: "completed",
result: {
input_path: path,
format,
root_artifact_sha256: snapshot.manifest.root_sha256,
inventory_manifest_id: snapshot.manifest.manifest_id,
inventory_graph_sha256: snapshot.manifest.graph_sha256,
integrity_contradictions: snapshot.integrity_contradictions,
graph,
semantic_graph: semanticGraph,
electron_summary: summarizeElectronBoundaries(analysis),
statistics: {
relevant_files: files.files.length,
nested_asar_containers: files.containers.length,
text_bytes_read: files.text_bytes_read,
invalid_utf8_files: files.invalid_utf8_files,
parsed_javascript_files: analysis.files.filter(
({ javascript }) => javascript !== null,
).length,
visited_ast_nodes: analysis.visited_ast_nodes,
findings: analysis.findings,
modules: analysis.modules,
parse_failures: analysis.parse_failures,
},
limitations: analysis.limitations,
await checkpointJavaScriptAnalysis(effectiveSignal);
const completed = {
input_path: path,
format,
root_artifact_sha256: snapshot.manifest.root_sha256,
inventory_manifest_id: snapshot.manifest.manifest_id,
inventory_graph_sha256: snapshot.manifest.graph_sha256,
integrity_contradictions: snapshot.integrity_contradictions,
graph,
semantic_graph: semanticGraph,
electron_summary: summarizeElectronBoundaries(analysis),
statistics: {
relevant_files: files.files.length,
nested_asar_containers: files.containers.length,
text_bytes_read: files.text_bytes_read,
invalid_utf8_files: files.invalid_utf8_files,
parsed_javascript_files: analysis.files.filter(
({ javascript }) => javascript !== null,
).length,
visited_ast_nodes: analysis.visited_ast_nodes,
findings: analysis.findings,
modules: analysis.modules,
parse_failures: analysis.parse_failures,
},
limitations: analysis.limitations,
};
if (interruption !== null && interruption !== undefined) {
const { electron_summary: summary, ...application } = completed;
const partial = await completeJavaScriptAnalysisSteps(
parseOwnedJavaScriptApplicationAnalysisSteps({
...application,
summary,
limitations: completed.graph.limitations,
}),
);
const { max_heap_mb, analysis_timeout_ms, ...coreInput } = input;
const parameters = {
...coreInput,
...(max_heap_mb === undefined ? {} : { max_heap_mb }),
...(analysis_timeout_ms === undefined ? {} : { analysis_timeout_ms }),
};
const evidence =
await createOwnedJavaScriptApplicationEvidenceCooperatively(
parameters,
partial,
);
throw interruption.retainPartialObservation(evidence);
}
reconstructionOutcome = { kind: "completed", result: completed };
} catch (cause: unknown) {
outcome = { kind: "failed", cause };
reconstructionOutcome = { kind: "failed", cause };
}
const owner = {
kind: "reader" as const,
reader,
resource: `JavaScript artifact reader for ${path}`,
};
const cleanupAttempt = await scope.release(owner);
if (cleanupAttempt.kind === "failed") {
const cleanup = ArtifactReaderFailure.cleanupObservation(
cleanupAttempt.cause,
owner.resource,
);
const partialObservation: ArtifactInventoryPartialObservation = {
kind: "artifact-inventory",
inventory: snapshot,
};
throw ArtifactReaderFailure.withCleanup(
outcome.kind === "failed" ? outcome.cause : cleanupAttempt.cause,
cleanup,
partialObservation,
);
if (readerOpen) {
const cleanupAttempt = await scope.release(readerOwner);
if (cleanupAttempt.kind === "failed") {
const cleanup = ArtifactReaderFailure.cleanupObservation(
cleanupAttempt.cause,
readerOwner.resource,
);
const partialObservation: ArtifactInventoryPartialObservation = {
kind: "artifact-inventory",
inventory: snapshot,
};
throw ArtifactReaderFailure.withCleanup(
reconstructionOutcome.kind === "failed"
? reconstructionOutcome.cause
: cleanupAttempt.cause,
cleanup,
partialObservation,
);
}
}
if (outcome.kind === "failed") throw outcome.cause;
return outcome.result;
if (reconstructionOutcome.kind === "failed")
throw reconstructionOutcome.cause;
return reconstructionOutcome.result;
});
const OPERATION = "analyze_javascript_application";
@@ -1,11 +1,13 @@
import { z } from "zod";
import { artifactIntegrityPolicySchema } from "../../domain/artifactIntegrityPolicy.js";
import { javaScriptAnalysisResourceControlsSchema } from "../../domain/javascript/javascriptAnalysisResourceControls.js";
/** Local ASAR/directory reconstruction request. */
export const javascriptArtifactReconstructionInputSchema = z.strictObject({
input_path: z.string().min(1),
format: z.enum(["auto", "asar", "directory"]).default("auto"),
integrity_policy: artifactIntegrityPolicySchema,
...javaScriptAnalysisResourceControlsSchema.shape,
});
/** Parsed local reconstruction request. */
@@ -1,5 +1,6 @@
import {
createJavaScriptSemanticGraph,
createJavaScriptSemanticGraphUnknown,
sealTransferredJavaScriptSemanticGraphSteps,
type JavaScriptSemanticGraph,
} from "../../domain/javascript/javascriptSemanticGraph.js";
@@ -10,6 +11,7 @@ import type {
import {
JAVASCRIPT_SEMANTIC_RELATION_FAMILIES,
JAVASCRIPT_SEMANTIC_RELATION_FAMILY,
JAVASCRIPT_SEMANTIC_RELATIONS,
} from "../../domain/javascript/javascriptSemanticGraphSchemas.js";
import type {
JavaScriptSemanticCallArgument,
@@ -33,6 +35,13 @@ import { projectSemanticResources } from "./JavaScriptSemanticGraphResourceProje
import { projectSemanticObjects } from "./JavaScriptSemanticGraphObjectProjection.js";
import { projectSemanticValues } from "./JavaScriptSemanticGraphValueProjection.js";
import { inferredSemanticEvidenceAt } from "./JavaScriptSemanticGraphEvidence.js";
import { observedSemanticEvidence } from "./JavaScriptSemanticGraphEvidence.js";
import {
partialApplicationCoverage,
type ApplicationCoverage,
} from "../../domain/javascript/javascriptApplicationEvidenceSchemas.js";
import type { JsonValue } from "../../domain/jsonValue.js";
import { javaScriptAnalysisCoverageLimits } from "../../domain/javascript/javascriptAnalysisResourceControls.js";
import { projectSemanticFunctionFingerprints } from "./JavaScriptSemanticGraphFingerprintProjection.js";
import {
projectSemanticClosureCaptures,
@@ -55,6 +64,7 @@ import {
bindSemanticGraphApplicationNodes,
type SemanticGraphProjectionState as BuilderState,
} from "./JavaScriptSemanticGraphConstruction.js";
import type { JavaScriptSemanticFileProjection } from "../../domain/javascript/javascriptSemanticFileProjection.js";
interface BuilderInput {
readonly rootArtifactSha256: string;
@@ -126,6 +136,19 @@ export const buildJavaScriptSemanticGraph = ({
/** File-local semantic projection that does not retain consumed source IR. */
export interface JavaScriptSemanticGraphProjection {
/** Node capacity offered to the next independently analyzed source file. */
readonly remainingFileNodeBudget: () => number;
/** Adopt decoded file-local facts; final validation and proof remain in this realm. */
readonly adoptFileProjection: (
projection: JavaScriptSemanticFileProjection,
) => void;
/** Preserve source-bound unknowns for an execution or result-retention failure. */
readonly recordUnavailableFile: (
file: JavaScriptArtifactFile,
reason: string,
limits: Readonly<Record<string, JsonValue>> | null,
unknownReason: "resource-limit" | "incomplete-module",
) => void;
readonly projectFile: (
file: JavaScriptArtifactFile,
ir: JavaScriptSemanticIr,
@@ -151,6 +174,12 @@ export const createJavaScriptSemanticGraphProjection =
const state = emptyState();
const fingerprints: JavaScriptSemanticFingerprint[] = [];
let truncatedFiles = 0;
let unavailableFiles = 0;
let resourceTruncatedFiles = 0;
const executionLimits = new Map<
string,
ApplicationCoverage["limits"][number]
>();
const projectSourceSteps = function* (
file: JavaScriptArtifactFile,
ir: JavaScriptSemanticIr,
@@ -207,12 +236,15 @@ export const createJavaScriptSemanticGraphProjection =
fingerprints: [...fingerprints],
unknowns,
coverage: {
status: truncatedFiles > 0 ? "partial" : "unknown",
truncated: truncatedFiles > 0,
omitted_nodes: truncatedFiles > 0 ? null : 0,
omitted_relations: truncatedFiles > 0 ? null : 0,
limits:
truncatedFiles > 0
status:
truncatedFiles > 0 || unavailableFiles > 0 ? "partial" : "unknown",
truncated: truncatedFiles > 0 || resourceTruncatedFiles > 0,
omitted_nodes: truncatedFiles > 0 || unavailableFiles > 0 ? null : 0,
omitted_relations:
truncatedFiles > 0 || unavailableFiles > 0 ? null : 0,
limits: [
...executionLimits.values(),
...(truncatedFiles > 0
? [
{
name: "semantic_graph_node_ceiling",
@@ -220,12 +252,14 @@ export const createJavaScriptSemanticGraphProjection =
unit: "items" as const,
},
]
: [],
: []),
],
families: JAVASCRIPT_SEMANTIC_RELATION_FAMILIES.map((family) => ({
family,
status: semanticFamilyStatus(family),
retained_relations: retainedByFamily.get(family) ?? 0,
omitted_relations: truncatedFiles > 0 ? null : 0,
omitted_relations:
truncatedFiles > 0 || unavailableFiles > 0 ? null : 0,
unknown_ids: unknownIdsByFamily.get(family) ?? [],
})),
},
@@ -252,6 +286,87 @@ export const createJavaScriptSemanticGraphProjection =
return graph;
};
return {
remainingFileNodeBudget: () =>
Math.min(
SEMANTIC_GRAPH_FILE_NODE_CEILING,
Math.max(0, SEMANTIC_GRAPH_NODE_CEILING - state.nodes.size),
),
adoptFileProjection: (projection) => {
const newNodes = projection.nodes.filter(
({ node_id }) => !state.nodes.has(node_id),
);
if (
newNodes.length >
Math.min(
SEMANTIC_GRAPH_FILE_NODE_CEILING,
SEMANTIC_GRAPH_NODE_CEILING - state.nodes.size,
)
)
throw new RangeError(
"Transferred semantic nodes exceed the offered tree capacity",
);
for (const context of projection.evidenceContexts)
state.evidenceContexts.adopt(context);
for (const node of newNodes) state.nodes.set(node.node_id, node);
for (const relation of projection.relations)
state.relations.set(relation.relation_id, relation);
for (const unknown of projection.unknowns)
state.unknowns.set(unknown.unknown_id, unknown);
for (const root of projection.roots) state.roots.add(root);
fingerprints.push(...projection.fingerprints);
if (projection.truncated) truncatedFiles += 1;
},
recordUnavailableFile: (file, reason, limits, unknownReason) => {
const retainedLimits = javaScriptAnalysisCoverageLimits(limits);
for (const limit of retainedLimits) {
executionLimits.set(
JSON.stringify([limit.name, limit.value, limit.unit]),
limit,
);
}
state.fileNodeBudget = Math.min(
SEMANTIC_GRAPH_FILE_NODE_CEILING,
Math.max(0, SEMANTIC_GRAPH_NODE_CEILING - state.nodes.size),
);
const module = retainNode(state, file, {
kind: "module",
roleKey: "module",
location: null,
label: file.path,
functionNodeId: null,
});
state.fileNodeBudget = null;
if (module !== null) state.roots.add(module.node_id);
const evidence = {
...observedSemanticEvidence(file, null),
authority: "unknown" as const,
state: "unknown" as const,
confidence: "unknown" as const,
coverage: partialApplicationCoverage(retainedLimits, null),
limitations: [reason],
};
for (const family of JAVASCRIPT_SEMANTIC_RELATION_FAMILIES) {
const unknown = createJavaScriptSemanticGraphUnknown(
{
node_id: module?.node_id ?? null,
family,
relation_kinds: JAVASCRIPT_SEMANTIC_RELATIONS.filter(
(relation) =>
JAVASCRIPT_SEMANTIC_RELATION_FAMILY[relation] === family,
),
reason: unknownReason,
detail: reason,
candidate_node_ids: [],
evidence,
},
state.evidenceContexts,
);
state.unknowns.set(unknown.unknown_id, unknown);
}
unavailableFiles += 1;
if (unknownReason === "resource-limit" && retainedLimits.length > 0)
resourceTruncatedFiles += 1;
},
projectFile: (file, ir) => {
const steps = projectSourceSteps(file, ir);
while (steps.next().done !== true);
@@ -272,6 +387,35 @@ export const createJavaScriptSemanticGraphProjection =
};
};
/** Project one source with the parent's remaining node capacity, retaining no full IR. */
export function* projectJavaScriptSemanticFileSteps(
file: JavaScriptArtifactFile,
ir: JavaScriptSemanticIr,
nodeBudget: number,
): Generator<void, JavaScriptSemanticFileProjection> {
if (
!Number.isSafeInteger(nodeBudget) ||
nodeBudget < 0 ||
nodeBudget > SEMANTIC_GRAPH_FILE_NODE_CEILING
)
throw new RangeError(
"File semantic projection requires a valid offered node capacity",
);
const state = emptyState();
state.fileNodeBudget = nodeBudget;
state.fileNodesDropped = false;
const fingerprints = yield* projectFileSteps(file, ir, state);
return {
roots: [...state.roots],
evidenceContexts: state.evidenceContexts.contexts,
nodes: [...state.nodes.values()],
relations: [...state.relations.values()],
unknowns: [...state.unknowns.values()],
fingerprints,
truncated: Boolean(state.fileNodesDropped),
};
}
function* projectFileSteps(
file: JavaScriptArtifactFile,
ir: JavaScriptSemanticIr,