diff --git a/docs/javascript-artifact-reconstruction.md b/docs/javascript-artifact-reconstruction.md index 90dbb92efb..4707ef0859 100644 --- a/docs/javascript-artifact-reconstruction.md +++ b/docs/javascript-artifact-reconstruction.md @@ -125,6 +125,9 @@ SHA-256 digests, inventory IDs, ASAR container identity, and `.asar.unpacked` status. Direct ASAR inputs and filesystem-backed ASAR files nested beneath a directory are supported. +JavaScript and HTML source ranges retain an initial UTF-8 BOM as one UTF-16 +code unit, matching the original bytes identified by the artifact digest. + If an ASAR declares an unpacked companion entry but the corresponding `.unpacked` file is absent from the operator-supplied artifact set, REA keeps the ASAR occurrence with `hash_status: unavailable`, records an explicit diff --git a/src/artifacts/javascript/JavaScriptArtifactFiles.ts b/src/artifacts/javascript/JavaScriptArtifactFiles.ts index 6f9cdb845b..84107c366f 100644 --- a/src/artifacts/javascript/JavaScriptArtifactFiles.ts +++ b/src/artifacts/javascript/JavaScriptArtifactFiles.ts @@ -222,7 +222,12 @@ const readText = async ( try { return { included: true, - value: new TextDecoder("utf-8", { fatal: true }).decode(bytes), + value: new TextDecoder("utf-8", { + fatal: true, + ignoreBOM: + input.expected.kind === "javascript" || + input.expected.kind === "html", + }).decode(bytes), }; } catch (cause: unknown) { // Non-UTF8 bytes are counted; the fixed reason preserves the schema. diff --git a/tests/boundary/cli/javascriptArtifactBomRanges.test.ts b/tests/boundary/cli/javascriptArtifactBomRanges.test.ts new file mode 100644 index 0000000000..9a542062c8 --- /dev/null +++ b/tests/boundary/cli/javascriptArtifactBomRanges.test.ts @@ -0,0 +1,45 @@ +import { expect } from "vitest"; + +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; +import { + bomSourceCases, + createBomArtifact, + expectBomSourceRange, +} from "../../fixtures/javascriptArtifactBom.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +for (const format of ["directory", "asar"] as const) { + for (const command of ["analyze-javascript-application", "analyze"]) { + cliTest( + `${command} preserves original BOM coordinates for ${format}`, + async ({ cli }) => { + const files = Object.fromEntries( + bomSourceCases.map((fixture) => [ + fixture.path, + Buffer.from(`\uFEFF${fixture.source}`), + ]), + ); + const input = await createBomArtifact(format, { + ...files, + "dep.js": "export const dependency = true;", + }); + const output = await cli.run({ arguments: [command, input, "--json"] }); + expect(output.exitCode).toBe(0); + const result = javascriptApplicationAnalysisResultSchema.parse( + parseEvidence(output.json).normalized_result, + ); + expect(result.statistics).toMatchObject({ + parse_failures: 0, + invalid_utf8_files: 0, + }); + for (const fixture of bomSourceCases) + expectBomSourceRange( + result.graph, + fixture, + Buffer.from(`\uFEFF${fixture.source}`), + ); + }, + ); + } +} diff --git a/tests/boundary/filesystem/javascriptArtifactBomRanges.test.ts b/tests/boundary/filesystem/javascriptArtifactBomRanges.test.ts new file mode 100644 index 0000000000..01c1741914 --- /dev/null +++ b/tests/boundary/filesystem/javascriptArtifactBomRanges.test.ts @@ -0,0 +1,97 @@ +import { expect, it } from "vitest"; + +import { reconstructJavaScriptArtifact } from "../../../src/application/javascript/JavaScriptArtifactReconstruction.js"; +import { + bomSourceCases, + createBomArtifact, + expectBomSourceRange, +} from "../../fixtures/javascriptArtifactBom.js"; + +for (const format of ["directory", "asar"] as const) { + for (const fixture of bomSourceCases) { + for (const prefix of ["", "\uFEFF"] as const) { + it(`${format} preserves ${fixture.kind} original ranges with ${prefix === "" ? "no BOM" : "a UTF-8 BOM"}`, async () => { + const source = + fixture.kind === "html" + ? `${prefix}${fixture.source}` + : `${prefix}${fixture.source}\nglobalThis.__rea_bom_executed = true; throw new Error("must remain inert");`; + const bytes = Buffer.from(source); + const input = await createBomArtifact(format, { + [fixture.path]: bytes, + "dep.js": "export const dependency = true;", + }); + const result = await reconstructJavaScriptArtifact({ + input_path: input, + }); + expect(result.statistics).toMatchObject({ + parse_failures: 0, + invalid_utf8_files: 0, + }); + expect(Reflect.get(globalThis, "__rea_bom_executed")).toBeUndefined(); + expectBomSourceRange(result.graph, fixture, bytes); + }); + } + } + + it(`${format} preserves accepted BOM handling for package, JSON and source maps`, async () => { + const input = await createBomArtifact(format, { + "package.json": '\uFEFF{"name":"bom-package","main":"dep.js"}', + "data.json": '\uFEFF{"__proto__":"owned","control":true}', + "app.js.map": + '\uFEFF{"version":3,"sources":["original.js"],"sourcesContent":["export const original = true;"],"names":[],"mappings":"AAAA"}', + "dep.js": "export const dependency = true;", + }); + const result = await reconstructJavaScriptArtifact({ input_path: input }); + expect(result.statistics).toMatchObject({ + parse_failures: 0, + invalid_utf8_files: 0, + }); + const observations = result.graph.nodes.flatMap( + ({ observations }) => observations, + ); + expect( + observations.some( + ({ properties }) => + properties.name === "bom-package" && + properties.parse_status === "included", + ), + ).toBe(true); + expect( + observations.some( + ({ properties }) => + properties.path === "data.json" && + properties.json_parse_status === "included", + ), + ).toBe(true); + expect( + result.graph.nodes.some( + ({ kind, observations }) => + kind === "source-module" && + observations.some( + ({ properties }) => properties.source === "original.js", + ), + ), + ).toBe(true); + }); + + it(`${format} retains invalid UTF-8 as unavailable rather than replacing bytes`, async () => { + const invalid = Buffer.from([0xef, 0xbb, 0xbf, 0xff]); + const input = await createBomArtifact(format, { + "invalid.js": invalid, + "invalid.html": invalid, + }); + const result = await reconstructJavaScriptArtifact({ input_path: input }); + expect(result.statistics.invalid_utf8_files).toBe(2); + const observations = result.graph.nodes.flatMap( + ({ observations }) => observations, + ); + for (const path of ["invalid.js", "invalid.html"]) + expect( + observations.some( + ({ properties }) => + properties.path === path && + properties.text_status === "invalid-utf8", + ), + ).toBe(true); + }); +} diff --git a/tests/fixtures/javascriptArtifactBom.ts b/tests/fixtures/javascriptArtifactBom.ts new file mode 100644 index 0000000000..61e2f6b955 --- /dev/null +++ b/tests/fixtures/javascriptArtifactBom.ts @@ -0,0 +1,111 @@ +import { createHash } from "node:crypto"; +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { createPackageWithOptions } from "@electron/asar"; +import { parse as parseJavaScript } from "@babel/parser"; +import { + defaultTreeAdapter, + parse as parseHtml, + type DefaultTreeAdapterTypes, +} from "parse5"; +import { expect } from "vitest"; + +import type { JavaScriptSourceRange } from "../../src/domain/javascript/javascriptStaticAnalysisTypes.js"; +import type { JavaScriptApplicationGraph } from "../../src/domain/javascript/javascriptApplicationGraph.js"; +import { createTestTempDirectory } from "./temporaryDirectory.js"; + +export const bomSourceCases = [ + { path: "import.js", kind: "static-import", source: 'import "./dep.js";' }, + { path: "require.cjs", kind: "require", source: 'require("./dep.js");' }, + { + path: "index.html", + kind: "html", + source: '', + }, +] as const; + +export const createBomArtifact = async ( + format: "directory" | "asar", + files: Readonly>, +): Promise => { + const root = await createTestTempDirectory("rea-bom-artifact-"); + const directory = join(root, "source"); + await mkdir(directory); + await Promise.all( + Object.entries(files).map(([path, content]) => + writeFile(join(directory, path), content), + ), + ); + if (format === "directory") return directory; + const archive = join(root, "app.asar"); + await createPackageWithOptions(directory, archive, {}); + return archive; +}; + +/** The oracle parses original UTF-8 bytes with their BOM retained. */ +export const expectBomSourceRange = ( + graph: JavaScriptApplicationGraph, + fixture: (typeof bomSourceCases)[number], + bytes: Buffer, +): void => { + const original = bytes.toString("utf8"); + const digest = createHash("sha256").update(bytes).digest("hex"); + let expectedRange: JavaScriptSourceRange; + if (fixture.kind === "html") { + const pending: DefaultTreeAdapterTypes.Node[] = [ + parseHtml(original, { sourceCodeLocationInfo: true }), + ]; + let opening; + while (pending.length > 0) { + const node = pending.pop(); + if (node === undefined) break; + if (defaultTreeAdapter.isElementNode(node) && node.tagName === "script") { + opening = node.sourceCodeLocation?.startTag; + break; + } + if ("childNodes" in node) pending.push(...node.childNodes); + } + if (opening === undefined) throw new Error("Missing original HTML script"); + expectedRange = { + start: { line: opening.startLine, column: opening.startCol - 1 }, + end: { line: opening.endLine, column: opening.endCol - 1 }, + }; + } else { + const statement = parseJavaScript(original, { + sourceType: "unambiguous", + }).program.body[0]; + const target = + fixture.kind === "require" && statement?.type === "ExpressionStatement" + ? statement.expression + : statement; + if (target?.loc === undefined || target.loc === null) + throw new Error("Missing original JavaScript reference"); + expectedRange = { + start: { line: target.loc.start.line, column: target.loc.start.column }, + end: { line: target.loc.end.line, column: target.loc.end.column }, + }; + } + const edges = graph.edges.filter( + ({ relation, properties, evidence }) => + relation === (fixture.kind === "html" ? "loads" : "imports") && + (fixture.kind === "html" + ? properties.script_path === "./dep.js" + : properties.kind === fixture.kind && + properties.specifier === "./dep.js") && + evidence.location.available && + evidence.location.value.kind === "source-range" && + evidence.location.value.source === fixture.path, + ); + expect(edges).toHaveLength(1); + expect(edges[0]).toMatchObject({ + properties: { resolved_path: "dep.js" }, + evidence: { + artifact: { available: true, sha256: digest }, + location: { + available: true, + value: { kind: "source-range", source: fixture.path, ...expectedRange }, + }, + }, + }); +};