diff --git a/src/application/BinaryTargetResolver.ts b/src/application/BinaryTargetResolver.ts index 348454e72d..7189ed7dec 100644 --- a/src/application/BinaryTargetResolver.ts +++ b/src/application/BinaryTargetResolver.ts @@ -32,7 +32,11 @@ import { } from "../domain/rawBinary.js"; import type { BinaryArchitecture } from "../domain/binaryTargetTypes.js"; import { observeMzWindowsHeader } from "../domain/dosMz.js"; -import { hasBinaryPlistSignature } from "../domain/apple/appleArchiveFormat.js"; +import { + createXmlPlistSignatureProbe, + hasBinaryPlistSignature, + MAX_XML_PLIST_PROBE_BYTES, +} from "../domain/apple/appleArchiveFormat.js"; import { hasZipSignature, isZipPackageFormat, @@ -49,6 +53,8 @@ import { } from "../domain/binaryTarget.js"; import type { PeCodeViewRecord } from "../domain/binaryTargetTypes.js"; +const XML_PLIST_PROBE_CHUNK_BYTES = 64 * 1024; + /** Options for resolving a local binary target and its stable identity. */ export interface BinaryTargetResolverOptions { readonly cwd?: string; @@ -288,12 +294,12 @@ const detectArtifactFormat = async ( > => { throwIfTargetResolutionCancelled(signal); const lower = path.toLowerCase(); - const magic = Buffer.alloc(8); + const magic = Buffer.alloc(4096); const observed = await handle.read(magic, 0, magic.length, 0); throwIfTargetResolutionCancelled(signal); if ( - observed.bytesRead === 8 && - magic.equals(Buffer.from([0, 0x61, 0x73, 0x6d, 1, 0, 0, 0])) + observed.bytesRead >= 8 && + magic.subarray(0, 8).equals(Buffer.from([0, 0x61, 0x73, 0x6d, 1, 0, 0, 0])) ) throw new BinaryTargetError( path, @@ -329,6 +335,37 @@ const detectArtifactFormat = async ( return "dmg"; } } + // XML property lists can use archive-specific suffixes such as .archive. + // The named-format check above keeps .js, .map, and .asar classifications. + const xmlProbe = createXmlPlistSignatureProbe(); + let xmlState = xmlProbe.write(magic.subarray(0, observed.bytesRead)); + if (xmlState === "matched") return "plist"; + const chunk = Buffer.alloc(XML_PLIST_PROBE_CHUNK_BYTES); + let bytesRead = observed.bytesRead; + while (xmlState === "pending" && bytesRead < MAX_XML_PLIST_PROBE_BYTES) { + throwIfTargetResolutionCancelled(signal); + const read = await handle.read( + chunk, + 0, + Math.min(chunk.length, MAX_XML_PLIST_PROBE_BYTES - bytesRead), + bytesRead, + ); + throwIfTargetResolutionCancelled(signal); + if (read.bytesRead === 0) break; + bytesRead += read.bytesRead; + xmlState = xmlProbe.write(chunk.subarray(0, read.bytesRead)); + if (xmlState === "matched") return "plist"; + } + if (bytesRead === MAX_XML_PLIST_PROBE_BYTES) { + const { size } = await handle.stat(); + throwIfTargetResolutionCancelled(signal); + if (size > bytesRead) + throw new BinaryTargetError( + path, + "XML plist prolog was not found within the 1 MiB format probe limit", + { constraint: "file_format_rejected" }, + ); + } return undefined; }; diff --git a/src/application/ReferenceSourceImport.ts b/src/application/ReferenceSourceImport.ts index 769f8bb8cd..e07a381460 100644 --- a/src/application/ReferenceSourceImport.ts +++ b/src/application/ReferenceSourceImport.ts @@ -151,6 +151,7 @@ const createShouldExclude = throw new Error(`Ignored secret path has no matching rule: ${path}`); exclusions.push({ path, + entry_kind: kind, reason: "configured-secret", pattern: secretMatch.rule.pattern, }); @@ -167,7 +168,12 @@ const createShouldExclude = reason !== "caller-excluded" ) throw new Error(`Ignored path has unknown rule origin: ${path}`); - exclusions.push({ path, reason, pattern: match.rule.pattern }); + exclusions.push({ + path, + entry_kind: kind, + reason, + pattern: match.rule.pattern, + }); return true; }; diff --git a/src/application/ReferenceSourceVcsAdapter.ts b/src/application/ReferenceSourceVcsAdapter.ts index 346ae6ebe1..ac05ce0a4a 100644 --- a/src/application/ReferenceSourceVcsAdapter.ts +++ b/src/application/ReferenceSourceVcsAdapter.ts @@ -162,14 +162,21 @@ const resolveSourceHead = async ( // The library probes readFile() without a path to detect promise APIs. // Reject that invocation without attributing it to a source read. if (path === undefined) - throw new TypeError("Git filesystem read requires a path"); + throw new TypeError("Git metadata path is required"); try { const bytes = await readMetadataBytes(path); const encoding = - typeof options === "string" ? options : options?.encoding; - return encoding === undefined || encoding === null - ? bytes - : bytes.toString(encoding); + typeof options === "string" + ? options + : typeof options === "object" && + options !== null && + "encoding" in options + ? options.encoding + : undefined; + if (encoding === undefined || encoding === null) return bytes; + if (typeof encoding !== "string" || !Buffer.isEncoding(encoding)) + throw new TypeError("Unsupported Git metadata text encoding"); + return bytes.toString(encoding); } catch (cause: unknown) { if (cause instanceof ArtifactReaderFailure) { cleanupFailures.push(cause); diff --git a/src/artifacts/ArtifactHash.ts b/src/artifacts/ArtifactHash.ts index b3bf43bfab..c1234f847e 100644 --- a/src/artifacts/ArtifactHash.ts +++ b/src/artifacts/ArtifactHash.ts @@ -3,6 +3,10 @@ import type { Readable } from "node:stream"; import { ArtifactReaderFailure } from "./ArtifactReader.js"; import { streamChunkToBuffer } from "./StreamBytes.js"; +import { + createXmlPlistSignatureProbe, + MAX_XML_PLIST_PROBE_BYTES, +} from "../domain/apple/appleArchiveFormat.js"; /** Full stream digest and byte count with bounded classification evidence. */ export type HashResult = { @@ -31,19 +35,32 @@ export const hashReadable = async ( const hash = createHash("sha256"); const prefixes: Buffer[] = []; let prefixBytes = 0; + const xmlProbe = createXmlPlistSignatureProbe(); let bytes = 0; for await (const raw of stream) { abortIfNeeded(signal); const chunk = streamChunkToBuffer(raw); bytes += chunk.length; hash.update(chunk); - if (prefixBytes < ARTIFACT_CLASSIFICATION_PREFIX_BYTES) { + let chunkOffset = 0; + while ( + chunkOffset < chunk.length && + (prefixBytes < ARTIFACT_CLASSIFICATION_PREFIX_BYTES || + (xmlProbe.state === "pending" && + prefixBytes < MAX_XML_PLIST_PROBE_BYTES)) + ) { + const limit = + prefixBytes < ARTIFACT_CLASSIFICATION_PREFIX_BYTES + ? ARTIFACT_CLASSIFICATION_PREFIX_BYTES + : MAX_XML_PLIST_PROBE_BYTES; const selected = chunk.subarray( - 0, - ARTIFACT_CLASSIFICATION_PREFIX_BYTES - prefixBytes, + chunkOffset, + Math.min(chunkOffset + limit - prefixBytes, chunk.length), ); prefixes.push(selected); prefixBytes += selected.length; + chunkOffset += selected.length; + xmlProbe.write(selected); } } return { sha256: hash.digest("hex"), bytes, prefix: Buffer.concat(prefixes) }; diff --git a/src/artifacts/ArtifactPaths.ts b/src/artifacts/ArtifactPaths.ts index ea81ea41fa..b831659072 100644 --- a/src/artifacts/ArtifactPaths.ts +++ b/src/artifacts/ArtifactPaths.ts @@ -4,7 +4,7 @@ import { logicalPathNfc } from "../domain/artifactIdentity.js"; import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; import { ArtifactReaderFailure } from "./ArtifactReader.js"; -const DRIVE_OR_UNC = /^(?:[A-Za-z]:|\\|\/\/)/u; +const DRIVE_PREFIX = /^[A-Za-z]:/u; /** * English Unicode case folding used only to describe logical names a @@ -20,14 +20,9 @@ export const ENGLISH_UNICODE_CASE_INVENTORY_LIMITATION = `Logical archive names export const DESTINATION_CASE_COLLISION_PREFIX = "Destination filesystem cannot store both "; -/** Normalize an untrusted logical archive path without touching filesystem. */ +/** Normalize slash-separated logical names; backslashes are literal components. */ export const normalizeArtifactPath = (input: string): string => { - if ( - input.includes("\0") || - input.includes("\\") || - input.startsWith("/") || - DRIVE_OR_UNC.test(input) - ) + if (input.includes("\0") || input.startsWith("/") || DRIVE_PREFIX.test(input)) throw new ArtifactReaderFailure( "path", `Artifact path is absolute or unsafe: ${JSON.stringify(input)}`, diff --git a/src/artifacts/DirectoryArtifactReader.ts b/src/artifacts/DirectoryArtifactReader.ts index 399bf979b9..56ef4ccc76 100644 --- a/src/artifacts/DirectoryArtifactReader.ts +++ b/src/artifacts/DirectoryArtifactReader.ts @@ -30,6 +30,7 @@ interface DirectoryTraversalOptions extends ArtifactTraversalOptions { export class DirectoryArtifactReader implements ArtifactReader { readonly format = "directory" as const; #rootPromise: Promise | undefined; + #rootIdentity: { readonly dev: bigint; readonly ino: bigint } | undefined; readonly #rootPath: string; readonly #directories = new Set(); readonly #files = new Map(); @@ -52,7 +53,7 @@ export class DirectoryArtifactReader implements ArtifactReader { const directory = pending.pop(); if (directory === undefined) break; await withDirectoryIoContext("inspect directory", directory, () => - assertContainedDirectory(root, directory), + assertContainedDirectory(root, directory, this.#rootIdentity), ); for await (const entry of this.#readDirectory( directory, @@ -62,21 +63,26 @@ export class DirectoryArtifactReader implements ArtifactReader { )) yield entry; await withDirectoryIoContext("inspect directory", directory, () => - assertContainedDirectory(root, directory), + assertContainedDirectory(root, directory, this.#rootIdentity), ); } } async #resolveRoot(): Promise { - return (this.#rootPromise ??= realpath(this.#rootPath)).catch( - (cause: unknown) => { - throw directoryIoFailure( - "resolve artifact root", - this.#rootPath, - cause, + return (this.#rootPromise ??= (async () => { + const root = await realpath(this.#rootPath); + const metadata = await lstat(root, { bigint: true }); + if (metadata.isSymbolicLink() || !metadata.isDirectory()) + throw new ArtifactReaderFailure( + "path", + "Artifact root changed during traversal", ); - }, - ); + this.#rootIdentity = { dev: metadata.dev, ino: metadata.ino }; + return root; + })().catch((cause: unknown) => { + if (cause instanceof ArtifactReaderFailure) throw cause; + throw directoryIoFailure("resolve artifact root", this.#rootPath, cause); + })); } async *#readDirectory( @@ -100,6 +106,9 @@ export class DirectoryArtifactReader implements ArtifactReader { }; try { for (;;) { + await withDirectoryIoContext("inspect directory", directory, () => + assertContainedDirectory(root, directory, this.#rootIdentity), + ); const child = await handle.handle.read(); if (child === null) break; abortIfNeeded(signal); @@ -440,11 +449,16 @@ const abortIfNeeded = (signal?: AbortSignal): void => { const assertContainedDirectory = async ( root: string, directory: string, + rootIdentity: { readonly dev: bigint; readonly ino: bigint } | undefined, ): Promise => { const canonical = await realpath(directory); const fromRoot = relative(root, canonical); const metadata = await lstat(directory); + const currentRoot = await lstat(root, { bigint: true }); if ( + rootIdentity === undefined || + currentRoot.dev !== rootIdentity.dev || + currentRoot.ino !== rootIdentity.ino || metadata.isSymbolicLink() || !metadata.isDirectory() || fromRoot === ".." || diff --git a/src/artifacts/SafeOutputTree.ts b/src/artifacts/SafeOutputTree.ts index 73885ad05e..0cf4dab98c 100644 --- a/src/artifacts/SafeOutputTree.ts +++ b/src/artifacts/SafeOutputTree.ts @@ -580,6 +580,11 @@ const normalizeDestinationPath = ( platform: NodeJS.Platform, ): string => { const path = normalizeArtifactPath(relativePath); + if (platform === "win32" && path.includes("\\")) + throw new ArtifactReaderFailure( + "path", + `Windows destination cannot materialize artifact path: ${JSON.stringify(path)}; backslashes denote path separators.`, + ); if (platform === "win32" && path.includes(":")) throw new ArtifactReaderFailure( "path", diff --git a/src/artifacts/apple/InterfaceBuilderDecodeBudget.test.ts b/src/artifacts/apple/InterfaceBuilderDecodeBudget.test.ts new file mode 100644 index 0000000000..b02aee7459 --- /dev/null +++ b/src/artifacts/apple/InterfaceBuilderDecodeBudget.test.ts @@ -0,0 +1,106 @@ +import { createHash } from "node:crypto"; +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { analyzeInterfaceBuilderBundle } from "./InterfaceBuilderAnalysis.js"; +import { preflightPropertyListDecode } from "./InterfaceBuilderDecodeBudget.js"; +import { parseXmlPropertyList } from "../../domain/propertyListKeys.js"; +import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; + +const example = ""; +const archive = (content: string, comment = "") => + `${comment}$archiverNSKeyedArchiver$objects$null$classCF$UID2title${content}$classnameNSButton$classesNSButtonNSObject$toprootCF$UID1`; + +const inertDeclarations = [ + { + kind: "CDATA text", + xml: archive(``), + title: example, + }, + { + kind: "XML comment", + xml: archive("Build", ``), + title: "Build", + }, +] as const; + +describe("inert XML declaration examples in Interface Builder archives", () => { + it.each(inertDeclarations)( + "preflights $kind without treating it as entity expansion", + ({ xml }) => { + expect(() => parseXmlPropertyList(xml)).not.toThrow(); + expect( + preflightPropertyListDecode(Buffer.from(xml), 16 * 1024 * 1024), + ).toMatchObject({ + binaryPrototypeKeyObserved: false, + }); + }, + ); + + it.each(inertDeclarations)( + "retains $kind and archive provenance through bundle decoding", + async ({ xml, title }) => { + const root = await createTestTempDirectory("rea-ib-inert-declaration-"); + const bundle = join(root, "Example.app"); + await mkdir(bundle); + const bytes = Buffer.from(xml); + await writeFile(join(bundle, "Panel.nib"), bytes); + + const result = await analyzeInterfaceBuilderBundle({ + bundlePath: bundle, + targetSha256: "a".repeat(64), + }); + + expect(result.documents).toContainEqual( + expect.objectContaining({ + relative_path: "Panel.nib", + archive_sha256: createHash("sha256").update(bytes).digest("hex"), + }), + ); + expect(result.graph.nodes.map(({ name }) => name)).toContain(title); + expect(result.graph.coverage).toContainEqual( + expect.objectContaining({ + facet: "archive_decode", + status: "complete", + }), + ); + }, + ); + + it("continues to refuse an actual internal entity declaration before decoding", () => { + const xml = archive("&harmless;").replace( + "]> + preflightPropertyListDecode(Buffer.from(xml), 16 * 1024 * 1024), + ).toThrow( + "XML plist internal entity expansion exceeds the aggregate Interface Builder decode budget", + ); + }); + + it("does not mistake a bracket in a quoted external identifier for an internal subset", () => { + const xml = archive("Build").replace( + " parseXmlPropertyList(xml)).not.toThrow(); + expect(() => + preflightPropertyListDecode(Buffer.from(xml), 16 * 1024 * 1024), + ).not.toThrow(); + }); + + it.each([ + '[edition].dtd">', + "[edition].dtd'>", + " ?>", + ])("consumes quoted or opaque declaration text once: %s", (declaration) => { + const xml = archive("Build").replace(" parseXmlPropertyList(xml)).not.toThrow(); + expect( + preflightPropertyListDecode(Buffer.from(xml), 16 * 1024 * 1024), + ).toMatchObject({ binaryPrototypeKeyObserved: false }); + }); +}); diff --git a/src/artifacts/apple/InterfaceBuilderDecodeBudget.ts b/src/artifacts/apple/InterfaceBuilderDecodeBudget.ts index 3593fa43e1..5d143f52a2 100644 --- a/src/artifacts/apple/InterfaceBuilderDecodeBudget.ts +++ b/src/artifacts/apple/InterfaceBuilderDecodeBudget.ts @@ -115,28 +115,23 @@ const estimateXmlPlistExpansion = ( xmlText?: string, ): number => { const xml = xmlText ?? decodeXmlPlistText(bytes); - if (/]*\[/iu.test(xml)) - throw new InterfaceBuilderDecodeBudgetExceeded( - "aggregate_decode_budget_exhausted", - "XML plist internal entity expansion exceeds the aggregate Interface Builder decode budget", - ); - const elementPattern = - /||]*>|<\?[^]*?\?>|<\/?[A-Za-z][^>]*>/giu; const decodedSourceBytes = xml.length * 2; let elementCount = 0; let xmlDepth = 0; - for (const match of xml.matchAll(elementPattern)) { - const token = match[0] ?? ""; - if ( - token.startsWith("" + : xml.startsWith("" + : xml.startsWith("" + : undefined; + if (opaqueEnd !== undefined) { + const end = xml.indexOf(opaqueEnd, start + 2); + if (end < 0) return; + cursor = end + opaqueEnd.length; + continue; + } + const doctype = /^") break; + else if (doctype && character === "[") internalSubset = true; + } + if (end === xml.length) return; + cursor = end + 1; + if (doctype || element) + yield { + doctype, + internalSubset, + closing: xml[start + 1] === "/", + selfClosing: xml[end - 1] === "/", + }; + } +} + /** Resource a binary plist preflight found over its budget, or a malformed reference cycle. */ export type BinaryPlistExpansionFailure = "cycle" | "depth" | "expansion"; diff --git a/src/artifacts/inventory/ArtifactGraphConstruction.test.ts b/src/artifacts/inventory/ArtifactGraphConstruction.test.ts index 925d3543ab..6531d4557a 100644 --- a/src/artifacts/inventory/ArtifactGraphConstruction.test.ts +++ b/src/artifacts/inventory/ArtifactGraphConstruction.test.ts @@ -101,3 +101,25 @@ describe("bounded FAT inventory classification", () => { }, ); }); + +it("preserves path-selected inventory roles for XML content", () => { + const xml = Buffer.from( + '', + ); + expect(classifyArtifactContent("state.archive", xml)).toEqual({ + kind: "plist", + format: "plist", + }); + expect(classifyArtifactContent("state.js", xml)).toEqual({ + kind: "javascript", + format: "javascript-bundle", + }); + expect(classifyArtifactContent("state.map", xml)).toEqual({ + kind: "source-map", + format: "source-map", + }); + expect(classifyArtifactContent("state.asar", xml)).toEqual({ + kind: "resource", + format: "file", + }); +}); diff --git a/src/artifacts/inventory/ArtifactGraphConstruction.ts b/src/artifacts/inventory/ArtifactGraphConstruction.ts index b7b1947447..f2564d254a 100644 --- a/src/artifacts/inventory/ArtifactGraphConstruction.ts +++ b/src/artifacts/inventory/ArtifactGraphConstruction.ts @@ -1,6 +1,10 @@ import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js"; +import { isJavaScriptSourcePath } from "../../domain/javascript/javascriptArtifactFiles.js"; import { digestCanonicalValue } from "../../domain/canonicalDigest.js"; -import { hasBinaryPlistSignature } from "../../domain/apple/appleArchiveFormat.js"; +import { + hasBinaryPlistSignature, + hasXmlPlistSignature, +} from "../../domain/apple/appleArchiveFormat.js"; import { artifactEdgeId, artifactIdForContent, @@ -365,7 +369,7 @@ export const classifyArtifactPath = ( const lower = path.toLowerCase(); if (lower.endsWith(".map")) return { kind: "source-map", format: "source-map" }; - if (/\.(?:m?js|cjs)$/u.test(lower)) + if (isJavaScriptSourcePath(path)) return { kind: "javascript", format: "javascript-bundle" }; if (/\.framework(?:\/|$)/u.test(lower)) return { kind: "framework", format: "file" }; @@ -390,6 +394,14 @@ export const classifyArtifactContent = ( readonly format: ArtifactOccurrence["artifact_format"]; } => { const format = classifyArtifactBytes(prefix, fileSize); + const byPath = classifyArtifactPath(path); + if ( + format === "plist" && + !hasBinaryPlistSignature(prefix) && + (["javascript", "source-map", "entitlements"].includes(byPath.kind) || + path.toLowerCase().endsWith(".asar")) + ) + return byPath; if ( format === "file" && path.toLowerCase().endsWith(".asar") && @@ -442,6 +454,7 @@ export const classifyArtifactBytes = ( ): ArtifactNode["format"] => { if (hasZipSignature(prefix)) return "zip"; if (hasBinaryPlistSignature(prefix)) return "plist"; + if (hasXmlPlistSignature(prefix)) return "plist"; const fatFormat = classifyFatArtifact(prefix, fileSize); if (fatFormat !== null) return fatFormat; if (prefix.length >= 4) { diff --git a/src/artifacts/inventory/classify.ts b/src/artifacts/inventory/classify.ts index 6322d60282..73cc6d760b 100644 --- a/src/artifacts/inventory/classify.ts +++ b/src/artifacts/inventory/classify.ts @@ -5,7 +5,14 @@ import type { StableRegularFileDescriptor } from "../../filesystem/RegularFile.j import type { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js"; import { classifyArtifactContent } from "./ArtifactGraphConstruction.js"; -import { ARTIFACT_CLASSIFICATION_PREFIX_BYTES } from "../ArtifactHash.js"; +import { + abortIfNeeded, + ARTIFACT_CLASSIFICATION_PREFIX_BYTES, +} from "../ArtifactHash.js"; +import { + createXmlPlistSignatureProbe, + MAX_XML_PLIST_PROBE_BYTES, +} from "../../domain/apple/appleArchiveFormat.js"; import type { ArtifactOccurrence } from "../../domain/artifactGraph.js"; import { hasZipSignature, @@ -104,7 +111,7 @@ export const classifyAndHashRootForInventory = async ( "integrity", `Root artifact changed before inventory: ${path}`, ); - const format = await classifyRootFormat(path, handle); + const format = await classifyRootFormat(path, handle, options.signal); const digest = await hashStableRootArtifactHandle( path, handle, @@ -144,10 +151,11 @@ export const classifyAndHashRootForInventory = async ( const classifyRootFormat = async ( path: string, handle: FileHandle, + signal?: AbortSignal, ): Promise => { const magic = Buffer.alloc(ARTIFACT_CLASSIFICATION_PREFIX_BYTES); const observed = await handle.read(magic, 0, magic.length, 0); - const prefix = magic.subarray(0, observed.bytesRead); + let prefix = magic.subarray(0, observed.bytesRead); const lower = path.toLowerCase(); if (hasZipSignature(prefix)) return zipRootPackageFormat(lower); if ( @@ -156,6 +164,27 @@ const classifyRootFormat = async ( ) return "pkg"; if (lower.endsWith(".dmg") && (await hasKolyTrailer(handle))) return "dmg"; + const xmlProbe = createXmlPlistSignatureProbe(); + xmlProbe.write(prefix); + const chunks = [prefix]; + let prefixBytes = prefix.length; + while ( + xmlProbe.state === "pending" && + prefixBytes < MAX_XML_PLIST_PROBE_BYTES + ) { + abortIfNeeded(signal); + const chunk = Buffer.alloc( + Math.min(64 * 1024, MAX_XML_PLIST_PROBE_BYTES - prefixBytes), + ); + const read = await handle.read(chunk, 0, chunk.length, prefixBytes); + abortIfNeeded(signal); + if (read.bytesRead === 0) break; + const selected = chunk.subarray(0, read.bytesRead); + chunks.push(selected); + prefixBytes += selected.length; + xmlProbe.write(selected); + } + prefix = Buffer.concat(chunks); // ASAR is chosen from its header pickle; any other suffix is a role hint. return classifyArtifactContent(path, prefix, (await handle.stat()).size) .format; diff --git a/src/artifacts/javascript/JavaScriptArtifactFiles.ts b/src/artifacts/javascript/JavaScriptArtifactFiles.ts index 78d8cf7501..d4845feacb 100644 --- a/src/artifacts/javascript/JavaScriptArtifactFiles.ts +++ b/src/artifacts/javascript/JavaScriptArtifactFiles.ts @@ -19,11 +19,14 @@ import { hashReadable } from "../ArtifactHash.js"; import type { ArtifactOccurrence } from "../../domain/artifactGraph.js"; import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySnapshot.js"; import { AnalysisInputError } from "../../domain/analysisErrorCore.js"; +import { isJavaScriptSourcePath } from "../../domain/javascript/javascriptArtifactFiles.js"; import type { JavaScriptArtifactFileKind, JavaScriptArtifactFile, JavaScriptArtifactContainer, + JavaScriptUnanalyzedArtifactFile, + JavaScriptUnanalyzedArtifactLookup, JavaScriptArtifactFileSet, } from "../../domain/javascript/javascriptArtifactFiles.js"; @@ -140,6 +143,10 @@ export const readJavaScriptArtifactFiles = async ( assertExpectedFilesWereVisited(expected, files); return { files, + unanalyzed_files: createUnanalyzedArtifactLookup( + snapshot, + context.containers, + ), containers: context.containers.sort((left, right) => compareUnicodeCodePoints(left.path, right.path), ), @@ -438,6 +445,118 @@ const expectedInventory = ( return { files, containers, unavailableFiles }; }; +/** Canonical snapshot arrays are already ordered by code-point identity. */ +const lowerBound = ( + rows: readonly T[], + key: string, + identity: (row: T) => string, +): number => { + let low = 0; + let high = rows.length; + while (low < high) { + const middle = low + Math.floor((high - low) / 2); + const row = rows[middle]; + if (row !== undefined && compareUnicodeCodePoints(identity(row), key) < 0) + low = middle + 1; + else high = middle; + } + return low; +}; + +const createUnanalyzedArtifactLookup = ( + snapshot: ArtifactInventorySnapshot, + containers: readonly JavaScriptArtifactContainer[], +): JavaScriptUnanalyzedArtifactLookup => { + const requested = new Map< + string, + JavaScriptUnanalyzedArtifactFile | undefined + >(); + const directories = new Map>(); + const containerIds = new Map( + containers.map((container) => [container.path, container.sha256]), + ); + const containerFor = (path: string): string => { + let prefix = path; + for (;;) { + const slash = prefix.lastIndexOf("/"); + if (slash < 0) return snapshot.manifest.root_sha256; + prefix = prefix.slice(0, slash); + const sha256 = containerIds.get(prefix); + if (sha256 !== undefined) return sha256; + } + }; + const isOpaque = (occurrence: ArtifactOccurrence): boolean => + occurrence.entry_kind === "file" && + relevantKind(occurrence.logical_path) === undefined; + return { + get(path) { + if (requested.has(path)) return requested.get(path); + const occurrence = + snapshot.occurrences[ + lowerBound(snapshot.occurrences, path, (entry) => entry.logical_path) + ]; + const id = + occurrence?.logical_path === path && isOpaque(occurrence) + ? occurrence.artifact_id + : undefined; + const candidate = + id == null + ? undefined + : snapshot.nodes[ + lowerBound(snapshot.nodes, id, (entry) => entry.artifact_id) + ]; + const node = candidate?.artifact_id === id ? candidate : undefined; + const file = + occurrence?.logical_path !== path || !isOpaque(occurrence) + ? undefined + : { + path, + container_sha256: containerFor(path), + sha256: node?.sha256 ?? null, + bytes: node?.size ?? occurrence.declared_size, + inventory_artifact_id: occurrence.artifact_id, + limitations: occurrence.limitations, + }; + requested.set(path, file); + return file; + }, + hasDirectory(path, containerSha256) { + let results = directories.get(path); + const previous = results?.get(containerSha256); + if (previous !== undefined) return previous; + const prefix = `${path}/`; + let present = false; + for ( + let index = lowerBound( + snapshot.occurrences, + prefix, + (entry) => entry.logical_path, + ); + index < snapshot.occurrences.length; + index += 1 + ) { + const entry = snapshot.occurrences[index]; + if (entry === undefined || !entry.logical_path.startsWith(prefix)) + break; + if ( + isOpaque(entry) && + (containerSha256 === undefined || + containerFor(entry.logical_path) === containerSha256) + ) { + present = true; + break; + } + } + if (results === undefined) { + results = new Map(); + directories.set(path, results); + } + results.set(containerSha256, present); + return present; + }, + }; +}; + const readAll = async ( stream: Readable, signal?: AbortSignal, @@ -471,7 +590,7 @@ const relevantKind = (path: string): JavaScriptArtifactFileKind | undefined => { if (lower === "package.json" || lower.endsWith("/package.json")) return "package-json"; if (lower.endsWith(".json")) return "json"; - if (/\.(?:cjs|cts|mjs|mts|js|jsx|ts|tsx)$/u.test(lower)) return "javascript"; + if (isJavaScriptSourcePath(path)) return "javascript"; if (/\.html?$/u.test(lower)) return "html"; if (lower.endsWith(".map")) return "source-map"; if (lower.endsWith(".node")) return "native-addon"; diff --git a/src/domain/android/androidAnalysis.ts b/src/domain/android/androidAnalysis.ts index eaf521a50f..83439518eb 100644 --- a/src/domain/android/androidAnalysis.ts +++ b/src/domain/android/androidAnalysis.ts @@ -187,8 +187,8 @@ export const androidResultSchemas = { top_class: z.string(), source_line: z.null(), dex_offset: z.null(), - /** false means transport-bounded instruction evidence; null/omitted is unknown. */ - direct_invokes_exhaustive: z.boolean().nullable().default(null), + /** false means transport-bounded instruction evidence; null is unknown. */ + direct_invokes_exhaustive: z.boolean().nullable(), /** Raw direct invoke instructions inside this using node, when available. */ direct_invokes: z .array( diff --git a/src/domain/android/androidApplication.ts b/src/domain/android/androidApplication.ts index 99b20c321d..6425c5bacc 100644 --- a/src/domain/android/androidApplication.ts +++ b/src/domain/android/androidApplication.ts @@ -18,6 +18,8 @@ import { projectionCoverage, projectionDigest, projectionEvidenceIdSchema, + unavailableComponentSchema, + unavailableComponents, } from "../mobileApplicationGraphProjection.js"; const evidenceIdSchema = projectionEvidenceIdSchema; @@ -42,6 +44,7 @@ export const androidApplicationProjectionResultSchema = z.strictObject({ javascript: z.array(componentSchema), signing: z.array(componentSchema), }), + unavailable_components: z.array(unavailableComponentSchema), runtime_families: z.array( z.enum([ "dalvik-art", @@ -92,6 +95,7 @@ export const projectAndroidApplication = ( `Android application projection requires APK inventory Evidence; the inventory root format is ${inventory.manifest.root_format}. APK is chosen for a ZIP archive whose name ends in .apk.`, ); const all = projectedComponents(inventory); + const unavailable = unavailableComponents(inventory); const classified = classify(all); const bridgeProjection = bridgeCandidates( [...classified.dex, ...classified.jvm_classes], @@ -118,6 +122,11 @@ export const projectAndroidApplication = ( `Bridge candidate pairs exceeded the projection safety budget; ${bridgeProjection.coverage.omitted_candidates} hypotheses are omitted. Component arrays still include every component from the supplied inventory pages.`, ] : []), + ...(unavailable.length > 0 + ? [ + "Some inventoried occurrences have no authenticated content identity. Their paths and limitations are retained in unavailable_components; component absence is unknown.", + ] + : []), ]; const withoutId = { root_sha256: inventory.manifest.root_sha256, @@ -126,12 +135,14 @@ export const projectAndroidApplication = ( .map(({ evidence_id: id }) => id) .sort(compareProjectionStrings), components, + unavailable_components: unavailable, runtime_families: runtimeFamilies(all), bridge_candidates: bridgeProjection.candidates, bridge_candidate_coverage: bridgeProjection.coverage, coverage: projectionCoverage( inventory, - bridgeProjection.coverage.status === "complete", + bridgeProjection.coverage.status === "complete" && + unavailable.length === 0, ), limitations, }; diff --git a/src/domain/apktool/apktoolResourceAnalysis.ts b/src/domain/apktool/apktoolResourceAnalysis.ts index df217d040b..4cbca07459 100644 --- a/src/domain/apktool/apktoolResourceAnalysis.ts +++ b/src/domain/apktool/apktoolResourceAnalysis.ts @@ -3,6 +3,32 @@ import { localPathStringSchema } from "../localPath.js"; import { digestSchema } from "../digests.js"; const MAX_STRINGS = 2_000; +const ANDROID_NON_LOCALE_QUALIFIERS = new Set([ + "car", + "desk", + "television", + "appliance", + "watch", + "vrheadset", + "night", + "notnight", +]); + +const localeQualifierSchema = z + .string() + .min(2) + .max(16) + .regex( + /^(?:[a-z]{2,3}(?:-(?:r?[A-Z]{2}|r?[0-9]{3}))?|b\+[a-z]{2,3}(?:\+[A-Za-z0-9]{2,8})*)$/u, + "Locale must look like de, pt-BR, pt-rBR, or b+zh+Hans+CN", + ) + .refine( + (value) => !ANDROID_NON_LOCALE_QUALIFIERS.has(value.toLowerCase()), + "Android resource configuration qualifiers are not locales", + ); + +/** Locale language tags admitted by Android resource table selection. */ +export const androidLocaleQualifierSchema = localeQualifierSchema; /** * Caller intent for Apktool-backed resource decoding. Apktool is @@ -19,14 +45,7 @@ export const apktoolInputSchemas = { .describe( `Project res/values/strings.xml entries (bounded to ${String(MAX_STRINGS)} entries)`, ), - locale: z - .string() - .min(2) - .max(16) - .regex( - /^(?:[a-z]{2,3}(?:-(?:r?[A-Z]{2}|r?[0-9]{3}))?|b\+[a-z]{2,3}(?:\+[A-Za-z0-9]{2,8})*)$/u, - "Locale must look like de, pt-BR, pt-rBR, or b+zh+Hans+CN", - ) + locale: androidLocaleQualifierSchema .optional() .describe( "Project strings from res/values-/strings.xml instead of the default table", diff --git a/src/domain/apple/appleArchiveFormat.ts b/src/domain/apple/appleArchiveFormat.ts index e831266f1b..204cc19f7e 100644 --- a/src/domain/apple/appleArchiveFormat.ts +++ b/src/domain/apple/appleArchiveFormat.ts @@ -3,10 +3,214 @@ const NIB_ARCHIVE_MAGIC = [ 0x4e, 0x49, 0x42, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, ]; -/** Recognize the exact binary plist version supported by the decoder, without validating its body. */ +/** Shared format-probe bound; this does not bound the plist document itself. */ +export const MAX_XML_PLIST_PROBE_BYTES = 1024 * 1024; + +/** Recognize the binary plist version supported by the decoder, without validating its body. */ export const hasBinaryPlistSignature = (bytes: Uint8Array): boolean => BINARY_PLIST_MAGIC.every((byte, index) => bytes[index] === byte); +export type XmlPlistProbeState = "pending" | "matched" | "rejected"; + +/** Incremental, bounded XML signature scanner. Consumed prolog text is discarded. */ +export const createXmlPlistSignatureProbe = (): { + write(bytes: Uint8Array): XmlPlistProbeState; + readonly state: XmlPlistProbeState; +} => { + let state: XmlPlistProbeState = "pending"; + let preview: TextDecoder | undefined; + let validator: TextDecoder | undefined; + let utf16 = false; + let bomBytes = 0; + let fedBytes = 0; + let decodedBytes = 0; + let preamble: Uint8Array = new Uint8Array(); + const scanner = createXmlPlistPrologScanner(); + const encodedLength = (text: string): number => + utf16 ? text.length * 2 : new TextEncoder().encode(text).length; + return { + get state() { + return state; + }, + write(bytes) { + if (state !== "pending" || bytes.length === 0) return state; + let selected = bytes.subarray( + 0, + MAX_XML_PLIST_PROBE_BYTES - fedBytes - preamble.length, + ); + if (preview === undefined) { + if (preamble.length !== 0 || selected.length < 4) { + const joined = new Uint8Array(preamble.length + selected.length); + joined.set(preamble); + joined.set(selected, preamble.length); + selected = joined; + if (selected.length < 4) { + preamble = selected; + return state; + } + preamble = new Uint8Array(); + } + const little = selected[0] === 0xff && selected[1] === 0xfe; + const big = selected[0] === 0xfe && selected[1] === 0xff; + utf16 = little || big; + bomBytes = utf16 + ? 2 + : selected[0] === 0xef && selected[1] === 0xbb && selected[2] === 0xbf + ? 3 + : 0; + const encoding = little ? "utf-16le" : big ? "utf-16be" : "utf-8"; + preview = new TextDecoder(encoding); + validator = new TextDecoder(encoding, { fatal: true }); + } + const text = preview.decode(selected, { stream: true }); + const result = scanner(text); + // Validate only through the observed signature. Invalid document bytes + // after that signature cannot undo the format observation. + const through = + result.state === "matched" + ? bomBytes + + decodedBytes + + encodedLength(text.slice(0, result.consumed)) - + fedBytes + : selected.length; + try { + validator!.decode(selected.subarray(0, through), { stream: true }); + state = result.state; + } catch { + state = "rejected"; + } + decodedBytes += encodedLength(text); + fedBytes += selected.length; + if (fedBytes === MAX_XML_PLIST_PROBE_BYTES && state === "pending") + state = "rejected"; + return state; + }, + }; +}; + +/** Recognize a plist prolog without claiming that its XML document is valid. */ +export const hasXmlPlistSignature = (bytes: Uint8Array): boolean => + createXmlPlistSignatureProbe().write(bytes) === "matched"; + +const createXmlPlistPrologScanner = (): ((chunk: string) => { + state: XmlPlistProbeState; + consumed: number; +}) => { + let text = ""; + let totalCharacters = 0; + let pending: "?>" | "-->" | undefined; + let doctypeName = false; + return (chunk) => { + const before = totalCharacters; + totalCharacters += chunk.length; + text += chunk; + let cursor = 0; + const result = (state: XmlPlistProbeState, through = cursor) => ({ + state, + consumed: totalCharacters - text.length + through - before, + }); + while (true) { + if (pending !== undefined) { + const end = text.indexOf(pending, cursor); + if (end < 0) { + text = text.slice(Math.max(cursor, text.length - pending.length + 1)); + return result("pending"); + } + cursor = end + pending.length; + pending = undefined; + } + while (cursor < text.length && isXmlWhitespace(text[cursor]!)) cursor++; + if (cursor === text.length) { + text = ""; + return result("pending"); + } + if (doctypeName) { + const token = tokenAt(text, cursor, "plist"); + if ( + token === "partial" || + (token === "match" && cursor + 5 === text.length) + ) { + text = text.slice(cursor); + return result("pending"); + } + const delimiter = text[cursor + 5]; + return result( + token === "match" && + (delimiter === ">" || + delimiter === "[" || + (delimiter !== undefined && isXmlWhitespace(delimiter))) + ? "matched" + : "rejected", + cursor + 6, + ); + } + for (const [token, terminator] of [ + [""], + [""], + ] as const) { + const match = tokenAt(text, cursor, token); + if (match === "partial") { + text = text.slice(cursor); + return result("pending"); + } + if (match === "match") { + pending = terminator; + cursor += token.length; + break; + } + } + if (pending !== undefined) continue; + const doctype = tokenAt(text, cursor, "" || + (delimiter !== undefined && isXmlWhitespace(delimiter))) + ? "matched" + : "rejected", + cursor + 7, + ); + } + }; +}; + +type TokenMatch = "match" | "partial" | "none"; +const tokenAt = (text: string, offset: number, token: string): TokenMatch => { + const available = text.slice(offset, offset + token.length); + return available === token + ? "match" + : token.startsWith(available) + ? "partial" + : "none"; +}; +const isXmlWhitespace = (character: string): boolean => + character === " " || + character === "\t" || + character === "\r" || + character === "\n"; + /** Recognize the serialized signature without claiming a valid NIB object table. */ export const hasNibArchiveSignature = (bytes: Uint8Array): boolean => NIB_ARCHIVE_MAGIC.every((byte, index) => bytes[index] === byte); diff --git a/src/domain/apple/interfaceBuilderGraph.test.ts b/src/domain/apple/interfaceBuilderGraph.test.ts index 96c6b49157..3105e1c5cc 100644 --- a/src/domain/apple/interfaceBuilderGraph.test.ts +++ b/src/domain/apple/interfaceBuilderGraph.test.ts @@ -195,7 +195,7 @@ describe("Interface Builder parser bounds", () => { }); }); -describe("keyed archive decoding and graph limits", () => { +describe("keyed archive identity decoding", () => { it("preserves keyed archive UID identities, arrays, and control actions", () => { const parsed = parseInterfaceBuilderRecords({ $archiver: "NSKeyedArchiver", @@ -245,7 +245,9 @@ describe("keyed archive decoding and graph limits", () => { }), ); }); +}); +describe("Interface Builder graph limits", () => { it("reports records omitted by the parser's hard object ceiling", () => { const objects = Object.fromEntries( Array.from({ length: 20_001 }, (_, index) => [ @@ -316,6 +318,46 @@ describe("keyed archive decoding and graph limits", () => { }), ); }); + + it("keeps a later document complete when an earlier hierarchy hits the limit", () => { + const document = ( + relativePath: string, + objectId: string, + references: number, + ) => ({ + relativePath, + archiveSha256: hash, + documentKind: "nib" as const, + raw: { + "com.apple.ibtool.document.objects": { + [objectId]: { class: "UIView" }, + }, + "com.apple.ibtool.document.hierarchy": Array.from( + { length: references }, + () => ({ objectID: objectId }), + ), + }, + }); + const result = buildInterfaceBuilderAnalysis({ + targetSha256: hash, + toolVersion: "test", + documents: [ + document("First.nib", "first", 5), + document("Second.nib", "second", 1), + ], + limits: interfaceBuilderLimitsSchema.parse({ max_objects: 4 }), + }); + + expect( + result.documents.map(({ hierarchy_complete }) => hierarchy_complete), + ).toEqual([false, true]); + expect(result.graph.coverage).toContainEqual( + expect.objectContaining({ + facet: "hierarchy:Second.nib", + status: "complete", + }), + ); + }); }); describe("keyed archive object references", () => { diff --git a/src/domain/apple/interfaceBuilderGraph.ts b/src/domain/apple/interfaceBuilderGraph.ts index b3e5216dc0..75bc6b7358 100644 --- a/src/domain/apple/interfaceBuilderGraph.ts +++ b/src/domain/apple/interfaceBuilderGraph.ts @@ -432,7 +432,7 @@ export const buildInterfaceBuilderAnalysis = (input: { parsed.unresolvedConnections, hierarchy_complete: parsed.hierarchy.length > 0 && - !truncated && + !hierarchyTruncated && parserOmittedHierarchyReferences === 0, }); coverage.push( diff --git a/src/domain/artifactGraph.ts b/src/domain/artifactGraph.ts index 5a804c8d6e..0d12f3812f 100644 --- a/src/domain/artifactGraph.ts +++ b/src/domain/artifactGraph.ts @@ -14,7 +14,6 @@ const boundedRelativePathSchema = z .refine( (path) => !path.startsWith("/") && - !path.includes("\\") && path.split("/").every((part) => part !== "" && part !== ".."), "Expected a normalized relative POSIX path without traversal", ); diff --git a/src/domain/artifactIdentity.ts b/src/domain/artifactIdentity.ts index 3107d9af0e..8321129a98 100644 --- a/src/domain/artifactIdentity.ts +++ b/src/domain/artifactIdentity.ts @@ -19,9 +19,7 @@ export const logicalPathNfc = (value: string): string => value.normalize("NFC"); */ export const normalizeJoinedLogicalPath = (value: string): string => { if (value.includes("\0")) return value; - return logicalPathNfc( - posix.normalize(value.replaceAll("\\", "/")).replace(/^\.\//u, ""), - ); + return logicalPathNfc(posix.normalize(value).replace(/^\.\//u, "")); }; /** Whether a joined logical path leaves the archive root. */ diff --git a/src/domain/artifactInspection.ts b/src/domain/artifactInspection.ts index ae0e3aa6eb..cb9c5ba48d 100644 --- a/src/domain/artifactInspection.ts +++ b/src/domain/artifactInspection.ts @@ -299,7 +299,53 @@ const probesFor = ( inventory: ArtifactInventoryResult, ): ArtifactInspectionResult["next_probes"] => { const format = inventory.manifest.root_format; - if (["asar", "javascript-bundle", "directory"].includes(format)) + return format === "directory" + ? directoryProbes(inventory) + : probesForFormat(format); +}; + +// An application bundle directory is only a container, so its probes follow +// the formats it holds: a bundle of Mach-O executables is not a JavaScript tree. +// Formats are ordered so a repeated operation keeps its most general rationale. +const DIRECTORY_MEMBER_FORMATS: readonly ArtifactInventoryResult["manifest"]["root_format"][] = + [ + "asar", + "javascript-bundle", + "mach-o-universal", + "mach-o", + "elf", + "pe", + "plist", + ]; + +const directoryProbes = ( + inventory: ArtifactInventoryResult, +): ArtifactInspectionResult["next_probes"] => { + // JavaScript acquisition does not follow symlinks. Other probes retain + // their own applicability rules; plist inspection follows readable links. + const present = new Set( + inventory.occurrences + .filter( + ({ entry_kind, artifact_format }) => + entry_kind !== "symlink" || artifact_format !== "javascript-bundle", + ) + .map(({ artifact_format }) => artifact_format), + ); + const members = DIRECTORY_MEMBER_FORMATS.filter((memberFormat) => + present.has(memberFormat), + ); + const probes = members.flatMap(probesForFormat); + return probes.filter( + (probe, index) => + probes.findIndex(({ operation }) => operation === probe.operation) === + index, + ); +}; + +const probesForFormat = ( + format: ArtifactInventoryResult["manifest"]["root_format"], +): ArtifactInspectionResult["next_probes"] => { + if (["asar", "javascript-bundle"].includes(format)) return [ { operation: "analyze_javascript_application", diff --git a/src/domain/referenceSourceClassification.ts b/src/domain/referenceSourceClassification.ts index ea53a18a4e..0c82cf6d69 100644 --- a/src/domain/referenceSourceClassification.ts +++ b/src/domain/referenceSourceClassification.ts @@ -213,7 +213,6 @@ const GENERATED_DIRECTORY_NAMES = new Set([ ".svelte-kit", ".astro", "storybook-static", - "docs/api", ".codex", ".devin", ".factory", @@ -225,6 +224,8 @@ const GENERATED_DIRECTORY_NAMES = new Set([ ".nyc_output", ]); +const GENERATED_DIRECTORY_PATHS = [["docs", "api"]] as const; + const VENDOR_DIRECTORY_NAMES = new Set([ "node_modules", "vendor", @@ -315,13 +316,18 @@ const basenameMatch = (filename: string, set: ReadonlySet): boolean => { return set.has(lower); }; -const isPathUnderAny = (path: string, names: ReadonlySet): boolean => { - for (const segment of path.split("/")) { - if (segment === "") continue; - if (names.has(segment.toLowerCase())) return true; - } - return false; -}; +const isPathUnderAny = ( + parts: readonly string[], + names: ReadonlySet, +): boolean => parts.some((segment) => names.has(segment)); + +const containsDirectoryPath = ( + parts: readonly string[], + directory: readonly string[], +): boolean => + parts.some((_, start) => + directory.every((segment, offset) => parts[start + offset] === segment), + ); /** Detect a source language from a POSIX path. */ export const detectReferenceSourceLanguage = (path: string): string | null => { @@ -344,33 +350,34 @@ export const classifyReferenceSourcePath = ( path: string, ): ReferenceSourceClassification[] => { const classifications: Set = new Set(); - const parts = path.split("/").filter((segment) => segment !== ""); + const parts = path + .toLowerCase() + .split("/") + .filter((segment) => segment !== ""); const filename = parts[parts.length - 1] ?? ""; - const lower = filename.toLowerCase(); - if (parts.length === 0) { classifications.add("source"); return [...classifications].sort(); } - addDirectoryClassifications(path, classifications); + addDirectoryClassifications(parts, classifications); addFilenameClassifications(filename, classifications); if ( - lower.endsWith(".json") || - lower.endsWith(".yaml") || - lower.endsWith(".yml") + filename.endsWith(".json") || + filename.endsWith(".yaml") || + filename.endsWith(".yml") ) { if (!classifications.has("manifest") && !classifications.has("config")) classifications.add("config"); } - if (isPathUnderAny(path, SOURCE_DIRECTORY_NAMES)) { + if (isPathUnderAny(parts, SOURCE_DIRECTORY_NAMES)) { if (!classifications.has("vendor") && !classifications.has("generated")) classifications.add("source"); } if (classifications.size === 0) { - if (lower === "license" || lower === "copying") + if (filename === "license" || filename === "copying") classifications.add("documentation"); else classifications.add("unknown"); } @@ -379,15 +386,20 @@ export const classifyReferenceSourcePath = ( }; const addDirectoryClassifications = ( - path: string, + parts: readonly string[], classifications: Set, ): void => { - if (isPathUnderAny(path, TEST_DIRECTORY_NAMES)) classifications.add("test"); - if (isPathUnderAny(path, DOCUMENTATION_DIRECTORY_NAMES)) + if (isPathUnderAny(parts, TEST_DIRECTORY_NAMES)) classifications.add("test"); + if (isPathUnderAny(parts, DOCUMENTATION_DIRECTORY_NAMES)) classifications.add("documentation"); - if (isPathUnderAny(path, VENDOR_DIRECTORY_NAMES)) + if (isPathUnderAny(parts, VENDOR_DIRECTORY_NAMES)) classifications.add("vendor"); - if (isPathUnderAny(path, GENERATED_DIRECTORY_NAMES)) + if ( + isPathUnderAny(parts, GENERATED_DIRECTORY_NAMES) || + GENERATED_DIRECTORY_PATHS.some((directory) => + containsDirectoryPath(parts, directory), + ) + ) classifications.add("generated"); }; diff --git a/src/domain/referenceSourceGraph.test.ts b/src/domain/referenceSourceGraph.test.ts index 1286cdde6b..272c2f62c7 100644 --- a/src/domain/referenceSourceGraph.test.ts +++ b/src/domain/referenceSourceGraph.test.ts @@ -434,6 +434,56 @@ describe("historical source graph validation", () => { }); }); +describe("historical source graph excluded symlink targets", () => { + it("retains internal symlinks into excluded trees without accepting sibling prefixes", () => { + const input = { ...graphInput(), inventory_state: "partial" as const }; + input.exclusions = [ + { + path: "dist", + entry_kind: "directory", + reason: "caller-excluded", + pattern: "dist/", + }, + ]; + input.entries[2] = symlinkEntry({ target: "../dist/generated.js" }); + const graph = createHistoricalSourceGraph(input); + expect(graph.entries[2]).toMatchObject({ + target: "../dist/generated.js", + target_state: "internal", + }); + + input.entries[2] = symlinkEntry({ target: "../dist-old/generated.js" }); + expect(() => createHistoricalSourceGraph(input)).toThrow( + /recorded exclusion/u, + ); + }); + + it("does not treat an excluded file or unknown-kind ancestor as a directory", () => { + const input = { ...graphInput(), inventory_state: "partial" as const }; + input.exclusions = [ + { + path: "secret.txt", + entry_kind: "file", + reason: "configured-secret", + pattern: "secret.txt", + }, + ]; + input.entries[2] = symlinkEntry({ target: "../secret.txt/child" }); + expect(() => createHistoricalSourceGraph(input)).toThrow( + /recorded exclusion/u, + ); + + input.exclusions[0] = { + path: "secret.txt", + reason: "configured-secret", + pattern: "secret.txt", + }; + expect(() => createHistoricalSourceGraph(input)).toThrow( + /recorded exclusion/u, + ); + }); +}); + describe("historical source graph symlink path identity", () => { it("joins internal symlink targets with the inventory NFC normalizer", () => { // "café" spelled NFC, as the inventory produces it, and the same @@ -486,3 +536,54 @@ describe("historical source graph symlink path identity", () => { ); }); }); + +it.each([ + "\0", + "/absolute.js", + "../outside.js", + "a/../b.js", + "a//b.js", + "./a.js", + "a/", +])("rejects unsafe historical logical entry spelling %j", (path) => { + const input = graphInput(); + input.entries = [fileEntry({ path })]; + input.relationships = []; + expect(() => createHistoricalSourceGraph(input)).toThrow(); +}); + +it("rejects NUL in observed symlink targets before graph identity", () => { + const input = graphInput(); + input.entries[2] = symlinkEntry({ target: "literal\0name.js" }); + expect(() => createHistoricalSourceGraph(input)).toThrow( + /normalized POSIX symlink target/u, + ); +}); + +it.each(["C:\\outside\\file.js", "\\\\server\\share.js"])( + "preserves unowned recorded Windows absolute target policy %j", + (target) => { + const external = graphInput(); + external.inventory_state = "partial"; + external.entries[2] = symlinkEntry({ target, target_state: "external" }); + expect(createHistoricalSourceGraph(external).entries).toContainEqual( + expect.objectContaining({ target, target_state: "external" }), + ); + const internal = graphInput(); + internal.entries[2] = symlinkEntry({ target, target_state: "internal" }); + expect(() => createHistoricalSourceGraph(internal)).toThrow( + /target path does not match its target state/u, + ); + }, +); + +it("refuses an absolute POSIX internal target even when lexical joining would name an inventoried file", () => { + const input = graphInput(); + input.entries[2] = symlinkEntry({ + target: "/main.ts", + target_state: "internal", + }); + expect(() => createHistoricalSourceGraph(input)).toThrow( + /target path does not match its target state/u, + ); +}); diff --git a/src/domain/referenceSourceGraph.ts b/src/domain/referenceSourceGraph.ts index 197e473bfc..0c1c7caafc 100644 --- a/src/domain/referenceSourceGraph.ts +++ b/src/domain/referenceSourceGraph.ts @@ -18,7 +18,7 @@ const relativePathSchema = z .refine( (path) => !path.startsWith("/") && - !path.includes("\\") && + !path.includes("\0") && path .split("/") .every((part) => part !== "" && part !== "." && part !== ".."), @@ -85,10 +85,9 @@ const symlinkTargetSchema = z .min(1) .refine( (target) => - isPortableAbsoluteReferenceTarget(target) || - (!target.startsWith("/") && - !target.includes("\\") && - posix.normalize(target) === target), + !target.includes("\0") && + (isPortableAbsoluteReferenceTarget(target) || + (!target.startsWith("/") && posix.normalize(target) === target)), "Expected a normalized POSIX symlink target", ); @@ -130,6 +129,9 @@ const parseFailureSchema = z.strictObject({ const exclusionSchema = z.discriminatedUnion("reason", [ z.strictObject({ path: relativePathSchema, + entry_kind: z + .enum(["file", "directory", "symlink", "other"]) + .exactOptional(), reason: z.enum([ "configured-secret", "project-ignored", @@ -140,6 +142,9 @@ const exclusionSchema = z.discriminatedUnion("reason", [ }), z.strictObject({ path: relativePathSchema, + entry_kind: z + .enum(["file", "directory", "symlink", "other"]) + .exactOptional(), reason: z.enum(["symlink-escape", "unreadable"]), }), ]); @@ -320,10 +325,50 @@ const checkSymlinks = ( context: z.RefinementCtx, ): void => { const lookupPath = createReferenceSourcePathLookup(paths); + const excludedPaths = createReferenceSourcePathLookup( + graph.exclusions.map(({ path }) => path), + ); + const excludedKinds = new Map< + string, + Set<"file" | "directory" | "symlink" | "other" | undefined> + >(); + for (const { path, entry_kind } of graph.exclusions) { + const kinds = excludedKinds.get(path) ?? new Set(); + kinds.add(entry_kind); + excludedKinds.set(path, kinds); + } for (const [index, entry] of graph.entries.entries()) { if (entry.kind !== "symlink") continue; if (entry.target === null) continue; - const absolute = isPortableAbsoluteReferenceTarget(entry.target); + const resolved = posix.normalize( + posix.join(posix.dirname(entry.path), entry.target), + ); + const escapesRoot = resolved.startsWith("../") || resolved === ".."; + // The root is implicit; every other internal target needs an exact or + // uniquely Unicode-equivalent recorded inventory identity. + const inventoried = + resolved === "." || lookupPath.resolve(resolved) !== undefined; + const exactExcluded = excludedPaths.resolve(resolved); + let excluded = exactExcluded !== undefined; + for ( + let candidate = posix.dirname(resolved); + candidate !== "." && candidate !== "/"; + candidate = posix.dirname(candidate) + ) { + const excludedAncestor = excludedPaths.resolve(candidate); + if (excludedAncestor === undefined) continue; + const kinds = excludedKinds.get(excludedAncestor); + if (kinds?.size !== 1 || !kinds.has("directory")) { + excluded = false; + break; + } + excluded = true; + } + const ownedRelativeTarget = + !entry.target.startsWith("/") && !escapesRoot && inventoried; + const absolute = + isPortableAbsoluteReferenceTarget(entry.target) && + !(entry.target_state === "internal" && ownedRelativeTarget); if ( (entry.target_state === "external" && !absolute) || (entry.target_state === "internal" && absolute) @@ -334,18 +379,11 @@ const checkSymlinks = ( path: ["entries", index, "target"], }); if (entry.target_state !== "internal") continue; - const resolved = posix.normalize( - posix.join(posix.dirname(entry.path), entry.target), - ); - if ( - resolved.startsWith("../") || - resolved === ".." || - // The inventory root is implicit rather than an entry with path ".". - (resolved !== "." && lookupPath.resolve(resolved) === undefined) - ) + if (escapesRoot || (!inventoried && !excluded)) context.addIssue({ code: "custom", - message: "Internal symlink target must resolve to an inventoried entry", + message: + "Internal symlink target must resolve to an inventoried entry or a recorded exclusion", path: ["entries", index, "target"], }); } diff --git a/src/domain/referenceSourcePathIdentity.ts b/src/domain/referenceSourcePathIdentity.ts index cde64c71a3..4ac6963cf8 100644 --- a/src/domain/referenceSourcePathIdentity.ts +++ b/src/domain/referenceSourcePathIdentity.ts @@ -1,6 +1,10 @@ +import { posix } from "node:path"; + export interface ReferenceSourcePathLookup { readonly resolve: (path: string) => string | undefined; readonly resolveCandidates: (paths: readonly string[]) => string | undefined; + /** Resolve a path to the closest recorded parent identity. */ + readonly resolveAncestor: (path: string) => string | undefined; } /** @@ -18,11 +22,24 @@ export const createReferenceSourcePathLookup = ( } const equivalent = (path: string): string | undefined => canonical.get(path.normalize("NFC")) ?? undefined; + const resolve = (path: string): string | undefined => + exact.has(path) ? path : equivalent(path); return { - resolve: (path) => (exact.has(path) ? path : equivalent(path)), + resolve, resolveCandidates: (candidates) => candidates.find((path) => exact.has(path)) ?? candidates.map(equivalent).find((path) => path !== undefined), + resolveAncestor: (path) => { + for ( + let candidate = path; + candidate !== "." && candidate !== "/"; + candidate = posix.dirname(candidate) + ) { + const found = resolve(candidate); + if (found !== undefined) return found; + } + return undefined; + }, }; }; diff --git a/src/javascript/recovery/RecoveryFiles.ts b/src/javascript/recovery/RecoveryFiles.ts index c64f011b48..cd1dcb6669 100644 --- a/src/javascript/recovery/RecoveryFiles.ts +++ b/src/javascript/recovery/RecoveryFiles.ts @@ -167,6 +167,11 @@ export const snapshotRecoveryInputs = async ( ? "bundle.js" : relative(commonDirectory, resolve(path)).split(sep).join("/"); try { + // Wakaru's multi-input public paths interpret backslashes as separators. + if (relativePath.includes("\\")) + throw new TypeError( + "Wakaru cannot preserve literal backslashes in input paths", + ); if (normalizeArtifactPath(relativePath) !== relativePath) throw new TypeError( "Input names must already use NFC Unicode spelling", diff --git a/src/reference/ReferenceSourceReaderErrors.ts b/src/reference/ReferenceSourceReaderErrors.ts index 6ddd02a3f2..4444b43ba9 100644 --- a/src/reference/ReferenceSourceReaderErrors.ts +++ b/src/reference/ReferenceSourceReaderErrors.ts @@ -29,6 +29,9 @@ const FILESYSTEM_ERROR_CODES = new Set([ "EISDIR", "ELOOP", "EMFILE", + // A traversed child can exceed the platform's absolute path limit even + // though the selected root and every component are valid. + "ENAMETOOLONG", "ENFILE", "ENOENT", "ENOTDIR", diff --git a/tests/boundary/cli/jsonInputSize.test.ts b/tests/boundary/cli/jsonInputSize.test.ts index 0a4aad8eed..33996b42f2 100644 --- a/tests/boundary/cli/jsonInputSize.test.ts +++ b/tests/boundary/cli/jsonInputSize.test.ts @@ -8,14 +8,14 @@ import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { cliTest } from "../../support/cli/cliFixture.js"; cliTest( - "rejects malformed oversized JSON through both CLI and Evidence readers", + "rejects malformed oversized input through CLI and Evidence readers", async ({ cli, processes }) => { const root = await createTestTempDirectory("rea-json-string-limit-"); const input = join(root, "oversized.json"); const file = await open(input, "wx"); try { - // Sparse zero bytes are valid UTF-8 but invalid JSON. Both readers - // stream the input and reject the syntax before assembling a value. + // Sparse zero bytes are valid UTF-8. Both streaming readers reject + // their invalid JSON syntax without assembling a whole-document string. await file.truncate(constants.MAX_STRING_LENGTH + 1); } finally { await file.close(); @@ -57,7 +57,7 @@ cliTest( cliTest( "traces valid Evidence beyond the whole-document string limit", - async ({ cli }) => { + async ({ cli, processes }) => { const root = await createTestTempDirectory("rea-json-streamed-evidence-"); const source = join(root, "application"); await mkdir(source); @@ -175,7 +175,7 @@ cliTest( cliTest( "returns a typed string assembly constraint under a bounded heap", - async ({ cli }) => { + async ({ cli, processes }) => { const root = await createTestTempDirectory("rea-json-string-headroom-"); const input = join(root, "long-string.json"); const file = await open(input, "wx"); @@ -207,5 +207,21 @@ cliTest( action: expect.stringContaining("re-analyze a smaller selection"), }, }); + const evidence = await processes.run(process.execPath, [ + "--max-old-space-size=768", + "--input-type=module", + "-e", + `import { readJsonFile } from "./dist/application/JsonFiles.js"; + import { projectAnalysisError } from "./dist/domain/analysisErrorProjection.js"; + const result = await readJsonFile(process.argv[1]); + console.log(JSON.stringify(result.ok ? result : projectAnalysisError(result.error)));`, + input, + ]); + expect(JSON.parse(evidence.stdout)).toMatchObject({ + code: "resource_constraint", + details: { operation: "read_evidence_file", resource: "memory" }, + }); + expect(evidence.stderr).toBe(""); + expect(evidence.exitCode).toBe(0); }, ); diff --git a/tests/boundary/cli/referenceSourceExclusions.test.ts b/tests/boundary/cli/referenceSourceExclusions.test.ts index cbc0632856..87b8541166 100644 --- a/tests/boundary/cli/referenceSourceExclusions.test.ts +++ b/tests/boundary/cli/referenceSourceExclusions.test.ts @@ -1,4 +1,10 @@ -import { mkdir, writeFile } from "node:fs/promises"; +import { + mkdir, + readFile, + readlink, + symlink, + writeFile, +} from "node:fs/promises"; import { join } from "node:path"; import { describe, expect } from "vitest"; @@ -10,6 +16,56 @@ import { cliTest } from "../../support/cli/cliFixture.js"; describe.skipIf(process.platform === "win32")( "compiled reference-source exclusion provenance", () => { + cliTest( + "imports source beside a link into a default-excluded directory", + async ({ cli }) => { + const root = await createTestTempDirectory( + "rea-reference-link-exclusion-", + ); + await mkdir(join(root, "dist")); + const source = "export const value = 1;\n"; + await writeFile(join(root, "source.ts"), source); + await symlink("dist", join(root, "alias")); + const result = await cli.run({ + arguments: ["import-reference-source", root, "--json"], + cwd: root, + environment: { + HOME: root, + USERPROFILE: root, + XDG_CONFIG_HOME: root, + XDG_CACHE_HOME: root, + }, + }); + expect(result.exitCode).toBe(0); + const graph = historicalSourceGraphSchema.parse(result.json); + expect(graph.inventory_state).toBe("partial"); + expect(graph.exclusions).toContainEqual({ + path: "dist", + entry_kind: "directory", + reason: "default-ignored", + pattern: "dist/", + }); + expect(graph.entries).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + path: "alias", + kind: "symlink", + target: "dist", + target_state: "internal", + }), + expect.objectContaining({ + path: "source.ts", + kind: "file", + content_state: "hashed", + }), + ]), + ); + expect(graph.entries.some(({ path }) => path === "dist")).toBe(false); + expect(await readFile(join(root, "source.ts"), "utf8")).toBe(source); + expect(await readlink(join(root, "alias"))).toBe("dist"); + }, + ); + cliTest( "reports project, default, and explicit secret exclusions", async ({ cli }) => { @@ -42,16 +98,19 @@ describe.skipIf(process.platform === "win32")( expect.arrayContaining([ { path: "project.tmp", + entry_kind: "file", reason: "project-ignored", pattern: "project.tmp", }, { path: "logs/application.log", + entry_kind: "file", reason: "default-ignored", pattern: "*.log", }, { path: "private.txt", + entry_kind: "file", reason: "configured-secret", pattern: "private.txt", }, diff --git a/tests/boundary/cli/referenceSourceUtf8.test.ts b/tests/boundary/cli/referenceSourceUtf8.test.ts index 06093f427e..0a1243a913 100644 --- a/tests/boundary/cli/referenceSourceUtf8.test.ts +++ b/tests/boundary/cli/referenceSourceUtf8.test.ts @@ -1,5 +1,11 @@ import { createHash } from "node:crypto"; -import { writeFile } from "node:fs/promises"; +import { + mkdir, + readFile, + readlink, + symlink, + writeFile, +} from "node:fs/promises"; import { join } from "node:path"; import { expect } from "vitest"; @@ -39,7 +45,9 @@ cliTest( arguments: ["import-reference-source", root, "--json"], environment: { HOME: root, XDG_CONFIG_HOME: root, XDG_CACHE_HOME: root }, }); - expect(result.exitCode).toBe(0); + expect(result.exitCode, result.stderr + JSON.stringify(result.json)).toBe( + 0, + ); const graph = historicalSourceGraphSchema.parse(result.json); expect(graph.parse_failures).toEqual([ { @@ -75,3 +83,139 @@ cliTest( ); }, ); + +cliTest.skipIf(process.platform === "win32")( + "preserves historical literal-backslash entries and their exact owning import base", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-reference-literal-paths-"); + const config = await createTestTempDirectory( + "rea-reference-literal-config-", + ); + await mkdir(join(root, "literal")); + await mkdir(join(root, "literal\\dir")); + const sources = new Map([ + ["literal\\name.js", "export const value = 'literal';\n"], + ["literal/name.js", "export const value = 'slash';\n"], + ["literal\\dir/main.js", 'import "./dep.js";\n'], + ["literal\\dir/dep.js", "export const value = 'owned';\n"], + ]); + for (const [path, bytes] of sources) + await writeFile(join(root, path), bytes); + const result = await cli.run({ + arguments: ["import-reference-source", root, "--json"], + environment: { + HOME: config, + XDG_CONFIG_HOME: config, + XDG_CACHE_HOME: config, + }, + }); + expect(result.exitCode, result.stderr + JSON.stringify(result.json)).toBe( + 0, + ); + const graph = historicalSourceGraphSchema.parse(result.json); + for (const [path, bytes] of sources) + expect(graph.entries).toContainEqual( + expect.objectContaining({ + path, + sha256: createHash("sha256").update(bytes).digest("hex"), + content_state: "hashed", + }), + ); + for (const [path, bytes] of sources) + expect(await readFile(join(root, path), "utf8")).toBe(bytes); + expect(graph.relationships).toContainEqual({ + from_path: "literal\\dir/main.js", + to: "literal\\dir/dep.js", + kind: "imports", + resolution: "internal", + parse_state: "parsed", + }); + }, +); + +cliTest.skipIf(process.platform === "win32")( + "preserves an observed historical symlink target with a literal backslash", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-reference-literal-link-"); + const config = await createTestTempDirectory( + "rea-reference-literal-link-config-", + ); + await writeFile( + join(root, "literal\\name.js"), + "export const value = 42;\n", + ); + await symlink("literal\\name.js", join(root, "link.js")); + const result = await cli.run({ + arguments: ["import-reference-source", root, "--json"], + environment: { + HOME: config, + XDG_CONFIG_HOME: config, + XDG_CACHE_HOME: config, + }, + }); + expect(result.exitCode, result.stderr + JSON.stringify(result.json)).toBe( + 0, + ); + const graph = historicalSourceGraphSchema.parse(result.json); + expect(await readlink(join(root, "link.js"))).toBe("literal\\name.js"); + expect(await readFile(join(root, "literal\\name.js"), "utf8")).toBe( + "export const value = 42;\n", + ); + expect(graph.entries).toContainEqual( + expect.objectContaining({ + kind: "symlink", + path: "link.js", + target: "literal\\name.js", + target_state: "internal", + }), + ); + }, +); + +cliTest.skipIf(process.platform === "win32")( + "imports an inventory-owned POSIX UNC-looking symlink target", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-reference-unc-literal-"); + const config = await createTestTempDirectory("rea-reference-unc-config-"); + const literal = "\\\\server\\share.js"; + const literalBytes = "export const value = 'literal';\n"; + const slashBytes = "export const value = 'slash';\n"; + await mkdir(join(root, "server")); + await writeFile(join(root, literal), literalBytes); + await writeFile(join(root, "server", "share.js"), slashBytes); + await symlink(literal, join(root, "link.js")); + const result = await cli.run({ + arguments: ["import-reference-source", root, "--json"], + environment: { + HOME: config, + XDG_CONFIG_HOME: config, + XDG_CACHE_HOME: config, + }, + }); + expect(result.exitCode, result.stderr + JSON.stringify(result.json)).toBe( + 0, + ); + const graph = historicalSourceGraphSchema.parse(result.json); + expect(graph.entries).toContainEqual( + expect.objectContaining({ + kind: "symlink", + path: "link.js", + target: literal, + target_state: "internal", + }), + ); + for (const [path, bytes] of [ + [literal, literalBytes], + ["server/share.js", slashBytes], + ]) { + expect(graph.entries).toContainEqual( + expect.objectContaining({ + path, + sha256: createHash("sha256").update(bytes!).digest("hex"), + }), + ); + expect(await readFile(join(root, path!), "utf8")).toBe(bytes); + } + expect(await readlink(join(root, "link.js"))).toBe(literal); + }, +); diff --git a/tests/boundary/filesystem/appleXmlDeclarations.test.ts b/tests/boundary/filesystem/appleXmlDeclarations.test.ts index e066655eba..56763d8b00 100644 --- a/tests/boundary/filesystem/appleXmlDeclarations.test.ts +++ b/tests/boundary/filesystem/appleXmlDeclarations.test.ts @@ -1,5 +1,7 @@ import { writeFile } from "node:fs/promises"; +import { execFile } from "node:child_process"; import { join } from "node:path"; +import { promisify } from "node:util"; import { build } from "plist"; import { expect, it } from "vitest"; @@ -14,6 +16,32 @@ const archive = build({ $top: { root: { CF$UID: 1 } }, }); +it("preflights repeated unterminated quoted declarations without suffix rescans", async () => { + const preflightUrl = new URL( + "../../../dist/artifacts/apple/InterfaceBuilderDecodeBudget.js", + import.meta.url, + ).href; + const { stdout } = await promisify(execFile)( + process.execPath, + [ + "--input-type=module", + "-e", + ` + import { preflightPropertyListDecode } from ${JSON.stringify(preflightUrl)}; + const xml = 'x" '.repeat(70000); + const observed = preflightPropertyListDecode(Buffer.from(xml), 16 * 1024 * 1024); + process.stdout.write(JSON.stringify({ observed, expected: xml.length * 4 })); + `, + ], + { timeout: 5000 }, + ); + const result = JSON.parse(stdout) as { + observed: { estimatedBytes: number }; + expected: number; + }; + expect(result.observed.estimatedBytes).toBe(result.expected); +}, 10000); + const fixture = async (declaration: string) => { const bundlePath = await createTestTempDirectory("rea-xml-declaration-"); await writeFile( diff --git a/tests/boundary/filesystem/artifactLiteralBackslash.test.ts b/tests/boundary/filesystem/artifactLiteralBackslash.test.ts new file mode 100644 index 0000000000..02105ad604 --- /dev/null +++ b/tests/boundary/filesystem/artifactLiteralBackslash.test.ts @@ -0,0 +1,222 @@ +import { createHash } from "node:crypto"; +import { access, mkdir, readFile, readdir, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { Readable } from "node:stream"; + +import { createPackage } from "@electron/asar"; +import { expect, it } from "vitest"; + +import { extractArtifact } from "../../../src/artifacts/extraction/ArtifactExtraction.js"; +import { SafeOutputTree } from "../../../src/artifacts/SafeOutputTree.js"; +import { + inventoryArtifact, + withArtifactResourceScope, +} from "../../fixtures/artifactInventory.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { analyzeJavaScriptApplication } from "../../support/javascriptApplicationScope.js"; +import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; + +const NAMES = [ + "literal\\name.txt", + "\\leading.txt", + "\\\\server\\share.txt", + "ordinary.txt", + "literal/name.txt", + "nested/literal\\name.txt", +]; +const CONTENT = Buffer.from("exact member bytes\n"); +const DIGEST = createHash("sha256").update(CONTENT).digest("hex"); + +it.skipIf(process.platform === "win32").each(["directory", "asar"] as const)( + "inventories and extracts literal POSIX backslashes from %s without changing names", + async (format) => { + const root = await createTestTempDirectory("rea-literal-backslash-"); + const source = join(root, "source"); + await mkdir(source); + for (const name of NAMES) { + await mkdir(dirname(join(source, name)), { recursive: true }); + await writeFile(join(source, name), CONTENT); + } + const input = format === "directory" ? source : join(root, "input.asar"); + if (format === "asar") await createPackage(source, input); + + const inventory = await inventoryArtifact(input); + for (const name of NAMES) { + const occurrence = inventory.occurrences.find( + ({ logical_path }) => logical_path === name, + ); + expect(occurrence).toMatchObject({ + entry_kind: "file", + hash_status: "verified", + }); + expect( + inventory.nodes.find( + ({ artifact_id }) => artifact_id === occurrence?.artifact_id, + ), + ).toMatchObject({ sha256: DIGEST, size: CONTENT.length }); + } + + const output = join(root, "output"); + const extracted = await withArtifactResourceScope((resourceScope) => + extractArtifact({ + inputPath: input, + // The owner checks lstat first and selects the directory reader; + // BinaryTarget formats have no separate directory tag. + inputFormat: "asar", + outputRoot: output, + environment: {}, + integrityPolicy: "fail", + resourceScope, + }), + ); + expect( + extracted.artifacts.map(({ relative_path }) => relative_path).sort(), + ).toEqual([...NAMES].sort()); + expect((await readdir(output)).sort()).toEqual( + [...new Set(NAMES.map((name) => name.split("/")[0]))].sort(), + ); + for (const name of NAMES) + expect(await readFile(join(output, name))).toEqual(CONTENT); + }, +); + +it.each(NAMES.filter((name) => name.includes("\\")))( + "rejects Windows separator interpretation before materializing %j", + async (name) => { + const root = await createTestTempDirectory("rea-backslash-output-"); + const output = join(root, "output"); + const tree = await SafeOutputTree.create(output, "win32"); + const stream = Readable.from(CONTENT); + try { + await expect( + tree.write(name, stream, { sha256: DIGEST, bytes: CONTENT.length }), + ).rejects.toMatchObject({ + reason: "path", + message: expect.stringContaining( + "Windows destination cannot materialize artifact path", + ), + }); + expect(stream.destroyed).toBe(true); + expect(await readdir(output)).toEqual([]); + } finally { + expect(await tree.rollback()).toMatchObject({ status: "complete" }); + } + await expect(access(output)).rejects.toMatchObject({ code: "ENOENT" }); + }, +); + +it.each([ + "../outside", + "safe/../outside", + "/outside", + "//server/share", + "C:\\outside", + "C:/outside", + "safe/\0bad", +])("keeps output containment rejection for %j", async (name) => { + const root = await createTestTempDirectory("rea-backslash-containment-"); + const output = join(root, "output"); + const tree = await SafeOutputTree.create(output); + try { + await expect( + tree.write(name, Readable.from(CONTENT), { + sha256: DIGEST, + bytes: CONTENT.length, + }), + ).rejects.toMatchObject({ reason: "path" }); + expect(await readdir(output)).toEqual([]); + } finally { + expect(await tree.rollback()).toMatchObject({ status: "complete" }); + } +}); + +it.skipIf(process.platform === "win32").each(["directory", "asar"] as const)( + "retains literal POSIX JavaScript names in application and semantic graphs from %s", + async (format) => { + const root = await createTestTempDirectory("rea-backslash-js-graph-"); + const source = join(root, "source"); + await mkdir(source); + const names = [ + "literal\\name.js", + "\\leading.js", + "\\\\server\\share.js", + "nested/literal\\name.js", + ]; + for (const name of names) { + await mkdir(dirname(join(source, name)), { recursive: true }); + await writeFile(join(source, name), "export const marker = 42;\n"); + } + const input = format === "directory" ? source : join(root, "input.asar"); + if (format === "asar") await createPackage(source, input); + const result = await analyzeJavaScriptApplication({ + input_path: input, + format, + }); + if (!result.ok) throw result.error; + const parsed = javascriptApplicationAnalysisResultSchema.parse( + result.value.normalized_result, + ); + const observed = parsed.graph.nodes.flatMap(({ observations }) => + observations.map(({ properties }) => properties.path), + ); + expect(observed).toEqual(expect.arrayContaining(names)); + const semantic = parsed.semantic_graph?.nodes.map( + ({ identity }) => identity.module_path, + ); + expect(semantic).toEqual(expect.arrayContaining(names)); + }, +); + +it.skipIf(process.platform === "win32").each(["directory", "asar"] as const)( + "resolves ordinary relative imports inside a literal POSIX directory from %s", + async (format) => { + const root = await createTestTempDirectory("rea-backslash-import-base-"); + const source = join(root, "source"); + const literal = "literal\\dir"; + await mkdir(join(source, literal), { recursive: true }); + await mkdir(join(source, "literal", "dir"), { recursive: true }); + const body = "export const marker = 'literal';\n"; + await writeFile( + join(source, literal, "a.js"), + "import { marker } from './b.js'; export { marker };\n", + ); + await writeFile(join(source, literal, "b.js"), body); + await writeFile( + join(source, "literal", "dir", "b.js"), + "export const marker = 'decoy';\n", + ); + const input = format === "directory" ? source : join(root, "input.asar"); + if (format === "asar") await createPackage(source, input); + const result = await analyzeJavaScriptApplication({ + input_path: input, + format, + }); + if (!result.ok) throw result.error; + const parsed = javascriptApplicationAnalysisResultSchema.parse( + result.value.normalized_result, + ); + const imports = parsed.graph.edges.filter( + ({ properties }) => properties.specifier === "./b.js", + ); + expect(imports.length).toBeGreaterThan(0); + expect(imports.map(({ properties }) => properties.resolved_path)).toEqual( + imports.map(() => `${literal}/b.js`), + ); + expect(parsed.graph.nodes).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + identity: { + strategy: "content-digest", + stability: "global-exact", + sha256: createHash("sha256").update(body).digest("hex"), + }, + observations: expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ path: `${literal}/b.js` }), + }), + ]), + }), + ]), + ); + }, +); diff --git a/tests/boundary/filesystem/asarHeaderAdmission.test.ts b/tests/boundary/filesystem/asarHeaderAdmission.test.ts new file mode 100644 index 0000000000..e16e2ea2de --- /dev/null +++ b/tests/boundary/filesystem/asarHeaderAdmission.test.ts @@ -0,0 +1,115 @@ +import { mkdir, open, readFile, truncate, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { createPackage } from "@electron/asar"; +import { expect, it, vi } from "vitest"; + +import { + admitAsarHeader, + MAX_ASAR_HEADER_BYTES, +} from "../../../src/artifacts/AsarHeader.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const createFixture = async () => { + const root = await createTestTempDirectory("rea-asar-header-admission-"); + const source = join(root, "source"); + const archive = join(root, "fixture.asar"); + await mkdir(source); + await writeFile(join(source, "main.js"), "export const answer = 42;\n"); + await createPackage(source, archive); + return { archive, original: await readFile(archive) }; +}; + +const admit = async (path: string) => { + const handle = await open(path, "r"); + const read = vi.spyOn(handle, "read"); + try { + return await admitAsarHeader(handle); + } finally { + // Admission must inspect only the fixed prefix, even for forged extents. + const calls = [...read.mock.calls]; + read.mockRestore(); + await handle.close(); + expect(handle.fd).toBe(-1); + expect(calls).toEqual([[expect.any(Buffer), 0, 16, 0]]); + } +}; + +it("admits a header produced by the upstream ASAR serializer", async () => { + const { archive, original } = await createFixture(); + await expect(admit(archive)).resolves.toEqual({ + archiveBytes: original.length, + headerBytes: original.readUInt32LE(4), + jsonBytes: original.readUInt32LE(12), + }); + expect(await readFile(archive)).toEqual(original); +}); + +it("rejects every truncated prefix of a real serialized header", async () => { + const { archive, original } = await createFixture(); + const headerEnd = 8 + original.readUInt32LE(4); + for (let length = 0; length < headerEnd; length++) { + await writeFile(archive, original.subarray(0, length)); + await expect( + admit(archive), + `truncated at byte ${length}`, + ).rejects.toMatchObject({ + reason: "format", + }); + } + // Member data is outside header admission; an intact header still passes. + await writeFile(archive, original.subarray(0, headerEnd)); + await expect(admit(archive)).resolves.toMatchObject({ + archiveBytes: headerEnd, + }); +}); + +it("rejects contradictory and unsigned-boundary extents before decoding", async () => { + const { archive, original } = await createFixture(); + const mutations = [ + { offset: 0, values: [0, 1, 3, 5, 0xffffffff] }, + { offset: 4, values: [0, 4, 7, original.length, 0x80000000, 0xffffffff] }, + { offset: 8, values: [0, 1, original.readUInt32LE(8) + 4, 0xffffffff] }, + { offset: 12, values: [original.readUInt32LE(8), 0x80000000, 0xffffffff] }, + ]; + for (const { offset, values } of mutations) { + for (const value of values) { + const bytes = Buffer.from(original); + bytes.writeUInt32LE(value, offset); + await writeFile(archive, bytes); + await expect( + admit(archive), + `uint32 at ${offset} = ${value}`, + ).rejects.toMatchObject({ + reason: "format", + }); + } + } +}); + +it("separates the header decode budget from container validity at its exact boundary", async () => { + const root = await createTestTempDirectory("rea-asar-header-budget-"); + const archive = join(root, "budget.asar"); + expect(MAX_ASAR_HEADER_BYTES).toBe(16 * 1024 * 1024); + for (const headerBytes of [ + MAX_ASAR_HEADER_BYTES, + MAX_ASAR_HEADER_BYTES + 4, + ]) { + const prefix = Buffer.alloc(16); + prefix.writeUInt32LE(4, 0); + prefix.writeUInt32LE(headerBytes, 4); + prefix.writeUInt32LE(headerBytes - 4, 8); + prefix.writeUInt32LE(2, 12); + await writeFile(archive, prefix); + // Supply the declared extent without allocating a header-sized JS buffer. + // This tests extent admission only; the padding is not valid archive JSON. + await truncate(archive, headerBytes + 8); + if (headerBytes === MAX_ASAR_HEADER_BYTES) + await expect(admit(archive)).resolves.toEqual({ + archiveBytes: headerBytes + 8, + headerBytes, + jsonBytes: 2, + }); + else + await expect(admit(archive)).rejects.toMatchObject({ reason: "limit" }); + } +}); diff --git a/tests/boundary/filesystem/binaryTarget.test.ts b/tests/boundary/filesystem/binaryTarget.test.ts index eb5977a541..93ef57afa4 100644 --- a/tests/boundary/filesystem/binaryTarget.test.ts +++ b/tests/boundary/filesystem/binaryTarget.test.ts @@ -26,6 +26,7 @@ import { thinMach, } from "../../../src/domain/binaryTarget.fixture.js"; import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; +import { hasXmlPlistSignature } from "../../../src/domain/apple/appleArchiveFormat.js"; import { parseExecutableHeader } from "../../../src/domain/binaryTarget.js"; import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; @@ -258,6 +259,46 @@ describe("binary target I/O: package classification", () => { }); }); + it("classifies XML plists by content while preserving named formats", async () => { + const directory = await createTestTempDirectory("rea-xml-plist-target-"); + const xml = Buffer.from( + '$archiverNSKeyedArchiver', + ); + const archivePath = join(directory, "Model.archive"); + const javascriptPath = join(directory, "Model.js"); + await writeFile(archivePath, xml); + await writeFile(javascriptPath, xml); + + const archive = await parseBinaryTarget(archivePath); + const javascript = await parseBinaryTarget(javascriptPath); + expect(archive.ok && archive.value).toMatchObject({ + kind: "artifact", + format: "plist", + }); + expect(javascript.ok && javascript.value).toMatchObject({ + kind: "artifact", + format: "javascript", + }); + }); + + it("continues a bounded XML plist probe beyond the initial file window", async () => { + const directory = await createTestTempDirectory("rea-xml-long-prolog-"); + const path = join(directory, "Model.archive"); + await writeFile( + path, + `${" ".repeat(8192)}`, + ); + + const result = await parseBinaryTarget(path); + expect(result.ok && result.value).toMatchObject({ + kind: "artifact", + format: "plist", + }); + expect(hasXmlPlistSignature(Buffer.from(``))).toBe( + false, + ); + }); + it("does not trust a ZIP-family extension without ZIP magic", async () => { const directory = await createTestTempDirectory("rea-fake-archive-"); const path = join(directory, "fake.zip"); diff --git a/tests/boundary/filesystem/directoryArtifactReaderOwnership.test.ts b/tests/boundary/filesystem/directoryArtifactReaderOwnership.test.ts index 4145e7bca2..0ca2e791ae 100644 --- a/tests/boundary/filesystem/directoryArtifactReaderOwnership.test.ts +++ b/tests/boundary/filesystem/directoryArtifactReaderOwnership.test.ts @@ -1,4 +1,4 @@ -import { writeFile, type FileHandle } from "node:fs/promises"; +import { mkdir, rename, writeFile, type FileHandle } from "node:fs/promises"; import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; @@ -47,6 +47,30 @@ afterEach(() => { }); describe("DirectoryArtifactReader resource ownership", () => { + it("rejects descendants after the selected root directory is replaced", async () => { + const { root, reader } = await makeFixture(); + const subdirectory = join(root, "sub"); + await mkdir(subdirectory); + await writeFile(join(subdirectory, "old.txt"), "original tree\n"); + const movedRoot = `${root}-original`; + try { + const entries = reader.entries()[Symbol.asyncIterator](); + expect((await entries.next()).value?.path).toBe("sub"); + expect((await entries.next()).value?.path).toBe("sample.txt"); + + await rename(root, movedRoot); + await mkdir(root); + await mkdir(subdirectory); + await writeFile(join(subdirectory, "new.txt"), "replacement tree\n"); + await expect(entries.next()).rejects.toMatchObject({ + reason: "path", + message: "Directory traversal escaped or changed its artifact root", + }); + } finally { + await reader.close(); + } + }); + it("closes the exact directory owner on early iterator return", async () => { const { reader } = await makeFixture(); let closeCalls = 0; diff --git a/tests/boundary/filesystem/keyedArchiveXmlNesting.test.ts b/tests/boundary/filesystem/keyedArchiveXmlNesting.test.ts index e2dbe084dc..c1a2790e1d 100644 --- a/tests/boundary/filesystem/keyedArchiveXmlNesting.test.ts +++ b/tests/boundary/filesystem/keyedArchiveXmlNesting.test.ts @@ -2,6 +2,7 @@ import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect, it } from "vitest"; +import { parse } from "plist"; import { inspectBundleKeyedArchive } from "../../../src/artifacts/apple/KeyedArchiveReader.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; @@ -27,3 +28,76 @@ it.each([129, 3000])( }); }, ); + +it.each(["(}", "{)", "(@", "{key @", "(x @", "(x ", "{key ", "{key = x,"])( + "preserves the pinned OpenStep syntax diagnosis before a deep suffix after %s", + async (prefix) => { + const text = prefix + "(".repeat(130); + let pinned: unknown; + try { + parse(text); + } catch (cause) { + pinned = cause; + } + if (!(pinned instanceof Error) || pinned instanceof RangeError) + throw new Error( + "Malformed control must fail in the pinned syntax decoder", + ); + const root = await createTestTempDirectory("rea-keyed-openstep-malformed-"); + await writeFile(join(root, "Malformed.plist"), text); + const rejected = await inspectBundleKeyedArchive({ + bundlePath: root, + targetSha256: "a".repeat(64), + parameters: { path: "Malformed.plist" }, + platform: "linux", + }).catch((cause: unknown) => cause); + expect(rejected).toMatchObject({ + reason: "format", + message: `Cannot decode selected Foundation keyed archive: ${pinned.message}`, + }); + }, +); + +it("retains the limit classification for syntactically valid deep OpenStep values", async () => { + const root = await createTestTempDirectory("rea-keyed-openstep-limit-"); + await writeFile( + join(root, "Deep.plist"), + "(".repeat(129) + "x" + ")".repeat(129), + ); + const rejected = await inspectBundleKeyedArchive({ + bundlePath: root, + targetSha256: "a".repeat(64), + parameters: { path: "Deep.plist" }, + platform: "linux", + }).catch((cause: unknown) => cause); + expect(rejected).toMatchObject({ + reason: "limit", + message: + "Cannot decode selected Foundation keyed archive: Plist exceeds 128 nesting levels", + }); +}); + +it.each([ + ["array sibling", (deep: string) => `((x),${deep})`], + ["compound dictionary key", (deep: string) => `{(key)=${deep};}`], +] as const)( + "retains the named OpenStep depth limit after a completed %s", + async (_name, document) => { + const root = await createTestTempDirectory( + "rea-keyed-openstep-sibling-depth-", + ); + const deep = "(".repeat(30000) + "x" + ")".repeat(30000); + await writeFile(join(root, "Deep.plist"), document(deep)); + const rejected = await inspectBundleKeyedArchive({ + bundlePath: root, + targetSha256: "a".repeat(64), + parameters: { path: "Deep.plist" }, + platform: "linux", + }).catch((cause: unknown) => cause); + expect(rejected).toMatchObject({ + reason: "limit", + message: + "Cannot decode selected Foundation keyed archive: Plist exceeds 128 nesting levels", + }); + }, +); diff --git a/tests/boundary/filesystem/recoveryInputPaths.test.ts b/tests/boundary/filesystem/recoveryInputPaths.test.ts new file mode 100644 index 0000000000..6fb6afa211 --- /dev/null +++ b/tests/boundary/filesystem/recoveryInputPaths.test.ts @@ -0,0 +1,72 @@ +import { access, mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { AnalysisInputError } from "../../../src/domain/analysisErrorCore.js"; +import { snapshotRecoveryInputs } from "../../../src/javascript/recovery/RecoveryFiles.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +it + .skipIf(process.platform === "win32") + .each(["cafe\u0301.js", "chunk\\name.js"])( + "refuses an unsupported Wakaru multi-input spelling %s before snapshotting", + async (name) => { + const root = await createTestTempDirectory("rea-recovery-input-path-"); + const selected = join(root, name); + const ordinary = join(root, "main.js"); + const staging = join(root, "snapshots"); + await writeFile(selected, "export const value = 'literal';"); + await writeFile(ordinary, "export const value = 'ordinary';"); + const slashDecoy = join(root, "chunk", "name.js"); + await mkdir(join(root, "chunk")); + await writeFile(slashDecoy, "export const value = 'slash decoy';"); + await expect( + snapshotRecoveryInputs([selected, ordinary], staging), + ).rejects.toBeInstanceOf(AnalysisInputError); + await expect(access(staging)).rejects.toMatchObject({ code: "ENOENT" }); + expect(await readFile(selected, "utf8")).toBe( + "export const value = 'literal';", + ); + expect(await readFile(ordinary, "utf8")).toBe( + "export const value = 'ordinary';", + ); + expect(await readFile(slashDecoy, "utf8")).toBe( + "export const value = 'slash decoy';", + ); + }, +); + +it.skipIf(process.platform === "win32")( + "keeps ordinary slash layout and safely renames a single literal input", + async () => { + const root = await createTestTempDirectory("rea-recovery-input-path-"); + const literal = join(root, "chunk\\name.js"); + const slash = join(root, "chunk", "name.js"); + const main = join(root, "main.js"); + await mkdir(join(root, "chunk")); + await writeFile(literal, "export const value = 'literal';"); + await writeFile(slash, "export const value = 'slash';"); + await writeFile(main, "export const value = 'main';"); + const single = await snapshotRecoveryInputs( + [literal], + join(root, "single"), + ); + expect(single[0].relative_path).toBe("bundle.js"); + expect(await readFile(single[0].snapshot_path, "utf8")).toBe( + "export const value = 'literal';", + ); + const multiple = await snapshotRecoveryInputs( + [main, slash], + join(root, "multiple"), + ); + expect(multiple.map(({ relative_path }) => relative_path)).toEqual([ + "main.js", + "chunk/name.js", + ]); + expect(await readFile(multiple[1]?.snapshot_path ?? "", "utf8")).toBe( + "export const value = 'slash';", + ); + expect(await readFile(literal, "utf8")).toBe( + "export const value = 'literal';", + ); + }, +); diff --git a/tests/boundary/filesystem/referenceSourceFailureDetails.test.ts b/tests/boundary/filesystem/referenceSourceFailureDetails.test.ts index dd87ea92a2..d379f01c62 100644 --- a/tests/boundary/filesystem/referenceSourceFailureDetails.test.ts +++ b/tests/boundary/filesystem/referenceSourceFailureDetails.test.ts @@ -117,6 +117,7 @@ describe("reference import native entry failure details", () => { ).toBeUndefined(); expect(imported.value.exclusions).toContainEqual({ path: "secret-events.pipe", + entry_kind: "other", reason: "configured-secret", pattern: "*secret*", }); @@ -133,3 +134,62 @@ describe("reference import native entry failure details", () => { ).toMatchObject({ kind: "file", content_state: "hashed", limitations: [] }); }); }); + +describe("reference import paths beyond the platform limit", () => { + it.skipIf(process.platform === "win32")( + "reports an entry beyond the absolute path limit as an I/O failure", + async () => { + const root = await createTestTempDirectory("rea-import-deep-path-"); + await writeFile(join(root, "kept.txt"), "k"); + // Relative creation lets the tree exceed PATH_MAX (1024 on macOS, 4096 + // on Linux) while each 200-byte component stays below NAME_MAX. + const deepTree = (action: "create" | "remove") => + promisify(execFile)(process.execPath, [ + "-e", + `const fs = require("node:fs"); + const names = Array.from({ length: 24 }, (_, i) => "d" + String(i).padStart(2, "0") + "x".repeat(197)); + process.chdir(${JSON.stringify(root)}); + if (${JSON.stringify(action)} === "create") { + for (const name of names) { fs.mkdirSync(name); process.chdir(name); } + fs.writeFileSync("deep.txt", "d"); + } else { + for (const name of names) process.chdir(name); + fs.rmSync("deep.txt"); + for (const name of [...names].reverse()) { process.chdir(".."); fs.rmdirSync(name); } + }`, + ]); + await deepTree("create"); + try { + const read = await readReferenceSource(root); + if (!read.ok) throw new Error(read.error.message); + expect(read.value.entries).toContainEqual( + expect.objectContaining({ path: "kept.txt", status: "read" }), + ); + const failed = read.value.entries.find( + (entry) => entry.status === "failed", + ); + expect(failed).toMatchObject({ + status: "failed", + code: "io", + message: expect.stringContaining("ENAMETOOLONG"), + }); + const imported = await importTree(root); + if (!imported.ok) throw new Error(imported.error.message); + expect(imported.value.entries).toContainEqual( + expect.objectContaining({ + path: "kept.txt", + content_state: "hashed", + }), + ); + const deep = imported.value.entries.find( + (entry) => entry.path === failed?.path, + ); + expect(deep?.limitations.join(" ")).toContain( + failed?.status === "failed" ? failed.message : "missing failure", + ); + } finally { + await deepTree("remove"); + } + }, + ); +}); diff --git a/tests/boundary/filesystem/referenceSourceImport.test.ts b/tests/boundary/filesystem/referenceSourceImport.test.ts index 8443dfbdd3..f49ef6ba4c 100644 --- a/tests/boundary/filesystem/referenceSourceImport.test.ts +++ b/tests/boundary/filesystem/referenceSourceImport.test.ts @@ -89,6 +89,90 @@ describe("reference source import projections", () => { }); describe("reference source symlink import", () => { + it.each([ + { target: "dist", reason: "default-ignored", pattern: "dist/" }, + { target: "generated", reason: "project-ignored", pattern: "generated/" }, + { target: "cache", reason: "caller-excluded", pattern: "cache/" }, + ])( + "retains links into a $reason directory without importing it", + async ({ target, reason, pattern }) => { + const root = await createTestTempDirectory( + "rea-reference-excluded-links-", + ); + await mkdir(join(root, target, "nested"), { recursive: true }); + await writeFile( + join(root, target, "nested", "output.js"), + "export const generated = true;\n", + ); + await writeFile(join(root, "source.ts"), "export const value = 1;\n"); + if (reason === "project-ignored") + await writeFile(join(root, ".gitignore"), `${pattern}\n`); + await symlink(target, join(root, "alias")); + await symlink(`${target}/nested/output.js`, join(root, "output-link")); + + const result = await importReferenceSource({ + root, + caller: "reference-import-test", + policy: { secretPatterns: [] }, + ...(reason === "caller-excluded" ? { excludePaths: [pattern] } : {}), + }); + if (!result.ok) throw result.error; + expect(result.value.inventory_state).toBe("partial"); + expect(result.value.exclusions).toContainEqual({ + path: target, + entry_kind: "directory", + reason, + pattern, + }); + expect(result.value.entries).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + path: "source.ts", + kind: "file", + content_state: "hashed", + }), + expect.objectContaining({ + path: "alias", + kind: "symlink", + target, + target_state: "internal", + }), + expect.objectContaining({ + path: "output-link", + kind: "symlink", + target: `${target}/nested/output.js`, + target_state: "internal", + }), + ]), + ); + expect( + result.value.entries.some( + ({ path }) => path === target || path.startsWith(`${target}/`), + ), + ).toBe(false); + }, + ); + + it("does not treat an excluded file as an excluded directory for a child link", async () => { + const root = await createTestTempDirectory( + "rea-reference-excluded-file-link-", + ); + await writeFile(join(root, "secret.txt"), "secret\n"); + await writeFile(join(root, "source.ts"), "export const value = 1;\n"); + await symlink("secret.txt/child", join(root, "alias")); + + const result = await importReferenceSource({ + root, + caller: "reference-import-test", + policy: { secretPatterns: [] }, + excludePaths: ["secret.txt"], + }); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error.code).toBe("parse"); + expect(result.error.message).toMatch(/recorded exclusion/u); + }); + it("preserves an unreadable target as unknown instead of a synthetic path", () => { const read: ReferenceSourceRead = { root: "/reference", @@ -160,6 +244,24 @@ describe("reference source manifest inventory", () => { ["out/widget_spec.js", "JavaScript", ["generated", "source", "test"]], ["vendor/widget_test.go", "Go", ["source", "test", "vendor"]], ["settings_spec.json", "JSON", ["config", "test"]], + [ + "docs/api/client.ts", + "TypeScript", + ["documentation", "generated", "source"], + ], + [ + "packages/widget/Docs/API/client.ts", + "TypeScript", + ["documentation", "generated", "source"], + ], + ["docs/apis/client.ts", "TypeScript", ["documentation", "source"]], + ["mydocs/api/client.ts", "TypeScript", ["source"]], + ["docs/manual/api/client.ts", "TypeScript", ["documentation", "source"]], + ["docs/api.ts", "TypeScript", ["documentation", "source"]], + ["public/client.ts", "TypeScript", ["generated", "source"]], + [".github", null, ["config"]], + [".husky", null, ["config"]], + [".gitignore", null, ["config"]], ] as const; await Promise.all( files.map(async ([path]) => { @@ -184,6 +286,11 @@ describe("reference source manifest inventory", () => { language, classifications, }); + for (const path of ["docs/api", "packages/widget/Docs/API"]) + expect(entries.get(path), path).toMatchObject({ + kind: "directory", + classifications: ["documentation", "generated"], + }); }); }); @@ -305,6 +412,7 @@ describe("reference source import behavior", () => { expect(left.value.parse_failures).toHaveLength(1); expect(left.value.exclusions).toContainEqual({ path: ".env", + entry_kind: "file", reason: "configured-secret", pattern: ".env", }); @@ -505,46 +613,55 @@ describe.skipIf(process.platform === "win32")( expect.arrayContaining([ { path: "project.tmp", + entry_kind: "file", reason: "project-ignored", pattern: "project.tmp", }, { path: "project.log", + entry_kind: "file", reason: "configured-secret", pattern: "project.log", }, { path: "keep.log", + entry_kind: "file", reason: "default-ignored", pattern: "*.log", }, { path: "dist", + entry_kind: "directory", reason: "default-ignored", pattern: "dist/", }, { path: "node_modules", + entry_kind: "directory", reason: "default-ignored", pattern: "node_modules/", }, { path: "empty-ignored", + entry_kind: "directory", reason: "project-ignored", pattern: "empty-ignored/", }, { path: "project-parent", + entry_kind: "directory", reason: "project-ignored", pattern: "project-parent/", }, { path: "caller-parent", + entry_kind: "directory", reason: "caller-excluded", pattern: "caller-parent/", }, { path: "secret-dir", + entry_kind: "directory", reason: "configured-secret", pattern: "secret-dir/", }, diff --git a/tests/boundary/filesystem/referenceSourceVcs.test.ts b/tests/boundary/filesystem/referenceSourceVcs.test.ts index dbfa9ff2b9..3e46be0ce1 100644 --- a/tests/boundary/filesystem/referenceSourceVcs.test.ts +++ b/tests/boundary/filesystem/referenceSourceVcs.test.ts @@ -1,4 +1,4 @@ -import { mkdir, writeFile } from "node:fs/promises"; +import { chmod, mkdir, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect, it } from "vitest"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; @@ -108,3 +108,27 @@ it("reads a linked worktree's own branch and detached HEAD through shared Git re dirty: null, }); }); + +it.skipIf(process.platform === "win32" || process.getuid?.() === 0)( + "keeps unreadable loose refs unknown instead of falling back to packed refs", + async () => { + const root = await createTestTempDirectory("rea-reference-loose-ref-"); + const gitdir = join(root, ".git"); + const looseRef = join(gitdir, "refs", "heads", "main"); + await mkdir(join(gitdir, "refs", "heads"), { recursive: true }); + const head = "1234567890abcdef1234567890abcdef12345678"; + await writeFile(join(gitdir, "HEAD"), "ref: refs/heads/main\n"); + await writeFile(join(gitdir, "packed-refs"), `${head} refs/heads/main\n`); + await writeFile(looseRef, `${head}\n`); + await chmod(looseRef, 0); + try { + expect(await readVcs(root)).toEqual({ + kind: "unknown", + head: null, + dirty: null, + }); + } finally { + await chmod(looseRef, 0o600); + } + }, +);