From d6e4fe4a4dd8914ec3db50d2a7fc3d0802649262 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 16 Oct 2026 04:41:40 +0800 Subject: [PATCH] fix(reference): preserve observed directories and unreadable links Continue validating child directories observed before a parent enumeration failure. Reconcile unobserved link targets at the reader using canonical path identity and the nearest failed ancestor, preserving exclusions and cancellation. Refs: #4013, #4053, #4060 Co-authored-by: XY-code1 <1611761391@qq.com> --- src/reference/ReferenceSourceReader.test.ts | 113 ++++++++++++++++++ src/reference/ReferenceSourceReader.ts | 104 ++++++++++++++++ src/reference/ReferenceSourceReaderEntries.ts | 21 +++- src/reference/ReferenceSourceReaderTypes.ts | 4 + .../referenceSourceQueuedDirectories.test.ts | 107 +++++++++++++++++ ...ferenceSourceUnreadableDirectories.test.ts | 56 ++++++++- 6 files changed, 401 insertions(+), 4 deletions(-) create mode 100644 src/reference/ReferenceSourceReader.test.ts create mode 100644 tests/boundary/filesystem/referenceSourceQueuedDirectories.test.ts diff --git a/src/reference/ReferenceSourceReader.test.ts b/src/reference/ReferenceSourceReader.test.ts new file mode 100644 index 000000000..2eef9ec31 --- /dev/null +++ b/src/reference/ReferenceSourceReader.test.ts @@ -0,0 +1,113 @@ +import { describe, expect, it } from "vitest"; + +import type { + ReferenceSourceEntry, + TraversalState, +} from "./ReferenceSourceReaderTypes.js"; +import { reconcileUnobservedSymlinkTargets } from "./ReferenceSourceReader.js"; + +const failedDirectory = ( + path: string, + code: "changed" | "io" | "symlink", +): ReferenceSourceEntry => ({ + status: "failed", + kind: "directory", + path, + code, + message: `directory failure ${code}`, +}); + +const internalLink = (path: string, target: string): ReferenceSourceEntry => ({ + status: "read", + kind: "symlink", + path, + target, + targetState: "internal", +}); + +const exclusions = ( + ...items: TraversalState["exclusions"][number][] +): TraversalState["exclusions"] => items; + +describe("unobserved internal symlink reconciliation", () => { + it("retains unreadable targets below the nearest failed directory", () => { + const entries: ReferenceSourceEntry[] = [ + internalLink("alias.txt", "hidden/target.txt"), + failedDirectory("hidden", "io"), + { + status: "read", + kind: "file", + path: "visible.txt", + bytes: new Uint8Array([1]), + size: 1, + }, + ]; + + reconcileUnobservedSymlinkTargets(entries); + + expect(entries[0]).toMatchObject({ + status: "failed", + kind: "symlink", + code: "io", + message: expect.stringContaining("directory failure io"), + }); + expect(entries[2]).toMatchObject({ status: "read", path: "visible.txt" }); + }); + + it("matches Unicode-equivalent failed ancestor identities", () => { + const entries: ReferenceSourceEntry[] = [ + internalLink("alias.txt", "hidden/cafe\u0301/target.txt"), + failedDirectory("hidden/caf\u00e9", "changed"), + ]; + + reconcileUnobservedSymlinkTargets(entries); + + expect(entries[0]).toMatchObject({ + status: "failed", + kind: "symlink", + code: "changed", + message: expect.stringContaining("directory failure changed"), + }); + }); + + it("stops at a complete nearer directory or an exclusion of unknown kind", () => { + const entries: ReferenceSourceEntry[] = [ + internalLink("to-missing", "outer/complete/missing.txt"), + internalLink("to-excluded", "outer/excluded/missing.txt"), + failedDirectory("outer", "io"), + { status: "read", kind: "directory", path: "outer/complete" }, + ]; + + reconcileUnobservedSymlinkTargets( + entries, + exclusions({ path: "outer/excluded", kind: "unknown" }), + ); + + expect(entries[0]).toMatchObject({ + status: "read", + targetState: "internal", + }); + expect(entries[1]).toMatchObject({ + status: "read", + targetState: "internal", + }); + }); + + it("keeps exact exclusions and leaves observations untouched after cancellation", () => { + const entries = [internalLink("alias.txt", "hidden/target.txt")]; + reconcileUnobservedSymlinkTargets( + entries, + exclusions({ path: "hidden/target.txt", kind: "file" }), + ); + expect(entries[0]).toMatchObject({ targetState: "internal" }); + + const cancelledEntries: ReferenceSourceEntry[] = [ + internalLink("alias.txt", "hidden/target.txt"), + failedDirectory("hidden", "io"), + ]; + const controller = new AbortController(); + controller.abort(); + reconcileUnobservedSymlinkTargets(cancelledEntries, [], controller.signal); + expect(cancelledEntries[0]).toMatchObject({ targetState: "internal" }); + }); +}); diff --git a/src/reference/ReferenceSourceReader.ts b/src/reference/ReferenceSourceReader.ts index e8e8650a1..6420e3f34 100644 --- a/src/reference/ReferenceSourceReader.ts +++ b/src/reference/ReferenceSourceReader.ts @@ -1,10 +1,14 @@ +import { posix } from "node:path"; + import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; +import { createReferenceSourcePathLookup } from "../domain/referenceSourcePathIdentity.js"; import { err, ok } from "../domain/result.js"; import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; import { traverseDirectory } from "./ReferenceSourceReaderEntries.js"; import type { ReferenceSourceReaderOptions, + ReferenceSourceEntry, ReferenceSourceRead, ReferenceSourceResult, TraversalState, @@ -61,10 +65,20 @@ const readSource = async ( ? {} : { shouldExclude: options.shouldExclude }), entries: [], + exclusions: [], pending: [{ path: canonicalRoot }], bytesRead: 0, }; const traversal = await traverse(state); + if ( + !isAborted(state.signal) && + (traversal.ok || traversal.error.code !== "cancelled") + ) + reconcileUnobservedSymlinkTargets( + state.entries, + state.exclusions, + state.signal, + ); state.entries.sort((left, right) => compareUnicodeCodePoints(left.path, right.path), ); @@ -77,6 +91,96 @@ const readSource = async ( return traversal.ok ? ok(read) : err({ ...traversal.error, partial: read }); }; +/** + * Resolve internal link observations against the completed reader inventory. + * If the closest containing directory failed before enumeration completed, + * the target remains unknown; a broader failed ancestor cannot override a + * successfully enumerated directory or a recorded exclusion. + */ +export const reconcileUnobservedSymlinkTargets = ( + entries: ReferenceSourceEntry[], + exclusions: TraversalState["exclusions"] = [], + signal?: AbortSignal, +): void => { + const failedDirectories = new Map( + entries.flatMap((entry) => + entry.status === "failed" && + entry.kind === "directory" && + entry.code !== "cancelled" + ? [[entry.path, entry] as const] + : [], + ), + ); + if (failedDirectories.size === 0 || isAborted(signal)) return; + const directoryEntries = entries.filter( + (entry) => entry.kind === "directory", + ); + const directoryPaths = createReferenceSourcePathLookup( + directoryEntries.map(({ path }) => path), + ); + const observedPaths = createReferenceSourcePathLookup( + entries.map(({ path }) => path), + ); + const excludedPaths = createReferenceSourcePathLookup( + exclusions.map(({ path }) => path), + ); + for (const [index, entry] of entries.entries()) { + if (isAborted(signal)) return; + if ( + entry.status !== "read" || + entry.kind !== "symlink" || + entry.targetState !== "internal" || + observedPaths.resolve(entry.target) !== undefined || + excludedPaths.resolve(entry.target) !== undefined + ) + continue; + const ancestor = nearestFailedAncestor( + entry.target, + directoryPaths, + failedDirectories, + excludedPaths, + signal, + ); + if (ancestor === undefined) continue; + entries[index] = { + status: "failed", + kind: "symlink", + path: entry.path, + code: ancestor.code, + message: `Symbolic link target is inside a directory that could not be enumerated: ${ancestor.message}`, + }; + } +}; + +const nearestFailedAncestor = ( + target: string, + directoryPaths: ReturnType, + failedDirectories: ReadonlyMap< + string, + Extract + >, + excludedPaths: ReturnType, + signal: AbortSignal | undefined, +): Extract | undefined => { + for ( + let candidate = posix.dirname(target); + candidate !== "." && candidate !== "/"; + candidate = posix.dirname(candidate) + ) { + if (isAborted(signal)) return undefined; + if (excludedPaths.resolve(candidate) !== undefined) return undefined; + const observedDirectory = directoryPaths.resolve(candidate); + if (observedDirectory !== undefined) + return failedDirectories.get(observedDirectory); + } + if (isAborted(signal)) return undefined; + if (excludedPaths.resolve(".") !== undefined) return undefined; + const observedRoot = directoryPaths.resolve("."); + return observedRoot === undefined + ? undefined + : failedDirectories.get(observedRoot); +}; + const traverse = async ( state: TraversalState, ): Promise> => { diff --git a/src/reference/ReferenceSourceReaderEntries.ts b/src/reference/ReferenceSourceReaderEntries.ts index a7ae837e8..1ca0ce3e2 100644 --- a/src/reference/ReferenceSourceReaderEntries.ts +++ b/src/reference/ReferenceSourceReaderEntries.ts @@ -134,6 +134,7 @@ export const traverseDirectory = async ( readFailure, ), ); + scheduleObservedDirectories(state, directories); return { ok: true, value: undefined }; } const after = await validateDirectory( @@ -152,6 +153,7 @@ export const traverseDirectory = async ( "Directory changed while it was read", ), ); + scheduleObservedDirectories(state, directories); return { ok: true, value: undefined }; } if (current.path !== state.root) @@ -160,9 +162,16 @@ export const traverseDirectory = async ( kind: "directory", path: pathFromRoot(state.root, current.path), }); + scheduleObservedDirectories(state, directories); + return { ok: true, value: undefined }; +}; + +const scheduleObservedDirectories = ( + state: TraversalState, + directories: PendingDirectory[], +): void => { directories.reverse(); state.pending.push(...directories); - return { ok: true, value: undefined }; }; const directoryReadFailure = (cause: unknown): string => { @@ -207,7 +216,10 @@ const processEntry = async ( ok: false, error: failure("io", "Reference source exclusion check failed"), }; - if (excluded.value) return { ok: true, value: undefined }; + if (excluded.value) { + state.exclusions.push({ path, kind: "unknown" }); + return { ok: true, value: undefined }; + } state.entries.push(entryFailure(path, "unknown", "io", metadata.message)); return { ok: true, value: undefined }; } @@ -228,7 +240,10 @@ const processEntry = async ( message: "Reference source exclusion check failed", }, }; - if (excluded.value) return { ok: true, value: undefined }; + if (excluded.value) { + state.exclusions.push({ path, kind }); + return { ok: true, value: undefined }; + } if (isAborted(state.signal)) return { ok: false, error: cancelled() }; if (kind === "symlink") state.entries.push( diff --git a/src/reference/ReferenceSourceReaderTypes.ts b/src/reference/ReferenceSourceReaderTypes.ts index 1fec38313..85d6d7602 100644 --- a/src/reference/ReferenceSourceReaderTypes.ts +++ b/src/reference/ReferenceSourceReaderTypes.ts @@ -93,6 +93,10 @@ export type TraversalState = { kind: ReferenceSourceExclusionKind, ) => boolean; readonly entries: ReferenceSourceEntry[]; + readonly exclusions: Array<{ + readonly path: string; + readonly kind: ReferenceSourceExclusionKind; + }>; readonly pending: PendingDirectory[]; bytesRead: number; }; diff --git a/tests/boundary/filesystem/referenceSourceQueuedDirectories.test.ts b/tests/boundary/filesystem/referenceSourceQueuedDirectories.test.ts new file mode 100644 index 000000000..6bc4be707 --- /dev/null +++ b/tests/boundary/filesystem/referenceSourceQueuedDirectories.test.ts @@ -0,0 +1,107 @@ +import type { Dir, Dirent } from "node:fs"; +import { mkdir, symlink, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect, it, vi } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { importReferenceSource } from "../../support/referenceSourceResourceScope.js"; + +const { opendirMock } = vi.hoisted(() => ({ opendirMock: vi.fn() })); + +vi.mock("node:fs/promises", async (importOriginal) => ({ + ...(await importOriginal()), + opendir: opendirMock, +})); + +const directoryHandle = (items: readonly (string | Error | null)[]): Dir => { + let index = 0; + return { + async read() { + const item = items[index++]; + if (item instanceof Error) throw item; + return item === null || item === undefined + ? null + : ({ name: item } as Dirent); + }, + async close() {}, + } as unknown as Dir; +}; + +it.skipIf(process.platform === "win32")( + "traverses directories lstat-observed before their parent enumeration failed", + async () => { + const root = await createTestTempDirectory( + "rea-queued-source-directories-", + ); + const container = join(root, "container"); + const hidden = join(container, "hidden"); + await mkdir(hidden, { recursive: true }); + await writeFile(join(container, "visible.txt"), "visible\n"); + await writeFile(join(hidden, "target.txt"), "target\n"); + await symlink("hidden/target.txt", join(container, "alias-file.txt")); + await symlink("hidden", join(container, "alias-directory")); + + const readFailure = Object.assign( + new Error("injected directory read failure"), + { code: "EIO" }, + ); + opendirMock.mockImplementation(async (path: string) => { + if (path === root) return directoryHandle(["container", null]); + if (path === container) + return directoryHandle([ + "visible.txt", + "hidden", + "alias-file.txt", + "alias-directory", + readFailure, + ]); + if (path === hidden) return directoryHandle(["target.txt", null]); + throw new Error(`Unexpected directory traversal: ${path}`); + }); + + const imported = await importReferenceSource({ + root, + caller: "queued-directory-read-failure-test", + policy: { secretPatterns: [] }, + }); + if (!imported.ok) throw new Error(imported.error.message); + expect(imported.value.inventory_state).toBe("partial"); + expect(imported.value.entries).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + path: "container", + kind: "directory", + tree_state: "unreadable", + }), + expect.objectContaining({ + path: "container/hidden", + kind: "directory", + tree_state: "enumerated", + }), + expect.objectContaining({ + path: "container/visible.txt", + kind: "file", + content_state: "hashed", + }), + expect.objectContaining({ + path: "container/hidden/target.txt", + kind: "file", + content_state: "hashed", + }), + expect.objectContaining({ + path: "container/alias-file.txt", + kind: "symlink", + target: "hidden/target.txt", + target_state: "internal", + }), + expect.objectContaining({ + path: "container/alias-directory", + kind: "symlink", + target: "hidden", + target_state: "internal", + }), + ]), + ); + }, +); diff --git a/tests/boundary/filesystem/referenceSourceUnreadableDirectories.test.ts b/tests/boundary/filesystem/referenceSourceUnreadableDirectories.test.ts index ec868052b..ae54e16ce 100644 --- a/tests/boundary/filesystem/referenceSourceUnreadableDirectories.test.ts +++ b/tests/boundary/filesystem/referenceSourceUnreadableDirectories.test.ts @@ -1,4 +1,4 @@ -import { chmod, mkdir, writeFile } from "node:fs/promises"; +import { chmod, mkdir, symlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; @@ -62,6 +62,60 @@ describe("reference source native unreadable directories", () => { }, ); + it.skipIf(permissionChecksUnavailable)( + "keeps links into an unreadable subtree uncertain while preserving readable siblings", + async () => { + const root = await createTestTempDirectory("rea-source-unreadable-link-"); + const hidden = join(root, "hidden"); + await mkdir(hidden); + await writeFile(join(root, "visible.txt"), "visible\n"); + await writeFile(join(hidden, "target.txt"), "target\n"); + await symlink("hidden/target.txt", join(root, "alias.txt")); + await chmod(hidden, 0o111); + try { + const read = await readReferenceSource(root); + if (!read.ok) throw new Error(read.error.message); + expect(read.value.entries).toContainEqual( + expect.objectContaining({ + status: "failed", + kind: "directory", + path: "hidden", + code: "io", + }), + ); + expect(read.value.entries).toContainEqual( + expect.objectContaining({ + status: "failed", + kind: "symlink", + path: "alias.txt", + code: "io", + }), + ); + + const imported = await importTree(root); + if (!imported.ok) throw new Error(imported.error.message); + expect(imported.value.inventory_state).toBe("partial"); + expect(imported.value.entries).toContainEqual( + expect.objectContaining({ + path: "visible.txt", + kind: "file", + content_state: "hashed", + }), + ); + expect(imported.value.entries).toContainEqual( + expect.objectContaining({ + path: "alias.txt", + kind: "symlink", + target: null, + target_state: "unreadable", + }), + ); + } finally { + await chmod(hidden, 0o700); + } + }, + ); + it("keeps readable and empty directories enumerated exactly once", async () => { const root = await createTestTempDirectory("rea-source-readable-dirs-"); await mkdir(join(root, "src", "empty"), { recursive: true });