fix(reference): preserve observed directories and unreadable links

Continue validating child directories observed before a parent enumeration failure. Reconcile unobserved link targets at the reader using canonical path identity and the nearest failed ancestor, preserving exclusions and cancellation.

Refs: #4013, #4053, #4060

Co-authored-by: XY-code1 <1611761391@qq.com>

src/reference/ReferenceSourceReader.test.ts

@@ -0,0 +1,113 @@
+import { describe, expect, it } from "vitest";
+
+import type {
+  ReferenceSourceEntry,
+  TraversalState,
+} from "./ReferenceSourceReaderTypes.js";
+import { reconcileUnobservedSymlinkTargets } from "./ReferenceSourceReader.js";
+
+const failedDirectory = (
+  path: string,
+  code: "changed" | "io" | "symlink",
+): ReferenceSourceEntry => ({
+  status: "failed",
+  kind: "directory",
+  path,
+  code,
+  message: `directory failure ${code}`,
+});
+
+const internalLink = (path: string, target: string): ReferenceSourceEntry => ({
+  status: "read",
+  kind: "symlink",
+  path,
+  target,
+  targetState: "internal",
+});
+
+const exclusions = (
+  ...items: TraversalState["exclusions"][number][]
+): TraversalState["exclusions"] => items;
+
+describe("unobserved internal symlink reconciliation", () => {
+  it("retains unreadable targets below the nearest failed directory", () => {
+    const entries: ReferenceSourceEntry[] = [
+      internalLink("alias.txt", "hidden/target.txt"),
+      failedDirectory("hidden", "io"),
+      {
+        status: "read",
+        kind: "file",
+        path: "visible.txt",
+        bytes: new Uint8Array([1]),
+        size: 1,
+      },
+    ];
+
+    reconcileUnobservedSymlinkTargets(entries);
+
+    expect(entries[0]).toMatchObject({
+      status: "failed",
+      kind: "symlink",
+      code: "io",
+      message: expect.stringContaining("directory failure io"),
+    });
+    expect(entries[2]).toMatchObject({ status: "read", path: "visible.txt" });
+  });
+
+  it("matches Unicode-equivalent failed ancestor identities", () => {
+    const entries: ReferenceSourceEntry[] = [
+      internalLink("alias.txt", "hidden/cafe\u0301/target.txt"),
+      failedDirectory("hidden/caf\u00e9", "changed"),
+    ];
+
+    reconcileUnobservedSymlinkTargets(entries);
+
+    expect(entries[0]).toMatchObject({
+      status: "failed",
+      kind: "symlink",
+      code: "changed",
+      message: expect.stringContaining("directory failure changed"),
+    });
+  });
+
+  it("stops at a complete nearer directory or an exclusion of unknown kind", () => {
+    const entries: ReferenceSourceEntry[] = [
+      internalLink("to-missing", "outer/complete/missing.txt"),
+      internalLink("to-excluded", "outer/excluded/missing.txt"),
+      failedDirectory("outer", "io"),
+      { status: "read", kind: "directory", path: "outer/complete" },
+    ];
+
+    reconcileUnobservedSymlinkTargets(
+      entries,
+      exclusions({ path: "outer/excluded", kind: "unknown" }),
+    );
+
+    expect(entries[0]).toMatchObject({
+      status: "read",
+      targetState: "internal",
+    });
+    expect(entries[1]).toMatchObject({
+      status: "read",
+      targetState: "internal",
+    });
+  });
+
+  it("keeps exact exclusions and leaves observations untouched after cancellation", () => {
+    const entries = [internalLink("alias.txt", "hidden/target.txt")];
+    reconcileUnobservedSymlinkTargets(
+      entries,
+      exclusions({ path: "hidden/target.txt", kind: "file" }),
+    );
+    expect(entries[0]).toMatchObject({ targetState: "internal" });
+
+    const cancelledEntries: ReferenceSourceEntry[] = [
+      internalLink("alias.txt", "hidden/target.txt"),
+      failedDirectory("hidden", "io"),
+    ];
+    const controller = new AbortController();
+    controller.abort();
+    reconcileUnobservedSymlinkTargets(cancelledEntries, [], controller.signal);
+    expect(cancelledEntries[0]).toMatchObject({ targetState: "internal" });
+  });
+});

src/reference/ReferenceSourceReader.ts

@@ -1,10 +1,14 @@
+import { posix } from "node:path";
+
 import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js";
+import { createReferenceSourcePathLookup } from "../domain/referenceSourcePathIdentity.js";
 import { err, ok } from "../domain/result.js";
 import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js";
 import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js";
 import { traverseDirectory } from "./ReferenceSourceReaderEntries.js";
 import type {
   ReferenceSourceReaderOptions,
+  ReferenceSourceEntry,
   ReferenceSourceRead,
   ReferenceSourceResult,
   TraversalState,
@@ -61,10 +65,20 @@ const readSource = async (
       ? {}
       : { shouldExclude: options.shouldExclude }),
     entries: [],
+    exclusions: [],
     pending: [{ path: canonicalRoot }],
     bytesRead: 0,
   };
   const traversal = await traverse(state);
+  if (
+    !isAborted(state.signal) &&
+    (traversal.ok || traversal.error.code !== "cancelled")
+  )
+    reconcileUnobservedSymlinkTargets(
+      state.entries,
+      state.exclusions,
+      state.signal,
+    );
   state.entries.sort((left, right) =>
     compareUnicodeCodePoints(left.path, right.path),
   );
@@ -77,6 +91,96 @@ const readSource = async (
   return traversal.ok ? ok(read) : err({ ...traversal.error, partial: read });
 };
 
+/**
+ * Resolve internal link observations against the completed reader inventory.
+ * If the closest containing directory failed before enumeration completed,
+ * the target remains unknown; a broader failed ancestor cannot override a
+ * successfully enumerated directory or a recorded exclusion.
+ */
+export const reconcileUnobservedSymlinkTargets = (
+  entries: ReferenceSourceEntry[],
+  exclusions: TraversalState["exclusions"] = [],
+  signal?: AbortSignal,
+): void => {
+  const failedDirectories = new Map(
+    entries.flatMap((entry) =>
+      entry.status === "failed" &&
+      entry.kind === "directory" &&
+      entry.code !== "cancelled"
+        ? [[entry.path, entry] as const]
+        : [],
+    ),
+  );
+  if (failedDirectories.size === 0 || isAborted(signal)) return;
+  const directoryEntries = entries.filter(
+    (entry) => entry.kind === "directory",
+  );
+  const directoryPaths = createReferenceSourcePathLookup(
+    directoryEntries.map(({ path }) => path),
+  );
+  const observedPaths = createReferenceSourcePathLookup(
+    entries.map(({ path }) => path),
+  );
+  const excludedPaths = createReferenceSourcePathLookup(
+    exclusions.map(({ path }) => path),
+  );
+  for (const [index, entry] of entries.entries()) {
+    if (isAborted(signal)) return;
+    if (
+      entry.status !== "read" ||
+      entry.kind !== "symlink" ||
+      entry.targetState !== "internal" ||
+      observedPaths.resolve(entry.target) !== undefined ||
+      excludedPaths.resolve(entry.target) !== undefined
+    )
+      continue;
+    const ancestor = nearestFailedAncestor(
+      entry.target,
+      directoryPaths,
+      failedDirectories,
+      excludedPaths,
+      signal,
+    );
+    if (ancestor === undefined) continue;
+    entries[index] = {
+      status: "failed",
+      kind: "symlink",
+      path: entry.path,
+      code: ancestor.code,
+      message: `Symbolic link target is inside a directory that could not be enumerated: ${ancestor.message}`,
+    };
+  }
+};
+
+const nearestFailedAncestor = (
+  target: string,
+  directoryPaths: ReturnType<typeof createReferenceSourcePathLookup>,
+  failedDirectories: ReadonlyMap<
+    string,
+    Extract<ReferenceSourceEntry, { status: "failed" }>
+  >,
+  excludedPaths: ReturnType<typeof createReferenceSourcePathLookup>,
+  signal: AbortSignal | undefined,
+): Extract<ReferenceSourceEntry, { status: "failed" }> | undefined => {
+  for (
+    let candidate = posix.dirname(target);
+    candidate !== "." && candidate !== "/";
+    candidate = posix.dirname(candidate)
+  ) {
+    if (isAborted(signal)) return undefined;
+    if (excludedPaths.resolve(candidate) !== undefined) return undefined;
+    const observedDirectory = directoryPaths.resolve(candidate);
+    if (observedDirectory !== undefined)
+      return failedDirectories.get(observedDirectory);
+  }
+  if (isAborted(signal)) return undefined;
+  if (excludedPaths.resolve(".") !== undefined) return undefined;
+  const observedRoot = directoryPaths.resolve(".");
+  return observedRoot === undefined
+    ? undefined
+    : failedDirectories.get(observedRoot);
+};
+
 const traverse = async (
   state: TraversalState,
 ): Promise<ReferenceSourceResult<undefined>> => {

src/reference/ReferenceSourceReaderEntries.ts

@@ -134,6 +134,7 @@ export const traverseDirectory = async (
         readFailure,
       ),
     );
+    scheduleObservedDirectories(state, directories);
     return { ok: true, value: undefined };
   }
   const after = await validateDirectory(
@@ -152,6 +153,7 @@ export const traverseDirectory = async (
         "Directory changed while it was read",
       ),
     );
+    scheduleObservedDirectories(state, directories);
     return { ok: true, value: undefined };
   }
   if (current.path !== state.root)
@@ -160,9 +162,16 @@ export const traverseDirectory = async (
       kind: "directory",
       path: pathFromRoot(state.root, current.path),
     });
+  scheduleObservedDirectories(state, directories);
+  return { ok: true, value: undefined };
+};
+
+const scheduleObservedDirectories = (
+  state: TraversalState,
+  directories: PendingDirectory[],
+): void => {
   directories.reverse();
   state.pending.push(...directories);
-  return { ok: true, value: undefined };
 };
 
 const directoryReadFailure = (cause: unknown): string => {
@@ -207,7 +216,10 @@ const processEntry = async (
         ok: false,
         error: failure("io", "Reference source exclusion check failed"),
       };
-    if (excluded.value) return { ok: true, value: undefined };
+    if (excluded.value) {
+      state.exclusions.push({ path, kind: "unknown" });
+      return { ok: true, value: undefined };
+    }
     state.entries.push(entryFailure(path, "unknown", "io", metadata.message));
     return { ok: true, value: undefined };
   }
@@ -228,7 +240,10 @@ const processEntry = async (
         message: "Reference source exclusion check failed",
       },
     };
-  if (excluded.value) return { ok: true, value: undefined };
+  if (excluded.value) {
+    state.exclusions.push({ path, kind });
+    return { ok: true, value: undefined };
+  }
   if (isAborted(state.signal)) return { ok: false, error: cancelled() };
   if (kind === "symlink")
     state.entries.push(

src/reference/ReferenceSourceReaderTypes.ts

@@ -93,6 +93,10 @@ export type TraversalState = {
     kind: ReferenceSourceExclusionKind,
   ) => boolean;
   readonly entries: ReferenceSourceEntry[];
+  readonly exclusions: Array<{
+    readonly path: string;
+    readonly kind: ReferenceSourceExclusionKind;
+  }>;
   readonly pending: PendingDirectory[];
   bytesRead: number;
 };

tests/boundary/filesystem/referenceSourceQueuedDirectories.test.ts

@@ -0,0 +1,107 @@
+import type { Dir, Dirent } from "node:fs";
+import { mkdir, symlink, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+import { expect, it, vi } from "vitest";
+
+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
+import { importReferenceSource } from "../../support/referenceSourceResourceScope.js";
+
+const { opendirMock } = vi.hoisted(() => ({ opendirMock: vi.fn() }));
+
+vi.mock("node:fs/promises", async (importOriginal) => ({
+  ...(await importOriginal<typeof import("node:fs/promises")>()),
+  opendir: opendirMock,
+}));
+
+const directoryHandle = (items: readonly (string | Error | null)[]): Dir => {
+  let index = 0;
+  return {
+    async read() {
+      const item = items[index++];
+      if (item instanceof Error) throw item;
+      return item === null || item === undefined
+        ? null
+        : ({ name: item } as Dirent);
+    },
+    async close() {},
+  } as unknown as Dir;
+};
+
+it.skipIf(process.platform === "win32")(
+  "traverses directories lstat-observed before their parent enumeration failed",
+  async () => {
+    const root = await createTestTempDirectory(
+      "rea-queued-source-directories-",
+    );
+    const container = join(root, "container");
+    const hidden = join(container, "hidden");
+    await mkdir(hidden, { recursive: true });
+    await writeFile(join(container, "visible.txt"), "visible\n");
+    await writeFile(join(hidden, "target.txt"), "target\n");
+    await symlink("hidden/target.txt", join(container, "alias-file.txt"));
+    await symlink("hidden", join(container, "alias-directory"));
+
+    const readFailure = Object.assign(
+      new Error("injected directory read failure"),
+      { code: "EIO" },
+    );
+    opendirMock.mockImplementation(async (path: string) => {
+      if (path === root) return directoryHandle(["container", null]);
+      if (path === container)
+        return directoryHandle([
+          "visible.txt",
+          "hidden",
+          "alias-file.txt",
+          "alias-directory",
+          readFailure,
+        ]);
+      if (path === hidden) return directoryHandle(["target.txt", null]);
+      throw new Error(`Unexpected directory traversal: ${path}`);
+    });
+
+    const imported = await importReferenceSource({
+      root,
+      caller: "queued-directory-read-failure-test",
+      policy: { secretPatterns: [] },
+    });
+    if (!imported.ok) throw new Error(imported.error.message);
+    expect(imported.value.inventory_state).toBe("partial");
+    expect(imported.value.entries).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          path: "container",
+          kind: "directory",
+          tree_state: "unreadable",
+        }),
+        expect.objectContaining({
+          path: "container/hidden",
+          kind: "directory",
+          tree_state: "enumerated",
+        }),
+        expect.objectContaining({
+          path: "container/visible.txt",
+          kind: "file",
+          content_state: "hashed",
+        }),
+        expect.objectContaining({
+          path: "container/hidden/target.txt",
+          kind: "file",
+          content_state: "hashed",
+        }),
+        expect.objectContaining({
+          path: "container/alias-file.txt",
+          kind: "symlink",
+          target: "hidden/target.txt",
+          target_state: "internal",
+        }),
+        expect.objectContaining({
+          path: "container/alias-directory",
+          kind: "symlink",
+          target: "hidden",
+          target_state: "internal",
+        }),
+      ]),
+    );
+  },
+);

tests/boundary/filesystem/referenceSourceUnreadableDirectories.test.ts

@@ -1,4 +1,4 @@
-import { chmod, mkdir, writeFile } from "node:fs/promises";
+import { chmod, mkdir, symlink, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 
 import { describe, expect, it } from "vitest";
@@ -62,6 +62,60 @@ describe("reference source native unreadable directories", () => {
     },
   );
 
+  it.skipIf(permissionChecksUnavailable)(
+    "keeps links into an unreadable subtree uncertain while preserving readable siblings",
+    async () => {
+      const root = await createTestTempDirectory("rea-source-unreadable-link-");
+      const hidden = join(root, "hidden");
+      await mkdir(hidden);
+      await writeFile(join(root, "visible.txt"), "visible\n");
+      await writeFile(join(hidden, "target.txt"), "target\n");
+      await symlink("hidden/target.txt", join(root, "alias.txt"));
+      await chmod(hidden, 0o111);
+      try {
+        const read = await readReferenceSource(root);
+        if (!read.ok) throw new Error(read.error.message);
+        expect(read.value.entries).toContainEqual(
+          expect.objectContaining({
+            status: "failed",
+            kind: "directory",
+            path: "hidden",
+            code: "io",
+          }),
+        );
+        expect(read.value.entries).toContainEqual(
+          expect.objectContaining({
+            status: "failed",
+            kind: "symlink",
+            path: "alias.txt",
+            code: "io",
+          }),
+        );
+
+        const imported = await importTree(root);
+        if (!imported.ok) throw new Error(imported.error.message);
+        expect(imported.value.inventory_state).toBe("partial");
+        expect(imported.value.entries).toContainEqual(
+          expect.objectContaining({
+            path: "visible.txt",
+            kind: "file",
+            content_state: "hashed",
+          }),
+        );
+        expect(imported.value.entries).toContainEqual(
+          expect.objectContaining({
+            path: "alias.txt",
+            kind: "symlink",
+            target: null,
+            target_state: "unreadable",
+          }),
+        );
+      } finally {
+        await chmod(hidden, 0o700);
+      }
+    },
+  );
+
   it("keeps readable and empty directories enumerated exactly once", async () => {
     const root = await createTestTempDirectory("rea-source-readable-dirs-");
     await mkdir(join(root, "src", "empty"), { recursive: true });