From 71ae02c6efceb6b59e703c0c4e7367d7ea8c6c03 Mon Sep 17 00:00:00 2001 From: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> Date: Tue, 20 Oct 2026 03:27:43 +0800 Subject: [PATCH] fix(process): retain child scope and bound native exit waits (#5280) Retain original parent/child scope and prior native-generation proof across exit settlement and token retries. Keep historical scope separate from signal authority, count each supplied snapshot once, and share one existing native-exit deadline throughout cleanup. Validated 387 affected tests (one Darwin-only skip), five controls repeated three times, and 30 synced-main integration tests (one platform skip). Synced-head static, architecture, docs and discovery checks pass; real GDB qualification passes 44 CLI/MCP cases. Native-observer edge controls are simulated. Assisted-by: Reimu (gensokyo-skills:reimu-incident-triage) Assisted-by: Kogasa (gensokyo-skills:kogasa-surprise-testing) --- src/process/ProcessLifecycle.ts | 5 + .../ProcessOwnership.exitScope.test.ts | 190 ++++++++++++++++++ src/process/ProcessOwnership.ts | 47 ++++- src/process/ProcessOwnershipEvidence.ts | 38 +++- src/process/ProcessOwnershipObservation.ts | 31 ++- 5 files changed, 293 insertions(+), 18 deletions(-) create mode 100644 src/process/ProcessOwnership.exitScope.test.ts diff --git a/src/process/ProcessLifecycle.ts b/src/process/ProcessLifecycle.ts index 9c0d0f52e1..df6619016d 100644 --- a/src/process/ProcessLifecycle.ts +++ b/src/process/ProcessLifecycle.ts @@ -10,6 +10,11 @@ export type ProcessLifecycleObservation = export const PROCESS_CLEANUP_VERIFICATION_GRACE_MS = 1_000; export const PROCESS_CLEANUP_VERIFICATION_INTERVAL_MS = 25; +/** One operation owns a lazily started native-exit waiting window. */ +export interface ProcessExitSettlementBudget { + deadline: number | undefined; +} + export const processIdentityHasExited = ( identity: ProcessIdentityObservation | undefined, ): boolean => diff --git a/src/process/ProcessOwnership.exitScope.test.ts b/src/process/ProcessOwnership.exitScope.test.ts new file mode 100644 index 0000000000..64df933d6b --- /dev/null +++ b/src/process/ProcessOwnership.exitScope.test.ts @@ -0,0 +1,190 @@ +import { performance } from "node:perf_hooks"; +import { expect, it, vi } from "vitest"; +import { + cleanupOwnedProcessGroup, + verifyNoTokenOwnedProcesses, + type ProcessIdentityObservation, + type ProcessOwnershipHost, + type ProcessRunTokenObservation, + type ProcessTableEntry, +} from "./ProcessOwnership.js"; +import { ProcessOwnershipEvidenceOwner } from "./ProcessOwnershipEvidence.js"; + +const row = (pid: number, parentPid: number): ProcessTableEntry => ({ + pid, + parentPid, + processGroupId: pid, + state: "R", + command: "/fixture/node", +}); +const identity = ( + process: ProcessTableEntry, + exiting: boolean, +): ProcessIdentityObservation => ({ + state: "readable", + identity: `start:${process.pid}`, + lifecycle: { + state: "observed", + lifecycle: exiting ? "exiting" : "non-zombie", + }, + relation: { + state: "observed", + parentPid: process.parentPid, + processGroupId: process.processGroupId, + }, +}); + +it.each(["readable", "retried", "reused", "retained-parent"] as const)( + "keeps child scope tied to its native generation after parent exit (%s)", + async (mode) => { + const parent = row(200, 1); + const child = row(300, parent.pid); + const owner = new ProcessOwnershipEvidenceOwner(); + if (mode === "retained-parent") + owner.reconcile( + [ + { + process: parent, + state: "same-generation", + identityBefore: identity(parent, false), + token: { state: "readable", runId: "run" }, + identityAfter: identity(parent, false), + }, + ], + { + runId: "run", + listedRows: 1, + listedPids: new Set([parent.pid]), + leaderPid: 100, + processGroupIds: new Set([100]), + }, + ); + const priorSnapshots = owner.coverage.snapshotCount; + let parentReads = 0; + let childTokenReads = 0; + let reparented = false; + const host: ProcessOwnershipHost = { + listProcesses: async () => [parent, child], + environment: async (pid) => + pid === parent.pid ? { REA_PROCESS_RUN_ID: "run" } : {}, + runTokens: async (selected) => + new Map( + selected.map((process) => [ + process.pid, + process.pid === parent.pid + ? mode === "retained-parent" + ? { state: "unavailable", reason: "EACCES" } + : { state: "readable", runId: "run" } + : ++childTokenReads === 1 && mode !== "readable" + ? { state: "unavailable", reason: "EACCES" } + : { state: "readable", runId: undefined }, + ]), + ), + processIdentities: async (selected) => + new Map( + selected.map((process) => { + if (process.pid === parent.pid && ++parentReads > 2) { + reparented = true; + return [ + process.pid, + { state: "absent", reason: "not-in-snapshot" }, + ]; + } + const observed = + process.pid === child.pid && reparented + ? { ...process, parentPid: 1 } + : process; + const current = identity( + observed, + process.pid === parent.pid && parentReads === 2, + ); + return [ + process.pid, + process.pid === child.pid && + reparented && + mode === "reused" && + current.state === "readable" + ? { ...current, identity: "start:replacement" } + : current, + ]; + }), + ), + signalGroup: vi.fn(), + }; + const result = await verifyNoTokenOwnedProcesses("run", host, undefined, { + leaderPid: 100, + processGroupId: 100, + evidenceOwner: owner, + }); + if (mode === "reused") { + // A replacement process cannot inherit the original child's history. + expect(result.cleaned).toBe(true); + expect(owner.candidates).not.toContainEqual( + expect.objectContaining({ pid: child.pid }), + ); + } else { + // The original child stays live even though its parent has exited. + expect(result.cleaned).toBe(false); + expect(owner.candidates).toContainEqual( + expect.objectContaining({ + pid: child.pid, + identity: "start:300", + applicability: "related-ownership-unresolved", + }), + ); + } + expect(host.signalGroup).not.toHaveBeenCalled(); + expect(owner.coverage.snapshotCount).toBe(priorSnapshots + 1); + }, +); + +it("shares native exit waiting before signaling healthy groups in one cleanup attempt", async () => { + const root = row(100, 1); + const rows = [ + root, + row(200, root.pid), + row(300, root.pid), + row(400, root.pid), + ]; + let clock = 0; + let firstSignalAt: number | undefined; + const nativeClock = vi + .spyOn(performance, "now") + .mockImplementation(() => clock); + const host: ProcessOwnershipHost = { + listProcesses: async () => rows, + environment: async () => ({ REA_PROCESS_RUN_ID: "run" }), + processIdentities: async (selected) => { + clock += 25; + return new Map( + selected.map((process) => [ + process.pid, + identity(process, process.pid !== root.pid), + ]), + ); + }, + signalGroup: vi.fn((group) => { + expect(group).toBe(root.pid); + firstSignalAt ??= clock; + }), + }; + try { + const result = await cleanupOwnedProcessGroup( + { + runId: "run", + leaderPid: root.pid, + processGroupId: root.pid, + sweepTokenOwnedProcesses: true, + evidenceOwner: new ProcessOwnershipEvidenceOwner(), + }, + host, + ); + expect(result.cleaned).toBe(false); + expect(firstSignalAt).toBeDefined(); + // Controlled native-observation time, not a wall-clock timeout: three + // stuck helpers must not each add another full second before root signal. + expect(firstSignalAt).toBeLessThan(2_000); + } finally { + nativeClock.mockRestore(); + } +}); diff --git a/src/process/ProcessOwnership.ts b/src/process/ProcessOwnership.ts index 95a80b22dc..c8a74a83af 100644 --- a/src/process/ProcessOwnership.ts +++ b/src/process/ProcessOwnership.ts @@ -22,6 +22,7 @@ import { processIdentityHasExited, processIdentityIsExiting, type ProcessLifecycleObservation, + type ProcessExitSettlementBudget, } from "./ProcessLifecycle.js"; const execFileAsync = promisify(execFile); @@ -117,7 +118,7 @@ export type ProcessNativeRelationObservation = | { readonly state: "unavailable"; readonly reason: string }; /** One caller-supplied process row validated across identity/token/identity reads. */ -export interface ProcessOwnershipFact { +interface ProcessOwnershipFactSnapshot { /** Caller-supplied candidate row; its parent/group may predate this read. */ readonly process: ProcessTableEntry; /** Identity relation only; token ownership still requires matching readable runId. */ @@ -129,6 +130,11 @@ export interface ProcessOwnershipFact { readonly identityAfter: ProcessIdentityObservation; } +export interface ProcessOwnershipFact extends ProcessOwnershipFactSnapshot { + /** Bracketed ownership proof before a later native exit observation. */ + readonly beforeExitSettlement?: ProcessOwnershipFactSnapshot; +} + /** Token-verified process lineage retained for one owned provider run. */ export interface OwnedProcessLineage { readonly runId: string; @@ -327,6 +333,7 @@ export const cleanupOwnedProcessGroup = async ( ownership: OwnedProcessGroup, host: ProcessOwnershipHost = systemHost, ): Promise => { + const exitSettlement: ProcessExitSettlementBudget = { deadline: undefined }; const processTable = await readLiveProcessTable(host); if (!processTable.available) return { @@ -337,6 +344,7 @@ export const cleanupOwnedProcessGroup = async ( ownership, processTable.processes, host, + exitSettlement, ); if ("cleaned" in plan) return plan; let signaled = false; @@ -346,6 +354,7 @@ export const cleanupOwnedProcessGroup = async ( ownership, host, plan.tokenOwnedIdentities, + exitSettlement, ); if ("cleaned" in revalidation) return mergeUnverifiedObservations(revalidation, plan.unverified); @@ -386,6 +395,7 @@ export const cleanupOwnedProcessGroup = async ( host, ownership.captureBaseline, relation, + exitSettlement, ); while (!verification.cleaned) { const remaining = deadline - performance.now(); @@ -398,6 +408,7 @@ export const cleanupOwnedProcessGroup = async ( host, ownership.captureBaseline, relation, + exitSettlement, ); } } catch (cause: unknown) { @@ -429,6 +440,7 @@ export const verifyNoTokenOwnedProcesses = async ( host: ProcessOwnershipHost = systemHost, captureBaseline?: ProcessOwnershipBaseline, relation?: OwnershipSweepRelation, + exitSettlement: ProcessExitSettlementBudget = { deadline: undefined }, ): Promise => { const processTable = await readLiveProcessTable(host); if (!processTable.available) @@ -442,6 +454,7 @@ export const verifyNoTokenOwnedProcesses = async ( host, captureBaseline, relation, + exitSettlement, ); if (scan.failures.length > 0) return cleanupValidationFailure(scan.failures, scan.unverified); @@ -555,6 +568,7 @@ const settleLauncherExit = async ( ownership: OwnedProcessGroup, processes: readonly ProcessTableEntry[], host: ProcessOwnershipHost, + exitSettlement: ProcessExitSettlementBudget, ): Promise< | readonly ProcessTableEntry[] | Extract @@ -571,6 +585,8 @@ const settleLauncherExit = async ( [launcher], host, initialIdentities, + undefined, + exitSettlement, ); const identity = identities.get(launcher.pid); const initialIdentity = initialIdentities.get(launcher.pid); @@ -647,8 +663,14 @@ const createOwnedCleanupPlan = async ( ownership: OwnedProcessGroup, initialProcesses: readonly ProcessTableEntry[], host: ProcessOwnershipHost, + exitSettlement: ProcessExitSettlementBudget, ): Promise => { - const processes = await settleLauncherExit(ownership, initialProcesses, host); + const processes = await settleLauncherExit( + ownership, + initialProcesses, + host, + exitSettlement, + ); if ("cleaned" in processes) return processes; const relationSnapshot = await readProcessRelationSnapshot( ownership, @@ -676,6 +698,7 @@ const createOwnedCleanupPlan = async ( ? {} : { sampledProcessGroupIds: [...sampledProcessGroupIds] }), }, + exitSettlement, ) : { owned: [], failures: [], unverified: [], identities: new Map() }; const tokenOwnedGroupIds = new Set( @@ -764,6 +787,7 @@ const createOwnedCleanupPlan = async ( ownership.runId, host, tokenOwned.identities, + exitSettlement, ); if (failures.length === 0 && unverifiableSampledGroups.has(processGroupId)) unresolved.push( @@ -839,6 +863,7 @@ const revalidateOwnedProcessGroup = async ( ownership: OwnedProcessGroup, host: ProcessOwnershipHost, expectedIdentities: ReadonlyMap, + exitSettlement: ProcessExitSettlementBudget, ): Promise<{ readonly empty: boolean } | ProcessCleanupResult> => { let members: readonly ProcessTableEntry[]; try { @@ -846,6 +871,7 @@ const revalidateOwnedProcessGroup = async ( ownership, liveProcesses(await host.listProcesses()), host, + exitSettlement, ); if ("cleaned" in processes) return processes; const relationSnapshot = await readProcessRelationSnapshot( @@ -870,6 +896,7 @@ const revalidateOwnedProcessGroup = async ( ownership.runId, host, expectedIdentities, + exitSettlement, ); if (failures.length > 0) return cleanupValidationFailure(failures); if ( @@ -982,9 +1009,12 @@ const processOwnershipFailures = async ( runId: string, host: ProcessOwnershipHost, expectedIdentities: ReadonlyMap = new Map(), + exitSettlement: ProcessExitSettlementBudget = { deadline: undefined }, ): Promise => { if (host.processIdentities !== undefined) { - const facts = await readProcessOwnershipFacts(members, host); + const facts = await readProcessOwnershipFacts(members, host, undefined, { + exitSettlement, + }); const failures: ProcessOwnershipValidationFailure[] = []; for (const fact of facts) { if (processIdentityHasExited(fact.identityAfter)) continue; @@ -1173,6 +1203,7 @@ const scanTokenOwnedProcesses = async ( host: ProcessOwnershipHost, captureBaseline?: ProcessOwnershipBaseline, relation?: OwnershipSweepRelation, + exitSettlement: ProcessExitSettlementBudget = { deadline: undefined }, ): Promise => { if (host.processIdentities !== undefined) return scanWithOwnershipEvidence( @@ -1181,6 +1212,7 @@ const scanTokenOwnedProcesses = async ( host, captureBaseline, relation, + exitSettlement, ); const candidates = processes; @@ -1273,6 +1305,7 @@ const scanWithOwnershipEvidence = async ( host: ProcessOwnershipHost, captureBaseline: ProcessOwnershipBaseline | undefined, relation: OwnershipSweepRelation | undefined, + exitSettlement: ProcessExitSettlementBudget, ): Promise => { const evidenceOwner = relation === undefined @@ -1323,6 +1356,7 @@ const scanWithOwnershipEvidence = async ( undefined, { identityBefore: firstIdentities, + exitSettlement, }, ); let facts = initialFacts; @@ -1337,7 +1371,12 @@ const scanWithOwnershipEvidence = async ( .map(({ process }) => process); if (retryRows.length === 0) break; await delay(TOKEN_READ_RETRY_DELAY_MS); - const retries = await readProcessOwnershipFacts(retryRows, host); + const retries = await readProcessOwnershipFacts( + retryRows, + host, + undefined, + { exitSettlement }, + ); const retryByPid = new Map(retries.map((fact) => [fact.process.pid, fact])); facts = facts.map((fact) => retryByPid.get(fact.process.pid) ?? fact); } diff --git a/src/process/ProcessOwnershipEvidence.ts b/src/process/ProcessOwnershipEvidence.ts index fd93b78042..10752d6b3b 100644 --- a/src/process/ProcessOwnershipEvidence.ts +++ b/src/process/ProcessOwnershipEvidence.ts @@ -165,32 +165,50 @@ export class ProcessOwnershipEvidenceOwner { number, Extract >(); - const scopeTokenMatchedPids = new Set(); - for (const fact of options.scopeFacts ?? []) { - const current = sameGeneration.get(fact.process.pid); + const scopeOwnedPids = new Set(); + // Keep the original batch together: retries can reparent a child after its + // verified parent exits. Historical parent proof is scope evidence only; + // it neither establishes current liveness nor authorizes a signal. + for (const observed of options.scopeFacts ?? facts) { + const fact = observed.beforeExitSettlement ?? observed; if ( fact.state !== "same-generation" || - fact.identityAfter.state !== "readable" || - current?.identity !== fact.identityAfter.identity + fact.identityAfter.state !== "readable" ) continue; scopeGenerations.set(fact.process.pid, fact.identityAfter); if ( fact.identityBefore.state === "readable" && - current?.identity === fact.identityBefore.identity + fact.identityAfter.identity === fact.identityBefore.identity ) scopeBeforeGenerations.set(fact.process.pid, fact.identityBefore); - if (fact.token.state === "readable" && fact.token.runId === options.runId) - scopeTokenMatchedPids.add(fact.process.pid); + // Capture earlier proof before applying terminal observations below. + // Its generation must match this historical native observation exactly. + const previous = this.#candidates.get(fact.process.pid); + if ( + (fact.token.state === "readable" && + fact.token.runId === options.runId) || + (previous?.applicability === "token-verified" && + previous.identity === fact.identityAfter.identity) + ) + scopeOwnedPids.add(fact.process.pid); } const related = (pid: number) => { const current = findScopeEvidence(pid, sameGeneration, tokenMatchedPids) ?? findScopeEvidence(pid, sameGenerationBefore, tokenMatchedPids); if (current !== undefined) return current; + // The child must still be the generation observed in that original + // batch. A reused PID cannot inherit its predecessor's historical scope. + const original = scopeGenerations.get(pid); + if ( + original === undefined || + sameGeneration.get(pid)?.identity !== original.identity + ) + return undefined; const initial = - findScopeEvidence(pid, scopeGenerations, scopeTokenMatchedPids) ?? - findScopeEvidence(pid, scopeBeforeGenerations, scopeTokenMatchedPids); + findScopeEvidence(pid, scopeGenerations, scopeOwnedPids) ?? + findScopeEvidence(pid, scopeBeforeGenerations, scopeOwnedPids); return initial === undefined ? undefined : "previously-related-generation"; diff --git a/src/process/ProcessOwnershipObservation.ts b/src/process/ProcessOwnershipObservation.ts index ec1ae9ddb6..fd287a031f 100644 --- a/src/process/ProcessOwnershipObservation.ts +++ b/src/process/ProcessOwnershipObservation.ts @@ -6,6 +6,7 @@ import { processIdentityIsExiting, PROCESS_CLEANUP_VERIFICATION_GRACE_MS, PROCESS_CLEANUP_VERIFICATION_INTERVAL_MS, + type ProcessExitSettlementBudget, } from "./ProcessLifecycle.js"; import { parseLinuxProcessStat } from "./LinuxProcessStat.js"; import { parseLinuxProcessEffectiveUid } from "./LinuxProcessStatus.js"; @@ -539,6 +540,7 @@ export const settleExitingProcessIdentities = async ( host: ProcessOwnershipHost, initial: ReadonlyMap, signal?: AbortSignal, + budget: ProcessExitSettlementBudget = { deadline: undefined }, ): Promise> => { signal?.throwIfAborted(); let pending = processes.filter(({ pid }) => @@ -546,7 +548,8 @@ export const settleExitingProcessIdentities = async ( ); if (pending.length === 0) return initial; const identities = new Map(initial); - const deadline = performance.now() + PROCESS_CLEANUP_VERIFICATION_GRACE_MS; + const deadline = (budget.deadline ??= + performance.now() + PROCESS_CLEANUP_VERIFICATION_GRACE_MS); while (pending.length > 0) { const remaining = deadline - performance.now(); if (remaining <= 0) break; @@ -589,6 +592,7 @@ export const readProcessOwnershipFacts = async ( signal?: AbortSignal, options: { readonly identityBefore?: ReadonlyMap; + readonly exitSettlement?: ProcessExitSettlementBudget; } = {}, ): Promise => { signal?.throwIfAborted(); @@ -627,19 +631,30 @@ export const readProcessOwnershipFacts = async ( .filter(({ pid }) => tokens.get(pid)?.state !== "readable") .map(({ pid }) => pid), ); + const observedAfter = await readIdentityFacts( + processes, + host, + "after", + signal, + effectiveUidPids, + ); const identityAfter = await settleExitingProcessIdentities( processes, host, - await readIdentityFacts(processes, host, "after", signal, effectiveUidPids), + observedAfter, signal, + options.exitSettlement, ); signal?.throwIfAborted(); - const facts = processes.map((process): ProcessOwnershipFact => { + const deriveFact = ( + process: ProcessTableEntry, + observation: ProcessIdentityObservation | undefined, + ): ProcessOwnershipFact => { const before = identityBefore.get(process.pid) ?? { state: "unavailable" as const, reason: "process identity was not returned during before observation", }; - const after = identityAfter.get(process.pid) ?? { + const after = observation ?? { state: "unavailable" as const, reason: "process identity was not returned during after observation", }; @@ -664,6 +679,14 @@ export const readProcessOwnershipFacts = async ( token, identityAfter: after, }; + }; + const facts = processes.map((process): ProcessOwnershipFact => { + const after = identityAfter.get(process.pid); + const original = observedAfter.get(process.pid); + const fact = deriveFact(process, after); + return after === original + ? fact + : { ...fact, beforeExitSettlement: deriveFact(process, original) }; }); signal?.throwIfAborted(); return facts;