import { describe, expect, it } from "vitest"; import { recordWorkflowUnknowns, createWorkflowEvidence, executeComposedWorkflow, workflowSnapshotRecord, } from "./WorkflowEvidence.js"; import { InvestigationRecords } from "./investigation/InvestigationRecords.js"; import { UnknownRegistryError } from "../domain/unknownRegistryError.js"; import { err, ok } from "../domain/result.js"; import { createAnalysisExecution, type AnalysisOperationPort, } from "./AnalysisProvider.js"; import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; import { nativeApiBoundarySchema } from "../domain/native/nativeApiBoundary.js"; import { projectNativeApiInspection } from "./native/NativeApiInspection.js"; import { functionDossierSchema } from "../domain/hopperValues.js"; import { ghidraFunctionDossier } from "../domain/ghidraValues.fixture.js"; import { ANALYSIS_SNAPSHOT_PROFILE } from "../domain/analysisSnapshot.fixture.js"; it.each([false, true])( "propagates transient=%s through composed snapshot decisions while retaining Evidence", async (transient) => { let calls = 0; const composed = await executeComposedWorkflow( { execute: async () => ok( createAnalysisExecution( "observed", provider, transient && calls++ === 0 ? { cachePolicy: "live" as const } : {}, ), ), }, async (observed) => { await observed.execute("analyze_function", { procedure: "0x1000" }); const last = await observed.execute("procedure_info", { procedure: "0x1000", }); return last.ok ? ok(last.value.result) : last; }, ); if (!composed.ok) throw composed.error; const evidence = createWorkflowEvidence({ target: undefined, operation: "inspect_native_api", parameters: {}, result: composed.value.result, upstreamProfile: ANALYSIS_SNAPSHOT_PROFILE, upstreamLimitations: composed.value.limitations, }); expect(new InvestigationRecords().recordEvidence(evidence).ok).toBe(true); expect( workflowSnapshotRecord( evidence, "inspect_native_api", ANALYSIS_SNAPSHOT_PROFILE, composed.value.cachePolicy, ) === undefined, ).toBe(transient); expect(evidence.normalized_result).toBe("observed"); }, ); const provider = { id: "fixture", name: "Fixture", version: "1" }; it("keeps Evidence identity stable when dependent branches finish in opposite orders", async () => { const run = async (order: readonly number[]) => { const gate = () => { let resolve: () => void = () => { throw new Error("gate not initialized"); }; const promise = new Promise((release) => { resolve = release; }); return { promise, resolve }; }; const parents = [gate(), gate()]; const children = [gate(), gate()]; const analysis: AnalysisOperationPort = { execute: async (operation, parameters) => { const index = Number(parameters.index); if (operation === "procedure_address") { await parents[index]!.promise; return ok(createAnalysisExecution(String(index), provider)); } children[index]!.resolve(); return ok( createAnalysisExecution(String(index), provider, { limitations: [ index === 0 ? "first child caveat" : "second child caveat", ], }), ); }, }; const composed = await executeComposedWorkflow( analysis, async (observed) => { const branches = Promise.all( [0, 1].map(async (index) => { await observed.execute("procedure_address", { index }); const child = await observed.execute("address_name", { index }); if (!child.ok) throw child.error; return child.value.result; }), ); for (const index of order) { parents[index]!.resolve(); await children[index]!.promise; } return ok(await branches); }, ); if (!composed.ok) throw composed.error; return createWorkflowEvidence({ target: undefined, operation: "batch_decompile", parameters: {}, result: composed.value.result, upstreamProfile: undefined, upstreamLimitations: composed.value.limitations, }); }; const left = await run([0, 1]); const right = await run([1, 0]); expect(left.normalized_result).toEqual(right.normalized_result); expect(left.limitations).toEqual(right.limitations); expect(left.evidence_id).toBe(right.evidence_id); }); it("retains unique upstream limits in canonical order across concurrent observations", async () => { let release: () => void = () => { throw new Error("gate not initialized"); }; const gate = new Promise((resolve) => { release = resolve; }); let calls = 0; const options = { signal: new AbortController().signal }; const analysis: AnalysisOperationPort = { execute: async (operation, parameters, forwarded) => { expect(parameters).toEqual({ marker: true }); expect(forwarded).toBe(options); const call = calls++; if (call === 0) await gate; else release(); const limitation = operation === "list_strings" ? ["first", "shared"] : ["second", "shared"]; return ok( createAnalysisExecution([], provider, { limitations: limitation }), ); }, }; const result = await executeComposedWorkflow(analysis, async (observed) => { await Promise.all([ observed.execute("list_strings", { marker: true }, options), observed.execute("list_procedures", { marker: true }, options), ]); return ok({ count: 0 }); }); expect(calls).toBe(2); expect(result).toEqual( ok({ result: { count: 0 }, limitations: [ "Fixture (fixture 1) list_procedures, list_strings: shared", "Fixture (fixture 1) list_procedures: second", "Fixture (fixture 1) list_strings: first", ], }), ); }); it("deduplicates exact source caveats without merging providers or operations", async () => { const firstProvider = { id: "first", name: "First", version: "1" }; const secondProvider = { id: "second", name: "Second", version: "1" }; const result = await executeComposedWorkflow( { execute: async (operation, parameters) => ok( createAnalysisExecution( null, parameters.provider === "second" ? secondProvider : firstProvider, { limitations: ["same source caveat"] }, ), ), }, async (observed) => { await observed.execute("list_strings", { provider: "first" }); await observed.execute("list_strings", { provider: "first" }); await observed.execute("list_strings", { provider: "second" }); await observed.execute("list_procedures", { provider: "first" }); return ok({ count: 0 }); }, ); expect(result).toEqual( ok({ result: { count: 0 }, limitations: [ "First (first 1) list_procedures, list_strings: same source caveat", "Second (second 1) list_strings: same source caveat", ], }), ); }); it("preserves a failed composed operation without an extra provider read", async () => { const failure = err(new AnalysisCancelledError("list_strings")); let calls = 0; const result = await executeComposedWorkflow( { execute: async () => { calls++; return failure; }, }, async (observed) => { const observedResult = await observed.execute("list_strings", {}); return observedResult.ok ? ok(observedResult.value.result) : observedResult; }, ); expect(result).toBe(failure); expect(calls).toBe(1); }); describe("workflow residual question retention", () => { it("keeps questions linked to workflow Evidence and records repeats idempotently", () => { const records = new InvestigationRecords(); const result = { residual_unknowns: ["Which provider resolves this boundary?"], }; const evidence = createWorkflowEvidence({ target: undefined, operation: "inspect_native_api", parameters: { procedure: "0x1000" }, result, upstreamProfile: undefined, upstreamLimitations: [], }); expect(records.recordEvidence(evidence).ok).toBe(true); const input = { name: "inspect_native_api", result, evidenceId: evidence.evidence_id, recordUnknown: ( unknown: Parameters[0], ) => records.recordUnknown(unknown), }; expect(recordWorkflowUnknowns(input).ok).toBe(true); expect(recordWorkflowUnknowns(input).ok).toBe(true); expect(records.listUnknowns()).toMatchObject([ { question: `${result.residual_unknowns[0]} (Evidence ${evidence.evidence_id})`, domain: "native-api", supporting_evidence_ids: [evidence.evidence_id], required_authority: "shipped-artifact", }, ]); }); it("retains one identical residual question per distinct workflow Evidence", () => { const records = new InvestigationRecords(); const result = { residual_unknowns: ["Which provider resolves this boundary?"], }; const evidence = ["0x1000", "0x2000"].map((procedure) => createWorkflowEvidence({ target: undefined, operation: "inspect_native_api", parameters: { procedure }, result, upstreamProfile: undefined, upstreamLimitations: [], }), ); expect(evidence[0]?.evidence_id).not.toBe(evidence[1]?.evidence_id); for (const item of evidence) { expect(records.recordEvidence(item).ok).toBe(true); const input = { name: "inspect_native_api", result, evidenceId: item.evidence_id, recordUnknown: ( unknown: Parameters[0], ) => records.recordUnknown(unknown), }; expect(recordWorkflowUnknowns(input).ok).toBe(true); expect(recordWorkflowUnknowns(input).ok).toBe(true); } expect(records.listUnknowns()).toHaveLength(2); expect( records .listUnknowns() .map(({ supporting_evidence_ids }) => supporting_evidence_ids), ).toEqual( expect.arrayContaining(evidence.map(({ evidence_id }) => [evidence_id])), ); }); it.each([ ["spaces", " Which boundary? "], ["line feed and tab", "\nWhich boundary?\t"], ["byte order mark", "\uFEFFWhich boundary?\uFEFF"], ])( "normalizes edge whitespace from %s without changing workflow Evidence", (_, question) => { const records = new InvestigationRecords(); const result = { residual_unknowns: [question] }; const evidence = createWorkflowEvidence({ target: undefined, operation: "inspect_native_api", parameters: {}, result, upstreamProfile: undefined, upstreamLimitations: [], }); expect(records.recordEvidence(evidence).ok).toBe(true); expect( recordWorkflowUnknowns({ name: "inspect_native_api", result, evidenceId: evidence.evidence_id, recordUnknown: (unknown) => records.recordUnknown(unknown), }).ok, ).toBe(true); expect(records.listUnknowns()).toMatchObject([ { question: `Which boundary? (Evidence ${evidence.evidence_id})` }, ]); expect(evidence.normalized_result).toEqual(result); }, ); it("rejects questions that are empty after whitespace normalization", () => { const records = new InvestigationRecords(); const result = { residual_unknowns: [" \n\t "] }; const evidence = createWorkflowEvidence({ target: undefined, operation: "inspect_native_api", parameters: {}, result, upstreamProfile: undefined, upstreamLimitations: [], }); expect(records.recordEvidence(evidence).ok).toBe(true); expect( recordWorkflowUnknowns({ name: "inspect_native_api", result, evidenceId: evidence.evidence_id, recordUnknown: (unknown) => records.recordUnknown(unknown), }).ok, ).toBe(false); expect(records.listUnknowns()).toEqual([]); expect(evidence.normalized_result).toEqual(result); }); it("preserves registry failures rather than presenting a completed workflow", () => { const error = new UnknownRegistryError("limit"); expect( recordWorkflowUnknowns({ name: "trace_feature", result: { residual_unknowns: ["Which branch handles the request?"] }, evidenceId: "unretained", recordUnknown: () => err(error), }), ).toEqual(err(error)); }); }); it.each([ " Which boundary? ", "\nWhich boundary?\t", "\uFEFFWhich boundary?\uFEFF", ])( "retains contract-accepted question %j without rewriting workflow Evidence", (question) => { const boundary = nativeApiBoundarySchema.parse({ available: false, reason: "Synthetic provider unavailable boundary", residual_unknowns: [question], }); const dossier = functionDossierSchema.parse(ghidraFunctionDossier()); const result = projectNativeApiInspection({ ...dossier, native_api: boundary, }); const evidence = createWorkflowEvidence({ target: undefined, operation: "inspect_native_api", parameters: { procedure: "0x1000" }, result, upstreamProfile: undefined, upstreamLimitations: [], }); const records = new InvestigationRecords(); expect(records.recordEvidence(evidence).ok).toBe(true); expect( recordWorkflowUnknowns({ name: "inspect_native_api", result, evidenceId: evidence.evidence_id, recordUnknown: (input) => records.recordUnknown(input), }).ok, ).toBe(true); expect(records.listUnknowns()).toMatchObject([ { question: `Which boundary? (Evidence ${evidence.evidence_id})` }, ]); expect(records.evidenceById(evidence.evidence_id)).toEqual(evidence); expect(evidence.normalized_result).toMatchObject({ residual_unknowns: [question], }); }, );