morluto
0ba4d6f43a
Retain native-only dispatch workflows without requiring a deep provider profile. Validate exact target, operation, and current reader bindings during open, import, recording, and export; preserve selected deep bindings and caller limits. Validated public CLI and stdio MCP save/reopen, exact CLI replay, changed-limit cache partitioning, baseline failure and candidate success on an Apple-clang fixture, snapshot lifecycle regressions, typecheck and lint. No target or deep engine execution. Fixes #5529.
429 lines
14 KiB
TypeScript
429 lines
14 KiB
TypeScript
import { describe, expect, it } from "vitest";
|
|
|
|
import {
|
|
recordWorkflowUnknowns,
|
|
createWorkflowEvidence,
|
|
executeComposedWorkflow,
|
|
workflowSnapshotRecord,
|
|
} from "./WorkflowEvidence.js";
|
|
import { InvestigationRecords } from "./investigation/InvestigationRecords.js";
|
|
import { UnknownRegistryError } from "../domain/unknownRegistryError.js";
|
|
import { err, ok } from "../domain/result.js";
|
|
import {
|
|
createAnalysisExecution,
|
|
type AnalysisOperationPort,
|
|
} from "./AnalysisProvider.js";
|
|
import { AnalysisCancelledError } from "../domain/analysisErrorCore.js";
|
|
import { nativeApiBoundarySchema } from "../domain/native/nativeApiBoundary.js";
|
|
import { projectNativeApiInspection } from "./native/NativeApiInspection.js";
|
|
import { functionDossierSchema } from "../domain/hopperValues.js";
|
|
import { ghidraFunctionDossier } from "../domain/ghidraValues.fixture.js";
|
|
import { ANALYSIS_SNAPSHOT_PROFILE } from "../domain/analysisSnapshot.fixture.js";
|
|
|
|
it.each([false, true])(
|
|
"propagates transient=%s through composed snapshot decisions while retaining Evidence",
|
|
async (transient) => {
|
|
let calls = 0;
|
|
const composed = await executeComposedWorkflow(
|
|
{
|
|
execute: async () =>
|
|
ok(
|
|
createAnalysisExecution(
|
|
"observed",
|
|
provider,
|
|
transient && calls++ === 0
|
|
? { cachePolicy: "live" as const }
|
|
: {},
|
|
),
|
|
),
|
|
},
|
|
async (observed) => {
|
|
await observed.execute("analyze_function", { procedure: "0x1000" });
|
|
const last = await observed.execute("procedure_info", {
|
|
procedure: "0x1000",
|
|
});
|
|
return last.ok ? ok(last.value.result) : last;
|
|
},
|
|
);
|
|
if (!composed.ok) throw composed.error;
|
|
const evidence = createWorkflowEvidence({
|
|
target: undefined,
|
|
operation: "inspect_native_api",
|
|
parameters: {},
|
|
result: composed.value.result,
|
|
upstreamProfile: ANALYSIS_SNAPSHOT_PROFILE,
|
|
upstreamLimitations: composed.value.limitations,
|
|
});
|
|
expect(new InvestigationRecords().recordEvidence(evidence).ok).toBe(true);
|
|
expect(
|
|
workflowSnapshotRecord(
|
|
evidence,
|
|
"inspect_native_api",
|
|
ANALYSIS_SNAPSHOT_PROFILE,
|
|
composed.value.cachePolicy,
|
|
) === undefined,
|
|
).toBe(transient);
|
|
expect(evidence.normalized_result).toBe("observed");
|
|
},
|
|
);
|
|
|
|
const provider = { id: "fixture", name: "Fixture", version: "1" };
|
|
|
|
it("keeps Evidence identity stable when dependent branches finish in opposite orders", async () => {
|
|
const run = async (order: readonly number[]) => {
|
|
const gate = () => {
|
|
let resolve: () => void = () => {
|
|
throw new Error("gate not initialized");
|
|
};
|
|
const promise = new Promise<void>((release) => {
|
|
resolve = release;
|
|
});
|
|
return { promise, resolve };
|
|
};
|
|
const parents = [gate(), gate()];
|
|
const children = [gate(), gate()];
|
|
const analysis: AnalysisOperationPort = {
|
|
execute: async (operation, parameters) => {
|
|
const index = Number(parameters.index);
|
|
if (operation === "procedure_address") {
|
|
await parents[index]!.promise;
|
|
return ok(createAnalysisExecution(String(index), provider));
|
|
}
|
|
children[index]!.resolve();
|
|
return ok(
|
|
createAnalysisExecution(String(index), provider, {
|
|
limitations: [
|
|
index === 0 ? "first child caveat" : "second child caveat",
|
|
],
|
|
}),
|
|
);
|
|
},
|
|
};
|
|
const composed = await executeComposedWorkflow(
|
|
analysis,
|
|
async (observed) => {
|
|
const branches = Promise.all(
|
|
[0, 1].map(async (index) => {
|
|
await observed.execute("procedure_address", { index });
|
|
const child = await observed.execute("address_name", { index });
|
|
if (!child.ok) throw child.error;
|
|
return child.value.result;
|
|
}),
|
|
);
|
|
for (const index of order) {
|
|
parents[index]!.resolve();
|
|
await children[index]!.promise;
|
|
}
|
|
return ok(await branches);
|
|
},
|
|
);
|
|
if (!composed.ok) throw composed.error;
|
|
return createWorkflowEvidence({
|
|
target: undefined,
|
|
operation: "batch_decompile",
|
|
parameters: {},
|
|
result: composed.value.result,
|
|
upstreamProfile: undefined,
|
|
upstreamLimitations: composed.value.limitations,
|
|
});
|
|
};
|
|
const left = await run([0, 1]);
|
|
const right = await run([1, 0]);
|
|
expect(left.normalized_result).toEqual(right.normalized_result);
|
|
expect(left.limitations).toEqual(right.limitations);
|
|
expect(left.evidence_id).toBe(right.evidence_id);
|
|
});
|
|
|
|
it("retains unique upstream limits in canonical order across concurrent observations", async () => {
|
|
let release: () => void = () => {
|
|
throw new Error("gate not initialized");
|
|
};
|
|
const gate = new Promise<void>((resolve) => {
|
|
release = resolve;
|
|
});
|
|
let calls = 0;
|
|
const options = { signal: new AbortController().signal };
|
|
const analysis: AnalysisOperationPort = {
|
|
execute: async (operation, parameters, forwarded) => {
|
|
expect(parameters).toEqual({ marker: true });
|
|
expect(forwarded).toBe(options);
|
|
const call = calls++;
|
|
if (call === 0) await gate;
|
|
else release();
|
|
const limitation =
|
|
operation === "list_strings"
|
|
? ["first", "shared"]
|
|
: ["second", "shared"];
|
|
return ok(
|
|
createAnalysisExecution([], provider, { limitations: limitation }),
|
|
);
|
|
},
|
|
};
|
|
const result = await executeComposedWorkflow(analysis, async (observed) => {
|
|
await Promise.all([
|
|
observed.execute("list_strings", { marker: true }, options),
|
|
observed.execute("list_procedures", { marker: true }, options),
|
|
]);
|
|
return ok({ count: 0 });
|
|
});
|
|
expect(calls).toBe(2);
|
|
expect(result).toEqual(
|
|
ok({
|
|
result: { count: 0 },
|
|
limitations: [
|
|
"Fixture (fixture 1) list_procedures, list_strings: shared",
|
|
"Fixture (fixture 1) list_procedures: second",
|
|
"Fixture (fixture 1) list_strings: first",
|
|
],
|
|
}),
|
|
);
|
|
});
|
|
|
|
it("deduplicates exact source caveats without merging providers or operations", async () => {
|
|
const firstProvider = { id: "first", name: "First", version: "1" };
|
|
const secondProvider = { id: "second", name: "Second", version: "1" };
|
|
const result = await executeComposedWorkflow(
|
|
{
|
|
execute: async (operation, parameters) =>
|
|
ok(
|
|
createAnalysisExecution(
|
|
null,
|
|
parameters.provider === "second" ? secondProvider : firstProvider,
|
|
{ limitations: ["same source caveat"] },
|
|
),
|
|
),
|
|
},
|
|
async (observed) => {
|
|
await observed.execute("list_strings", { provider: "first" });
|
|
await observed.execute("list_strings", { provider: "first" });
|
|
await observed.execute("list_strings", { provider: "second" });
|
|
await observed.execute("list_procedures", { provider: "first" });
|
|
return ok({ count: 0 });
|
|
},
|
|
);
|
|
|
|
expect(result).toEqual(
|
|
ok({
|
|
result: { count: 0 },
|
|
limitations: [
|
|
"First (first 1) list_procedures, list_strings: same source caveat",
|
|
"Second (second 1) list_strings: same source caveat",
|
|
],
|
|
}),
|
|
);
|
|
});
|
|
|
|
it("preserves a failed composed operation without an extra provider read", async () => {
|
|
const failure = err(new AnalysisCancelledError("list_strings"));
|
|
let calls = 0;
|
|
const result = await executeComposedWorkflow(
|
|
{
|
|
execute: async () => {
|
|
calls++;
|
|
return failure;
|
|
},
|
|
},
|
|
async (observed) => {
|
|
const observedResult = await observed.execute("list_strings", {});
|
|
return observedResult.ok
|
|
? ok(observedResult.value.result)
|
|
: observedResult;
|
|
},
|
|
);
|
|
expect(result).toBe(failure);
|
|
expect(calls).toBe(1);
|
|
});
|
|
|
|
describe("workflow residual question retention", () => {
|
|
it("keeps questions linked to workflow Evidence and records repeats idempotently", () => {
|
|
const records = new InvestigationRecords();
|
|
const result = {
|
|
residual_unknowns: ["Which provider resolves this boundary?"],
|
|
};
|
|
const evidence = createWorkflowEvidence({
|
|
target: undefined,
|
|
operation: "inspect_native_api",
|
|
parameters: { procedure: "0x1000" },
|
|
result,
|
|
upstreamProfile: undefined,
|
|
upstreamLimitations: [],
|
|
});
|
|
expect(records.recordEvidence(evidence).ok).toBe(true);
|
|
const input = {
|
|
name: "inspect_native_api",
|
|
result,
|
|
evidenceId: evidence.evidence_id,
|
|
recordUnknown: (
|
|
unknown: Parameters<InvestigationRecords["recordUnknown"]>[0],
|
|
) => records.recordUnknown(unknown),
|
|
};
|
|
expect(recordWorkflowUnknowns(input).ok).toBe(true);
|
|
expect(recordWorkflowUnknowns(input).ok).toBe(true);
|
|
expect(records.listUnknowns()).toMatchObject([
|
|
{
|
|
question: `${result.residual_unknowns[0]} (Evidence ${evidence.evidence_id})`,
|
|
domain: "native-api",
|
|
supporting_evidence_ids: [evidence.evidence_id],
|
|
required_authority: "shipped-artifact",
|
|
},
|
|
]);
|
|
});
|
|
|
|
it("retains one identical residual question per distinct workflow Evidence", () => {
|
|
const records = new InvestigationRecords();
|
|
const result = {
|
|
residual_unknowns: ["Which provider resolves this boundary?"],
|
|
};
|
|
const evidence = ["0x1000", "0x2000"].map((procedure) =>
|
|
createWorkflowEvidence({
|
|
target: undefined,
|
|
operation: "inspect_native_api",
|
|
parameters: { procedure },
|
|
result,
|
|
upstreamProfile: undefined,
|
|
upstreamLimitations: [],
|
|
}),
|
|
);
|
|
expect(evidence[0]?.evidence_id).not.toBe(evidence[1]?.evidence_id);
|
|
|
|
for (const item of evidence) {
|
|
expect(records.recordEvidence(item).ok).toBe(true);
|
|
const input = {
|
|
name: "inspect_native_api",
|
|
result,
|
|
evidenceId: item.evidence_id,
|
|
recordUnknown: (
|
|
unknown: Parameters<InvestigationRecords["recordUnknown"]>[0],
|
|
) => records.recordUnknown(unknown),
|
|
};
|
|
expect(recordWorkflowUnknowns(input).ok).toBe(true);
|
|
expect(recordWorkflowUnknowns(input).ok).toBe(true);
|
|
}
|
|
|
|
expect(records.listUnknowns()).toHaveLength(2);
|
|
expect(
|
|
records
|
|
.listUnknowns()
|
|
.map(({ supporting_evidence_ids }) => supporting_evidence_ids),
|
|
).toEqual(
|
|
expect.arrayContaining(evidence.map(({ evidence_id }) => [evidence_id])),
|
|
);
|
|
});
|
|
|
|
it.each([
|
|
["spaces", " Which boundary? "],
|
|
["line feed and tab", "\nWhich boundary?\t"],
|
|
["byte order mark", "\uFEFFWhich boundary?\uFEFF"],
|
|
])(
|
|
"normalizes edge whitespace from %s without changing workflow Evidence",
|
|
(_, question) => {
|
|
const records = new InvestigationRecords();
|
|
const result = { residual_unknowns: [question] };
|
|
const evidence = createWorkflowEvidence({
|
|
target: undefined,
|
|
operation: "inspect_native_api",
|
|
parameters: {},
|
|
result,
|
|
upstreamProfile: undefined,
|
|
upstreamLimitations: [],
|
|
});
|
|
|
|
expect(records.recordEvidence(evidence).ok).toBe(true);
|
|
expect(
|
|
recordWorkflowUnknowns({
|
|
name: "inspect_native_api",
|
|
result,
|
|
evidenceId: evidence.evidence_id,
|
|
recordUnknown: (unknown) => records.recordUnknown(unknown),
|
|
}).ok,
|
|
).toBe(true);
|
|
expect(records.listUnknowns()).toMatchObject([
|
|
{ question: `Which boundary? (Evidence ${evidence.evidence_id})` },
|
|
]);
|
|
expect(evidence.normalized_result).toEqual(result);
|
|
},
|
|
);
|
|
|
|
it("rejects questions that are empty after whitespace normalization", () => {
|
|
const records = new InvestigationRecords();
|
|
const result = { residual_unknowns: [" \n\t "] };
|
|
const evidence = createWorkflowEvidence({
|
|
target: undefined,
|
|
operation: "inspect_native_api",
|
|
parameters: {},
|
|
result,
|
|
upstreamProfile: undefined,
|
|
upstreamLimitations: [],
|
|
});
|
|
|
|
expect(records.recordEvidence(evidence).ok).toBe(true);
|
|
expect(
|
|
recordWorkflowUnknowns({
|
|
name: "inspect_native_api",
|
|
result,
|
|
evidenceId: evidence.evidence_id,
|
|
recordUnknown: (unknown) => records.recordUnknown(unknown),
|
|
}).ok,
|
|
).toBe(false);
|
|
expect(records.listUnknowns()).toEqual([]);
|
|
expect(evidence.normalized_result).toEqual(result);
|
|
});
|
|
|
|
it("preserves registry failures rather than presenting a completed workflow", () => {
|
|
const error = new UnknownRegistryError("limit");
|
|
expect(
|
|
recordWorkflowUnknowns({
|
|
name: "trace_feature",
|
|
result: { residual_unknowns: ["Which branch handles the request?"] },
|
|
evidenceId: "unretained",
|
|
recordUnknown: () => err(error),
|
|
}),
|
|
).toEqual(err(error));
|
|
});
|
|
});
|
|
|
|
it.each([
|
|
" Which boundary? ",
|
|
"\nWhich boundary?\t",
|
|
"\uFEFFWhich boundary?\uFEFF",
|
|
])(
|
|
"retains contract-accepted question %j without rewriting workflow Evidence",
|
|
(question) => {
|
|
const boundary = nativeApiBoundarySchema.parse({
|
|
available: false,
|
|
reason: "Synthetic provider unavailable boundary",
|
|
residual_unknowns: [question],
|
|
});
|
|
const dossier = functionDossierSchema.parse(ghidraFunctionDossier());
|
|
const result = projectNativeApiInspection({
|
|
...dossier,
|
|
native_api: boundary,
|
|
});
|
|
const evidence = createWorkflowEvidence({
|
|
target: undefined,
|
|
operation: "inspect_native_api",
|
|
parameters: { procedure: "0x1000" },
|
|
result,
|
|
upstreamProfile: undefined,
|
|
upstreamLimitations: [],
|
|
});
|
|
const records = new InvestigationRecords();
|
|
expect(records.recordEvidence(evidence).ok).toBe(true);
|
|
expect(
|
|
recordWorkflowUnknowns({
|
|
name: "inspect_native_api",
|
|
result,
|
|
evidenceId: evidence.evidence_id,
|
|
recordUnknown: (input) => records.recordUnknown(input),
|
|
}).ok,
|
|
).toBe(true);
|
|
expect(records.listUnknowns()).toMatchObject([
|
|
{ question: `Which boundary? (Evidence ${evidence.evidence_id})` },
|
|
]);
|
|
expect(records.evidenceById(evidence.evidence_id)).toEqual(evidence);
|
|
expect(evidence.normalized_result).toMatchObject({
|
|
residual_unknowns: [question],
|
|
});
|
|
},
|
|
);
|