rea/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.ts

import { parseMcpToolError } from "../../fixtures/mcpToolError.js";
import { writeFile } from "node:fs/promises";
import { join } from "node:path";

import { Client, InMemoryTransport } from "@modelcontextprotocol/client";
import { expect, it, onTestFinished } from "vitest";

import { compareProcessEvidenceFiles } from "../../../src/application/process/ProcessCli.js";
import { PROCESS_PROVIDER } from "../../../src/domain/process/processEvidenceProvider.js";
import {
  EMPTY_PROCESS_CAPTURE_EXAMPLE,
  FINALIZED_PROCESS_CAPTURE_EXAMPLE,
} from "../../../src/domain/process/processCaptureExample.js";
import {
  capture,
  partialSpecification,
  processStarted,
  terminal,
} from "../../../src/domain/process/processTraceComparison.fixture.js";
import {
  parseProcessCapture,
  type ProcessCapture,
} from "../../../src/domain/process/processCaptureParsing.js";
import type { ProcessTraceSpecification } from "../../../src/domain/process/processTraceSpecification.js";
import {
  createEvidence,
  parseEvidence,
  type EvidenceProvider,
} from "../../../src/domain/evidence.js";
import { jsonValueSchema } from "../../../src/domain/jsonValue.js";
import { digestProcessCommitment } from "../../../src/domain/process/processScenario.js";
import { createServer } from "../../../src/server/createServer.js";
import { createTestBinarySession } from "../../fixtures/binarySession.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";

const captureEvidence = (
  side: "left" | "right",
  comparisonContract: Readonly<Record<string, string>> = {},
  provider: EvidenceProvider = PROCESS_PROVIDER,
  observation: ProcessCapture = parseProcessCapture(
    EMPTY_PROCESS_CAPTURE_EXAMPLE,
  ),
) =>
  createEvidence(undefined, provider, {
    predicateType: "rea.process-capture",
    operation: "capture_process_scenario",
    parameters: { side },
    result: jsonValueSchema.parse({
      ...observation,
      manifest: {
        ...observation.manifest,
        comparison_contract: comparisonContract,
        comparison_contract_sha256: digestProcessCommitment(comparisonContract),
      },
    }),
    confidence: "observed",
    authority: "controlled-replay",
  });

const left = captureEvidence("left", { working_directory: "/work/reference" });
const right = captureEvidence("right", {
  working_directory: "/work/reconstruction",
});

const connect = async (
  session = createTestBinarySession(() => {
    throw new Error("Process comparison must not launch a provider");
  }),
) => {
  const server = createServer({ kind: "session", session });
  const client = new Client({ name: "process-comparison-test", version: "1" });
  onTestFinished(async () => {
    await Promise.allSettled([client.close(), server.close(), session.close()]);
  });
  const [clientTransport, serverTransport] =
    InMemoryTransport.createLinkedPair();
  await server.connect(serverTransport);
  await client.connect(clientTransport);
  return client;
};

const compareViaCli = async (
  left: ReturnType<typeof captureEvidence>,
  right: ReturnType<typeof captureEvidence>,
  specification?: ProcessTraceSpecification,
) => {
  const root = await createTestTempDirectory("rea-process-provider-");
  const leftPath = join(root, "left.json");
  const rightPath = join(root, "right.json");
  await Promise.all([
    writeFile(leftPath, JSON.stringify(left)),
    writeFile(rightPath, JSON.stringify(right)),
  ]);
  const tracePath = join(root, "trace.json");
  if (specification !== undefined)
    await writeFile(tracePath, JSON.stringify(specification));
  return compareProcessEvidenceFiles(
    leftPath,
    rightPath,
    specification === undefined ? undefined : tracePath,
  );
};

it.each(
  (["left", "right"] as const).flatMap((missingWorker) =>
    (["left", "right", "both", "neither"] as const).map((omittedRendered) => ({
      missingWorker,
      omittedRendered,
    })),
  ),
)(
  "preserves terminal coverage when $missingWorker lacks a worker and $omittedRendered omits rendered output",
  async ({ missingWorker, omittedRendered }) => {
    const observation = (side: "left" | "right") => {
      const hasWorker = side !== missingWorker;
      const complete = capture({
        frames: [terminal],
        process_samples: hasWorker ? [processStarted] : [],
        event_journal: [
          { collection: "frames", index: 0 },
          ...(hasWorker
            ? [{ collection: "process_samples" as const, index: 0 }]
            : []),
        ],
      });
      const omitted = omittedRendered === "both" || omittedRendered === side;
      return captureEvidence(
        side,
        {},
        PROCESS_PROVIDER,
        omitted
          ? parseProcessCapture({
              ...complete,
              truncated: true,
              truncation_details: {
                ...complete.truncation_details,
                rendered_terminal: {
                  budget_bytes: 5,
                  observed_bytes: 0,
                  retained_bytes: 0,
                  observed_frames: 1,
                  retained_frames: 0,
                  unknown_size_frames: 1,
                },
              },
            })
          : complete,
      );
    };
    const left = observation("left");
    const right = observation("right");
    const terminalStatus =
      omittedRendered === "neither" ? "unchanged" : "unknown";
    const plain = parseEvidence(await compareViaCli(left, right));
    expect(plain.normalized_result).toMatchObject({
      terminal: terminalStatus,
      process: missingWorker === "left" ? "added" : "removed",
      status: "changed",
    });

    const specification = partialSpecification();
    const cli = parseEvidence(await compareViaCli(left, right, specification));
    const client = await connect();
    const response = await client.callTool({
      name: "compare_process_captures",
      arguments: { left, right, trace_spec: specification },
    });
    expect(response.isError).not.toBe(true);
    expect(parseEvidence(response.structuredContent)).toEqual(cli);
    expect(cli.evidence_links).toEqual([left.evidence_id, right.evidence_id]);
    expect(cli.normalized_result).toMatchObject({
      terminal: terminalStatus,
      process: "changed",
      status: "changed",
      trace: {
        verdict: "different",
        diagnostic: {
          kind: "cardinality",
          side: missingWorker,
          event_ids: ["worker"],
          locations: [],
        },
      },
    });
  },
);

it.each(
  (["left", "right"] as const).flatMap((side) =>
    [
      { id: "unrelated-provider" },
      { version: "1" },
      { name: "unrelated provider" },
    ].map((change) => ({ side, change })),
  ),
)(
  "rejects an authenticated $side capture with provider $change",
  async ({ side, change }) => {
    const wrongProvider = captureEvidence(
      side,
      {},
      { ...PROCESS_PROVIDER, ...change },
    );
    expect(parseEvidence(wrongProvider)).toEqual(wrongProvider);
    const left = side === "left" ? wrongProvider : captureEvidence("left");
    const right = side === "right" ? wrongProvider : captureEvidence("right");
    const issue = {
      path: [side, "provider"],
      reason: "invalid_value",
      message: expect.stringContaining("process capture provider"),
    };
    expect(await compareViaCli(left, right)).toMatchObject({
      category: "invalid_input",
      details: { operation: "compare_process_captures", issues: [issue] },
    });
    const client = await connect();
    const response = await client.callTool({
      name: "compare_process_captures",
      arguments: { left, right },
    });
    expect(response.isError).toBe(true);
    expect(parseMcpToolError(response)).toMatchObject({
      error: { code: "invalid_request", details: { issues: [issue] } },
    });
    await client.ping();
  },
);

it.each([PROCESS_PROVIDER.name, "REA deterministic process harness"])(
  "preserves authenticated CLI/MCP comparison Evidence for provider name %s",
  async (name) => {
    const provider = { ...PROCESS_PROVIDER, name };
    const left = captureEvidence("left", {}, provider);
    const right = captureEvidence("right", {}, provider);
    const cli = parseEvidence(await compareViaCli(left, right));
    const client = await connect();
    const response = await client.callTool({
      name: "compare_process_captures",
      arguments: { left, right },
    });
    expect(response.isError).not.toBe(true);
    expect(parseEvidence(response.structuredContent)).toEqual(cli);
    const text = response.content.find((block) => block.type === "text");
    expect(text?.type).toBe("text");
    if (text?.type !== "text") throw new Error("Missing comparison text");
    expect(JSON.parse(text.text)).toEqual(cli);
    expect(cli.evidence_links).toEqual([left.evidence_id, right.evidence_id]);
  },
);

it.each(["left", "right"] as const)(
  "authenticates the %s Evidence before provider admission",
  async (side) => {
    const mismatched = captureEvidence(
      side,
      {},
      { ...PROCESS_PROVIDER, id: "unrelated-provider" },
    );
    const tampered = { ...mismatched, parameters: { side: "tampered" } };
    const left = side === "left" ? tampered : captureEvidence("left");
    const right = side === "right" ? tampered : captureEvidence("right");
    expect(await compareViaCli(left, right)).toMatchObject({
      category: "invalid_input",
      details: {
        issues: [
          {
            path: ["evidence_id"],
            message: expect.stringContaining(
              "Evidence semantic identifier does not match its record",
            ),
          },
        ],
      },
    });
    const client = await connect();
    const response = await client.callTool({
      name: "compare_process_captures",
      arguments: { left, right },
    });
    expect(parseMcpToolError(response)).toMatchObject({
      error: { code: "evidence_integrity_mismatch" },
    });
  },
);

it("names differing comparison contract fields through MCP and the CLI", async () => {
  const client = await connect();
  const response = await client.callTool({
    name: "compare_process_captures",
    arguments: { left, right },
  });
  expect(response.isError).toBe(true);
  const issue = {
    path: ["right"],
    reason: "invalid_value",
    message: expect.stringContaining(
      "these scenario fields differ: working_directory",
    ),
    expected: ["working_directory"],
  };
  expect(parseMcpToolError(response)).toMatchObject({
    error: { code: "invalid_request", details: { issues: [issue] } },
  });

  const root = await createTestTempDirectory("rea-process-contract-");
  const leftPath = join(root, "left.json");
  const rightPath = join(root, "right.json");
  await writeFile(leftPath, JSON.stringify(left));
  await writeFile(rightPath, JSON.stringify(right));
  expect(await compareProcessEvidenceFiles(leftPath, rightPath)).toMatchObject({
    error: "Process command failed",
    code: "invalid_request",
    details: { operation: "compare_process_captures", issues: [issue] },
  });
});

it("names the stale capture that exceeds max_capture_age_ms", async () => {
  const client = await connect();
  const response = await client.callTool({
    name: "compare_process_captures",
    arguments: {
      left: captureEvidence("left"),
      right: captureEvidence("right"),
      max_capture_age_ms: 1000,
    },
  });
  expect(response.isError).toBe(true);
  expect(parseMcpToolError(response)).toMatchObject({
    error: {
      code: "invalid_request",
      details: {
        issues: [
          {
            path: ["max_capture_age_ms"],
            reason: "out_of_range",
            message: expect.stringContaining(
              `left completed at ${EMPTY_PROCESS_CAPTURE_EXAMPLE.manifest.completed_at}`,
            ),
          },
        ],
      },
    },
  });
});

it("retains a selected max_capture_age_ms requirement in comparison Evidence parameters", async () => {
  const session = createTestBinarySession(() => {
    throw new Error("Process comparison must not launch a provider");
  });
  const client = await connect(session);
  const unconstrained = await client.callTool({
    name: "compare_process_captures",
    arguments: {
      left: captureEvidence("left"),
      right: captureEvidence("right"),
    },
  });
  const selected = await client.callTool({
    name: "compare_process_captures",
    arguments: {
      left: captureEvidence("left"),
      right: captureEvidence("right"),
      max_capture_age_ms: Number.MAX_SAFE_INTEGER,
    },
  });
  expect(unconstrained.isError).not.toBe(true);
  expect(selected.isError).not.toBe(true);
  const before = parseEvidence(unconstrained.structuredContent);
  const after = parseEvidence(selected.structuredContent);
  expect(before.parameters).not.toHaveProperty("max_capture_age_ms");
  expect(after.parameters).toHaveProperty(
    "max_capture_age_ms",
    Number.MAX_SAFE_INTEGER,
  );
  expect(after.evidence_id).not.toBe(before.evidence_id);
  expect(session.evidenceById(after.evidence_id)?.parameters).toHaveProperty(
    "max_capture_age_ms",
    Number.MAX_SAFE_INTEGER,
  );
});

it("reports CLI trace specification issues like the MCP trace_spec input", async () => {
  const root = await createTestTempDirectory("rea-process-trace-spec-");
  const capturePath = join(root, "capture.json");
  const specPath = join(root, "spec.json");
  await writeFile(capturePath, JSON.stringify(captureEvidence("left")));
  await writeFile(specPath, JSON.stringify({ language: {} }));
  expect(
    await compareProcessEvidenceFiles(capturePath, capturePath, specPath),
  ).toMatchObject({
    code: "invalid_request",
    details: {
      operation: "compare_process_captures",
      issues: expect.arrayContaining([
        expect.objectContaining({ path: ["trace_spec", "events"] }),
      ]),
    },
  });
});

it("rejects a deep trace cycle as an input error through CLI and MCP", async () => {
  const events = Array.from({ length: 6000 }, (_, index) => ({
    id: `e${String(index)}`,
    source: "lifecycle",
    exact: { marker: index },
  }));
  const trace = {
    events,
    language: {
      kind: "partial_order",
      happens_before: events.map((event, index) => ({
        before: event.id,
        after: events[(index + 1) % events.length]!.id,
      })),
      not_before: events.map((event, index) => ({
        event: event.id,
        anchor: events[(index + 1) % events.length]!.id,
      })),
      unordered_groups: [{ events: events.map(({ id }) => id) }],
    },
  };
  const issue = {
    path: ["trace_spec", "language"],
    message: "happens-before graph must be acyclic",
  };
  const capture = captureEvidence("left");
  const root = await createTestTempDirectory("rea-process-trace-cycle-");
  const capturePath = join(root, "capture.json");
  const specPath = join(root, "spec.json");
  await writeFile(capturePath, JSON.stringify(capture));
  await writeFile(specPath, JSON.stringify(trace));
  expect(
    await compareProcessEvidenceFiles(capturePath, capturePath, specPath),
  ).toMatchObject({
    code: "invalid_request",
    details: { issues: [issue] },
  });

  const client = await connect();
  const response = await client.callTool({
    name: "compare_process_captures",
    arguments: { left: capture, right: capture, trace_spec: trace },
  });
  expect(response.isError).toBe(true);
  expect(response.content).toEqual([
    { type: "text", text: expect.stringContaining(issue.message) },
  ]);
  await client.ping();
});

it("names the side and constraint when Evidence is not a usable capture", async () => {
  const client = await connect();
  const capture = captureEvidence("right");
  const unrelated = createEvidence(undefined, PROCESS_PROVIDER, {
    operation: "other",
    parameters: {},
    result: {},
  });
  const wrongKind = await client.callTool({
    name: "compare_process_captures",
    arguments: { left: unrelated, right: capture },
  });
  expect(parseMcpToolError(wrongKind)).toMatchObject({
    error: {
      code: "invalid_request",
      details: {
        issues: [
          {
            path: ["left"],
            reason: "invalid_value",
            message: expect.stringContaining(
              "the left Evidence has operation other",
            ),
          },
        ],
      },
    },
  });

  const tampered = await client.callTool({
    name: "compare_process_captures",
    arguments: {
      left: capture,
      right: { ...capture, parameters: { side: "changed" } },
    },
  });
  expect(parseMcpToolError(tampered)).toMatchObject({
    error: {
      code: "evidence_integrity_mismatch",
      message: expect.stringContaining(
        "The right Evidence failed validation (at evidence_id: Evidence semantic identifier does not match its record)",
      ),
    },
  });

  const invalidResult = createEvidence(undefined, PROCESS_PROVIDER, {
    predicateType: "rea.process-capture",
    operation: "capture_process_scenario",
    parameters: {},
    result: { manifest: {} },
  });
  const malformed = await client.callTool({
    name: "compare_process_captures",
    arguments: { left: capture, right: invalidResult },
  });
  expect(parseMcpToolError(malformed)).toMatchObject({
    error: {
      code: "evidence_integrity_mismatch",
      message: expect.stringContaining(
        "The right Evidence has an invalid process capture result (at ",
      ),
    },
  });

  const root = await createTestTempDirectory("rea-process-malformed-");
  const leftPath = join(root, "left.json");
  const rightPath = join(root, "right.json");
  await writeFile(leftPath, JSON.stringify(capture));
  await writeFile(rightPath, JSON.stringify(invalidResult));
  expect(await compareProcessEvidenceFiles(leftPath, rightPath)).toMatchObject({
    category: "invalid_input",
    message: expect.stringContaining("Capture evidence is malformed (at "),
  });
});

it("keeps structural capture issues alongside semantic ones through MCP and the CLI", async () => {
  const client = await connect();
  const capture = captureEvidence("left");
  const {
    manifest: { completed_at: _completedAt, ...manifest },
    exit,
    ...rest
  } = FINALIZED_PROCESS_CAPTURE_EXAMPLE;
  const mixed = createEvidence(undefined, PROCESS_PROVIDER, {
    predicateType: "rea.process-capture",
    operation: "capture_process_scenario",
    parameters: {},
    result: jsonValueSchema.parse({
      ...rest,
      manifest,
      exit: {
        ...exit,
        finalization: {
          ...exit.finalization,
          signals: [
            { ...exit.finalization.signals[0], reason: "not allowed here" },
          ],
        },
      },
    }),
  });
  const described =
    "(at manifest.completed_at: Invalid input: expected string, received undefined (1 more issues))";

  const response = await client.callTool({
    name: "compare_process_captures",
    arguments: { left: capture, right: mixed },
  });
  expect(parseMcpToolError(response)).toMatchObject({
    error: {
      code: "evidence_integrity_mismatch",
      message: expect.stringContaining(
        `The right Evidence has an invalid process capture result ${described}`,
      ),
    },
  });
  expect(await compareViaCli(capture, mixed)).toMatchObject({
    code: "invalid_request",
    category: "invalid_input",
    message: expect.stringContaining(
      `Capture evidence is malformed ${described}`,
    ),
  });
});