rea/experiments/analysis-efficiency/mcp-input-catalog.mjs

// Compare complete public MCP tasks with caller-selected input advertisements.
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { mkdir, readFile, writeFile } from "node:fs/promises";
import { join, resolve } from "node:path";
import { parseArgs } from "node:util";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { Ajv2020 } from "ajv/dist/2020.js";
import {
  captureRuntimeBinding,
  assertRuntimeBinding,
} from "./managed-cli-runtime.mjs";

const { values } = parseArgs({
  options: {
    runtime: { type: "string" },
    artifact: { type: "string" },
    oracle: { type: "string" },
    output: { type: "string" },
    profile: { type: "string", default: "default" },
    phase: { type: "string", default: "analysis" },
    cache: { type: "string", default: "cold" },
    "cache-root": { type: "string" },
    bindings: { type: "string" },
    role: { type: "string" },
  },
});
assert.ok(["analysis", "discovery"].includes(values.phase));
assert.ok(["cold", "warm"].includes(values.cache));
for (const key of values.phase === "analysis"
  ? ["runtime", "artifact", "oracle", "output"]
  : ["runtime", "output"])
  assert.ok(
    values[key] && resolve(values[key]) === values[key],
    `${key} must be absolute`,
  );
assert.ok(["default", "full", "compact"].includes(values.profile));
if (values.cache === "warm")
  assert.ok(
    values["cache-root"] &&
      resolve(values["cache-root"]) === values["cache-root"],
  );
await mkdir(values.output, { recursive: false });
const hash = (value) => createHash("sha256").update(value).digest("hex");
let oracle;
let artifactSha;
if (values.phase === "analysis") {
  oracle = JSON.parse(await readFile(values.oracle, "utf8"));
  artifactSha = hash(await readFile(values.artifact));
  assert.equal(artifactSha, oracle.artifact_sha256);
}
let expectedBinding;
let bindingBefore;
let auditBeforeMs;
let measurementInputsSha256;
const producerSources = {};
if (values.bindings !== undefined) {
  assert.ok(["baseline", "candidate"].includes(values.role));
  const inputBytes = await readFile(values.bindings);
  measurementInputsSha256 = hash(inputBytes);
  const inputs = JSON.parse(inputBytes.toString("utf8"));
  for (const name of ["mcp-input-catalog.mjs", "managed-cli-runtime.mjs"]) {
    producerSources[name] = hash(
      await readFile(new URL(name, import.meta.url)),
    );
    assert.equal(producerSources[name], inputs.harness_sources_sha256[name]);
  }
  expectedBinding = inputs.runtime_bindings[values.role];
  const start = performance.now();
  bindingBefore = await captureRuntimeBinding(values.runtime, "mcp");
  auditBeforeMs = performance.now() - start;
  assertRuntimeBinding(bindingBefore, expectedBinding, values.role);
}
const environment = {
  PATH: process.env.PATH,
  HOME: values.output,
  REA_LOG_LEVEL: "silent",
  UV_THREADPOOL_SIZE: "1",
  NODE_OPTIONS: "--max-old-space-size=2048",
  ...(values.cache === "cold"
    ? { NODE_DISABLE_COMPILE_CACHE: "1" }
    : { NODE_COMPILE_CACHE: values["cache-root"] }),
  ...(values.profile === "default"
    ? {}
    : { REA_MCP_INPUT_SCHEMA_PROFILE: values.profile }),
};
const transport = new StdioClientTransport({
  command: process.execPath,
  args: [join(values.runtime, "scripts/rea.mjs"), "mcp"],
  cwd: values.output,
  env: environment,
  stderr: "pipe",
});
const protocolRequests = {};
const send = transport.send.bind(transport);
transport.send = (...args) => {
  const method = args[0].method;
  if (typeof method === "string")
    protocolRequests[method] = (protocolRequests[method] ?? 0) + 1;
  return send(...args);
};
const client = new Client({ name: "rea-catalog-task", version: "1" });
const report = {
  profile: values.profile,
  phase: values.phase,
  cache: values.cache,
  cache_root: values["cache-root"] ?? null,
  role: values.role,
  measurement_inputs_sha256: measurementInputsSha256,
  producer_sources_sha256: producerSources,
  artifact_sha256: artifactSha,
  calls: [],
  checks: [],
  status: "incomplete",
  cleanup_verified: false,
  protocol_requests: protocolRequests,
};
const transcript = [];
let stderr = "";
transport.stderr?.on("data", (value) => {
  stderr += value;
});
const began = performance.now();
try {
  await client.connect(transport);
  report.protocol_era = client.getProtocolEra();
  await writeFile(
    join(values.output, "server.json"),
    JSON.stringify({
      info: client.getServerVersion(),
      capabilities: client.getServerCapabilities(),
      instructions: client.getInstructions(),
    }),
  );
  const catalog = await client.listTools();
  assert.equal(catalog.nextCursor, undefined);
  const definitions = new Map(catalog.tools.map((tool) => [tool.name, tool]));
  assert.equal(definitions.size, catalog.tools.length);
  const ajv = new Ajv2020({ strict: false, validateFormats: false });
  for (const tool of catalog.tools)
    assert.ok(
      ajv.validateSchema(tool.inputSchema),
      `Invalid schema: ${tool.name}`,
    );
  {
    const encoded = JSON.stringify(catalog);
    await writeFile(join(values.output, "catalog.json"), encoded);
    report.catalog_sha256 = hash(encoded);
  }
  report.tool_names = [...definitions.keys()];
  if (values.phase === "discovery") {
    const prompts = await client.listPrompts();
    assert.equal(prompts.nextCursor, undefined);
    await writeFile(
      join(values.output, "prompts.json"),
      JSON.stringify(prompts),
    );
    report.prompts_sha256 = hash(JSON.stringify(prompts));
    report.answer = {
      tools: report.tool_names,
      prompts: prompts.prompts.map((prompt) => prompt.name),
    };
    report.status = "passed";
  } else {
    const call = async (name, args) => {
      const definition = definitions.get(name);
      assert.ok(definition, `Missing named tool ${name}`);
      const start = performance.now();
      const reply = await client.callTool(
        { name, arguments: args },
        { toolDefinition: definition },
      );
      report.calls.push({
        name,
        elapsed_ms: performance.now() - start,
        is_error: reply.isError === true,
      });
      transcript.push({ name, arguments: args, reply });
      return reply;
    };
    const input = {
      path: values.artifact,
      view: {
        collection: "packages",
        names: ["serde"],
        fields: ["name", "version", "kind"],
      },
    };
    const first = await call("inspect_rust_binary", input);
    assert.notEqual(first.isError, true);
    const selected = first.structuredContent;
    assert.ok(selected?.normalized_result?.page);
    assert.deepEqual(selected.normalized_result.page.rows, oracle.serde_rows);
    assert.equal(selected.subject.digest.sha256, artifactSha);
    const parentId = selected.normalized_result.parent_evidence_id;
    for (let index = 0; index < 2; index++) {
      const reply = await call("inspect_analysis_view", {
        source: { kind: "retained-evidence", evidence_id: parentId },
        view: {
          kind: "rust",
          collection: "packages",
          names: ["serde"],
          fields: ["name", "version", "kind"],
        },
      });
      assert.notEqual(reply.isError, true);
      assert.deepEqual(
        reply.structuredContent.normalized_result.page.rows,
        oracle.serde_rows,
      );
      assert.equal(
        reply.structuredContent.normalized_result.parent_evidence_id,
        parentId,
      );
    }
    const malformed = await call("inspect_rust_binary", {
      path: values.artifact,
      view: { collection: "packages", offset: -1 },
    });
    assert.equal(malformed.isError, true);
    report.checks.push(
      "Independent package rows and digest; repeated retained identity; canonical negative-offset refusal",
    );
    const exported = await call("export_evidence_bundle", {
      path: join(values.output, "evidence.json"),
    });
    assert.notEqual(exported.isError, true);
    const bundle = JSON.parse(
      await readFile(join(values.output, "evidence.json"), "utf8"),
    );
    const parent = bundle.records.find(
      (record) => record.evidence_id === parentId,
    );
    assert.ok(parent, "Complete original Evidence is retained");
    assert.equal(parent.subject.digest.sha256, artifactSha);
    assert.deepEqual(
      parent.normalized_result.auditable_metadata.packages,
      oracle.packages,
    );
    assert.deepEqual(
      parent.normalized_result.auditable_metadata.dependency_edges,
      oracle.edges,
    );
    report.parent_evidence_id = parentId;
    report.parent_semantic_sha256 = hash(JSON.stringify(parent));
    report.answer = oracle.serde_rows;
    assert.deepEqual(
      await client.listTools(),
      catalog,
      "Advertisement changed after canonical materialization",
    );
    report.checks.push(
      "Catalog remains identical after canonical materialization",
    );
    report.status = "passed";
  }
} finally {
  await client.close();
  await transport.close();
  report.cleanup_verified = transport.pid === null;
  assert.ok(report.cleanup_verified);
  report.elapsed_ms = performance.now() - began;
  if (expectedBinding !== undefined) {
    const start = performance.now();
    const after = await captureRuntimeBinding(values.runtime, "mcp");
    report.runtime_audit_ms = {
      before: auditBeforeMs,
      after: performance.now() - start,
    };
    assertRuntimeBinding(after, expectedBinding, values.role);
    assert.deepEqual(after, bindingBefore);
    assert.equal(
      hash(await readFile(values.bindings)),
      measurementInputsSha256,
    );
    for (const [name, digest] of Object.entries(producerSources))
      assert.equal(
        hash(await readFile(new URL(name, import.meta.url))),
        digest,
      );
    report.runtime_binding = after;
    report.runtime_binding_verified = true;
  }
  await writeFile(
    join(values.output, "report.json"),
    JSON.stringify(report, null, 2) + "\n",
  );
  await writeFile(
    join(values.output, "transcript.json"),
    JSON.stringify(transcript),
  );
  await writeFile(join(values.output, "server.log"), stderr);
}