fix(evaluation): align scenario prompts and outcome gates

docs/client-verification.md

@@ -475,10 +475,16 @@ is removed after confirmed process cleanup even with
 `REA_AGENT_EVAL_KEEP_FIXTURES=true`. No user configuration,
 rules, plugins or history is copied; shell snapshots and additional agents are
 disabled. Existing API-key environment authentication remains available.
-The native missing-provider scenario preapproves only `open_binary` and
+The native provider scenario preapproves only `open_binary` and
 `close_binary` in the disposable client so it reaches REA's actual provider
-boundary instead of stopping at Codex approval. This does not validate a real
-native engine or authorize arbitrary runtime execution.
+boundary instead of stopping at Codex approval. Its scenario contract accepts
+one `provider_unavailable` open refusal without a close, or one successful open
+followed by a successful close. Other errors, an unclosed session, and a close
+without acquisition fail. These outcomes and the prompt are owned together in
+`src/evaluation/AgentEvaluationScenarios.ts`; generic successful-tool sequences
+still apply to scenarios that require them. The provider scenario checks routing,
+reported limitations, and cleanup; free-form answer correctness remains
+unassessed. It does not validate a real native engine or authorize runtime execution.
 
 Select a smaller run with `REA_AGENT_EVAL_SCENARIOS`, a comma-separated list of
 scenario IDs. `asar-zh` exercises the packaged desktop rubric from a Chinese
@@ -491,8 +497,10 @@ limit for the whole process tree when testing on a constrained host; a Node
 heap limit alone does not bound client and child-process memory.
 
 `missing-target` leaves desktop fixtures visible without selecting one in the
-request. It requires zero REA calls and a structured request for an app name or
-artifact path. Its `expectedFirstTool` is `null`, and
+request. It requires zero REA calls and a JSON-only request for an app name or
+artifact path, with no preceding prose or Markdown fences. Its prompt and
+whole-answer validator are defined together in `AgentEvaluationScenarios.ts`.
+Its `expectedFirstTool` is `null`, and
 `targetClarificationPassed` grades that limited routing outcome independently
 of analysis prose heuristics. Analyzing a nearby example cannot pass this case.
 
@@ -554,7 +562,10 @@ tool results and final answers for review.
 Report schema version 4 adds negative routing scenarios with nullable
 `expectedFirstTool` and a `targetClarificationPassed` outcome. Consumers must
 allow an expected absence of REA calls; this does not make an analysis scenario
-pass without its required tools. Version 3 changed the top-level
+pass without its required tools. Native provider records additionally report
+`scenarioOutcome` (`provider-unavailable`, `opened-and-closed`, or `failed`) and
+`scenarioOutcomePassed`; a failed outcome fails the gate independently of prose
+heuristics. Version 3 changed the top-level
 `factualCorrectness` from a constant
 string to an assessment summary with status, scope, and assessed/passed/failed/
 not-assessed scenario counts. Scenario records include the factual assessment and

scripts/verify-agent-experience.mjs

@@ -11,9 +11,12 @@ import {
 import { homedir } from "node:os";
 import { delimiter, dirname, join, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
-import { z } from "zod";
 
 import { evaluateCodexEvents } from "../dist/evaluation/CodexAgentEval.js";
+import {
+  MISSING_TARGET_SCENARIO,
+  nativeProviderScenario,
+} from "../dist/evaluation/AgentEvaluationScenarios.js";
 import {
   createAgentEvaluationFixtures,
   agentFixtureClaims,
@@ -136,17 +139,7 @@ try {
     requestedScenarioIds.size === 0 || requestedScenarioIds.has("managed"),
   );
   const allScenarios = [
-    {
-      id: "native",
-      expectedFirstTool: "open_binary",
-      requiresEvidence: false,
-      requiredAnswerTermGroups: [["unavailable", "could not", "provider"]],
-      // The verifier authorizes these named session operations, not arbitrary
-      // writes or runtime execution. Missing-engine behavior stays observable.
-      approvedTools: ["open_binary", "close_binary"],
-      requiredToolSubsequence: ["open_binary", "close_binary"],
-      prompt: `Explain what static artifact evidence establishes about the shipped native program at ${targets.native}. Do not execute the target or install tools. Clearly state unavailable provider evidence, and close any analysis session you opened before finishing.`,
-    },
+    nativeProviderScenario(targets.native),
     {
       id: "asar",
       expectedFirstTool: "analyze_javascript_application",
@@ -194,14 +187,7 @@ try {
       ],
       prompt: `Inspect the recorded module identity, exports and observations for main.js in the shipped JavaScript tree ${targets.largeJavascript}. Use REA without directly reading target files or executing the target. Keep results focused on that module, cite Evidence, and state the limits of what its module view establishes.`,
     },
-    {
-      id: "missing-target",
-      expectedFirstTool: null,
-      requiresEvidence: false,
-      requiredAnswerTermGroups: [],
-      prompt:
-        'Reverse engineer my desktop app and explain how it saves user settings. Use static analysis; do not install software. End with one JSON object containing status ("needs_target" or "analyzed"), requested_input ("app_name_or_artifact_path" or null), and question (a string or null).',
-    },
+    MISSING_TARGET_SCENARIO,
     {
       id: "javascript-export-shape",
       expectedFirstTool: "analyze_javascript_application",
@@ -313,8 +299,9 @@ try {
     // Their existence does not identify the unnamed app in this request.
     const targetClarificationPassed =
       scenario.expectedFirstTool === null
-        ? validTargetClarification(metrics.finalMessage)
+        ? scenario.validTargetClarification(metrics.finalMessage)
         : undefined;
+    const scenarioOutcome = scenario.assess?.(metrics);
     const result = {
       id: scenario.id,
       expectedFirstTool: scenario.expectedFirstTool,
@@ -325,6 +312,12 @@ try {
       exitCode: execution.exitCode,
       stderr: execution.stderr,
       ...metrics,
+      ...(scenarioOutcome === undefined
+        ? {}
+        : {
+            scenarioOutcome,
+            scenarioOutcomePassed: scenarioOutcome !== "failed",
+          }),
       ...(targetClarificationPassed === undefined
         ? {}
         : { targetClarificationPassed }),
@@ -488,17 +481,3 @@ async function createTargets(root, includeManaged) {
       : {}),
   };
 }
-
-function validTargetClarification(message) {
-  const schema = z.strictObject({
-    status: z.literal("needs_target"),
-    requested_input: z.literal("app_name_or_artifact_path"),
-    question: z.string().refine((value) => value.trim().length > 0),
-  });
-  try {
-    return schema.safeParse(JSON.parse(message)).success;
-  } catch (cause) {
-    if (cause instanceof SyntaxError) return false;
-    throw cause;
-  }
-}

src/evaluation/AgentEvaluationReport.ts

@@ -6,6 +6,8 @@ export interface AgentEvaluationGateResult {
   /** Explicit null requires a target clarification with no REA calls. */
   readonly expectedFirstTool?: string | null;
   readonly targetClarificationPassed?: boolean;
+  /** A scenario-specific expected refusal or acquisition/cleanup outcome. */
+  readonly scenarioOutcomePassed?: boolean;
   readonly exitCode: number;
   readonly naturalUse: boolean;
   readonly correctFirstTool: boolean;
@@ -30,6 +32,7 @@ export const agentEvaluationPassed = (
   result.repeatedCallCount === 0 &&
   result.inputValidationFailureCount === 0 &&
   result.requiredToolSubsequenceMet &&
+  result.scenarioOutcomePassed !== false &&
   result.inputTokens > 0 &&
   (result.expectedFirstTool === null
     ? result.targetClarificationPassed === true

src/evaluation/AgentEvaluationScenarios.ts

@@ -0,0 +1,65 @@
+import { z } from "zod";
+
+import type { CodexAgentMetrics } from "./CodexAgentEval.js";
+
+const clarificationSchema = z.strictObject({
+  status: z.literal("needs_target"),
+  requested_input: z.literal("app_name_or_artifact_path"),
+  question: z.string().refine((value) => value.trim().length > 0),
+});
+
+/** The missing-target prompt and its whole-answer grammar share one owner. */
+export const MISSING_TARGET_SCENARIO = {
+  id: "missing-target",
+  expectedFirstTool: null,
+  requiresEvidence: false,
+  requiredAnswerTermGroups: [],
+  prompt:
+    'Reverse engineer my desktop app and explain how it saves user settings. Use static analysis; do not install software. Return only one JSON object, without prose or Markdown fences, containing status ("needs_target" or "analyzed"), requested_input ("app_name_or_artifact_path" or null), and question (a string or null).',
+  validTargetClarification(message: string): boolean {
+    try {
+      return clarificationSchema.safeParse(JSON.parse(message)).success;
+    } catch (cause: unknown) {
+      if (cause instanceof SyntaxError) return false;
+      throw cause;
+    }
+  },
+} as const;
+
+type NativeSessionOutcome =
+  | "provider-unavailable"
+  | "opened-and-closed"
+  | "failed";
+
+/** Expected refusal is distinct from acquisition, which always requires close. */
+const nativeSessionOutcome = ({
+  reaCalls,
+}: CodexAgentMetrics): NativeSessionOutcome => {
+  if (reaCalls.some(({ tool, error }) => error && tool !== "open_binary"))
+    return "failed";
+  const lifecycle = reaCalls.filter(
+    ({ tool }) => tool === "open_binary" || tool === "close_binary",
+  );
+  const open = lifecycle[0];
+  if (open?.tool !== "open_binary") return "failed";
+  if (open.error)
+    return lifecycle.length === 1 && open.errorCode === "provider_unavailable"
+      ? "provider-unavailable"
+      : "failed";
+  return lifecycle.length === 2 &&
+    lifecycle[1]?.tool === "close_binary" &&
+    !lifecycle[1].error
+    ? "opened-and-closed"
+    : "failed";
+};
+
+/** Routing/cleanup trial; it does not establish real-engine analysis correctness. */
+export const nativeProviderScenario = (target: string) => ({
+  id: "native",
+  expectedFirstTool: "open_binary",
+  requiresEvidence: false,
+  requiredAnswerTermGroups: [],
+  approvedTools: ["open_binary", "close_binary"],
+  assess: nativeSessionOutcome,
+  prompt: `Explain what static artifact evidence establishes about the shipped native program at ${target}. Do not execute the target or install tools. If the provider is unavailable, state that limitation and that no session was opened. If a session opens, report the available evidence and close it before finishing.`,
+});

tests/evaluation/model-evals/agentEvaluationScenarios.test.ts

@@ -0,0 +1,145 @@
+import { describe, expect, it } from "vitest";
+
+import { evaluateCodexEvents } from "../../../src/evaluation/CodexAgentEval.js";
+import { agentEvaluationPassed } from "../../../src/evaluation/AgentEvaluationReport.js";
+import {
+  MISSING_TARGET_SCENARIO,
+  nativeProviderScenario,
+} from "../../../src/evaluation/AgentEvaluationScenarios.js";
+
+const call = (id: string, tool: string, errorCode?: string) => ({
+  type: "item.completed",
+  item: {
+    id,
+    type: "mcp_tool_call",
+    server: "rea",
+    tool,
+    arguments: tool === "open_binary" ? { path: "/bin/true" } : {},
+    status: "completed",
+    result:
+      errorCode === undefined
+        ? { content: [{ type: "text", text: "Operation completed." }] }
+        : { isError: true, structuredContent: { error: { code: errorCode } } },
+  },
+});
+
+const completion = (text: string) => [
+  { type: "item.completed", item: { type: "agent_message", text } },
+  { type: "turn.completed", usage: { input_tokens: 100, output_tokens: 30 } },
+];
+
+describe("native provider scenario", () => {
+  const scenario = nativeProviderScenario("/bin/true");
+  const opened = call("open", "open_binary");
+  const closed = call("close", "close_binary");
+  const unavailable = call("open", "open_binary", "provider_unavailable");
+
+  it.each([
+    ["expected provider refusal", [unavailable], "provider-unavailable"],
+    ["opened session with cleanup", [opened, closed], "opened-and-closed"],
+    ["unclosed session", [opened], "failed"],
+    ["cleanup without acquisition", [unavailable, closed], "failed"],
+    [
+      "failed cleanup",
+      [opened, call("close", "close_binary", "cleanup_incomplete")],
+      "failed",
+    ],
+    [
+      "unexpected provider error",
+      [call("open", "open_binary", "configuration_invalid")],
+      "failed",
+    ],
+    [
+      "invalid request",
+      [call("open", "open_binary", "invalid_request")],
+      "failed",
+    ],
+    ["no attempted open", [], "failed"],
+    [
+      "second unclosed session",
+      [opened, closed, call("reopen", "open_binary")],
+      "failed",
+    ],
+  ] as const)(
+    "grades %s through the transcript parser and release gate",
+    (_, calls, expected) => {
+      const final =
+        expected === "provider-unavailable"
+          ? "The native provider is unavailable, so no analysis session was opened. Static provider evidence remains unknown; the target was not executed."
+          : "The returned static artifact evidence describes the selected native program. Runtime behavior remains unknown; the analysis session was closed.";
+      const metrics = evaluateCodexEvents(
+        [...calls, ...completion(final)],
+        scenario.expectedFirstTool,
+        scenario,
+      );
+      const outcome = scenario.assess(metrics);
+      expect(outcome).toBe(expected);
+      expect(
+        agentEvaluationPassed({
+          ...metrics,
+          expectedFirstTool: scenario.expectedFirstTool,
+          exitCode: 0,
+          scenarioOutcomePassed: outcome !== "failed",
+        }),
+      ).toBe(expected !== "failed");
+    },
+  );
+});
+
+describe("missing-target scenario", () => {
+  const answer = JSON.stringify({
+    status: "needs_target",
+    requested_input: "app_name_or_artifact_path",
+    question: "Which app name or artifact path should I inspect?",
+  });
+
+  it("advertises and grades the same JSON-only clarification grammar", () => {
+    expect(MISSING_TARGET_SCENARIO.prompt).toContain(
+      "Return only one JSON object, without prose or Markdown fences",
+    );
+    const metrics = evaluateCodexEvents(completion(answer), null);
+    expect(
+      agentEvaluationPassed({
+        ...metrics,
+        expectedFirstTool: null,
+        exitCode: 0,
+        targetClarificationPassed:
+          MISSING_TARGET_SCENARIO.validTargetClarification(
+            metrics.finalMessage,
+          ),
+      }),
+    ).toBe(true);
+    for (const message of [
+      `Please choose an app.\n${answer}`,
+      `\`\`\`json\n${answer}\n\`\`\``,
+      answer.replace("needs_target", "analyzed"),
+      answer.replace("app_name_or_artifact_path", "unknown"),
+      JSON.stringify({
+        status: "needs_target",
+        requested_input: "app_name_or_artifact_path",
+        question: "   ",
+      }),
+    ])
+      expect(MISSING_TARGET_SCENARIO.validTargetClarification(message)).toBe(
+        false,
+      );
+  });
+
+  it("rejects analysis of a nearby target even with a valid final clarification", () => {
+    const metrics = evaluateCodexEvents(
+      [call("open", "open_binary"), ...completion(answer)],
+      null,
+    );
+    expect(
+      agentEvaluationPassed({
+        ...metrics,
+        expectedFirstTool: null,
+        exitCode: 0,
+        targetClarificationPassed:
+          MISSING_TARGET_SCENARIO.validTargetClarification(
+            metrics.finalMessage,
+          ),
+      }),
+    ).toBe(false);
+  });
+});