From d3700f709bd962e9a096351c123597e62ceed06c Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 24 Oct 2026 02:34:29 +0800 Subject: [PATCH] fix(evaluation): align scenario prompts and outcome gates --- docs/client-verification.md | 23 ++- scripts/verify-agent-experience.mjs | 49 ++---- src/evaluation/AgentEvaluationReport.ts | 3 + src/evaluation/AgentEvaluationScenarios.ts | 65 ++++++++ .../agentEvaluationScenarios.test.ts | 145 ++++++++++++++++++ 5 files changed, 244 insertions(+), 41 deletions(-) create mode 100644 src/evaluation/AgentEvaluationScenarios.ts create mode 100644 tests/evaluation/model-evals/agentEvaluationScenarios.test.ts diff --git a/docs/client-verification.md b/docs/client-verification.md index 6759ab5180..117e369cda 100644 --- a/docs/client-verification.md +++ b/docs/client-verification.md @@ -475,10 +475,16 @@ is removed after confirmed process cleanup even with `REA_AGENT_EVAL_KEEP_FIXTURES=true`. No user configuration, rules, plugins or history is copied; shell snapshots and additional agents are disabled. Existing API-key environment authentication remains available. -The native missing-provider scenario preapproves only `open_binary` and +The native provider scenario preapproves only `open_binary` and `close_binary` in the disposable client so it reaches REA's actual provider -boundary instead of stopping at Codex approval. This does not validate a real -native engine or authorize arbitrary runtime execution. +boundary instead of stopping at Codex approval. Its scenario contract accepts +one `provider_unavailable` open refusal without a close, or one successful open +followed by a successful close. Other errors, an unclosed session, and a close +without acquisition fail. These outcomes and the prompt are owned together in +`src/evaluation/AgentEvaluationScenarios.ts`; generic successful-tool sequences +still apply to scenarios that require them. The provider scenario checks routing, +reported limitations, and cleanup; free-form answer correctness remains +unassessed. It does not validate a real native engine or authorize runtime execution. Select a smaller run with `REA_AGENT_EVAL_SCENARIOS`, a comma-separated list of scenario IDs. `asar-zh` exercises the packaged desktop rubric from a Chinese @@ -491,8 +497,10 @@ limit for the whole process tree when testing on a constrained host; a Node heap limit alone does not bound client and child-process memory. `missing-target` leaves desktop fixtures visible without selecting one in the -request. It requires zero REA calls and a structured request for an app name or -artifact path. Its `expectedFirstTool` is `null`, and +request. It requires zero REA calls and a JSON-only request for an app name or +artifact path, with no preceding prose or Markdown fences. Its prompt and +whole-answer validator are defined together in `AgentEvaluationScenarios.ts`. +Its `expectedFirstTool` is `null`, and `targetClarificationPassed` grades that limited routing outcome independently of analysis prose heuristics. Analyzing a nearby example cannot pass this case. @@ -554,7 +562,10 @@ tool results and final answers for review. Report schema version 4 adds negative routing scenarios with nullable `expectedFirstTool` and a `targetClarificationPassed` outcome. Consumers must allow an expected absence of REA calls; this does not make an analysis scenario -pass without its required tools. Version 3 changed the top-level +pass without its required tools. Native provider records additionally report +`scenarioOutcome` (`provider-unavailable`, `opened-and-closed`, or `failed`) and +`scenarioOutcomePassed`; a failed outcome fails the gate independently of prose +heuristics. Version 3 changed the top-level `factualCorrectness` from a constant string to an assessment summary with status, scope, and assessed/passed/failed/ not-assessed scenario counts. Scenario records include the factual assessment and diff --git a/scripts/verify-agent-experience.mjs b/scripts/verify-agent-experience.mjs index 3ac69d1d27..fbb5b47ccb 100644 --- a/scripts/verify-agent-experience.mjs +++ b/scripts/verify-agent-experience.mjs @@ -11,9 +11,12 @@ import { import { homedir } from "node:os"; import { delimiter, dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; -import { z } from "zod"; import { evaluateCodexEvents } from "../dist/evaluation/CodexAgentEval.js"; +import { + MISSING_TARGET_SCENARIO, + nativeProviderScenario, +} from "../dist/evaluation/AgentEvaluationScenarios.js"; import { createAgentEvaluationFixtures, agentFixtureClaims, @@ -136,17 +139,7 @@ try { requestedScenarioIds.size === 0 || requestedScenarioIds.has("managed"), ); const allScenarios = [ - { - id: "native", - expectedFirstTool: "open_binary", - requiresEvidence: false, - requiredAnswerTermGroups: [["unavailable", "could not", "provider"]], - // The verifier authorizes these named session operations, not arbitrary - // writes or runtime execution. Missing-engine behavior stays observable. - approvedTools: ["open_binary", "close_binary"], - requiredToolSubsequence: ["open_binary", "close_binary"], - prompt: `Explain what static artifact evidence establishes about the shipped native program at ${targets.native}. Do not execute the target or install tools. Clearly state unavailable provider evidence, and close any analysis session you opened before finishing.`, - }, + nativeProviderScenario(targets.native), { id: "asar", expectedFirstTool: "analyze_javascript_application", @@ -194,14 +187,7 @@ try { ], prompt: `Inspect the recorded module identity, exports and observations for main.js in the shipped JavaScript tree ${targets.largeJavascript}. Use REA without directly reading target files or executing the target. Keep results focused on that module, cite Evidence, and state the limits of what its module view establishes.`, }, - { - id: "missing-target", - expectedFirstTool: null, - requiresEvidence: false, - requiredAnswerTermGroups: [], - prompt: - 'Reverse engineer my desktop app and explain how it saves user settings. Use static analysis; do not install software. End with one JSON object containing status ("needs_target" or "analyzed"), requested_input ("app_name_or_artifact_path" or null), and question (a string or null).', - }, + MISSING_TARGET_SCENARIO, { id: "javascript-export-shape", expectedFirstTool: "analyze_javascript_application", @@ -313,8 +299,9 @@ try { // Their existence does not identify the unnamed app in this request. const targetClarificationPassed = scenario.expectedFirstTool === null - ? validTargetClarification(metrics.finalMessage) + ? scenario.validTargetClarification(metrics.finalMessage) : undefined; + const scenarioOutcome = scenario.assess?.(metrics); const result = { id: scenario.id, expectedFirstTool: scenario.expectedFirstTool, @@ -325,6 +312,12 @@ try { exitCode: execution.exitCode, stderr: execution.stderr, ...metrics, + ...(scenarioOutcome === undefined + ? {} + : { + scenarioOutcome, + scenarioOutcomePassed: scenarioOutcome !== "failed", + }), ...(targetClarificationPassed === undefined ? {} : { targetClarificationPassed }), @@ -488,17 +481,3 @@ async function createTargets(root, includeManaged) { : {}), }; } - -function validTargetClarification(message) { - const schema = z.strictObject({ - status: z.literal("needs_target"), - requested_input: z.literal("app_name_or_artifact_path"), - question: z.string().refine((value) => value.trim().length > 0), - }); - try { - return schema.safeParse(JSON.parse(message)).success; - } catch (cause) { - if (cause instanceof SyntaxError) return false; - throw cause; - } -} diff --git a/src/evaluation/AgentEvaluationReport.ts b/src/evaluation/AgentEvaluationReport.ts index 4af002f40c..5b909dc6d6 100644 --- a/src/evaluation/AgentEvaluationReport.ts +++ b/src/evaluation/AgentEvaluationReport.ts @@ -6,6 +6,8 @@ export interface AgentEvaluationGateResult { /** Explicit null requires a target clarification with no REA calls. */ readonly expectedFirstTool?: string | null; readonly targetClarificationPassed?: boolean; + /** A scenario-specific expected refusal or acquisition/cleanup outcome. */ + readonly scenarioOutcomePassed?: boolean; readonly exitCode: number; readonly naturalUse: boolean; readonly correctFirstTool: boolean; @@ -30,6 +32,7 @@ export const agentEvaluationPassed = ( result.repeatedCallCount === 0 && result.inputValidationFailureCount === 0 && result.requiredToolSubsequenceMet && + result.scenarioOutcomePassed !== false && result.inputTokens > 0 && (result.expectedFirstTool === null ? result.targetClarificationPassed === true diff --git a/src/evaluation/AgentEvaluationScenarios.ts b/src/evaluation/AgentEvaluationScenarios.ts new file mode 100644 index 0000000000..0b491e890f --- /dev/null +++ b/src/evaluation/AgentEvaluationScenarios.ts @@ -0,0 +1,65 @@ +import { z } from "zod"; + +import type { CodexAgentMetrics } from "./CodexAgentEval.js"; + +const clarificationSchema = z.strictObject({ + status: z.literal("needs_target"), + requested_input: z.literal("app_name_or_artifact_path"), + question: z.string().refine((value) => value.trim().length > 0), +}); + +/** The missing-target prompt and its whole-answer grammar share one owner. */ +export const MISSING_TARGET_SCENARIO = { + id: "missing-target", + expectedFirstTool: null, + requiresEvidence: false, + requiredAnswerTermGroups: [], + prompt: + 'Reverse engineer my desktop app and explain how it saves user settings. Use static analysis; do not install software. Return only one JSON object, without prose or Markdown fences, containing status ("needs_target" or "analyzed"), requested_input ("app_name_or_artifact_path" or null), and question (a string or null).', + validTargetClarification(message: string): boolean { + try { + return clarificationSchema.safeParse(JSON.parse(message)).success; + } catch (cause: unknown) { + if (cause instanceof SyntaxError) return false; + throw cause; + } + }, +} as const; + +type NativeSessionOutcome = + | "provider-unavailable" + | "opened-and-closed" + | "failed"; + +/** Expected refusal is distinct from acquisition, which always requires close. */ +const nativeSessionOutcome = ({ + reaCalls, +}: CodexAgentMetrics): NativeSessionOutcome => { + if (reaCalls.some(({ tool, error }) => error && tool !== "open_binary")) + return "failed"; + const lifecycle = reaCalls.filter( + ({ tool }) => tool === "open_binary" || tool === "close_binary", + ); + const open = lifecycle[0]; + if (open?.tool !== "open_binary") return "failed"; + if (open.error) + return lifecycle.length === 1 && open.errorCode === "provider_unavailable" + ? "provider-unavailable" + : "failed"; + return lifecycle.length === 2 && + lifecycle[1]?.tool === "close_binary" && + !lifecycle[1].error + ? "opened-and-closed" + : "failed"; +}; + +/** Routing/cleanup trial; it does not establish real-engine analysis correctness. */ +export const nativeProviderScenario = (target: string) => ({ + id: "native", + expectedFirstTool: "open_binary", + requiresEvidence: false, + requiredAnswerTermGroups: [], + approvedTools: ["open_binary", "close_binary"], + assess: nativeSessionOutcome, + prompt: `Explain what static artifact evidence establishes about the shipped native program at ${target}. Do not execute the target or install tools. If the provider is unavailable, state that limitation and that no session was opened. If a session opens, report the available evidence and close it before finishing.`, +}); diff --git a/tests/evaluation/model-evals/agentEvaluationScenarios.test.ts b/tests/evaluation/model-evals/agentEvaluationScenarios.test.ts new file mode 100644 index 0000000000..eab15fdf6c --- /dev/null +++ b/tests/evaluation/model-evals/agentEvaluationScenarios.test.ts @@ -0,0 +1,145 @@ +import { describe, expect, it } from "vitest"; + +import { evaluateCodexEvents } from "../../../src/evaluation/CodexAgentEval.js"; +import { agentEvaluationPassed } from "../../../src/evaluation/AgentEvaluationReport.js"; +import { + MISSING_TARGET_SCENARIO, + nativeProviderScenario, +} from "../../../src/evaluation/AgentEvaluationScenarios.js"; + +const call = (id: string, tool: string, errorCode?: string) => ({ + type: "item.completed", + item: { + id, + type: "mcp_tool_call", + server: "rea", + tool, + arguments: tool === "open_binary" ? { path: "/bin/true" } : {}, + status: "completed", + result: + errorCode === undefined + ? { content: [{ type: "text", text: "Operation completed." }] } + : { isError: true, structuredContent: { error: { code: errorCode } } }, + }, +}); + +const completion = (text: string) => [ + { type: "item.completed", item: { type: "agent_message", text } }, + { type: "turn.completed", usage: { input_tokens: 100, output_tokens: 30 } }, +]; + +describe("native provider scenario", () => { + const scenario = nativeProviderScenario("/bin/true"); + const opened = call("open", "open_binary"); + const closed = call("close", "close_binary"); + const unavailable = call("open", "open_binary", "provider_unavailable"); + + it.each([ + ["expected provider refusal", [unavailable], "provider-unavailable"], + ["opened session with cleanup", [opened, closed], "opened-and-closed"], + ["unclosed session", [opened], "failed"], + ["cleanup without acquisition", [unavailable, closed], "failed"], + [ + "failed cleanup", + [opened, call("close", "close_binary", "cleanup_incomplete")], + "failed", + ], + [ + "unexpected provider error", + [call("open", "open_binary", "configuration_invalid")], + "failed", + ], + [ + "invalid request", + [call("open", "open_binary", "invalid_request")], + "failed", + ], + ["no attempted open", [], "failed"], + [ + "second unclosed session", + [opened, closed, call("reopen", "open_binary")], + "failed", + ], + ] as const)( + "grades %s through the transcript parser and release gate", + (_, calls, expected) => { + const final = + expected === "provider-unavailable" + ? "The native provider is unavailable, so no analysis session was opened. Static provider evidence remains unknown; the target was not executed." + : "The returned static artifact evidence describes the selected native program. Runtime behavior remains unknown; the analysis session was closed."; + const metrics = evaluateCodexEvents( + [...calls, ...completion(final)], + scenario.expectedFirstTool, + scenario, + ); + const outcome = scenario.assess(metrics); + expect(outcome).toBe(expected); + expect( + agentEvaluationPassed({ + ...metrics, + expectedFirstTool: scenario.expectedFirstTool, + exitCode: 0, + scenarioOutcomePassed: outcome !== "failed", + }), + ).toBe(expected !== "failed"); + }, + ); +}); + +describe("missing-target scenario", () => { + const answer = JSON.stringify({ + status: "needs_target", + requested_input: "app_name_or_artifact_path", + question: "Which app name or artifact path should I inspect?", + }); + + it("advertises and grades the same JSON-only clarification grammar", () => { + expect(MISSING_TARGET_SCENARIO.prompt).toContain( + "Return only one JSON object, without prose or Markdown fences", + ); + const metrics = evaluateCodexEvents(completion(answer), null); + expect( + agentEvaluationPassed({ + ...metrics, + expectedFirstTool: null, + exitCode: 0, + targetClarificationPassed: + MISSING_TARGET_SCENARIO.validTargetClarification( + metrics.finalMessage, + ), + }), + ).toBe(true); + for (const message of [ + `Please choose an app.\n${answer}`, + `\`\`\`json\n${answer}\n\`\`\``, + answer.replace("needs_target", "analyzed"), + answer.replace("app_name_or_artifact_path", "unknown"), + JSON.stringify({ + status: "needs_target", + requested_input: "app_name_or_artifact_path", + question: " ", + }), + ]) + expect(MISSING_TARGET_SCENARIO.validTargetClarification(message)).toBe( + false, + ); + }); + + it("rejects analysis of a nearby target even with a valid final clarification", () => { + const metrics = evaluateCodexEvents( + [call("open", "open_binary"), ...completion(answer)], + null, + ); + expect( + agentEvaluationPassed({ + ...metrics, + expectedFirstTool: null, + exitCode: 0, + targetClarificationPassed: + MISSING_TARGET_SCENARIO.validTargetClarification( + metrics.finalMessage, + ), + }), + ).toBe(false); + }); +});