diff --git a/experiments/analysis-efficiency/agent-trial.py b/experiments/analysis-efficiency/agent-trial.py index 824cb43dab..8fbd1eff1a 100644 --- a/experiments/analysis-efficiency/agent-trial.py +++ b/experiments/analysis-efficiency/agent-trial.py @@ -99,6 +99,8 @@ def main(): help='Opt-in exact-name discovery guidance for selected REA tools') parser.add_argument('--normalize-mcp-guidance', action='store_true', help='Opt-in common envelope/stopping guidance, identical across protocol lanes') + parser.add_argument('--guide-long-waits', action='store_true', + help='Opt-in common client wait guidance; does not change analysis contracts or deadlines') args = parser.parse_args() if args.guide_enabled_tools and args.enabled_tools is None: parser.error('--guide-enabled-tools requires --enabled-tools') @@ -235,6 +237,15 @@ def main(): 'return only their metadata. Preserve the original analyst question, ' 'dependency order and source Evidence.' ) + if args.guide_long_waits: + instruction += ( + ' When the client executor reports a running cell during a long analysis, ' + 'resume that same cell with wait and yield_time_ms:30000, within the ' + 'advertised tool schema, rather than repeated one-second polls. ' + 'The wait may return early when the result is ready. Do not sleep or ' + 'start duplicate analysis calls just to check completion. Keep the ' + 'original analysis dependencies, evidence and stopping requirements.' + ) quote = json.dumps config = '\n'.join([ 'approval_policy = "never"', 'sandbox_mode = "read-only"', @@ -290,6 +301,7 @@ def main(): 'capture_tool_context': args.capture_tool_context, 'guide_enabled_tools': args.guide_enabled_tools, 'normalize_mcp_guidance': args.normalize_mcp_guidance, + 'guide_long_waits': args.guide_long_waits, 'driver_commit': subprocess.check_output( ['git', 'rev-parse', 'HEAD'], cwd=driver_repository, text=True).strip(), 'runtime_revision': args.revision, diff --git a/experiments/atomic-codemode/client-wait-study.md b/experiments/atomic-codemode/client-wait-study.md new file mode 100644 index 0000000000..f454c3a2aa --- /dev/null +++ b/experiments/atomic-codemode/client-wait-study.md @@ -0,0 +1,51 @@ +# Client waiting and whole-task cost + +The [selected-case pair](switch-study.md) exposes one-second client polls after +the snapshot execution yields. Direct makes six waits (three one-second then +three ten-second requests); Code Mode makes three one-second requests. Five +direct and two Code Mode wait responses still report a running cell. Initial +execution yields after about 31 seconds in both lanes. This is observed client +behavior, separate from MCP attempts and native source-capture time. + +Native latency remains variable, including in sequential scripted MCP runs +without models. The [overlap canary](overlap-study.md) did not establish a cause. +No analysis acceleration or cache invalidation change is proposed here. + +## Frozen diagnostic + +The opt-in runner flag `--guide-long-waits` appends exactly the same client wait +policy in both lanes: resume an already-running cell with a 30,000 ms wait, +respect the advertised schema, and do not sleep or duplicate analysis merely +to check progress. It changes neither the initial execution yield policy nor +the provider/Code Mode deadlines. It supplies no analysis facts. The policy, +flag and instruction digest are recorded in each trial's settings. Older runs +without this flag remain a separate stratum. + +Run Code Mode first, then direct, with the unchanged +[question](sqlite-switch-question.txt) and [oracle](switch-oracle.md), final +`a9ca8a8a...` engine, complete four/eighteen-tool catalogs and the same exact +discovery/envelope guidance as the first pair. Use fresh leases and isolated +model workspaces, requested `gpt-6.1-sol/high`, Codex 0.161.0 and the same one-CPU, +4 GiB hard / 3 GiB early-stop resource policy. No runtime edits during the pair. +Local records use `sqlite-switch-{codemode,direct}-v2*`. + +Review complete answers against the frozen independent machine oracle and +retained source, including selector widths, all requested cases, signed target +arithmetic, case/edge identities, native default versus machine out-of-range +guard, provenance and unknown runtime behavior. Require one VM capture and +focused reads; a lower token count without a correct answer is not a win. + +Record actual wait arguments and running/completed responses, model responses, +all input/cached/uncached/output/reasoning tokens, MCP/API attempts, wall time, +family CPU, kernel family peak, native source-capture time, identity/freshness +and cleanup/OOM status. Count discovery, guidance and unsuccessful waits in +whole-task usage. Fewer polls would establish adherence in this client, not +causally lower tokens or native latency. A before/after pair remains subject to +model choices, prompt caches, host contention and run order; do not subtract +native or polling intervals to manufacture an adapter speedup. + +The [official Codex CLI reference](https://learn.chatgpt.com/docs/cli/reference) +documents client configuration but does not establish the `wait` contract used +here. The actual captured client calls and subsequent run establish behavior; +no configuration feature flag is treated as proof of the available surface. +The native/API contracts are unchanged. Results are pending.