diff --git a/experiments/atomic-codemode/summarize-trials.py b/experiments/atomic-codemode/summarize-trials.py new file mode 100644 index 000000000..9dae1cc52 --- /dev/null +++ b/experiments/atomic-codemode/summarize-trials.py @@ -0,0 +1,76 @@ +"""Summarize actual model usage and owned process-family counters, without grading answers.""" + +import argparse +import collections +import json +from pathlib import Path + + +def summarize(directory): + summary = json.loads((directory / 'summary.json').read_text()) + resource_records = [json.loads(line) for line in directory.with_suffix('.log').read_text().splitlines() + if line.startswith('{"resource_scope":')] + if len(resource_records) != 1: + raise ValueError(f'{directory}: expected one process-family resource receipt') + resources = resource_records[0] + observation = resources['payload_resource_observation'] + if observation['phase'] != 'payload-exited-observer-alive': + raise ValueError(f'{directory}: payload-exit counters were not observed') + usage = collections.Counter() + for turn in summary['usage']: + usage.update(turn) + calls = summary['completed_mcp_calls'] # This runner key also retains failed attempts. + statuses = collections.Counter(call['status'] for call in calls) + methods = collections.Counter() + source_ids = set() + api_failures = [] + trace_path = directory.with_suffix('.trace.jsonl') + if trace_path.exists(): + for line in trace_path.read_text().splitlines(): + entry = json.loads(line) + result = entry['result'] + if entry['tool'] == 'execute_javascript': + for call in result.get('calls', []): + methods[call['method']] += 1 + if call['status'] != 'ok': + api_failures.append(call) + source_ids.update(result.get('evidence_ids', [])) + usage['uncached_input_tokens'] = usage['input_tokens'] - usage['cached_input_tokens'] + cleanup = json.loads((directory / 'cleanup.json').read_text()) + return { + 'trial': directory.name, + 'runtime_revision': summary['runtime_revision'], + 'requested_model': summary['requested_model'], + 'reasoning': summary['reasoning'], + 'usage': dict(usage), + 'outer_attempts': len(calls), + 'outer_statuses': dict(statuses), + 'outer_failures': [{k: call[k] for k in ('tool', 'status', 'error')} + for call in calls if call['status'] != 'completed'], + 'inner_code_mode_calls': dict(methods), + 'inner_api_failures': api_failures, + 'code_mode_evidence_count': len(source_ids), + 'model_wall_seconds': summary['wall_seconds'], + 'family_wall_seconds': resources['elapsed_seconds'], + 'family_cpu_seconds': observation['cpu_stat']['usage_usec'] / 1_000_000, + 'family_memory_peak_bytes': observation['memory_peak_bytes'], + 'memory_events': observation['resource_events']['memory.events'], + 'resource_abort': resources['abort'], + 'payload_cleanup_verified': observation['payload_cleanup_verified'], + 'profile_cleanup': cleanup, + 'target_sha256': summary['selected_artifact_sha256_before'], + 'target_unchanged': summary['selected_artifact_bytes_unchanged'], + 'correctness': 'requires separate answer/source-oracle review', + 'scope': 'Actual cumulative usage includes discovery/retries; cached input is included in input, reasoning in output. Memory is kernel cgroup peak including charged file cache. Server-side compute and resolved model build are unmeasured.', + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('directories', type=Path, nargs='+') + options = parser.parse_args() + print(json.dumps([summarize(path) for path in options.directories], indent=2)) + + +if __name__ == '__main__': + main() diff --git a/experiments/atomic-codemode/trials.md b/experiments/atomic-codemode/trials.md index 9f80fd64d..d124a1035 100644 --- a/experiments/atomic-codemode/trials.md +++ b/experiments/atomic-codemode/trials.md @@ -58,5 +58,57 @@ Private captures remain under `.cache/atomic-codemode/zlib-codemode-v1/`, with trial settings and cleanup proof. The adjacent trace retains actual model code and inner API calls; the `*-evidence/` directory retains the three complete native sources and their identity/digest metadata. Credentials and raw captures are not -committed. Direct atomic, revised Code Mode and traditional-provider trials have -not yet established an equivalent completed-task comparison. +committed. + +The second round used the pinned `c5cefd622` runtime, the same target, question, +model alias/reasoning and client version, with the empty model profiles outside +the repository under `/tmp`. Both atomic modes use the same eight fact methods; +only the calling adapter changes. The traditional comparison forces the real +Kuna provider and selects nine relevant production tools in the client: +`open_binary`, `close_binary`, `binary_session`, `search_strings`, `xrefs`, +`procedure_address`, `procedure_pseudo_code`, `procedure_assembly`, +`analyze_function`. The production catalog remains complete. This is a favorable +focused baseline, not the default full-catalog experience. + +| Measurement | Atomic direct v2 | Atomic + Code Mode v2 | Traditional v2 | +| --- | ---: | ---: | ---: | +| Outer MCP attempts, including failures | 27 | 12 | 9 | +| Completed / failed outer attempts | 27 / 0 | 12 / 0 | 7 / 2 | +| Internal Code Mode API reads | — | 21 | — | +| Cumulative model input tokens | 590,872 | 453,734 | 344,105 | +| Cached input tokens (included) | 496,512 | 401,664 | 297,344 | +| Uncached input tokens | 94,360 | 52,070 | 46,761 | +| Output tokens | 5,874 | 4,738 | 3,076 | +| Reasoning tokens (included in output) | 2,228 | 1,716 | 832 | +| Model runner wall seconds | 170.813 | 148.397 | 106.352 | +| Bounded family elapsed seconds | 171.801 | 149.585 | 106.747 | +| Bounded family CPU seconds | 11.898 | 22.129 | 20.666 | +| Kernel process-family memory peak MiB | 578.2 | 544.3 | 843.4 | +| OOM kills | 0 | 0 | 0 | + +The traditional trace includes a client approval-policy rejection for +`binary_session` and a failed name lookup for `inflate`; the model recovered +through string/xref/address discovery. Both attempts and their costs remain in +the comparison. A completed transport call does not itself establish a valid +analysis result; answers and the retained calls were reviewed separately. + +All three answers identify the same two local error branches and modulo-31 +condition, with supporting addresses and limitations consistent with the +independent instruction/arithmetic checks. This establishes a completed local +task comparison. It does not prove global runtime reachability or correctness +on arbitrary binaries. All target digests remained unchanged, temporary model +profiles were removed, and observed payload cleanup succeeded. + +Code Mode reduces direct atomic round trips on this trial, but traditional tools +win both cumulative input tokens and wall time. The prototype still makes the +model assemble incomplete CFG/phi/value relationships and repeatedly inspect +operations or C. Fixing the arbitrary C page cap reduced internal C reads from +16 to 2; the overall first-to-second trial changes are not a controlled causal +measurement because return contracts and profile placement changed too. + +These are single stochastic samples, with fresh provider/session caches, warm +and uncontrolled OS file caches, and actual model prompt-cache usage reported. +No warm-session paired reruns, confidence interval, server-side model compute or +pricing was measured. Reproduce the ledger with `summarize-trials.py` over the +private trial directories; it reads actual model usage and the payload-exit +cgroup receipt and deliberately does not grade answers.