gensokyo-dev
4bfdadb7de
Add opt-in Frida session status summaries linked to complete retained Evidence. Share canonical session facts across contracts and preserve observed state, source identity and capture truncation. Validated with native SDK/client workflows, default delivery compatibility, lifecycle and provenance tests. The completed large-history task uses 50.9% fewer estimated request tokens; no general CPU or latency improvement is claimed. Fixes #5640
82 lines
6.0 KiB
Python
82 lines
6.0 KiB
Python
"""Count complete actual-client requests, not only smaller tool-reply bytes."""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import tiktoken
|
|
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--lab', type=Path, required=True)
|
|
parser.add_argument('--output', type=Path, required=True)
|
|
args = parser.parse_args()
|
|
enc = tiktoken.get_encoding('cl100k_base')
|
|
|
|
|
|
def tokens(value):
|
|
return len(enc.encode(json.dumps(value, ensure_ascii=False, separators=(',', ':'), allow_nan=False), disallowed_special=()))
|
|
|
|
|
|
def resources(root):
|
|
receipt = json.loads(Path(str(root) + '.log').read_text().splitlines()[-1])
|
|
payload = receipt['payload_resource_observation']
|
|
assert receipt['abort'] is None and receipt['payload_resource_observation_verified']
|
|
assert payload['payload_exit_code'] == 0 and payload['payload_cleanup_verified']
|
|
for events in payload['resource_events'].values():
|
|
assert not any(events.values())
|
|
return {'wall_seconds': receipt['elapsed_seconds'], 'cpu_seconds': payload['cpu_stat']['usage_usec'] / 1e6,
|
|
'family_peak_mib': payload['memory_peak_bytes'] / 1048576,
|
|
'payload_cleanup_verified': True, 'final_whole_scope_counters_verified': receipt['resource_counters_verified']}
|
|
|
|
|
|
wire, sdk = [], []
|
|
for root in sorted(args.lab.glob('wire-v2-*')):
|
|
if not root.is_dir():
|
|
continue
|
|
report = json.loads((root / 'report.json').read_text())
|
|
assert report['status'] in ['passed', 'expected-incomplete-task'] and not report['cleanup_failures']
|
|
assert report['appserver_exit_code'] == 0 and report['owned_scope_drain']['remaining_processes'] == []
|
|
requests = json.loads((root / 'model-requests.json').read_text())
|
|
details = []
|
|
for captured in requests:
|
|
body = captured['body']
|
|
namespace = [n for n in body.get('tools', []) if n.get('type') == 'namespace' and n.get('name') == 'mcp__rea_audit']
|
|
assert len(namespace) == 1 and len(namespace[0]['tools']) == 211, 'Unexpected compaction/catalog: audit before comparing'
|
|
details.append({'whole_json_tokens_proxy': tokens(body), 'rea_namespace_tokens_proxy': tokens(namespace[0]), 'rea_functions': 211})
|
|
wire.append({'case': root.name, 'task_completed': report['task_completed'], 'mode': report['mode'],
|
|
'budget': report['observations']['tool_output_budget'], 'requests': len(details), 'compactions': report['compaction_requests'],
|
|
'whole_json_tokens_proxy': sum(r['whole_json_tokens_proxy'] for r in details), 'request_details': details, **resources(root)})
|
|
for root in sorted(args.lab.glob('sdk-*')):
|
|
if not root.is_dir():
|
|
continue
|
|
report = json.loads((root / 'report.json').read_text())
|
|
assert report['status'] == 'passed' and not report['cleanup_failures'] and not report['receipt_failures']
|
|
transcript = json.loads((root / 'transcript.json').read_text())
|
|
catalog = json.loads((root / 'catalog.json').read_text())
|
|
assert len(catalog['tools']) == 211 and report['original_target_exited']
|
|
sdk.append({'case': root.name, 'detail': report['detail'], 'capture': report['capture'], 'tool_calls': len(report['calls']),
|
|
'exported_bundle_bytes': report['exported_bundle_bytes'], 'answer': report['answer'],
|
|
'status_calls': [c for c in report['calls'] if c['name'] == 'frida_session_status'],
|
|
'one_catalog_plus_all_sdk_calls_tokens_proxy': tokens(catalog) + sum(tokens({'name': t['name'], 'input': t['input'], 'reply': t['reply']}) for t in transcript), **resources(root)})
|
|
assert len(sdk) == len(wire) == 4, 'A partial cohort cannot qualify a comparison'
|
|
for capture in ['small', 'libc']:
|
|
pair = [r for r in sdk if r['capture'] == capture]
|
|
assert len(pair) == 2 and {r['detail'] for r in pair} == {'complete', 'summary'}
|
|
assert {k: v for k, v in pair[0]['answer'].items() if k != 'pid'} == {k: v for k, v in pair[1]['answer'].items() if k != 'pid'}
|
|
reports = [json.loads((args.lab / r['case'] / 'report.json').read_text()) for r in pair]
|
|
assert reports[0]['load']['source_sha256'] == reports[1]['load']['source_sha256']
|
|
completed = [r for r in wire if r['budget'] == 300000]
|
|
assert len(completed) == 2 and all(r['task_completed'] for r in completed)
|
|
answers = [json.loads((args.lab / r['case'] / 'report.json').read_text())['observations']['answers'] for r in completed]
|
|
comparable_answers = [[{k: v for k, v in a.items() if k != 'parent_evidence_id'} for a in run] for run in answers]
|
|
assert comparable_answers[0] == comparable_answers[1]
|
|
result = {
|
|
'scope': 'Real local Frida binding with controlled sleep target and caller-authored libc export inventory. Actual Codex0.163 app-server uses a deterministic local SSE fixture: no AI inference, model usage, quality or billing claim. Whole compact JSON token estimates include all 211 schemas and every request replay. Incomplete default baseline is excluded from completed-task gains. SDK exchange estimates are not model-input measurements. Fresh processes have unflushed OS caches; both status reads query live session observations. Timings include the eight-second target lifetime and cleanup; single pairs establish no general speedup. Payload-exit cgroup counters are verified; final whole-scope counters may be unavailable.',
|
|
'tokenizer': {'package': tiktoken.__version__, 'encoding': enc.name},
|
|
'runtime_pins': {v: {k: x for k, x in json.loads((args.lab / (v + '-manifest.json')).read_text()).items() if k != 'files'} for v in ['baseline', 'candidate']},
|
|
'native': json.loads((args.lab / 'native-prerequisite-manifest.json').read_text()),
|
|
'script_pins': {p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in Path(__file__).parent.glob('frida*') if p.suffix in ['.py', '.mjs']},
|
|
'wire': wire, 'sdk': sdk,
|
|
}
|
|
args.output.write_text(json.dumps(result, indent=2) + '\n')
|
|
print(json.dumps({'wire': [{k: v for k, v in r.items() if k != 'request_details'} for r in wire], 'sdk': [{k: v for k, v in r.items() if k not in ['status_calls', 'answer']} for r in sdk]}, indent=2))
|