gensokyo-dev
0583fc69be
Select completed source-to-bundle mappings and candidates through the shared view owner, preserve reference-file provenance, and retain full Evidence. Reuse bounded JSON reading, heap admission, and cancellation before session writes. Verified public Redux Toolkit source/map facts with an independent oracle, equivalent CLI/MCP workflows, focused negative controls, and complete CI/review. Historical paired SDK context estimates fall about 64%; timing/resource tradeoffs and final verification receipts are recorded without a general speedup or actual model-token claim. Fixes #6112.
35 lines
3.4 KiB
Python
35 lines
3.4 KiB
Python
import argparse, json, statistics
|
|
from pathlib import Path
|
|
import tiktoken
|
|
parser=argparse.ArgumentParser()
|
|
parser.add_argument('root', type=Path)
|
|
parser.add_argument('--pattern', default='paired-*')
|
|
parser.add_argument('--output', type=Path)
|
|
args=parser.parse_args()
|
|
root=args.root
|
|
enc=tiktoken.get_encoding('cl100k_base')
|
|
compact=lambda v: json.dumps(v,ensure_ascii=False,separators=(',',':'))
|
|
count=lambda v: len(enc.encode(v,disallowed_special=()))
|
|
results=[]
|
|
for case in sorted(root.glob(args.pattern)):
|
|
if not case.is_dir(): continue
|
|
report=json.loads((case/'report.json').read_text())
|
|
assert report['status']=='observed' and report['cleanup_verified']
|
|
scope=next(json.loads(line) for line in reversed(case.with_name(case.name+'.log').read_text().splitlines()) if line.startswith('{"resource_scope":'))
|
|
observed=scope['payload_resource_observation']
|
|
assert scope['abort'] is None and scope['payload_resource_observation_verified'] and observed['payload_cleanup_verified'] and observed['payload_exit_code']==0
|
|
catalog=json.loads((case/'catalog.json').read_text())
|
|
transcript=json.loads((case/'transcript.json').read_text())
|
|
inputs=[{k: t[k] for k in ['name','description','inputSchema']} for t in catalog['tools']]
|
|
arg=sum(count(compact({'name':r['name'],'arguments':r['arguments']})) for r in transcript)
|
|
text=sum(count(b['text']) for r in transcript for b in r['reply']['content'] if b['type']=='text')
|
|
results.append({'case':case.name,'mode':report['mode'],'output_schema_profile':report.get('output_schema_profile','full'),'runtime_revision':report['runtime_revision'],'runtime_worktree_status':report['runtime_worktree_status'],'token_estimates':{'input_catalog_once':count(compact(inputs)),'arguments':arg,'text_replies':text,'whole_task':count(compact(inputs))+arg+text},'actual_model_usage':None,'wall_seconds':scope['elapsed_seconds'],'cpu_seconds':observed['cpu_stat']['usage_usec']/1e6,'peak_process_family_bytes_including_page_cache':observed['memory_peak_bytes'],'calls':report['calls'],'retained_records':report['retained_records'],'proof':json.loads((case/'independent-proof.json').read_text()),'resource_events':observed['resource_events'],'cleanup_verified':True})
|
|
summary={}
|
|
for mode in ['complete','selected']:
|
|
rows=[r for r in results if r['mode']==mode]
|
|
summary[mode]={key:statistics.median(r[key] for r in rows) for key in ['wall_seconds','cpu_seconds','peak_process_family_bytes_including_page_cache']}
|
|
summary[mode]['whole_task_tokens_estimated']=[r['token_estimates']['whole_task'] for r in rows]
|
|
out={'status':'passed','tokenizer':'cl100k_base, tiktoken 0.12.0; estimates only; one input catalog plus arguments plus MCP text replies; not actual model usage','cold_warm':'Fresh server process and isolated HOME each run; alternating complete/selected, selected/complete, complete/selected. OS page caches are warm/uncontrolled; not an OS-cold claim.','tarball_sha256':'f2517c466b50424b661a27e48039f15db7783e5822f46b5c7d39f20104ac67a9','preparation_scope':'Reference-source import is performed once outside these repeated SDK trials and reported separately. Both workflows use the explicit existing semantic cap of one for the static correspondence question; full semantic coverage is not claimed.','summary':summary,'runs':results}
|
|
(args.output or root/'costs.json').write_text(json.dumps(out,indent=2)+'\n')
|
|
print(json.dumps(summary,indent=2))
|