gensokyo-dev
aa51e43439
Use compact MCP input advertisements by default while preserving explicit full, canonical validation, output contracts and complete Evidence. Require an explicit selected catalog for runtime identity and honor both profiles in CLI status. Real ripgrep/Zellij SDK tasks, independent ELF oracle, actual client trials, profile parity workflows and CI are recorded in experiments/analysis-efficiency/mcp-input-catalog.md. SDK token estimates improve; actual Codex token savings and general latency/memory gains are not established. Refs #5861 BREAKING CHANGE: MCP input advertisements default to compact. Set REA_MCP_INPUT_SCHEMA_PROFILE=full for complete advertised guidance and nested constraints.
91 lines
4.9 KiB
Python
91 lines
4.9 KiB
Python
"""Account for catalog, complete SDK tasks and separately recorded model trials."""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import tiktoken
|
|
|
|
|
|
def compact(value):
|
|
return json.dumps(value, ensure_ascii=False, separators=(",", ":"), allow_nan=False)
|
|
|
|
|
|
def resource(case):
|
|
path = case.with_name(case.name + ".log")
|
|
scope = next(json.loads(line) for line in reversed(path.read_text().splitlines())
|
|
if line.startswith('{"resource_scope":'))
|
|
observation = scope["payload_resource_observation"]
|
|
assert scope["abort"] is None and scope["payload_resource_observation_verified"]
|
|
assert observation["payload_cleanup_verified"] and observation["payload_exit_code"] == 0
|
|
assert all(observation["resource_events"]["memory.events"][key] == 0
|
|
for key in ["high", "max", "oom", "oom_kill"])
|
|
return {"wall_seconds": scope["elapsed_seconds"],
|
|
"cpu_seconds": observation["cpu_stat"]["usage_usec"] / 1e6,
|
|
"kernel_family_peak_bytes_including_page_cache": observation["memory_peak_bytes"],
|
|
"resource_stop": scope["abort"], "cleanup_verified": True,
|
|
"final_whole_scope_counters": scope["resource_events_final"],
|
|
"guard_sources_sha256": scope["payload_observer_sources_sha256"],
|
|
"log_sha256": hashlib.sha256(path.read_bytes()).hexdigest()}
|
|
|
|
|
|
def sdk(case, count):
|
|
report = json.loads((case / "report.json").read_text())
|
|
assert report["status"] == "passed" and report["cleanup_verified"]
|
|
catalog = json.loads((case / "catalog.json").read_text())
|
|
calls = json.loads((case / "transcript.json").read_text())
|
|
inputs = [{key: tool[key] for key in ["name", "description", "inputSchema"]}
|
|
for tool in catalog["tools"]]
|
|
arguments = sum(count(compact({"name": call["name"], "arguments": call["arguments"]}))
|
|
for call in calls)
|
|
text = sum(count(block["text"]) for call in calls for block in call["reply"]["content"]
|
|
if block["type"] == "text")
|
|
return {"case": case.name, "kind": "SDK", "profile": report["profile"],
|
|
"artifact_sha256": report["artifact_sha256"],
|
|
"tool_count": len(catalog["tools"]), "calls": report["calls"],
|
|
"parent_evidence_id": report["parent_evidence_id"],
|
|
"parent_semantic_sha256": report["parent_semantic_sha256"],
|
|
"token_estimates": {"input_catalog_once": count(compact(inputs)),
|
|
"arguments": arguments, "text_replies": text,
|
|
"whole_task": count(compact(inputs)) + arguments + text,
|
|
"raw_catalog_including_output_schemas": count(compact(catalog))},
|
|
"actual_model_usage": None, **resource(case),
|
|
"receipt_sha256": {name: hashlib.sha256((case / name).read_bytes()).hexdigest()
|
|
for name in ["report.json", "catalog.json", "transcript.json"]}}
|
|
|
|
|
|
def model(case):
|
|
summary = json.loads((case / "summary.json").read_text())
|
|
assert summary["exit_code"] == 0 and not summary["timeout"]
|
|
assert summary["selected_artifact_bytes_unchanged"]
|
|
assert json.loads((case / "cleanup.json").read_text())["profile_removed"]
|
|
return {"case": case.name, "kind": "model", "client": summary["codex_version"],
|
|
"requested_model": summary["requested_model"], "reasoning": summary["reasoning"],
|
|
"model_identity_scope": summary["model_identity_scope"],
|
|
"actual_model_usage": summary["usage"],
|
|
"artifact_sha256": summary["selected_artifact_sha256_before"],
|
|
"calls": summary["completed_mcp_calls"],
|
|
"factual_correctness": "requires independent oracle and answer review",
|
|
**resource(case),
|
|
"receipt_sha256": {name: hashlib.sha256((case / name).read_bytes()).hexdigest()
|
|
for name in ["summary.json", "answer.txt", "events.jsonl",
|
|
"trial-settings.json", "context-receipt.json"]}}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
parser.add_argument("cases", type=Path, nargs="+")
|
|
args = parser.parse_args()
|
|
encoding = tiktoken.get_encoding("cl100k_base")
|
|
count = lambda value: len(encoding.encode(value, disallowed_special=()))
|
|
result = {"tokenizer": f"tiktoken {tiktoken.__version__}", "encoding": encoding.name,
|
|
"estimate_scope": "One complete name/description/input-schema catalog, every RPC argument and text reply; raw output-schema transport counted separately. Model usage is separate and is not inferred from estimates.",
|
|
"records": [sdk(case, count) if (case / "report.json").exists() else model(case)
|
|
for case in args.cases]}
|
|
args.output.write_text(json.dumps(result, indent=2) + "\n")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|