lee/rea
1
0 Fork 0
mirror of https://github.com/morluto/rea.git synced 2026-10-23 15:12:06 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
fb6785dcafe5d4ab59eaabf503801d8e8a9767fa
rea/experiments/analysis-efficiency/retained-investigation-results.json
T
Ysopwnablegensokyo-devKaoru0822
 47d3c6b1a1 test(perf): verify authentic retained investigation queries (#4814) 
Verify retained queries against authentic SQLite ELF and npm histories, retain failed RPC/cleanup/output costs, and make mixed-history verification repeatable. Record the separate validation-reuse bottleneck in #4810.

Co-authored-by: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com>
Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
2026-10-18 02:32:58 +00:00

636 lines
19 KiB
JSON
Raw Blame History

{
"runtime_revision": "f4322b15ed1423d736bd2572cf7a8eadf2b466e1",
"host": "Linux x86-64",
"node": "24.18.0",
"npm": "11.16.0",
"codex_cli": "0.161.0",
"requested_model": "gpt-6.1-sol",
"reasoning": "high",
"resolved_model_identity": "unknown",
"code_mode": false,
"client_enabled_tools": [
"import_evidence_bundle",
"get_evidence_bundle",
"inspect_analysis_view",
"list_unknowns"
],
"catalog": {
"tools": 207,
"serialized_bytes": 3152125,
"sha256": "4f31d9d56c400bc48f5639a9dc4b4cdad7faf862e4be81641d94fa70552438e4"
},
"bounds": {
"cpu_quota_percent": 100,
"family_hard_mib": 8192,
"early_stop_mib": 7168,
"swap": false,
"tasks": 192
},
"counter_scope": "Payload-exit counters with observer alive; kernel family peaks include file cache; terminal whole-scope counters and remote model compute unknown",
"sources": {
"native": {
"target_sha256": "2e6eef9a727f081f0d453b4e5e6cbd8b9ef8b6f86cbf7681cbad444d3b0b55c8",
"sections": 26,
"symbols": 1479,
"relocations": 2963,
"all_symbol_locations_names_values_agree_with_independent_ELF_parser": true
},
"javascript": {
"package": "npm",
"version": "11.16.0",
"source_files": [
{
"path": "package.json",
"sha256": "60ff022e32b5d8291d3e09849fd0cee6bbbff4e4d8352ad258214b1adf070714"
},
{
"path": "bin/npm-cli.js",
"sha256": "8e5f6f3429f8cdbe693cdc29904e9d5a7b127a494bd15c804bd54c7403bfcbe7"
},
{
"path": "lib/cli.js",
"sha256": "67666f06479f9b0bbc01412c198caadd4287d34d5ed74a02871ea78f186451e7"
},
{
"path": "lib/cli/entry.js",
"sha256": "667a44c1fa8795fdbba8ad843736ca7f8f24e64511985e9a7060b002123560c6"
},
{
"path": "lib/commands/install.js",
"sha256": "f7a60e8b40ab1bece76a2441b8b15378d199f05c6050eebe455cd9961f67b816"
}
],
"lib_cli_module": {
"node_id": "jag_node_131257ad52f3333971c2a5ed325c64357e52604489e42ccc67a8849770cb5b10",
"kind": "javascript-module",
"path": "lib/cli.js"
},
"source_bytes_checked": true,
"inventory_completeness": "producer-relative, not independently established",
"unselected_sources": 1416
}
},
"sdk": [
{
"name": "sdk-native-complete-1",
"status": "passed",
"answer_sha256": "4e28c2c2a25620576d27abe846a05835ec49386b889779e33410c31971523b54",
"query_ms": 1328.7,
"query_reply_bytes": 11224227,
"rpc_count": 7,
"elapsed_seconds": 14.117,
"cpu_seconds": 13.915,
"peak_family_mib": 1160.8,
"memory_events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0
},
"payload_cleanup_verified": true,
"response_capacity_refused": false,
"warm_page_ms": []
},
{
"name": "sdk-native-complete-2",
"status": "passed",
"answer_sha256": "4e28c2c2a25620576d27abe846a05835ec49386b889779e33410c31971523b54",
"query_ms": 1385.4,
"query_reply_bytes": 11224227,
"rpc_count": 7,
"elapsed_seconds": 14.722,
"cpu_seconds": 14.485,
"peak_family_mib": 1109.9,
"memory_events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0
},
"payload_cleanup_verified": true,
"response_capacity_refused": false,
"warm_page_ms": []
},
{
"name": "sdk-native-focused-1",
"status": "passed",
"answer_sha256": "4e28c2c2a25620576d27abe846a05835ec49386b889779e33410c31971523b54",
"query_ms": 529.7,
"query_reply_bytes": 167985,
"rpc_count": 25,
"elapsed_seconds": 12.907,
"cpu_seconds": 12.679,
"peak_family_mib": 934.8,
"memory_events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0
},
"payload_cleanup_verified": true,
"response_capacity_refused": false,
"warm_page_ms": [44.1, 59.6, 24.0, 30.8, 26.7, 47.4]
},
{
"name": "sdk-native-focused-2",
"status": "passed",
"answer_sha256": "4e28c2c2a25620576d27abe846a05835ec49386b889779e33410c31971523b54",
"query_ms": 572.2,
"query_reply_bytes": 167985,
"rpc_count": 25,
"elapsed_seconds": 13.316,
"cpu_seconds": 13.116,
"peak_family_mib": 1000.0,
"memory_events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0
},
"payload_cleanup_verified": true,
"response_capacity_refused": false,
"warm_page_ms": [44.0, 64.6, 26.8, 33.2, 29.8, 50.4]
},
{
"name": "sdk-javascript-complete-1",
"status": "passed",
"answer_sha256": "0fe3ad249eb1a7233a4966a56652c173667c48fdaf395b6c08fc6901ef3b46f7",
"query_ms": 11241.5,
"query_reply_bytes": 212856,
"rpc_count": 31,
"elapsed_seconds": 26.3,
"cpu_seconds": 26.1,
"peak_family_mib": 1333.5,
"memory_events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0
},
"payload_cleanup_verified": true,
"response_capacity_refused": true,
"warm_page_ms": [1124.6, 1073.8, 1474.7, 1074.7, 1081.6, 1109.5]
},
{
"name": "sdk-javascript-complete-2",
"status": "passed",
"answer_sha256": "0fe3ad249eb1a7233a4966a56652c173667c48fdaf395b6c08fc6901ef3b46f7",
"query_ms": 11938.1,
"query_reply_bytes": 212856,
"rpc_count": 31,
"elapsed_seconds": 27.105,
"cpu_seconds": 26.87,
"peak_family_mib": 1320.9,
"memory_events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0
},
"payload_cleanup_verified": true,
"response_capacity_refused": true,
"warm_page_ms": [1155.4, 1096.2, 1489.3, 1121.3, 1088.3, 1484.6]
},
{
"name": "sdk-javascript-focused-2",
"status": "passed",
"answer_sha256": "0fe3ad249eb1a7233a4966a56652c173667c48fdaf395b6c08fc6901ef3b46f7",
"query_ms": 10967.3,
"query_reply_bytes": 208350,
"rpc_count": 28,
"elapsed_seconds": 25.708,
"cpu_seconds": 25.505,
"peak_family_mib": 1344.0,
"memory_events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0
},
"payload_cleanup_verified": true,
"response_capacity_refused": false,
"warm_page_ms": [1342.7, 1085.1, 1055.0, 1034.6, 1082.7, 1440.7]
},
{
"name": "sdk-javascript-focused-3",
"status": "passed",
"answer_sha256": "0fe3ad249eb1a7233a4966a56652c173667c48fdaf395b6c08fc6901ef3b46f7",
"query_ms": 10990.4,
"query_reply_bytes": 208350,
"rpc_count": 28,
"elapsed_seconds": 26.197,
"cpu_seconds": 25.98,
"peak_family_mib": 1336.0,
"memory_events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0
},
"payload_cleanup_verified": true,
"response_capacity_refused": false,
"warm_page_ms": [1338.1, 1088.5, 1069.1, 1038.3, 1087.5, 1483.0]
},
{
"name": "sdk-mixed-native",
"status": "passed",
"answer_sha256": "4e28c2c2a25620576d27abe846a05835ec49386b889779e33410c31971523b54",
"query_ms": 560.0,
"query_reply_bytes": 167991,
"rpc_count": 25,
"elapsed_seconds": 16.933,
"cpu_seconds": 16.714,
"peak_family_mib": 1261.4,
"memory_events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0
},
"payload_cleanup_verified": true,
"response_capacity_refused": false,
"warm_page_ms": [46.9, 51.0, 36.3, 24.4, 38.6, 43.8]
},
{
"name": "sdk-mixed-javascript",
"status": "passed",
"answer_sha256": "0fe3ad249eb1a7233a4966a56652c173667c48fdaf395b6c08fc6901ef3b46f7",
"query_ms": 10904.5,
"query_reply_bytes": 208356,
"rpc_count": 28,
"elapsed_seconds": 26.5,
"cpu_seconds": 26.297,
"peak_family_mib": 1346.5,
"memory_events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0
},
"payload_cleanup_verified": true,
"response_capacity_refused": false,
"warm_page_ms": [1321.9, 1063.3, 1070.8, 1309.9, 1041.7, 1074.2]
}
],
"model_trials": [
{
"name": "model-javascript-complete-1",
"facts_pass": true,
"elapsed_seconds": 76.49,
"cpu_seconds": 24.687,
"peak_family_mib": 818.7,
"input_tokens": 454491,
"cached_input_tokens": 396288,
"uncached_input_tokens": 58203,
"output_tokens": 2033,
"reported_reasoning_output_tokens": 207,
"mcp_calls_including_discovery": 9,
"payload_cleanup_verified": true,
"memory_events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0
}
},
{
"name": "model-javascript-complete-2",
"facts_pass": true,
"elapsed_seconds": 58.335,
"cpu_seconds": 14.993,
"peak_family_mib": 971.4,
"input_tokens": 328066,
"cached_input_tokens": 280704,
"uncached_input_tokens": 47362,
"output_tokens": 1676,
"reported_reasoning_output_tokens": 129,
"mcp_calls_including_discovery": 9,
"payload_cleanup_verified": true,
"memory_events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0
}
},
{
"name": "model-javascript-focused-1",
"facts_pass": false,
"elapsed_seconds": 34.206,
"cpu_seconds": 16.31,
"peak_family_mib": 811.9,
"input_tokens": 167470,
"cached_input_tokens": 136320,
"uncached_input_tokens": 31150,
"output_tokens": 633,
"reported_reasoning_output_tokens": 172,
"mcp_calls_including_discovery": 4,
"payload_cleanup_verified": true,
"memory_events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0
}
},
{
"name": "model-javascript-focused-2",
"facts_pass": true,
"elapsed_seconds": 66.455,
"cpu_seconds": 14.337,
"peak_family_mib": 955.3,
"input_tokens": 339528,
"cached_input_tokens": 307456,
"uncached_input_tokens": 32072,
"output_tokens": 2051,
"reported_reasoning_output_tokens": 169,
"mcp_calls_including_discovery": 10,
"payload_cleanup_verified": true,
"memory_events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0
}
},
{
"name": "model-javascript-guided-complete-1",
"facts_pass": true,
"elapsed_seconds": 69.206,
"cpu_seconds": 13.65,
"peak_family_mib": 888.8,
"input_tokens": 168862,
"cached_input_tokens": 121728,
"uncached_input_tokens": 47134,
"output_tokens": 2385,
"reported_reasoning_output_tokens": 737,
"mcp_calls_including_discovery": 6,
"payload_cleanup_verified": true,
"memory_events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0
}
},
{
"name": "model-javascript-guided-complete-2",
"facts_pass": false,
"elapsed_seconds": 29.187,
"cpu_seconds": 15.105,
"peak_family_mib": 697.4,
"input_tokens": 28696,
"cached_input_tokens": 23296,
"uncached_input_tokens": 5400,
"output_tokens": 610,
"reported_reasoning_output_tokens": 379,
"mcp_calls_including_discovery": 0,
"payload_cleanup_verified": true,
"memory_events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0
}
},
{
"name": "model-javascript-guided-focused-1",
"facts_pass": true,
"elapsed_seconds": 53.464,
"cpu_seconds": 12.059,
"peak_family_mib": 944.5,
"input_tokens": 124218,
"cached_input_tokens": 92416,
"uncached_input_tokens": 31802,
"output_tokens": 1746,
"reported_reasoning_output_tokens": 330,
"mcp_calls_including_discovery": 5,
"payload_cleanup_verified": true,
"memory_events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0
}
},
{
"name": "model-javascript-guided-focused-2",
"facts_pass": true,
"elapsed_seconds": 58.526,
"cpu_seconds": 22.852,
"peak_family_mib": 924.2,
"input_tokens": 129134,
"cached_input_tokens": 97536,
"uncached_input_tokens": 31598,
"output_tokens": 1478,
"reported_reasoning_output_tokens": 69,
"mcp_calls_including_discovery": 5,
"payload_cleanup_verified": true,
"memory_events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0
}
},
{
"name": "model-native-complete-1",
"facts_pass": true,
"elapsed_seconds": 52.103,
"cpu_seconds": 8.686,
"peak_family_mib": 1028.5,
"input_tokens": 287913,
"cached_input_tokens": 249984,
"uncached_input_tokens": 37929,
"output_tokens": 1724,
"reported_reasoning_output_tokens": 121,
"mcp_calls_including_discovery": 4,
"payload_cleanup_verified": true,
"memory_events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0
}
},
{
"name": "model-native-complete-2",
"facts_pass": true,
"elapsed_seconds": 58.261,
"cpu_seconds": 9.102,
"peak_family_mib": 837.4,
"input_tokens": 348976,
"cached_input_tokens": 286976,
"uncached_input_tokens": 62000,
"output_tokens": 2009,
"reported_reasoning_output_tokens": 246,
"mcp_calls_including_discovery": 5,
"payload_cleanup_verified": true,
"memory_events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0
}
},
{
"name": "model-native-focused-1",
"facts_pass": true,
"elapsed_seconds": 62.078,
"cpu_seconds": 10.198,
"peak_family_mib": 823.9,
"input_tokens": 335445,
"cached_input_tokens": 293120,
"uncached_input_tokens": 42325,
"output_tokens": 1924,
"reported_reasoning_output_tokens": 87,
"mcp_calls_including_discovery": 8,
"payload_cleanup_verified": true,
"memory_events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0
}
},
{
"name": "model-native-focused-2",
"facts_pass": true,
"elapsed_seconds": 57.933,
"cpu_seconds": 8.058,
"peak_family_mib": 821.3,
"input_tokens": 286938,
"cached_input_tokens": 256384,
"uncached_input_tokens": 30554,
"output_tokens": 1831,
"reported_reasoning_output_tokens": 106,
"mcp_calls_including_discovery": 8,
"payload_cleanup_verified": true,
"memory_events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0
}
}
],
"failed_trials": {
"model-javascript-focused-1": "Wrong connector discovery; approval-policy refusal; no source facts established.",
"model-javascript-guided-complete-2": "Premature agent refusal with no REA RPC; asserted tool unavailability not established by a connection/discovery observation."
},
"pilot_failures": [
"Private pilot oracle read view.item instead of view.items; corrected before the committed SDK cohorts. No product failure inferred."
],
"decision": "Existing query APIs suffice for these bounded questions; no general token/time advantage established. Separate imported immutable-format validation optimization is tracked in #4810.",
"review_followup_verification": {
"review-mixed-native": {
"status": "passed",
"checks": {
"heads_checked": 64,
"pages": 32,
"revised_unknown_id": "unk_057c8f2dbf30ec98851a66ee054366bc6e2e168874581e6f99cb80673d0ff9b8",
"revision_checked": 2,
"stale_update_rejected": true,
"tampered_import_rejected_atomically": true,
"unknowns_after_close": 0,
"old_reference_refused": true
},
"original_records_preserved": true,
"elapsed_seconds": 18.444,
"cpu_seconds": 18.16515,
"peak_family_mib": 1401.6328125,
"oom_events_at_payload_exit": 0,
"payload_cleanup_verified": true
},
"review-mixed-javascript": {
"status": "passed",
"checks": {
"heads_checked": 64,
"pages": 32,
"revised_unknown_id": "unk_057c8f2dbf30ec98851a66ee054366bc6e2e168874581e6f99cb80673d0ff9b8",
"revision_checked": 2,
"stale_update_rejected": true,
"tampered_import_rejected_atomically": true,
"unknowns_after_close": 0,
"old_reference_refused": true
},
"original_records_preserved": true,
"elapsed_seconds": 27.617,
"cpu_seconds": 27.275299,
"peak_family_mib": 1444.30859375,
"oom_events_at_payload_exit": 0,
"payload_cleanup_verified": true
},
"deterministic_receipt_tests": {
"passed": 4,
"scope": "Rejected SDK call, protocol error, receipt-write failure, simultaneous call/write failure; not real-provider evidence"
},
"cleanup_receipt_regressions": {
"passed": 6,
"scope": "Real stream/file refusals, independent peer close, and injected partial publication at the filesystem boundary. Original failure and call costs survive; failed cleanup retains workspace ownership. Harness failure checks, not real-provider evidence.",
"baseline_reproduction": {
"revision": "b025a919b36ec091ca888339df6b24e78e89b79b",
"method": "Execute the prior committed finalization block with deterministic workspace or log rejection; not a production-provider reproduction.",
"workspace_error": {
"file_receipt_attempts": 0,
"stdout_receipt_attempts": 0
},
"log_error": {
"file_receipt_attempts": 0,
"stdout_receipt_attempts": 0
}
},
"partial_retry_baseline": {
"revision": "5d6259e658ee54fc9829b9143a8721a2ab426e80",
"regression_exit_code": 1,
"observed_saved_report": "partial",
"method": "Run the unchanged partial-write regression against the prior committed finalizer with deterministic filesystem-boundary failure injection."
}
}
}
}
Reference in new issue View Git Blame Copy Permalink