gensokyo-dev
7d8e4e4338
Validate omitted GGUF strings without materializing discarded text while preserving full UTF-8 checks and retained Evidence. Record real-model costs and workflow coverage; preserve failed experiment receipts through cleanup. Fixes #5774.
436 lines
14 KiB
JSON
436 lines
14 KiB
JSON
{
|
|
"baseline_source": "264862bf2e3612c9cb6fcfb861843cbba4e1893f",
|
|
"candidate_code": "c2550343173c5abd8f34e925eac1bcc12b7ee6a2",
|
|
"toolchain": {
|
|
"node": "24.18.0",
|
|
"npm": "11.16.0"
|
|
},
|
|
"runtime_filetables_sha256": {
|
|
"baseline": "81c293ff8997017cfe9db2a65a8b70b188e3e6d20cef07902599c69ade5d9fac",
|
|
"candidate": "b4b654a7151f6affbb3ab8541bff74b643d9f78073d338b8feb2b095dd12b6b5"
|
|
},
|
|
"lock_sha256": "699a14749e3825c62430faeb18224190fee1c9347760494e38e334e180bea6ec",
|
|
"gguf_source_sha256": {
|
|
"baseline": "9c387d5fba10db7830e74ec74a73f4b3d2e1347cca665e948b7891881b186007",
|
|
"candidate": "c43e5f8eebe04c0f82e0a970606f20ddf409c7a5fd0e298c88b09bcd0e7864a9"
|
|
},
|
|
"driver_sha256": "9732f45b8b550eef47c1fc993b435129108934f3ec3501a34df1784e6ee37b44",
|
|
"resource_guard_sha256": {
|
|
"resource-run.py": "7a9642ab86ba34290564f636eb080cacf38b4185a254028680b238bff172993b",
|
|
"resource_payload.py": "6f089e645459eba80c5d73e39c21606c78173a44daf49fabc3b88c4631ce51c6"
|
|
},
|
|
"oracle": {
|
|
"gguf": "0.19.0",
|
|
"safetensors": "0.8.0",
|
|
"receipts": [
|
|
{
|
|
"file": "stories15M-q4_0.gguf",
|
|
"sha256": "66967fbece6dbe97886593fdbb73589584927e29119ec31f08090732d1861739",
|
|
"metadata": 20,
|
|
"tensors": 57,
|
|
"status": "matched"
|
|
},
|
|
{
|
|
"file": "stories260K.gguf",
|
|
"sha256": "270cba1bd5109f42d03350f60406024560464db173c0e387d91f0426d3bd256d",
|
|
"metadata": 19,
|
|
"tensors": 48,
|
|
"status": "matched"
|
|
},
|
|
{
|
|
"file": "tiny-bert.safetensors",
|
|
"sha256": "965f02b6a7e5520fc12f710e4e3b6132f697f1c8f648819553c5ade86752d2de",
|
|
"metadata": 2,
|
|
"tensors": 100,
|
|
"status": "matched"
|
|
}
|
|
]
|
|
},
|
|
"tokenizer": "tiktoken 0.12.0 cl100k_base; independent compact JSON chunks, catalog once + each named request/reply; not actual model usage or initialization/framing",
|
|
"cases": [
|
|
{
|
|
"artifact": {
|
|
"repo": "ggml-org/models",
|
|
"revision": "499bc8821c6b12b4e53c5bffcb21ec206f212d81",
|
|
"member": "tinyllamas/stories15M-q4_0.gguf",
|
|
"file": "stories15M-q4_0.gguf",
|
|
"url": "https://huggingface.co/ggml-org/models/resolve/499bc8821c6b12b4e53c5bffcb21ec206f212d81/tinyllamas/stories15M-q4_0.gguf",
|
|
"bytes": 19077344
|
|
},
|
|
"normalized_sha256": "a894614da7ce9486b07eb2ede05ce11307d4726089722278bc4c95043feb6b66",
|
|
"answers_sha256": "64277f01f43440bc99a94ab0787a4ddde22866ed20140d87d3942b1723cbb067",
|
|
"catalog_sha256": "859b1433ef7c45ea23ba4073715d25ed7a76fd9df6d74285d20a5ea39e8e2898",
|
|
"mcp": [
|
|
{
|
|
"pair": 1,
|
|
"side": "baseline",
|
|
"wall_seconds": 6.667,
|
|
"cpu_seconds": 6.480601,
|
|
"peak_family_mib": 769.34,
|
|
"catalog_plus_calls_tokens": 895583,
|
|
"calls_ms": [367.272, 1936.653, 162.868, 12.072, 19.236]
|
|
},
|
|
{
|
|
"pair": 1,
|
|
"side": "candidate",
|
|
"wall_seconds": 6.867,
|
|
"cpu_seconds": 6.690645,
|
|
"peak_family_mib": 816.34,
|
|
"catalog_plus_calls_tokens": 895583,
|
|
"calls_ms": [324.806, 1822.885, 153.167, 11.988, 18.093]
|
|
},
|
|
{
|
|
"pair": 2,
|
|
"side": "baseline",
|
|
"wall_seconds": 6.77,
|
|
"cpu_seconds": 6.555891,
|
|
"peak_family_mib": 798.98,
|
|
"catalog_plus_calls_tokens": 895583,
|
|
"calls_ms": [456.467, 1823.786, 160.038, 11.296, 18.607]
|
|
},
|
|
{
|
|
"pair": 2,
|
|
"side": "candidate",
|
|
"wall_seconds": 6.565,
|
|
"cpu_seconds": 6.3467,
|
|
"peak_family_mib": 813.566,
|
|
"catalog_plus_calls_tokens": 895583,
|
|
"calls_ms": [329.862, 1767.086, 152.299, 10.948, 17.523]
|
|
},
|
|
{
|
|
"pair": 3,
|
|
"side": "baseline",
|
|
"wall_seconds": 6.666,
|
|
"cpu_seconds": 6.49779,
|
|
"peak_family_mib": 771.02,
|
|
"catalog_plus_calls_tokens": 895583,
|
|
"calls_ms": [345.895, 1854.963, 156.895, 11.476, 15.852]
|
|
},
|
|
{
|
|
"pair": 3,
|
|
"side": "candidate",
|
|
"wall_seconds": 6.567,
|
|
"cpu_seconds": 6.386103,
|
|
"peak_family_mib": 818.891,
|
|
"catalog_plus_calls_tokens": 895583,
|
|
"calls_ms": [342.734, 1827.835, 157.363, 10.122, 17.6]
|
|
}
|
|
],
|
|
"phase": [
|
|
{
|
|
"pair": 1,
|
|
"side": "baseline",
|
|
"first_ms": 154.027,
|
|
"repeat_median_ms": 56.14,
|
|
"wall_seconds": 1.028,
|
|
"cpu_seconds": 0.87953,
|
|
"peak_family_mib": 75.867
|
|
},
|
|
{
|
|
"pair": 1,
|
|
"side": "candidate",
|
|
"first_ms": 121.41,
|
|
"repeat_median_ms": 35.429,
|
|
"wall_seconds": 0.925,
|
|
"cpu_seconds": 0.717987,
|
|
"peak_family_mib": 77.945
|
|
},
|
|
{
|
|
"pair": 2,
|
|
"side": "baseline",
|
|
"first_ms": 159.766,
|
|
"repeat_median_ms": 53.575,
|
|
"wall_seconds": 1.027,
|
|
"cpu_seconds": 0.854739,
|
|
"peak_family_mib": 75.586
|
|
},
|
|
{
|
|
"pair": 2,
|
|
"side": "candidate",
|
|
"first_ms": 121.383,
|
|
"repeat_median_ms": 38.773,
|
|
"wall_seconds": 0.925,
|
|
"cpu_seconds": 0.733397,
|
|
"peak_family_mib": 75.125
|
|
},
|
|
{
|
|
"pair": 3,
|
|
"side": "baseline",
|
|
"first_ms": 148.447,
|
|
"repeat_median_ms": 53.58,
|
|
"wall_seconds": 1.026,
|
|
"cpu_seconds": 0.848173,
|
|
"peak_family_mib": 75.363
|
|
},
|
|
{
|
|
"pair": 3,
|
|
"side": "candidate",
|
|
"first_ms": 121.712,
|
|
"repeat_median_ms": 37.191,
|
|
"wall_seconds": 0.926,
|
|
"cpu_seconds": 0.728712,
|
|
"peak_family_mib": 77.887
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"artifact": {
|
|
"repo": "ggml-org/models",
|
|
"revision": "499bc8821c6b12b4e53c5bffcb21ec206f212d81",
|
|
"member": "tinyllamas/stories260K.gguf",
|
|
"file": "stories260K.gguf",
|
|
"url": "https://huggingface.co/ggml-org/models/resolve/499bc8821c6b12b4e53c5bffcb21ec206f212d81/tinyllamas/stories260K.gguf",
|
|
"bytes": 1185376
|
|
},
|
|
"normalized_sha256": "15f4dbdc30dd9bd936747057031c6391c38b4c1717d56ae33ed2f0cba4184405",
|
|
"answers_sha256": "99257428a01aa513b435c95ca7217c69f4641d9021372a6333e7cb9e2cb35d63",
|
|
"catalog_sha256": "859b1433ef7c45ea23ba4073715d25ed7a76fd9df6d74285d20a5ea39e8e2898",
|
|
"mcp": [
|
|
{
|
|
"pair": 1,
|
|
"side": "baseline",
|
|
"wall_seconds": 6.568,
|
|
"cpu_seconds": 6.375324,
|
|
"peak_family_mib": 743.887,
|
|
"catalog_plus_calls_tokens": 895421,
|
|
"calls_ms": [223.11, 1938.73, 102.018, 10.759, 16.11]
|
|
},
|
|
{
|
|
"pair": 1,
|
|
"side": "candidate",
|
|
"wall_seconds": 6.566,
|
|
"cpu_seconds": 6.338327,
|
|
"peak_family_mib": 744.852,
|
|
"catalog_plus_calls_tokens": 895421,
|
|
"calls_ms": [248.979, 1886.809, 105.801, 10.738, 15.653]
|
|
},
|
|
{
|
|
"pair": 2,
|
|
"side": "baseline",
|
|
"wall_seconds": 6.365,
|
|
"cpu_seconds": 6.206216,
|
|
"peak_family_mib": 783.84,
|
|
"catalog_plus_calls_tokens": 895421,
|
|
"calls_ms": [248.196, 1845.737, 112.874, 10.724, 17.594]
|
|
},
|
|
{
|
|
"pair": 2,
|
|
"side": "candidate",
|
|
"wall_seconds": 7.383,
|
|
"cpu_seconds": 7.211823,
|
|
"peak_family_mib": 742.281,
|
|
"catalog_plus_calls_tokens": 895421,
|
|
"calls_ms": [233.155, 1909.721, 104.803, 11.603, 19.357]
|
|
},
|
|
{
|
|
"pair": 3,
|
|
"side": "baseline",
|
|
"wall_seconds": 6.565,
|
|
"cpu_seconds": 6.323702,
|
|
"peak_family_mib": 740.84,
|
|
"catalog_plus_calls_tokens": 895421,
|
|
"calls_ms": [233.664, 1918.301, 104.389, 14.561, 15.227]
|
|
},
|
|
{
|
|
"pair": 3,
|
|
"side": "candidate",
|
|
"wall_seconds": 6.566,
|
|
"cpu_seconds": 6.315785,
|
|
"peak_family_mib": 778.133,
|
|
"catalog_plus_calls_tokens": 895421,
|
|
"calls_ms": [232.37, 1851.929, 78.318, 21.857, 16.438]
|
|
}
|
|
],
|
|
"phase": [
|
|
{
|
|
"pair": 1,
|
|
"side": "baseline",
|
|
"first_ms": 35.366,
|
|
"repeat_median_ms": 7.784,
|
|
"wall_seconds": 0.625,
|
|
"cpu_seconds": 0.429065,
|
|
"peak_family_mib": 62.922
|
|
},
|
|
{
|
|
"pair": 1,
|
|
"side": "candidate",
|
|
"first_ms": 30.009,
|
|
"repeat_median_ms": 7.934,
|
|
"wall_seconds": 0.624,
|
|
"cpu_seconds": 0.392848,
|
|
"peak_family_mib": 59.664
|
|
},
|
|
{
|
|
"pair": 2,
|
|
"side": "baseline",
|
|
"first_ms": 32.0,
|
|
"repeat_median_ms": 8.772,
|
|
"wall_seconds": 0.624,
|
|
"cpu_seconds": 0.412059,
|
|
"peak_family_mib": 60.422
|
|
},
|
|
{
|
|
"pair": 2,
|
|
"side": "candidate",
|
|
"first_ms": 32.032,
|
|
"repeat_median_ms": 9.36,
|
|
"wall_seconds": 0.624,
|
|
"cpu_seconds": 0.412412,
|
|
"peak_family_mib": 59.441
|
|
},
|
|
{
|
|
"pair": 3,
|
|
"side": "baseline",
|
|
"first_ms": 33.628,
|
|
"repeat_median_ms": 9.496,
|
|
"wall_seconds": 0.624,
|
|
"cpu_seconds": 0.396232,
|
|
"peak_family_mib": 60.41
|
|
},
|
|
{
|
|
"pair": 3,
|
|
"side": "candidate",
|
|
"first_ms": 30.506,
|
|
"repeat_median_ms": 8.172,
|
|
"wall_seconds": 0.622,
|
|
"cpu_seconds": 0.404398,
|
|
"peak_family_mib": 61.672
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"artifact": {
|
|
"repo": "hf-internal-testing/tiny-random-bert",
|
|
"revision": "f171d7baecaf37b5da5a3616d8833b9969753535",
|
|
"member": "model.safetensors",
|
|
"file": "tiny-bert.safetensors",
|
|
"url": "https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/f171d7baecaf37b5da5a3616d8833b9969753535/model.safetensors",
|
|
"bytes": 520212
|
|
},
|
|
"normalized_sha256": "8658d90261cdbaaf1f5f8d2e9408ad301b40ad3e6b7aab7ac9c5d4be71ca36d5",
|
|
"answers_sha256": "82072404a0aa24e3b42979397421f455da2589c2ea649f28ba7279d43e6c3bc8",
|
|
"catalog_sha256": "859b1433ef7c45ea23ba4073715d25ed7a76fd9df6d74285d20a5ea39e8e2898",
|
|
"mcp": [
|
|
{
|
|
"pair": 1,
|
|
"side": "baseline",
|
|
"wall_seconds": 6.667,
|
|
"cpu_seconds": 6.46146,
|
|
"peak_family_mib": 741.602,
|
|
"catalog_plus_calls_tokens": 893755,
|
|
"calls_ms": [251.201, 1965.785, 114.055, 10.837, 28.862]
|
|
},
|
|
{
|
|
"pair": 1,
|
|
"side": "candidate",
|
|
"wall_seconds": 6.767,
|
|
"cpu_seconds": 6.519606,
|
|
"peak_family_mib": 736.367,
|
|
"catalog_plus_calls_tokens": 893755,
|
|
"calls_ms": [255.557, 1979.227, 86.726, 10.093, 24.531]
|
|
},
|
|
{
|
|
"pair": 2,
|
|
"side": "baseline",
|
|
"wall_seconds": 6.565,
|
|
"cpu_seconds": 6.379249,
|
|
"peak_family_mib": 730.273,
|
|
"catalog_plus_calls_tokens": 893755,
|
|
"calls_ms": [245.517, 1900.802, 94.351, 10.0, 25.635]
|
|
},
|
|
{
|
|
"pair": 2,
|
|
"side": "candidate",
|
|
"wall_seconds": 6.569,
|
|
"cpu_seconds": 6.388131,
|
|
"peak_family_mib": 785.746,
|
|
"catalog_plus_calls_tokens": 893755,
|
|
"calls_ms": [250.715, 1846.568, 103.314, 21.618, 27.261]
|
|
},
|
|
{
|
|
"pair": 3,
|
|
"side": "baseline",
|
|
"wall_seconds": 6.771,
|
|
"cpu_seconds": 6.530512,
|
|
"peak_family_mib": 745.098,
|
|
"catalog_plus_calls_tokens": 893755,
|
|
"calls_ms": [255.441, 1949.189, 110.897, 10.679, 24.345]
|
|
},
|
|
{
|
|
"pair": 3,
|
|
"side": "candidate",
|
|
"wall_seconds": 6.466,
|
|
"cpu_seconds": 6.296937,
|
|
"peak_family_mib": 741.348,
|
|
"catalog_plus_calls_tokens": 893755,
|
|
"calls_ms": [243.919, 1895.978, 113.995, 10.66, 27.269]
|
|
}
|
|
],
|
|
"phase": [
|
|
{
|
|
"pair": 1,
|
|
"side": "baseline",
|
|
"first_ms": 27.648,
|
|
"repeat_median_ms": 8.11,
|
|
"wall_seconds": 0.623,
|
|
"cpu_seconds": 0.394405,
|
|
"peak_family_mib": 57.281
|
|
},
|
|
{
|
|
"pair": 1,
|
|
"side": "candidate",
|
|
"first_ms": 27.003,
|
|
"repeat_median_ms": 8.082,
|
|
"wall_seconds": 0.523,
|
|
"cpu_seconds": 0.383678,
|
|
"peak_family_mib": 57.266
|
|
},
|
|
{
|
|
"pair": 2,
|
|
"side": "baseline",
|
|
"first_ms": 26.941,
|
|
"repeat_median_ms": 8.04,
|
|
"wall_seconds": 0.626,
|
|
"cpu_seconds": 0.401321,
|
|
"peak_family_mib": 59.504
|
|
},
|
|
{
|
|
"pair": 2,
|
|
"side": "candidate",
|
|
"first_ms": 28.012,
|
|
"repeat_median_ms": 8.006,
|
|
"wall_seconds": 0.625,
|
|
"cpu_seconds": 0.392541,
|
|
"peak_family_mib": 59.012
|
|
},
|
|
{
|
|
"pair": 3,
|
|
"side": "baseline",
|
|
"first_ms": 28.734,
|
|
"repeat_median_ms": 8.499,
|
|
"wall_seconds": 0.624,
|
|
"cpu_seconds": 0.411118,
|
|
"peak_family_mib": 59.23
|
|
},
|
|
{
|
|
"pair": 3,
|
|
"side": "candidate",
|
|
"first_ms": 29.961,
|
|
"repeat_median_ms": 8.632,
|
|
"wall_seconds": 0.625,
|
|
"cpu_seconds": 0.41875,
|
|
"peak_family_mib": 57.5
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"published_equivalent_source": "2f091ccc734d7bd945bd61432d41cf5ba68be12b",
|
|
"candidate_reproduction": "Use baseline_source plus the byte-identical src/model/GgufHeader.ts from published_equivalent_source; driver SHA-256 is pinned separately. Measured local candidate predates the integration rebase.",
|
|
"post_review_receipt_fix": {
|
|
"measured_driver_version": "2f091ccc734d7bd945bd61432d41cf5ba68be12b",
|
|
"driver_sha256": "449860fb973418f2e3e51183a81be1e3111ffa6453de8208a4b175d3d410322f",
|
|
"publication_owner_sha256": "901092ae6dc7fda1a8869d62362a0a9f560e03d5ef5667a5c6ed660db000b9a2",
|
|
"validation": "3 new fault-injection regressions reject original driver and pass current; 11 shared receipt regressions pass. Real stories15M MCP and seven-inspection phase reruns qualify; original paired samples retain their measured driver hash."
|
|
}
|
|
}
|