{ "measurement": "Startup floor re-verified after the 2.1.229 -> 2.1.231 version bump", "claude_code_version": "2.1.231", "supersedes": "baseline-floor-2-1-229.json, which the /cost/ calculator was still quoting after the version moved.", "why": "The calculator carries a BASELINE constant and deploy.sh gated it against the 2.1.229 corpus. That gate proves the constant matches its corpus and is structurally blind to the corpus being measured on a version nobody is running, so it printed ok on every deploy while the figures aged. The gate now derives the corpus path from the calculator's own version stamp and fails when the installed claude --version differs from it.", "method": { "reconstructed_from": "The published method block of baseline-floor-2-1-229.json. The harness itself had been lost with a wiped scratchpad; the method was rebuilt from what the corpus published, and the 521-token .usage/modelUsage constant reproduced, which is the check that the reconstruction is faithful.", "floor": "An EMPTY temp directory, created fresh per measurement and ASSERTED empty, with --setting-sources project asserted onto the command line so nothing user-scope loads. Prompt: 'Reply with only: OK'.", "tool_search_off": "The same floor with ENABLE_TOOL_SEARCH=false in the child environment, asserted present only on that arm.", "loaded": "The same prompt with --setting-sources asserted ABSENT, so the arms cannot silently be the same measurement.", "tokens": "inputTokens + cacheCreationInputTokens + cacheReadInputTokens from modelUsage. .usage recorded alongside.", "estimator": "MODE over all 10 repetitions per cell, not the minimum. modelUsage on this trivial call is BIMODAL: a 521-token Haiku call fires in 9 of 10 repetitions, so the minimum picks the rare run where it did not and publishes a floor 521 tokens too low. The 2.1.229 corpus used the minimum and did not hit this, its runs being uniform. Reporting the mode with its count is the fix.", "controls": { "noise_floor": "Every cell measured TWICE per round, the gap kept as that round's noise. Published per round.", "same_round": "All arms run inside one round before the next begins.", "flag_both_ways": "--setting-sources project asserted PRESENT on isolated cells and ABSENT on loaded ones; ENABLE_TOOL_SEARCH asserted present only where intended.", "empty_dir": "The sandbox is asserted to contain no files before each call, so the floor is a floor.", "arrival": "The reply must be exactly OK. 15 of 15 cells passed.", "no_exclusions": "All 15 cells scored; the outlying repetitions are reported, not dropped." } }, "results": { "floor_tool_search_on": 22404, "floor_tool_search_on_repetitions": "9 of 10", "floor_tool_search_off": 40209, "floor_tool_search_off_repetitions": "10 of 10", "loaded_machine_total": 60206, "loaded_repetitions": "9 of 10", "tool_search_delta": 17805, "change_since_2_1_229": { "floor_tool_search_on": "22,340 -> 22,404 (+64)", "floor_tool_search_off": "40,144 -> 40,209 (+65)", "tool_search_delta": "17,804 -> 17,805 (+1)", "reading": "The floor rose about 64 tokens on BOTH arms while the tool-search delta held to within a token, which is what a slightly larger base system prompt looks like rather than a change to the tool-search mechanism." }, "all_repetitions": { "floor_tool_search_on": { "modelUsage": { "22404": 9, "21883": 1 }, "usage": { "21883": 10 } }, "floor_tool_search_off": { "modelUsage": { "40209": 10 }, "usage": { "39688": 10 } }, "loaded": { "modelUsage": { "60206": 9, "60056": 1 }, "usage": { "60206": 9, "60056": 1 } } } }, "known_limits": { "one_machine": "One machine, one account. The floor arms exclude user-scope config by design; the loaded arm is this machine's configuration and transfers to nobody.", "bimodal_modelusage": "modelUsage on a trivial isolated call is bimodal at 22,404 / 21,883 depending on whether a Haiku call fires. Any single-run floor measurement has roughly a one-in-ten chance of landing 521 tokens low.", "loaded_arm_also_moved": "The loaded total moved to 60,206 from 59,862. That arm mixes the version change with any change to this machine's user-scope configuration and the two are not separated here." }, "runs": [ { "cell": "floor_ts_on", "round": 0, "isolated": true, "tool_search_off": false, "status": "ok", "model_usage": [ 22404, 22404 ], "dot_usage": [ 21883, 21883 ], "model_min": 22404, "model_noise": 0, "dot_min": 21883, "dot_noise": 0, "replies": [ "OK", "OK" ] }, { "cell": "floor_ts_off", "round": 0, "isolated": true, "tool_search_off": true, "status": "ok", "model_usage": [ 40209, 40209 ], "dot_usage": [ 39688, 39688 ], "model_min": 40209, "model_noise": 0, "dot_min": 39688, "dot_noise": 0, "replies": [ "OK", "OK" ] }, { "cell": "loaded", "round": 0, "isolated": false, "tool_search_off": false, "status": "ok", "model_usage": [ 60206, 60056 ], "dot_usage": [ 60206, 60056 ], "model_min": 60056, "model_noise": 150, "dot_min": 60056, "dot_noise": 150, "replies": [ "OK", "OK" ] }, { "cell": "floor_ts_on", "round": 1, "isolated": true, "tool_search_off": false, "status": "ok", "model_usage": [ 22404, 22404 ], "dot_usage": [ 21883, 21883 ], "model_min": 22404, "model_noise": 0, "dot_min": 21883, "dot_noise": 0, "replies": [ "OK", "OK" ] }, { "cell": "floor_ts_off", "round": 1, "isolated": true, "tool_search_off": true, "status": "ok", "model_usage": [ 40209, 40209 ], "dot_usage": [ 39688, 39688 ], "model_min": 40209, "model_noise": 0, "dot_min": 39688, "dot_noise": 0, "replies": [ "OK", "OK" ] }, { "cell": "loaded", "round": 1, "isolated": false, "tool_search_off": false, "status": "ok", "model_usage": [ 60206, 60206 ], "dot_usage": [ 60206, 60206 ], "model_min": 60206, "model_noise": 0, "dot_min": 60206, "dot_noise": 0, "replies": [ "OK", "OK" ] }, { "cell": "floor_ts_on", "round": 2, "isolated": true, "tool_search_off": false, "status": "ok", "model_usage": [ 22404, 22404 ], "dot_usage": [ 21883, 21883 ], "model_min": 22404, "model_noise": 0, "dot_min": 21883, "dot_noise": 0, "replies": [ "OK", "OK" ] }, { "cell": "floor_ts_off", "round": 2, "isolated": true, "tool_search_off": true, "status": "ok", "model_usage": [ 40209, 40209 ], "dot_usage": [ 39688, 39688 ], "model_min": 40209, "model_noise": 0, "dot_min": 39688, "dot_noise": 0, "replies": [ "OK", "OK" ] }, { "cell": "loaded", "round": 2, "isolated": false, "tool_search_off": false, "status": "ok", "model_usage": [ 60206, 60206 ], "dot_usage": [ 60206, 60206 ], "model_min": 60206, "model_noise": 0, "dot_min": 60206, "dot_noise": 0, "replies": [ "OK", "OK" ] }, { "cell": "floor_ts_on", "round": 3, "isolated": true, "tool_search_off": false, "status": "ok", "model_usage": [ 22404, 22404 ], "dot_usage": [ 21883, 21883 ], "model_min": 22404, "model_noise": 0, "dot_min": 21883, "dot_noise": 0, "replies": [ "OK", "OK" ] }, { "cell": "floor_ts_off", "round": 3, "isolated": true, "tool_search_off": true, "status": "ok", "model_usage": [ 40209, 40209 ], "dot_usage": [ 39688, 39688 ], "model_min": 40209, "model_noise": 0, "dot_min": 39688, "dot_noise": 0, "replies": [ "OK", "OK" ] }, { "cell": "loaded", "round": 3, "isolated": false, "tool_search_off": false, "status": "ok", "model_usage": [ 60206, 60206 ], "dot_usage": [ 60206, 60206 ], "model_min": 60206, "model_noise": 0, "dot_min": 60206, "dot_noise": 0, "replies": [ "OK", "OK" ] }, { "cell": "floor_ts_on", "round": 4, "isolated": true, "tool_search_off": false, "status": "ok", "model_usage": [ 21883, 22404 ], "dot_usage": [ 21883, 21883 ], "model_min": 21883, "model_noise": 521, "dot_min": 21883, "dot_noise": 0, "replies": [ "OK", "OK" ] }, { "cell": "floor_ts_off", "round": 4, "isolated": true, "tool_search_off": true, "status": "ok", "model_usage": [ 40209, 40209 ], "dot_usage": [ 39688, 39688 ], "model_min": 40209, "model_noise": 0, "dot_min": 39688, "dot_noise": 0, "replies": [ "OK", "OK" ] }, { "cell": "loaded", "round": 4, "isolated": false, "tool_search_off": false, "status": "ok", "model_usage": [ 60206, 60206 ], "dot_usage": [ 60206, 60206 ], "model_min": 60206, "model_noise": 0, "dot_min": 60206, "dot_noise": 0, "replies": [ "OK", "OK" ] } ] }