{
 "measurement": "What a book actually costs in Claude Code context, against its tiktoken cl100k_base count",
 "claude_code_version": "2.1.238",
 "why": "Token-cost claims about Claude tooling are routinely computed with OpenAI's cl100k_base, because tiktoken is the tokeniser that is easy to install. A widely shared repository prices four books that way to reach '24x-51x fewer tokens than dumping the book into context'. The gap between that count and what Claude Code actually charges for the same bytes is in no vendor doc, so it can only be settled by running it.",
 "method": {
  "shape": "Each slice is written as a project-scope CLAUDE.md and the context is read back from modelUsage, paired against a floor re-measured in the same round. Prompt: 'Reply with only: OK'.",
  "estimator": "Guarded mode of four rounds on the floor-paired delta, declared once and used for every figure. It refuses a cell where nothing repeated or the top count is tied; no cell here refused.",
  "headline_is_a_slope": "A CLAUDE.md costs a fixed framing charge plus a per-byte rate. The published ratio is the SLOPE between the smallest and largest slice, which cancels that intercept. A ratio of single totals would include it and overstate the gap.",
  "two_texts_on_purpose": "This site has already measured that the byte-to-token rate is a property of the text, not of the file. A literary book and a technical book with code blocks are the two shapes the claim covers.",
  "controls": {
   "paired": "The floor is re-measured every round and every delta is against THAT round.",
   "fixture": "The byte count is read back off disk per run and asserted equal to the slice.",
   "arrival": "Every reply had to be exactly OK; 28 of 28 arrived.",
   "isolation": "--setting-sources project asserted onto the command line for every call.",
   "utf8_boundary": "Slices back off to a valid UTF-8 boundary and then to whitespace, asserted per run.",
   "single_release": "All 28 run directories verified 2.1.238 from their transcripts."
  }
 },
 "results": {
  "per_cell": {
   "moby": {
    "20000": {
     "mode": 8809,
     "estimator_refused": null,
     "rounds": 4,
     "min": 8110,
     "max": 8809,
     "spread": 699,
     "all_rounds": [
      8110,
      8809,
      8809,
      8809
     ],
     "mode_seen": 3,
     "bytes_on_disk": 19992,
     "chars": 19500,
     "cl100k_tokens": 5425,
     "claude_tokens_per_byte": 0.4406,
     "cl100k_tokens_per_byte": 0.2714
    },
    "60000": {
     "mode": 22948,
     "estimator_refused": null,
     "rounds": 4,
     "min": 22948,
     "max": 23146,
     "spread": 198,
     "all_rounds": [
      22948,
      22948,
      22948,
      23146
     ],
     "mode_seen": 3,
     "bytes_on_disk": 59997,
     "chars": 58845,
     "cl100k_tokens": 15425,
     "claude_tokens_per_byte": 0.3825,
     "cl100k_tokens_per_byte": 0.2571
    },
    "120000": {
     "mode": 44106,
     "estimator_refused": null,
     "rounds": 4,
     "min": 44106,
     "max": 44304,
     "spread": 198,
     "all_rounds": [
      44106,
      44106,
      44106,
      44304
     ],
     "mode_seen": 3,
     "bytes_on_disk": 119997,
     "chars": 118055,
     "cl100k_tokens": 30379,
     "claude_tokens_per_byte": 0.3676,
     "cl100k_tokens_per_byte": 0.2532
    }
   },
   "progit": {
    "20000": {
     "mode": 9891,
     "estimator_refused": null,
     "rounds": 4,
     "min": 9693,
     "max": 10089,
     "spread": 396,
     "all_rounds": [
      9693,
      9891,
      9891,
      10089
     ],
     "mode_seen": 2,
     "bytes_on_disk": 20000,
     "chars": 20000,
     "cl100k_tokens": 9300,
     "claude_tokens_per_byte": 0.4945,
     "cl100k_tokens_per_byte": 0.465
    },
    "60000": {
     "mode": 22818,
     "estimator_refused": null,
     "rounds": 4,
     "min": 22818,
     "max": 23016,
     "spread": 198,
     "all_rounds": [
      22818,
      22818,
      22818,
      23016
     ],
     "mode_seen": 3,
     "bytes_on_disk": 59998,
     "chars": 59747,
     "cl100k_tokens": 18207,
     "claude_tokens_per_byte": 0.3803,
     "cl100k_tokens_per_byte": 0.3035
    },
    "120000": {
     "mode": 42091,
     "estimator_refused": null,
     "rounds": 4,
     "min": 42091,
     "max": 42289,
     "spread": 198,
     "all_rounds": [
      42091,
      42091,
      42091,
      42289
     ],
     "mode_seen": 3,
     "bytes_on_disk": 119997,
     "chars": 119292,
     "cl100k_tokens": 32144,
     "claude_tokens_per_byte": 0.3508,
     "cl100k_tokens_per_byte": 0.2679
    }
   }
  },
  "slopes": {
   "moby": {
    "span_bytes": [
     19992,
     119997
    ],
    "claude_tokens_per_byte": 0.353,
    "cl100k_tokens_per_byte": 0.2495,
    "ratio_claude_over_cl100k": 1.414,
    "per_segment": [
     {
      "from_bytes": 19992,
      "to_bytes": 59997,
      "claude_tokens_per_byte": 0.3534,
      "cl100k_tokens_per_byte": 0.25,
      "ratio": 1.414
     },
     {
      "from_bytes": 59997,
      "to_bytes": 119997,
      "claude_tokens_per_byte": 0.3526,
      "cl100k_tokens_per_byte": 0.2492,
      "ratio": 1.415
     }
    ]
   },
   "progit": {
    "span_bytes": [
     20000,
     119997
    ],
    "claude_tokens_per_byte": 0.322,
    "cl100k_tokens_per_byte": 0.2284,
    "ratio_claude_over_cl100k": 1.41,
    "per_segment": [
     {
      "from_bytes": 20000,
      "to_bytes": 59998,
      "claude_tokens_per_byte": 0.3232,
      "cl100k_tokens_per_byte": 0.2227,
      "ratio": 1.451
     },
     {
      "from_bytes": 59998,
      "to_bytes": 119997,
      "claude_tokens_per_byte": 0.3212,
      "cl100k_tokens_per_byte": 0.2323,
      "ratio": 1.383
     }
    ]
   }
  },
  "framing_bound": {
   "model": "real_rate = cl100k_rate * T + F",
   "tokeniser_factor_T": 1.4678,
   "framing_tokens_per_byte_F": -0.01331,
   "F_as_share_of_smallest_measured_rate": -0.0413,
   "exactly_determined": true,
   "reading": "F is negative, and a framing charge cannot be negative, so any per-byte framing component is indistinguishable from zero."
  },
  "published_figure_this_disagrees_with": {
   "source": "anthropics/skills, skills/claude-api/shared/token-counting.md",
   "quote": "Do not use `tiktoken`. It's OpenAI's tokenizer. It undercounts Claude tokens by ~15-20% on typical text, and by much more on code or non-English input.",
   "implied_ratio": "1.18x to 1.25x",
   "measured_here": "1.41x",
   "reading": "The published figure is for the TOKENISER; this corpus measures what a CLAUDE.md of the same text adds to a real Claude Code context, which includes any framing the product applies. They are not the same quantity and the gap between them is not resolved here. The published guidance is the conservative end."
  },
  "field_observations_on_the_tool": {
   "note": "Recorded because the post cites them. These are hands-on observations of book-to-skill v1.4.0 at commit 3a97a71, NOT part of the 28-run measurement.",
   "extraction": "501-page Pro Git PDF to structured text in 0.55-0.68s wall clock over six runs.",
   "chapter_count_is_a_false_positive": "The extractor reported 'chapters: 93 (structural)' for Pro Git, which has ten chapters and three appendices. All 93 are depth-1 ATX matches on '#' lines lifted from code samples and terminal transcripts; none is a chapter heading. The fenced-code guard that would exclude them found one closed fence, six lines, because the pdftotext output preserved only two ``` markers. tools/discovery_tax.py on the same file reports 'No chapters detected'. The project's own docs give this book's chapter cell as a dash, which is the honest value.",
   "the_low_count_warning_cannot_fire_at_zero": "The warning beside the count requires chapters_method == 'structural', and the method is set to 'none' whenever the count is 0, so it fires only at exactly 1. Verified by running the real detect_structure on three fixtures: 93/structural/no-warn, 1/structural/warn, 0/none/no-warn.",
   "cl100k_is_stale_for_openai_hosts_too": "tiktoken 0.12.0 maps gpt-4o, gpt-4.1, gpt-5 and o3 to o200k_base and reserves cl100k_base for the original gpt-4, gpt-3.5-turbo and the embedding models. On these two fixtures o200k differs from cl100k by +0.21% and -0.69%, so for an OpenAI-backed host the choice is nearly immaterial; the 1.41x gap measured here is specific to Claude Code."
  },
  "floor": {
   "rounds": 4,
   "all_rounds": [
    22778,
    22976,
    22976,
    22976
   ]
  },
  "headline": "Claude Code charges 1.41x what cl100k_base predicts for the same bytes, on both books. Every segment measured falls between 1.382x and 1.452x."
 },
 "known_limits": {
  "this_is_a_cost_not_a_mechanism": "The delta is what a CLAUDE.md of that text costs in Claude Code context. Whether the excess over cl100k_base is the tokeniser being denser on this text or Claude Code adding framing that scales with content size is NOT separated here. A constant framing charge is cancelled by using the slope; a size-proportional one would not be.",
  "what_bounds_the_framing_confound": "Model the measured rate as real = cl100k_rate * T + F, where T is a tokeniser factor and F is any framing charge proportional to bytes. The two books have DIFFERENT cl100k rates (0.2495 and 0.2284 tokens per byte), so they give two equations: solving them yields T = 1.4678 and F = -0.01331 tokens per byte, which is -4.1% of the smaller measured rate. A negative framing charge is unphysical, so F is indistinguishable from zero and cannot be positive. With only two texts the system is exactly determined, so this is a derivation with no residual left over to test it, not a fit. It bounds the confound; it does not eliminate it. An earlier draft argued this from line density instead, which was wrong: the two texts differ five-fold in lines per kB only over the first 20,000 bytes, and the slope subtracts that region out. Over the interval the slope actually covers they differ 1.33-fold (23.0 against 17.4 lines per kB), which bounds almost nothing.",
  "extraction_timing_is_wall_clock_not_a_benchmark": "The book-to-skill extraction of the 501-page Pro Git PDF took 0.64s and 0.55s wall clock over two runs on one machine, timed with /usr/bin/time. It is recorded because the post quotes it; it is a stopwatch reading, not a benchmark, and no control was run against it.",
  "the_verifier_matches_one_stale_directory": "verify-release.py matched 29 transcript directories against the 28 this run created. The extra is a 'probe' directory left at the same path prefix by earlier work. It also reports 2.1.238, so the single-release conclusion is unaffected, but the honest count of THIS trial's directories is 28.",
  "measured_to_120kb_not_to_a_whole_book": "Slices run to 120,000 bytes. Both books are seven to nine times larger. Applying the rate to a whole book is an extrapolation beyond the measured range and is labelled as one wherever this site does it.",
  "one_delivery_mechanism": "Every byte arrived as a CLAUDE.md. A book pasted into a chat turn, read with a tool call, or loaded as a skill body may be framed differently and is not measured here.",
  "cl100k_is_not_the_only_estimator_in_that_project": "The repository's own CLI prints a words/0.75 heuristic, not tiktoken, so the number it shows a user and the number in its performance table are computed differently. Both are recorded in this trial.",
  "the_model_is_not_controlled": "One machine, Claude Code 2.1.238, project-scope fixtures only. The harness SUMS modelUsage across every model appearing in a run rather than pinning one, so the session's configured model is not a property this measurement enforced. A run that routed part of its work to another model is included as measured.",
  "the_tool_this_adjudicates_is_not_a_claude_tool": "book-to-skill targets any host implementing the Agent Skills standard; its README names GitHub Copilot CLI and Amp beside Claude Code. cl100k_base is therefore a defensible choice for some of its hosts and the wrong one for Claude Code. This corpus measures the gap for Claude Code only and implies nothing about the others."
 },
 "runs": [
  {
   "round": 0,
   "kind": "floor",
   "status": "ok",
   "context": 22976,
   "claude_code_version_at_runtime": "2.1.238 (Claude Code)"
  },
  {
   "round": 0,
   "kind": "progit",
   "requested_bytes": 20000,
   "bytes_on_disk": 20000,
   "chars": 20000,
   "cl100k_tokens": 9300,
   "status": "ok",
   "context": 32669,
   "delta_vs_floor": 9693,
   "claude_code_version_at_runtime": "2.1.238 (Claude Code)"
  },
  {
   "round": 0,
   "kind": "progit",
   "requested_bytes": 60000,
   "bytes_on_disk": 59998,
   "chars": 59747,
   "cl100k_tokens": 18207,
   "status": "ok",
   "context": 45794,
   "delta_vs_floor": 22818,
   "claude_code_version_at_runtime": "2.1.238 (Claude Code)"
  },
  {
   "round": 0,
   "kind": "progit",
   "requested_bytes": 120000,
   "bytes_on_disk": 119997,
   "chars": 119292,
   "cl100k_tokens": 32144,
   "status": "ok",
   "context": 65067,
   "delta_vs_floor": 42091,
   "claude_code_version_at_runtime": "2.1.238 (Claude Code)"
  },
  {
   "round": 0,
   "kind": "moby",
   "requested_bytes": 20000,
   "bytes_on_disk": 19992,
   "chars": 19500,
   "cl100k_tokens": 5425,
   "status": "ok",
   "context": 31785,
   "delta_vs_floor": 8809,
   "claude_code_version_at_runtime": "2.1.238 (Claude Code)"
  },
  {
   "round": 0,
   "kind": "moby",
   "requested_bytes": 60000,
   "bytes_on_disk": 59997,
   "chars": 58845,
   "cl100k_tokens": 15425,
   "status": "ok",
   "context": 45924,
   "delta_vs_floor": 22948,
   "claude_code_version_at_runtime": "2.1.238 (Claude Code)"
  },
  {
   "round": 0,
   "kind": "moby",
   "requested_bytes": 120000,
   "bytes_on_disk": 119997,
   "chars": 118055,
   "cl100k_tokens": 30379,
   "status": "ok",
   "context": 67082,
   "delta_vs_floor": 44106,
   "claude_code_version_at_runtime": "2.1.238 (Claude Code)"
  },
  {
   "round": 1,
   "kind": "floor",
   "status": "ok",
   "context": 22976,
   "claude_code_version_at_runtime": "2.1.238 (Claude Code)"
  },
  {
   "round": 1,
   "kind": "progit",
   "requested_bytes": 20000,
   "bytes_on_disk": 20000,
   "chars": 20000,
   "cl100k_tokens": 9300,
   "status": "ok",
   "context": 32867,
   "delta_vs_floor": 9891,
   "claude_code_version_at_runtime": "2.1.238 (Claude Code)"
  },
  {
   "round": 1,
   "kind": "progit",
   "requested_bytes": 60000,
   "bytes_on_disk": 59998,
   "chars": 59747,
   "cl100k_tokens": 18207,
   "status": "ok",
   "context": 45794,
   "delta_vs_floor": 22818,
   "claude_code_version_at_runtime": "2.1.238 (Claude Code)"
  },
  {
   "round": 1,
   "kind": "progit",
   "requested_bytes": 120000,
   "bytes_on_disk": 119997,
   "chars": 119292,
   "cl100k_tokens": 32144,
   "status": "ok",
   "context": 65067,
   "delta_vs_floor": 42091,
   "claude_code_version_at_runtime": "2.1.238 (Claude Code)"
  },
  {
   "round": 1,
   "kind": "moby",
   "requested_bytes": 20000,
   "bytes_on_disk": 19992,
   "chars": 19500,
   "cl100k_tokens": 5425,
   "status": "ok",
   "context": 31785,
   "delta_vs_floor": 8809,
   "claude_code_version_at_runtime": "2.1.238 (Claude Code)"
  },
  {
   "round": 1,
   "kind": "moby",
   "requested_bytes": 60000,
   "bytes_on_disk": 59997,
   "chars": 58845,
   "cl100k_tokens": 15425,
   "status": "ok",
   "context": 45924,
   "delta_vs_floor": 22948,
   "claude_code_version_at_runtime": "2.1.238 (Claude Code)"
  },
  {
   "round": 1,
   "kind": "moby",
   "requested_bytes": 120000,
   "bytes_on_disk": 119997,
   "chars": 118055,
   "cl100k_tokens": 30379,
   "status": "ok",
   "context": 67082,
   "delta_vs_floor": 44106,
   "claude_code_version_at_runtime": "2.1.238 (Claude Code)"
  },
  {
   "round": 2,
   "kind": "floor",
   "status": "ok",
   "context": 22778,
   "claude_code_version_at_runtime": "2.1.238 (Claude Code)"
  },
  {
   "round": 2,
   "kind": "progit",
   "requested_bytes": 20000,
   "bytes_on_disk": 20000,
   "chars": 20000,
   "cl100k_tokens": 9300,
   "status": "ok",
   "context": 32867,
   "delta_vs_floor": 10089,
   "claude_code_version_at_runtime": "2.1.238 (Claude Code)"
  },
  {
   "round": 2,
   "kind": "progit",
   "requested_bytes": 60000,
   "bytes_on_disk": 59998,
   "chars": 59747,
   "cl100k_tokens": 18207,
   "status": "ok",
   "context": 45794,
   "delta_vs_floor": 23016,
   "claude_code_version_at_runtime": "2.1.238 (Claude Code)"
  },
  {
   "round": 2,
   "kind": "progit",
   "requested_bytes": 120000,
   "bytes_on_disk": 119997,
   "chars": 119292,
   "cl100k_tokens": 32144,
   "status": "ok",
   "context": 65067,
   "delta_vs_floor": 42289,
   "claude_code_version_at_runtime": "2.1.238 (Claude Code)"
  },
  {
   "round": 2,
   "kind": "moby",
   "requested_bytes": 20000,
   "bytes_on_disk": 19992,
   "chars": 19500,
   "cl100k_tokens": 5425,
   "status": "ok",
   "context": 30888,
   "delta_vs_floor": 8110,
   "claude_code_version_at_runtime": "2.1.238 (Claude Code)"
  },
  {
   "round": 2,
   "kind": "moby",
   "requested_bytes": 60000,
   "bytes_on_disk": 59997,
   "chars": 58845,
   "cl100k_tokens": 15425,
   "status": "ok",
   "context": 45924,
   "delta_vs_floor": 23146,
   "claude_code_version_at_runtime": "2.1.238 (Claude Code)"
  },
  {
   "round": 2,
   "kind": "moby",
   "requested_bytes": 120000,
   "bytes_on_disk": 119997,
   "chars": 118055,
   "cl100k_tokens": 30379,
   "status": "ok",
   "context": 67082,
   "delta_vs_floor": 44304,
   "claude_code_version_at_runtime": "2.1.238 (Claude Code)"
  },
  {
   "round": 3,
   "kind": "floor",
   "status": "ok",
   "context": 22976,
   "claude_code_version_at_runtime": "2.1.238 (Claude Code)"
  },
  {
   "round": 3,
   "kind": "progit",
   "requested_bytes": 20000,
   "bytes_on_disk": 20000,
   "chars": 20000,
   "cl100k_tokens": 9300,
   "status": "ok",
   "context": 32867,
   "delta_vs_floor": 9891,
   "claude_code_version_at_runtime": "2.1.238 (Claude Code)"
  },
  {
   "round": 3,
   "kind": "progit",
   "requested_bytes": 60000,
   "bytes_on_disk": 59998,
   "chars": 59747,
   "cl100k_tokens": 18207,
   "status": "ok",
   "context": 45794,
   "delta_vs_floor": 22818,
   "claude_code_version_at_runtime": "2.1.238 (Claude Code)"
  },
  {
   "round": 3,
   "kind": "progit",
   "requested_bytes": 120000,
   "bytes_on_disk": 119997,
   "chars": 119292,
   "cl100k_tokens": 32144,
   "status": "ok",
   "context": 65067,
   "delta_vs_floor": 42091,
   "claude_code_version_at_runtime": "2.1.238 (Claude Code)"
  },
  {
   "round": 3,
   "kind": "moby",
   "requested_bytes": 20000,
   "bytes_on_disk": 19992,
   "chars": 19500,
   "cl100k_tokens": 5425,
   "status": "ok",
   "context": 31785,
   "delta_vs_floor": 8809,
   "claude_code_version_at_runtime": "2.1.238 (Claude Code)"
  },
  {
   "round": 3,
   "kind": "moby",
   "requested_bytes": 60000,
   "bytes_on_disk": 59997,
   "chars": 58845,
   "cl100k_tokens": 15425,
   "status": "ok",
   "context": 45924,
   "delta_vs_floor": 22948,
   "claude_code_version_at_runtime": "2.1.238 (Claude Code)"
  },
  {
   "round": 3,
   "kind": "moby",
   "requested_bytes": 120000,
   "bytes_on_disk": 119997,
   "chars": 118055,
   "cl100k_tokens": 30379,
   "status": "ok",
   "context": 67082,
   "delta_vs_floor": 44106,
   "claude_code_version_at_runtime": "2.1.238 (Claude Code)"
  }
 ]
}