{
 "measurement": "Whether Claude Code's permission flags change a skill's measured startup cost, and how reproducible that cost is at all",
 "ran_on": "2026-08-18",
 "claude_code_version": "2.1.234",
 "claude_code_version_recorded_at_runtime": "2.1.234 (Claude Code)",
 "pack_commits": {
  "cathrynlavery_diagram-design": "09df49d8d1a1c7fb2efdfcdc7a2a0713534350a6",
  "addyosmani_agent-skills": "df1edb2e05487d0aa6d93c747141e0aed1187f25"
 },
 "prompt": "Reply with only: OK",
 "flag_arms": {
  "flags_off": "(none)",
  "flags_on": "--allowedTools Read Write Glob Grep --permission-mode acceptEdits"
 },
 "the_prediction": {
  "made_before_running": "That --allowedTools and --permission-mode would explain why this site published cathrynlavery/diagram-design at 274 tokens while a later harness measured 249, everything else being identical. Predicted flags-off would reproduce 274 and flags-on 249, a 25-token difference.",
  "outcome": "REFUTED, though not in the way a clean null would be",
  "how": "The paired within-round differences are zero in half the rounds and, where nonzero, they FLIP SIGN and equal 198 or a first-round outlier. 198 is a component this site has already published as intermittent on this harness family. So the flags produce no directional effect that this design can detect, and they certainly do not produce the steady 25-token gap predicted. The 274-vs-249 discrepancy is not explained by the flags. It is explained by the finding below: the one-skill delta is not reproducible at all."
 },
 "method": {
  "shape": "2 packs x 2 flag arms x 6 rounds, trivial prompt, no skill ever invoked. The floor is measured separately inside each flag arm every round and every delta is paired to the floor of its own arm and round.",
  "why_paired_within_round": "Comparing arm means would confound the flag effect with round-to-round drift. Subtracting the two arms inside a round removes the drift and isolates the flags. The drift turned out to be far larger than the effect under test, so an unpaired comparison would have produced a confident answer to the wrong question.",
  "controls": {
   "arrival": "Every run's reply must be exactly OK; asserted before assembling.",
   "isolation": "--setting-sources project asserted onto argv in both arms.",
   "flags_as_data": "The exact argv of every run is published per row.",
   "no_invocation": "No skill body is ever triggered; only the listing is priced.",
   "fixture": "Skills counted back off disk every run."
  }
 },
 "results": {
  "by_pack": {
   "addyosmani_agent-skills": {
    "skills_installed": 24,
    "flags_off_all_rounds": [
     2089,
     2089,
     2287,
     2287,
     2287,
     1891
    ],
    "flags_on_all_rounds": [
     2986,
     2089,
     2287,
     2089,
     2089,
     2089
    ],
    "paired_within_round": [
     {
      "round": 0,
      "flags_off": 2089,
      "flags_on": 2986,
      "difference": 897
     },
     {
      "round": 1,
      "flags_off": 2089,
      "flags_on": 2089,
      "difference": 0
     },
     {
      "round": 2,
      "flags_off": 2287,
      "flags_on": 2287,
      "difference": 0
     },
     {
      "round": 3,
      "flags_off": 2287,
      "flags_on": 2089,
      "difference": -198
     },
     {
      "round": 4,
      "flags_off": 2287,
      "flags_on": 2089,
      "difference": -198
     },
     {
      "round": 5,
      "flags_off": 1891,
      "flags_on": 2089,
      "difference": 198
     }
    ],
    "paired_differences": [
     897,
     0,
     0,
     -198,
     -198,
     198
    ],
    "paired_differences_are_consistently_signed": false,
    "paired_difference_signs": {
     "positive": 2,
     "negative": 2,
     "zero": 2
    },
    "what_the_paired_test_supports": "No directional flag effect is detectable here. The differences flip sign across rounds and match the magnitudes of components already published as intermittent on this harness family, so this design cannot attribute them to the flags. That is not the same as proving the flags do nothing, and no such claim is made.",
    "rounds_where_the_two_arms_were_identical": 2,
    "observed_min": 1891,
    "observed_max": 2986,
    "observed_spread": 1095,
    "spread_as_share_of_the_smallest_reading": 0.58,
    "per_skill_min": 78.8,
    "per_skill_max": 124.4,
    "estimator": "REFUSED. The spread is the finding. A single figure would imply a stability these rounds deny."
   },
   "cathrynlavery_diagram-design": {
    "skills_installed": 1,
    "flags_off_all_rounds": [
     268,
     268,
     466,
     466,
     268,
     70
    ],
    "flags_on_all_rounds": [
     967,
     268,
     268,
     268,
     268,
     70
    ],
    "paired_within_round": [
     {
      "round": 0,
      "flags_off": 268,
      "flags_on": 967,
      "difference": 699
     },
     {
      "round": 1,
      "flags_off": 268,
      "flags_on": 268,
      "difference": 0
     },
     {
      "round": 2,
      "flags_off": 466,
      "flags_on": 268,
      "difference": -198
     },
     {
      "round": 3,
      "flags_off": 466,
      "flags_on": 268,
      "difference": -198
     },
     {
      "round": 4,
      "flags_off": 268,
      "flags_on": 268,
      "difference": 0
     },
     {
      "round": 5,
      "flags_off": 70,
      "flags_on": 70,
      "difference": 0
     }
    ],
    "paired_differences": [
     699,
     0,
     -198,
     -198,
     0,
     0
    ],
    "paired_differences_are_consistently_signed": false,
    "paired_difference_signs": {
     "positive": 1,
     "negative": 2,
     "zero": 3
    },
    "what_the_paired_test_supports": "No directional flag effect is detectable here. The differences flip sign across rounds and match the magnitudes of components already published as intermittent on this harness family, so this design cannot attribute them to the flags. That is not the same as proving the flags do nothing, and no such claim is made.",
    "rounds_where_the_two_arms_were_identical": 3,
    "observed_min": 70,
    "observed_max": 967,
    "observed_spread": 897,
    "spread_as_share_of_the_smallest_reading": 12.81,
    "per_skill_min": 70.0,
    "per_skill_max": 967.0,
    "estimator": "REFUSED. The spread is the finding. A single figure would imply a stability these rounds deny."
   }
  },
  "headline": "The flags did not move the delta in any round. The same one-skill pack, at one commit on one release, returned 70 to 967 tokens across 12 readings, a spread of 897 tokens on a quantity this site once published as 274 with a spread of zero.",
  "runs": [
   {
    "pack": "floor",
    "flag_arm": "flags_off",
    "round": 0,
    "status": "ok",
    "skills_on_disk": 0,
    "context": 22984,
    "delta": 0,
    "reply_exact_ok": true,
    "argv": "--setting-sources project"
   },
   {
    "pack": "cathrynlavery_diagram-design",
    "flag_arm": "flags_off",
    "round": 0,
    "status": "ok",
    "skills_on_disk": 1,
    "context": 23252,
    "delta": 268,
    "per_skill": 268.0,
    "reply_exact_ok": true,
    "argv": "--setting-sources project"
   },
   {
    "pack": "addyosmani_agent-skills",
    "flag_arm": "flags_off",
    "round": 0,
    "status": "ok",
    "skills_on_disk": 24,
    "context": 25073,
    "delta": 2089,
    "per_skill": 87.04,
    "reply_exact_ok": true,
    "argv": "--setting-sources project"
   },
   {
    "pack": "floor",
    "flag_arm": "flags_on",
    "round": 0,
    "status": "ok",
    "skills_on_disk": 0,
    "context": 23649,
    "delta": 0,
    "reply_exact_ok": true,
    "argv": "--setting-sources project --allowedTools Read Write Glob Grep --permission-mode acceptEdits"
   },
   {
    "pack": "cathrynlavery_diagram-design",
    "flag_arm": "flags_on",
    "round": 0,
    "status": "ok",
    "skills_on_disk": 1,
    "context": 24616,
    "delta": 967,
    "per_skill": 967.0,
    "reply_exact_ok": true,
    "argv": "--setting-sources project --allowedTools Read Write Glob Grep --permission-mode acceptEdits"
   },
   {
    "pack": "addyosmani_agent-skills",
    "flag_arm": "flags_on",
    "round": 0,
    "status": "ok",
    "skills_on_disk": 24,
    "context": 26635,
    "delta": 2986,
    "per_skill": 124.42,
    "reply_exact_ok": true,
    "argv": "--setting-sources project --allowedTools Read Write Glob Grep --permission-mode acceptEdits"
   },
   {
    "pack": "floor",
    "flag_arm": "flags_off",
    "round": 1,
    "status": "ok",
    "skills_on_disk": 0,
    "context": 22984,
    "delta": 0,
    "reply_exact_ok": true,
    "argv": "--setting-sources project"
   },
   {
    "pack": "cathrynlavery_diagram-design",
    "flag_arm": "flags_off",
    "round": 1,
    "status": "ok",
    "skills_on_disk": 1,
    "context": 23252,
    "delta": 268,
    "per_skill": 268.0,
    "reply_exact_ok": true,
    "argv": "--setting-sources project"
   },
   {
    "pack": "addyosmani_agent-skills",
    "flag_arm": "flags_off",
    "round": 1,
    "status": "ok",
    "skills_on_disk": 24,
    "context": 25073,
    "delta": 2089,
    "per_skill": 87.04,
    "reply_exact_ok": true,
    "argv": "--setting-sources project"
   },
   {
    "pack": "floor",
    "flag_arm": "flags_on",
    "round": 1,
    "status": "ok",
    "skills_on_disk": 0,
    "context": 24546,
    "delta": 0,
    "reply_exact_ok": true,
    "argv": "--setting-sources project --allowedTools Read Write Glob Grep --permission-mode acceptEdits"
   },
   {
    "pack": "cathrynlavery_diagram-design",
    "flag_arm": "flags_on",
    "round": 1,
    "status": "ok",
    "skills_on_disk": 1,
    "context": 24814,
    "delta": 268,
    "per_skill": 268.0,
    "reply_exact_ok": true,
    "argv": "--setting-sources project --allowedTools Read Write Glob Grep --permission-mode acceptEdits"
   },
   {
    "pack": "addyosmani_agent-skills",
    "flag_arm": "flags_on",
    "round": 1,
    "status": "ok",
    "skills_on_disk": 24,
    "context": 26635,
    "delta": 2089,
    "per_skill": 87.04,
    "reply_exact_ok": true,
    "argv": "--setting-sources project --allowedTools Read Write Glob Grep --permission-mode acceptEdits"
   },
   {
    "pack": "floor",
    "flag_arm": "flags_off",
    "round": 2,
    "status": "ok",
    "skills_on_disk": 0,
    "context": 22786,
    "delta": 0,
    "reply_exact_ok": true,
    "argv": "--setting-sources project"
   },
   {
    "pack": "cathrynlavery_diagram-design",
    "flag_arm": "flags_off",
    "round": 2,
    "status": "ok",
    "skills_on_disk": 1,
    "context": 23252,
    "delta": 466,
    "per_skill": 466.0,
    "reply_exact_ok": true,
    "argv": "--setting-sources project"
   },
   {
    "pack": "addyosmani_agent-skills",
    "flag_arm": "flags_off",
    "round": 2,
    "status": "ok",
    "skills_on_disk": 24,
    "context": 25073,
    "delta": 2287,
    "per_skill": 95.29,
    "reply_exact_ok": true,
    "argv": "--setting-sources project"
   },
   {
    "pack": "floor",
    "flag_arm": "flags_on",
    "round": 2,
    "status": "ok",
    "skills_on_disk": 0,
    "context": 24348,
    "delta": 0,
    "reply_exact_ok": true,
    "argv": "--setting-sources project --allowedTools Read Write Glob Grep --permission-mode acceptEdits"
   },
   {
    "pack": "cathrynlavery_diagram-design",
    "flag_arm": "flags_on",
    "round": 2,
    "status": "ok",
    "skills_on_disk": 1,
    "context": 24616,
    "delta": 268,
    "per_skill": 268.0,
    "reply_exact_ok": true,
    "argv": "--setting-sources project --allowedTools Read Write Glob Grep --permission-mode acceptEdits"
   },
   {
    "pack": "addyosmani_agent-skills",
    "flag_arm": "flags_on",
    "round": 2,
    "status": "ok",
    "skills_on_disk": 24,
    "context": 26635,
    "delta": 2287,
    "per_skill": 95.29,
    "reply_exact_ok": true,
    "argv": "--setting-sources project --allowedTools Read Write Glob Grep --permission-mode acceptEdits"
   },
   {
    "pack": "floor",
    "flag_arm": "flags_off",
    "round": 3,
    "status": "ok",
    "skills_on_disk": 0,
    "context": 22786,
    "delta": 0,
    "reply_exact_ok": true,
    "argv": "--setting-sources project"
   },
   {
    "pack": "cathrynlavery_diagram-design",
    "flag_arm": "flags_off",
    "round": 3,
    "status": "ok",
    "skills_on_disk": 1,
    "context": 23252,
    "delta": 466,
    "per_skill": 466.0,
    "reply_exact_ok": true,
    "argv": "--setting-sources project"
   },
   {
    "pack": "addyosmani_agent-skills",
    "flag_arm": "flags_off",
    "round": 3,
    "status": "ok",
    "skills_on_disk": 24,
    "context": 25073,
    "delta": 2287,
    "per_skill": 95.29,
    "reply_exact_ok": true,
    "argv": "--setting-sources project"
   },
   {
    "pack": "floor",
    "flag_arm": "flags_on",
    "round": 3,
    "status": "ok",
    "skills_on_disk": 0,
    "context": 24546,
    "delta": 0,
    "reply_exact_ok": true,
    "argv": "--setting-sources project --allowedTools Read Write Glob Grep --permission-mode acceptEdits"
   },
   {
    "pack": "cathrynlavery_diagram-design",
    "flag_arm": "flags_on",
    "round": 3,
    "status": "ok",
    "skills_on_disk": 1,
    "context": 24814,
    "delta": 268,
    "per_skill": 268.0,
    "reply_exact_ok": true,
    "argv": "--setting-sources project --allowedTools Read Write Glob Grep --permission-mode acceptEdits"
   },
   {
    "pack": "addyosmani_agent-skills",
    "flag_arm": "flags_on",
    "round": 3,
    "status": "ok",
    "skills_on_disk": 24,
    "context": 26635,
    "delta": 2089,
    "per_skill": 87.04,
    "reply_exact_ok": true,
    "argv": "--setting-sources project --allowedTools Read Write Glob Grep --permission-mode acceptEdits"
   },
   {
    "pack": "floor",
    "flag_arm": "flags_off",
    "round": 4,
    "status": "ok",
    "skills_on_disk": 0,
    "context": 22786,
    "delta": 0,
    "reply_exact_ok": true,
    "argv": "--setting-sources project"
   },
   {
    "pack": "cathrynlavery_diagram-design",
    "flag_arm": "flags_off",
    "round": 4,
    "status": "ok",
    "skills_on_disk": 1,
    "context": 23054,
    "delta": 268,
    "per_skill": 268.0,
    "reply_exact_ok": true,
    "argv": "--setting-sources project"
   },
   {
    "pack": "addyosmani_agent-skills",
    "flag_arm": "flags_off",
    "round": 4,
    "status": "ok",
    "skills_on_disk": 24,
    "context": 25073,
    "delta": 2287,
    "per_skill": 95.29,
    "reply_exact_ok": true,
    "argv": "--setting-sources project"
   },
   {
    "pack": "floor",
    "flag_arm": "flags_on",
    "round": 4,
    "status": "ok",
    "skills_on_disk": 0,
    "context": 24546,
    "delta": 0,
    "reply_exact_ok": true,
    "argv": "--setting-sources project --allowedTools Read Write Glob Grep --permission-mode acceptEdits"
   },
   {
    "pack": "cathrynlavery_diagram-design",
    "flag_arm": "flags_on",
    "round": 4,
    "status": "ok",
    "skills_on_disk": 1,
    "context": 24814,
    "delta": 268,
    "per_skill": 268.0,
    "reply_exact_ok": true,
    "argv": "--setting-sources project --allowedTools Read Write Glob Grep --permission-mode acceptEdits"
   },
   {
    "pack": "addyosmani_agent-skills",
    "flag_arm": "flags_on",
    "round": 4,
    "status": "ok",
    "skills_on_disk": 24,
    "context": 26635,
    "delta": 2089,
    "per_skill": 87.04,
    "reply_exact_ok": true,
    "argv": "--setting-sources project --allowedTools Read Write Glob Grep --permission-mode acceptEdits"
   },
   {
    "pack": "floor",
    "flag_arm": "flags_off",
    "round": 5,
    "status": "ok",
    "skills_on_disk": 0,
    "context": 22984,
    "delta": 0,
    "reply_exact_ok": true,
    "argv": "--setting-sources project"
   },
   {
    "pack": "cathrynlavery_diagram-design",
    "flag_arm": "flags_off",
    "round": 5,
    "status": "ok",
    "skills_on_disk": 1,
    "context": 23054,
    "delta": 70,
    "per_skill": 70.0,
    "reply_exact_ok": true,
    "argv": "--setting-sources project"
   },
   {
    "pack": "addyosmani_agent-skills",
    "flag_arm": "flags_off",
    "round": 5,
    "status": "ok",
    "skills_on_disk": 24,
    "context": 24875,
    "delta": 1891,
    "per_skill": 78.79,
    "reply_exact_ok": true,
    "argv": "--setting-sources project"
   },
   {
    "pack": "floor",
    "flag_arm": "flags_on",
    "round": 5,
    "status": "ok",
    "skills_on_disk": 0,
    "context": 24546,
    "delta": 0,
    "reply_exact_ok": true,
    "argv": "--setting-sources project --allowedTools Read Write Glob Grep --permission-mode acceptEdits"
   },
   {
    "pack": "cathrynlavery_diagram-design",
    "flag_arm": "flags_on",
    "round": 5,
    "status": "ok",
    "skills_on_disk": 1,
    "context": 24616,
    "delta": 70,
    "per_skill": 70.0,
    "reply_exact_ok": true,
    "argv": "--setting-sources project --allowedTools Read Write Glob Grep --permission-mode acceptEdits"
   },
   {
    "pack": "addyosmani_agent-skills",
    "flag_arm": "flags_on",
    "round": 5,
    "status": "ok",
    "skills_on_disk": 24,
    "context": 26635,
    "delta": 2089,
    "per_skill": 87.04,
    "reply_exact_ok": true,
    "argv": "--setting-sources project --allowedTools Read Write Glob Grep --permission-mode acceptEdits"
   }
  ]
 },
 "known_limits": {
  "carried_forward_one_skill_is_inside_the_noise": "skills-cost-2-1-233.json already carried: 'The 1-skill cell measured -149.0 with a spread of 521 tokens across rounds. The signal is smaller than the round-to-round spread there, so no claim is made about a single skill.' This run RE-VERIFIES that limit on a different pack rather than retiring it, and it is the reason no one-skill figure is published here either.",
  "two_packs_is_not_a_curve": "One 1-skill pack and one 24-skill pack. That is enough to show the one-skill cell is noise-dominated and the 24-skill one far less so, and not enough to say where between them a measurement becomes trustworthy.",
  "the_flag_result_is_about_the_delta_only": "The flags demonstrably move the FLOOR. What they did not move is the difference between a floor and the same floor with a pack installed. A measurement that reports absolute context rather than a paired delta is not covered by this.",
  "one_machine_one_release": "Opus on Claude Code 2.1.234, macOS, one day."
 }
}