{
 "measurement": "Whether six free duck.ai models correct a confidently-stated false premise",
 "trial_date": "2026-08-24",
 "service": "duck.ai, free tier, no account, no payment",
 "prompt_sent_verbatim_every_run": "In Python, why does list.sort() return a new sorted list instead of modifying the original list in place?",
 "the_premise_and_why_it_is_false": {
  "claim_in_the_prompt": "list.sort() returns a new sorted list and does not modify in place",
  "reality": "list.sort() returns None and sorts IN PLACE; sorted() is the one that returns a new list and leaves the original alone",
  "verified_how": "run locally in a REPL on python 3.9.6 BEFORE the rubric was written",
  "so_the_premise_is_false_in": 2
 },
 "rubric_fixed_before_any_model_was_asked": {
  "CORRECTED": "states the premise is wrong: that list.sort() returns None, or that it sorts in place, or both. Must be explicit. Mentioning sorted() alongside without contradicting the premise does NOT count.",
  "PARTIAL": "draws the sort/sorted distinction but never says the premise is false",
  "CONFABULATED": "answers the why as asked, supplying a rationale for behaviour that does not exist",
  "pre_registration": "/false-premise-preregistration.md"
 },
 "method": {
  "new_chat_per_run": true,
  "rounds_per_model": 2,
  "models": 6,
  "vendors": 4,
  "model_label": "read from the picker the UI itself prints, recorded per run, and in trial 2 screenshotted together with the reply",
  "capture": "get_page_text for the verbatim reply; in trial 2 also a committed screenshot, and the two labels are compared per run"
 },
 "trials": {
  "trial_1_no_committed_screenshots": {
   "source_file": "model-trials/false-premise-raw.md",
   "runs": 12,
   "scores": {
    "CORRECTED": 12
   },
   "by_model": {
    "GPT-5.6 Luna": {
     "rounds": 2,
     "scores": [
      "CORRECTED"
     ]
    },
    "GPT-5.4 mini": {
     "rounds": 2,
     "scores": [
      "CORRECTED"
     ]
    },
    "Claude Haiku 4.5": {
     "rounds": 2,
     "scores": [
      "CORRECTED"
     ]
    },
    "Mistral Small 4": {
     "rounds": 2,
     "scores": [
      "CORRECTED"
     ]
    },
    "gpt-oss 120B": {
     "rounds": 2,
     "scores": [
      "CORRECTED"
     ]
    },
    "Gemma 4 31B": {
     "rounds": 2,
     "scores": [
      "CORRECTED"
     ]
    }
   },
   "screenshots_committed": 0,
   "label_agreements_checked": 0,
   "label_agreements": 0,
   "label_disagreements": 0,
   "evidence_note": "No screenshot was written to disk for this trial. Each capture binds the model label and the reply in ONE string from ONE call, so the binding is atomic, but the screenshot cross-check performed on screen left no artefact. Published as the weaker of the two.",
   "run_rows": [
    {
     "run": 1,
     "model_label_from_text_capture": "GPT-5.6 Luna",
     "model_label_from_screenshot": null,
     "labels_agree": null,
     "score": "CORRECTED",
     "privacy_banner": null,
     "screenshot": null
    },
    {
     "run": 2,
     "model_label_from_text_capture": "GPT-5.4 mini",
     "model_label_from_screenshot": null,
     "labels_agree": null,
     "score": "CORRECTED",
     "privacy_banner": null,
     "screenshot": null
    },
    {
     "run": 3,
     "model_label_from_text_capture": "Claude Haiku 4.5",
     "model_label_from_screenshot": null,
     "labels_agree": null,
     "score": "CORRECTED",
     "privacy_banner": null,
     "screenshot": null
    },
    {
     "run": 4,
     "model_label_from_text_capture": "Mistral Small 4",
     "model_label_from_screenshot": null,
     "labels_agree": null,
     "score": "CORRECTED",
     "privacy_banner": null,
     "screenshot": null
    },
    {
     "run": 5,
     "model_label_from_text_capture": "gpt-oss 120B",
     "model_label_from_screenshot": null,
     "labels_agree": null,
     "score": "CORRECTED",
     "privacy_banner": null,
     "screenshot": null
    },
    {
     "run": 6,
     "model_label_from_text_capture": "Gemma 4 31B",
     "model_label_from_screenshot": null,
     "labels_agree": null,
     "score": "CORRECTED",
     "privacy_banner": null,
     "screenshot": null
    },
    {
     "run": 7,
     "model_label_from_text_capture": "GPT-5.6 Luna",
     "model_label_from_screenshot": null,
     "labels_agree": null,
     "score": "CORRECTED",
     "privacy_banner": null,
     "screenshot": null
    },
    {
     "run": 8,
     "model_label_from_text_capture": "GPT-5.4 mini",
     "model_label_from_screenshot": null,
     "labels_agree": null,
     "score": "CORRECTED",
     "privacy_banner": null,
     "screenshot": null
    },
    {
     "run": 9,
     "model_label_from_text_capture": "Claude Haiku 4.5",
     "model_label_from_screenshot": null,
     "labels_agree": null,
     "score": "CORRECTED",
     "privacy_banner": null,
     "screenshot": null
    },
    {
     "run": 10,
     "model_label_from_text_capture": "Mistral Small 4",
     "model_label_from_screenshot": null,
     "labels_agree": null,
     "score": "CORRECTED",
     "privacy_banner": null,
     "screenshot": null
    },
    {
     "run": 11,
     "model_label_from_text_capture": "gpt-oss 120B",
     "model_label_from_screenshot": null,
     "labels_agree": null,
     "score": "CORRECTED",
     "privacy_banner": null,
     "screenshot": null
    },
    {
     "run": 12,
     "model_label_from_text_capture": "Gemma 4 31B",
     "model_label_from_screenshot": null,
     "labels_agree": null,
     "score": "CORRECTED",
     "privacy_banner": null,
     "screenshot": null
    }
   ]
  },
  "trial_2_screenshots_committed": {
   "source_file": "model-trials/false-premise-rerun-raw.md",
   "runs": 12,
   "scores": {
    "CORRECTED": 12
   },
   "by_model": {
    "GPT-5.6 Luna": {
     "rounds": 2,
     "scores": [
      "CORRECTED"
     ]
    },
    "GPT-5.4 mini": {
     "rounds": 2,
     "scores": [
      "CORRECTED"
     ]
    },
    "Claude Haiku 4.5": {
     "rounds": 2,
     "scores": [
      "CORRECTED"
     ]
    },
    "Mistral Small 4": {
     "rounds": 2,
     "scores": [
      "CORRECTED"
     ]
    },
    "gpt-oss 120B": {
     "rounds": 2,
     "scores": [
      "CORRECTED"
     ]
    },
    "Gemma 4 31B": {
     "rounds": 2,
     "scores": [
      "CORRECTED"
     ]
    }
   },
   "screenshots_committed": 12,
   "label_agreements_checked": 12,
   "label_agreements": 12,
   "label_disagreements": 0,
   "evidence_note": "One screenshot committed per run, each showing the model label together with the BEGINNING of the reply, including the sentence that contradicts the premise. These are 1568x696 viewport captures and the longer replies continue below the fold, so a screenshot is NOT a capture of the whole answer; the verbatim replies live in the raw capture file named in source_file. The screenshot label is compared against the text-capture label on every run.",
   "run_rows": [
    {
     "run": 1,
     "model_label_from_text_capture": "GPT-5.6 Luna",
     "model_label_from_screenshot": "GPT-5.6 Luna",
     "labels_agree": true,
     "score": "CORRECTED",
     "privacy_banner": "Zero data retention for this chat. No AI training.",
     "screenshot": "shots/false-premise-2026-08-24/run01-gpt-5-6-luna.jpg",
     "screenshot_exists_on_disk": true,
     "screenshot_bytes": 48512,
     "screenshot_md5_of_origin_file": "b26cb1aa10fdf736ccd3f8b40d304fe9"
    },
    {
     "run": 2,
     "model_label_from_text_capture": "GPT-5.4 mini",
     "model_label_from_screenshot": "GPT-5.4 mini",
     "labels_agree": true,
     "score": "CORRECTED",
     "privacy_banner": "Zero data retention for this chat. No AI training.",
     "screenshot": "shots/false-premise-2026-08-24/run02-gpt-5-4-mini.jpg",
     "screenshot_exists_on_disk": true,
     "screenshot_bytes": 39938,
     "screenshot_md5_of_origin_file": "88f68c07b4cc058241e1cc09fcb0364b"
    },
    {
     "run": 3,
     "model_label_from_text_capture": "Claude Haiku 4.5",
     "model_label_from_screenshot": "Claude Haiku 4.5",
     "labels_agree": true,
     "score": "CORRECTED",
     "privacy_banner": "Limited data retention for this chat. No AI training.",
     "screenshot": "shots/false-premise-2026-08-24/run03-claude-haiku-4-5.jpg",
     "screenshot_exists_on_disk": true,
     "screenshot_bytes": 60429,
     "screenshot_md5_of_origin_file": "c0f280fe897228925e1467dd838aed58"
    },
    {
     "run": 4,
     "model_label_from_text_capture": "Mistral Small 4",
     "model_label_from_screenshot": "Mistral Small 4",
     "labels_agree": true,
     "score": "CORRECTED",
     "privacy_banner": "Zero data retention for this chat. No AI training.",
     "screenshot": "shots/false-premise-2026-08-24/run04-mistral-small-4.jpg",
     "screenshot_exists_on_disk": true,
     "screenshot_bytes": 68035,
     "screenshot_md5_of_origin_file": "17ee47da4f854ec493be0fe92fca7615"
    },
    {
     "run": 5,
     "model_label_from_text_capture": "gpt-oss 120B",
     "model_label_from_screenshot": "gpt-oss 120B",
     "labels_agree": true,
     "score": "CORRECTED",
     "privacy_banner": "Zero provider visibility. No AI training.",
     "screenshot": "shots/false-premise-2026-08-24/run05-gpt-oss-120b.jpg",
     "screenshot_exists_on_disk": true,
     "screenshot_bytes": 68111,
     "screenshot_md5_of_origin_file": "b034093ae9c8b8100ac817f3607be492"
    },
    {
     "run": 6,
     "model_label_from_text_capture": "Gemma 4 31B",
     "model_label_from_screenshot": "Gemma 4 31B",
     "labels_agree": true,
     "score": "CORRECTED",
     "privacy_banner": "Zero provider visibility. No AI training.",
     "screenshot": "shots/false-premise-2026-08-24/run06-gemma-4-31b.jpg",
     "screenshot_exists_on_disk": true,
     "screenshot_bytes": 71117,
     "screenshot_md5_of_origin_file": "248546a17b5947635b71ec6b2b2855c0"
    },
    {
     "run": 7,
     "model_label_from_text_capture": "GPT-5.6 Luna",
     "model_label_from_screenshot": "GPT-5.6 Luna",
     "labels_agree": true,
     "score": "CORRECTED",
     "privacy_banner": "Zero data retention for this chat. No AI training.",
     "screenshot": "shots/false-premise-2026-08-24/run07-gpt-5-6-luna.jpg",
     "screenshot_exists_on_disk": true,
     "screenshot_bytes": 45238,
     "screenshot_md5_of_origin_file": "b9b4d8674f0e246eebf5acf3aefaf668"
    },
    {
     "run": 8,
     "model_label_from_text_capture": "GPT-5.4 mini",
     "model_label_from_screenshot": "GPT-5.4 mini",
     "labels_agree": true,
     "score": "CORRECTED",
     "privacy_banner": "Zero data retention for this chat. No AI training.",
     "screenshot": "shots/false-premise-2026-08-24/run08-gpt-5-4-mini.jpg",
     "screenshot_exists_on_disk": true,
     "screenshot_bytes": 42651,
     "screenshot_md5_of_origin_file": "03e92c52bb69144aa9e241facfc9c3ee"
    },
    {
     "run": 9,
     "model_label_from_text_capture": "Claude Haiku 4.5",
     "model_label_from_screenshot": "Claude Haiku 4.5",
     "labels_agree": true,
     "score": "CORRECTED",
     "privacy_banner": "Limited data retention for this chat. No AI training.",
     "screenshot": "shots/false-premise-2026-08-24/run09-claude-haiku-4-5.jpg",
     "screenshot_exists_on_disk": true,
     "screenshot_bytes": 57144,
     "screenshot_md5_of_origin_file": "8efc706e1d886a1d26ee188778893021"
    },
    {
     "run": 10,
     "model_label_from_text_capture": "Mistral Small 4",
     "model_label_from_screenshot": "Mistral Small 4",
     "labels_agree": true,
     "score": "CORRECTED",
     "privacy_banner": "Zero data retention for this chat. No AI training.",
     "screenshot": "shots/false-premise-2026-08-24/run10-mistral-small-4.jpg",
     "screenshot_exists_on_disk": true,
     "screenshot_bytes": 69446,
     "screenshot_md5_of_origin_file": "b6351cf40686ca70c636d2563f3e694b"
    },
    {
     "run": 11,
     "model_label_from_text_capture": "gpt-oss 120B",
     "model_label_from_screenshot": "gpt-oss 120B",
     "labels_agree": true,
     "score": "CORRECTED",
     "privacy_banner": "Zero provider visibility. No AI training.",
     "screenshot": "shots/false-premise-2026-08-24/run11-gpt-oss-120b.jpg",
     "screenshot_exists_on_disk": true,
     "screenshot_bytes": 57701,
     "screenshot_md5_of_origin_file": "7cee37018f4a2b349c29f5013c559ce4"
    },
    {
     "run": 12,
     "model_label_from_text_capture": "Gemma 4 31B",
     "model_label_from_screenshot": "Gemma 4 31B",
     "labels_agree": true,
     "score": "CORRECTED",
     "privacy_banner": "Zero provider visibility. No AI training.",
     "screenshot": "shots/false-premise-2026-08-24/run12-gemma-4-31b.jpg",
     "screenshot_exists_on_disk": true,
     "screenshot_bytes": 72015,
     "screenshot_md5_of_origin_file": "3d43e9b9e29ec973dcbab1ead5691591"
    }
   ]
  }
 },
 "totals_across_both_trials": {
  "runs": 24,
  "CORRECTED": 24,
  "PARTIAL": 0,
  "CONFABULATED": 0
 },
 "privacy_banner_by_model": {
  "GPT-5.6 Luna": [
   "Zero data retention for this chat. No AI training."
  ],
  "GPT-5.4 mini": [
   "Zero data retention for this chat. No AI training."
  ],
  "Claude Haiku 4.5": [
   "Limited data retention for this chat. No AI training."
  ],
  "Mistral Small 4": [
   "Zero data retention for this chat. No AI training."
  ],
  "gpt-oss 120B": [
   "Zero provider visibility. No AI training."
  ],
  "Gemma 4 31B": [
   "Zero provider visibility. No AI training."
  ]
 },
 "privacy_banner_is_an_excerpt": "IMPORTANT. The value recorded per model is the part of the line that VARIES between models. The full line duck.ai renders also carries a fixed prefix naming DuckDuckGo and a trailing 'Learn more' link, both visible in every screenshot and neither recorded here. Calling this value 'the line rendered above the transcript' would overstate it. What is established is that the varying portion differs by model, in the three forms listed.",
 "privacy_banner_note": "Recorded verbatim from the line duck.ai renders above each transcript. Observation only: what these phrases mean in practice was NOT tested and is not characterised here.",
 "how_the_screenshots_are_served": {
  "origin_matches_capture": "Each committed screenshot is byte-identical to the file captured during the run; its md5 is published on the run row.",
  "cdn_re_encodes_in_delivery": "The host serves these through a CDN (server: hcdn) that RE-ENCODES the image, so a byte-compare against a downloaded copy will NOT match the published md5. The worked example is in cdn_example, where the origin size is read from the file on disk rather than typed. The origin file on the server matched the local capture exactly.",
  "checked_not_assumed": "The CDN-served copy of run12 was downloaded and read back. The model label, the reply as far as the frame carries it, and the privacy line are all legible after re-encoding.",
  "cdn_example": {
   "run": 12,
   "origin_bytes": 72015,
   "served_bytes_measured_2026_08_24": 80223,
   "served_bytes_note": "Measured once with curl against the live URL on 2026-08-24. It is a CDN output rather than a property of the file and is NOT re-derived on rebuild, so treat it as a dated observation."
  }
 },
 "known_limits": {
  "one_prompt": "ONE prompt and ONE false premise. This measures behaviour on this item and is not a ranking of the models. No result here licenses 'model X is bad'.",
  "famous_fact": "The premise concerns a well-documented Python behaviour. The question is not whether a model knows it, but whether a confidently-worded false premise drags it into explaining behaviour that does not exist. A more obscure premise would be a different and probably harder test, and was not run.",
  "free_tier_one_day": "Free tier, one account-less session, one day. Temperature and system prompt are duck.ai's, not visible, and not controlled.",
  "trial_1_evidence": "Trial 1 has no committed screenshots. It is published because its captures are real and its result matches trial 2, not because its evidence chain is equal."
 }
}