{
  "generated_at_utc": "2026-09-11T00:38:19.281560+00:00",
  "groups": [
    {
      "provider": "gpt",
      "condition": "original",
      "subset": "Hard20",
      "model": "gpt-5.6-luna",
      "questions": 20,
      "expected_slots": 80,
      "nonempty_answers": 80,
      "human_adjudicated_slots": 0,
      "valid_labels": 80,
      "correct_slots": 4,
      "incorrect_slots": 76,
      "first_roll_correct": 2,
      "pass_at_4_questions": 3,
      "stable_at_4_questions": 0,
      "acc_at_1": 0.1,
      "mean_acc_4": 0.05,
      "pass_at_4": 0.15,
      "stable_at_4": 0.0,
      "correct_roll_distribution": {
        "0": 17,
        "1": 2,
        "2": 1,
        "3": 0,
        "4": 0
      },
      "missing_slots": 0,
      "judge_error_slots": 0,
      "metric_question_ids": [
        "GT-0019",
        "GT-0050",
        "GT-0061",
        "GT-0102",
        "GT-0164",
        "GT-0216",
        "GT-0244",
        "GT-0248",
        "GT-0288",
        "GT-0291",
        "GT-0302",
        "GT-0361",
        "GT-0382",
        "GT-0395",
        "GT-0411",
        "GT-0514",
        "GT-0516",
        "GT-0554",
        "GT-0560",
        "GT-0585"
      ],
      "coverage": {
        "planned_questions": 20,
        "planned_slots": 80,
        "valid_labels": 80,
        "missing_slot_keys": [],
        "archived_complete_answers": 80,
        "human_adjudicated_slots": 0,
        "technical_missing_slot_keys": [],
        "complete_question_ids": [
          "GT-0019",
          "GT-0050",
          "GT-0061",
          "GT-0102",
          "GT-0164",
          "GT-0216",
          "GT-0244",
          "GT-0248",
          "GT-0288",
          "GT-0291",
          "GT-0302",
          "GT-0361",
          "GT-0382",
          "GT-0395",
          "GT-0411",
          "GT-0514",
          "GT-0516",
          "GT-0554",
          "GT-0560",
          "GT-0585"
        ],
        "incomplete_question_ids": [],
        "excluded_valid_labels_from_metrics": 0,
        "missing_is_not_incorrect": true
      },
      "confidence_intervals_95": {
        "mean_acc_4": [
          0.0,
          0.1125
        ],
        "acc_at_1": [
          0.027866481213768224,
          0.3010336452284872
        ],
        "pass_at_4": [
          0.05236874589621657,
          0.3604188647407569
        ],
        "stable_at_4": [
          0.0,
          0.16112515805281927
        ]
      }
    },
    {
      "provider": "gpt",
      "condition": "text_only",
      "subset": "Hard20",
      "model": "gpt-5.6-luna",
      "questions": 20,
      "expected_slots": 80,
      "nonempty_answers": 80,
      "human_adjudicated_slots": 0,
      "valid_labels": 80,
      "correct_slots": 0,
      "incorrect_slots": 80,
      "first_roll_correct": 0,
      "pass_at_4_questions": 0,
      "stable_at_4_questions": 0,
      "acc_at_1": 0.0,
      "mean_acc_4": 0.0,
      "pass_at_4": 0.0,
      "stable_at_4": 0.0,
      "correct_roll_distribution": {
        "0": 20,
        "1": 0,
        "2": 0,
        "3": 0,
        "4": 0
      },
      "missing_slots": 0,
      "judge_error_slots": 0,
      "metric_question_ids": [
        "GT-0019",
        "GT-0050",
        "GT-0061",
        "GT-0102",
        "GT-0164",
        "GT-0216",
        "GT-0244",
        "GT-0248",
        "GT-0288",
        "GT-0291",
        "GT-0302",
        "GT-0361",
        "GT-0382",
        "GT-0395",
        "GT-0411",
        "GT-0514",
        "GT-0516",
        "GT-0554",
        "GT-0560",
        "GT-0585"
      ],
      "coverage": {
        "planned_questions": 20,
        "planned_slots": 80,
        "valid_labels": 80,
        "missing_slot_keys": [],
        "archived_complete_answers": 80,
        "human_adjudicated_slots": 0,
        "technical_missing_slot_keys": [],
        "complete_question_ids": [
          "GT-0019",
          "GT-0050",
          "GT-0061",
          "GT-0102",
          "GT-0164",
          "GT-0216",
          "GT-0244",
          "GT-0248",
          "GT-0288",
          "GT-0291",
          "GT-0302",
          "GT-0361",
          "GT-0382",
          "GT-0395",
          "GT-0411",
          "GT-0514",
          "GT-0516",
          "GT-0554",
          "GT-0560",
          "GT-0585"
        ],
        "incomplete_question_ids": [],
        "excluded_valid_labels_from_metrics": 0,
        "missing_is_not_incorrect": true
      },
      "confidence_intervals_95": {
        "mean_acc_4": [
          0.0,
          0.0
        ],
        "acc_at_1": [
          0.0,
          0.16112515805281927
        ],
        "pass_at_4": [
          0.0,
          0.16112515805281927
        ],
        "stable_at_4": [
          0.0,
          0.16112515805281927
        ]
      }
    },
    {
      "provider": "gemini",
      "condition": "original",
      "subset": "Hard20",
      "model": "gemini-3.6-flash",
      "questions": 20,
      "expected_slots": 80,
      "nonempty_answers": 80,
      "human_adjudicated_slots": 0,
      "valid_labels": 80,
      "correct_slots": 2,
      "incorrect_slots": 78,
      "first_roll_correct": 1,
      "pass_at_4_questions": 1,
      "stable_at_4_questions": 0,
      "acc_at_1": 0.05,
      "mean_acc_4": 0.025,
      "pass_at_4": 0.05,
      "stable_at_4": 0.0,
      "correct_roll_distribution": {
        "0": 19,
        "1": 0,
        "2": 1,
        "3": 0,
        "4": 0
      },
      "missing_slots": 0,
      "judge_error_slots": 0,
      "metric_question_ids": [
        "GT-0019",
        "GT-0050",
        "GT-0061",
        "GT-0102",
        "GT-0164",
        "GT-0216",
        "GT-0244",
        "GT-0248",
        "GT-0288",
        "GT-0291",
        "GT-0302",
        "GT-0361",
        "GT-0382",
        "GT-0395",
        "GT-0411",
        "GT-0514",
        "GT-0516",
        "GT-0554",
        "GT-0560",
        "GT-0585"
      ],
      "coverage": {
        "planned_questions": 20,
        "planned_slots": 80,
        "valid_labels": 80,
        "missing_slot_keys": [],
        "archived_complete_answers": 80,
        "human_adjudicated_slots": 0,
        "technical_missing_slot_keys": [],
        "complete_question_ids": [
          "GT-0019",
          "GT-0050",
          "GT-0061",
          "GT-0102",
          "GT-0164",
          "GT-0216",
          "GT-0244",
          "GT-0248",
          "GT-0288",
          "GT-0291",
          "GT-0302",
          "GT-0361",
          "GT-0382",
          "GT-0395",
          "GT-0411",
          "GT-0514",
          "GT-0516",
          "GT-0554",
          "GT-0560",
          "GT-0585"
        ],
        "incomplete_question_ids": [],
        "excluded_valid_labels_from_metrics": 0,
        "missing_is_not_incorrect": true
      },
      "confidence_intervals_95": {
        "mean_acc_4": [
          0.0,
          0.075
        ],
        "acc_at_1": [
          0.00888144880079543,
          0.23613119344674194
        ],
        "pass_at_4": [
          0.00888144880079543,
          0.23613119344674194
        ],
        "stable_at_4": [
          0.0,
          0.16112515805281927
        ]
      }
    },
    {
      "provider": "gemini",
      "condition": "text_only",
      "subset": "Hard20",
      "model": "gemini-3.6-flash",
      "questions": 20,
      "expected_slots": 80,
      "nonempty_answers": 80,
      "human_adjudicated_slots": 0,
      "valid_labels": 80,
      "correct_slots": 0,
      "incorrect_slots": 80,
      "first_roll_correct": 0,
      "pass_at_4_questions": 0,
      "stable_at_4_questions": 0,
      "acc_at_1": 0.0,
      "mean_acc_4": 0.0,
      "pass_at_4": 0.0,
      "stable_at_4": 0.0,
      "correct_roll_distribution": {
        "0": 20,
        "1": 0,
        "2": 0,
        "3": 0,
        "4": 0
      },
      "missing_slots": 0,
      "judge_error_slots": 0,
      "metric_question_ids": [
        "GT-0019",
        "GT-0050",
        "GT-0061",
        "GT-0102",
        "GT-0164",
        "GT-0216",
        "GT-0244",
        "GT-0248",
        "GT-0288",
        "GT-0291",
        "GT-0302",
        "GT-0361",
        "GT-0382",
        "GT-0395",
        "GT-0411",
        "GT-0514",
        "GT-0516",
        "GT-0554",
        "GT-0560",
        "GT-0585"
      ],
      "coverage": {
        "planned_questions": 20,
        "planned_slots": 80,
        "valid_labels": 80,
        "missing_slot_keys": [],
        "archived_complete_answers": 80,
        "human_adjudicated_slots": 0,
        "technical_missing_slot_keys": [],
        "complete_question_ids": [
          "GT-0019",
          "GT-0050",
          "GT-0061",
          "GT-0102",
          "GT-0164",
          "GT-0216",
          "GT-0244",
          "GT-0248",
          "GT-0288",
          "GT-0291",
          "GT-0302",
          "GT-0361",
          "GT-0382",
          "GT-0395",
          "GT-0411",
          "GT-0514",
          "GT-0516",
          "GT-0554",
          "GT-0560",
          "GT-0585"
        ],
        "incomplete_question_ids": [],
        "excluded_valid_labels_from_metrics": 0,
        "missing_is_not_incorrect": true
      },
      "confidence_intervals_95": {
        "mean_acc_4": [
          0.0,
          0.0
        ],
        "acc_at_1": [
          0.0,
          0.16112515805281927
        ],
        "pass_at_4": [
          0.0,
          0.16112515805281927
        ],
        "stable_at_4": [
          0.0,
          0.16112515805281927
        ]
      }
    },
    {
      "provider": "claude",
      "condition": "original",
      "subset": "Hard20",
      "model": "claude-opus-4-8",
      "questions": 20,
      "expected_slots": 80,
      "nonempty_answers": 80,
      "human_adjudicated_slots": 0,
      "valid_labels": 80,
      "correct_slots": 1,
      "incorrect_slots": 79,
      "first_roll_correct": 0,
      "pass_at_4_questions": 1,
      "stable_at_4_questions": 0,
      "acc_at_1": 0.0,
      "mean_acc_4": 0.0125,
      "pass_at_4": 0.05,
      "stable_at_4": 0.0,
      "correct_roll_distribution": {
        "0": 19,
        "1": 1,
        "2": 0,
        "3": 0,
        "4": 0
      },
      "missing_slots": 0,
      "judge_error_slots": 0,
      "metric_question_ids": [
        "GT-0019",
        "GT-0050",
        "GT-0061",
        "GT-0102",
        "GT-0164",
        "GT-0216",
        "GT-0244",
        "GT-0248",
        "GT-0288",
        "GT-0291",
        "GT-0302",
        "GT-0361",
        "GT-0382",
        "GT-0395",
        "GT-0411",
        "GT-0514",
        "GT-0516",
        "GT-0554",
        "GT-0560",
        "GT-0585"
      ],
      "coverage": {
        "planned_questions": 20,
        "planned_slots": 80,
        "valid_labels": 80,
        "missing_slot_keys": [],
        "archived_complete_answers": 80,
        "human_adjudicated_slots": 0,
        "technical_missing_slot_keys": [],
        "complete_question_ids": [
          "GT-0019",
          "GT-0050",
          "GT-0061",
          "GT-0102",
          "GT-0164",
          "GT-0216",
          "GT-0244",
          "GT-0248",
          "GT-0288",
          "GT-0291",
          "GT-0302",
          "GT-0361",
          "GT-0382",
          "GT-0395",
          "GT-0411",
          "GT-0514",
          "GT-0516",
          "GT-0554",
          "GT-0560",
          "GT-0585"
        ],
        "incomplete_question_ids": [],
        "excluded_valid_labels_from_metrics": 0,
        "missing_is_not_incorrect": true
      },
      "confidence_intervals_95": {
        "mean_acc_4": [
          0.0,
          0.0375
        ],
        "acc_at_1": [
          0.0,
          0.16112515805281927
        ],
        "pass_at_4": [
          0.00888144880079543,
          0.23613119344674194
        ],
        "stable_at_4": [
          0.0,
          0.16112515805281927
        ]
      }
    },
    {
      "provider": "claude",
      "condition": "text_only",
      "subset": "Hard20",
      "model": "claude-opus-4-8",
      "questions": 20,
      "expected_slots": 80,
      "nonempty_answers": 79,
      "human_adjudicated_slots": 1,
      "valid_labels": 80,
      "correct_slots": 4,
      "incorrect_slots": 76,
      "first_roll_correct": 1,
      "pass_at_4_questions": 1,
      "stable_at_4_questions": 1,
      "acc_at_1": 0.05,
      "mean_acc_4": 0.05,
      "pass_at_4": 0.05,
      "stable_at_4": 0.05,
      "correct_roll_distribution": {
        "0": 19,
        "1": 0,
        "2": 0,
        "3": 0,
        "4": 1
      },
      "missing_slots": 0,
      "judge_error_slots": 0,
      "metric_question_ids": [
        "GT-0019",
        "GT-0050",
        "GT-0061",
        "GT-0102",
        "GT-0164",
        "GT-0216",
        "GT-0244",
        "GT-0248",
        "GT-0288",
        "GT-0291",
        "GT-0302",
        "GT-0361",
        "GT-0382",
        "GT-0395",
        "GT-0411",
        "GT-0514",
        "GT-0516",
        "GT-0554",
        "GT-0560",
        "GT-0585"
      ],
      "coverage": {
        "planned_questions": 20,
        "planned_slots": 80,
        "valid_labels": 80,
        "missing_slot_keys": [],
        "archived_complete_answers": 79,
        "human_adjudicated_slots": 1,
        "technical_missing_slot_keys": [
          "claude::text_only::GT-0560::2"
        ],
        "complete_question_ids": [
          "GT-0019",
          "GT-0050",
          "GT-0061",
          "GT-0102",
          "GT-0164",
          "GT-0216",
          "GT-0244",
          "GT-0248",
          "GT-0288",
          "GT-0291",
          "GT-0302",
          "GT-0361",
          "GT-0382",
          "GT-0395",
          "GT-0411",
          "GT-0514",
          "GT-0516",
          "GT-0554",
          "GT-0560",
          "GT-0585"
        ],
        "incomplete_question_ids": [],
        "excluded_valid_labels_from_metrics": 0,
        "missing_is_not_incorrect": true
      },
      "confidence_intervals_95": {
        "mean_acc_4": [
          0.0,
          0.15
        ],
        "acc_at_1": [
          0.00888144880079543,
          0.23613119344674194
        ],
        "pass_at_4": [
          0.00888144880079543,
          0.23613119344674194
        ],
        "stable_at_4": [
          0.00888144880079543,
          0.23613119344674194
        ]
      }
    },
    {
      "provider": "gpt",
      "condition": "original",
      "subset": "Challenge62",
      "model": "gpt-5.6-luna",
      "questions": 62,
      "expected_slots": 248,
      "nonempty_answers": 248,
      "human_adjudicated_slots": 0,
      "valid_labels": 248,
      "correct_slots": 79,
      "incorrect_slots": 169,
      "first_roll_correct": 21,
      "pass_at_4_questions": 34,
      "stable_at_4_questions": 8,
      "acc_at_1": 0.3387096774193548,
      "mean_acc_4": 0.3185483870967742,
      "pass_at_4": 0.5483870967741935,
      "stable_at_4": 0.12903225806451613,
      "correct_roll_distribution": {
        "0": 28,
        "1": 11,
        "2": 9,
        "3": 6,
        "4": 8
      },
      "missing_slots": 0,
      "judge_error_slots": 0,
      "metric_question_ids": [
        "GT-0004",
        "GT-0008",
        "GT-0010",
        "GT-0012",
        "GT-0016",
        "GT-0018",
        "GT-0019",
        "GT-0049",
        "GT-0050",
        "GT-0061",
        "GT-0070",
        "GT-0081",
        "GT-0084",
        "GT-0102",
        "GT-0138",
        "GT-0163",
        "GT-0164",
        "GT-0167",
        "GT-0168",
        "GT-0207",
        "GT-0208",
        "GT-0216",
        "GT-0219",
        "GT-0230",
        "GT-0239",
        "GT-0244",
        "GT-0248",
        "GT-0255",
        "GT-0277",
        "GT-0282",
        "GT-0288",
        "GT-0291",
        "GT-0302",
        "GT-0305",
        "GT-0317",
        "GT-0325",
        "GT-0361",
        "GT-0372",
        "GT-0374",
        "GT-0382",
        "GT-0385",
        "GT-0394",
        "GT-0395",
        "GT-0402",
        "GT-0411",
        "GT-0424",
        "GT-0426",
        "GT-0436",
        "GT-0437",
        "GT-0495",
        "GT-0503",
        "GT-0514",
        "GT-0516",
        "GT-0524",
        "GT-0526",
        "GT-0543",
        "GT-0547",
        "GT-0554",
        "GT-0560",
        "GT-0572",
        "GT-0573",
        "GT-0585"
      ],
      "coverage": {
        "planned_questions": 62,
        "planned_slots": 248,
        "valid_labels": 248,
        "missing_slot_keys": [],
        "archived_complete_answers": 248,
        "human_adjudicated_slots": 0,
        "technical_missing_slot_keys": [],
        "complete_question_ids": [
          "GT-0004",
          "GT-0008",
          "GT-0010",
          "GT-0012",
          "GT-0016",
          "GT-0018",
          "GT-0019",
          "GT-0049",
          "GT-0050",
          "GT-0061",
          "GT-0070",
          "GT-0081",
          "GT-0084",
          "GT-0102",
          "GT-0138",
          "GT-0163",
          "GT-0164",
          "GT-0167",
          "GT-0168",
          "GT-0207",
          "GT-0208",
          "GT-0216",
          "GT-0219",
          "GT-0230",
          "GT-0239",
          "GT-0244",
          "GT-0248",
          "GT-0255",
          "GT-0277",
          "GT-0282",
          "GT-0288",
          "GT-0291",
          "GT-0302",
          "GT-0305",
          "GT-0317",
          "GT-0325",
          "GT-0361",
          "GT-0372",
          "GT-0374",
          "GT-0382",
          "GT-0385",
          "GT-0394",
          "GT-0395",
          "GT-0402",
          "GT-0411",
          "GT-0424",
          "GT-0426",
          "GT-0436",
          "GT-0437",
          "GT-0495",
          "GT-0503",
          "GT-0514",
          "GT-0516",
          "GT-0524",
          "GT-0526",
          "GT-0543",
          "GT-0547",
          "GT-0554",
          "GT-0560",
          "GT-0572",
          "GT-0573",
          "GT-0585"
        ],
        "incomplete_question_ids": [],
        "excluded_valid_labels_from_metrics": 0,
        "missing_is_not_incorrect": true
      },
      "confidence_intervals_95": {
        "mean_acc_4": [
          0.22983870967741934,
          0.4112903225806452
        ],
        "acc_at_1": [
          0.23341701875208903,
          0.46282300450641806
        ],
        "pass_at_4": [
          0.4253251246984431,
          0.6658028683240047
        ],
        "stable_at_4": [
          0.06685878059283037,
          0.23449327290173624
        ]
      }
    },
    {
      "provider": "gemini",
      "condition": "original",
      "subset": "Challenge62",
      "model": "gemini-3.6-flash",
      "questions": 62,
      "expected_slots": 248,
      "nonempty_answers": 248,
      "human_adjudicated_slots": 0,
      "valid_labels": 248,
      "correct_slots": 71,
      "incorrect_slots": 177,
      "first_roll_correct": 17,
      "pass_at_4_questions": 27,
      "stable_at_4_questions": 9,
      "acc_at_1": 0.27419354838709675,
      "mean_acc_4": 0.2862903225806452,
      "pass_at_4": 0.43548387096774194,
      "stable_at_4": 0.14516129032258066,
      "correct_roll_distribution": {
        "0": 35,
        "1": 6,
        "2": 7,
        "3": 5,
        "4": 9
      },
      "missing_slots": 0,
      "judge_error_slots": 0,
      "metric_question_ids": [
        "GT-0004",
        "GT-0008",
        "GT-0010",
        "GT-0012",
        "GT-0016",
        "GT-0018",
        "GT-0019",
        "GT-0049",
        "GT-0050",
        "GT-0061",
        "GT-0070",
        "GT-0081",
        "GT-0084",
        "GT-0102",
        "GT-0138",
        "GT-0163",
        "GT-0164",
        "GT-0167",
        "GT-0168",
        "GT-0207",
        "GT-0208",
        "GT-0216",
        "GT-0219",
        "GT-0230",
        "GT-0239",
        "GT-0244",
        "GT-0248",
        "GT-0255",
        "GT-0277",
        "GT-0282",
        "GT-0288",
        "GT-0291",
        "GT-0302",
        "GT-0305",
        "GT-0317",
        "GT-0325",
        "GT-0361",
        "GT-0372",
        "GT-0374",
        "GT-0382",
        "GT-0385",
        "GT-0394",
        "GT-0395",
        "GT-0402",
        "GT-0411",
        "GT-0424",
        "GT-0426",
        "GT-0436",
        "GT-0437",
        "GT-0495",
        "GT-0503",
        "GT-0514",
        "GT-0516",
        "GT-0524",
        "GT-0526",
        "GT-0543",
        "GT-0547",
        "GT-0554",
        "GT-0560",
        "GT-0572",
        "GT-0573",
        "GT-0585"
      ],
      "coverage": {
        "planned_questions": 62,
        "planned_slots": 248,
        "valid_labels": 248,
        "missing_slot_keys": [],
        "archived_complete_answers": 248,
        "human_adjudicated_slots": 0,
        "technical_missing_slot_keys": [],
        "complete_question_ids": [
          "GT-0004",
          "GT-0008",
          "GT-0010",
          "GT-0012",
          "GT-0016",
          "GT-0018",
          "GT-0019",
          "GT-0049",
          "GT-0050",
          "GT-0061",
          "GT-0070",
          "GT-0081",
          "GT-0084",
          "GT-0102",
          "GT-0138",
          "GT-0163",
          "GT-0164",
          "GT-0167",
          "GT-0168",
          "GT-0207",
          "GT-0208",
          "GT-0216",
          "GT-0219",
          "GT-0230",
          "GT-0239",
          "GT-0244",
          "GT-0248",
          "GT-0255",
          "GT-0277",
          "GT-0282",
          "GT-0288",
          "GT-0291",
          "GT-0302",
          "GT-0305",
          "GT-0317",
          "GT-0325",
          "GT-0361",
          "GT-0372",
          "GT-0374",
          "GT-0382",
          "GT-0385",
          "GT-0394",
          "GT-0395",
          "GT-0402",
          "GT-0411",
          "GT-0424",
          "GT-0426",
          "GT-0436",
          "GT-0437",
          "GT-0495",
          "GT-0503",
          "GT-0514",
          "GT-0516",
          "GT-0524",
          "GT-0526",
          "GT-0543",
          "GT-0547",
          "GT-0554",
          "GT-0560",
          "GT-0572",
          "GT-0573",
          "GT-0585"
        ],
        "incomplete_question_ids": [],
        "excluded_valid_labels_from_metrics": 0,
        "missing_is_not_incorrect": true
      },
      "confidence_intervals_95": {
        "mean_acc_4": [
          0.1935483870967742,
          0.38306451612903225
        ],
        "acc_at_1": [
          0.17881063614202042,
          0.39592539641988966
        ],
        "pass_at_4": [
          0.31942588902489055,
          0.5590701202785122
        ],
        "stable_at_4": [
          0.07829411405317008,
          0.25343393711554585
        ]
      }
    },
    {
      "provider": "claude",
      "condition": "original",
      "subset": "Challenge62",
      "model": "claude-opus-4-8",
      "questions": 62,
      "expected_slots": 248,
      "nonempty_answers": 248,
      "human_adjudicated_slots": 0,
      "valid_labels": 248,
      "correct_slots": 27,
      "incorrect_slots": 221,
      "first_roll_correct": 6,
      "pass_at_4_questions": 14,
      "stable_at_4_questions": 1,
      "acc_at_1": 0.0967741935483871,
      "mean_acc_4": 0.10887096774193548,
      "pass_at_4": 0.22580645161290322,
      "stable_at_4": 0.016129032258064516,
      "correct_roll_distribution": {
        "0": 48,
        "1": 5,
        "2": 6,
        "3": 2,
        "4": 1
      },
      "missing_slots": 0,
      "judge_error_slots": 0,
      "metric_question_ids": [
        "GT-0004",
        "GT-0008",
        "GT-0010",
        "GT-0012",
        "GT-0016",
        "GT-0018",
        "GT-0019",
        "GT-0049",
        "GT-0050",
        "GT-0061",
        "GT-0070",
        "GT-0081",
        "GT-0084",
        "GT-0102",
        "GT-0138",
        "GT-0163",
        "GT-0164",
        "GT-0167",
        "GT-0168",
        "GT-0207",
        "GT-0208",
        "GT-0216",
        "GT-0219",
        "GT-0230",
        "GT-0239",
        "GT-0244",
        "GT-0248",
        "GT-0255",
        "GT-0277",
        "GT-0282",
        "GT-0288",
        "GT-0291",
        "GT-0302",
        "GT-0305",
        "GT-0317",
        "GT-0325",
        "GT-0361",
        "GT-0372",
        "GT-0374",
        "GT-0382",
        "GT-0385",
        "GT-0394",
        "GT-0395",
        "GT-0402",
        "GT-0411",
        "GT-0424",
        "GT-0426",
        "GT-0436",
        "GT-0437",
        "GT-0495",
        "GT-0503",
        "GT-0514",
        "GT-0516",
        "GT-0524",
        "GT-0526",
        "GT-0543",
        "GT-0547",
        "GT-0554",
        "GT-0560",
        "GT-0572",
        "GT-0573",
        "GT-0585"
      ],
      "coverage": {
        "planned_questions": 62,
        "planned_slots": 248,
        "valid_labels": 248,
        "missing_slot_keys": [],
        "archived_complete_answers": 248,
        "human_adjudicated_slots": 0,
        "technical_missing_slot_keys": [],
        "complete_question_ids": [
          "GT-0004",
          "GT-0008",
          "GT-0010",
          "GT-0012",
          "GT-0016",
          "GT-0018",
          "GT-0019",
          "GT-0049",
          "GT-0050",
          "GT-0061",
          "GT-0070",
          "GT-0081",
          "GT-0084",
          "GT-0102",
          "GT-0138",
          "GT-0163",
          "GT-0164",
          "GT-0167",
          "GT-0168",
          "GT-0207",
          "GT-0208",
          "GT-0216",
          "GT-0219",
          "GT-0230",
          "GT-0239",
          "GT-0244",
          "GT-0248",
          "GT-0255",
          "GT-0277",
          "GT-0282",
          "GT-0288",
          "GT-0291",
          "GT-0302",
          "GT-0305",
          "GT-0317",
          "GT-0325",
          "GT-0361",
          "GT-0372",
          "GT-0374",
          "GT-0382",
          "GT-0385",
          "GT-0394",
          "GT-0395",
          "GT-0402",
          "GT-0411",
          "GT-0424",
          "GT-0426",
          "GT-0436",
          "GT-0437",
          "GT-0495",
          "GT-0503",
          "GT-0514",
          "GT-0516",
          "GT-0524",
          "GT-0526",
          "GT-0543",
          "GT-0547",
          "GT-0554",
          "GT-0560",
          "GT-0572",
          "GT-0573",
          "GT-0585"
        ],
        "incomplete_question_ids": [],
        "excluded_valid_labels_from_metrics": 0,
        "missing_is_not_incorrect": true
      },
      "confidence_intervals_95": {
        "mean_acc_4": [
          0.056451612903225805,
          0.1693548387096774
        ],
        "acc_at_1": [
          0.04511187202319393,
          0.19548818612307417
        ],
        "pass_at_4": [
          0.1395518303858828,
          0.3440562091535795
        ],
        "stable_at_4": [
          0.002852867168621827,
          0.0858672026068999
        ]
      }
    }
  ],
  "paired_comparisons": [
    {
      "subset": "Hard20",
      "kind": "input_condition",
      "first": "gpt::text_only",
      "second": "gpt::original",
      "metric": "mean_acc_4",
      "difference": -0.05,
      "first_mean_acc_4": 0.0,
      "second_mean_acc_4": 0.05,
      "questions": 20,
      "question_ids": [
        "GT-0019",
        "GT-0050",
        "GT-0061",
        "GT-0102",
        "GT-0164",
        "GT-0216",
        "GT-0244",
        "GT-0248",
        "GT-0288",
        "GT-0291",
        "GT-0302",
        "GT-0361",
        "GT-0382",
        "GT-0395",
        "GT-0411",
        "GT-0514",
        "GT-0516",
        "GT-0554",
        "GT-0560",
        "GT-0585"
      ],
      "excluded_question_ids": [],
      "confidence_interval_95": [
        -0.1125,
        0.0
      ]
    },
    {
      "subset": "Hard20",
      "kind": "input_condition",
      "first": "gemini::text_only",
      "second": "gemini::original",
      "metric": "mean_acc_4",
      "difference": -0.025,
      "first_mean_acc_4": 0.0,
      "second_mean_acc_4": 0.025,
      "questions": 20,
      "question_ids": [
        "GT-0019",
        "GT-0050",
        "GT-0061",
        "GT-0102",
        "GT-0164",
        "GT-0216",
        "GT-0244",
        "GT-0248",
        "GT-0288",
        "GT-0291",
        "GT-0302",
        "GT-0361",
        "GT-0382",
        "GT-0395",
        "GT-0411",
        "GT-0514",
        "GT-0516",
        "GT-0554",
        "GT-0560",
        "GT-0585"
      ],
      "excluded_question_ids": [],
      "confidence_interval_95": [
        -0.075,
        0.0
      ]
    },
    {
      "subset": "Hard20",
      "kind": "input_condition",
      "first": "claude::text_only",
      "second": "claude::original",
      "metric": "mean_acc_4",
      "difference": 0.0375,
      "first_mean_acc_4": 0.05,
      "second_mean_acc_4": 0.0125,
      "questions": 20,
      "question_ids": [
        "GT-0019",
        "GT-0050",
        "GT-0061",
        "GT-0102",
        "GT-0164",
        "GT-0216",
        "GT-0244",
        "GT-0248",
        "GT-0288",
        "GT-0291",
        "GT-0302",
        "GT-0361",
        "GT-0382",
        "GT-0395",
        "GT-0411",
        "GT-0514",
        "GT-0516",
        "GT-0554",
        "GT-0560",
        "GT-0585"
      ],
      "excluded_question_ids": [],
      "confidence_interval_95": [
        -0.0375,
        0.15
      ]
    },
    {
      "subset": "Hard20",
      "kind": "model_comparison",
      "first": "gpt::original",
      "second": "gemini::original",
      "metric": "mean_acc_4",
      "difference": 0.025,
      "first_mean_acc_4": 0.05,
      "second_mean_acc_4": 0.025,
      "questions": 20,
      "question_ids": [
        "GT-0019",
        "GT-0050",
        "GT-0061",
        "GT-0102",
        "GT-0164",
        "GT-0216",
        "GT-0244",
        "GT-0248",
        "GT-0288",
        "GT-0291",
        "GT-0302",
        "GT-0361",
        "GT-0382",
        "GT-0395",
        "GT-0411",
        "GT-0514",
        "GT-0516",
        "GT-0554",
        "GT-0560",
        "GT-0585"
      ],
      "excluded_question_ids": [],
      "confidence_interval_95": [
        -0.05,
        0.1
      ]
    },
    {
      "subset": "Hard20",
      "kind": "model_comparison",
      "first": "gpt::original",
      "second": "claude::original",
      "metric": "mean_acc_4",
      "difference": 0.0375,
      "first_mean_acc_4": 0.05,
      "second_mean_acc_4": 0.0125,
      "questions": 20,
      "question_ids": [
        "GT-0019",
        "GT-0050",
        "GT-0061",
        "GT-0102",
        "GT-0164",
        "GT-0216",
        "GT-0244",
        "GT-0248",
        "GT-0288",
        "GT-0291",
        "GT-0302",
        "GT-0361",
        "GT-0382",
        "GT-0395",
        "GT-0411",
        "GT-0514",
        "GT-0516",
        "GT-0554",
        "GT-0560",
        "GT-0585"
      ],
      "excluded_question_ids": [],
      "confidence_interval_95": [
        -0.025,
        0.1
      ]
    },
    {
      "subset": "Hard20",
      "kind": "model_comparison",
      "first": "gemini::original",
      "second": "claude::original",
      "metric": "mean_acc_4",
      "difference": 0.0125,
      "first_mean_acc_4": 0.025,
      "second_mean_acc_4": 0.0125,
      "questions": 20,
      "question_ids": [
        "GT-0019",
        "GT-0050",
        "GT-0061",
        "GT-0102",
        "GT-0164",
        "GT-0216",
        "GT-0244",
        "GT-0248",
        "GT-0288",
        "GT-0291",
        "GT-0302",
        "GT-0361",
        "GT-0382",
        "GT-0395",
        "GT-0411",
        "GT-0514",
        "GT-0516",
        "GT-0554",
        "GT-0560",
        "GT-0585"
      ],
      "excluded_question_ids": [],
      "confidence_interval_95": [
        -0.0375,
        0.075
      ]
    },
    {
      "subset": "Challenge62",
      "kind": "model_comparison",
      "first": "gpt::original",
      "second": "gemini::original",
      "metric": "mean_acc_4",
      "difference": 0.03225806451612903,
      "first_mean_acc_4": 0.3185483870967742,
      "second_mean_acc_4": 0.2862903225806452,
      "questions": 62,
      "question_ids": [
        "GT-0004",
        "GT-0008",
        "GT-0010",
        "GT-0012",
        "GT-0016",
        "GT-0018",
        "GT-0019",
        "GT-0049",
        "GT-0050",
        "GT-0061",
        "GT-0070",
        "GT-0081",
        "GT-0084",
        "GT-0102",
        "GT-0138",
        "GT-0163",
        "GT-0164",
        "GT-0167",
        "GT-0168",
        "GT-0207",
        "GT-0208",
        "GT-0216",
        "GT-0219",
        "GT-0230",
        "GT-0239",
        "GT-0244",
        "GT-0248",
        "GT-0255",
        "GT-0277",
        "GT-0282",
        "GT-0288",
        "GT-0291",
        "GT-0302",
        "GT-0305",
        "GT-0317",
        "GT-0325",
        "GT-0361",
        "GT-0372",
        "GT-0374",
        "GT-0382",
        "GT-0385",
        "GT-0394",
        "GT-0395",
        "GT-0402",
        "GT-0411",
        "GT-0424",
        "GT-0426",
        "GT-0436",
        "GT-0437",
        "GT-0495",
        "GT-0503",
        "GT-0514",
        "GT-0516",
        "GT-0524",
        "GT-0526",
        "GT-0543",
        "GT-0547",
        "GT-0554",
        "GT-0560",
        "GT-0572",
        "GT-0573",
        "GT-0585"
      ],
      "excluded_question_ids": [],
      "confidence_interval_95": [
        -0.06854838709677419,
        0.13306451612903225
      ]
    },
    {
      "subset": "Challenge62",
      "kind": "model_comparison",
      "first": "gpt::original",
      "second": "claude::original",
      "metric": "mean_acc_4",
      "difference": 0.20967741935483872,
      "first_mean_acc_4": 0.3185483870967742,
      "second_mean_acc_4": 0.10887096774193548,
      "questions": 62,
      "question_ids": [
        "GT-0004",
        "GT-0008",
        "GT-0010",
        "GT-0012",
        "GT-0016",
        "GT-0018",
        "GT-0019",
        "GT-0049",
        "GT-0050",
        "GT-0061",
        "GT-0070",
        "GT-0081",
        "GT-0084",
        "GT-0102",
        "GT-0138",
        "GT-0163",
        "GT-0164",
        "GT-0167",
        "GT-0168",
        "GT-0207",
        "GT-0208",
        "GT-0216",
        "GT-0219",
        "GT-0230",
        "GT-0239",
        "GT-0244",
        "GT-0248",
        "GT-0255",
        "GT-0277",
        "GT-0282",
        "GT-0288",
        "GT-0291",
        "GT-0302",
        "GT-0305",
        "GT-0317",
        "GT-0325",
        "GT-0361",
        "GT-0372",
        "GT-0374",
        "GT-0382",
        "GT-0385",
        "GT-0394",
        "GT-0395",
        "GT-0402",
        "GT-0411",
        "GT-0424",
        "GT-0426",
        "GT-0436",
        "GT-0437",
        "GT-0495",
        "GT-0503",
        "GT-0514",
        "GT-0516",
        "GT-0524",
        "GT-0526",
        "GT-0543",
        "GT-0547",
        "GT-0554",
        "GT-0560",
        "GT-0572",
        "GT-0573",
        "GT-0585"
      ],
      "excluded_question_ids": [],
      "confidence_interval_95": [
        0.11290322580645161,
        0.3064516129032258
      ]
    },
    {
      "subset": "Challenge62",
      "kind": "model_comparison",
      "first": "gemini::original",
      "second": "claude::original",
      "metric": "mean_acc_4",
      "difference": 0.1774193548387097,
      "first_mean_acc_4": 0.2862903225806452,
      "second_mean_acc_4": 0.10887096774193548,
      "questions": 62,
      "question_ids": [
        "GT-0004",
        "GT-0008",
        "GT-0010",
        "GT-0012",
        "GT-0016",
        "GT-0018",
        "GT-0019",
        "GT-0049",
        "GT-0050",
        "GT-0061",
        "GT-0070",
        "GT-0081",
        "GT-0084",
        "GT-0102",
        "GT-0138",
        "GT-0163",
        "GT-0164",
        "GT-0167",
        "GT-0168",
        "GT-0207",
        "GT-0208",
        "GT-0216",
        "GT-0219",
        "GT-0230",
        "GT-0239",
        "GT-0244",
        "GT-0248",
        "GT-0255",
        "GT-0277",
        "GT-0282",
        "GT-0288",
        "GT-0291",
        "GT-0302",
        "GT-0305",
        "GT-0317",
        "GT-0325",
        "GT-0361",
        "GT-0372",
        "GT-0374",
        "GT-0382",
        "GT-0385",
        "GT-0394",
        "GT-0395",
        "GT-0402",
        "GT-0411",
        "GT-0424",
        "GT-0426",
        "GT-0436",
        "GT-0437",
        "GT-0495",
        "GT-0503",
        "GT-0514",
        "GT-0516",
        "GT-0524",
        "GT-0526",
        "GT-0543",
        "GT-0547",
        "GT-0554",
        "GT-0560",
        "GT-0572",
        "GT-0573",
        "GT-0585"
      ],
      "excluded_question_ids": [],
      "confidence_interval_95": [
        0.10483870967741937,
        0.25806451612903225
      ]
    }
  ],
  "confidence_interval_methods": {
    "mean_acc_4": "question_percentile_bootstrap",
    "paired_mean_differences": "paired_question_percentile_bootstrap",
    "acc_at_1": "wilson_score_question_binary",
    "pass_at_4": "wilson_score_question_binary",
    "stable_at_4": "wilson_score_question_binary"
  },
  "bootstrap": {
    "replicates": 100000,
    "seed": 20260910,
    "method": "percentile_linear_quantile",
    "unit": "question, all four rolls jointly; common question draw across compared models and conditions",
    "multiplicity_adjusted": false,
    "p_values_reported": false,
    "scope": "conditional on selected benchmark; not a random sample of all visual mathematics"
  },
  "scope": "Saved current Challenge62/Hard20 statistics. Offline summarize recomputes four point estimates; the recorded intervals are supplied as saved paper results."
}