{
  "version": 1,
  "generatedAt": "2026-10-05T14:09:37.345Z",
  "quality": {
    "protocol": "original-weight-expansion-v4",
    "models": {
      "english": {
        "label": "Laya English",
        "origin": "historical_reused",
        "metrics": {
          "refund": {
            "correct": 24,
            "n": 96,
            "score": 0.25
          },
          "access": {
            "correct": 33,
            "n": 96,
            "score": 0.34375
          },
          "routing": {
            "correct": 69,
            "n": 96,
            "score": 0.71875
          },
          "policy_action": {
            "correct": 126,
            "n": 288,
            "score": 0.4375
          },
          "legal": {
            "correct": 56,
            "n": 144,
            "score": 0.3888888888888889
          },
          "science": {
            "correct": 171,
            "n": 339,
            "score": 0.504424778761062
          },
          "action_head": {
            "correct": 126,
            "n": 288,
            "score": 0.4375
          },
          "review_head": {
            "correct": 184,
            "n": 288,
            "score": 0.6388888888888888
          },
          "severity_head": {
            "correct": 106,
            "n": 288,
            "score": 0.3680555555555556
          },
          "all_heads": {
            "correct": 32,
            "n": 288,
            "score": 0.1111111111111111
          },
          "policy_counterfactual": {
            "correct": 29,
            "n": 288,
            "score": 0.10069444444444445
          },
          "natural_reversal": {
            "score": 0.4126413470993117,
            "components": {
              "legal": {
                "correct": 53,
                "n": 144,
                "score": 0.3680555555555556
              },
              "science": {
                "correct": 155,
                "n": 339,
                "score": 0.45722713864306785
              }
            }
          },
          "overall_domain_equal": {
            "score": 0.443604555883317,
            "ci95": [
              0.4102465363829256,
              0.4761933832674341
            ]
          },
          "overall_task_equal": {
            "score": 0.4411627335299902
          }
        },
        "decisions": 4905,
        "errors": 0,
        "known_training_overlap": "not established"
      },
      "multilingual": {
        "label": "Laya Multilingual",
        "origin": "historical_reused",
        "metrics": {
          "refund": {
            "correct": 24,
            "n": 96,
            "score": 0.25
          },
          "access": {
            "correct": 28,
            "n": 96,
            "score": 0.2916666666666667
          },
          "routing": {
            "correct": 23,
            "n": 96,
            "score": 0.23958333333333334
          },
          "policy_action": {
            "correct": 75,
            "n": 288,
            "score": 0.2604166666666667
          },
          "legal": {
            "correct": 49,
            "n": 144,
            "score": 0.3402777777777778
          },
          "science": {
            "correct": 150,
            "n": 339,
            "score": 0.4424778761061947
          },
          "action_head": {
            "correct": 75,
            "n": 288,
            "score": 0.2604166666666667
          },
          "review_head": {
            "correct": 171,
            "n": 288,
            "score": 0.59375
          },
          "severity_head": {
            "correct": 90,
            "n": 288,
            "score": 0.3125
          },
          "all_heads": {
            "correct": 23,
            "n": 288,
            "score": 0.0798611111111111
          },
          "policy_counterfactual": {
            "correct": 0,
            "n": 288,
            "score": 0
          },
          "natural_reversal": {
            "score": 0.3375737463126844,
            "components": {
              "legal": {
                "correct": 42,
                "n": 144,
                "score": 0.2916666666666667
              },
              "science": {
                "correct": 130,
                "n": 339,
                "score": 0.3834808259587021
              }
            }
          },
          "overall_domain_equal": {
            "score": 0.34772410685021304,
            "ci95": [
              0.3148346380306405,
              0.38150650342941483
            ]
          },
          "overall_task_equal": {
            "score": 0.3128011307767945
          }
        },
        "decisions": 4905,
        "errors": 0,
        "known_training_overlap": "not established"
      },
      "llama31_8b_instruct": {
        "label": "Llama-3.1-8B",
        "origin": "historical_reused",
        "metrics": {
          "refund": {
            "correct": 32,
            "n": 96,
            "score": 0.3333333333333333
          },
          "access": {
            "correct": 33,
            "n": 96,
            "score": 0.34375
          },
          "routing": {
            "correct": 24,
            "n": 96,
            "score": 0.25
          },
          "policy_action": {
            "correct": 89,
            "n": 288,
            "score": 0.3090277777777778
          },
          "legal": {
            "correct": 44,
            "n": 144,
            "score": 0.3055555555555556
          },
          "science": {
            "correct": 182,
            "n": 339,
            "score": 0.5368731563421829
          },
          "action_head": {
            "correct": 89,
            "n": 288,
            "score": 0.3090277777777778
          },
          "review_head": {
            "correct": 171,
            "n": 288,
            "score": 0.59375
          },
          "severity_head": {
            "correct": 180,
            "n": 288,
            "score": 0.625
          },
          "all_heads": {
            "correct": 38,
            "n": 288,
            "score": 0.13194444444444445
          },
          "policy_counterfactual": {
            "correct": 0,
            "n": 288,
            "score": 0
          },
          "natural_reversal": {
            "score": 0.3284476401179941,
            "components": {
              "legal": {
                "correct": 33,
                "n": 144,
                "score": 0.22916666666666666
              },
              "science": {
                "correct": 145,
                "n": 339,
                "score": 0.4277286135693215
              }
            }
          },
          "overall_domain_equal": {
            "score": 0.38381882989183874,
            "ci95": [
              0.350314927060029,
              0.4163333221735177
            ]
          },
          "overall_task_equal": {
            "score": 0.35390240904621434
          }
        },
        "decisions": 4905,
        "errors": 0,
        "known_training_overlap": "not established"
      },
      "qwen3_8b": {
        "label": "Qwen3-8B",
        "origin": "historical_reused",
        "metrics": {
          "refund": {
            "correct": 24,
            "n": 96,
            "score": 0.25
          },
          "access": {
            "correct": 64,
            "n": 96,
            "score": 0.6666666666666666
          },
          "routing": {
            "correct": 47,
            "n": 96,
            "score": 0.4895833333333333
          },
          "policy_action": {
            "correct": 135,
            "n": 288,
            "score": 0.46875
          },
          "legal": {
            "correct": 65,
            "n": 144,
            "score": 0.4513888888888889
          },
          "science": {
            "correct": 231,
            "n": 339,
            "score": 0.6814159292035398
          },
          "action_head": {
            "correct": 135,
            "n": 288,
            "score": 0.46875
          },
          "review_head": {
            "correct": 184,
            "n": 288,
            "score": 0.6388888888888888
          },
          "severity_head": {
            "correct": 194,
            "n": 288,
            "score": 0.6736111111111112
          },
          "all_heads": {
            "correct": 67,
            "n": 288,
            "score": 0.2326388888888889
          },
          "policy_counterfactual": {
            "correct": 73,
            "n": 288,
            "score": 0.2534722222222222
          },
          "natural_reversal": {
            "score": 0.45141961651917406,
            "components": {
              "legal": {
                "correct": 51,
                "n": 144,
                "score": 0.3541666666666667
              },
              "science": {
                "correct": 186,
                "n": 339,
                "score": 0.5486725663716814
              }
            }
          },
          "overall_domain_equal": {
            "score": 0.5338516060308095,
            "ci95": [
              0.4980686097205295,
              0.5708505146295304
            ]
          },
          "overall_task_equal": {
            "score": 0.5078109636184858
          }
        },
        "decisions": 4905,
        "errors": 0,
        "known_training_overlap": "not established"
      },
      "jev-1.13.0": {
        "label": "Jev 1.13.0 API",
        "origin": "historical_reused",
        "metrics": {
          "refund": {
            "correct": 96,
            "n": 96,
            "score": 1
          },
          "access": {
            "correct": 96,
            "n": 96,
            "score": 1
          },
          "routing": {
            "correct": 52,
            "n": 96,
            "score": 0.5416666666666666
          },
          "policy_action": {
            "correct": 244,
            "n": 288,
            "score": 0.8472222222222222
          },
          "legal": {
            "correct": 111,
            "n": 144,
            "score": 0.7708333333333334
          },
          "science": {
            "correct": 291,
            "n": 339,
            "score": 0.8584070796460177
          },
          "action_head": {
            "correct": 244,
            "n": 288,
            "score": 0.8472222222222222
          },
          "review_head": {
            "correct": 278,
            "n": 288,
            "score": 0.9652777777777778
          },
          "severity_head": {
            "correct": 280,
            "n": 288,
            "score": 0.9722222222222222
          },
          "all_heads": {
            "correct": 228,
            "n": 288,
            "score": 0.7916666666666666
          },
          "policy_counterfactual": {
            "correct": 220,
            "n": 288,
            "score": 0.7638888888888888
          },
          "natural_reversal": {
            "score": 0.7977814650934121,
            "components": {
              "legal": {
                "correct": 107,
                "n": 144,
                "score": 0.7430555555555556
              },
              "science": {
                "correct": 289,
                "n": 339,
                "score": 0.8525073746312685
              }
            }
          },
          "overall_domain_equal": {
            "score": 0.8254875450671911,
            "ci95": [
              0.7941993420722927,
              0.8550427007357582
            ]
          },
          "overall_task_equal": {
            "score": 0.8341814159292035
          }
        },
        "decisions": 4905,
        "errors": 0,
        "known_training_overlap": "not established"
      },
      "kev_08b": {
        "label": "Kev-0.8B",
        "origin": "historical_reused",
        "metrics": {
          "refund": {
            "correct": 54,
            "n": 96,
            "score": 0.5625
          },
          "access": {
            "correct": 64,
            "n": 96,
            "score": 0.6666666666666666
          },
          "routing": {
            "correct": 73,
            "n": 96,
            "score": 0.7604166666666666
          },
          "policy_action": {
            "correct": 191,
            "n": 288,
            "score": 0.6631944444444444
          },
          "legal": {
            "correct": 70,
            "n": 144,
            "score": 0.4861111111111111
          },
          "science": {
            "correct": 168,
            "n": 339,
            "score": 0.49557522123893805
          },
          "action_head": {
            "correct": 191,
            "n": 288,
            "score": 0.6631944444444444
          },
          "review_head": {
            "correct": 192,
            "n": 288,
            "score": 0.6666666666666666
          },
          "severity_head": {
            "correct": 120,
            "n": 288,
            "score": 0.4166666666666667
          },
          "all_heads": {
            "correct": 60,
            "n": 288,
            "score": 0.20833333333333334
          },
          "policy_counterfactual": {
            "correct": 97,
            "n": 288,
            "score": 0.3368055555555556
          },
          "natural_reversal": {
            "score": 0.3990290068829892,
            "components": {
              "legal": {
                "correct": 58,
                "n": 144,
                "score": 0.4027777777777778
              },
              "science": {
                "correct": 134,
                "n": 339,
                "score": 0.3952802359882006
              }
            }
          },
          "overall_domain_equal": {
            "score": 0.5482935922648312,
            "ci95": [
              0.5097296310387976,
              0.585566597860793
            ]
          },
          "overall_task_equal": {
            "score": 0.5942539331366765
          }
        },
        "decisions": 4905,
        "errors": 0,
        "known_training_overlap": "not established"
      },
      "kev_4b": {
        "label": "Kev-4B",
        "origin": "historical_reused",
        "metrics": {
          "refund": {
            "correct": 84,
            "n": 96,
            "score": 0.875
          },
          "access": {
            "correct": 93,
            "n": 96,
            "score": 0.96875
          },
          "routing": {
            "correct": 78,
            "n": 96,
            "score": 0.8125
          },
          "policy_action": {
            "correct": 255,
            "n": 288,
            "score": 0.8854166666666666
          },
          "legal": {
            "correct": 95,
            "n": 144,
            "score": 0.6597222222222222
          },
          "science": {
            "correct": 274,
            "n": 339,
            "score": 0.8082595870206489
          },
          "action_head": {
            "correct": 255,
            "n": 288,
            "score": 0.8854166666666666
          },
          "review_head": {
            "correct": 242,
            "n": 288,
            "score": 0.8402777777777778
          },
          "severity_head": {
            "correct": 189,
            "n": 288,
            "score": 0.65625
          },
          "all_heads": {
            "correct": 145,
            "n": 288,
            "score": 0.5034722222222222
          },
          "policy_counterfactual": {
            "correct": 241,
            "n": 288,
            "score": 0.8368055555555556
          },
          "natural_reversal": {
            "score": 0.7068276794493609,
            "components": {
              "legal": {
                "correct": 91,
                "n": 144,
                "score": 0.6319444444444444
              },
              "science": {
                "correct": 265,
                "n": 339,
                "score": 0.7817109144542773
              }
            }
          },
          "overall_domain_equal": {
            "score": 0.7844661586365126,
            "ci95": [
              0.7548825796162193,
              0.8158839270178802
            ]
          },
          "overall_task_equal": {
            "score": 0.8248463618485742
          }
        },
        "decisions": 4905,
        "errors": 0,
        "known_training_overlap": "not established"
      },
      "kev_9b": {
        "label": "Kev-9B",
        "origin": "historical_reused",
        "metrics": {
          "refund": {
            "correct": 96,
            "n": 96,
            "score": 1
          },
          "access": {
            "correct": 95,
            "n": 96,
            "score": 0.9895833333333334
          },
          "routing": {
            "correct": 68,
            "n": 96,
            "score": 0.7083333333333334
          },
          "policy_action": {
            "correct": 259,
            "n": 288,
            "score": 0.8993055555555556
          },
          "legal": {
            "correct": 101,
            "n": 144,
            "score": 0.7013888888888888
          },
          "science": {
            "correct": 273,
            "n": 339,
            "score": 0.8053097345132744
          },
          "action_head": {
            "correct": 259,
            "n": 288,
            "score": 0.8993055555555556
          },
          "review_head": {
            "correct": 223,
            "n": 288,
            "score": 0.7743055555555556
          },
          "severity_head": {
            "correct": 234,
            "n": 288,
            "score": 0.8125
          },
          "all_heads": {
            "correct": 169,
            "n": 288,
            "score": 0.5868055555555556
          },
          "policy_counterfactual": {
            "correct": 245,
            "n": 288,
            "score": 0.8506944444444444
          },
          "natural_reversal": {
            "score": 0.7351278269419862,
            "components": {
              "legal": {
                "correct": 100,
                "n": 144,
                "score": 0.6944444444444444
              },
              "science": {
                "correct": 263,
                "n": 339,
                "score": 0.775811209439528
              }
            }
          },
          "overall_domain_equal": {
            "score": 0.8020013929859062,
            "ci95": [
              0.7713708673052941,
              0.8330769804966933
            ]
          },
          "overall_task_equal": {
            "score": 0.840923058013766
          }
        },
        "decisions": 4905,
        "errors": 0,
        "known_training_overlap": "not established"
      },
      "nanojev": {
        "label": "NanoJev",
        "origin": "historical_reused",
        "metrics": {
          "refund": {
            "correct": 24,
            "n": 96,
            "score": 0.25
          },
          "access": {
            "correct": 33,
            "n": 96,
            "score": 0.34375
          },
          "routing": {
            "correct": 24,
            "n": 96,
            "score": 0.25
          },
          "policy_action": {
            "correct": 81,
            "n": 288,
            "score": 0.28125
          },
          "legal": {
            "correct": 46,
            "n": 144,
            "score": 0.3194444444444444
          },
          "science": {
            "correct": 126,
            "n": 339,
            "score": 0.37168141592920356
          },
          "action_head": {
            "correct": 81,
            "n": 288,
            "score": 0.28125
          },
          "review_head": {
            "correct": 171,
            "n": 288,
            "score": 0.59375
          },
          "severity_head": {
            "correct": 92,
            "n": 288,
            "score": 0.3194444444444444
          },
          "all_heads": {
            "correct": 22,
            "n": 288,
            "score": 0.0763888888888889
          },
          "policy_counterfactual": {
            "correct": 0,
            "n": 288,
            "score": 0
          },
          "natural_reversal": {
            "score": 0.345562930186824,
            "components": {
              "legal": {
                "correct": 46,
                "n": 144,
                "score": 0.3194444444444444
              },
              "science": {
                "correct": 126,
                "n": 339,
                "score": 0.37168141592920356
              }
            }
          },
          "overall_domain_equal": {
            "score": 0.324125286791216,
            "ci95": [
              0.2928072713787588,
              0.3563181060172533
            ]
          },
          "overall_task_equal": {
            "score": 0.3069751720747296
          }
        },
        "decisions": 4905,
        "errors": 0,
        "known_training_overlap": "not established"
      },
      "qwen35_9b": {
        "label": "Qwen3.5-9B",
        "origin": "historical_reused",
        "metrics": {
          "refund": {
            "correct": 70,
            "n": 96,
            "score": 0.7291666666666666
          },
          "access": {
            "correct": 86,
            "n": 96,
            "score": 0.8958333333333334
          },
          "routing": {
            "correct": 86,
            "n": 96,
            "score": 0.8958333333333334
          },
          "policy_action": {
            "correct": 242,
            "n": 288,
            "score": 0.8402777777777778
          },
          "legal": {
            "correct": 109,
            "n": 144,
            "score": 0.7569444444444444
          },
          "science": {
            "correct": 283,
            "n": 339,
            "score": 0.8348082595870207
          },
          "action_head": {
            "correct": 242,
            "n": 288,
            "score": 0.8402777777777778
          },
          "review_head": {
            "correct": 222,
            "n": 288,
            "score": 0.7708333333333334
          },
          "severity_head": {
            "correct": 181,
            "n": 288,
            "score": 0.6284722222222222
          },
          "all_heads": {
            "correct": 113,
            "n": 288,
            "score": 0.3923611111111111
          },
          "policy_counterfactual": {
            "correct": 228,
            "n": 288,
            "score": 0.7916666666666666
          },
          "natural_reversal": {
            "score": 0.7124815634218289,
            "components": {
              "legal": {
                "correct": 99,
                "n": 144,
                "score": 0.6875
              },
              "science": {
                "correct": 250,
                "n": 339,
                "score": 0.7374631268436578
              }
            }
          },
          "overall_domain_equal": {
            "score": 0.8106768272697477,
            "ci95": [
              0.7790994020347797,
              0.8409093649476931
            ]
          },
          "overall_task_equal": {
            "score": 0.8225172074729598
          }
        },
        "decisions": 4905,
        "errors": 0,
        "known_training_overlap": "not established"
      },
      "kev_27b": {
        "label": "Kev-27B v2",
        "origin": "historical_reused",
        "metrics": {
          "refund": {
            "correct": 96,
            "n": 96,
            "score": 1
          },
          "access": {
            "correct": 96,
            "n": 96,
            "score": 1
          },
          "routing": {
            "correct": 96,
            "n": 96,
            "score": 1
          },
          "policy_action": {
            "correct": 288,
            "n": 288,
            "score": 1
          },
          "legal": {
            "correct": 116,
            "n": 144,
            "score": 0.8055555555555556
          },
          "science": {
            "correct": 285,
            "n": 339,
            "score": 0.8407079646017699
          },
          "action_head": {
            "correct": 288,
            "n": 288,
            "score": 1
          },
          "review_head": {
            "correct": 288,
            "n": 288,
            "score": 1
          },
          "severity_head": {
            "correct": 285,
            "n": 288,
            "score": 0.9895833333333334
          },
          "all_heads": {
            "correct": 285,
            "n": 288,
            "score": 0.9895833333333334
          },
          "policy_counterfactual": {
            "correct": 288,
            "n": 288,
            "score": 1
          },
          "natural_reversal": {
            "score": 0.8117625368731562,
            "components": {
              "legal": {
                "correct": 114,
                "n": 144,
                "score": 0.7916666666666666
              },
              "science": {
                "correct": 282,
                "n": 339,
                "score": 0.831858407079646
              }
            }
          },
          "overall_domain_equal": {
            "score": 0.8820878400524418,
            "ci95": [
              0.8586668574980462,
              0.9048829623902519
            ]
          },
          "overall_task_equal": {
            "score": 0.9292527040314651
          }
        },
        "decisions": 4905,
        "errors": 0,
        "known_training_overlap": "not established"
      },
      "qwen35_08b": {
        "label": "Qwen3.5-0.8B",
        "origin": "historical_reused",
        "metrics": {
          "refund": {
            "correct": 24,
            "n": 96,
            "score": 0.25
          },
          "access": {
            "correct": 50,
            "n": 96,
            "score": 0.5208333333333334
          },
          "routing": {
            "correct": 24,
            "n": 96,
            "score": 0.25
          },
          "policy_action": {
            "correct": 98,
            "n": 288,
            "score": 0.3402777777777778
          },
          "legal": {
            "correct": 44,
            "n": 144,
            "score": 0.3055555555555556
          },
          "science": {
            "correct": 138,
            "n": 339,
            "score": 0.40707964601769914
          },
          "action_head": {
            "correct": 98,
            "n": 288,
            "score": 0.3402777777777778
          },
          "review_head": {
            "correct": 117,
            "n": 288,
            "score": 0.40625
          },
          "severity_head": {
            "correct": 109,
            "n": 288,
            "score": 0.3784722222222222
          },
          "all_heads": {
            "correct": 12,
            "n": 288,
            "score": 0.041666666666666664
          },
          "policy_counterfactual": {
            "correct": 0,
            "n": 288,
            "score": 0
          },
          "natural_reversal": {
            "score": 0.22308259587020648,
            "components": {
              "legal": {
                "correct": 12,
                "n": 144,
                "score": 0.08333333333333333
              },
              "science": {
                "correct": 123,
                "n": 339,
                "score": 0.36283185840707965
              }
            }
          },
          "overall_domain_equal": {
            "score": 0.35097099311701085,
            "ci95": [
              0.3176144024234149,
              0.38406213320704047
            ]
          },
          "overall_task_equal": {
            "score": 0.34669370698131763
          }
        },
        "decisions": 4905,
        "errors": 0,
        "known_training_overlap": "not established"
      },
      "qwen35_4b": {
        "label": "Qwen3.5-4B",
        "origin": "historical_reused",
        "metrics": {
          "refund": {
            "correct": 53,
            "n": 96,
            "score": 0.5520833333333334
          },
          "access": {
            "correct": 76,
            "n": 96,
            "score": 0.7916666666666666
          },
          "routing": {
            "correct": 55,
            "n": 96,
            "score": 0.5729166666666666
          },
          "policy_action": {
            "correct": 184,
            "n": 288,
            "score": 0.6388888888888888
          },
          "legal": {
            "correct": 86,
            "n": 144,
            "score": 0.5972222222222222
          },
          "science": {
            "correct": 225,
            "n": 339,
            "score": 0.6637168141592921
          },
          "action_head": {
            "correct": 184,
            "n": 288,
            "score": 0.6388888888888888
          },
          "review_head": {
            "correct": 130,
            "n": 288,
            "score": 0.4513888888888889
          },
          "severity_head": {
            "correct": 163,
            "n": 288,
            "score": 0.5659722222222222
          },
          "all_heads": {
            "correct": 57,
            "n": 288,
            "score": 0.19791666666666666
          },
          "policy_counterfactual": {
            "correct": 133,
            "n": 288,
            "score": 0.4618055555555556
          },
          "natural_reversal": {
            "score": 0.5658185840707964,
            "components": {
              "legal": {
                "correct": 78,
                "n": 144,
                "score": 0.5416666666666666
              },
              "science": {
                "correct": 200,
                "n": 339,
                "score": 0.5899705014749262
              }
            }
          },
          "overall_domain_equal": {
            "score": 0.6332759750901343,
            "ci95": [
              0.5955717667542068,
              0.6691788318715134
            ]
          },
          "overall_task_equal": {
            "score": 0.6355211406096362
          }
        },
        "decisions": 4905,
        "errors": 0,
        "known_training_overlap": "not established"
      },
      "qwen38_27b": {
        "label": "Qwen3.8-27B",
        "origin": "historical_reused",
        "metrics": {
          "refund": {
            "correct": 74,
            "n": 96,
            "score": 0.7708333333333334
          },
          "access": {
            "correct": 93,
            "n": 96,
            "score": 0.96875
          },
          "routing": {
            "correct": 78,
            "n": 96,
            "score": 0.8125
          },
          "policy_action": {
            "correct": 245,
            "n": 288,
            "score": 0.8506944444444444
          },
          "legal": {
            "correct": 112,
            "n": 144,
            "score": 0.7777777777777778
          },
          "science": {
            "correct": 285,
            "n": 339,
            "score": 0.8407079646017699
          },
          "action_head": {
            "correct": 245,
            "n": 288,
            "score": 0.8506944444444444
          },
          "review_head": {
            "correct": 206,
            "n": 288,
            "score": 0.7152777777777778
          },
          "severity_head": {
            "correct": 194,
            "n": 288,
            "score": 0.6736111111111112
          },
          "all_heads": {
            "correct": 104,
            "n": 288,
            "score": 0.3611111111111111
          },
          "policy_counterfactual": {
            "correct": 241,
            "n": 288,
            "score": 0.8368055555555556
          },
          "natural_reversal": {
            "score": 0.764718534906588,
            "components": {
              "legal": {
                "correct": 103,
                "n": 144,
                "score": 0.7152777777777778
              },
              "science": {
                "correct": 276,
                "n": 339,
                "score": 0.8141592920353983
              }
            }
          },
          "overall_domain_equal": {
            "score": 0.823060062274664,
            "ci95": [
              0.794507256617321,
              0.8512164432948056
            ]
          },
          "overall_task_equal": {
            "score": 0.8341138151425762
          }
        },
        "decisions": 4905,
        "errors": 0,
        "known_training_overlap": "not established"
      },
      "startlux_4b": {
        "label": "StartLux-Decision-4B",
        "origin": "new_measured",
        "metrics": {
          "refund": {
            "correct": 96,
            "n": 96,
            "score": 1
          },
          "access": {
            "correct": 94,
            "n": 96,
            "score": 0.9791666666666666
          },
          "routing": {
            "correct": 50,
            "n": 96,
            "score": 0.5208333333333334
          },
          "policy_action": {
            "correct": 240,
            "n": 288,
            "score": 0.8333333333333334
          },
          "legal": {
            "correct": 126,
            "n": 144,
            "score": 0.875
          },
          "science": {
            "correct": 281,
            "n": 339,
            "score": 0.8289085545722714
          },
          "action_head": {
            "correct": 240,
            "n": 288,
            "score": 0.8333333333333334
          },
          "review_head": {
            "correct": 270,
            "n": 288,
            "score": 0.9375
          },
          "severity_head": {
            "correct": 264,
            "n": 288,
            "score": 0.9166666666666666
          },
          "all_heads": {
            "correct": 211,
            "n": 288,
            "score": 0.7326388888888888
          },
          "policy_counterfactual": {
            "correct": 211,
            "n": 288,
            "score": 0.7326388888888888
          },
          "natural_reversal": {
            "score": 0.841107423795477,
            "components": {
              "legal": {
                "correct": 125,
                "n": 144,
                "score": 0.8680555555555556
              },
              "science": {
                "correct": 276,
                "n": 339,
                "score": 0.8141592920353983
              }
            }
          },
          "overall_domain_equal": {
            "score": 0.8457472959685349,
            "ci95": [
              0.8200949166224746,
              0.8707581866842663
            ]
          },
          "overall_task_equal": {
            "score": 0.8407817109144542
          }
        },
        "decisions": 4905,
        "errors": 0,
        "known_training_overlap": "ContractNLI train split declared"
      },
      "startlux_9b": {
        "label": "StartLux-Decision-9B",
        "origin": "new_measured",
        "metrics": {
          "refund": {
            "correct": 96,
            "n": 96,
            "score": 1
          },
          "access": {
            "correct": 96,
            "n": 96,
            "score": 1
          },
          "routing": {
            "correct": 52,
            "n": 96,
            "score": 0.5416666666666666
          },
          "policy_action": {
            "correct": 244,
            "n": 288,
            "score": 0.8472222222222222
          },
          "legal": {
            "correct": 126,
            "n": 144,
            "score": 0.875
          },
          "science": {
            "correct": 287,
            "n": 339,
            "score": 0.8466076696165191
          },
          "action_head": {
            "correct": 244,
            "n": 288,
            "score": 0.8472222222222222
          },
          "review_head": {
            "correct": 266,
            "n": 288,
            "score": 0.9236111111111112
          },
          "severity_head": {
            "correct": 272,
            "n": 288,
            "score": 0.9444444444444444
          },
          "all_heads": {
            "correct": 212,
            "n": 288,
            "score": 0.7361111111111112
          },
          "policy_counterfactual": {
            "correct": 220,
            "n": 288,
            "score": 0.7638888888888888
          },
          "natural_reversal": {
            "score": 0.8529068338249755,
            "components": {
              "legal": {
                "correct": 125,
                "n": 144,
                "score": 0.8680555555555556
              },
              "science": {
                "correct": 284,
                "n": 339,
                "score": 0.8377581120943953
              }
            }
          },
          "overall_domain_equal": {
            "score": 0.8562766306129138,
            "ci95": [
              0.8311198424087641,
              0.8797954265420019
            ]
          },
          "overall_task_equal": {
            "score": 0.8526548672566371
          }
        },
        "decisions": 4905,
        "errors": 0,
        "known_training_overlap": "ContractNLI train split declared"
      },
      "startlux_27b": {
        "label": "StartLux-Decision-27B",
        "origin": "new_measured",
        "metrics": {
          "refund": {
            "correct": 96,
            "n": 96,
            "score": 1
          },
          "access": {
            "correct": 96,
            "n": 96,
            "score": 1
          },
          "routing": {
            "correct": 96,
            "n": 96,
            "score": 1
          },
          "policy_action": {
            "correct": 288,
            "n": 288,
            "score": 1
          },
          "legal": {
            "correct": 129,
            "n": 144,
            "score": 0.8958333333333334
          },
          "science": {
            "correct": 282,
            "n": 339,
            "score": 0.831858407079646
          },
          "action_head": {
            "correct": 288,
            "n": 288,
            "score": 1
          },
          "review_head": {
            "correct": 288,
            "n": 288,
            "score": 1
          },
          "severity_head": {
            "correct": 286,
            "n": 288,
            "score": 0.9930555555555556
          },
          "all_heads": {
            "correct": 286,
            "n": 288,
            "score": 0.9930555555555556
          },
          "policy_counterfactual": {
            "correct": 288,
            "n": 288,
            "score": 1
          },
          "natural_reversal": {
            "score": 0.8638458702064897,
            "components": {
              "legal": {
                "correct": 129,
                "n": 144,
                "score": 0.8958333333333334
              },
              "science": {
                "correct": 282,
                "n": 339,
                "score": 0.831858407079646
              }
            }
          },
          "overall_domain_equal": {
            "score": 0.9092305801376598,
            "ci95": [
              0.8871813745915279,
              0.9289481229991728
            ]
          },
          "overall_task_equal": {
            "score": 0.9455383480825958
          }
        },
        "decisions": 4905,
        "errors": 0,
        "known_training_overlap": "ContractNLI train split declared"
      },
      "intern_4b": {
        "label": "Intern-Decision-4B",
        "origin": "new_measured",
        "metrics": {
          "refund": {
            "correct": 92,
            "n": 96,
            "score": 0.9583333333333334
          },
          "access": {
            "correct": 86,
            "n": 96,
            "score": 0.8958333333333334
          },
          "routing": {
            "correct": 62,
            "n": 96,
            "score": 0.6458333333333334
          },
          "policy_action": {
            "correct": 240,
            "n": 288,
            "score": 0.8333333333333334
          },
          "legal": {
            "correct": 103,
            "n": 144,
            "score": 0.7152777777777778
          },
          "science": {
            "correct": 257,
            "n": 339,
            "score": 0.7581120943952803
          },
          "action_head": {
            "correct": 240,
            "n": 288,
            "score": 0.8333333333333334
          },
          "review_head": {
            "correct": 231,
            "n": 288,
            "score": 0.8020833333333334
          },
          "severity_head": {
            "correct": 228,
            "n": 288,
            "score": 0.7916666666666666
          },
          "all_heads": {
            "correct": 164,
            "n": 288,
            "score": 0.5694444444444444
          },
          "policy_counterfactual": {
            "correct": 212,
            "n": 288,
            "score": 0.7361111111111112
          },
          "natural_reversal": {
            "score": 0.6981624877089478,
            "components": {
              "legal": {
                "correct": 97,
                "n": 144,
                "score": 0.6736111111111112
              },
              "science": {
                "correct": 245,
                "n": 339,
                "score": 0.7227138643067846
              }
            }
          },
          "overall_domain_equal": {
            "score": 0.7689077351687971,
            "ci95": [
              0.7353167938626254,
              0.8025090435131468
            ]
          },
          "overall_task_equal": {
            "score": 0.7946779744346116
          }
        },
        "decisions": 4905,
        "errors": 0,
        "known_training_overlap": "not established"
      },
      "intern_08b": {
        "label": "Intern-Decision-0.8B",
        "origin": "new_original_weight_measured",
        "metrics": {
          "refund": {
            "correct": 72,
            "n": 96,
            "score": 0.75
          },
          "access": {
            "correct": 40,
            "n": 96,
            "score": 0.4166666666666667
          },
          "routing": {
            "correct": 56,
            "n": 96,
            "score": 0.5833333333333334
          },
          "policy_action": {
            "correct": 168,
            "n": 288,
            "score": 0.5833333333333334
          },
          "legal": {
            "correct": 61,
            "n": 144,
            "score": 0.4236111111111111
          },
          "science": {
            "correct": 149,
            "n": 339,
            "score": 0.43952802359882004
          },
          "action_head": {
            "correct": 168,
            "n": 288,
            "score": 0.5833333333333334
          },
          "review_head": {
            "correct": 184,
            "n": 288,
            "score": 0.6388888888888888
          },
          "severity_head": {
            "correct": 166,
            "n": 288,
            "score": 0.5763888888888888
          },
          "all_heads": {
            "correct": 51,
            "n": 288,
            "score": 0.17708333333333334
          },
          "policy_counterfactual": {
            "correct": 102,
            "n": 288,
            "score": 0.3541666666666667
          },
          "natural_reversal": {
            "score": 0.3444260078662733,
            "components": {
              "legal": {
                "correct": 41,
                "n": 144,
                "score": 0.2847222222222222
              },
              "science": {
                "correct": 137,
                "n": 339,
                "score": 0.40412979351032446
              }
            }
          },
          "overall_domain_equal": {
            "score": 0.4821574893477549,
            "ci95": [
              0.44415158529930904,
              0.5207920544400935
            ]
          },
          "overall_task_equal": {
            "score": 0.5226278269419862
          }
        },
        "decisions": 4905,
        "errors": 0,
        "known_training_overlap": "not established",
        "checkpoint": {
          "repo": "internlm/Intern-Decision-0.8B",
          "revision": "85a0cc5a99d67ea8d56dfe98115689212867171d"
        },
        "admission_sha256": "2d85d272d7cdf0de0cf71f1a57bd305468cbcd3d510a640ac67f4965e6860c28"
      },
      "intern_2b": {
        "label": "Intern-Decision-2B",
        "origin": "new_original_weight_measured",
        "metrics": {
          "refund": {
            "correct": 76,
            "n": 96,
            "score": 0.7916666666666666
          },
          "access": {
            "correct": 73,
            "n": 96,
            "score": 0.7604166666666666
          },
          "routing": {
            "correct": 70,
            "n": 96,
            "score": 0.7291666666666666
          },
          "policy_action": {
            "correct": 219,
            "n": 288,
            "score": 0.7604166666666666
          },
          "legal": {
            "correct": 67,
            "n": 144,
            "score": 0.4652777777777778
          },
          "science": {
            "correct": 206,
            "n": 339,
            "score": 0.6076696165191741
          },
          "action_head": {
            "correct": 219,
            "n": 288,
            "score": 0.7604166666666666
          },
          "review_head": {
            "correct": 189,
            "n": 288,
            "score": 0.65625
          },
          "severity_head": {
            "correct": 185,
            "n": 288,
            "score": 0.6423611111111112
          },
          "all_heads": {
            "correct": 86,
            "n": 288,
            "score": 0.2986111111111111
          },
          "policy_counterfactual": {
            "correct": 167,
            "n": 288,
            "score": 0.5798611111111112
          },
          "natural_reversal": {
            "score": 0.5226769911504424,
            "components": {
              "legal": {
                "correct": 66,
                "n": 144,
                "score": 0.4583333333333333
              },
              "science": {
                "correct": 199,
                "n": 339,
                "score": 0.5870206489675516
              }
            }
          },
          "overall_domain_equal": {
            "score": 0.6111213536545396,
            "ci95": [
              0.574717892102223,
              0.647214802829896
            ]
          },
          "overall_task_equal": {
            "score": 0.6708394788593903
          }
        },
        "decisions": 4905,
        "errors": 0,
        "known_training_overlap": "not established",
        "checkpoint": {
          "repo": "internlm/Intern-Decision-2B",
          "revision": "8797836c65fc91a2435b1fb6850b5f0aabd75cc3"
        },
        "admission_sha256": "6ce210a9581a07aea3365201b7079daa9c1df3c8da981a809a0cad1e015aff26"
      },
      "llama32_1b_public": {
        "label": "Llama-3.2-1B-Instruct (Unsloth BF16 distribution)",
        "origin": "new_original_weight_measured",
        "metrics": {
          "refund": {
            "correct": 24,
            "n": 96,
            "score": 0.25
          },
          "access": {
            "correct": 33,
            "n": 96,
            "score": 0.34375
          },
          "routing": {
            "correct": 29,
            "n": 96,
            "score": 0.3020833333333333
          },
          "policy_action": {
            "correct": 86,
            "n": 288,
            "score": 0.2986111111111111
          },
          "legal": {
            "correct": 45,
            "n": 144,
            "score": 0.3125
          },
          "science": {
            "correct": 120,
            "n": 339,
            "score": 0.35398230088495575
          },
          "action_head": {
            "correct": 86,
            "n": 288,
            "score": 0.2986111111111111
          },
          "review_head": {
            "correct": 171,
            "n": 288,
            "score": 0.59375
          },
          "severity_head": {
            "correct": 92,
            "n": 288,
            "score": 0.3194444444444444
          },
          "all_heads": {
            "correct": 18,
            "n": 288,
            "score": 0.0625
          },
          "policy_counterfactual": {
            "correct": 0,
            "n": 288,
            "score": 0
          },
          "natural_reversal": {
            "score": 0.027071042281219274,
            "components": {
              "legal": {
                "correct": 1,
                "n": 144,
                "score": 0.006944444444444444
              },
              "science": {
                "correct": 16,
                "n": 339,
                "score": 0.0471976401179941
              }
            }
          },
          "overall_domain_equal": {
            "score": 0.3216978039986889,
            "ci95": [
              0.2905593025270268,
              0.3531921855210093
            ]
          },
          "overall_task_equal": {
            "score": 0.31246312684365785
          }
        },
        "decisions": 4905,
        "errors": 0,
        "known_training_overlap": "not established",
        "checkpoint": {
          "repo": "unsloth/Llama-3.2-1B-Instruct",
          "revision": "5a8abab4a5d6f164389b1079fb721cfab8d7126c"
        },
        "admission_sha256": "1785bc154dd1c285313c846922b95b7b1a3cf3a36f80f8aa013aebe07f667b77"
      },
      "llama32_3b_public": {
        "label": "Llama-3.2-3B-Instruct (Unsloth BF16 distribution)",
        "origin": "new_original_weight_measured",
        "metrics": {
          "refund": {
            "correct": 24,
            "n": 96,
            "score": 0.25
          },
          "access": {
            "correct": 30,
            "n": 96,
            "score": 0.3125
          },
          "routing": {
            "correct": 42,
            "n": 96,
            "score": 0.4375
          },
          "policy_action": {
            "correct": 96,
            "n": 288,
            "score": 0.3333333333333333
          },
          "legal": {
            "correct": 47,
            "n": 144,
            "score": 0.3263888888888889
          },
          "science": {
            "correct": 149,
            "n": 339,
            "score": 0.43952802359882004
          },
          "action_head": {
            "correct": 96,
            "n": 288,
            "score": 0.3333333333333333
          },
          "review_head": {
            "correct": 171,
            "n": 288,
            "score": 0.59375
          },
          "severity_head": {
            "correct": 104,
            "n": 288,
            "score": 0.3611111111111111
          },
          "all_heads": {
            "correct": 29,
            "n": 288,
            "score": 0.10069444444444445
          },
          "policy_counterfactual": {
            "correct": 0,
            "n": 288,
            "score": 0
          },
          "natural_reversal": {
            "score": 0.22099311701081614,
            "components": {
              "legal": {
                "correct": 8,
                "n": 144,
                "score": 0.05555555555555555
              },
              "science": {
                "correct": 131,
                "n": 339,
                "score": 0.3864306784660767
              }
            }
          },
          "overall_domain_equal": {
            "score": 0.36641674860701406,
            "ci95": [
              0.3299162082346397,
              0.4042676826305124
            ]
          },
          "overall_task_equal": {
            "score": 0.3531833824975418
          }
        },
        "decisions": 4905,
        "errors": 0,
        "known_training_overlap": "not established",
        "checkpoint": {
          "repo": "unsloth/Llama-3.2-3B-Instruct",
          "revision": "006f5dcd1393c3add266de40994ba96225e9689d"
        },
        "admission_sha256": "1a74ead22621cc17d245cc42b0180f528e11c821d80796a5196b1af6e11d72d6"
      },
      "qwen3_17b": {
        "label": "Qwen3-1.7B",
        "origin": "new_original_weight_measured",
        "metrics": {
          "refund": {
            "correct": 24,
            "n": 96,
            "score": 0.25
          },
          "access": {
            "correct": 39,
            "n": 96,
            "score": 0.40625
          },
          "routing": {
            "correct": 24,
            "n": 96,
            "score": 0.25
          },
          "policy_action": {
            "correct": 87,
            "n": 288,
            "score": 0.3020833333333333
          },
          "legal": {
            "correct": 53,
            "n": 144,
            "score": 0.3680555555555556
          },
          "science": {
            "correct": 179,
            "n": 339,
            "score": 0.528023598820059
          },
          "action_head": {
            "correct": 87,
            "n": 288,
            "score": 0.3020833333333333
          },
          "review_head": {
            "correct": 171,
            "n": 288,
            "score": 0.59375
          },
          "severity_head": {
            "correct": 141,
            "n": 288,
            "score": 0.4895833333333333
          },
          "all_heads": {
            "correct": 28,
            "n": 288,
            "score": 0.09722222222222222
          },
          "policy_counterfactual": {
            "correct": 10,
            "n": 288,
            "score": 0.034722222222222224
          },
          "natural_reversal": {
            "score": 0.17093780727630287,
            "components": {
              "legal": {
                "correct": 11,
                "n": 144,
                "score": 0.0763888888888889
              },
              "science": {
                "correct": 90,
                "n": 339,
                "score": 0.26548672566371684
              }
            }
          },
          "overall_domain_equal": {
            "score": 0.3993874959029826,
            "ci95": [
              0.362730976483019,
              0.43454068022749315
            ]
          },
          "overall_task_equal": {
            "score": 0.3604658308751229
          }
        },
        "decisions": 4905,
        "errors": 0,
        "known_training_overlap": "not established",
        "checkpoint": {
          "repo": "Qwen/Qwen3-1.7B",
          "revision": "70d244cc86ccca08cf5af4e1e306ecf908b1ad5e"
        },
        "admission_sha256": "0aee91447e5c3f97729a33d9267e0160b5a10cc91d8981b388a08634058fe1c8"
      }
    },
    "rankings": {
      "refund": {
        "english": 16,
        "multilingual": 16,
        "llama31_8b_instruct": 15,
        "qwen3_8b": 16,
        "jev-1.13.0": 1,
        "kev_08b": 13,
        "kev_4b": 8,
        "kev_9b": 1,
        "nanojev": 16,
        "qwen35_9b": 12,
        "kev_27b": 1,
        "qwen35_08b": 16,
        "qwen35_4b": 14,
        "qwen38_27b": 10,
        "startlux_4b": 1,
        "startlux_9b": 1,
        "startlux_27b": 1,
        "intern_4b": 7,
        "intern_08b": 11,
        "intern_2b": 9,
        "llama32_1b_public": 16,
        "llama32_3b_public": 16,
        "qwen3_17b": 16
      },
      "access": {
        "english": 18,
        "multilingual": 23,
        "llama31_8b_instruct": 18,
        "qwen3_8b": 13,
        "jev-1.13.0": 1,
        "kev_08b": 13,
        "kev_4b": 7,
        "kev_9b": 5,
        "nanojev": 18,
        "qwen35_9b": 9,
        "kev_27b": 1,
        "qwen35_08b": 15,
        "qwen35_4b": 11,
        "qwen38_27b": 7,
        "startlux_4b": 6,
        "startlux_9b": 1,
        "startlux_27b": 1,
        "intern_4b": 9,
        "intern_08b": 16,
        "intern_2b": 12,
        "llama32_1b_public": 18,
        "llama32_3b_public": 22,
        "qwen3_17b": 17
      },
      "routing": {
        "english": 8,
        "multilingual": 23,
        "llama31_8b_instruct": 19,
        "qwen3_8b": 16,
        "jev-1.13.0": 13,
        "kev_08b": 6,
        "kev_4b": 4,
        "kev_9b": 9,
        "nanojev": 19,
        "qwen35_9b": 3,
        "kev_27b": 1,
        "qwen35_08b": 19,
        "qwen35_4b": 12,
        "qwen38_27b": 4,
        "startlux_4b": 15,
        "startlux_9b": 13,
        "startlux_27b": 1,
        "intern_4b": 10,
        "intern_08b": 11,
        "intern_2b": 7,
        "llama32_1b_public": 18,
        "llama32_3b_public": 17,
        "qwen3_17b": 19
      },
      "policy_action": {
        "english": 16,
        "multilingual": 23,
        "llama31_8b_instruct": 19,
        "qwen3_8b": 15,
        "jev-1.13.0": 6,
        "kev_08b": 12,
        "kev_4b": 4,
        "kev_9b": 3,
        "nanojev": 22,
        "qwen35_9b": 8,
        "kev_27b": 1,
        "qwen35_08b": 17,
        "qwen35_4b": 13,
        "qwen38_27b": 5,
        "startlux_4b": 9,
        "startlux_9b": 6,
        "startlux_27b": 1,
        "intern_4b": 9,
        "intern_08b": 14,
        "intern_2b": 11,
        "llama32_1b_public": 21,
        "llama32_3b_public": 18,
        "qwen3_17b": 20
      },
      "legal": {
        "english": 16,
        "multilingual": 18,
        "llama31_8b_instruct": 22,
        "qwen3_8b": 14,
        "jev-1.13.0": 6,
        "kev_08b": 12,
        "kev_4b": 10,
        "kev_9b": 9,
        "nanojev": 20,
        "qwen35_9b": 7,
        "kev_27b": 4,
        "qwen35_08b": 22,
        "qwen35_4b": 11,
        "qwen38_27b": 5,
        "startlux_4b": 2,
        "startlux_9b": 2,
        "startlux_27b": 1,
        "intern_4b": 8,
        "intern_08b": 15,
        "intern_2b": 13,
        "llama32_1b_public": 21,
        "llama32_3b_public": 19,
        "qwen3_17b": 17
      },
      "science": {
        "english": 16,
        "multilingual": 18,
        "llama31_8b_instruct": 14,
        "qwen3_8b": 11,
        "jev-1.13.0": 1,
        "kev_08b": 17,
        "kev_4b": 8,
        "kev_9b": 9,
        "nanojev": 22,
        "qwen35_9b": 5,
        "kev_27b": 3,
        "qwen35_08b": 21,
        "qwen35_4b": 12,
        "qwen38_27b": 3,
        "startlux_4b": 7,
        "startlux_9b": 2,
        "startlux_27b": 6,
        "intern_4b": 10,
        "intern_08b": 19,
        "intern_2b": 13,
        "llama32_1b_public": 23,
        "llama32_3b_public": 19,
        "qwen3_17b": 15
      },
      "action_head": {
        "english": 16,
        "multilingual": 23,
        "llama31_8b_instruct": 19,
        "qwen3_8b": 15,
        "jev-1.13.0": 6,
        "kev_08b": 12,
        "kev_4b": 4,
        "kev_9b": 3,
        "nanojev": 22,
        "qwen35_9b": 8,
        "kev_27b": 1,
        "qwen35_08b": 17,
        "qwen35_4b": 13,
        "qwen38_27b": 5,
        "startlux_4b": 9,
        "startlux_9b": 6,
        "startlux_27b": 1,
        "intern_4b": 9,
        "intern_08b": 14,
        "intern_2b": 11,
        "llama32_1b_public": 21,
        "llama32_3b_public": 18,
        "qwen3_17b": 20
      },
      "review_head": {
        "english": 13,
        "multilingual": 16,
        "llama31_8b_instruct": 16,
        "qwen3_8b": 13,
        "jev-1.13.0": 3,
        "kev_08b": 11,
        "kev_4b": 6,
        "kev_9b": 8,
        "nanojev": 16,
        "qwen35_9b": 9,
        "kev_27b": 1,
        "qwen35_08b": 23,
        "qwen35_4b": 22,
        "qwen38_27b": 10,
        "startlux_4b": 4,
        "startlux_9b": 5,
        "startlux_27b": 1,
        "intern_4b": 7,
        "intern_08b": 13,
        "intern_2b": 12,
        "llama32_1b_public": 16,
        "llama32_3b_public": 16,
        "qwen3_17b": 16
      },
      "severity_head": {
        "english": 19,
        "multilingual": 23,
        "llama31_8b_instruct": 13,
        "qwen3_8b": 8,
        "jev-1.13.0": 3,
        "kev_08b": 17,
        "kev_4b": 10,
        "kev_9b": 6,
        "nanojev": 21,
        "qwen35_9b": 12,
        "kev_27b": 2,
        "qwen35_08b": 18,
        "qwen35_4b": 15,
        "qwen38_27b": 8,
        "startlux_4b": 5,
        "startlux_9b": 4,
        "startlux_27b": 1,
        "intern_4b": 7,
        "intern_08b": 14,
        "intern_2b": 11,
        "llama32_1b_public": 21,
        "llama32_3b_public": 20,
        "qwen3_17b": 16
      },
      "all_heads": {
        "english": 17,
        "multilingual": 20,
        "llama31_8b_instruct": 16,
        "qwen3_8b": 12,
        "jev-1.13.0": 3,
        "kev_08b": 13,
        "kev_4b": 8,
        "kev_9b": 6,
        "nanojev": 21,
        "qwen35_9b": 9,
        "kev_27b": 2,
        "qwen35_08b": 23,
        "qwen35_4b": 14,
        "qwen38_27b": 10,
        "startlux_4b": 5,
        "startlux_9b": 4,
        "startlux_27b": 1,
        "intern_4b": 7,
        "intern_08b": 15,
        "intern_2b": 11,
        "llama32_1b_public": 22,
        "llama32_3b_public": 18,
        "qwen3_17b": 19
      },
      "policy_counterfactual": {
        "english": 16,
        "multilingual": 18,
        "llama31_8b_instruct": 18,
        "qwen3_8b": 15,
        "jev-1.13.0": 7,
        "kev_08b": 14,
        "kev_4b": 4,
        "kev_9b": 3,
        "nanojev": 18,
        "qwen35_9b": 6,
        "kev_27b": 1,
        "qwen35_08b": 18,
        "qwen35_4b": 12,
        "qwen38_27b": 4,
        "startlux_4b": 10,
        "startlux_9b": 7,
        "startlux_27b": 1,
        "intern_4b": 9,
        "intern_08b": 13,
        "intern_2b": 11,
        "llama32_1b_public": 18,
        "llama32_3b_public": 18,
        "qwen3_17b": 17
      },
      "natural_reversal": {
        "english": 14,
        "multilingual": 18,
        "llama31_8b_instruct": 19,
        "qwen3_8b": 13,
        "jev-1.13.0": 5,
        "kev_08b": 15,
        "kev_4b": 9,
        "kev_9b": 7,
        "nanojev": 16,
        "qwen35_9b": 8,
        "kev_27b": 4,
        "qwen35_08b": 20,
        "qwen35_4b": 11,
        "qwen38_27b": 6,
        "startlux_4b": 3,
        "startlux_9b": 2,
        "startlux_27b": 1,
        "intern_4b": 10,
        "intern_08b": 17,
        "intern_2b": 12,
        "llama32_1b_public": 23,
        "llama32_3b_public": 21,
        "qwen3_17b": 22
      },
      "overall_domain_equal": {
        "english": 16,
        "multilingual": 21,
        "llama31_8b_instruct": 18,
        "qwen3_8b": 14,
        "jev-1.13.0": 5,
        "kev_08b": 13,
        "kev_4b": 9,
        "kev_9b": 8,
        "nanojev": 22,
        "qwen35_9b": 7,
        "kev_27b": 2,
        "qwen35_08b": 20,
        "qwen35_4b": 11,
        "qwen38_27b": 6,
        "startlux_4b": 4,
        "startlux_9b": 3,
        "startlux_27b": 1,
        "intern_4b": 10,
        "intern_08b": 15,
        "intern_2b": 12,
        "llama32_1b_public": 23,
        "llama32_3b_public": 19,
        "qwen3_17b": 17
      },
      "overall_task_equal": {
        "english": 16,
        "multilingual": 21,
        "llama31_8b_instruct": 18,
        "qwen3_8b": 15,
        "jev-1.13.0": 6,
        "kev_08b": 13,
        "kev_4b": 8,
        "kev_9b": 4,
        "nanojev": 23,
        "qwen35_9b": 9,
        "kev_27b": 2,
        "qwen35_08b": 20,
        "qwen35_4b": 12,
        "qwen38_27b": 7,
        "startlux_4b": 5,
        "startlux_9b": 3,
        "startlux_27b": 1,
        "intern_4b": 10,
        "intern_08b": 14,
        "intern_2b": 11,
        "llama32_1b_public": 22,
        "llama32_3b_public": 19,
        "qwen3_17b": 17
      }
    },
    "matched_decisions_per_model": 4905,
    "protocol_sha256": "ac12f140dc6c5da53694452227813d8b1d4062634dda6abdfb0a8a765bb4f55f",
    "builder_sha256": "8f20ae1ca483593fb6f4d0f5a0a512156fa6cdefd21fca1955ec13f5c15a9c95",
    "historical_file_receipts": {
      "english": [
        [
          "research/confirmation_v1/results/english/policy_refund.json.gz",
          "6205b50ea6a481a0387d2f8057f08ae6ddea0cb4473f4d1dd2fce9596ac04212"
        ],
        [
          "research/confirmation_v1/results/english/policy_access.json.gz",
          "364a4adfc560fed15508ae8d25d0bf5694c9880da8b8d5579091e410affdaf4f"
        ],
        [
          "research/confirmation_v1/results/english/policy_routing.json.gz",
          "cdc7ce3593696c8dda94321ad8dec29c23053441b204597d33af892d0a55abf2"
        ],
        [
          "research/domain_expansion_v1/results/local/english/contractnli_base.json.gz",
          "5ea13826c3ed396ead5dab6cfed3c4c5700326f2d2de193456b44810d864d3d6"
        ],
        [
          "research/domain_expansion_v1/results/local/english/contractnli_exact_repeat.json.gz",
          "76053f3bbaf95793bab68dbd0d969038ffed9b712679d72944be2782fda7708f"
        ],
        [
          "research/domain_expansion_v1/results/local/english/contractnli_reversed_option_order.json.gz",
          "afe969a1052fb2eceef31fe259817095a22436fd73081443240bede6c61a945b"
        ],
        [
          "research/scifact3_census_v1/results/local/english/scifact3_base.json.gz",
          "3de6e613a7a9651233f7faaae797131aadac9ff2e9edfc309c7ccac58076fe3a"
        ],
        [
          "research/scifact3_census_v1/results/local/english/scifact3_exact_repeat.json.gz",
          "de75d39e5d5737b73a19efa1072458fa1edd02bc79114f6477864e3d317c8ada"
        ],
        [
          "research/scifact3_census_v1/results/local/english/scifact3_reversed_option_order.json.gz",
          "297e4cb02259ee273194de63ed619123516bc4808c2fcf2ff344723848cb87de"
        ]
      ],
      "multilingual": [
        [
          "research/confirmation_v1/results/multilingual/policy_refund.json.gz",
          "0a8acf822b712011822e448f03935fa46d5be9aa5634fe54f96842493df4e105"
        ],
        [
          "research/confirmation_v1/results/multilingual/policy_access.json.gz",
          "6173c1a4e646cb2abd9a8f3cc85dab76846d6dfcecdd59aa59933d2fec69fd9e"
        ],
        [
          "research/confirmation_v1/results/multilingual/policy_routing.json.gz",
          "bf6aaf1d69c62273f417cd714af6ef05f7aa83495cadd9b09ebaf1f70d947c2d"
        ],
        [
          "research/domain_expansion_v1/results/local/multilingual/contractnli_base.json.gz",
          "5ec118f58262c3402dd3fbcc8d1d32e51a9c7e940849020b437cb316bc25013c"
        ],
        [
          "research/domain_expansion_v1/results/local/multilingual/contractnli_exact_repeat.json.gz",
          "787b30209f0c7f7b0d15b65151773f0a41fc98e9b7a6d23b8998ee6e80608f8d"
        ],
        [
          "research/domain_expansion_v1/results/local/multilingual/contractnli_reversed_option_order.json.gz",
          "e1b3249719ceab5871eb04116e183de4ab6e5521c67e422f16b932cc723557a3"
        ],
        [
          "research/scifact3_census_v1/results/local/multilingual/scifact3_base.json.gz",
          "45fcb7f3ebb8f8c6a4495f38688bc200ae3c114a54a5cd7ad07639770f528e83"
        ],
        [
          "research/scifact3_census_v1/results/local/multilingual/scifact3_exact_repeat.json.gz",
          "608a0d43630ade5dc7d34e9f52865a197ef9b6656d44b2cb2077c616f7e63ce0"
        ],
        [
          "research/scifact3_census_v1/results/local/multilingual/scifact3_reversed_option_order.json.gz",
          "f8e6dc98421ec95a4f576012b58c974ce00469e516531df2906d167ce607a6db"
        ]
      ],
      "llama31_8b_instruct": [
        [
          "research/confirmation_v1/results/llama31_8b_instruct/policy_refund.json.gz",
          "5f90a9e5b23e5e9491db3475be221af6ec82c03922ad9d4c83b86824453cc7b3"
        ],
        [
          "research/confirmation_v1/results/llama31_8b_instruct/policy_access.json.gz",
          "486142e2f811b885d489d56ccc5cb4e97105c2617445fe4296dc61089e217b31"
        ],
        [
          "research/confirmation_v1/results/llama31_8b_instruct/policy_routing.json.gz",
          "347e55228859ad77abcf216b5c84aa3b55317826337871a0285924460a7146f9"
        ],
        [
          "research/domain_expansion_v1/results/local/llama31_8b_instruct/contractnli_base.json.gz",
          "387bc685b61d253747c9422c0f0e17819d58f2a48bbffd24d245bb6f8e0013d1"
        ],
        [
          "research/domain_expansion_v1/results/local/llama31_8b_instruct/contractnli_exact_repeat.json.gz",
          "dc464ed89bbf14db581a3241eee143fb11270ee26206299df8600d8f1b02eb48"
        ],
        [
          "research/domain_expansion_v1/results/local/llama31_8b_instruct/contractnli_reversed_option_order.json.gz",
          "b3c2c49e541a59a87c73c26c85a636d9e28e5c7f5e8bd48c709cb7692ea6aa2f"
        ],
        [
          "research/scifact3_census_v1/results/local/llama31_8b_instruct/scifact3_base.json.gz",
          "c2be2429c32b0c899e369834253200cc95b53799de18c951621236dfadd5a888"
        ],
        [
          "research/scifact3_census_v1/results/local/llama31_8b_instruct/scifact3_exact_repeat.json.gz",
          "2eb2d111a19c5f6437b54a8a1b7f73c8dd0deb8f9f023a6cd196d32822a5a423"
        ],
        [
          "research/scifact3_census_v1/results/local/llama31_8b_instruct/scifact3_reversed_option_order.json.gz",
          "b3f3c3a16008ad2d7c37ab4eee827a301bf936c432834cbf61a8a2f88d9618bb"
        ]
      ],
      "qwen3_8b": [
        [
          "research/confirmation_v1/results/qwen3_8b/policy_refund.json.gz",
          "7e1f0b1d40f0dc613456e551c87c14f6b1ca4fe19fe2311b1f881e0024c5fe45"
        ],
        [
          "research/confirmation_v1/results/qwen3_8b/policy_access.json.gz",
          "41b2e4f1f784b0ae9359d1830079d1e171ca4ea9bfcd5fdbc4fbffcf3c0d5a88"
        ],
        [
          "research/confirmation_v1/results/qwen3_8b/policy_routing.json.gz",
          "d3741aff94c09a61aadf569dfb1b707a6dff8681631d3c0fbe0e67b91580bd32"
        ],
        [
          "research/domain_expansion_v1/results/local/qwen3_8b/contractnli_base.json.gz",
          "7b79274b82c30671b07ef8a68645a8a2b01cd809f477f20763b4a11dd3c15d62"
        ],
        [
          "research/domain_expansion_v1/results/local/qwen3_8b/contractnli_exact_repeat.json.gz",
          "513d708efd55e57276bb9697d83e655e0024fd6f77b35cf8b528cc40f2bffe97"
        ],
        [
          "research/domain_expansion_v1/results/local/qwen3_8b/contractnli_reversed_option_order.json.gz",
          "267b6f2ff4c62e734e5b541aad6a087a4e870967e0758810bd8077b254c9a73c"
        ],
        [
          "research/scifact3_census_v1/results/local/qwen3_8b/scifact3_base.json.gz",
          "8117972364c9dd47b05be79d9b76c5c5a7a888a35425031b9f7f8c9f310b62d5"
        ],
        [
          "research/scifact3_census_v1/results/local/qwen3_8b/scifact3_exact_repeat.json.gz",
          "d6e6e7285f02999f2fb5be165b8cc43e2414b732836a6e5e7b4939fb730cbfba"
        ],
        [
          "research/scifact3_census_v1/results/local/qwen3_8b/scifact3_reversed_option_order.json.gz",
          "b60a4670e59feeddb2aaa0e7439fcf31bde7eadcf24e3ee738129634b0fc65f3"
        ]
      ],
      "jev-1.13.0": [
        [
          "research/confirmation_v1/api/jev-1.13.0/policy_refund.json.gz",
          "6ea83d4d1d578ee43f4b114f68d85bdaaaf6cf88df8bb0fa634ff7037d466a7c"
        ],
        [
          "research/confirmation_v1/api/jev-1.13.0/policy_access.json.gz",
          "58affde3e3a223624d8c769008d816992f69582feb86965f71d64812eee90fea"
        ],
        [
          "research/confirmation_v1/api/jev-1.13.0/policy_routing.json.gz",
          "290e95c91f37a30c92df117462685ff84267eae08774ffd3d3840018aef7e1aa"
        ],
        [
          "research/domain_expansion_v1/results/jev-1.13.0-continue-invalid/contractnli_base.json.gz",
          "ff6467568f5056722826c85aff32da175671b3612562297c4ffcb913a0ca0992"
        ],
        [
          "research/domain_expansion_v1/results/jev-1.13.0-continue-invalid/contractnli_exact_repeat.json.gz",
          "115c84f65be266f5a356fd29590a9574334abb9ba6a07ee47614acfba2b0833f"
        ],
        [
          "research/domain_expansion_v1/results/jev-1.13.0-continue-invalid/contractnli_reversed_option_order.json.gz",
          "71429d22065710d0a33a83edea3023e403f60118f5ae8b55759de8707078308f"
        ],
        [
          "research/scifact3_census_v1/results/jev-1.13.0/scifact3_base.json.gz",
          "5ea7b0bf558b0e754415a5c04c118df1c4b33f5c83297b770b73498f9f534bae"
        ],
        [
          "research/scifact3_census_v1/results/jev-1.13.0/scifact3_exact_repeat.json.gz",
          "3ba5d7cedc6936df4de77a4c6fcccf543824d31cb4fea7addef06624f919f6d5"
        ],
        [
          "research/scifact3_census_v1/results/jev-1.13.0/scifact3_reversed_option_order.json.gz",
          "19b34d57845c6de57801cd87ddceeb4c07c8b6858a84ace5945d92289d67d9d7"
        ]
      ]
    },
    "transfer_summary_sha256": "01183ad1fe818ae278bc1e369e47eb5d4e562b9a32fd045b32f688d0c5bae0d2",
    "previous_scores_sha256": "91c07ef3586deb526a811f86aa04ee92d62add43bf765edee476396020448cb3",
    "bootstrap": {
      "draws": 5000,
      "seed": 20261002,
      "unit": "policy state / contract document / science claim"
    },
    "expansion_summary_sha256": "12935896cdfb25f4a63e867a43271fa78dc1b7f821d1d5917c12bdea6eb67096"
  },
  "transfer": {
    "startlux_4b": {
      "receipt": {
        "raw_sha256": "76aa9bff2cd18d726f77695dd1d52c12e276a238faddf56848ccea7ca01d0bd5",
        "metadata_sha256": "3cdd8226b885736fff334f701a833734a54cdd074ed7ae1117e3c7390a8a7b5d",
        "count": 1152,
        "errors": 0
      },
      "metrics": {
        "cladder": {
          "original": {
            "n": 144,
            "correct": 92,
            "score": 0.6388888888888888,
            "ci95": [
              0.5594405594405595,
              0.7172633649618143
            ],
            "errors": 0
          },
          "reversed": {
            "n": 144,
            "correct": 94,
            "score": 0.6527777777777778,
            "ci95": [
              0.5734265734265734,
              0.7328767123287672
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 144,
            "correct": 91,
            "score": 0.6319444444444444,
            "ci95": [
              0.5517241379310345,
              0.7114323802519076
            ]
          },
          "prediction_flips": 4,
          "strata": {
            "1": {
              "n": 48,
              "correct": 29,
              "score": 0.6041666666666666,
              "ci95": [
                0.4583333333333333,
                0.7446808510638298
              ],
              "errors": 0
            },
            "2": {
              "n": 48,
              "correct": 32,
              "score": 0.6666666666666666,
              "ci95": [
                0.5416666666666666,
                0.7916666666666666
              ],
              "errors": 0
            },
            "3": {
              "n": 48,
              "correct": 31,
              "score": 0.6458333333333334,
              "ci95": [
                0.5,
                0.7708333333333334
              ],
              "errors": 0
            }
          }
        },
        "cruxeval": {
          "original": {
            "n": 128,
            "correct": 66,
            "score": 0.515625,
            "ci95": [
              0.4296875,
              0.6015625
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 76,
            "score": 0.59375,
            "ci95": [
              0.5078125,
              0.671875
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 58,
            "score": 0.453125,
            "ci95": [
              0.3671875,
              0.5390625
            ]
          },
          "prediction_flips": 40,
          "strata": {
            "all": {
              "n": 128,
              "correct": 66,
              "score": 0.515625,
              "ci95": [
                0.4296875,
                0.6015625
              ],
              "errors": 0
            }
          },
          "uniform_choice_chance": 0.3639136904761905,
          "candidate_counts": {
            "2": 56,
            "3": 28,
            "4": 23,
            "5": 8,
            "6": 5,
            "7": 5,
            "8": 2,
            "10": 1
          }
        },
        "finentity": {
          "original": {
            "n": 128,
            "correct": 120,
            "score": 0.9375,
            "ci95": [
              0.890625,
              0.9765625
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 119,
            "score": 0.9296875,
            "ci95": [
              0.8828125,
              0.96875
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 119,
            "score": 0.9296875,
            "ci95": [
              0.8828125,
              0.96875
            ]
          },
          "prediction_flips": 1,
          "strata": {
            "Negative": {
              "n": 43,
              "correct": 41,
              "score": 0.9534883720930233,
              "ci95": [
                0.8837209302325582,
                1
              ],
              "errors": 0
            },
            "Neutral": {
              "n": 42,
              "correct": 36,
              "score": 0.8571428571428571,
              "ci95": [
                0.7380952380952381,
                0.9523809523809523
              ],
              "errors": 0
            },
            "Positive": {
              "n": 43,
              "correct": 43,
              "score": 1,
              "ci95": [
                1,
                1
              ],
              "errors": 0
            }
          }
        },
        "when2call": {
          "original": {
            "n": 128,
            "correct": 105,
            "score": 0.8203125,
            "ci95": [
              0.75,
              0.8828125
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 104,
            "score": 0.8125,
            "ci95": [
              0.7421875,
              0.875
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 98,
            "score": 0.765625,
            "ci95": [
              0.6951171875,
              0.8359375
            ]
          },
          "prediction_flips": 15,
          "strata": {
            "cannot_answer": {
              "n": 43,
              "correct": 33,
              "score": 0.7674418604651163,
              "ci95": [
                0.627906976744186,
                0.8837209302325582
              ],
              "errors": 0
            },
            "request_for_info": {
              "n": 42,
              "correct": 37,
              "score": 0.8809523809523809,
              "ci95": [
                0.7857142857142857,
                0.9761904761904762
              ],
              "errors": 0
            },
            "tool_call": {
              "n": 43,
              "correct": 35,
              "score": 0.813953488372093,
              "ci95": [
                0.6976744186046512,
                0.9302325581395349
              ],
              "errors": 0
            }
          },
          "direct_false_selections": {
            "n": 128,
            "count": 3,
            "rate": 0.0234375,
            "gold_positives": 0
          }
        },
        "known_distribution": {
          "n": 96,
          "valid": 96,
          "errors": 0,
          "paired_settings": 48,
          "excess_brier": {
            "score": 0.05386751723392563,
            "ci95": [
              0.03510409707892261,
              0.07582645226157328
            ]
          },
          "expected_brier": {
            "score": 0.4990153016213081,
            "ci95": [
              0.43125138716952616,
              0.5592512279290983
            ]
          },
          "total_variation": {
            "score": 0.13977389642476534,
            "ci95": [
              0.11178902888286234,
              0.17006083487015186
            ]
          },
          "impossible_mass": {
            "score": 0.00761056017107856,
            "ci95": [
              0.0023672323024465447,
              0.014429895175351126
            ]
          },
          "uniform_all_candidates": {
            "n": 96,
            "valid": 96,
            "errors": 0,
            "paired_settings": 48,
            "excess_brier": {
              "score": 0.1439903922507942,
              "ci95": [
                0.09995881558499044,
                0.19733934748419735
              ]
            },
            "expected_brier": {
              "score": 0.5891381766381767,
              "ci95": [
                0.5522569444444446,
                0.6316912615740741
              ]
            },
            "total_variation": {
              "score": 0.2490848180933528,
              "ci95": [
                0.19992744552797803,
                0.30308455010232166
              ]
            },
            "impossible_mass": {
              "score": 0.07797364672364672,
              "ci95": [
                0.033561030982905984,
                0.13547743055555547
              ]
            }
          },
          "minus_uniform_excess_brier": {
            "n": 96,
            "delta": -0.09012287501686857,
            "ci95": [
              -0.14940152423361294,
              -0.043063484909669304
            ]
          },
          "categories": {
            "composition": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.0648165397229353,
                "ci95": [
                  0.02296355667294467,
                  0.11271724215939412
                ]
              },
              "expected_brier": {
                "score": 0.5780338120994785,
                "ci95": [
                  0.356675020996461,
                  0.7788984477126474
                ]
              },
              "total_variation": {
                "score": 0.1678778386349748,
                "ci95": [
                  0.08928494428635629,
                  0.24292863943381876
                ]
              },
              "impossible_mass": {
                "score": 0.012516027665174404,
                "ci95": [
                  0,
                  0.029965104820495327
                ]
              }
            },
            "daily_evidence": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.09382910483033194,
                "ci95": [
                  0.04414235773940951,
                  0.13752280260456295
                ]
              },
              "expected_brier": {
                "score": 0.48221860718669146,
                "ci95": [
                  0.41334880959194803,
                  0.5504914618485119
                ]
              },
              "total_variation": {
                "score": 0.18860455780262775,
                "ci95": [
                  0.109502443154347,
                  0.25287417268536366
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "direct": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.019241036468824215,
                "ci95": [
                  0.00916954040082154,
                  0.031141520529160912
                ]
              },
              "expected_brier": {
                "score": 0.3548824531354908,
                "ci95": [
                  0.09853043234223752,
                  0.6012243737278722
                ]
              },
              "total_variation": {
                "score": 0.09600716808429637,
                "ci95": [
                  0.05362554291810374,
                  0.13578963386850748
                ]
              },
              "impossible_mass": {
                "score": 0.0061499837763571204,
                "ci95": [
                  0.00046077023046278026,
                  0.013269309138238358
                ]
              }
            },
            "disclosure": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.09036943260582891,
                "ci95": [
                  0.03323845206524911,
                  0.17433446257129084
                ]
              },
              "expected_brier": {
                "score": 0.5092274572971869,
                "ci95": [
                  0.47096583939878117,
                  0.5485455644003161
                ]
              },
              "total_variation": {
                "score": 0.18034618538721664,
                "ci95": [
                  0.11383163849761294,
                  0.26117873109340256
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "history": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.014341251205107063,
                "ci95": [
                  0.004949862337678733,
                  0.02606367225813288
                ]
              },
              "expected_brier": {
                "score": 0.5238828726285258,
                "ci95": [
                  0.4237703745274628,
                  0.6287997161923167
                ]
              },
              "total_variation": {
                "score": 0.0856653210607829,
                "ci95": [
                  0.04334961015255435,
                  0.13580462713360716
                ]
              },
              "impossible_mass": {
                "score": 0.021784159570436598,
                "ci95": [
                  0,
                  0.05154118542430771
                ]
              }
            },
            "sequential": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.04060773857052637,
                "ci95": [
                  0.00881794500924082,
                  0.08347862814793154
                ]
              },
              "expected_brier": {
                "score": 0.5458466073804749,
                "ci95": [
                  0.4582026691438631,
                  0.640288750112181
                ]
              },
              "total_variation": {
                "score": 0.1201423075786936,
                "ci95": [
                  0.055393102924584284,
                  0.1949630124526093
                ]
              },
              "impossible_mass": {
                "score": 0.005213190014503235,
                "ci95": [
                  0,
                  0.012795911799584449
                ]
              }
            }
          }
        }
      },
      "temperature_sensitivity": {
        "temperature": 1.3872,
        "method": "recover softmax(logits) from released softmax(logits/T); no fitting; argmax invariant",
        "temperature_one": {
          "n": 96,
          "valid": 96,
          "errors": 0,
          "paired_settings": 48,
          "excess_brier": {
            "score": 0.07208586941069602,
            "ci95": [
              0.04789961133372403,
              0.09987666124794153
            ]
          },
          "expected_brier": {
            "score": 0.5172336537980784,
            "ci95": [
              0.4446937434974537,
              0.5829094556588443
            ]
          },
          "total_variation": {
            "score": 0.15853584479298613,
            "ci95": [
              0.12225656168071074,
              0.19346876046010783
            ]
          },
          "impossible_mass": {
            "score": 0.003486356768332891,
            "ci95": [
              0.0008804418959323893,
              0.006691188937487825
            ]
          }
        }
      }
    },
    "startlux_9b": {
      "receipt": {
        "raw_sha256": "b81599d517dcab1b9722dd43748ac6f015fe17d083e3b26b57cb5f53fa8d0dd5",
        "metadata_sha256": "38bec03ce42faab40bb423139906d865773096b045ed12b9d18e14ee849ce97c",
        "count": 1152,
        "errors": 0
      },
      "metrics": {
        "cladder": {
          "original": {
            "n": 144,
            "correct": 104,
            "score": 0.7222222222222222,
            "ci95": [
              0.6457714160839161,
              0.7959183673469388
            ],
            "errors": 0
          },
          "reversed": {
            "n": 144,
            "correct": 103,
            "score": 0.7152777777777778,
            "ci95": [
              0.6344827586206897,
              0.7902097902097902
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 144,
            "correct": 103,
            "score": 0.7152777777777778,
            "ci95": [
              0.6344827586206897,
              0.7902097902097902
            ]
          },
          "prediction_flips": 1,
          "strata": {
            "1": {
              "n": 48,
              "correct": 30,
              "score": 0.625,
              "ci95": [
                0.4791666666666667,
                0.7551020408163265
              ],
              "errors": 0
            },
            "2": {
              "n": 48,
              "correct": 36,
              "score": 0.75,
              "ci95": [
                0.625,
                0.875
              ],
              "errors": 0
            },
            "3": {
              "n": 48,
              "correct": 38,
              "score": 0.7916666666666666,
              "ci95": [
                0.6666666666666666,
                0.8958333333333334
              ],
              "errors": 0
            }
          }
        },
        "cruxeval": {
          "original": {
            "n": 128,
            "correct": 71,
            "score": 0.5546875,
            "ci95": [
              0.46875,
              0.640625
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 73,
            "score": 0.5703125,
            "ci95": [
              0.484375,
              0.65625
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 64,
            "score": 0.5,
            "ci95": [
              0.4140625,
              0.5859375
            ]
          },
          "prediction_flips": 25,
          "strata": {
            "all": {
              "n": 128,
              "correct": 71,
              "score": 0.5546875,
              "ci95": [
                0.46875,
                0.640625
              ],
              "errors": 0
            }
          },
          "uniform_choice_chance": 0.3639136904761905,
          "candidate_counts": {
            "2": 56,
            "3": 28,
            "4": 23,
            "5": 8,
            "6": 5,
            "7": 5,
            "8": 2,
            "10": 1
          }
        },
        "finentity": {
          "original": {
            "n": 128,
            "correct": 121,
            "score": 0.9453125,
            "ci95": [
              0.8984375,
              0.9765625
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 121,
            "score": 0.9453125,
            "ci95": [
              0.8984375,
              0.9765625
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 121,
            "score": 0.9453125,
            "ci95": [
              0.8984375,
              0.9765625
            ]
          },
          "prediction_flips": 0,
          "strata": {
            "Negative": {
              "n": 43,
              "correct": 41,
              "score": 0.9534883720930233,
              "ci95": [
                0.8837209302325582,
                1
              ],
              "errors": 0
            },
            "Neutral": {
              "n": 42,
              "correct": 38,
              "score": 0.9047619047619048,
              "ci95": [
                0.8095238095238095,
                0.9761904761904762
              ],
              "errors": 0
            },
            "Positive": {
              "n": 43,
              "correct": 42,
              "score": 0.9767441860465116,
              "ci95": [
                0.9302325581395349,
                1
              ],
              "errors": 0
            }
          }
        },
        "when2call": {
          "original": {
            "n": 128,
            "correct": 103,
            "score": 0.8046875,
            "ci95": [
              0.734375,
              0.8671875
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 106,
            "score": 0.828125,
            "ci95": [
              0.7578125,
              0.890625
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 102,
            "score": 0.796875,
            "ci95": [
              0.7265625,
              0.8671875
            ]
          },
          "prediction_flips": 7,
          "strata": {
            "cannot_answer": {
              "n": 43,
              "correct": 34,
              "score": 0.7906976744186046,
              "ci95": [
                0.6744186046511628,
                0.9069767441860465
              ],
              "errors": 0
            },
            "request_for_info": {
              "n": 42,
              "correct": 36,
              "score": 0.8571428571428571,
              "ci95": [
                0.7380952380952381,
                0.9523809523809523
              ],
              "errors": 0
            },
            "tool_call": {
              "n": 43,
              "correct": 33,
              "score": 0.7674418604651163,
              "ci95": [
                0.627906976744186,
                0.8837209302325582
              ],
              "errors": 0
            }
          },
          "direct_false_selections": {
            "n": 128,
            "count": 1,
            "rate": 0.0078125,
            "gold_positives": 0
          }
        },
        "known_distribution": {
          "n": 96,
          "valid": 96,
          "errors": 0,
          "paired_settings": 48,
          "excess_brier": {
            "score": 0.03219610021551663,
            "ci95": [
              0.021364887929261174,
              0.04535589749530563
            ]
          },
          "expected_brier": {
            "score": 0.47734388460289906,
            "ci95": [
              0.4107445039329619,
              0.5369944547085724
            ]
          },
          "total_variation": {
            "score": 0.11392898290242359,
            "ci95": [
              0.0907495383582697,
              0.1377263798630825
            ]
          },
          "impossible_mass": {
            "score": 0.007157543863237412,
            "ci95": [
              0.0012180624674691025,
              0.01705423702077944
            ]
          },
          "uniform_all_candidates": {
            "n": 96,
            "valid": 96,
            "errors": 0,
            "paired_settings": 48,
            "excess_brier": {
              "score": 0.1439903922507942,
              "ci95": [
                0.09995881558499044,
                0.19733934748419735
              ]
            },
            "expected_brier": {
              "score": 0.5891381766381767,
              "ci95": [
                0.5522569444444446,
                0.6316912615740741
              ]
            },
            "total_variation": {
              "score": 0.2490848180933528,
              "ci95": [
                0.19992744552797803,
                0.30308455010232166
              ]
            },
            "impossible_mass": {
              "score": 0.07797364672364672,
              "ci95": [
                0.033561030982905984,
                0.13547743055555547
              ]
            }
          },
          "minus_uniform_excess_brier": {
            "n": 96,
            "delta": -0.11179429203527758,
            "ci95": [
              -0.1669843744919962,
              -0.06600025472638457
            ]
          },
          "categories": {
            "composition": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.023087154772583184,
                "ci95": [
                  0.01095810803316336,
                  0.035726747672930305
                ]
              },
              "expected_brier": {
                "score": 0.5363044271491264,
                "ci95": [
                  0.33107205697972575,
                  0.7382457789633076
                ]
              },
              "total_variation": {
                "score": 0.116037891995587,
                "ci95": [
                  0.06369763188542439,
                  0.16546609116016175
                ]
              },
              "impossible_mass": {
                "score": 0.007154138973952904,
                "ci95": [
                  0,
                  0.01759239541192198
                ]
              }
            },
            "daily_evidence": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.07347128310001905,
                "ci95": [
                  0.030109715212106845,
                  0.12656806131553847
                ]
              },
              "expected_brier": {
                "score": 0.46186078545637854,
                "ci95": [
                  0.39111647643915115,
                  0.5225114271309578
                ]
              },
              "total_variation": {
                "score": 0.16442907794690234,
                "ci95": [
                  0.10123745493927329,
                  0.23457362658670744
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "direct": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.010434942296758383,
                "ci95": [
                  0.004801546695379982,
                  0.015944189505726784
                ]
              },
              "expected_brier": {
                "score": 0.34607635896342503,
                "ci95": [
                  0.0929058831336404,
                  0.5898964985247305
                ]
              },
              "total_variation": {
                "score": 0.07303892230397444,
                "ci95": [
                  0.03575980408548801,
                  0.1097700770888206
                ]
              },
              "impossible_mass": {
                "score": 0.002394450367405484,
                "ci95": [
                  0.00008071958192968254,
                  0.0057154844867138435
                ]
              }
            },
            "disclosure": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.02994868785824785,
                "ci95": [
                  0.010183559371568982,
                  0.05642782649751443
                ]
              },
              "expected_brier": {
                "score": 0.44880671254960586,
                "ci95": [
                  0.38483095947305535,
                  0.5071770036176337
                ]
              },
              "total_variation": {
                "score": 0.10366775418883417,
                "ci95": [
                  0.06414907984230452,
                  0.15269001197016502
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "history": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.02901695155126043,
                "ci95": [
                  0.011535664317418303,
                  0.049325224385860955
                ]
              },
              "expected_brier": {
                "score": 0.5385585729746791,
                "ci95": [
                  0.43234039054300555,
                  0.644964153323767
                ]
              },
              "total_variation": {
                "score": 0.11840893456113294,
                "ci95": [
                  0.06654675869598568,
                  0.17590951828070678
                ]
              },
              "impossible_mass": {
                "score": 0.03048171494435619,
                "ci95": [
                  0,
                  0.08054681861264737
                ]
              }
            },
            "sequential": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.02721758171423091,
                "ci95": [
                  0.008503379507991536,
                  0.05025584915643458
                ]
              },
              "expected_brier": {
                "score": 0.5324564505241793,
                "ci95": [
                  0.44992672099025544,
                  0.6045753572200561
                ]
              },
              "total_variation": {
                "score": 0.10799131641811063,
                "ci95": [
                  0.05790111608634881,
                  0.16178130249805456
                ]
              },
              "impossible_mass": {
                "score": 0.002914958893709901,
                "ci95": [
                  0,
                  0.00683412971083611
                ]
              }
            }
          }
        }
      },
      "temperature_sensitivity": {
        "temperature": 1.2294,
        "method": "recover softmax(logits) from released softmax(logits/T); no fitting; argmax invariant",
        "temperature_one": {
          "n": 96,
          "valid": 96,
          "errors": 0,
          "paired_settings": 48,
          "excess_brier": {
            "score": 0.037500950804638195,
            "ci95": [
              0.025280441317950927,
              0.051466920529329024
            ]
          },
          "expected_brier": {
            "score": 0.4826487351920206,
            "ci95": [
              0.41459052052247464,
              0.5441472717990703
            ]
          },
          "total_variation": {
            "score": 0.1215647532911267,
            "ci95": [
              0.09594394485283278,
              0.1475712970120501
            ]
          },
          "impossible_mass": {
            "score": 0.005104191490904411,
            "ci95": [
              0.0006147490726833205,
              0.012950246779009383
            ]
          }
        }
      }
    },
    "startlux_27b": {
      "receipt": {
        "raw_sha256": "241d83286cd5b978d263b6e2c309c85357ed9eeca4e76a1f0488df56016d994a",
        "metadata_sha256": "2975c8786df0b6bafd50c4649f394aec498dc88afd33ea2b6d29e2e7f0b60440",
        "count": 1152,
        "errors": 0
      },
      "metrics": {
        "cladder": {
          "original": {
            "n": 144,
            "correct": 108,
            "score": 0.75,
            "ci95": [
              0.673755171394799,
              0.8181970713849909
            ],
            "errors": 0
          },
          "reversed": {
            "n": 144,
            "correct": 106,
            "score": 0.7361111111111112,
            "ci95": [
              0.6619191338507234,
              0.8055555555555556
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 144,
            "correct": 105,
            "score": 0.7291666666666666,
            "ci95": [
              0.6524822695035462,
              0.7986458333333332
            ]
          },
          "prediction_flips": 4,
          "strata": {
            "1": {
              "n": 48,
              "correct": 37,
              "score": 0.7708333333333334,
              "ci95": [
                0.6382978723404256,
                0.8913043478260869
              ],
              "errors": 0
            },
            "2": {
              "n": 48,
              "correct": 36,
              "score": 0.75,
              "ci95": [
                0.625,
                0.8541666666666666
              ],
              "errors": 0
            },
            "3": {
              "n": 48,
              "correct": 35,
              "score": 0.7291666666666666,
              "ci95": [
                0.6041666666666666,
                0.8541666666666666
              ],
              "errors": 0
            }
          }
        },
        "cruxeval": {
          "original": {
            "n": 128,
            "correct": 102,
            "score": 0.796875,
            "ci95": [
              0.7265625,
              0.859375
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 97,
            "score": 0.7578125,
            "ci95": [
              0.6796875,
              0.828125
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 95,
            "score": 0.7421875,
            "ci95": [
              0.6640625,
              0.8125
            ]
          },
          "prediction_flips": 17,
          "strata": {
            "all": {
              "n": 128,
              "correct": 102,
              "score": 0.796875,
              "ci95": [
                0.7265625,
                0.859375
              ],
              "errors": 0
            }
          },
          "uniform_choice_chance": 0.3639136904761905,
          "candidate_counts": {
            "2": 56,
            "3": 28,
            "4": 23,
            "5": 8,
            "6": 5,
            "7": 5,
            "8": 2,
            "10": 1
          }
        },
        "finentity": {
          "original": {
            "n": 128,
            "correct": 122,
            "score": 0.953125,
            "ci95": [
              0.9140625,
              0.984375
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 121,
            "score": 0.9453125,
            "ci95": [
              0.90625,
              0.984375
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 120,
            "score": 0.9375,
            "ci95": [
              0.890625,
              0.9765625
            ]
          },
          "prediction_flips": 3,
          "strata": {
            "Negative": {
              "n": 43,
              "correct": 43,
              "score": 1,
              "ci95": [
                1,
                1
              ],
              "errors": 0
            },
            "Neutral": {
              "n": 42,
              "correct": 37,
              "score": 0.8809523809523809,
              "ci95": [
                0.7619047619047619,
                0.9761904761904762
              ],
              "errors": 0
            },
            "Positive": {
              "n": 43,
              "correct": 42,
              "score": 0.9767441860465116,
              "ci95": [
                0.9302325581395349,
                1
              ],
              "errors": 0
            }
          }
        },
        "when2call": {
          "original": {
            "n": 128,
            "correct": 111,
            "score": 0.8671875,
            "ci95": [
              0.8046875,
              0.921875
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 110,
            "score": 0.859375,
            "ci95": [
              0.796875,
              0.9140625
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 109,
            "score": 0.8515625,
            "ci95": [
              0.7890625,
              0.90625
            ]
          },
          "prediction_flips": 4,
          "strata": {
            "cannot_answer": {
              "n": 43,
              "correct": 37,
              "score": 0.8604651162790697,
              "ci95": [
                0.7441860465116279,
                0.9534883720930233
              ],
              "errors": 0
            },
            "request_for_info": {
              "n": 42,
              "correct": 37,
              "score": 0.8809523809523809,
              "ci95": [
                0.7857142857142857,
                0.9761904761904762
              ],
              "errors": 0
            },
            "tool_call": {
              "n": 43,
              "correct": 37,
              "score": 0.8604651162790697,
              "ci95": [
                0.7441860465116279,
                0.9534883720930233
              ],
              "errors": 0
            }
          },
          "direct_false_selections": {
            "n": 128,
            "count": 0,
            "rate": 0,
            "gold_positives": 0
          }
        },
        "known_distribution": {
          "n": 96,
          "valid": 96,
          "errors": 0,
          "paired_settings": 48,
          "excess_brier": {
            "score": 0.032992809170976155,
            "ci95": [
              0.02136166095566344,
              0.04591907099207463
            ]
          },
          "expected_brier": {
            "score": 0.47814059355835853,
            "ci95": [
              0.4095092903576941,
              0.5398815590131746
            ]
          },
          "total_variation": {
            "score": 0.10747623913914234,
            "ci95": [
              0.08379368185047548,
              0.13107639370335908
            ]
          },
          "impossible_mass": {
            "score": 0.0040842722676028564,
            "ci95": [
              0.0008984100586868677,
              0.008511597994289717
            ]
          },
          "uniform_all_candidates": {
            "n": 96,
            "valid": 96,
            "errors": 0,
            "paired_settings": 48,
            "excess_brier": {
              "score": 0.1439903922507942,
              "ci95": [
                0.09995881558499044,
                0.19733934748419735
              ]
            },
            "expected_brier": {
              "score": 0.5891381766381767,
              "ci95": [
                0.5522569444444446,
                0.6316912615740741
              ]
            },
            "total_variation": {
              "score": 0.2490848180933528,
              "ci95": [
                0.19992744552797803,
                0.30308455010232166
              ]
            },
            "impossible_mass": {
              "score": 0.07797364672364672,
              "ci95": [
                0.033561030982905984,
                0.13547743055555547
              ]
            }
          },
          "minus_uniform_excess_brier": {
            "n": 96,
            "delta": -0.11099758307981805,
            "ci95": [
              -0.16996054467628052,
              -0.06220985951742811
            ]
          },
          "categories": {
            "composition": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.033981539170156484,
                "ci95": [
                  0.00792061565669813,
                  0.060704313585229645
                ]
              },
              "expected_brier": {
                "score": 0.5471988115466997,
                "ci95": [
                  0.3302380389006236,
                  0.7599807892691055
                ]
              },
              "total_variation": {
                "score": 0.13123692301059964,
                "ci95": [
                  0.05719312725458406,
                  0.20844384942264826
                ]
              },
              "impossible_mass": {
                "score": 0.006170943764388306,
                "ci95": [
                  0,
                  0.015244367867964388
                ]
              }
            },
            "daily_evidence": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.05328541996787322,
                "ci95": [
                  0.013834577259144698,
                  0.10049516869635092
                ]
              },
              "expected_brier": {
                "score": 0.4416749223242328,
                "ci95": [
                  0.35583891618325414,
                  0.5211579900028632
                ]
              },
              "total_variation": {
                "score": 0.12793414864253028,
                "ci95": [
                  0.06775665325469822,
                  0.19730309995420803
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "direct": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.005269789149625236,
                "ci95": [
                  0.0009659325152973908,
                  0.011022379949064515
                ]
              },
              "expected_brier": {
                "score": 0.34091120581629186,
                "ci95": [
                  0.08594348819565358,
                  0.5838712082313042
                ]
              },
              "total_variation": {
                "score": 0.042337614417232425,
                "ci95": [
                  0.018440821995931278,
                  0.06935854845989539
                ]
              },
              "impossible_mass": {
                "score": 0.0038060196041678866,
                "ci95": [
                  0.00004997313333962033,
                  0.009126036068770067
                ]
              }
            },
            "disclosure": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.039666147047808846,
                "ci95": [
                  0.01578506318348625,
                  0.06933661670985501
                ]
              },
              "expected_brier": {
                "score": 0.45852417173916693,
                "ci95": [
                  0.36721112961446895,
                  0.5367390910557158
                ]
              },
              "total_variation": {
                "score": 0.12014279325081462,
                "ci95": [
                  0.0714516218513018,
                  0.1717762820447855
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "history": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.008789043546547963,
                "ci95": [
                  0.0031995920877799077,
                  0.01608166621690986
                ]
              },
              "expected_brier": {
                "score": 0.5183306649699667,
                "ci95": [
                  0.42884564539536757,
                  0.6122949346763763
                ]
              },
              "total_variation": {
                "score": 0.05976174577579063,
                "ci95": [
                  0.032289532427671436,
                  0.09049235900352705
                ]
              },
              "impossible_mass": {
                "score": 0.01351187322623253,
                "ci95": [
                  0,
                  0.035273484903506784
                ]
              }
            },
            "sequential": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.05696491614384515,
                "ci95": [
                  0.029318614924915245,
                  0.09159625364182777
                ]
              },
              "expected_brier": {
                "score": 0.5622037849537935,
                "ci95": [
                  0.4810175796459438,
                  0.644811409268605
                ]
              },
              "total_variation": {
                "score": 0.16344420973788643,
                "ci95": [
                  0.12545890376074165,
                  0.20670715927028074
                ]
              },
              "impossible_mass": {
                "score": 0.001016797010828416,
                "ci95": [
                  0,
                  0.0024440327640983203
                ]
              }
            }
          }
        }
      },
      "temperature_sensitivity": {
        "temperature": 1.2098,
        "method": "recover softmax(logits) from released softmax(logits/T); no fitting; argmax invariant",
        "temperature_one": {
          "n": 96,
          "valid": 96,
          "errors": 0,
          "paired_settings": 48,
          "excess_brier": {
            "score": 0.04417719179789557,
            "ci95": [
              0.028458773307182966,
              0.060866178604777416
            ]
          },
          "expected_brier": {
            "score": 0.489324976185278,
            "ci95": [
              0.41915032131060753,
              0.5527628923896005
            ]
          },
          "total_variation": {
            "score": 0.12297263297746115,
            "ci95": [
              0.09543409151959412,
              0.15006092926269957
            ]
          },
          "impossible_mass": {
            "score": 0.0024841484930913094,
            "ci95": [
              0.00044362556134322506,
              0.005516858444798334
            ]
          }
        }
      }
    },
    "intern_4b": {
      "receipt": {
        "raw_sha256": "6249fc9679e9eb9ca6fd1031d39c938449505f029a1fdbb3c54498d8626b8ab3",
        "metadata_sha256": "4c2f11440ac1564e0a6893fc6fe413354603e2093587a079027e5af6d570e66a",
        "count": 1152,
        "errors": 0
      },
      "metrics": {
        "cladder": {
          "original": {
            "n": 144,
            "correct": 90,
            "score": 0.625,
            "ci95": [
              0.5486111111111112,
              0.7013888888888888
            ],
            "errors": 0
          },
          "reversed": {
            "n": 144,
            "correct": 86,
            "score": 0.5972222222222222,
            "ci95": [
              0.5172413793103449,
              0.676056338028169
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 144,
            "correct": 74,
            "score": 0.5138888888888888,
            "ci95": [
              0.4333156028368794,
              0.5985915492957746
            ]
          },
          "prediction_flips": 28,
          "strata": {
            "1": {
              "n": 48,
              "correct": 23,
              "score": 0.4791666666666667,
              "ci95": [
                0.3404255319148936,
                0.6087843833185443
              ],
              "errors": 0
            },
            "2": {
              "n": 48,
              "correct": 36,
              "score": 0.75,
              "ci95": [
                0.625,
                0.8541666666666666
              ],
              "errors": 0
            },
            "3": {
              "n": 48,
              "correct": 31,
              "score": 0.6458333333333334,
              "ci95": [
                0.5,
                0.7708333333333334
              ],
              "errors": 0
            }
          }
        },
        "cruxeval": {
          "original": {
            "n": 128,
            "correct": 49,
            "score": 0.3828125,
            "ci95": [
              0.296875,
              0.46894531249999893
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 53,
            "score": 0.4140625,
            "ci95": [
              0.328125,
              0.5
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 34,
            "score": 0.265625,
            "ci95": [
              0.1875,
              0.34375
            ]
          },
          "prediction_flips": 60,
          "strata": {
            "all": {
              "n": 128,
              "correct": 49,
              "score": 0.3828125,
              "ci95": [
                0.296875,
                0.46894531249999893
              ],
              "errors": 0
            }
          },
          "uniform_choice_chance": 0.3639136904761905,
          "candidate_counts": {
            "2": 56,
            "3": 28,
            "4": 23,
            "5": 8,
            "6": 5,
            "7": 5,
            "8": 2,
            "10": 1
          }
        },
        "finentity": {
          "original": {
            "n": 128,
            "correct": 107,
            "score": 0.8359375,
            "ci95": [
              0.765625,
              0.8984375
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 110,
            "score": 0.859375,
            "ci95": [
              0.796875,
              0.9140625
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 107,
            "score": 0.8359375,
            "ci95": [
              0.765625,
              0.8984375
            ]
          },
          "prediction_flips": 4,
          "strata": {
            "Negative": {
              "n": 43,
              "correct": 41,
              "score": 0.9534883720930233,
              "ci95": [
                0.8837209302325582,
                1
              ],
              "errors": 0
            },
            "Neutral": {
              "n": 42,
              "correct": 25,
              "score": 0.5952380952380952,
              "ci95": [
                0.4523809523809524,
                0.7380952380952381
              ],
              "errors": 0
            },
            "Positive": {
              "n": 43,
              "correct": 41,
              "score": 0.9534883720930233,
              "ci95": [
                0.8837209302325582,
                1
              ],
              "errors": 0
            }
          }
        },
        "when2call": {
          "original": {
            "n": 128,
            "correct": 84,
            "score": 0.65625,
            "ci95": [
              0.5703125,
              0.734375
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 86,
            "score": 0.671875,
            "ci95": [
              0.59375,
              0.75
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 74,
            "score": 0.578125,
            "ci95": [
              0.4921875,
              0.6640625
            ]
          },
          "prediction_flips": 30,
          "strata": {
            "cannot_answer": {
              "n": 43,
              "correct": 13,
              "score": 0.3023255813953488,
              "ci95": [
                0.16279069767441862,
                0.4418604651162791
              ],
              "errors": 0
            },
            "request_for_info": {
              "n": 42,
              "correct": 30,
              "score": 0.7142857142857143,
              "ci95": [
                0.5714285714285714,
                0.8333333333333334
              ],
              "errors": 0
            },
            "tool_call": {
              "n": 43,
              "correct": 41,
              "score": 0.9534883720930233,
              "ci95": [
                0.8837209302325582,
                1
              ],
              "errors": 0
            }
          },
          "direct_false_selections": {
            "n": 128,
            "count": 11,
            "rate": 0.0859375,
            "gold_positives": 0
          }
        },
        "known_distribution": {
          "n": 96,
          "valid": 96,
          "errors": 0,
          "paired_settings": 48,
          "excess_brier": {
            "score": 0.1040689904644243,
            "ci95": [
              0.0761572720967581,
              0.1406975351891045
            ]
          },
          "expected_brier": {
            "score": 0.5492167748518068,
            "ci95": [
              0.4780668840903752,
              0.6162826334205874
            ]
          },
          "total_variation": {
            "score": 0.21463753040820857,
            "ci95": [
              0.18068664455905872,
              0.250699969921771
            ]
          },
          "impossible_mass": {
            "score": 0.015228805792558012,
            "ci95": [
              0.0059026792398698385,
              0.02659299395368231
            ]
          },
          "uniform_all_candidates": {
            "n": 96,
            "valid": 96,
            "errors": 0,
            "paired_settings": 48,
            "excess_brier": {
              "score": 0.1439903922507942,
              "ci95": [
                0.09995881558499044,
                0.19733934748419735
              ]
            },
            "expected_brier": {
              "score": 0.5891381766381767,
              "ci95": [
                0.5522569444444446,
                0.6316912615740741
              ]
            },
            "total_variation": {
              "score": 0.2490848180933528,
              "ci95": [
                0.19992744552797803,
                0.30308455010232166
              ]
            },
            "impossible_mass": {
              "score": 0.07797364672364672,
              "ci95": [
                0.033561030982905984,
                0.13547743055555547
              ]
            }
          },
          "minus_uniform_excess_brier": {
            "n": 96,
            "delta": -0.03992140178636989,
            "ci95": [
              -0.10108998255694324,
              0.014974672481117451
            ]
          },
          "categories": {
            "composition": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.06447649776479127,
                "ci95": [
                  0.03664329065985821,
                  0.09333260464277245
                ]
              },
              "expected_brier": {
                "score": 0.5776937701413345,
                "ci95": [
                  0.3635792261561169,
                  0.7804036173649934
                ]
              },
              "total_variation": {
                "score": 0.20449853836179524,
                "ci95": [
                  0.1439072890877559,
                  0.26490083041887874
                ]
              },
              "impossible_mass": {
                "score": 0.02338692655444105,
                "ci95": [
                  0,
                  0.05816644574022305
                ]
              }
            },
            "daily_evidence": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.18226213501883018,
                "ci95": [
                  0.07481733677779903,
                  0.34885277969588313
                ]
              },
              "expected_brier": {
                "score": 0.5706516373751898,
                "ci95": [
                  0.4466841347339555,
                  0.7142620617283725
                ]
              },
              "total_variation": {
                "score": 0.2558632420641219,
                "ci95": [
                  0.16642228605784903,
                  0.3682089078052497
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "direct": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.08338837821462987,
                "ci95": [
                  0.04011739904013228,
                  0.12204986321992214
                ]
              },
              "expected_brier": {
                "score": 0.41902979488129655,
                "ci95": [
                  0.1606028701058445,
                  0.6761384211556302
                ]
              },
              "total_variation": {
                "score": 0.20682154793946148,
                "ci95": [
                  0.11949315992013429,
                  0.2826183282702509
                ]
              },
              "impossible_mass": {
                "score": 0.029041162721634636,
                "ci95": [
                  0.0044140390288793805,
                  0.06435983624059943
                ]
              }
            },
            "disclosure": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.09375405135677209,
                "ci95": [
                  0.03312242332917428,
                  0.17319305368152194
                ]
              },
              "expected_brier": {
                "score": 0.5126120760481302,
                "ci95": [
                  0.4894633753046124,
                  0.5359146488310897
                ]
              },
              "total_variation": {
                "score": 0.18164719711752714,
                "ci95": [
                  0.10629099318447915,
                  0.2655024930156791
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "history": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.10292215976142198,
                "ci95": [
                  0.051481759821120875,
                  0.1545551411161299
                ]
              },
              "expected_brier": {
                "score": 0.6124637811848407,
                "ci95": [
                  0.49622794614886934,
                  0.7326179049955486
                ]
              },
              "total_variation": {
                "score": 0.22230764665726283,
                "ci95": [
                  0.14214510734005253,
                  0.30578513283332054
                ]
              },
              "impossible_mass": {
                "score": 0.0305310226072525,
                "ci95": [
                  0,
                  0.06694648030210786
                ]
              }
            },
            "sequential": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.09761072067010051,
                "ci95": [
                  0.049883668236316125,
                  0.15857431435995975
                ]
              },
              "expected_brier": {
                "score": 0.602849589480049,
                "ci95": [
                  0.5135935183713833,
                  0.6967438220562747
                ]
              },
              "total_variation": {
                "score": 0.21668701030908286,
                "ci95": [
                  0.14339455150449157,
                  0.29374374525583186
                ]
              },
              "impossible_mass": {
                "score": 0.008413722872019889,
                "ci95": [
                  0,
                  0.02098054806308345
                ]
              }
            }
          }
        }
      },
      "temperature_sensitivity": {
        "temperature": 1.9924182353655278,
        "method": "recover softmax(logits) from released softmax(logits/T); no fitting; argmax invariant",
        "temperature_one": {
          "n": 96,
          "valid": 96,
          "errors": 0,
          "paired_settings": 48,
          "excess_brier": {
            "score": 0.1801358647530662,
            "ci95": [
              0.13107124216537835,
              0.23869066469313144
            ]
          },
          "expected_brier": {
            "score": 0.6252836491404485,
            "ci95": [
              0.5289165984673778,
              0.7178306906700896
            ]
          },
          "total_variation": {
            "score": 0.26792977965768244,
            "ci95": [
              0.2166236163637504,
              0.3190038384958903
            ]
          },
          "impossible_mass": {
            "score": 0.003650149063444262,
            "ci95": [
              0.0008509875387356302,
              0.007129499711973073
            ]
          }
        }
      }
    },
    "kev_08b": {
      "receipt": {
        "raw_sha256": "eea9ec18a5f10bd5d7ed8fd2260da3ba66d1e5dd5ae144747b0460d4944e5db2",
        "metadata_sha256": "2e76b0d16cf2d8b929f2a363d67dc6237eae512824ac72a68fa02ccf70706c97",
        "count": 1152,
        "errors": 0
      },
      "metrics": {
        "cladder": {
          "original": {
            "n": 144,
            "correct": 74,
            "score": 0.5138888888888888,
            "ci95": [
              0.4315068493150685,
              0.5903095657276993
            ],
            "errors": 0
          },
          "reversed": {
            "n": 144,
            "correct": 70,
            "score": 0.4861111111111111,
            "ci95": [
              0.4041095890410959,
              0.5655172413793104
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 144,
            "correct": 67,
            "score": 0.4652777777777778,
            "ci95": [
              0.38461538461538464,
              0.5422535211267606
            ]
          },
          "prediction_flips": 10,
          "strata": {
            "1": {
              "n": 48,
              "correct": 24,
              "score": 0.5,
              "ci95": [
                0.3541666666666667,
                0.6382978723404256
              ],
              "errors": 0
            },
            "2": {
              "n": 48,
              "correct": 22,
              "score": 0.4583333333333333,
              "ci95": [
                0.3333333333333333,
                0.5838541666666639
              ],
              "errors": 0
            },
            "3": {
              "n": 48,
              "correct": 28,
              "score": 0.5833333333333334,
              "ci95": [
                0.4375,
                0.7291666666666666
              ],
              "errors": 0
            }
          }
        },
        "cruxeval": {
          "original": {
            "n": 128,
            "correct": 39,
            "score": 0.3046875,
            "ci95": [
              0.2265625,
              0.3828125
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 46,
            "score": 0.359375,
            "ci95": [
              0.2734375,
              0.4453125
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 28,
            "score": 0.21875,
            "ci95": [
              0.1484375,
              0.296875
            ]
          },
          "prediction_flips": 56,
          "strata": {
            "all": {
              "n": 128,
              "correct": 39,
              "score": 0.3046875,
              "ci95": [
                0.2265625,
                0.3828125
              ],
              "errors": 0
            }
          },
          "uniform_choice_chance": 0.3639136904761905,
          "candidate_counts": {
            "2": 56,
            "3": 28,
            "4": 23,
            "5": 8,
            "6": 5,
            "7": 5,
            "8": 2,
            "10": 1
          }
        },
        "finentity": {
          "original": {
            "n": 128,
            "correct": 92,
            "score": 0.71875,
            "ci95": [
              0.640625,
              0.796875
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 96,
            "score": 0.75,
            "ci95": [
              0.6794921875,
              0.8203125
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 90,
            "score": 0.703125,
            "ci95": [
              0.625,
              0.78125
            ]
          },
          "prediction_flips": 8,
          "strata": {
            "Negative": {
              "n": 43,
              "correct": 34,
              "score": 0.7906976744186046,
              "ci95": [
                0.6511627906976745,
                0.9069767441860465
              ],
              "errors": 0
            },
            "Neutral": {
              "n": 42,
              "correct": 25,
              "score": 0.5952380952380952,
              "ci95": [
                0.4523809523809524,
                0.7380952380952381
              ],
              "errors": 0
            },
            "Positive": {
              "n": 43,
              "correct": 33,
              "score": 0.7674418604651163,
              "ci95": [
                0.627906976744186,
                0.8837209302325582
              ],
              "errors": 0
            }
          }
        },
        "when2call": {
          "original": {
            "n": 128,
            "correct": 45,
            "score": 0.3515625,
            "ci95": [
              0.2734375,
              0.4296875
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 54,
            "score": 0.421875,
            "ci95": [
              0.3359375,
              0.5078125
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 34,
            "score": 0.265625,
            "ci95": [
              0.1875,
              0.34375
            ]
          },
          "prediction_flips": 49,
          "strata": {
            "cannot_answer": {
              "n": 43,
              "correct": 18,
              "score": 0.4186046511627907,
              "ci95": [
                0.27906976744186046,
                0.5581395348837209
              ],
              "errors": 0
            },
            "request_for_info": {
              "n": 42,
              "correct": 6,
              "score": 0.14285714285714285,
              "ci95": [
                0.047619047619047616,
                0.2619047619047619
              ],
              "errors": 0
            },
            "tool_call": {
              "n": 43,
              "correct": 21,
              "score": 0.4883720930232558,
              "ci95": [
                0.3488372093023256,
                0.627906976744186
              ],
              "errors": 0
            }
          },
          "direct_false_selections": {
            "n": 128,
            "count": 10,
            "rate": 0.078125,
            "gold_positives": 0
          }
        },
        "known_distribution": {
          "n": 96,
          "valid": 96,
          "errors": 0,
          "paired_settings": 48,
          "excess_brier": {
            "score": 0.14077419819659096,
            "ci95": [
              0.08973138273340817,
              0.21176605795443346
            ]
          },
          "expected_brier": {
            "score": 0.5859219825839733,
            "ci95": [
              0.5218196754583075,
              0.6485743964110388
            ]
          },
          "total_variation": {
            "score": 0.24198171784575337,
            "ci95": [
              0.202191790500321,
              0.28651734701468634
            ]
          },
          "impossible_mass": {
            "score": 0.03554053763512119,
            "ci95": [
              0.015610454968113103,
              0.059790766399936114
            ]
          },
          "uniform_all_candidates": {
            "n": 96,
            "valid": 96,
            "errors": 0,
            "paired_settings": 48,
            "excess_brier": {
              "score": 0.1439903922507942,
              "ci95": [
                0.09995881558499044,
                0.19733934748419735
              ]
            },
            "expected_brier": {
              "score": 0.5891381766381767,
              "ci95": [
                0.5522569444444446,
                0.6316912615740741
              ]
            },
            "total_variation": {
              "score": 0.2490848180933528,
              "ci95": [
                0.19992744552797803,
                0.30308455010232166
              ]
            },
            "impossible_mass": {
              "score": 0.07797364672364672,
              "ci95": [
                0.033561030982905984,
                0.13547743055555547
              ]
            }
          },
          "minus_uniform_excess_brier": {
            "n": 96,
            "delta": -0.0032161940542032585,
            "ci95": [
              -0.06364657855349792,
              0.055241551519561005
            ]
          },
          "categories": {
            "composition": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.09324738726393841,
                "ci95": [
                  0.0376279841279678,
                  0.16181526595567403
                ]
              },
              "expected_brier": {
                "score": 0.6064646596404817,
                "ci95": [
                  0.4483600040969148,
                  0.7646699639651939
                ]
              },
              "total_variation": {
                "score": 0.22727119366300833,
                "ci95": [
                  0.13363124921271358,
                  0.3082213395928658
                ]
              },
              "impossible_mass": {
                "score": 0.04241736445177289,
                "ci95": [
                  0,
                  0.10377292426762016
                ]
              }
            },
            "daily_evidence": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.10857077153525183,
                "ci95": [
                  0.05421959866603207,
                  0.18641078572789588
                ]
              },
              "expected_brier": {
                "score": 0.4969602738916114,
                "ci95": [
                  0.44876261527581607,
                  0.5363261446110055
                ]
              },
              "total_variation": {
                "score": 0.20900814358468894,
                "ci95": [
                  0.1475744422541308,
                  0.2806221444066459
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "direct": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.3297093402901171,
                "ci95": [
                  0.05646888430321107,
                  0.6885971125489538
                ]
              },
              "expected_brier": {
                "score": 0.6653507569567838,
                "ci95": [
                  0.37818122346127697,
                  0.9153761486257431
                ]
              },
              "total_variation": {
                "score": 0.35030858815058663,
                "ci95": [
                  0.19574244538142396,
                  0.5258399655820535
                ]
              },
              "impossible_mass": {
                "score": 0.0618276883799558,
                "ci95": [
                  0.015887447746425046,
                  0.10576493686749835
                ]
              }
            },
            "disclosure": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.10263253330455732,
                "ci95": [
                  0.03994569043243772,
                  0.17042959712312664
                ]
              },
              "expected_brier": {
                "score": 0.5214905579959155,
                "ci95": [
                  0.4081315208432562,
                  0.6313291532309145
                ]
              },
              "total_variation": {
                "score": 0.19118737992279466,
                "ci95": [
                  0.11314370192468078,
                  0.2692834081157267
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "history": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.08752750835166544,
                "ci95": [
                  0.045800677593207544,
                  0.1302500855560526
                ]
              },
              "expected_brier": {
                "score": 0.5970691297750842,
                "ci95": [
                  0.5038770959887473,
                  0.6983497332745465
                ]
              },
              "total_variation": {
                "score": 0.22814655740696252,
                "ci95": [
                  0.1402328096101641,
                  0.3120549028818024
                ]
              },
              "impossible_mass": {
                "score": 0.06729269201217766,
                "ci95": [
                  0,
                  0.16085359507921265
                ]
              }
            },
            "sequential": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.1229576484340155,
                "ci95": [
                  0.06963806159689627,
                  0.175785082326993
                ]
              },
              "expected_brier": {
                "score": 0.628196517243964,
                "ci95": [
                  0.5393813345037834,
                  0.7027362188609345
                ]
              },
              "total_variation": {
                "score": 0.24596844434647916,
                "ci95": [
                  0.16365297830668057,
                  0.3138661161664908
                ]
              },
              "impossible_mass": {
                "score": 0.041705480966820786,
                "ci95": [
                  0,
                  0.10232351011124444
                ]
              }
            }
          }
        }
      },
      "temperature_sensitivity": {
        "temperature": 2.3510958125672174,
        "method": "recover softmax(logits) from released softmax(logits/T); no fitting; argmax invariant",
        "temperature_one": {
          "n": 96,
          "valid": 96,
          "errors": 0,
          "paired_settings": 48,
          "excess_brier": {
            "score": 0.23391116020671077,
            "ci95": [
              0.15130574001561814,
              0.3434263858900774
            ]
          },
          "expected_brier": {
            "score": 0.6790589445940931,
            "ci95": [
              0.5814921428658002,
              0.7810625240770093
            ]
          },
          "total_variation": {
            "score": 0.30257474959253755,
            "ci95": [
              0.24767833039256867,
              0.3621824427689064
            ]
          },
          "impossible_mass": {
            "score": 0.015442074156551472,
            "ci95": [
              0.005217190645573698,
              0.02752806505448621
            ]
          }
        }
      }
    },
    "kev_4b": {
      "receipt": {
        "raw_sha256": "565543e5e5ea50a9f14b32311907a574ad3a6a9b7af63e9100ef95ca8955d67a",
        "metadata_sha256": "102af1866ac107a8204d77ba7d23440637bdaae994687d5d877e98702e99704a",
        "count": 1152,
        "errors": 0
      },
      "metrics": {
        "cladder": {
          "original": {
            "n": 144,
            "correct": 87,
            "score": 0.6041666666666666,
            "ci95": [
              0.5244755244755245,
              0.6827671199611461
            ],
            "errors": 0
          },
          "reversed": {
            "n": 144,
            "correct": 89,
            "score": 0.6180555555555556,
            "ci95": [
              0.5389934533551555,
              0.6986469010441613
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 144,
            "correct": 79,
            "score": 0.5486111111111112,
            "ci95": [
              0.46808510638297873,
              0.6301369863013698
            ]
          },
          "prediction_flips": 18,
          "strata": {
            "1": {
              "n": 48,
              "correct": 24,
              "score": 0.5,
              "ci95": [
                0.3617021276595745,
                0.6401458333333325
              ],
              "errors": 0
            },
            "2": {
              "n": 48,
              "correct": 33,
              "score": 0.6875,
              "ci95": [
                0.5625,
                0.8125
              ],
              "errors": 0
            },
            "3": {
              "n": 48,
              "correct": 30,
              "score": 0.625,
              "ci95": [
                0.4791666666666667,
                0.7708333333333334
              ],
              "errors": 0
            }
          }
        },
        "cruxeval": {
          "original": {
            "n": 128,
            "correct": 55,
            "score": 0.4296875,
            "ci95": [
              0.34375,
              0.515625
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 65,
            "score": 0.5078125,
            "ci95": [
              0.4140625,
              0.59375
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 48,
            "score": 0.375,
            "ci95": [
              0.2890625,
              0.4609375
            ]
          },
          "prediction_flips": 36,
          "strata": {
            "all": {
              "n": 128,
              "correct": 55,
              "score": 0.4296875,
              "ci95": [
                0.34375,
                0.515625
              ],
              "errors": 0
            }
          },
          "uniform_choice_chance": 0.3639136904761905,
          "candidate_counts": {
            "2": 56,
            "3": 28,
            "4": 23,
            "5": 8,
            "6": 5,
            "7": 5,
            "8": 2,
            "10": 1
          }
        },
        "finentity": {
          "original": {
            "n": 128,
            "correct": 114,
            "score": 0.890625,
            "ci95": [
              0.828125,
              0.9376953124999989
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 116,
            "score": 0.90625,
            "ci95": [
              0.8515625,
              0.953125
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 112,
            "score": 0.875,
            "ci95": [
              0.8125,
              0.9296875
            ]
          },
          "prediction_flips": 6,
          "strata": {
            "Negative": {
              "n": 43,
              "correct": 36,
              "score": 0.8372093023255814,
              "ci95": [
                0.7209302325581395,
                0.9302325581395349
              ],
              "errors": 0
            },
            "Neutral": {
              "n": 42,
              "correct": 41,
              "score": 0.9761904761904762,
              "ci95": [
                0.9285714285714286,
                1
              ],
              "errors": 0
            },
            "Positive": {
              "n": 43,
              "correct": 37,
              "score": 0.8604651162790697,
              "ci95": [
                0.7674418604651163,
                0.9534883720930233
              ],
              "errors": 0
            }
          }
        },
        "when2call": {
          "original": {
            "n": 128,
            "correct": 88,
            "score": 0.6875,
            "ci95": [
              0.6015625,
              0.765625
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 86,
            "score": 0.671875,
            "ci95": [
              0.5859375,
              0.75
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 78,
            "score": 0.609375,
            "ci95": [
              0.5234375,
              0.6875
            ]
          },
          "prediction_flips": 22,
          "strata": {
            "cannot_answer": {
              "n": 43,
              "correct": 17,
              "score": 0.3953488372093023,
              "ci95": [
                0.2558139534883721,
                0.5348837209302325
              ],
              "errors": 0
            },
            "request_for_info": {
              "n": 42,
              "correct": 36,
              "score": 0.8571428571428571,
              "ci95": [
                0.7380952380952381,
                0.9523809523809523
              ],
              "errors": 0
            },
            "tool_call": {
              "n": 43,
              "correct": 35,
              "score": 0.813953488372093,
              "ci95": [
                0.6976744186046512,
                0.9302325581395349
              ],
              "errors": 0
            }
          },
          "direct_false_selections": {
            "n": 128,
            "count": 2,
            "rate": 0.015625,
            "gold_positives": 0
          }
        },
        "known_distribution": {
          "n": 96,
          "valid": 96,
          "errors": 0,
          "paired_settings": 48,
          "excess_brier": {
            "score": 0.07780872773408205,
            "ci95": [
              0.05251126307902147,
              0.10746539378780055
            ]
          },
          "expected_brier": {
            "score": 0.5229565121214645,
            "ci95": [
              0.45560248901949446,
              0.582913498742746
            ]
          },
          "total_variation": {
            "score": 0.17032148512473633,
            "ci95": [
              0.13971338072001543,
              0.2022707453379183
            ]
          },
          "impossible_mass": {
            "score": 0.008913636329620656,
            "ci95": [
              0.002502658101886447,
              0.01747032123571397
            ]
          },
          "uniform_all_candidates": {
            "n": 96,
            "valid": 96,
            "errors": 0,
            "paired_settings": 48,
            "excess_brier": {
              "score": 0.1439903922507942,
              "ci95": [
                0.09995881558499044,
                0.19733934748419735
              ]
            },
            "expected_brier": {
              "score": 0.5891381766381767,
              "ci95": [
                0.5522569444444446,
                0.6316912615740741
              ]
            },
            "total_variation": {
              "score": 0.2490848180933528,
              "ci95": [
                0.19992744552797803,
                0.30308455010232166
              ]
            },
            "impossible_mass": {
              "score": 0.07797364672364672,
              "ci95": [
                0.033561030982905984,
                0.13547743055555547
              ]
            }
          },
          "minus_uniform_excess_brier": {
            "n": 96,
            "delta": -0.06618166451671215,
            "ci95": [
              -0.12433841350018264,
              -0.01518585157027395
            ]
          },
          "categories": {
            "composition": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.09653595268654325,
                "ci95": [
                  0.030504461510907816,
                  0.1785003582528666
                ]
              },
              "expected_brier": {
                "score": 0.6097532250630865,
                "ci95": [
                  0.3958366665151679,
                  0.7970302785803819
                ]
              },
              "total_variation": {
                "score": 0.21653772499714036,
                "ci95": [
                  0.14463945780011542,
                  0.2877494140955211
                ]
              },
              "impossible_mass": {
                "score": 0.008422263450237612,
                "ci95": [
                  0,
                  0.020388453078785337
                ]
              }
            },
            "daily_evidence": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.1592222877129712,
                "ci95": [
                  0.06591691007752179,
                  0.28765712490857126
                ]
              },
              "expected_brier": {
                "score": 0.5476117900693308,
                "ci95": [
                  0.4542955743996516,
                  0.6511714895605932
                ]
              },
              "total_variation": {
                "score": 0.2425145959103734,
                "ci95": [
                  0.1469749213109317,
                  0.3442926402566032
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "direct": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.05790043987785709,
                "ci95": [
                  0.013705307631813907,
                  0.11040486520711834
                ]
              },
              "expected_brier": {
                "score": 0.3935418565445238,
                "ci95": [
                  0.16564200274012186,
                  0.615774083487091
                ]
              },
              "total_variation": {
                "score": 0.13505300734108205,
                "ci95": [
                  0.0677964737178456,
                  0.2030467592302776
                ]
              },
              "impossible_mass": {
                "score": 0.006436312932194885,
                "ci95": [
                  0.00042874776177333944,
                  0.0149412521985211
                ]
              }
            },
            "disclosure": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.04621654366585724,
                "ci95": [
                  0.01808138661784271,
                  0.0917649849805984
                ]
              },
              "expected_brier": {
                "score": 0.4650745683572153,
                "ci95": [
                  0.425395448137555,
                  0.5019873067279241
                ]
              },
              "total_variation": {
                "score": 0.1214966759247341,
                "ci95": [
                  0.07637299359612051,
                  0.1832770141117622
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "history": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.026940294152959336,
                "ci95": [
                  0.008498325921401342,
                  0.04516910645458694
                ]
              },
              "expected_brier": {
                "score": 0.536481915576378,
                "ci95": [
                  0.43132168096950846,
                  0.6426280221106957
                ]
              },
              "total_variation": {
                "score": 0.11023011301682992,
                "ci95": [
                  0.05438151896462892,
                  0.1672315311283712
                ]
              },
              "impossible_mass": {
                "score": 0.03113988603412406,
                "ci95": [
                  0,
                  0.06355717450695832
                ]
              }
            },
            "sequential": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.08003684830830422,
                "ci95": [
                  0.04947072737551147,
                  0.11388446953219762
                ]
              },
              "expected_brier": {
                "score": 0.5852757171182527,
                "ci95": [
                  0.5011749934561525,
                  0.6628784978806757
                ]
              },
              "total_variation": {
                "score": 0.19609679355825826,
                "ci95": [
                  0.1455699994313933,
                  0.2517764296305354
                ]
              },
              "impossible_mass": {
                "score": 0.007483355561167375,
                "ci95": [
                  0,
                  0.018683787742745827
                ]
              }
            }
          }
        }
      },
      "temperature_sensitivity": {
        "temperature": 2.406050072164233,
        "method": "recover softmax(logits) from released softmax(logits/T); no fitting; argmax invariant",
        "temperature_one": {
          "n": 96,
          "valid": 96,
          "errors": 0,
          "paired_settings": 48,
          "excess_brier": {
            "score": 0.14736119579214293,
            "ci95": [
              0.10129120180868174,
              0.20078167326352672
            ]
          },
          "expected_brier": {
            "score": 0.5925089801795255,
            "ci95": [
              0.5053538864294956,
              0.6741335199277348
            ]
          },
          "total_variation": {
            "score": 0.23407601811770076,
            "ci95": [
              0.19213361559988,
              0.2773023988498695
            ]
          },
          "impossible_mass": {
            "score": 0.0008390603674574004,
            "ci95": [
              0.0001734866786826267,
              0.0017438479295331783
            ]
          }
        }
      }
    },
    "kev_9b": {
      "receipt": {
        "raw_sha256": "32152f4a1bdd2525d36793aa1f8bca84f841cac9e61fb86ce0f0b4346a32689e",
        "metadata_sha256": "81a1817038abb0b7928a44081491ef38c30434817b50742acf9b301536d2234c",
        "count": 1152,
        "errors": 0
      },
      "metrics": {
        "cladder": {
          "original": {
            "n": 144,
            "correct": 84,
            "score": 0.5833333333333334,
            "ci95": [
              0.5033163265306122,
              0.6619718309859155
            ],
            "errors": 0
          },
          "reversed": {
            "n": 144,
            "correct": 85,
            "score": 0.5902777777777778,
            "ci95": [
              0.5102622632345799,
              0.6689655172413793
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 144,
            "correct": 80,
            "score": 0.5555555555555556,
            "ci95": [
              0.472212441314554,
              0.6351658476658475
            ]
          },
          "prediction_flips": 9,
          "strata": {
            "1": {
              "n": 48,
              "correct": 23,
              "score": 0.4791666666666667,
              "ci95": [
                0.3333333333333333,
                0.625
              ],
              "errors": 0
            },
            "2": {
              "n": 48,
              "correct": 32,
              "score": 0.6666666666666666,
              "ci95": [
                0.5416666666666666,
                0.7916666666666666
              ],
              "errors": 0
            },
            "3": {
              "n": 48,
              "correct": 29,
              "score": 0.6041666666666666,
              "ci95": [
                0.4786458333333334,
                0.75
              ],
              "errors": 0
            }
          }
        },
        "cruxeval": {
          "original": {
            "n": 128,
            "correct": 59,
            "score": 0.4609375,
            "ci95": [
              0.375,
              0.5390625
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 62,
            "score": 0.484375,
            "ci95": [
              0.3984375,
              0.5625
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 48,
            "score": 0.375,
            "ci95": [
              0.2890625,
              0.453125
            ]
          },
          "prediction_flips": 40,
          "strata": {
            "all": {
              "n": 128,
              "correct": 59,
              "score": 0.4609375,
              "ci95": [
                0.375,
                0.5390625
              ],
              "errors": 0
            }
          },
          "uniform_choice_chance": 0.3639136904761905,
          "candidate_counts": {
            "2": 56,
            "3": 28,
            "4": 23,
            "5": 8,
            "6": 5,
            "7": 5,
            "8": 2,
            "10": 1
          }
        },
        "finentity": {
          "original": {
            "n": 128,
            "correct": 119,
            "score": 0.9296875,
            "ci95": [
              0.8828125,
              0.96875
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 118,
            "score": 0.921875,
            "ci95": [
              0.875,
              0.96875
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 117,
            "score": 0.9140625,
            "ci95": [
              0.8671875,
              0.9609375
            ]
          },
          "prediction_flips": 4,
          "strata": {
            "Negative": {
              "n": 43,
              "correct": 42,
              "score": 0.9767441860465116,
              "ci95": [
                0.9302325581395349,
                1
              ],
              "errors": 0
            },
            "Neutral": {
              "n": 42,
              "correct": 38,
              "score": 0.9047619047619048,
              "ci95": [
                0.8095238095238095,
                0.9761904761904762
              ],
              "errors": 0
            },
            "Positive": {
              "n": 43,
              "correct": 39,
              "score": 0.9069767441860465,
              "ci95": [
                0.813953488372093,
                0.9767441860465116
              ],
              "errors": 0
            }
          }
        },
        "when2call": {
          "original": {
            "n": 128,
            "correct": 79,
            "score": 0.6171875,
            "ci95": [
              0.53125,
              0.6953125
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 79,
            "score": 0.6171875,
            "ci95": [
              0.53125,
              0.6953125
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 70,
            "score": 0.546875,
            "ci95": [
              0.4609375,
              0.6328125
            ]
          },
          "prediction_flips": 23,
          "strata": {
            "cannot_answer": {
              "n": 43,
              "correct": 6,
              "score": 0.13953488372093023,
              "ci95": [
                0.046511627906976744,
                0.2558139534883721
              ],
              "errors": 0
            },
            "request_for_info": {
              "n": 42,
              "correct": 32,
              "score": 0.7619047619047619,
              "ci95": [
                0.6190476190476191,
                0.8809523809523809
              ],
              "errors": 0
            },
            "tool_call": {
              "n": 43,
              "correct": 41,
              "score": 0.9534883720930233,
              "ci95": [
                0.8837209302325582,
                1
              ],
              "errors": 0
            }
          },
          "direct_false_selections": {
            "n": 128,
            "count": 4,
            "rate": 0.03125,
            "gold_positives": 0
          }
        },
        "known_distribution": {
          "n": 96,
          "valid": 96,
          "errors": 0,
          "paired_settings": 48,
          "excess_brier": {
            "score": 0.08680597026068816,
            "ci95": [
              0.04598918621255591,
              0.13666963675930294
            ]
          },
          "expected_brier": {
            "score": 0.5319537546480706,
            "ci95": [
              0.4588592961621714,
              0.5982801722764216
            ]
          },
          "total_variation": {
            "score": 0.16771324205201343,
            "ci95": [
              0.12971586250577538,
              0.20639613662583928
            ]
          },
          "impossible_mass": {
            "score": 0.01675929465373291,
            "ci95": [
              0.005865912966110989,
              0.031210624274561265
            ]
          },
          "uniform_all_candidates": {
            "n": 96,
            "valid": 96,
            "errors": 0,
            "paired_settings": 48,
            "excess_brier": {
              "score": 0.1439903922507942,
              "ci95": [
                0.09995881558499044,
                0.19733934748419735
              ]
            },
            "expected_brier": {
              "score": 0.5891381766381767,
              "ci95": [
                0.5522569444444446,
                0.6316912615740741
              ]
            },
            "total_variation": {
              "score": 0.2490848180933528,
              "ci95": [
                0.19992744552797803,
                0.30308455010232166
              ]
            },
            "impossible_mass": {
              "score": 0.07797364672364672,
              "ci95": [
                0.033561030982905984,
                0.13547743055555547
              ]
            }
          },
          "minus_uniform_excess_brier": {
            "n": 96,
            "delta": -0.05718442199010604,
            "ci95": [
              -0.11907756428434671,
              0.004796596504653311
            ]
          },
          "categories": {
            "composition": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.028552853068023604,
                "ci95": [
                  0.019005252563042526,
                  0.038024401079304986
                ]
              },
              "expected_brier": {
                "score": 0.5417701254445668,
                "ci95": [
                  0.35154232719605116,
                  0.7309734390694251
                ]
              },
              "total_variation": {
                "score": 0.14058776757893687,
                "ci95": [
                  0.10312476959540957,
                  0.17260147677931956
                ]
              },
              "impossible_mass": {
                "score": 0.01620569077910313,
                "ci95": [
                  0,
                  0.04008363089769724
                ]
              }
            },
            "daily_evidence": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.29251903115460465,
                "ci95": [
                  0.08015564419507842,
                  0.5249128899873547
                ]
              },
              "expected_brier": {
                "score": 0.6809085335109643,
                "ci95": [
                  0.4654414846759497,
                  0.8948745120420977
                ]
              },
              "total_variation": {
                "score": 0.302294374600233,
                "ci95": [
                  0.13629040800748327,
                  0.4676486425324944
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "direct": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.07008326942660668,
                "ci95": [
                  0.014322928130377194,
                  0.1396458995635551
                ]
              },
              "expected_brier": {
                "score": 0.4057246860932734,
                "ci95": [
                  0.17749912659490683,
                  0.6235532046230706
                ]
              },
              "total_variation": {
                "score": 0.1626614786852445,
                "ci95": [
                  0.08498280438966538,
                  0.242386435841215
                ]
              },
              "impossible_mass": {
                "score": 0.021056953847517938,
                "ci95": [
                  0.003582196145833316,
                  0.04325620572523661
                ]
              }
            },
            "disclosure": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.0314526027015681,
                "ci95": [
                  0.0071470820141402705,
                  0.05839972262984262
                ]
              },
              "expected_brier": {
                "score": 0.45031062739292615,
                "ci95": [
                  0.39378563087377144,
                  0.5012147656859972
                ]
              },
              "total_variation": {
                "score": 0.09525067757776016,
                "ci95": [
                  0.047692118402720786,
                  0.14674491364119874
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "history": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.03620129695462357,
                "ci95": [
                  0.021766794345171554,
                  0.05191386461105231
                ]
              },
              "expected_brier": {
                "score": 0.5457429183780423,
                "ci95": [
                  0.45383423585629523,
                  0.6433291394410696
                ]
              },
              "total_variation": {
                "score": 0.14277537783703548,
                "ci95": [
                  0.10163900700331112,
                  0.19007345131388126
                ]
              },
              "impossible_mass": {
                "score": 0.048327041179213576,
                "ci95": [
                  0,
                  0.11487028126826
                ]
              }
            },
            "sequential": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.062026768258702264,
                "ci95": [
                  0.021687365487278294,
                  0.10964021285589313
                ]
              },
              "expected_brier": {
                "score": 0.5672656370686506,
                "ci95": [
                  0.48916727896773604,
                  0.6474907750939017
                ]
              },
              "total_variation": {
                "score": 0.16270977603287068,
                "ci95": [
                  0.09546500197681411,
                  0.2401093784738762
                ]
              },
              "impossible_mass": {
                "score": 0.014966082116562807,
                "ci95": [
                  0,
                  0.037266534299369866
                ]
              }
            }
          }
        }
      },
      "temperature_sensitivity": {
        "temperature": 2.2973967099940698,
        "method": "recover softmax(logits) from released softmax(logits/T); no fitting; argmax invariant",
        "temperature_one": {
          "n": 96,
          "valid": 96,
          "errors": 0,
          "paired_settings": 48,
          "excess_brier": {
            "score": 0.14137390126416025,
            "ci95": [
              0.07941908444893768,
              0.2202079589983764
            ]
          },
          "expected_brier": {
            "score": 0.5865216856515426,
            "ci95": [
              0.49099784387625134,
              0.6793951627618133
            ]
          },
          "total_variation": {
            "score": 0.21277156862919436,
            "ci95": [
              0.16683283857475042,
              0.26336769870193616
            ]
          },
          "impossible_mass": {
            "score": 0.0046381997021900856,
            "ci95": [
              0.0007016484630880327,
              0.011460701614069006
            ]
          }
        }
      }
    },
    "kev_27b": {
      "receipt": {
        "raw_sha256": "4470f397fac0ac7ed5e217ad4d1045aaac009f01684a6877cc7c5fbc4da1b32e",
        "metadata_sha256": "c3f97463f5870d2c9c5b018912e2de6e1c9ae0dc97ac2b83ceed51688ddc0f22",
        "count": 1152,
        "errors": 0
      },
      "metrics": {
        "cladder": {
          "original": {
            "n": 144,
            "correct": 105,
            "score": 0.7291666666666666,
            "ci95": [
              0.6549295774647887,
              0.8014036436492403
            ],
            "errors": 0
          },
          "reversed": {
            "n": 144,
            "correct": 103,
            "score": 0.7152777777777778,
            "ci95": [
              0.6407961658841941,
              0.7876712328767124
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 144,
            "correct": 101,
            "score": 0.7013888888888888,
            "ci95": [
              0.6232649200114954,
              0.7762438574938574
            ]
          },
          "prediction_flips": 6,
          "strata": {
            "1": {
              "n": 48,
              "correct": 38,
              "score": 0.7916666666666666,
              "ci95": [
                0.6666666666666666,
                0.8979591836734694
              ],
              "errors": 0
            },
            "2": {
              "n": 48,
              "correct": 33,
              "score": 0.6875,
              "ci95": [
                0.5625,
                0.8125
              ],
              "errors": 0
            },
            "3": {
              "n": 48,
              "correct": 34,
              "score": 0.7083333333333334,
              "ci95": [
                0.5625,
                0.8333333333333334
              ],
              "errors": 0
            }
          }
        },
        "cruxeval": {
          "original": {
            "n": 128,
            "correct": 92,
            "score": 0.71875,
            "ci95": [
              0.640625,
              0.7890625
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 97,
            "score": 0.7578125,
            "ci95": [
              0.6796875,
              0.828125
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 86,
            "score": 0.671875,
            "ci95": [
              0.5859375,
              0.75
            ]
          },
          "prediction_flips": 26,
          "strata": {
            "all": {
              "n": 128,
              "correct": 92,
              "score": 0.71875,
              "ci95": [
                0.640625,
                0.7890625
              ],
              "errors": 0
            }
          },
          "uniform_choice_chance": 0.3639136904761905,
          "candidate_counts": {
            "2": 56,
            "3": 28,
            "4": 23,
            "5": 8,
            "6": 5,
            "7": 5,
            "8": 2,
            "10": 1
          }
        },
        "finentity": {
          "original": {
            "n": 128,
            "correct": 118,
            "score": 0.921875,
            "ci95": [
              0.875,
              0.96875
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 118,
            "score": 0.921875,
            "ci95": [
              0.8671875,
              0.9609375
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 116,
            "score": 0.90625,
            "ci95": [
              0.8515625,
              0.953125
            ]
          },
          "prediction_flips": 4,
          "strata": {
            "Negative": {
              "n": 43,
              "correct": 42,
              "score": 0.9767441860465116,
              "ci95": [
                0.9302325581395349,
                1
              ],
              "errors": 0
            },
            "Neutral": {
              "n": 42,
              "correct": 36,
              "score": 0.8571428571428571,
              "ci95": [
                0.7380952380952381,
                0.9523809523809523
              ],
              "errors": 0
            },
            "Positive": {
              "n": 43,
              "correct": 40,
              "score": 0.9302325581395349,
              "ci95": [
                0.8604651162790697,
                1
              ],
              "errors": 0
            }
          }
        },
        "when2call": {
          "original": {
            "n": 128,
            "correct": 82,
            "score": 0.640625,
            "ci95": [
              0.5546875,
              0.71875
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 85,
            "score": 0.6640625,
            "ci95": [
              0.578125,
              0.75
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 79,
            "score": 0.6171875,
            "ci95": [
              0.53125,
              0.6955078124999989
            ]
          },
          "prediction_flips": 17,
          "strata": {
            "cannot_answer": {
              "n": 43,
              "correct": 9,
              "score": 0.20930232558139536,
              "ci95": [
                0.09302325581395349,
                0.32558139534883723
              ],
              "errors": 0
            },
            "request_for_info": {
              "n": 42,
              "correct": 34,
              "score": 0.8095238095238095,
              "ci95": [
                0.6904761904761905,
                0.9285714285714286
              ],
              "errors": 0
            },
            "tool_call": {
              "n": 43,
              "correct": 39,
              "score": 0.9069767441860465,
              "ci95": [
                0.813953488372093,
                0.9767441860465116
              ],
              "errors": 0
            }
          },
          "direct_false_selections": {
            "n": 128,
            "count": 11,
            "rate": 0.0859375,
            "gold_positives": 0
          }
        },
        "known_distribution": {
          "n": 96,
          "valid": 96,
          "errors": 0,
          "paired_settings": 48,
          "excess_brier": {
            "score": 0.03599078531561913,
            "ci95": [
              0.022650622976311808,
              0.05170129307225163
            ]
          },
          "expected_brier": {
            "score": 0.48113856970300156,
            "ci95": [
              0.41388467778645077,
              0.5408134035944583
            ]
          },
          "total_variation": {
            "score": 0.11473901850595551,
            "ci95": [
              0.09269356019360221,
              0.13765138907901783
            ]
          },
          "impossible_mass": {
            "score": 0.0035626323460658574,
            "ci95": [
              0.0011750973033817494,
              0.006506447829812356
            ]
          },
          "uniform_all_candidates": {
            "n": 96,
            "valid": 96,
            "errors": 0,
            "paired_settings": 48,
            "excess_brier": {
              "score": 0.1439903922507942,
              "ci95": [
                0.09995881558499044,
                0.19733934748419735
              ]
            },
            "expected_brier": {
              "score": 0.5891381766381767,
              "ci95": [
                0.5522569444444446,
                0.6316912615740741
              ]
            },
            "total_variation": {
              "score": 0.2490848180933528,
              "ci95": [
                0.19992744552797803,
                0.30308455010232166
              ]
            },
            "impossible_mass": {
              "score": 0.07797364672364672,
              "ci95": [
                0.033561030982905984,
                0.13547743055555547
              ]
            }
          },
          "minus_uniform_excess_brier": {
            "n": 96,
            "delta": -0.10799960693517507,
            "ci95": [
              -0.1665488139204093,
              -0.061949375301869024
            ]
          },
          "categories": {
            "composition": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.025602675073122506,
                "ci95": [
                  0.00732880497075285,
                  0.05571333143308455
                ]
              },
              "expected_brier": {
                "score": 0.5388199474496658,
                "ci95": [
                  0.3303212700838153,
                  0.7377860537024741
                ]
              },
              "total_variation": {
                "score": 0.11531782951326677,
                "ci95": [
                  0.05762085455405577,
                  0.1815961076305131
                ]
              },
              "impossible_mass": {
                "score": 0.008812254110991836,
                "ci95": [
                  0,
                  0.02061443108236829
                ]
              }
            },
            "daily_evidence": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.08249579473052916,
                "ci95": [
                  0.020960060779789434,
                  0.15198104678614474
                ]
              },
              "expected_brier": {
                "score": 0.47088529708688875,
                "ci95": [
                  0.3656877794726947,
                  0.5725391228855365
                ]
              },
              "total_variation": {
                "score": 0.16509222985009936,
                "ci95": [
                  0.09069298379711159,
                  0.24736699964961684
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "direct": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.03236055592626248,
                "ci95": [
                  0.010242523608855109,
                  0.05469977871995874
                ]
              },
              "expected_brier": {
                "score": 0.36800197259292916,
                "ci95": [
                  0.12147167396728216,
                  0.602540224786716
                ]
              },
              "total_variation": {
                "score": 0.11104815513169285,
                "ci95": [
                  0.057767089442848175,
                  0.1610678427638665
                ]
              },
              "impossible_mass": {
                "score": 0.005025608432505432,
                "ci95": [
                  0.0004437734571392536,
                  0.010870037671108272
                ]
              }
            },
            "disclosure": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.028386779874592986,
                "ci95": [
                  0.008919600758337193,
                  0.05252585414216802
                ]
              },
              "expected_brier": {
                "score": 0.44724480456595106,
                "ci95": [
                  0.38605079960823085,
                  0.4929668757458757
                ]
              },
              "total_variation": {
                "score": 0.0937742385726279,
                "ci95": [
                  0.054699862280438924,
                  0.1386799141262758
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "history": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.025962322022683736,
                "ci95": [
                  0.013710570544872522,
                  0.03822832735827967
                ]
              },
              "expected_brier": {
                "score": 0.5355039434461024,
                "ci95": [
                  0.43540483331207275,
                  0.6357197126954621
                ]
              },
              "total_variation": {
                "score": 0.10460959334158945,
                "ci95": [
                  0.07186820541280273,
                  0.13610814046155395
                ]
              },
              "impossible_mass": {
                "score": 0.00614389229033782,
                "ci95": [
                  0,
                  0.012440102692457013
                ]
              }
            },
            "sequential": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.02113658426652392,
                "ci95": [
                  0.011657968035661317,
                  0.03360676267162475
                ]
              },
              "expected_brier": {
                "score": 0.5263754530764724,
                "ci95": [
                  0.45090014426686753,
                  0.5998571833642043
                ]
              },
              "total_variation": {
                "score": 0.09859206462645669,
                "ci95": [
                  0.07232952200790874,
                  0.12980587041436628
                ]
              },
              "impossible_mass": {
                "score": 0.0013940392425600572,
                "ci95": [
                  0,
                  0.003363659191711604
                ]
              }
            }
          }
        }
      },
      "temperature_sensitivity": {
        "temperature": 1.319507910772894,
        "method": "recover softmax(logits) from released softmax(logits/T); no fitting; argmax invariant",
        "temperature_one": {
          "n": 96,
          "valid": 96,
          "errors": 0,
          "paired_settings": 48,
          "excess_brier": {
            "score": 0.047545897421694044,
            "ci95": [
              0.029548425495340813,
              0.06805441016290407
            ]
          },
          "expected_brier": {
            "score": 0.4926936818090765,
            "ci95": [
              0.4224605008759523,
              0.5569176710893796
            ]
          },
          "total_variation": {
            "score": 0.12963058157110802,
            "ci95": [
              0.10302935051556904,
              0.15679660404976534
            ]
          },
          "impossible_mass": {
            "score": 0.0015885680873173134,
            "ci95": [
              0.0003998912111868577,
              0.0032518291483126707
            ]
          }
        }
      }
    },
    "qwen35_08b": {
      "receipt": {
        "raw_sha256": "112d865f12a51046222fd6c8b32d6a521eef2bcc8bea3272e7ef0e675b791b06",
        "metadata_sha256": "5b663721d0e6d06378ff051d334303a7b310f9fb33ab65997a190013d2d2dd3a",
        "count": 1152,
        "errors": 0
      },
      "metrics": {
        "cladder": {
          "original": {
            "n": 144,
            "correct": 66,
            "score": 0.4583333333333333,
            "ci95": [
              0.379287045666356,
              0.5384615384615384
            ],
            "errors": 0
          },
          "reversed": {
            "n": 144,
            "correct": 74,
            "score": 0.5138888888888888,
            "ci95": [
              0.4314830669710807,
              0.5985915492957746
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 144,
            "correct": 34,
            "score": 0.2361111111111111,
            "ci95": [
              0.16901170536733917,
              0.3103448275862069
            ]
          },
          "prediction_flips": 72,
          "strata": {
            "1": {
              "n": 48,
              "correct": 22,
              "score": 0.4583333333333333,
              "ci95": [
                0.3191489361702128,
                0.6
              ],
              "errors": 0
            },
            "2": {
              "n": 48,
              "correct": 23,
              "score": 0.4791666666666667,
              "ci95": [
                0.3333333333333333,
                0.6041666666666666
              ],
              "errors": 0
            },
            "3": {
              "n": 48,
              "correct": 21,
              "score": 0.4375,
              "ci95": [
                0.2916666666666667,
                0.5833333333333334
              ],
              "errors": 0
            }
          }
        },
        "cruxeval": {
          "original": {
            "n": 128,
            "correct": 41,
            "score": 0.3203125,
            "ci95": [
              0.2421875,
              0.3984375
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 45,
            "score": 0.3515625,
            "ci95": [
              0.2732421875,
              0.4296875
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 12,
            "score": 0.09375,
            "ci95": [
              0.046875,
              0.1484375
            ]
          },
          "prediction_flips": 93,
          "strata": {
            "all": {
              "n": 128,
              "correct": 41,
              "score": 0.3203125,
              "ci95": [
                0.2421875,
                0.3984375
              ],
              "errors": 0
            }
          },
          "uniform_choice_chance": 0.3639136904761905,
          "candidate_counts": {
            "2": 56,
            "3": 28,
            "4": 23,
            "5": 8,
            "6": 5,
            "7": 5,
            "8": 2,
            "10": 1
          }
        },
        "finentity": {
          "original": {
            "n": 128,
            "correct": 76,
            "score": 0.59375,
            "ci95": [
              0.5078125,
              0.6796875
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 66,
            "score": 0.515625,
            "ci95": [
              0.421875,
              0.6015625
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 55,
            "score": 0.4296875,
            "ci95": [
              0.34375,
              0.515625
            ]
          },
          "prediction_flips": 47,
          "strata": {
            "Negative": {
              "n": 43,
              "correct": 37,
              "score": 0.8604651162790697,
              "ci95": [
                0.7441860465116279,
                0.9534883720930233
              ],
              "errors": 0
            },
            "Neutral": {
              "n": 42,
              "correct": 3,
              "score": 0.07142857142857142,
              "ci95": [
                0,
                0.14285714285714285
              ],
              "errors": 0
            },
            "Positive": {
              "n": 43,
              "correct": 36,
              "score": 0.8372093023255814,
              "ci95": [
                0.7209302325581395,
                0.9302325581395349
              ],
              "errors": 0
            }
          }
        },
        "when2call": {
          "original": {
            "n": 128,
            "correct": 34,
            "score": 0.265625,
            "ci95": [
              0.1875,
              0.34375
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 37,
            "score": 0.2890625,
            "ci95": [
              0.2109375,
              0.375
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 24,
            "score": 0.1875,
            "ci95": [
              0.125,
              0.2578125
            ]
          },
          "prediction_flips": 68,
          "strata": {
            "cannot_answer": {
              "n": 43,
              "correct": 0,
              "score": 0,
              "ci95": [
                0,
                0
              ],
              "errors": 0
            },
            "request_for_info": {
              "n": 42,
              "correct": 3,
              "score": 0.07142857142857142,
              "ci95": [
                0,
                0.16666666666666666
              ],
              "errors": 0
            },
            "tool_call": {
              "n": 43,
              "correct": 31,
              "score": 0.7209302325581395,
              "ci95": [
                0.5813953488372093,
                0.8372093023255814
              ],
              "errors": 0
            }
          },
          "direct_false_selections": {
            "n": 128,
            "count": 40,
            "rate": 0.3125,
            "gold_positives": 0
          }
        },
        "known_distribution": {
          "n": 96,
          "valid": 96,
          "errors": 0,
          "paired_settings": 48,
          "excess_brier": {
            "score": 0.2451569391171097,
            "ci95": [
              0.1867317561851743,
              0.3084842431904455
            ]
          },
          "expected_brier": {
            "score": 0.6903047235044921,
            "ci95": [
              0.616724630094609,
              0.7630425303544841
            ]
          },
          "total_variation": {
            "score": 0.33811711967641317,
            "ci95": [
              0.28783142719464677,
              0.3917127475328489
            ]
          },
          "impossible_mass": {
            "score": 0.0800842440731979,
            "ci95": [
              0.036644724306582284,
              0.1309686868816915
            ]
          },
          "uniform_all_candidates": {
            "n": 96,
            "valid": 96,
            "errors": 0,
            "paired_settings": 48,
            "excess_brier": {
              "score": 0.1439903922507942,
              "ci95": [
                0.09995881558499044,
                0.19733934748419735
              ]
            },
            "expected_brier": {
              "score": 0.5891381766381767,
              "ci95": [
                0.5522569444444446,
                0.6316912615740741
              ]
            },
            "total_variation": {
              "score": 0.2490848180933528,
              "ci95": [
                0.19992744552797803,
                0.30308455010232166
              ]
            },
            "impossible_mass": {
              "score": 0.07797364672364672,
              "ci95": [
                0.033561030982905984,
                0.13547743055555547
              ]
            }
          },
          "minus_uniform_excess_brier": {
            "n": 96,
            "delta": 0.1011665468663155,
            "ci95": [
              0.040993727116045736,
              0.15372568902169942
            ]
          },
          "categories": {
            "composition": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.22647193387979914,
                "ci95": [
                  0.14715711869406967,
                  0.2966134850337411
                ]
              },
              "expected_brier": {
                "score": 0.7396892062563424,
                "ci95": [
                  0.4910034477088421,
                  0.9848531557270096
                ]
              },
              "total_variation": {
                "score": 0.3952595158314505,
                "ci95": [
                  0.2766419056260907,
                  0.503117381106925
                ]
              },
              "impossible_mass": {
                "score": 0.11889789537815096,
                "ci95": [
                  0,
                  0.2935797467797191
                ]
              }
            },
            "daily_evidence": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.20583825146690954,
                "ci95": [
                  0.11597751157091894,
                  0.30537777020824114
                ]
              },
              "expected_brier": {
                "score": 0.5942277538232691,
                "ci95": [
                  0.507274388012999,
                  0.6736420939580744
                ]
              },
              "total_variation": {
                "score": 0.2750949755005192,
                "ci95": [
                  0.20568134562207113,
                  0.3502054958125963
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "direct": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.43850280779990597,
                "ci95": [
                  0.2347815433591044,
                  0.6694354122829864
                ]
              },
              "expected_brier": {
                "score": 0.7741442244665726,
                "ci95": [
                  0.5443262380497288,
                  0.9766823741554652
                ]
              },
              "total_variation": {
                "score": 0.486166748563218,
                "ci95": [
                  0.37862757951130044,
                  0.5905825288541399
                ]
              },
              "impossible_mass": {
                "score": 0.1646537591946543,
                "ci95": [
                  0.03738543891013309,
                  0.27999016265854415
                ]
              }
            },
            "disclosure": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.18208619208146212,
                "ci95": [
                  0.05848625689140135,
                  0.35476297138959006
                ]
              },
              "expected_brier": {
                "score": 0.6009442167728202,
                "ci95": [
                  0.5263238891672161,
                  0.707115418830269
                ]
              },
              "total_variation": {
                "score": 0.2400387660314995,
                "ci95": [
                  0.140105666853967,
                  0.3644564923844228
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "history": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.12153331291438371,
                "ci95": [
                  0.05991861641690213,
                  0.18880632370301617
                ]
              },
              "expected_brier": {
                "score": 0.6310749343378025,
                "ci95": [
                  0.48614256455632077,
                  0.7931950026415511
                ]
              },
              "total_variation": {
                "score": 0.26009665576567115,
                "ci95": [
                  0.1386594631404047,
                  0.4033978619614026
                ]
              },
              "impossible_mass": {
                "score": 0.12355409565462383,
                "ci95": [
                  0,
                  0.2563585216651323
                ]
              }
            },
            "sequential": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.29650913656019784,
                "ci95": [
                  0.16596518457668166,
                  0.4421791533427398
                ]
              },
              "expected_brier": {
                "score": 0.8017480053701462,
                "ci95": [
                  0.6645221457754272,
                  0.9512289279964585
                ]
              },
              "total_variation": {
                "score": 0.3720460563661203,
                "ci95": [
                  0.25887242190076326,
                  0.4954773824813741
                ]
              },
              "impossible_mass": {
                "score": 0.07339971421175827,
                "ci95": [
                  0,
                  0.1805866330555897
                ]
              }
            }
          }
        }
      }
    },
    "qwen35_4b": {
      "receipt": {
        "raw_sha256": "edc63d19477b8ccb4bbbfdcf2ce6d29d42f3c9a39cb1e24ea63e0d7559282469",
        "metadata_sha256": "7172086c15af0242479db516b01fe642900fa06b530d571a38c6310c00b25a5b",
        "count": 1152,
        "errors": 0
      },
      "metrics": {
        "cladder": {
          "original": {
            "n": 144,
            "correct": 82,
            "score": 0.5694444444444444,
            "ci95": [
              0.4896551724137931,
              0.6482758620689655
            ],
            "errors": 0
          },
          "reversed": {
            "n": 144,
            "correct": 82,
            "score": 0.5694444444444444,
            "ci95": [
              0.4930531103286385,
              0.6503496503496503
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 144,
            "correct": 47,
            "score": 0.3263888888888889,
            "ci95": [
              0.2534246575342466,
              0.40559440559440557
            ]
          },
          "prediction_flips": 70,
          "strata": {
            "1": {
              "n": 48,
              "correct": 24,
              "score": 0.5,
              "ci95": [
                0.3617021276595745,
                0.6382978723404256
              ],
              "errors": 0
            },
            "2": {
              "n": 48,
              "correct": 32,
              "score": 0.6666666666666666,
              "ci95": [
                0.5208333333333334,
                0.7916666666666666
              ],
              "errors": 0
            },
            "3": {
              "n": 48,
              "correct": 26,
              "score": 0.5416666666666666,
              "ci95": [
                0.3958333333333333,
                0.6875
              ],
              "errors": 0
            }
          }
        },
        "cruxeval": {
          "original": {
            "n": 128,
            "correct": 48,
            "score": 0.375,
            "ci95": [
              0.296875,
              0.453125
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 56,
            "score": 0.4375,
            "ci95": [
              0.34375,
              0.5234375
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 33,
            "score": 0.2578125,
            "ci95": [
              0.1796875,
              0.328125
            ]
          },
          "prediction_flips": 59,
          "strata": {
            "all": {
              "n": 128,
              "correct": 48,
              "score": 0.375,
              "ci95": [
                0.296875,
                0.453125
              ],
              "errors": 0
            }
          },
          "uniform_choice_chance": 0.3639136904761905,
          "candidate_counts": {
            "2": 56,
            "3": 28,
            "4": 23,
            "5": 8,
            "6": 5,
            "7": 5,
            "8": 2,
            "10": 1
          }
        },
        "finentity": {
          "original": {
            "n": 128,
            "correct": 94,
            "score": 0.734375,
            "ci95": [
              0.65625,
              0.8125
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 90,
            "score": 0.703125,
            "ci95": [
              0.6171875,
              0.7890625
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 89,
            "score": 0.6953125,
            "ci95": [
              0.609375,
              0.78125
            ]
          },
          "prediction_flips": 9,
          "strata": {
            "Negative": {
              "n": 43,
              "correct": 42,
              "score": 0.9767441860465116,
              "ci95": [
                0.9302325581395349,
                1
              ],
              "errors": 0
            },
            "Neutral": {
              "n": 42,
              "correct": 11,
              "score": 0.2619047619047619,
              "ci95": [
                0.14285714285714285,
                0.40476190476190477
              ],
              "errors": 0
            },
            "Positive": {
              "n": 43,
              "correct": 41,
              "score": 0.9534883720930233,
              "ci95": [
                0.8837209302325582,
                1
              ],
              "errors": 0
            }
          }
        },
        "when2call": {
          "original": {
            "n": 128,
            "correct": 60,
            "score": 0.46875,
            "ci95": [
              0.3828125,
              0.5546875
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 58,
            "score": 0.453125,
            "ci95": [
              0.3671875,
              0.5390625
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 45,
            "score": 0.3515625,
            "ci95": [
              0.2734375,
              0.4375
            ]
          },
          "prediction_flips": 42,
          "strata": {
            "cannot_answer": {
              "n": 43,
              "correct": 7,
              "score": 0.16279069767441862,
              "ci95": [
                0.06976744186046512,
                0.27906976744186046
              ],
              "errors": 0
            },
            "request_for_info": {
              "n": 42,
              "correct": 12,
              "score": 0.2857142857142857,
              "ci95": [
                0.16666666666666666,
                0.42857142857142855
              ],
              "errors": 0
            },
            "tool_call": {
              "n": 43,
              "correct": 41,
              "score": 0.9534883720930233,
              "ci95": [
                0.8837209302325582,
                1
              ],
              "errors": 0
            }
          },
          "direct_false_selections": {
            "n": 128,
            "count": 9,
            "rate": 0.0703125,
            "gold_positives": 0
          }
        },
        "known_distribution": {
          "n": 96,
          "valid": 96,
          "errors": 0,
          "paired_settings": 48,
          "excess_brier": {
            "score": 0.13931325354457835,
            "ci95": [
              0.09858833041271937,
              0.1896721479048436
            ]
          },
          "expected_brier": {
            "score": 0.5844610379319608,
            "ci95": [
              0.5027009862300417,
              0.6659298367127117
            ]
          },
          "total_variation": {
            "score": 0.2440126023546232,
            "ci95": [
              0.19634043282045555,
              0.29980097432120334
            ]
          },
          "impossible_mass": {
            "score": 0.03592554085931116,
            "ci95": [
              0.013026286181999575,
              0.06468427922537133
            ]
          },
          "uniform_all_candidates": {
            "n": 96,
            "valid": 96,
            "errors": 0,
            "paired_settings": 48,
            "excess_brier": {
              "score": 0.1439903922507942,
              "ci95": [
                0.09995881558499044,
                0.19733934748419735
              ]
            },
            "expected_brier": {
              "score": 0.5891381766381767,
              "ci95": [
                0.5522569444444446,
                0.6316912615740741
              ]
            },
            "total_variation": {
              "score": 0.2490848180933528,
              "ci95": [
                0.19992744552797803,
                0.30308455010232166
              ]
            },
            "impossible_mass": {
              "score": 0.07797364672364672,
              "ci95": [
                0.033561030982905984,
                0.13547743055555547
              ]
            }
          },
          "minus_uniform_excess_brier": {
            "n": 96,
            "delta": -0.004677138706215846,
            "ci95": [
              -0.07550177449764726,
              0.06175117666781063
            ]
          },
          "categories": {
            "composition": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.15004519188649681,
                "ci95": [
                  0.06316112218530937,
                  0.24231350801879048
                ]
              },
              "expected_brier": {
                "score": 0.66326246426304,
                "ci95": [
                  0.3986084070202375,
                  0.8972712667520227
                ]
              },
              "total_variation": {
                "score": 0.2897744423311933,
                "ci95": [
                  0.14872031530933616,
                  0.42783763697512356
                ]
              },
              "impossible_mass": {
                "score": 0.08645610182014897,
                "ci95": [
                  0,
                  0.19941217932863953
                ]
              }
            },
            "daily_evidence": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.1549525528991169,
                "ci95": [
                  0.05797856392407299,
                  0.2960073063920167
                ]
              },
              "expected_brier": {
                "score": 0.5433420552554764,
                "ci95": [
                  0.44270566638484676,
                  0.6577166329415322
                ]
              },
              "total_variation": {
                "score": 0.2321988406207233,
                "ci95": [
                  0.14257404551466304,
                  0.33958398657928274
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "direct": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.1310210513203216,
                "ci95": [
                  0.05612678122708029,
                  0.21547273014891286
                ]
              },
              "expected_brier": {
                "score": 0.4666624679869883,
                "ci95": [
                  0.17190064776035446,
                  0.7721647668886509
                ]
              },
              "total_variation": {
                "score": 0.2582707228323835,
                "ci95": [
                  0.12568157105627656,
                  0.3984923953409316
                ]
              },
              "impossible_mass": {
                "score": 0.0451800745059822,
                "ci95": [
                  0.0015661763307765135,
                  0.10233984710226347
                ]
              }
            },
            "disclosure": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.05849840187557101,
                "ci95": [
                  0.0153879939857087,
                  0.12485953982968281
                ]
              },
              "expected_brier": {
                "score": 0.47735642656692906,
                "ci95": [
                  0.4499814803299445,
                  0.5014039264314089
                ]
              },
              "total_variation": {
                "score": 0.12949204654505894,
                "ci95": [
                  0.06862680535458965,
                  0.2062691773046215
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "history": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.113801516820327,
                "ci95": [
                  0.051838918072006884,
                  0.17707153528988728
                ]
              },
              "expected_brier": {
                "score": 0.6233431382437458,
                "ci95": [
                  0.48525243739675006,
                  0.7761662583250104
                ]
              },
              "total_variation": {
                "score": 0.24710121741942265,
                "ci95": [
                  0.13870224159180283,
                  0.3695320420620432
                ]
              },
              "impossible_mass": {
                "score": 0.05332251809217842,
                "ci95": [
                  0,
                  0.11435524191382944
                ]
              }
            },
            "sequential": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.22756080646563676,
                "ci95": [
                  0.08696321050626299,
                  0.4074027547321081
                ]
              },
              "expected_brier": {
                "score": 0.7327996752755852,
                "ci95": [
                  0.6223795096463597,
                  0.8592790652995713
                ]
              },
              "total_variation": {
                "score": 0.3072383443789576,
                "ci95": [
                  0.20529132288991392,
                  0.42024440427588317
                ]
              },
              "impossible_mass": {
                "score": 0.030594550737557423,
                "ci95": [
                  0,
                  0.07128677344023752
                ]
              }
            }
          }
        }
      }
    },
    "qwen35_9b": {
      "receipt": {
        "raw_sha256": "1d7833fefff416a9ba77eccb29bb1accbc009ee3f95d0ecaa905001e522c052a",
        "metadata_sha256": "7bb8115ec3daeb5be8b078e1d6de1035ec18b0c36877573a6d89bf536c0bfe9d",
        "count": 1152,
        "errors": 0
      },
      "metrics": {
        "cladder": {
          "original": {
            "n": 144,
            "correct": 80,
            "score": 0.5555555555555556,
            "ci95": [
              0.4725932267884323,
              0.6363636363636364
            ],
            "errors": 0
          },
          "reversed": {
            "n": 144,
            "correct": 85,
            "score": 0.5902777777777778,
            "ci95": [
              0.5069444444444444,
              0.6734729308390022
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 144,
            "correct": 72,
            "score": 0.5,
            "ci95": [
              0.41843971631205673,
              0.5815760711162925
            ]
          },
          "prediction_flips": 21,
          "strata": {
            "1": {
              "n": 48,
              "correct": 26,
              "score": 0.5416666666666666,
              "ci95": [
                0.3958333333333333,
                0.6808510638297872
              ],
              "errors": 0
            },
            "2": {
              "n": 48,
              "correct": 28,
              "score": 0.5833333333333334,
              "ci95": [
                0.4583333333333333,
                0.7083333333333334
              ],
              "errors": 0
            },
            "3": {
              "n": 48,
              "correct": 26,
              "score": 0.5416666666666666,
              "ci95": [
                0.3958333333333333,
                0.6671874999999972
              ],
              "errors": 0
            }
          }
        },
        "cruxeval": {
          "original": {
            "n": 128,
            "correct": 48,
            "score": 0.375,
            "ci95": [
              0.2890625,
              0.4609375
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 62,
            "score": 0.484375,
            "ci95": [
              0.3984375,
              0.5625
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 30,
            "score": 0.234375,
            "ci95": [
              0.1640625,
              0.3046875
            ]
          },
          "prediction_flips": 68,
          "strata": {
            "all": {
              "n": 128,
              "correct": 48,
              "score": 0.375,
              "ci95": [
                0.2890625,
                0.4609375
              ],
              "errors": 0
            }
          },
          "uniform_choice_chance": 0.3639136904761905,
          "candidate_counts": {
            "2": 56,
            "3": 28,
            "4": 23,
            "5": 8,
            "6": 5,
            "7": 5,
            "8": 2,
            "10": 1
          }
        },
        "finentity": {
          "original": {
            "n": 128,
            "correct": 107,
            "score": 0.8359375,
            "ci95": [
              0.7732421875,
              0.8984375
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 102,
            "score": 0.796875,
            "ci95": [
              0.7265625,
              0.859375
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 99,
            "score": 0.7734375,
            "ci95": [
              0.6953125,
              0.84375
            ]
          },
          "prediction_flips": 11,
          "strata": {
            "Negative": {
              "n": 43,
              "correct": 42,
              "score": 0.9767441860465116,
              "ci95": [
                0.9302325581395349,
                1
              ],
              "errors": 0
            },
            "Neutral": {
              "n": 42,
              "correct": 24,
              "score": 0.5714285714285714,
              "ci95": [
                0.42857142857142855,
                0.7142857142857143
              ],
              "errors": 0
            },
            "Positive": {
              "n": 43,
              "correct": 41,
              "score": 0.9534883720930233,
              "ci95": [
                0.8837209302325582,
                1
              ],
              "errors": 0
            }
          }
        },
        "when2call": {
          "original": {
            "n": 128,
            "correct": 75,
            "score": 0.5859375,
            "ci95": [
              0.5,
              0.671875
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 77,
            "score": 0.6015625,
            "ci95": [
              0.515625,
              0.6875
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 63,
            "score": 0.4921875,
            "ci95": [
              0.40625,
              0.578125
            ]
          },
          "prediction_flips": 38,
          "strata": {
            "cannot_answer": {
              "n": 43,
              "correct": 11,
              "score": 0.2558139534883721,
              "ci95": [
                0.13953488372093023,
                0.3953488372093023
              ],
              "errors": 0
            },
            "request_for_info": {
              "n": 42,
              "correct": 23,
              "score": 0.5476190476190477,
              "ci95": [
                0.40476190476190477,
                0.6904761904761905
              ],
              "errors": 0
            },
            "tool_call": {
              "n": 43,
              "correct": 41,
              "score": 0.9534883720930233,
              "ci95": [
                0.8837209302325582,
                1
              ],
              "errors": 0
            }
          },
          "direct_false_selections": {
            "n": 128,
            "count": 5,
            "rate": 0.0390625,
            "gold_positives": 0
          }
        },
        "known_distribution": {
          "n": 96,
          "valid": 96,
          "errors": 0,
          "paired_settings": 48,
          "excess_brier": {
            "score": 0.14721344427083752,
            "ci95": [
              0.10362575173326492,
              0.1932419252207373
            ]
          },
          "expected_brier": {
            "score": 0.5923612286582199,
            "ci95": [
              0.49995435073484806,
              0.6767848309930526
            ]
          },
          "total_variation": {
            "score": 0.24753237226627697,
            "ci95": [
              0.1954725928811696,
              0.3008978668024852
            ]
          },
          "impossible_mass": {
            "score": 0.027116460921176196,
            "ci95": [
              0.008958104020933683,
              0.051212095981848504
            ]
          },
          "uniform_all_candidates": {
            "n": 96,
            "valid": 96,
            "errors": 0,
            "paired_settings": 48,
            "excess_brier": {
              "score": 0.1439903922507942,
              "ci95": [
                0.09995881558499044,
                0.19733934748419735
              ]
            },
            "expected_brier": {
              "score": 0.5891381766381767,
              "ci95": [
                0.5522569444444446,
                0.6316912615740741
              ]
            },
            "total_variation": {
              "score": 0.2490848180933528,
              "ci95": [
                0.19992744552797803,
                0.30308455010232166
              ]
            },
            "impossible_mass": {
              "score": 0.07797364672364672,
              "ci95": [
                0.033561030982905984,
                0.13547743055555547
              ]
            }
          },
          "minus_uniform_excess_brier": {
            "n": 96,
            "delta": 0.003223052020043308,
            "ci95": [
              -0.07385652080803214,
              0.07204950935528784
            ]
          },
          "categories": {
            "composition": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.10462546645986631,
                "ci95": [
                  0.026072792697027274,
                  0.1983647057237389
                ]
              },
              "expected_brier": {
                "score": 0.6178427388364095,
                "ci95": [
                  0.3576264276933521,
                  0.8736116969844785
                ]
              },
              "total_variation": {
                "score": 0.23923145191664782,
                "ci95": [
                  0.10358437040382372,
                  0.3902251771880042
                ]
              },
              "impossible_mass": {
                "score": 0.03372065847172919,
                "ci95": [
                  0,
                  0.08262113330568194
                ]
              }
            },
            "daily_evidence": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.2998212231886916,
                "ci95": [
                  0.15019693241963744,
                  0.45775900142668063
                ]
              },
              "expected_brier": {
                "score": 0.6882107255450511,
                "ci95": [
                  0.5093187560805804,
                  0.847034783291385
                ]
              },
              "total_variation": {
                "score": 0.3465139508116583,
                "ci95": [
                  0.23156366431170367,
                  0.4556028982972181
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "direct": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.15996923944011054,
                "ci95": [
                  0.04092459450518735,
                  0.29839701821583897
                ]
              },
              "expected_brier": {
                "score": 0.49561065610677724,
                "ci95": [
                  0.1255147590926473,
                  0.8657609751149028
                ]
              },
              "total_variation": {
                "score": 0.2702719819783742,
                "ci95": [
                  0.10916664767236116,
                  0.44618367222616095
                ]
              },
              "impossible_mass": {
                "score": 0.021573392370990173,
                "ci95": [
                  0.0025006748708571416,
                  0.046905724218213304
                ]
              }
            },
            "disclosure": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.14890186185580162,
                "ci95": [
                  0.09323606360891593,
                  0.20645161215145127
                ]
              },
              "expected_brier": {
                "score": 0.5677598865471596,
                "ci95": [
                  0.49338275479623434,
                  0.643333065733505
                ]
              },
              "total_variation": {
                "score": 0.22571354995446183,
                "ci95": [
                  0.17759291636107702,
                  0.2814941230707824
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "history": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.06337787836866393,
                "ci95": [
                  0.022078548896320334,
                  0.11122357176479279
                ]
              },
              "expected_brier": {
                "score": 0.5729194997920827,
                "ci95": [
                  0.45417492514253455,
                  0.7128925119541866
                ]
              },
              "total_variation": {
                "score": 0.18064403746283314,
                "ci95": [
                  0.07422820088593719,
                  0.3032708953708396
                ]
              },
              "impossible_mass": {
                "score": 0.08218430959573518,
                "ci95": [
                  0,
                  0.19209981259832726
                ]
              }
            },
            "sequential": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.10658499631189097,
                "ci95": [
                  0.04744367424584325,
                  0.17347263773131166
                ]
              },
              "expected_brier": {
                "score": 0.6118238651218394,
                "ci95": [
                  0.5322583886932177,
                  0.7203787359744598
                ]
              },
              "total_variation": {
                "score": 0.22281926147368636,
                "ci95": [
                  0.13364632777391638,
                  0.315899940205585
                ]
              },
              "impossible_mass": {
                "score": 0.025220405088602606,
                "ci95": [
                  0,
                  0.06025429206303859
                ]
              }
            }
          }
        }
      }
    },
    "qwen38_27b": {
      "receipt": {
        "raw_sha256": "dfc54025db97551cd42f1e7cb88ca7186da6657b456cc378f2dce83ce9f56941",
        "metadata_sha256": "cbf29c8f59eca030560b7d8178f39d7af97dbe432a3468f68c4e1299dfe13b45",
        "count": 1152,
        "errors": 0
      },
      "metrics": {
        "cladder": {
          "original": {
            "n": 144,
            "correct": 85,
            "score": 0.5902777777777778,
            "ci95": [
              0.5104895104895105,
              0.668921298058622
            ],
            "errors": 0
          },
          "reversed": {
            "n": 144,
            "correct": 80,
            "score": 0.5555555555555556,
            "ci95": [
              0.47586206896551725,
              0.6388888888888888
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 144,
            "correct": 40,
            "score": 0.2777777777777778,
            "ci95": [
              0.21085700014271444,
              0.35617635788868657
            ]
          },
          "prediction_flips": 85,
          "strata": {
            "1": {
              "n": 48,
              "correct": 24,
              "score": 0.5,
              "ci95": [
                0.3617021276595745,
                0.6382978723404256
              ],
              "errors": 0
            },
            "2": {
              "n": 48,
              "correct": 30,
              "score": 0.625,
              "ci95": [
                0.4791666666666667,
                0.75
              ],
              "errors": 0
            },
            "3": {
              "n": 48,
              "correct": 31,
              "score": 0.6458333333333334,
              "ci95": [
                0.5,
                0.7708333333333334
              ],
              "errors": 0
            }
          }
        },
        "cruxeval": {
          "original": {
            "n": 128,
            "correct": 70,
            "score": 0.546875,
            "ci95": [
              0.4609375,
              0.6328125
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 72,
            "score": 0.5625,
            "ci95": [
              0.4765625,
              0.640625
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 55,
            "score": 0.4296875,
            "ci95": [
              0.34375,
              0.515625
            ]
          },
          "prediction_flips": 48,
          "strata": {
            "all": {
              "n": 128,
              "correct": 70,
              "score": 0.546875,
              "ci95": [
                0.4609375,
                0.6328125
              ],
              "errors": 0
            }
          },
          "uniform_choice_chance": 0.3639136904761905,
          "candidate_counts": {
            "2": 56,
            "3": 28,
            "4": 23,
            "5": 8,
            "6": 5,
            "7": 5,
            "8": 2,
            "10": 1
          }
        },
        "finentity": {
          "original": {
            "n": 128,
            "correct": 113,
            "score": 0.8828125,
            "ci95": [
              0.8203125,
              0.9375
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 114,
            "score": 0.890625,
            "ci95": [
              0.8357421875,
              0.9375
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 110,
            "score": 0.859375,
            "ci95": [
              0.796875,
              0.921875
            ]
          },
          "prediction_flips": 7,
          "strata": {
            "Negative": {
              "n": 43,
              "correct": 43,
              "score": 1,
              "ci95": [
                1,
                1
              ],
              "errors": 0
            },
            "Neutral": {
              "n": 42,
              "correct": 29,
              "score": 0.6904761904761905,
              "ci95": [
                0.5476190476190477,
                0.8333333333333334
              ],
              "errors": 0
            },
            "Positive": {
              "n": 43,
              "correct": 41,
              "score": 0.9534883720930233,
              "ci95": [
                0.8837209302325582,
                1
              ],
              "errors": 0
            }
          }
        },
        "when2call": {
          "original": {
            "n": 128,
            "correct": 89,
            "score": 0.6953125,
            "ci95": [
              0.6171875,
              0.7734375
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 97,
            "score": 0.7578125,
            "ci95": [
              0.6796875,
              0.828125
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 87,
            "score": 0.6796875,
            "ci95": [
              0.6015625,
              0.7578125
            ]
          },
          "prediction_flips": 19,
          "strata": {
            "cannot_answer": {
              "n": 43,
              "correct": 15,
              "score": 0.3488372093023256,
              "ci95": [
                0.20930232558139536,
                0.5116279069767442
              ],
              "errors": 0
            },
            "request_for_info": {
              "n": 42,
              "correct": 35,
              "score": 0.8333333333333334,
              "ci95": [
                0.7142857142857143,
                0.9285714285714286
              ],
              "errors": 0
            },
            "tool_call": {
              "n": 43,
              "correct": 39,
              "score": 0.9069767441860465,
              "ci95": [
                0.813953488372093,
                0.9767441860465116
              ],
              "errors": 0
            }
          },
          "direct_false_selections": {
            "n": 128,
            "count": 3,
            "rate": 0.0234375,
            "gold_positives": 0
          }
        },
        "known_distribution": {
          "n": 96,
          "valid": 96,
          "errors": 0,
          "paired_settings": 48,
          "excess_brier": {
            "score": 0.10729454824532587,
            "ci95": [
              0.08337661226164945,
              0.13699550180141254
            ]
          },
          "expected_brier": {
            "score": 0.5524423326327083,
            "ci95": [
              0.4809669656686567,
              0.6205733798007553
            ]
          },
          "total_variation": {
            "score": 0.22350004322671702,
            "ci95": [
              0.18646251022762905,
              0.26454460642358113
            ]
          },
          "impossible_mass": {
            "score": 0.026614895231161954,
            "ci95": [
              0.010035584099124051,
              0.04684313372879213
            ]
          },
          "uniform_all_candidates": {
            "n": 96,
            "valid": 96,
            "errors": 0,
            "paired_settings": 48,
            "excess_brier": {
              "score": 0.1439903922507942,
              "ci95": [
                0.09995881558499044,
                0.19733934748419735
              ]
            },
            "expected_brier": {
              "score": 0.5891381766381767,
              "ci95": [
                0.5522569444444446,
                0.6316912615740741
              ]
            },
            "total_variation": {
              "score": 0.2490848180933528,
              "ci95": [
                0.19992744552797803,
                0.30308455010232166
              ]
            },
            "impossible_mass": {
              "score": 0.07797364672364672,
              "ci95": [
                0.033561030982905984,
                0.13547743055555547
              ]
            }
          },
          "minus_uniform_excess_brier": {
            "n": 96,
            "delta": -0.03669584400546832,
            "ci95": [
              -0.09705450455017783,
              0.011641603798305339
            ]
          },
          "categories": {
            "composition": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.15246038862560596,
                "ci95": [
                  0.08102238528599996,
                  0.25720171411409043
                ]
              },
              "expected_brier": {
                "score": 0.6656776610021492,
                "ci95": [
                  0.43669341501164116,
                  0.9115437901282994
                ]
              },
              "total_variation": {
                "score": 0.2911157297507766,
                "ci95": [
                  0.1936345440554913,
                  0.4261447594591552
                ]
              },
              "impossible_mass": {
                "score": 0.032192009325702334,
                "ci95": [
                  0,
                  0.08079038058369996
                ]
              }
            },
            "daily_evidence": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.11074191332982422,
                "ci95": [
                  0.056945955125556925,
                  0.18233303786482147
                ]
              },
              "expected_brier": {
                "score": 0.4991314156861838,
                "ci95": [
                  0.45153702843086085,
                  0.5461040882981185
                ]
              },
              "total_variation": {
                "score": 0.20264725203609157,
                "ci95": [
                  0.14355821826206025,
                  0.2704340001634967
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "direct": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.09995137726661688,
                "ci95": [
                  0.04672196929063341,
                  0.15266889702045416
                ]
              },
              "expected_brier": {
                "score": 0.4355927939332836,
                "ci95": [
                  0.1611455882721003,
                  0.7092451561499615
                ]
              },
              "total_variation": {
                "score": 0.23553700275271355,
                "ci95": [
                  0.12180076359613982,
                  0.35804502958177686
                ]
              },
              "impossible_mass": {
                "score": 0.03710263168545819,
                "ci95": [
                  0.0011128551769034752,
                  0.08429681909553678
                ]
              }
            },
            "disclosure": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.07565290966893566,
                "ci95": [
                  0.032131114292414914,
                  0.12307614252687643
                ]
              },
              "expected_brier": {
                "score": 0.4945109343602937,
                "ci95": [
                  0.4635979457446883,
                  0.5192002902512511
                ]
              },
              "total_variation": {
                "score": 0.1609179119571701,
                "ci95": [
                  0.10806497998184539,
                  0.21816612000520663
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "history": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.09306573239873761,
                "ci95": [
                  0.05519847924752272,
                  0.1356022378025492
                ]
              },
              "expected_brier": {
                "score": 0.6026073538221564,
                "ci95": [
                  0.5066706308508784,
                  0.708269405066072
                ]
              },
              "total_variation": {
                "score": 0.22140551588125879,
                "ci95": [
                  0.146630405029174,
                  0.30213357951849173
                ]
              },
              "impossible_mass": {
                "score": 0.06277420546746808,
                "ci95": [
                  0,
                  0.12889331556153
                ]
              }
            },
            "sequential": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.11189496818223496,
                "ci95": [
                  0.05570040272856666,
                  0.17607307182859352
                ]
              },
              "expected_brier": {
                "score": 0.6171338369921835,
                "ci95": [
                  0.5285219437077477,
                  0.7117263037811484
                ]
              },
              "total_variation": {
                "score": 0.22937684698229144,
                "ci95": [
                  0.15000194398267191,
                  0.31500100726447877
                ]
              },
              "impossible_mass": {
                "score": 0.02762052490834312,
                "ci95": [
                  0,
                  0.06626052096189215
                ]
              }
            }
          }
        }
      }
    },
    "jev": {
      "receipt": {
        "raw_sha256": "8dca5fe8815e60512425bd500b0b91671a4a3ea24d2eef7b22e158f73f41dcdc",
        "metadata_sha256": "7bf3225d4ee72c442f9a47a3e7cc16542d79588f407e17a112ddfbe092690fcb",
        "count": 1152,
        "errors": 3
      },
      "metrics": {
        "cladder": {
          "original": {
            "n": 144,
            "correct": 111,
            "score": 0.7708333333333334,
            "ci95": [
              0.6993006993006993,
              0.8369033927544565
            ],
            "errors": 0
          },
          "reversed": {
            "n": 144,
            "correct": 107,
            "score": 0.7430555555555556,
            "ci95": [
              0.6690140845070423,
              0.8137931034482758
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 144,
            "correct": 103,
            "score": 0.7152777777777778,
            "ci95": [
              0.640810338219795,
              0.7862068965517242
            ]
          },
          "prediction_flips": 12,
          "strata": {
            "1": {
              "n": 48,
              "correct": 40,
              "score": 0.8333333333333334,
              "ci95": [
                0.723404255319149,
                0.9361702127659575
              ],
              "errors": 0
            },
            "2": {
              "n": 48,
              "correct": 34,
              "score": 0.7083333333333334,
              "ci95": [
                0.5833333333333334,
                0.8333333333333334
              ],
              "errors": 0
            },
            "3": {
              "n": 48,
              "correct": 37,
              "score": 0.7708333333333334,
              "ci95": [
                0.6458333333333334,
                0.875
              ],
              "errors": 0
            }
          }
        },
        "cruxeval": {
          "original": {
            "n": 128,
            "correct": 85,
            "score": 0.6640625,
            "ci95": [
              0.578125,
              0.7421875
            ],
            "errors": 1
          },
          "reversed": {
            "n": 128,
            "correct": 86,
            "score": 0.671875,
            "ci95": [
              0.5859375,
              0.75
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 80,
            "score": 0.625,
            "ci95": [
              0.5390625,
              0.703125
            ]
          },
          "prediction_flips": 21,
          "strata": {
            "all": {
              "n": 128,
              "correct": 85,
              "score": 0.6640625,
              "ci95": [
                0.578125,
                0.7421875
              ],
              "errors": 1
            }
          },
          "uniform_choice_chance": 0.3639136904761905,
          "candidate_counts": {
            "2": 56,
            "3": 28,
            "4": 23,
            "5": 8,
            "6": 5,
            "7": 5,
            "8": 2,
            "10": 1
          }
        },
        "finentity": {
          "original": {
            "n": 128,
            "correct": 115,
            "score": 0.8984375,
            "ci95": [
              0.84375,
              0.9453125
            ],
            "errors": 1
          },
          "reversed": {
            "n": 128,
            "correct": 118,
            "score": 0.921875,
            "ci95": [
              0.875,
              0.9609375
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 115,
            "score": 0.8984375,
            "ci95": [
              0.84375,
              0.9453125
            ]
          },
          "prediction_flips": 3,
          "strata": {
            "Negative": {
              "n": 43,
              "correct": 42,
              "score": 0.9767441860465116,
              "ci95": [
                0.9302325581395349,
                1
              ],
              "errors": 0
            },
            "Neutral": {
              "n": 42,
              "correct": 32,
              "score": 0.7619047619047619,
              "ci95": [
                0.6428571428571429,
                0.8809523809523809
              ],
              "errors": 1
            },
            "Positive": {
              "n": 43,
              "correct": 41,
              "score": 0.9534883720930233,
              "ci95": [
                0.8837209302325582,
                1
              ],
              "errors": 0
            }
          }
        },
        "when2call": {
          "original": {
            "n": 128,
            "correct": 96,
            "score": 0.75,
            "ci95": [
              0.671875,
              0.8203125
            ],
            "errors": 1
          },
          "reversed": {
            "n": 128,
            "correct": 94,
            "score": 0.734375,
            "ci95": [
              0.65625,
              0.8046875
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 91,
            "score": 0.7109375,
            "ci95": [
              0.6328125,
              0.7890625
            ]
          },
          "prediction_flips": 14,
          "strata": {
            "cannot_answer": {
              "n": 43,
              "correct": 26,
              "score": 0.6046511627906976,
              "ci95": [
                0.46511627906976744,
                0.7441860465116279
              ],
              "errors": 1
            },
            "request_for_info": {
              "n": 42,
              "correct": 35,
              "score": 0.8333333333333334,
              "ci95": [
                0.7142857142857143,
                0.9285714285714286
              ],
              "errors": 0
            },
            "tool_call": {
              "n": 43,
              "correct": 35,
              "score": 0.813953488372093,
              "ci95": [
                0.6976744186046512,
                0.9302325581395349
              ],
              "errors": 0
            }
          },
          "direct_false_selections": {
            "n": 128,
            "count": 3,
            "rate": 0.0234375,
            "gold_positives": 0
          }
        },
        "known_distribution": {
          "n": 96,
          "valid": 96,
          "errors": 0,
          "paired_settings": 48,
          "excess_brier": {
            "score": 0.1481170265884406,
            "ci95": [
              0.09912016218446497,
              0.19575586776284878
            ]
          },
          "expected_brier": {
            "score": 0.5932648109758231,
            "ci95": [
              0.4926053347260852,
              0.6837201889159268
            ]
          },
          "total_variation": {
            "score": 0.24123491344563697,
            "ci95": [
              0.1847368337741121,
              0.2928304071799004
            ]
          },
          "impossible_mass": {
            "score": 0.005,
            "ci95": [
              0.0007291666666666667,
              0.012398437499999988
            ]
          },
          "uniform_all_candidates": {
            "n": 96,
            "valid": 96,
            "errors": 0,
            "paired_settings": 48,
            "excess_brier": {
              "score": 0.1439903922507942,
              "ci95": [
                0.09995881558499044,
                0.19733934748419735
              ]
            },
            "expected_brier": {
              "score": 0.5891381766381767,
              "ci95": [
                0.5522569444444446,
                0.6316912615740741
              ]
            },
            "total_variation": {
              "score": 0.2490848180933528,
              "ci95": [
                0.19992744552797803,
                0.30308455010232166
              ]
            },
            "impossible_mass": {
              "score": 0.07797364672364672,
              "ci95": [
                0.033561030982905984,
                0.13547743055555547
              ]
            }
          },
          "minus_uniform_excess_brier": {
            "n": 96,
            "delta": 0.004126634337646398,
            "ci95": [
              -0.07942406147457541,
              0.07532429548887816
            ]
          },
          "categories": {
            "composition": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.1721993942901235,
                "ci95": [
                  0.05018435156250001,
                  0.3398214563078704
                ]
              },
              "expected_brier": {
                "score": 0.6854166666666668,
                "ci95": [
                  0.3827490625,
                  1.0091762499999999
                ]
              },
              "total_variation": {
                "score": 0.27547916666666666,
                "ci95": [
                  0.1241171875,
                  0.4469583333333334
                ]
              },
              "impossible_mass": {
                "score": 0.00125,
                "ci95": [
                  0,
                  0.003125
                ]
              }
            },
            "daily_evidence": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.21935925226666989,
                "ci95": [
                  0.07282208792016233,
                  0.3627106979470008
                ]
              },
              "expected_brier": {
                "score": 0.6077487546230294,
                "ci95": [
                  0.4400089080740305,
                  0.7656429991515836
                ]
              },
              "total_variation": {
                "score": 0.26461166553027954,
                "ci95": [
                  0.12134154373024235,
                  0.4005260180995475
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "direct": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.15482405496888074,
                "ci95": [
                  0.038489800000000005,
                  0.2741900411877614
                ]
              },
              "expected_brier": {
                "score": 0.49046547163554743,
                "ci95": [
                  0.13332500000000003,
                  0.8393894849377613
                ]
              },
              "total_variation": {
                "score": 0.24746338383838384,
                "ci95": [
                  0.08431458333333336,
                  0.4091414772727272
                ]
              },
              "impossible_mass": {
                "score": 0.000625,
                "ci95": [
                  0,
                  0.001875
                ]
              }
            },
            "disclosure": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.05630586419753087,
                "ci95": [
                  0.016918672839506172,
                  0.11250868441358024
                ]
              },
              "expected_brier": {
                "score": 0.4751638888888889,
                "ci95": [
                  0.3753012152777779,
                  0.5733025000000002
                ]
              },
              "total_variation": {
                "score": 0.13875,
                "ci95": [
                  0.07895312500000001,
                  0.20937499999999998
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "history": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.14590191565544441,
                "ci95": [
                  0.07024223459805723,
                  0.23523143482599645
                ]
              },
              "expected_brier": {
                "score": 0.6554435370788632,
                "ci95": [
                  0.5148924313005835,
                  0.7984619196428571
                ]
              },
              "total_variation": {
                "score": 0.26233072181441747,
                "ci95": [
                  0.18707062864671564,
                  0.3414287067099567
                ]
              },
              "impossible_mass": {
                "score": 0.02125,
                "ci95": [
                  0,
                  0.05750000000000001
                ]
              }
            },
            "sequential": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.14011167815199424,
                "ci95": [
                  0.06309150940297739,
                  0.22104261329812883
                ]
              },
              "expected_brier": {
                "score": 0.6453505469619427,
                "ci95": [
                  0.516372502767335,
                  0.7593008460286458
                ]
              },
              "total_variation": {
                "score": 0.2587745428240741,
                "ci95": [
                  0.15700481481481482,
                  0.35439040060763854
                ]
              },
              "impossible_mass": {
                "score": 0.006875000000000001,
                "ci95": [
                  0,
                  0.01625
                ]
              }
            }
          }
        }
      }
    },
    "intern_08b": {
      "label": "Intern-Decision-0.8B",
      "receipt": {
        "raw_sha256": "3de543aa2684fb28668bf7a4c66879f51437e11ac9aa4f781ca27cde200bc7dc",
        "metadata_sha256": "2022b139055d2969d0a45bd59dd7ef93c47eb78274ac6cbde997718ea00e5fa6",
        "count": 1152,
        "errors": 0
      },
      "metrics": {
        "cladder": {
          "original": {
            "n": 144,
            "correct": 72,
            "score": 0.5,
            "ci95": [
              0.42066192428261395,
              0.5845070422535211
            ],
            "errors": 0
          },
          "reversed": {
            "n": 144,
            "correct": 72,
            "score": 0.5,
            "ci95": [
              0.4184239288837074,
              0.5862068965517241
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 144,
            "correct": 44,
            "score": 0.3055555555555556,
            "ci95": [
              0.23076923076923078,
              0.3835616438356164
            ]
          },
          "prediction_flips": 56,
          "strata": {
            "1": {
              "n": 48,
              "correct": 23,
              "score": 0.4791666666666667,
              "ci95": [
                0.3478039041703638,
                0.6122448979591837
              ],
              "errors": 0
            },
            "2": {
              "n": 48,
              "correct": 23,
              "score": 0.4791666666666667,
              "ci95": [
                0.3333333333333333,
                0.625
              ],
              "errors": 0
            },
            "3": {
              "n": 48,
              "correct": 26,
              "score": 0.5416666666666666,
              "ci95": [
                0.3958333333333333,
                0.6875
              ],
              "errors": 0
            }
          }
        },
        "cruxeval": {
          "original": {
            "n": 128,
            "correct": 34,
            "score": 0.265625,
            "ci95": [
              0.1875,
              0.34375
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 37,
            "score": 0.2890625,
            "ci95": [
              0.2109375,
              0.3671875
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 15,
            "score": 0.1171875,
            "ci95": [
              0.0625,
              0.171875
            ]
          },
          "prediction_flips": 73,
          "strata": {
            "all": {
              "n": 128,
              "correct": 34,
              "score": 0.265625,
              "ci95": [
                0.1875,
                0.34375
              ],
              "errors": 0
            }
          },
          "uniform_choice_chance": 0.3639136904761905,
          "candidate_counts": {
            "2": 56,
            "3": 28,
            "4": 23,
            "5": 8,
            "6": 5,
            "7": 5,
            "8": 2,
            "10": 1
          }
        },
        "finentity": {
          "original": {
            "n": 128,
            "correct": 98,
            "score": 0.765625,
            "ci95": [
              0.6875,
              0.8359375
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 94,
            "score": 0.734375,
            "ci95": [
              0.65625,
              0.8125
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 92,
            "score": 0.71875,
            "ci95": [
              0.640625,
              0.796875
            ]
          },
          "prediction_flips": 11,
          "strata": {
            "Negative": {
              "n": 43,
              "correct": 41,
              "score": 0.9534883720930233,
              "ci95": [
                0.8837209302325582,
                1
              ],
              "errors": 0
            },
            "Neutral": {
              "n": 42,
              "correct": 18,
              "score": 0.42857142857142855,
              "ci95": [
                0.2857142857142857,
                0.5714285714285714
              ],
              "errors": 0
            },
            "Positive": {
              "n": 43,
              "correct": 39,
              "score": 0.9069767441860465,
              "ci95": [
                0.813953488372093,
                0.9767441860465116
              ],
              "errors": 0
            }
          }
        },
        "when2call": {
          "original": {
            "n": 128,
            "correct": 62,
            "score": 0.484375,
            "ci95": [
              0.3984375,
              0.5703125
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 61,
            "score": 0.4765625,
            "ci95": [
              0.390625,
              0.5625
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 48,
            "score": 0.375,
            "ci95": [
              0.2890625,
              0.4609375
            ]
          },
          "prediction_flips": 34,
          "strata": {
            "cannot_answer": {
              "n": 43,
              "correct": 14,
              "score": 0.32558139534883723,
              "ci95": [
                0.18604651162790697,
                0.46511627906976744
              ],
              "errors": 0
            },
            "request_for_info": {
              "n": 42,
              "correct": 11,
              "score": 0.2619047619047619,
              "ci95": [
                0.14285714285714285,
                0.40476190476190477
              ],
              "errors": 0
            },
            "tool_call": {
              "n": 43,
              "correct": 37,
              "score": 0.8604651162790697,
              "ci95": [
                0.7441860465116279,
                0.9534883720930233
              ],
              "errors": 0
            }
          },
          "direct_false_selections": {
            "n": 128,
            "count": 9,
            "rate": 0.0703125,
            "gold_positives": 0
          }
        },
        "known_distribution": {
          "n": 96,
          "valid": 96,
          "errors": 0,
          "paired_settings": 48,
          "excess_brier": {
            "score": 0.10512949569284695,
            "ci95": [
              0.07638774070421098,
              0.13682395547541518
            ]
          },
          "expected_brier": {
            "score": 0.5502772800802295,
            "ci95": [
              0.5040829759324102,
              0.5959278430388997
            ]
          },
          "total_variation": {
            "score": 0.21934831094194884,
            "ci95": [
              0.1844118019066735,
              0.25409576369966724
            ]
          },
          "impossible_mass": {
            "score": 0.043022888994107035,
            "ci95": [
              0.01907672873559927,
              0.0736236472120092
            ]
          },
          "uniform_all_candidates": {
            "n": 96,
            "valid": 96,
            "errors": 0,
            "paired_settings": 48,
            "excess_brier": {
              "score": 0.1439903922507942,
              "ci95": [
                0.09995881558499044,
                0.19733934748419735
              ]
            },
            "expected_brier": {
              "score": 0.5891381766381767,
              "ci95": [
                0.5522569444444446,
                0.6316912615740741
              ]
            },
            "total_variation": {
              "score": 0.2490848180933528,
              "ci95": [
                0.19992744552797803,
                0.30308455010232166
              ]
            },
            "impossible_mass": {
              "score": 0.07797364672364672,
              "ci95": [
                0.033561030982905984,
                0.13547743055555547
              ]
            }
          },
          "minus_uniform_excess_brier": {
            "n": 96,
            "delta": -0.03886089655794724,
            "ci95": [
              -0.07721830374199622,
              -0.01041863749704549
            ]
          },
          "categories": {
            "composition": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.09165839749104603,
                "ci95": [
                  0.029671884024523584,
                  0.16648497502469325
                ]
              },
              "expected_brier": {
                "score": 0.6048756698675892,
                "ci95": [
                  0.45856373752335755,
                  0.7517827360760194
                ]
              },
              "total_variation": {
                "score": 0.22911724851221674,
                "ci95": [
                  0.14338059888278618,
                  0.3067958855493767
                ]
              },
              "impossible_mass": {
                "score": 0.04274138658753628,
                "ci95": [
                  0,
                  0.10294892411462549
                ]
              }
            },
            "daily_evidence": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.13111545481196535,
                "ci95": [
                  0.06781024962794296,
                  0.20053809723022056
                ]
              },
              "expected_brier": {
                "score": 0.519504957168325,
                "ci95": [
                  0.4881407811626321,
                  0.5485378255067741
                ]
              },
              "total_variation": {
                "score": 0.23289083195997373,
                "ci95": [
                  0.161599590894772,
                  0.30472467690661775
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "direct": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.17578923107081318,
                "ci95": [
                  0.08359297998810922,
                  0.29997995959305773
                ]
              },
              "expected_brier": {
                "score": 0.5114306477374798,
                "ci95": [
                  0.3230486738115482,
                  0.6963883949084181
                ]
              },
              "total_variation": {
                "score": 0.3063320441003193,
                "ci95": [
                  0.24464131313123874,
                  0.37852929977297345
                ]
              },
              "impossible_mass": {
                "score": 0.10057688945246139,
                "ci95": [
                  0.03268893964317166,
                  0.18345740348647213
                ]
              }
            },
            "disclosure": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.06928727874474427,
                "ci95": [
                  0.020099013939960962,
                  0.13632232306866932
                ]
              },
              "expected_brier": {
                "score": 0.48814530343610235,
                "ci95": [
                  0.46769859549212345,
                  0.5060102767392282
                ]
              },
              "total_variation": {
                "score": 0.15145155400658752,
                "ci95": [
                  0.08541172371962122,
                  0.22784489856957962
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "history": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.05610760103931669,
                "ci95": [
                  0.0188989517296306,
                  0.10519640073485777
                ]
              },
              "expected_brier": {
                "score": 0.5656492224627354,
                "ci95": [
                  0.4893262871809951,
                  0.6497959257682357
                ]
              },
              "total_variation": {
                "score": 0.1658575755928322,
                "ci95": [
                  0.08560284815489504,
                  0.25468657496703245
                ]
              },
              "impossible_mass": {
                "score": 0.0770064022525301,
                "ci95": [
                  0,
                  0.17003647488963552
                ]
              }
            },
            "sequential": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.10681901099919622,
                "ci95": [
                  0.054016987179714586,
                  0.16782201292121549
                ]
              },
              "expected_brier": {
                "score": 0.6120578798091447,
                "ci95": [
                  0.552624354412515,
                  0.6712326828983838
                ]
              },
              "total_variation": {
                "score": 0.23044061147976352,
                "ci95": [
                  0.15508311350526438,
                  0.3026405190892391
                ]
              },
              "impossible_mass": {
                "score": 0.03781265567211445,
                "ci95": [
                  0,
                  0.0918583606552163
                ]
              }
            }
          }
        }
      }
    },
    "intern_2b": {
      "label": "Intern-Decision-2B",
      "receipt": {
        "raw_sha256": "dc7db1275c25ddeda5c3f3f30874368062bea3bf41e254c84d660ae48eb08ca2",
        "metadata_sha256": "a8c58844932a7a2400d0c903858a8296e30fdbb013e82e17a660db4a4957a8a1",
        "count": 1152,
        "errors": 0
      },
      "metrics": {
        "cladder": {
          "original": {
            "n": 144,
            "correct": 78,
            "score": 0.5416666666666666,
            "ci95": [
              0.4588898401826484,
              0.6223776223776224
            ],
            "errors": 0
          },
          "reversed": {
            "n": 144,
            "correct": 77,
            "score": 0.5347222222222222,
            "ci95": [
              0.45575139146567717,
              0.6127039825157842
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 144,
            "correct": 58,
            "score": 0.4027777777777778,
            "ci95": [
              0.32413793103448274,
              0.4822757030203838
            ]
          },
          "prediction_flips": 39,
          "strata": {
            "1": {
              "n": 48,
              "correct": 25,
              "score": 0.5208333333333334,
              "ci95": [
                0.375,
                0.6595744680851063
              ],
              "errors": 0
            },
            "2": {
              "n": 48,
              "correct": 30,
              "score": 0.625,
              "ci95": [
                0.5,
                0.75
              ],
              "errors": 0
            },
            "3": {
              "n": 48,
              "correct": 23,
              "score": 0.4791666666666667,
              "ci95": [
                0.3333333333333333,
                0.625
              ],
              "errors": 0
            }
          }
        },
        "cruxeval": {
          "original": {
            "n": 128,
            "correct": 44,
            "score": 0.34375,
            "ci95": [
              0.2578125,
              0.42207031249999893
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 41,
            "score": 0.3203125,
            "ci95": [
              0.25,
              0.3984375
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 27,
            "score": 0.2109375,
            "ci95": [
              0.1482421875,
              0.28125
            ]
          },
          "prediction_flips": 67,
          "strata": {
            "all": {
              "n": 128,
              "correct": 44,
              "score": 0.34375,
              "ci95": [
                0.2578125,
                0.42207031249999893
              ],
              "errors": 0
            }
          },
          "uniform_choice_chance": 0.3639136904761905,
          "candidate_counts": {
            "2": 56,
            "3": 28,
            "4": 23,
            "5": 8,
            "6": 5,
            "7": 5,
            "8": 2,
            "10": 1
          }
        },
        "finentity": {
          "original": {
            "n": 128,
            "correct": 99,
            "score": 0.7734375,
            "ci95": [
              0.6953125,
              0.84375
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 103,
            "score": 0.8046875,
            "ci95": [
              0.734375,
              0.8671875
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 94,
            "score": 0.734375,
            "ci95": [
              0.65625,
              0.8125
            ]
          },
          "prediction_flips": 14,
          "strata": {
            "Negative": {
              "n": 43,
              "correct": 39,
              "score": 0.9069767441860465,
              "ci95": [
                0.813953488372093,
                0.9767441860465116
              ],
              "errors": 0
            },
            "Neutral": {
              "n": 42,
              "correct": 19,
              "score": 0.4523809523809524,
              "ci95": [
                0.30952380952380953,
                0.5952380952380952
              ],
              "errors": 0
            },
            "Positive": {
              "n": 43,
              "correct": 41,
              "score": 0.9534883720930233,
              "ci95": [
                0.8837209302325582,
                1
              ],
              "errors": 0
            }
          }
        },
        "when2call": {
          "original": {
            "n": 128,
            "correct": 78,
            "score": 0.609375,
            "ci95": [
              0.5234375,
              0.6953125
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 74,
            "score": 0.578125,
            "ci95": [
              0.4921875,
              0.6640625
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 61,
            "score": 0.4765625,
            "ci95": [
              0.390625,
              0.5625
            ]
          },
          "prediction_flips": 36,
          "strata": {
            "cannot_answer": {
              "n": 43,
              "correct": 16,
              "score": 0.37209302325581395,
              "ci95": [
                0.23255813953488372,
                0.5116279069767442
              ],
              "errors": 0
            },
            "request_for_info": {
              "n": 42,
              "correct": 23,
              "score": 0.5476190476190477,
              "ci95": [
                0.40476190476190477,
                0.6904761904761905
              ],
              "errors": 0
            },
            "tool_call": {
              "n": 43,
              "correct": 39,
              "score": 0.9069767441860465,
              "ci95": [
                0.813953488372093,
                0.9767441860465116
              ],
              "errors": 0
            }
          },
          "direct_false_selections": {
            "n": 128,
            "count": 11,
            "rate": 0.0859375,
            "gold_positives": 0
          }
        },
        "known_distribution": {
          "n": 96,
          "valid": 96,
          "errors": 0,
          "paired_settings": 48,
          "excess_brier": {
            "score": 0.08302916215818212,
            "ci95": [
              0.05552848090144894,
              0.11557170035342235
            ]
          },
          "expected_brier": {
            "score": 0.5281769465455646,
            "ci95": [
              0.4581842755502323,
              0.5915194513745216
            ]
          },
          "total_variation": {
            "score": 0.1810678851245884,
            "ci95": [
              0.143783275009988,
              0.21698348921465205
            ]
          },
          "impossible_mass": {
            "score": 0.019715427218037908,
            "ci95": [
              0.007125262036665827,
              0.03438254871545184
            ]
          },
          "uniform_all_candidates": {
            "n": 96,
            "valid": 96,
            "errors": 0,
            "paired_settings": 48,
            "excess_brier": {
              "score": 0.1439903922507942,
              "ci95": [
                0.09995881558499044,
                0.19733934748419735
              ]
            },
            "expected_brier": {
              "score": 0.5891381766381767,
              "ci95": [
                0.5522569444444446,
                0.6316912615740741
              ]
            },
            "total_variation": {
              "score": 0.2490848180933528,
              "ci95": [
                0.19992744552797803,
                0.30308455010232166
              ]
            },
            "impossible_mass": {
              "score": 0.07797364672364672,
              "ci95": [
                0.033561030982905984,
                0.13547743055555547
              ]
            }
          },
          "minus_uniform_excess_brier": {
            "n": 96,
            "delta": -0.06096123009261208,
            "ci95": [
              -0.12216192813362506,
              -0.008238366404621309
            ]
          },
          "categories": {
            "composition": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.05398809065137184,
                "ci95": [
                  0.017596254697120357,
                  0.0893157663950974
                ]
              },
              "expected_brier": {
                "score": 0.567205363027915,
                "ci95": [
                  0.3467053810433989,
                  0.7836290551661907
                ]
              },
              "total_variation": {
                "score": 0.18642897784545776,
                "ci95": [
                  0.08522308488735116,
                  0.2807823080510299
                ]
              },
              "impossible_mass": {
                "score": 0.04565017177213071,
                "ci95": [
                  0,
                  0.11080893915895347
                ]
              }
            },
            "daily_evidence": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.2064728804939276,
                "ci95": [
                  0.10648079718918275,
                  0.3285380433689636
                ]
              },
              "expected_brier": {
                "score": 0.5948623828502871,
                "ci95": [
                  0.49650822471663375,
                  0.6963992100209063
                ]
              },
              "total_variation": {
                "score": 0.2863255820623647,
                "ci95": [
                  0.18741663466915542,
                  0.38057514194298064
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "direct": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.035044266619522244,
                "ci95": [
                  0.013598070176122025,
                  0.05742113084657618
                ]
              },
              "expected_brier": {
                "score": 0.3706856832861889,
                "ci95": [
                  0.09486186658678863,
                  0.6320344983313464
                ]
              },
              "total_variation": {
                "score": 0.1358772976179569,
                "ci95": [
                  0.07567256520125001,
                  0.19958474022901992
                ]
              },
              "impossible_mass": {
                "score": 0.02129740749157193,
                "ci95": [
                  0.004585271549448098,
                  0.03916063019030956
                ]
              }
            },
            "disclosure": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.08777820289082565,
                "ci95": [
                  0.03534080200664184,
                  0.16266866525039594
                ]
              },
              "expected_brier": {
                "score": 0.5066362275821836,
                "ci95": [
                  0.45669516544547095,
                  0.5569956709089157
                ]
              },
              "total_variation": {
                "score": 0.17335010333575274,
                "ci95": [
                  0.11096569686971394,
                  0.25314572948316416
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "history": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.0507980447946607,
                "ci95": [
                  0.016379137813787296,
                  0.10154725844518285
                ]
              },
              "expected_brier": {
                "score": 0.5603396662180795,
                "ci95": [
                  0.4883793761874817,
                  0.6501920161263619
                ]
              },
              "total_variation": {
                "score": 0.1429633155940713,
                "ci95": [
                  0.07526982622659754,
                  0.21676871783132723
                ]
              },
              "impossible_mass": {
                "score": 0.0372955482234122,
                "ci95": [
                  0,
                  0.08320830935676535
                ]
              }
            },
            "sequential": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.06409348749878478,
                "ci95": [
                  0.022838299699909536,
                  0.1040656395644386
                ]
              },
              "expected_brier": {
                "score": 0.5693323563087332,
                "ci95": [
                  0.4973431497662345,
                  0.6416222015064468
                ]
              },
              "total_variation": {
                "score": 0.1614620342919272,
                "ci95": [
                  0.08061437972192267,
                  0.24286827793532595
                ]
              },
              "impossible_mass": {
                "score": 0.0140494358211126,
                "ci95": [
                  0,
                  0.033629893674263486
                ]
              }
            }
          }
        }
      }
    },
    "llama32_1b_public": {
      "label": "Llama-3.2-1B-Instruct (Unsloth BF16 distribution)",
      "receipt": {
        "raw_sha256": "39315cbc58dc1ddf275cf68e65a0184fdb6f714ecae90b703f1810c1d16388e5",
        "metadata_sha256": "2e5530acee4560ef8635dc25f685aab94760eee65bc2cc8c95d453cd5e574738",
        "count": 1152,
        "errors": 0
      },
      "metrics": {
        "cladder": {
          "original": {
            "n": 144,
            "correct": 72,
            "score": 0.5,
            "ci95": [
              0.4154929577464789,
              0.5833333333333334
            ],
            "errors": 0
          },
          "reversed": {
            "n": 144,
            "correct": 74,
            "score": 0.5138888888888888,
            "ci95": [
              0.4305555555555556,
              0.5986734693877549
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 144,
            "correct": 31,
            "score": 0.2152777777777778,
            "ci95": [
              0.14580818965517242,
              0.28474702380952366
            ]
          },
          "prediction_flips": 84,
          "strata": {
            "1": {
              "n": 48,
              "correct": 25,
              "score": 0.5208333333333334,
              "ci95": [
                0.3673469387755102,
                0.673469387755102
              ],
              "errors": 0
            },
            "2": {
              "n": 48,
              "correct": 24,
              "score": 0.5,
              "ci95": [
                0.3541666666666667,
                0.6458333333333334
              ],
              "errors": 0
            },
            "3": {
              "n": 48,
              "correct": 23,
              "score": 0.4791666666666667,
              "ci95": [
                0.3333333333333333,
                0.625
              ],
              "errors": 0
            }
          }
        },
        "cruxeval": {
          "original": {
            "n": 128,
            "correct": 40,
            "score": 0.3125,
            "ci95": [
              0.234375,
              0.3984375
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 54,
            "score": 0.421875,
            "ci95": [
              0.34375,
              0.5078125
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 18,
            "score": 0.140625,
            "ci95": [
              0.0859375,
              0.203125
            ]
          },
          "prediction_flips": 90,
          "strata": {
            "all": {
              "n": 128,
              "correct": 40,
              "score": 0.3125,
              "ci95": [
                0.234375,
                0.3984375
              ],
              "errors": 0
            }
          },
          "uniform_choice_chance": 0.3639136904761905,
          "candidate_counts": {
            "2": 56,
            "3": 28,
            "4": 23,
            "5": 8,
            "6": 5,
            "7": 5,
            "8": 2,
            "10": 1
          }
        },
        "finentity": {
          "original": {
            "n": 128,
            "correct": 63,
            "score": 0.4921875,
            "ci95": [
              0.40625,
              0.578125
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 63,
            "score": 0.4921875,
            "ci95": [
              0.40625,
              0.578125
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 40,
            "score": 0.3125,
            "ci95": [
              0.234375,
              0.3984375
            ]
          },
          "prediction_flips": 65,
          "strata": {
            "Negative": {
              "n": 43,
              "correct": 26,
              "score": 0.6046511627906976,
              "ci95": [
                0.46511627906976744,
                0.7441860465116279
              ],
              "errors": 0
            },
            "Neutral": {
              "n": 42,
              "correct": 6,
              "score": 0.14285714285714285,
              "ci95": [
                0.047619047619047616,
                0.2619047619047619
              ],
              "errors": 0
            },
            "Positive": {
              "n": 43,
              "correct": 31,
              "score": 0.7209302325581395,
              "ci95": [
                0.5813953488372093,
                0.8372093023255814
              ],
              "errors": 0
            }
          }
        },
        "when2call": {
          "original": {
            "n": 128,
            "correct": 28,
            "score": 0.21875,
            "ci95": [
              0.1484375,
              0.296875
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 28,
            "score": 0.21875,
            "ci95": [
              0.1484375,
              0.2890625
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 12,
            "score": 0.09375,
            "ci95": [
              0.046875,
              0.1484375
            ]
          },
          "prediction_flips": 82,
          "strata": {
            "cannot_answer": {
              "n": 43,
              "correct": 1,
              "score": 0.023255813953488372,
              "ci95": [
                0,
                0.06976744186046512
              ],
              "errors": 0
            },
            "request_for_info": {
              "n": 42,
              "correct": 5,
              "score": 0.11904761904761904,
              "ci95": [
                0.023809523809523808,
                0.21428571428571427
              ],
              "errors": 0
            },
            "tool_call": {
              "n": 43,
              "correct": 22,
              "score": 0.5116279069767442,
              "ci95": [
                0.37209302325581395,
                0.6511627906976745
              ],
              "errors": 0
            }
          },
          "direct_false_selections": {
            "n": 128,
            "count": 43,
            "rate": 0.3359375,
            "gold_positives": 0
          }
        },
        "known_distribution": {
          "n": 96,
          "valid": 96,
          "errors": 0,
          "paired_settings": 48,
          "excess_brier": {
            "score": 0.46290859690869657,
            "ci95": [
              0.38615885464444627,
              0.5462251690035556
            ]
          },
          "expected_brier": {
            "score": 0.908056381296079,
            "ci95": [
              0.8348463962581297,
              0.976870747757802
            ]
          },
          "total_variation": {
            "score": 0.46630991031320296,
            "ci95": [
              0.4191037152844758,
              0.5129743786924007
            ]
          },
          "impossible_mass": {
            "score": 0.09011336500075152,
            "ci95": [
              0.03604318446176053,
              0.15407323947541662
            ]
          },
          "uniform_all_candidates": {
            "n": 96,
            "valid": 96,
            "errors": 0,
            "paired_settings": 48,
            "excess_brier": {
              "score": 0.1439903922507942,
              "ci95": [
                0.09995881558499044,
                0.19733934748419735
              ]
            },
            "expected_brier": {
              "score": 0.5891381766381767,
              "ci95": [
                0.5522569444444446,
                0.6316912615740741
              ]
            },
            "total_variation": {
              "score": 0.2490848180933528,
              "ci95": [
                0.19992744552797803,
                0.30308455010232166
              ]
            },
            "impossible_mass": {
              "score": 0.07797364672364672,
              "ci95": [
                0.033561030982905984,
                0.13547743055555547
              ]
            }
          },
          "minus_uniform_excess_brier": {
            "n": 96,
            "delta": 0.3189182046579024,
            "ci95": [
              0.24660323669824133,
              0.3833145855433969
            ]
          },
          "categories": {
            "composition": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.4963454938298113,
                "ci95": [
                  0.375599490327063,
                  0.6117664402910763
                ]
              },
              "expected_brier": {
                "score": 1.0095627662063544,
                "ci95": [
                  0.807666019051906,
                  1.2146011604773337
                ]
              },
              "total_variation": {
                "score": 0.530282289798541,
                "ci95": [
                  0.4117440277033012,
                  0.6495535628350143
                ]
              },
              "impossible_mass": {
                "score": 0.11548209268763297,
                "ci95": [
                  0,
                  0.31407599759958
                ]
              }
            },
            "daily_evidence": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.4245031940904254,
                "ci95": [
                  0.25331296107469853,
                  0.5959668444725154
                ]
              },
              "expected_brier": {
                "score": 0.8128926964467851,
                "ci95": [
                  0.624635659360778,
                  0.9750842599825481
                ]
              },
              "total_variation": {
                "score": 0.40403858490332517,
                "ci95": [
                  0.2874677400307427,
                  0.5124191488491036
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "direct": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.7062627036408151,
                "ci95": [
                  0.39967405078087836,
                  1.0385530740926077
                ]
              },
              "expected_brier": {
                "score": 1.041904120307482,
                "ci95": [
                  0.8029974303979556,
                  1.2448196147474457
                ]
              },
              "total_variation": {
                "score": 0.5936132753156476,
                "ci95": [
                  0.4731513693358782,
                  0.7223396623192829
                ]
              },
              "impossible_mass": {
                "score": 0.2353448422772747,
                "ci95": [
                  0.0451296088143589,
                  0.4581984342958348
                ]
              }
            },
            "disclosure": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.519484496890174,
                "ci95": [
                  0.437482539612009,
                  0.6136063140187343
                ]
              },
              "expected_brier": {
                "score": 0.9383425215815321,
                "ci95": [
                  0.8989882815322874,
                  0.9704498500339941
                ]
              },
              "total_variation": {
                "score": 0.4527291499745276,
                "ci95": [
                  0.41671864763586103,
                  0.48109234876874235
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "history": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.2661723960709872,
                "ci95": [
                  0.1580655175750782,
                  0.3503176238919696
                ]
              },
              "expected_brier": {
                "score": 0.7757140174944059,
                "ci95": [
                  0.5996295973437767,
                  0.9243917630563113
                ]
              },
              "total_variation": {
                "score": 0.3862470012436564,
                "ci95": [
                  0.273674792233247,
                  0.49044894297055225
                ]
              },
              "impossible_mass": {
                "score": 0.11526044295701122,
                "ci95": [
                  0,
                  0.2754774707356816
                ]
              }
            },
            "sequential": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.36468329692996665,
                "ci95": [
                  0.24957724338954534,
                  0.4951239654838767
                ]
              },
              "expected_brier": {
                "score": 0.8699221657399151,
                "ci95": [
                  0.7939955317179593,
                  0.952680523844846
                ]
              },
              "total_variation": {
                "score": 0.4309491606435203,
                "ci95": [
                  0.34711078044554133,
                  0.5189067413618409
                ]
              },
              "impossible_mass": {
                "score": 0.0745928120825903,
                "ci95": [
                  0,
                  0.1745470678953328
                ]
              }
            }
          }
        }
      }
    },
    "llama32_3b_public": {
      "label": "Llama-3.2-3B-Instruct (Unsloth BF16 distribution)",
      "receipt": {
        "raw_sha256": "a8530a864536228435b8eeb2b5a3f74468a0571dcddad10a4fde7b36e03f074f",
        "metadata_sha256": "39ea272b809d02a405c3163b9faf2a50d91489362cd36f4ceb6a6c37bbe77c1a",
        "count": 1152,
        "errors": 0
      },
      "metrics": {
        "cladder": {
          "original": {
            "n": 144,
            "correct": 72,
            "score": 0.5,
            "ci95": [
              0.41956388206388207,
              0.5810930611462525
            ],
            "errors": 0
          },
          "reversed": {
            "n": 144,
            "correct": 71,
            "score": 0.4930555555555556,
            "ci95": [
              0.4121417960513705,
              0.5724391126115262
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 144,
            "correct": 46,
            "score": 0.3194444444444444,
            "ci95": [
              0.2413793103448276,
              0.3971631205673759
            ]
          },
          "prediction_flips": 51,
          "strata": {
            "1": {
              "n": 48,
              "correct": 23,
              "score": 0.4791666666666667,
              "ci95": [
                0.32653061224489793,
                0.6170957446808507
              ],
              "errors": 0
            },
            "2": {
              "n": 48,
              "correct": 27,
              "score": 0.5625,
              "ci95": [
                0.4166666666666667,
                0.7083333333333334
              ],
              "errors": 0
            },
            "3": {
              "n": 48,
              "correct": 22,
              "score": 0.4583333333333333,
              "ci95": [
                0.3333333333333333,
                0.6041666666666666
              ],
              "errors": 0
            }
          }
        },
        "cruxeval": {
          "original": {
            "n": 128,
            "correct": 45,
            "score": 0.3515625,
            "ci95": [
              0.2734375,
              0.4296875
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 48,
            "score": 0.375,
            "ci95": [
              0.2890625,
              0.4609375
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 16,
            "score": 0.125,
            "ci95": [
              0.0703125,
              0.1796875
            ]
          },
          "prediction_flips": 84,
          "strata": {
            "all": {
              "n": 128,
              "correct": 45,
              "score": 0.3515625,
              "ci95": [
                0.2734375,
                0.4296875
              ],
              "errors": 0
            }
          },
          "uniform_choice_chance": 0.3639136904761905,
          "candidate_counts": {
            "2": 56,
            "3": 28,
            "4": 23,
            "5": 8,
            "6": 5,
            "7": 5,
            "8": 2,
            "10": 1
          }
        },
        "finentity": {
          "original": {
            "n": 128,
            "correct": 80,
            "score": 0.625,
            "ci95": [
              0.5390625,
              0.7109375
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 70,
            "score": 0.546875,
            "ci95": [
              0.4609375,
              0.6328125
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 52,
            "score": 0.40625,
            "ci95": [
              0.3203125,
              0.4921875
            ]
          },
          "prediction_flips": 52,
          "strata": {
            "Negative": {
              "n": 43,
              "correct": 31,
              "score": 0.7209302325581395,
              "ci95": [
                0.5813953488372093,
                0.8372093023255814
              ],
              "errors": 0
            },
            "Neutral": {
              "n": 42,
              "correct": 20,
              "score": 0.47619047619047616,
              "ci95": [
                0.3333333333333333,
                0.6190476190476191
              ],
              "errors": 0
            },
            "Positive": {
              "n": 43,
              "correct": 29,
              "score": 0.6744186046511628,
              "ci95": [
                0.5348837209302325,
                0.813953488372093
              ],
              "errors": 0
            }
          }
        },
        "when2call": {
          "original": {
            "n": 128,
            "correct": 35,
            "score": 0.2734375,
            "ci95": [
              0.203125,
              0.3515625
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 40,
            "score": 0.3125,
            "ci95": [
              0.234375,
              0.3984375
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 26,
            "score": 0.203125,
            "ci95": [
              0.1328125,
              0.2734375
            ]
          },
          "prediction_flips": 54,
          "strata": {
            "cannot_answer": {
              "n": 43,
              "correct": 0,
              "score": 0,
              "ci95": [
                0,
                0
              ],
              "errors": 0
            },
            "request_for_info": {
              "n": 42,
              "correct": 7,
              "score": 0.16666666666666666,
              "ci95": [
                0.07083333333333336,
                0.2857142857142857
              ],
              "errors": 0
            },
            "tool_call": {
              "n": 43,
              "correct": 28,
              "score": 0.6511627906976745,
              "ci95": [
                0.5116279069767442,
                0.7906976744186046
              ],
              "errors": 0
            }
          },
          "direct_false_selections": {
            "n": 128,
            "count": 36,
            "rate": 0.28125,
            "gold_positives": 0
          }
        },
        "known_distribution": {
          "n": 96,
          "valid": 96,
          "errors": 0,
          "paired_settings": 48,
          "excess_brier": {
            "score": 0.3796016725380868,
            "ci95": [
              0.3054813685341776,
              0.45905404493071605
            ]
          },
          "expected_brier": {
            "score": 0.8247494569254693,
            "ci95": [
              0.7318837307465494,
              0.9152649825265194
            ]
          },
          "total_variation": {
            "score": 0.4126297179506167,
            "ci95": [
              0.36220354216896034,
              0.46421872131566816
            ]
          },
          "impossible_mass": {
            "score": 0.06566362378505866,
            "ci95": [
              0.024628972179858634,
              0.11742888934014228
            ]
          },
          "uniform_all_candidates": {
            "n": 96,
            "valid": 96,
            "errors": 0,
            "paired_settings": 48,
            "excess_brier": {
              "score": 0.1439903922507942,
              "ci95": [
                0.09995881558499044,
                0.19733934748419735
              ]
            },
            "expected_brier": {
              "score": 0.5891381766381767,
              "ci95": [
                0.5522569444444446,
                0.6316912615740741
              ]
            },
            "total_variation": {
              "score": 0.2490848180933528,
              "ci95": [
                0.19992744552797803,
                0.30308455010232166
              ]
            },
            "impossible_mass": {
              "score": 0.07797364672364672,
              "ci95": [
                0.033561030982905984,
                0.13547743055555547
              ]
            }
          },
          "minus_uniform_excess_brier": {
            "n": 96,
            "delta": 0.23561128028729264,
            "ci95": [
              0.1464970888655711,
              0.3179383495179255
            ]
          },
          "categories": {
            "composition": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.3061860135292246,
                "ci95": [
                  0.18386255513961586,
                  0.4270401282956459
                ]
              },
              "expected_brier": {
                "score": 0.8194032859057678,
                "ci95": [
                  0.5063025640226261,
                  1.119695335664786
                ]
              },
              "total_variation": {
                "score": 0.40018736149908507,
                "ci95": [
                  0.24238797983356872,
                  0.5610613014826609
                ]
              },
              "impossible_mass": {
                "score": 0.18020122349765186,
                "ci95": [
                  0,
                  0.4460987983086974
                ]
              }
            },
            "daily_evidence": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.4674770501689833,
                "ci95": [
                  0.2767807320589181,
                  0.6769527648512301
                ]
              },
              "expected_brier": {
                "score": 0.8558665525253428,
                "ci95": [
                  0.6409455285195154,
                  1.0447715000843478
                ]
              },
              "total_variation": {
                "score": 0.42175937114313466,
                "ci95": [
                  0.2936308243095942,
                  0.5394658475885862
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "direct": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.5433728980017782,
                "ci95": [
                  0.2874161108335287,
                  0.8275916048676888
                ]
              },
              "expected_brier": {
                "score": 0.879014314668445,
                "ci95": [
                  0.5707970795634891,
                  1.1424309533786123
                ]
              },
              "total_variation": {
                "score": 0.5155735988268153,
                "ci95": [
                  0.38856557016968807,
                  0.6326050826751506
                ]
              },
              "impossible_mass": {
                "score": 0.14041491878788115,
                "ci95": [
                  0.03727129963743878,
                  0.2418749969069645
                ]
              }
            },
            "disclosure": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.2361205709832308,
                "ci95": [
                  0.12388401499637203,
                  0.3788966161516016
                ]
              },
              "expected_brier": {
                "score": 0.6549785956745888,
                "ci95": [
                  0.5579351136751479,
                  0.753391924785391
                ]
              },
              "total_variation": {
                "score": 0.2816431886955661,
                "ci95": [
                  0.1842925881179844,
                  0.38947927219131595
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "history": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.2520276468429946,
                "ci95": [
                  0.16842003066056513,
                  0.322766292472743
                ]
              },
              "expected_brier": {
                "score": 0.7615692682664134,
                "ci95": [
                  0.6236545668244482,
                  0.8881617527859286
                ]
              },
              "total_variation": {
                "score": 0.3682994821700386,
                "ci95": [
                  0.29881815393087907,
                  0.43175832806327397
                ]
              },
              "impossible_mass": {
                "score": 0.04201778487696906,
                "ci95": [
                  0,
                  0.0945260442353046
                ]
              }
            },
            "sequential": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.4724258557023094,
                "ci95": [
                  0.3433336826734061,
                  0.6427867376309626
                ]
              },
              "expected_brier": {
                "score": 0.9776647245122579,
                "ci95": [
                  0.8597390934217827,
                  1.104356179795631
                ]
              },
              "total_variation": {
                "score": 0.48831530536906076,
                "ci95": [
                  0.419578128365615,
                  0.5623621848394058
                ]
              },
              "impossible_mass": {
                "score": 0.03134781554784992,
                "ci95": [
                  0,
                  0.07411764212500833
                ]
              }
            }
          }
        }
      }
    },
    "qwen3_17b": {
      "label": "Qwen3-1.7B",
      "receipt": {
        "raw_sha256": "1db2032a4ad1a1d8ca64675d112330996f7981e83e93d8ccb3b1ff7e08ccbb21",
        "metadata_sha256": "c04535ad32b9d5d41f7ae73e95f1d74d80d1beaf57ef8141e82d1c8534bf4ed7",
        "count": 1152,
        "errors": 0
      },
      "metrics": {
        "cladder": {
          "original": {
            "n": 144,
            "correct": 69,
            "score": 0.4791666666666667,
            "ci95": [
              0.3986013986013986,
              0.5594405594405595
            ],
            "errors": 0
          },
          "reversed": {
            "n": 144,
            "correct": 72,
            "score": 0.5,
            "ci95": [
              0.4154929577464789,
              0.5874125874125874
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 144,
            "correct": 53,
            "score": 0.3680555555555556,
            "ci95": [
              0.2867132867132867,
              0.4482758620689655
            ]
          },
          "prediction_flips": 35,
          "strata": {
            "1": {
              "n": 48,
              "correct": 25,
              "score": 0.5208333333333334,
              "ci95": [
                0.3829787234042553,
                0.6595851063829786
              ],
              "errors": 0
            },
            "2": {
              "n": 48,
              "correct": 23,
              "score": 0.4791666666666667,
              "ci95": [
                0.3541666666666667,
                0.6041666666666666
              ],
              "errors": 0
            },
            "3": {
              "n": 48,
              "correct": 21,
              "score": 0.4375,
              "ci95": [
                0.3125,
                0.5625
              ],
              "errors": 0
            }
          }
        },
        "cruxeval": {
          "original": {
            "n": 128,
            "correct": 35,
            "score": 0.2734375,
            "ci95": [
              0.1953125,
              0.3515625
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 54,
            "score": 0.421875,
            "ci95": [
              0.3359375,
              0.5078125
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 15,
            "score": 0.1171875,
            "ci95": [
              0.0625,
              0.171875
            ]
          },
          "prediction_flips": 81,
          "strata": {
            "all": {
              "n": 128,
              "correct": 35,
              "score": 0.2734375,
              "ci95": [
                0.1953125,
                0.3515625
              ],
              "errors": 0
            }
          },
          "uniform_choice_chance": 0.3639136904761905,
          "candidate_counts": {
            "2": 56,
            "3": 28,
            "4": 23,
            "5": 8,
            "6": 5,
            "7": 5,
            "8": 2,
            "10": 1
          }
        },
        "finentity": {
          "original": {
            "n": 128,
            "correct": 57,
            "score": 0.4453125,
            "ci95": [
              0.359375,
              0.53125
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 50,
            "score": 0.390625,
            "ci95": [
              0.3046875,
              0.4765625
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 35,
            "score": 0.2734375,
            "ci95": [
              0.1953125,
              0.3515625
            ]
          },
          "prediction_flips": 39,
          "strata": {
            "Negative": {
              "n": 43,
              "correct": 19,
              "score": 0.4418604651162791,
              "ci95": [
                0.3023255813953488,
                0.6046511627906976
              ],
              "errors": 0
            },
            "Neutral": {
              "n": 42,
              "correct": 13,
              "score": 0.30952380952380953,
              "ci95": [
                0.16666666666666666,
                0.4523809523809524
              ],
              "errors": 0
            },
            "Positive": {
              "n": 43,
              "correct": 25,
              "score": 0.5813953488372093,
              "ci95": [
                0.4418604651162791,
                0.7209302325581395
              ],
              "errors": 0
            }
          }
        },
        "when2call": {
          "original": {
            "n": 128,
            "correct": 44,
            "score": 0.34375,
            "ci95": [
              0.2578125,
              0.4296875
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 44,
            "score": 0.34375,
            "ci95": [
              0.265625,
              0.4296875
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 38,
            "score": 0.296875,
            "ci95": [
              0.21875,
              0.3828125
            ]
          },
          "prediction_flips": 39,
          "strata": {
            "cannot_answer": {
              "n": 43,
              "correct": 4,
              "score": 0.09302325581395349,
              "ci95": [
                0.023255813953488372,
                0.18604651162790697
              ],
              "errors": 0
            },
            "request_for_info": {
              "n": 42,
              "correct": 1,
              "score": 0.023809523809523808,
              "ci95": [
                0,
                0.07142857142857142
              ],
              "errors": 0
            },
            "tool_call": {
              "n": 43,
              "correct": 39,
              "score": 0.9069767441860465,
              "ci95": [
                0.813953488372093,
                0.9767441860465116
              ],
              "errors": 0
            }
          },
          "direct_false_selections": {
            "n": 128,
            "count": 22,
            "rate": 0.171875,
            "gold_positives": 0
          }
        },
        "known_distribution": {
          "n": 96,
          "valid": 96,
          "errors": 0,
          "paired_settings": 48,
          "excess_brier": {
            "score": 0.5657946399022175,
            "ci95": [
              0.44563976842767555,
              0.6869396848144962
            ]
          },
          "expected_brier": {
            "score": 1.0109424242896,
            "ci95": [
              0.8751880317430326,
              1.1400450623657064
            ]
          },
          "total_variation": {
            "score": 0.5044753996494343,
            "ci95": [
              0.4351294878217941,
              0.5705228182137644
            ]
          },
          "impossible_mass": {
            "score": 0.03902364671226373,
            "ci95": [
              0.005667057672823128,
              0.08240083926266449
            ]
          },
          "uniform_all_candidates": {
            "n": 96,
            "valid": 96,
            "errors": 0,
            "paired_settings": 48,
            "excess_brier": {
              "score": 0.1439903922507942,
              "ci95": [
                0.09995881558499044,
                0.19733934748419735
              ]
            },
            "expected_brier": {
              "score": 0.5891381766381767,
              "ci95": [
                0.5522569444444446,
                0.6316912615740741
              ]
            },
            "total_variation": {
              "score": 0.2490848180933528,
              "ci95": [
                0.19992744552797803,
                0.30308455010232166
              ]
            },
            "impossible_mass": {
              "score": 0.07797364672364672,
              "ci95": [
                0.033561030982905984,
                0.13547743055555547
              ]
            }
          },
          "minus_uniform_excess_brier": {
            "n": 96,
            "delta": 0.42180424765142327,
            "ci95": [
              0.29295218842421755,
              0.5455504039118306
            ]
          },
          "categories": {
            "composition": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.37810080264707624,
                "ci95": [
                  0.20936353451389972,
                  0.5355035385488193
                ]
              },
              "expected_brier": {
                "score": 0.8913180750236193,
                "ci95": [
                  0.5435493777329093,
                  1.213203506719122
                ]
              },
              "total_variation": {
                "score": 0.46279358818692573,
                "ci95": [
                  0.2718785001872054,
                  0.636756759491361
                ]
              },
              "impossible_mass": {
                "score": 0.10481338562064615,
                "ci95": [
                  0,
                  0.2970935134517773
                ]
              }
            },
            "daily_evidence": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.6487465000535872,
                "ci95": [
                  0.3688106238777683,
                  0.9522289089160082
                ]
              },
              "expected_brier": {
                "score": 1.0371360024099467,
                "ci95": [
                  0.7439719226897396,
                  1.3225980134588236
                ]
              },
              "total_variation": {
                "score": 0.517170786813037,
                "ci95": [
                  0.36773938009161566,
                  0.6612640669064951
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "direct": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.7566449655930083,
                "ci95": [
                  0.24845286825725066,
                  1.312694991496466
                ]
              },
              "expected_brier": {
                "score": 1.092286382259675,
                "ci95": [
                  0.5432922549122333,
                  1.607705973106307
                ]
              },
              "total_variation": {
                "score": 0.5456491053727134,
                "ci95": [
                  0.2704139539415348,
                  0.8039998830268199
                ]
              },
              "impossible_mass": {
                "score": 0.1050687698593985,
                "ci95": [
                  2.364166855392344e-8,
                  0.24991930392205486
                ]
              }
            },
            "disclosure": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.5555688426474366,
                "ci95": [
                  0.4535795648958006,
                  0.6598742630010909
                ]
              },
              "expected_brier": {
                "score": 0.9744268673387946,
                "ci95": [
                  0.9058028499252853,
                  1.0448157374159028
                ]
              },
              "total_variation": {
                "score": 0.48373278668431763,
                "ci95": [
                  0.4344907030965348,
                  0.5281292151299687
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "history": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.3819108730299729,
                "ci95": [
                  0.24707405258633322,
                  0.5058414658738453
                ]
              },
              "expected_brier": {
                "score": 0.8914524944533916,
                "ci95": [
                  0.674244115312286,
                  1.095852334945545
                ]
              },
              "total_variation": {
                "score": 0.44397308090138476,
                "ci95": [
                  0.33107677423319987,
                  0.5578589222460937
                ]
              },
              "impossible_mass": {
                "score": 0.018556114731344338,
                "ci95": [
                  0,
                  0.04365649119759099
                ]
              }
            },
            "sequential": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.673795855442224,
                "ci95": [
                  0.4179820675483123,
                  0.9444813933837308
                ]
              },
              "expected_brier": {
                "score": 1.1790347242521724,
                "ci95": [
                  0.9505804462198371,
                  1.4119612325614168
                ]
              },
              "total_variation": {
                "score": 0.5735330499382275,
                "ci95": [
                  0.4479876941798706,
                  0.7037176782339468
                ]
              },
              "impossible_mass": {
                "score": 0.005703610062193403,
                "ci95": [
                  0,
                  0.016931567697962523
                ]
              }
            }
          }
        }
      }
    },
    "qwen3_8b": {
      "label": "Qwen3-8B",
      "receipt": {
        "raw_sha256": "3361ebd5e6025fb059bb678176439b06e9e734069230c74cfb36143efed61955",
        "metadata_sha256": "a411bec64f017cbc4a456ee890838e0eb827ae59a7c98a33df4734ae84ef36f4",
        "count": 1152,
        "errors": 0
      },
      "metrics": {
        "cladder": {
          "original": {
            "n": 144,
            "correct": 76,
            "score": 0.5277777777777778,
            "ci95": [
              0.4489619985925405,
              0.6111111111111112
            ],
            "errors": 0
          },
          "reversed": {
            "n": 144,
            "correct": 76,
            "score": 0.5277777777777778,
            "ci95": [
              0.4482577767060526,
              0.6083916083916084
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 144,
            "correct": 34,
            "score": 0.2361111111111111,
            "ci95": [
              0.1702127659574468,
              0.3103448275862069
            ]
          },
          "prediction_flips": 84,
          "strata": {
            "1": {
              "n": 48,
              "correct": 24,
              "score": 0.5,
              "ci95": [
                0.3541666666666667,
                0.6458333333333334
              ],
              "errors": 0
            },
            "2": {
              "n": 48,
              "correct": 27,
              "score": 0.5625,
              "ci95": [
                0.4166666666666667,
                0.6875
              ],
              "errors": 0
            },
            "3": {
              "n": 48,
              "correct": 25,
              "score": 0.5208333333333334,
              "ci95": [
                0.375,
                0.6666666666666666
              ],
              "errors": 0
            }
          }
        },
        "cruxeval": {
          "original": {
            "n": 128,
            "correct": 49,
            "score": 0.3828125,
            "ci95": [
              0.296875,
              0.4609375
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 54,
            "score": 0.421875,
            "ci95": [
              0.3359375,
              0.5078125
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 27,
            "score": 0.2109375,
            "ci95": [
              0.140625,
              0.28125
            ]
          },
          "prediction_flips": 69,
          "strata": {
            "all": {
              "n": 128,
              "correct": 49,
              "score": 0.3828125,
              "ci95": [
                0.296875,
                0.4609375
              ],
              "errors": 0
            }
          },
          "uniform_choice_chance": 0.3639136904761905,
          "candidate_counts": {
            "2": 56,
            "3": 28,
            "4": 23,
            "5": 8,
            "6": 5,
            "7": 5,
            "8": 2,
            "10": 1
          }
        },
        "finentity": {
          "original": {
            "n": 128,
            "correct": 102,
            "score": 0.796875,
            "ci95": [
              0.71875,
              0.859375
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 102,
            "score": 0.796875,
            "ci95": [
              0.7265625,
              0.8671875
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 95,
            "score": 0.7421875,
            "ci95": [
              0.6640625,
              0.8203125
            ]
          },
          "prediction_flips": 15,
          "strata": {
            "Negative": {
              "n": 43,
              "correct": 39,
              "score": 0.9069767441860465,
              "ci95": [
                0.813953488372093,
                0.9767441860465116
              ],
              "errors": 0
            },
            "Neutral": {
              "n": 42,
              "correct": 24,
              "score": 0.5714285714285714,
              "ci95": [
                0.40476190476190477,
                0.7142857142857143
              ],
              "errors": 0
            },
            "Positive": {
              "n": 43,
              "correct": 39,
              "score": 0.9069767441860465,
              "ci95": [
                0.813953488372093,
                0.9767441860465116
              ],
              "errors": 0
            }
          }
        },
        "when2call": {
          "original": {
            "n": 128,
            "correct": 57,
            "score": 0.4453125,
            "ci95": [
              0.359375,
              0.53125
            ],
            "errors": 0
          },
          "reversed": {
            "n": 128,
            "correct": 61,
            "score": 0.4765625,
            "ci95": [
              0.390625,
              0.5625
            ],
            "errors": 0
          },
          "paired_both_correct": {
            "n": 128,
            "correct": 50,
            "score": 0.390625,
            "ci95": [
              0.3046875,
              0.4765625
            ]
          },
          "prediction_flips": 24,
          "strata": {
            "cannot_answer": {
              "n": 43,
              "correct": 4,
              "score": 0.09302325581395349,
              "ci95": [
                0.023255813953488372,
                0.18604651162790697
              ],
              "errors": 0
            },
            "request_for_info": {
              "n": 42,
              "correct": 10,
              "score": 0.23809523809523808,
              "ci95": [
                0.11904761904761904,
                0.35714285714285715
              ],
              "errors": 0
            },
            "tool_call": {
              "n": 43,
              "correct": 43,
              "score": 1,
              "ci95": [
                1,
                1
              ],
              "errors": 0
            }
          },
          "direct_false_selections": {
            "n": 128,
            "count": 6,
            "rate": 0.046875,
            "gold_positives": 0
          }
        },
        "known_distribution": {
          "n": 96,
          "valid": 96,
          "errors": 0,
          "paired_settings": 48,
          "excess_brier": {
            "score": 0.47537556433769135,
            "ci95": [
              0.3878008051862168,
              0.5627995106841388
            ]
          },
          "expected_brier": {
            "score": 0.9205233487250738,
            "ci95": [
              0.7952328126185755,
              1.035524954539892
            ]
          },
          "total_variation": {
            "score": 0.45986711493911264,
            "ci95": [
              0.39325125101086655,
              0.5215012028054075
            ]
          },
          "impossible_mass": {
            "score": 0.021689239830020857,
            "ci95": [
              0.0019070679961196496,
              0.04852011887598158
            ]
          },
          "uniform_all_candidates": {
            "n": 96,
            "valid": 96,
            "errors": 0,
            "paired_settings": 48,
            "excess_brier": {
              "score": 0.1439903922507942,
              "ci95": [
                0.09995881558499044,
                0.19733934748419735
              ]
            },
            "expected_brier": {
              "score": 0.5891381766381767,
              "ci95": [
                0.5522569444444446,
                0.6316912615740741
              ]
            },
            "total_variation": {
              "score": 0.2490848180933528,
              "ci95": [
                0.19992744552797803,
                0.30308455010232166
              ]
            },
            "impossible_mass": {
              "score": 0.07797364672364672,
              "ci95": [
                0.033561030982905984,
                0.13547743055555547
              ]
            }
          },
          "minus_uniform_excess_brier": {
            "n": 96,
            "delta": 0.33138517208689716,
            "ci95": [
              0.2214494632001824,
              0.43622900076931836
            ]
          },
          "categories": {
            "composition": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.41842585955765943,
                "ci95": [
                  0.2148825963307316,
                  0.595106595907898
                ]
              },
              "expected_brier": {
                "score": 0.9316431319342027,
                "ci95": [
                  0.5504282300034455,
                  1.2688786257689335
                ]
              },
              "total_variation": {
                "score": 0.4818274576461068,
                "ci95": [
                  0.2715901188878043,
                  0.6773949020446651
                ]
              },
              "impossible_mass": {
                "score": 0.09520801951167075,
                "ci95": [
                  0,
                  0.22335023665769177
                ]
              }
            },
            "daily_evidence": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.5275239562873666,
                "ci95": [
                  0.27496731248895195,
                  0.778692557447401
                ]
              },
              "expected_brier": {
                "score": 0.9159134586437262,
                "ci95": [
                  0.6455781423683729,
                  1.1604610701342037
                ]
              },
              "total_variation": {
                "score": 0.44421893179366734,
                "ci95": [
                  0.2820918082463811,
                  0.5912334200739133
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "direct": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.5218822755520941,
                "ci95": [
                  0.27721118539076006,
                  0.7433143217986541
                ]
              },
              "expected_brier": {
                "score": 0.8575236922187608,
                "ci95": [
                  0.4862905530324336,
                  1.1972933648799886
                ]
              },
              "total_variation": {
                "score": 0.4300064676128563,
                "ci95": [
                  0.2473884693825073,
                  0.5955860014144756
                ]
              },
              "impossible_mass": {
                "score": 9.114244008829676e-8,
                "ci95": [
                  5.44294729482252e-11,
                  2.2816532015004008e-7
                ]
              }
            },
            "disclosure": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.3281525604388471,
                "ci95": [
                  0.18439986103902103,
                  0.4413511949083437
                ]
              },
              "expected_brier": {
                "score": 0.7470105851302051,
                "ci95": [
                  0.5505865976650736,
                  0.9235400240029864
                ]
              },
              "total_variation": {
                "score": 0.35574605883628196,
                "ci95": [
                  0.254066202904525,
                  0.4471000615416315
                ]
              },
              "impossible_mass": {
                "score": 0,
                "ci95": [
                  0,
                  0
                ]
              }
            },
            "history": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.40244963169646364,
                "ci95": [
                  0.25260266527971065,
                  0.5260401259946407
                ]
              },
              "expected_brier": {
                "score": 0.9119912531198824,
                "ci95": [
                  0.6850306064021011,
                  1.1036556114588172
                ]
              },
              "total_variation": {
                "score": 0.4646993493711,
                "ci95": [
                  0.34102637060273866,
                  0.571220787230528
                ]
              },
              "impossible_mass": {
                "score": 0.03463886244908362,
                "ci95": [
                  0,
                  0.0811005011329345
                ]
              }
            },
            "sequential": {
              "n": 16,
              "valid": 16,
              "errors": 0,
              "paired_settings": 8,
              "excess_brier": {
                "score": 0.6538191024937173,
                "ci95": [
                  0.4195244800764231,
                  0.8885714877641562
                ]
              },
              "expected_brier": {
                "score": 1.1590579713036657,
                "ci95": [
                  0.9425541410768289,
                  1.373582424971019
                ]
              },
              "total_variation": {
                "score": 0.5827044243746635,
                "ci95": [
                  0.47122450806727834,
                  0.692020647692232
                ]
              },
              "impossible_mass": {
                "score": 0.0002884658769306894,
                "ci95": [
                  0,
                  0.0007969407567548748
                ]
              }
            }
          }
        }
      }
    }
  },
  "speed": {
    "intern_08b": {
      "status": "DONE",
      "model": "intern_08b",
      "workloads": {
        "one_field": {
          "n": 100,
          "fields": 1,
          "mean_ms": 26.427123780000002,
          "median_ms": 26.214923,
          "p95_ms": 28.709862599999997,
          "mean_round_ci95_ms": [
            26.36787504,
            26.509690010000003
          ],
          "requests_per_second": 37.83991055268747,
          "fields_per_second": 37.83991055268747,
          "round_mean_ms": [
            26.412260250000003,
            26.58168815,
            26.377398499999998,
            26.425987099999997,
            26.338284899999998
          ],
          "distinct_prompt_encodings": 1,
          "prompt_token_lengths": [
            255
          ],
          "cpu_contention": {
            "interval_seconds": 3.8573595150373876,
            "core_busy_fraction": {
              "20": 0.6921833418926524,
              "21": 0.049256492494233695,
              "22": 0.11925256077551316,
              "23": 0.028516916707187926,
              "84": 0.0025924469733807206,
              "85": 0.007777340920142162,
              "86": 0.0025924469733807206,
              "87": 0.0025924469733807206
            },
            "worker_cpu_seconds": 2.69,
            "other_busy_fraction_on_worker_cores": 0.04796026900754335,
            "smt_sibling_busy_fraction": 0.003888670460071081
          }
        },
        "three_fields": {
          "n": 100,
          "fields": 3,
          "mean_ms": 32.76918626,
          "median_ms": 32.609107,
          "p95_ms": 33.12209214999999,
          "mean_round_ci95_ms": [
            32.75160769,
            32.79242649
          ],
          "requests_per_second": 30.51647337427658,
          "fields_per_second": 91.54942012282974,
          "round_mean_ms": [
            32.75315395,
            32.7526653,
            32.77932489999999,
            32.74977695,
            32.8110102
          ],
          "distinct_prompt_encodings": 1,
          "prompt_token_lengths": [
            348
          ],
          "cpu_contention": {
            "interval_seconds": 4.492521625012159,
            "core_busy_fraction": {
              "20": 0.7345540601579337,
              "21": 0.11129606972089906,
              "22": 0.04451842788835962,
              "23": 0.011129606972089905,
              "84": 0,
              "85": 0,
              "86": 0,
              "87": 0
            },
            "worker_cpu_seconds": 3.31,
            "other_busy_fraction_on_worker_cores": 0.041179545796732636,
            "smt_sibling_busy_fraction": 0
          }
        },
        "eight_fields": {
          "n": 100,
          "fields": 8,
          "mean_ms": 48.23476776999999,
          "median_ms": 48.0602095,
          "p95_ms": 49.30125195,
          "mean_round_ci95_ms": [
            48.1976084,
            48.27192713999999
          ],
          "requests_per_second": 20.73193354570182,
          "fields_per_second": 165.85546836561457,
          "round_mean_ms": [
            48.2831925,
            48.1936319,
            48.2261064,
            48.28357215,
            48.1873359
          ],
          "distinct_prompt_encodings": 1,
          "prompt_token_lengths": [
            593
          ],
          "cpu_contention": {
            "interval_seconds": 6.023701285012066,
            "core_busy_fraction": {
              "20": 0.8068129161870059,
              "21": 0.03154206873982122,
              "22": 0.05976391971755599,
              "23": 0.028221850977734775,
              "84": 0.001660108881043222,
              "85": 0.001660108881043222,
              "86": 0.00830054440521611,
              "87": 0.001660108881043222
            },
            "worker_cpu_seconds": 4.85,
            "other_busy_fraction_on_worker_cores": 0.030296987079038818,
            "smt_sibling_busy_fraction": 0.003320217762086444
          }
        }
      },
      "raw_sha256": "29013f26dbed836a1bd219cc0b31ed4f9ded6a8c9dc5b418486f7359e2818bbd",
      "metadata_sha256": "865c993d7d059d8fd918c7eda03b834c8e88887af18fed20011675f5b84c918b",
      "telemetry_sha256": "51bc09733d6fadcd4b369d9cb970ff38d386b19453610bd1cee3617f01b9a110",
      "finished_at": "2026-10-05T13:18:45.588942+00:00",
      "hardware": "NVIDIA A100 80GB PCIe",
      "adapter": "intern_official_hf",
      "dtype": "bfloat16",
      "source": "research/latency_v2/results/intern_08b/summary.json",
      "protocol": "research/latency_v2/PROTOCOL.md"
    },
    "intern_2b": {
      "status": "DONE",
      "model": "intern_2b",
      "workloads": {
        "one_field": {
          "n": 100,
          "fields": 1,
          "mean_ms": 42.98134063,
          "median_ms": 42.868157999999994,
          "p95_ms": 43.8840074,
          "mean_round_ci95_ms": [
            42.91777244,
            43.04490882
          ],
          "requests_per_second": 23.26590993539235,
          "fields_per_second": 23.26590993539235,
          "round_mean_ms": [
            42.883880649999995,
            43.05162005000001,
            43.066691000000006,
            42.987922000000005,
            42.91658945
          ],
          "distinct_prompt_encodings": 1,
          "prompt_token_lengths": [
            255
          ],
          "cpu_contention": {
            "interval_seconds": 5.531490524765104,
            "core_busy_fraction": {
              "20": 0.009039154957627495,
              "21": 0.7827908193305411,
              "22": 0.10846985949152993,
              "23": 0.023501802889831485,
              "84": 0.003615661983050998,
              "85": 0.001807830991525499,
              "86": 0,
              "87": 0
            },
            "worker_cpu_seconds": 4.33,
            "other_busy_fraction_on_worker_cores": 0.03525270433474724,
            "smt_sibling_busy_fraction": 0.001355873243644124
          }
        },
        "three_fields": {
          "n": 100,
          "fields": 3,
          "mean_ms": 47.21791988000001,
          "median_ms": 47.7472655,
          "p95_ms": 49.69420685,
          "mean_round_ci95_ms": [
            45.711103179999995,
            48.031923369999994
          ],
          "requests_per_second": 21.178400118883,
          "fields_per_second": 63.535200356649,
          "round_mean_ms": [
            47.929288,
            47.91917735,
            48.12116114999999,
            47.8668454,
            44.2531275
          ],
          "distinct_prompt_encodings": 1,
          "prompt_token_lengths": [
            348
          ],
          "cpu_contention": {
            "interval_seconds": 5.917993112932891,
            "core_busy_fraction": {
              "20": 0.006759048082125336,
              "21": 0.8043267217729149,
              "22": 0.09293691112922338,
              "23": 0.02872595434903268,
              "84": 0.006759048082125336,
              "85": 0,
              "86": 0.001689762020531334,
              "87": 0.005069286061594002
            },
            "worker_cpu_seconds": 4.76,
            "other_busy_fraction_on_worker_cores": 0.03210547839009534,
            "smt_sibling_busy_fraction": 0.003379524041062668
          }
        },
        "eight_fields": {
          "n": 100,
          "fields": 8,
          "mean_ms": 54.55080113000001,
          "median_ms": 54.391701999999995,
          "p95_ms": 54.930698449999994,
          "mean_round_ci95_ms": [
            54.52816863,
            54.57856663
          ],
          "requests_per_second": 18.33153646299163,
          "fields_per_second": 146.65229170393303,
          "round_mean_ms": [
            54.603282549999996,
            54.55849775,
            54.524487750000006,
            54.52413005,
            54.543607550000004
          ],
          "distinct_prompt_encodings": 1,
          "prompt_token_lengths": [
            593
          ],
          "cpu_contention": {
            "interval_seconds": 6.683865564875305,
            "core_busy_fraction": {
              "20": 0.014961402055348733,
              "21": 0.8213809728386454,
              "22": 0.0837838515099529,
              "23": 0.011969121644278987,
              "84": 0.0059845608221394935,
              "85": 0.0029922804110697468,
              "86": 0.0029922804110697468,
              "87": 0.0014961402055348734
            },
            "worker_cpu_seconds": 5.49,
            "other_busy_fraction_on_worker_cores": 0.027678593802395162,
            "smt_sibling_busy_fraction": 0.0033663154624534648
          }
        }
      },
      "raw_sha256": "5bb3e2011bc2d816c4ecc8b88536f669056b87f79e14d1128e63e8cbe939feb8",
      "metadata_sha256": "dec5c19065d3129db420ffaf994416dbceb6d1c631b1be66141496fb7cc81927",
      "telemetry_sha256": "e1265c2608ea734babe1fcf2f5bab8c8127ddb4d97dc829ffb4fd3c3ea61bba3",
      "finished_at": "2026-10-05T13:19:24.256575+00:00",
      "hardware": "NVIDIA A100 80GB PCIe",
      "adapter": "intern_official_hf",
      "dtype": "bfloat16",
      "source": "research/latency_v2/results/intern_2b/summary.json",
      "protocol": "research/latency_v2/PROTOCOL.md"
    },
    "intern_4b": {
      "status": "DONE",
      "model": "intern_4b",
      "workloads": {
        "one_field": {
          "n": 100,
          "fields": 1,
          "mean_ms": 55.83101796000001,
          "median_ms": 55.974517,
          "p95_ms": 58.416013449999994,
          "mean_round_ci95_ms": [
            55.03401093,
            56.259888190000005
          ],
          "requests_per_second": 17.911190527037277,
          "fields_per_second": 17.911190527037277,
          "round_mean_ms": [
            56.30746355,
            56.218946,
            56.1808315,
            56.19621880000001,
            54.25162995000001
          ],
          "distinct_prompt_encodings": 1,
          "prompt_token_lengths": [
            255
          ],
          "cpu_contention": {
            "interval_seconds": 6.798845157958567,
            "core_busy_fraction": {
              "20": 0.033829274627731075,
              "21": 0.06030435911899886,
              "22": 0.03971262673690169,
              "23": 0.8251401333111796,
              "84": 0.005883352109170621,
              "85": 0,
              "86": 0,
              "87": 0.0029416760545853105
            },
            "worker_cpu_seconds": 5.62,
            "other_busy_fraction_on_worker_cores": 0.033093855614084754,
            "smt_sibling_busy_fraction": 0.0022062570409389828
          }
        },
        "three_fields": {
          "n": 100,
          "fields": 3,
          "mean_ms": 62.29413261999999,
          "median_ms": 62.1125105,
          "p95_ms": 62.487936,
          "mean_round_ci95_ms": [
            62.279001490000006,
            62.30680409
          ],
          "requests_per_second": 16.052876217092432,
          "fields_per_second": 48.158628651277304,
          "round_mean_ms": [
            62.31363740000002,
            62.28728745000001,
            62.30468555,
            62.297374549999994,
            62.26767815000001
          ],
          "distinct_prompt_encodings": 1,
          "prompt_token_lengths": [
            348
          ],
          "cpu_contention": {
            "interval_seconds": 7.476346523035318,
            "core_busy_fraction": {
              "20": 0.020063275496639443,
              "21": 0.05885227479014237,
              "22": 0.04146410269305485,
              "23": 0.8373073640597528,
              "84": 0.005350206799103852,
              "85": 0.001337551699775963,
              "86": 0.002675103399551926,
              "87": 0.001337551699775963
            },
            "worker_cpu_seconds": 6.27,
            "other_busy_fraction_on_worker_cores": 0.029760525320015194,
            "smt_sibling_busy_fraction": 0.002675103399551926
          }
        },
        "eight_fields": {
          "n": 100,
          "fields": 8,
          "mean_ms": 93.97478066999999,
          "median_ms": 93.78115,
          "p95_ms": 94.26965899999999,
          "mean_round_ci95_ms": [
            93.9529221415,
            93.99809776999999
          ],
          "requests_per_second": 10.641152795148098,
          "fields_per_second": 85.12922236118479,
          "round_mean_ms": [
            93.94421405,
            93.96875705,
            93.99267425,
            93.95059975,
            94.01765825
          ],
          "distinct_prompt_encodings": 1,
          "prompt_token_lengths": [
            593
          ],
          "cpu_contention": {
            "interval_seconds": 10.613127864897251,
            "core_busy_fraction": {
              "20": 0.888522226438972,
              "21": 0.029209108186222837,
              "22": 0.021671273815584688,
              "23": 0.023555732408244223,
              "84": 0.0018844585926595379,
              "85": 0.0018844585926595379,
              "86": 0.0018844585926595379,
              "87": 0.0009422292963297689
            },
            "worker_cpu_seconds": 9.44,
            "other_busy_fraction_on_worker_cores": 0.01837347127843048,
            "smt_sibling_busy_fraction": 0.0016489012685770958
          }
        }
      },
      "raw_sha256": "42fa721c3f16470d501b94c80744b874c434fe482b828843762a3f7599afe761",
      "metadata_sha256": "f6f658a60281727f46073b2b72f8ca7cb768fa4277894556b2c5d78f46fcd737",
      "telemetry_sha256": "424975a22c307b41ab53017e576977e4f31935e32e1a0ad85fbf53d47e4e9221",
      "finished_at": "2026-10-05T13:20:57.788296+00:00",
      "hardware": "NVIDIA A100 80GB PCIe",
      "adapter": "intern_official_hf",
      "dtype": "bfloat16",
      "source": "research/latency_v2/results/intern_4b/summary.json",
      "protocol": "research/latency_v2/PROTOCOL.md"
    },
    "kev_08b": {
      "status": "DONE",
      "model": "kev_08b",
      "workloads": {
        "one_field": {
          "n": 100,
          "fields": 1,
          "mean_ms": 42.597877880000006,
          "median_ms": 42.339505,
          "p95_ms": 44.287744599999996,
          "mean_round_ci95_ms": [
            42.52666386,
            42.69990876999999
          ],
          "requests_per_second": 23.475347828759023,
          "fields_per_second": 23.475347828759023,
          "round_mean_ms": [
            42.5579663,
            42.78116455,
            42.494208349999994,
            42.543468149999995,
            42.61258205
          ],
          "distinct_prompt_encodings": 1,
          "prompt_token_lengths": [
            115
          ],
          "cpu_contention": {
            "interval_seconds": 5.498886509332806,
            "core_busy_fraction": {
              "20": 0.7801579451983484,
              "21": 0.15639529903743113,
              "22": 0.021822599865688064,
              "23": 0.009092749944036695,
              "84": 0.0036370999776146776,
              "85": 0.0018185499888073388,
              "86": 0.01454839991045871,
              "87": 0
            },
            "worker_cpu_seconds": 4.3,
            "other_busy_fraction_on_worker_cores": 0.04637302471458716,
            "smt_sibling_busy_fraction": 0.005001012469220182
          }
        },
        "three_fields": {
          "n": 100,
          "fields": 3,
          "mean_ms": 45.081031700000004,
          "median_ms": 43.0117995,
          "p95_ms": 46.380637750000005,
          "mean_round_ci95_ms": [
            43.20908352000001,
            48.70776451999999
          ],
          "requests_per_second": 22.18227849474882,
          "fields_per_second": 66.54683548424647,
          "round_mean_ms": [
            43.1182474,
            52.2986246,
            43.3453377,
            43.24804135,
            43.39490744999999
          ],
          "distinct_prompt_encodings": 1,
          "prompt_token_lengths": [
            172
          ],
          "cpu_contention": {
            "interval_seconds": 5.619477470871061,
            "core_busy_fraction": {
              "20": 0.20642488665768977,
              "21": 0.04448812212450211,
              "22": 0.016015723964820757,
              "23": 0.6530856327876909,
              "84": 0.003559049769960169,
              "85": 0,
              "86": 0,
              "87": 0
            },
            "worker_cpu_seconds": 4.55,
            "other_busy_fraction_on_worker_cores": 0.02758263571719131,
            "smt_sibling_busy_fraction": 0.0008897624424900422
          }
        },
        "eight_fields": {
          "n": 100,
          "fields": 8,
          "mean_ms": 48.434735810000014,
          "median_ms": 48.1718835,
          "p95_ms": 51.29846875,
          "mean_round_ci95_ms": [
            48.37743886999999,
            48.49115125000001
          ],
          "requests_per_second": 20.646339518043508,
          "fields_per_second": 165.17071614434806,
          "round_mean_ms": [
            48.346713949999994,
            48.48080125,
            48.49911135000001,
            48.48836615,
            48.358686350000006
          ],
          "distinct_prompt_encodings": 1,
          "prompt_token_lengths": [
            328
          ],
          "cpu_contention": {
            "interval_seconds": 5.942849627695978,
            "core_busy_fraction": {
              "20": 0.074038555165426,
              "21": 0.0185096387913565,
              "22": 0.010096166613467182,
              "23": 0.8211548845619974,
              "84": 0.006730777742311455,
              "85": 0,
              "86": 0,
              "87": 0
            },
            "worker_cpu_seconds": 4.88,
            "other_busy_fraction_on_worker_cores": 0.025661090142562435,
            "smt_sibling_busy_fraction": 0.0016826944355778637
          }
        }
      },
      "raw_sha256": "5aed9dc2ca0490d9aa3c57bff2fdbfca8820bfd58703ad1bf0d35518e3197c5c",
      "metadata_sha256": "77404b72e6673f81d0dd84b7fbf90d7c83f5451c1e0c158fd6fe792e370da14a",
      "telemetry_sha256": "846f6b3963f04070403f3e9b4bee53db469f350b72bb5f013d8ee5b2b52c3d67",
      "finished_at": "2026-10-05T13:21:34.635252+00:00",
      "hardware": "NVIDIA A100 80GB PCIe",
      "adapter": "kev_official_pointer",
      "dtype": "bfloat16",
      "source": "research/latency_v2/results/kev_08b/summary.json",
      "protocol": "research/latency_v2/PROTOCOL.md"
    },
    "kev_4b": {
      "status": "DONE",
      "model": "kev_4b",
      "workloads": {
        "one_field": {
          "n": 100,
          "fields": 1,
          "mean_ms": 99.23258267999998,
          "median_ms": 101.27772999999999,
          "p95_ms": 103.94928605,
          "mean_round_ci95_ms": [
            94.38135412,
            101.90699549000001
          ],
          "requests_per_second": 10.077335215840822,
          "fields_per_second": 10.077335215840822,
          "round_mean_ms": [
            102.28915915000002,
            101.64494855,
            101.61142674999999,
            101.05607325,
            89.5613057
          ],
          "distinct_prompt_encodings": 1,
          "prompt_token_lengths": [
            115
          ],
          "cpu_contention": {
            "interval_seconds": 11.19232222577557,
            "core_busy_fraction": {
              "20": 0.7085214167384725,
              "21": 0.2135392416147477,
              "22": 0.042886542248986984,
              "23": 0.012508574822621205,
              "84": 0.004467348150936144,
              "85": 0.004467348150936144,
              "86": 0.0035738785207489153,
              "87": 0.004467348150936144
            },
            "worker_cpu_seconds": 9.96,
            "other_busy_fraction_on_worker_cores": 0.021890005939587075,
            "smt_sibling_busy_fraction": 0.004243980743389337
          }
        },
        "three_fields": {
          "n": 100,
          "fields": 3,
          "mean_ms": 64.4697318,
          "median_ms": 64.288661,
          "p95_ms": 66.01180344999999,
          "mean_round_ci95_ms": [
            64.42642874,
            64.514738
          ],
          "requests_per_second": 15.511154957216682,
          "fields_per_second": 46.53346487165005,
          "round_mean_ms": [
            64.45347315000001,
            64.53651239999999,
            64.44770079999999,
            64.39452065,
            64.516452
          ],
          "distinct_prompt_encodings": 1,
          "prompt_token_lengths": [
            172
          ],
          "cpu_contention": {
            "interval_seconds": 7.6181500176899135,
            "core_busy_fraction": {
              "20": 0.8492875547181634,
              "21": 0.047255567186791164,
              "22": 0.03544167539009337,
              "23": 0.014439201084852856,
              "84": 0.0026253092881550646,
              "85": 0.010501237152620259,
              "86": 0.006563273220387662,
              "87": 0.009188582508542728
            },
            "worker_cpu_seconds": 6.49,
            "other_busy_fraction_on_worker_cores": 0.023627783593395575,
            "smt_sibling_busy_fraction": 0.007219600542426428
          }
        },
        "eight_fields": {
          "n": 100,
          "fields": 8,
          "mean_ms": 87.27558497999999,
          "median_ms": 87.124052,
          "p95_ms": 87.58579189999999,
          "mean_round_ci95_ms": [
            87.24084712999999,
            87.29784089
          ],
          "requests_per_second": 11.45795814750665,
          "fields_per_second": 91.6636651800532,
          "round_mean_ms": [
            87.29047175000001,
            87.3013538,
            87.28032275000001,
            87.29801255,
            87.20776405000001
          ],
          "distinct_prompt_encodings": 1,
          "prompt_token_lengths": [
            328
          ],
          "cpu_contention": {
            "interval_seconds": 9.9217598028481,
            "core_busy_fraction": {
              "20": 0.8808921172926532,
              "21": 0.05442582875721198,
              "22": 0.01411040004816607,
              "23": 0.0040315428709045914,
              "84": 0.005039428588630739,
              "85": 0.009070971459535329,
              "86": 0.016126171483618366,
              "87": 0.009070971459535329
            },
            "worker_cpu_seconds": 8.77,
            "other_busy_fraction_on_worker_cores": 0.017386028630776082,
            "smt_sibling_busy_fraction": 0.00982688574782994
          }
        }
      },
      "raw_sha256": "e75351132242f23d8b4727cc639fa4342fcc7168dd1c5f1db4c5cd76711a613e",
      "metadata_sha256": "673085db369ae5c22913e4900eb27e74187e92fc9c8c7e655e3c7e835f04b964",
      "telemetry_sha256": "cdec34ee1c15424133d82ae0159bb7486cec7e891ba147f6c5501e0e4913cd10",
      "finished_at": "2026-10-05T13:49:19.402866+00:00",
      "hardware": "NVIDIA A100 80GB PCIe",
      "adapter": "kev_official_pointer",
      "dtype": "bfloat16",
      "source": "research/latency_v2/results/kev_4b/summary.json",
      "protocol": "research/latency_v2/PROTOCOL.md"
    },
    "kev_9b": {
      "status": "DONE",
      "model": "kev_9b",
      "workloads": {
        "one_field": {
          "n": 100,
          "fields": 1,
          "mean_ms": 67.75896436000001,
          "median_ms": 67.5902445,
          "p95_ms": 69.87579174999999,
          "mean_round_ci95_ms": [
            67.71544054,
            67.82552210000001
          ],
          "requests_per_second": 14.758194866837838,
          "fields_per_second": 14.758194866837838,
          "round_mean_ms": [
            67.7053405,
            67.7310068,
            67.88853230000002,
            67.75218475,
            67.71775745
          ],
          "distinct_prompt_encodings": 1,
          "prompt_token_lengths": [
            115
          ],
          "cpu_contention": {
            "interval_seconds": 7.886475238017738,
            "core_busy_fraction": {
              "20": 0.030431846009361704,
              "21": 0.02282388450702128,
              "22": 0.8609676433481915,
              "23": 0.46662163881021285,
              "84": 0.002535987167446809,
              "85": 0.006339967918617022,
              "86": 0.002535987167446809,
              "87": 0.0012679935837234045
            },
            "worker_cpu_seconds": 6.81,
            "other_busy_fraction_on_worker_cores": 0.1293353455397873,
            "smt_sibling_busy_fraction": 0.003169983959308511
          }
        },
        "three_fields": {
          "n": 100,
          "fields": 3,
          "mean_ms": 90.76081257,
          "median_ms": 100.834314,
          "p95_ms": 102.42253229999999,
          "mean_round_ci95_ms": [
            80.06992034000002,
            101.45094420000001
          ],
          "requests_per_second": 11.017970990825386,
          "fields_per_second": 33.053912972476155,
          "round_mean_ms": [
            74.70936450000002,
            74.8036415,
            101.3273927,
            101.54579744999998,
            101.4178667
          ],
          "distinct_prompt_encodings": 1,
          "prompt_token_lengths": [
            172
          ],
          "cpu_contention": {
            "interval_seconds": 10.255095972213894,
            "core_busy_fraction": {
              "20": 0.6133535967915569,
              "21": 0.036079623340679816,
              "22": 0.29936336123212715,
              "23": 0.49926397703859643,
              "84": 0.0009751249551535087,
              "85": 0.006825874686074561,
              "86": 0.0019502499103070173,
              "87": 0.0009751249551535087
            },
            "worker_cpu_seconds": 9.12,
            "other_busy_fraction_on_worker_cores": 0.1396866498257401,
            "smt_sibling_busy_fraction": 0.002681593626672149
          }
        },
        "eight_fields": {
          "n": 100,
          "fields": 8,
          "mean_ms": 106.06799624000001,
          "median_ms": 104.2839755,
          "p95_ms": 120.78709315,
          "mean_round_ci95_ms": [
            104.40632252,
            109.33982375999999
          ],
          "requests_per_second": 9.427914502479151,
          "fields_per_second": 75.42331601983321,
          "round_mean_ms": [
            112.59898515,
            104.40057889999999,
            104.39162210000002,
            104.51054125,
            104.43825380000001
          ],
          "distinct_prompt_encodings": 1,
          "prompt_token_lengths": [
            328
          ],
          "cpu_contention": {
            "interval_seconds": 11.722284673247486,
            "core_busy_fraction": {
              "20": 0.21497515802112585,
              "21": 0.025592280716800697,
              "22": 0.7165838600704195,
              "23": 0.3437896376290227,
              "84": 0.0017061520477867132,
              "85": 0.005118456143360139,
              "86": 0.0034123040955734265,
              "87": 0.0008530760238933566
            },
            "worker_cpu_seconds": 10.65,
            "other_busy_fraction_on_worker_cores": 0.09810374274773599,
            "smt_sibling_busy_fraction": 0.002772497077653409
          }
        }
      },
      "raw_sha256": "57e6d8057d2078f864f9b715a33ad46ce072c36f04ee29b55e7a04c66d287d23",
      "metadata_sha256": "4898796d653a813ad552e328d79c2f2806e80c3fa35c4b913eef0350237f7320",
      "telemetry_sha256": "815e535b5012503c209acb5c7520454a3b9ec33758ec4d615de8e0be5d731f9c",
      "finished_at": "2026-10-05T14:07:39.411612+00:00",
      "hardware": "NVIDIA A100 80GB PCIe",
      "adapter": "kev_official_pointer",
      "dtype": "bfloat16",
      "source": "research/latency_v2/results/kev_9b/summary.json",
      "protocol": "research/latency_v2/PROTOCOL.md"
    },
    "llama32_1b_public": {
      "status": "DONE",
      "model": "llama32_1b_public",
      "workloads": {
        "one_field": {
          "n": 100,
          "fields": 1,
          "mean_ms": 9.4868187,
          "median_ms": 9.309929,
          "p95_ms": 11.23720805,
          "mean_round_ci95_ms": [
            9.409260380000001,
            9.60814502
          ],
          "requests_per_second": 105.40941401146414,
          "fields_per_second": 105.40941401146414,
          "round_mean_ms": [
            9.724125750000002,
            9.413631849999998,
            9.39265135,
            9.4756965,
            9.42798805
          ],
          "distinct_prompt_encodings": 1,
          "prompt_token_lengths": [
            217
          ],
          "cpu_contention": {
            "interval_seconds": 2.0650448258966208,
            "core_busy_fraction": {
              "20": 0.05811011872243367,
              "21": 0.3922433013764273,
              "22": 0.22759796499619853,
              "23": 0.15011780669962033,
              "84": 0.014527529680608418,
              "85": 0,
              "86": 0,
              "87": 0
            },
            "worker_cpu_seconds": 0.97,
            "other_busy_fraction_on_worker_cores": 0.08958643303041858,
            "smt_sibling_busy_fraction": 0.0036318824201521044
          }
        },
        "three_fields": {
          "n": 100,
          "fields": 3,
          "mean_ms": 27.395698700000004,
          "median_ms": 27.226708000000002,
          "p95_ms": 29.00851745,
          "mean_round_ci95_ms": [
            27.343317620000004,
            27.468530599999998
          ],
          "requests_per_second": 36.502080525509655,
          "fields_per_second": 109.50624157652895,
          "round_mean_ms": [
            27.32188175,
            27.36307205,
            27.5313152,
            27.3878708,
            27.3743537
          ],
          "distinct_prompt_encodings": 3,
          "prompt_token_lengths": [
            204,
            216,
            217
          ],
          "cpu_contention": {
            "interval_seconds": 3.8232139591127634,
            "core_busy_fraction": {
              "20": 0.01569360246161163,
              "21": 0,
              "22": 0.7245213136444034,
              "23": 0.12554881969289303,
              "84": 0.0052312008205372094,
              "85": 0,
              "86": 0,
              "87": 0.0052312008205372094
            },
            "worker_cpu_seconds": 2.78,
            "other_busy_fraction_on_worker_cores": 0.034656705436059025,
            "smt_sibling_busy_fraction": 0.0026156004102686047
          }
        },
        "eight_fields": {
          "n": 100,
          "fields": 8,
          "mean_ms": 72.55486637999999,
          "median_ms": 72.0747805,
          "p95_ms": 74.89822175,
          "mean_round_ci95_ms": [
            72.05164334,
            73.0133904
          ],
          "requests_per_second": 13.78267302929874,
          "fields_per_second": 110.26138423438992,
          "round_mean_ms": [
            72.5423573,
            71.6984981,
            73.30574165,
            72.60736975,
            72.6203651
          ],
          "distinct_prompt_encodings": 3,
          "prompt_token_lengths": [
            204,
            216,
            217
          ],
          "cpu_contention": {
            "interval_seconds": 8.546528353355825,
            "core_busy_fraction": {
              "20": 0.05148289244570661,
              "21": 0.005850328687012116,
              "22": 0.8541479883037688,
              "23": 0.035101972122072694,
              "84": 0.007020394424414538,
              "85": 0.004680262949609692,
              "86": 0.002340131474804846,
              "87": 0.008190460161816963
            },
            "worker_cpu_seconds": 7.3,
            "other_busy_fraction_on_worker_cores": 0.023108798313697856,
            "smt_sibling_busy_fraction": 0.005557812252661509
          }
        }
      },
      "raw_sha256": "8f03cd2ff95d30eba328ca205a908b2dd1543d7da32f45e1a2bff3f4a5f3da85",
      "metadata_sha256": "38c03a4a91e75bf783df5a7b2b5961ca8dafd6e66966747788448d16c45945d0",
      "telemetry_sha256": "34dfd2e4ada78cacc9072b359ce33430f9d333ad116f316cc6afe1cc1dbb6683",
      "finished_at": "2026-10-05T13:38:26.315907+00:00",
      "hardware": "NVIDIA A100 80GB PCIe",
      "adapter": "llama_constrained_next_token",
      "dtype": "bfloat16",
      "source": "research/latency_v2/results/llama32_1b_public/summary.json",
      "protocol": "research/latency_v2/PROTOCOL.md"
    },
    "llama32_3b_public": {
      "status": "DONE",
      "model": "llama32_3b_public",
      "workloads": {
        "one_field": {
          "n": 100,
          "fields": 1,
          "mean_ms": 16.76089338,
          "median_ms": 16.6481235,
          "p95_ms": 18.11478185,
          "mean_round_ci95_ms": [
            16.71687168,
            16.80917664
          ],
          "requests_per_second": 59.66269084398889,
          "fields_per_second": 59.66269084398889,
          "round_mean_ms": [
            16.85424455,
            16.76322045,
            16.719929099999998,
            16.698678249999997,
            16.768394549999996
          ],
          "distinct_prompt_encodings": 1,
          "prompt_token_lengths": [
            217
          ],
          "cpu_contention": {
            "interval_seconds": 2.903879217337817,
            "core_busy_fraction": {
              "20": 0.588867467279742,
              "21": 0.196289155759914,
              "22": 0.044767702190857586,
              "23": 0.0034436693992967375,
              "84": 0.0034436693992967375,
              "85": 0.0034436693992967375,
              "86": 0.006887338798593475,
              "87": 0.0034436693992967375
            },
            "worker_cpu_seconds": 1.7,
            "other_busy_fraction_on_worker_cores": 0.06198604918734127,
            "smt_sibling_busy_fraction": 0.0043045867491209215
          }
        },
        "three_fields": {
          "n": 100,
          "fields": 3,
          "mean_ms": 49.36688435,
          "median_ms": 49.1867505,
          "p95_ms": 51.16267025,
          "mean_round_ci95_ms": [
            49.2985762,
            49.43519249999999
          ],
          "requests_per_second": 20.25649406817386,
          "fields_per_second": 60.76948220452158,
          "round_mean_ms": [
            49.46047564999999,
            49.26217785,
            49.2884234,
            49.43166635,
            49.3916785
          ],
          "distinct_prompt_encodings": 3,
          "prompt_token_lengths": [
            204,
            216,
            217
          ],
          "cpu_contention": {
            "interval_seconds": 6.113344276789576,
            "core_busy_fraction": {
              "20": 0.8113398780486718,
              "21": 0.10305325063924661,
              "22": 0.013086127065301158,
              "23": 0.0016357658831626448,
              "84": 0.0032715317663252895,
              "85": 0.006543063532650579,
              "86": 0.0032715317663252895,
              "87": 0.006543063532650579
            },
            "worker_cpu_seconds": 4.98,
            "other_busy_fraction_on_worker_cores": 0.02862590295534625,
            "smt_sibling_busy_fraction": 0.004907297649487934
          }
        },
        "eight_fields": {
          "n": 100,
          "fields": 8,
          "mean_ms": 124.55161529999998,
          "median_ms": 124.35305500000001,
          "p95_ms": 126.73805855,
          "mean_round_ci95_ms": [
            124.50448148999999,
            124.59469550000001
          ],
          "requests_per_second": 8.028799928377966,
          "fields_per_second": 64.23039942702373,
          "round_mean_ms": [
            124.50513234999998,
            124.48008075,
            124.62358125,
            124.57703285,
            124.57224930000002
          ],
          "distinct_prompt_encodings": 3,
          "prompt_token_lengths": [
            204,
            216,
            217
          ],
          "cpu_contention": {
            "interval_seconds": 13.8028669860214,
            "core_busy_fraction": {
              "20": 0.9056089588242172,
              "21": 0.042744742856503044,
              "22": 0.007969358837653111,
              "23": 0.002897948668237495,
              "84": 0.0014489743341187474,
              "85": 0.016663204842365595,
              "86": 0.002897948668237495,
              "87": 0.003622435835296869
            },
            "worker_cpu_seconds": 12.5,
            "other_busy_fraction_on_worker_cores": 0.013403012590598418,
            "smt_sibling_busy_fraction": 0.006158140920004677
          }
        }
      },
      "raw_sha256": "914f9865dc9b9fa73c54eced7611775c943c6ac1a3cbf0203c7685f81cba2ad4",
      "metadata_sha256": "3c0c2558c6234841b91eb575cd8d1d68b65ba2860b1b39d9f437b6fe5a2267c0",
      "telemetry_sha256": "f786cd65dd06fb34f338f83ff2a376227eceb39ff21786f31bac99eda6804e16",
      "finished_at": "2026-10-05T14:02:37.843972+00:00",
      "hardware": "NVIDIA A100 80GB PCIe",
      "adapter": "llama_constrained_next_token",
      "dtype": "bfloat16",
      "source": "research/latency_v2/results/llama32_3b_public/summary.json",
      "protocol": "research/latency_v2/PROTOCOL.md"
    },
    "nanojev": {
      "status": "DONE",
      "model": "nanojev",
      "workloads": {
        "one_field": {
          "n": 100,
          "fields": 1,
          "mean_ms": 25.523704239999997,
          "median_ms": 25.3593345,
          "p95_ms": 27.409630500000002,
          "mean_round_ci95_ms": [
            25.49907252,
            25.548650010750002
          ],
          "requests_per_second": 39.17926608916073,
          "fields_per_second": 39.17926608916073,
          "round_mean_ms": [
            25.5566508,
            25.5310106,
            25.49678575,
            25.55182265,
            25.482251400000003
          ],
          "distinct_prompt_encodings": 1,
          "prompt_token_lengths": [
            107
          ],
          "cpu_contention": {
            "interval_seconds": 3.736527998931706,
            "core_busy_fraction": {
              "20": 0.005352562594397288,
              "21": 0.6931568559744488,
              "22": 0.17128200302071322,
              "23": 0.013381406485993222,
              "84": 0.005352562594397288,
              "85": 0.002676281297198644,
              "86": 0.005352562594397288,
              "87": 0.010705125188794577
            },
            "worker_cpu_seconds": 2.58,
            "other_busy_fraction_on_worker_cores": 0.04817306334957558,
            "smt_sibling_busy_fraction": 0.006021632918696949
          }
        },
        "three_fields": {
          "n": 100,
          "fields": 3,
          "mean_ms": 27.304662769999997,
          "median_ms": 27.120568,
          "p95_ms": 29.1977447,
          "mean_round_ci95_ms": [
            27.244269159999998,
            27.392081339999997
          ],
          "requests_per_second": 36.6237813820837,
          "fields_per_second": 109.8713441462511,
          "round_mean_ms": [
            27.3172631,
            27.263075199999996,
            27.243586049999998,
            27.47187705,
            27.22751245
          ],
          "distinct_prompt_encodings": 3,
          "prompt_token_lengths": [
            106,
            107,
            121
          ],
          "cpu_contention": {
            "interval_seconds": 3.9114864957518876,
            "core_busy_fraction": {
              "20": 0,
              "21": 0.7081706668317502,
              "22": 0.15595094107125185,
              "23": 0.02045258243557401,
              "84": 0.005113145608893503,
              "85": 0.0025565728044467514,
              "86": 0.0025565728044467514,
              "87": 0.007669718413340254
            },
            "worker_cpu_seconds": 2.78,
            "other_busy_fraction_on_worker_cores": 0.04346173767559479,
            "smt_sibling_busy_fraction": 0.004474002407781816
          }
        },
        "eight_fields": {
          "n": 100,
          "fields": 8,
          "mean_ms": 43.495489909999996,
          "median_ms": 43.374487,
          "p95_ms": 43.8286766,
          "mean_round_ci95_ms": [
            43.483157829999996,
            43.507821990000004
          ],
          "requests_per_second": 22.99088944782965,
          "fields_per_second": 183.9271155826372,
          "round_mean_ms": [
            43.5066981,
            43.51444405,
            43.49682565,
            43.48403625,
            43.4754455
          ],
          "distinct_prompt_encodings": 3,
          "prompt_token_lengths": [
            106,
            107,
            121
          ],
          "cpu_contention": {
            "interval_seconds": 5.509350057691336,
            "core_busy_fraction": {
              "20": 0.016335865221407626,
              "21": 0.7950121074418379,
              "22": 0.09438499905702186,
              "23": 0.012705672949983713,
              "84": 0,
              "85": 0.0018150961357119588,
              "86": 0.0018150961357119588,
              "87": 0.005445288407135876
            },
            "worker_cpu_seconds": 4.39,
            "other_busy_fraction_on_worker_cores": 0.030402860273175306,
            "smt_sibling_busy_fraction": 0.0022688701696399484
          }
        }
      },
      "raw_sha256": "f2b8d3b2fda70aacbfeff807db7b3e0eb921795acebb353b29d75d7c258c6a40",
      "metadata_sha256": "c7c65ca0e5e76540df9d766a964347cb5175477ba47d06692e005ddba02b526b",
      "telemetry_sha256": "82467f7c3f8d3f3c051d584ef962c2ea16ca2626fd9920637fa6977aed7e8086",
      "finished_at": "2026-10-05T13:42:11.392507+00:00",
      "hardware": "NVIDIA A100 80GB PCIe",
      "adapter": "nanojev_official_candidate_paths",
      "dtype": "fp32_storage_bf16_autocast",
      "source": "research/latency_v2/results/nanojev/summary.json",
      "protocol": "research/latency_v2/PROTOCOL.md"
    },
    "qwen35_08b": {
      "status": "DONE",
      "model": "qwen35_08b",
      "workloads": {
        "one_field": {
          "n": 100,
          "fields": 1,
          "mean_ms": 25.59647532,
          "median_ms": 25.448262,
          "p95_ms": 25.828415599999996,
          "mean_round_ci95_ms": [
            25.581836120000002,
            25.613142420000003
          ],
          "requests_per_second": 39.06787897545575,
          "fields_per_second": 39.06787897545575,
          "round_mean_ms": [
            25.604495199999995,
            25.5799793,
            25.627452299999998,
            25.591677600000004,
            25.5787722
          ],
          "distinct_prompt_encodings": 1,
          "prompt_token_lengths": [
            204
          ],
          "cpu_contention": {
            "interval_seconds": 3.6159370471723378,
            "core_busy_fraction": {
              "20": 0.05531069744601886,
              "21": 0.02765534872300943,
              "22": 0.7135079970536433,
              "23": 0.08849711591363017,
              "84": 0,
              "85": 0.005531069744601886,
              "86": 0.002765534872300943,
              "87": 0.002765534872300943
            },
            "worker_cpu_seconds": 2.6,
            "other_busy_fraction_on_worker_cores": 0.04148302308451415,
            "smt_sibling_busy_fraction": 0.002765534872300943
          }
        },
        "three_fields": {
          "n": 100,
          "fields": 3,
          "mean_ms": 73.18267184999999,
          "median_ms": 72.9901165,
          "p95_ms": 74.92228465,
          "mean_round_ci95_ms": [
            73.10471537999999,
            73.24799598999999
          ],
          "requests_per_second": 13.664436877211392,
          "fields_per_second": 40.99331063163417,
          "round_mean_ms": [
            73.2237458,
            73.27924825000001,
            73.1265982,
            73.05660585000001,
            73.22716114999999
          ],
          "distinct_prompt_encodings": 3,
          "prompt_token_lengths": [
            191,
            203,
            204
          ],
          "cpu_contention": {
            "interval_seconds": 8.461523588746786,
            "core_busy_fraction": {
              "20": 0.04727281036384169,
              "21": 0.01063638233186438,
              "22": 0.868637890435591,
              "23": 0.03427278751378522,
              "84": 0.004727281036384169,
              "85": 0.005909101295480211,
              "86": 0.0023636405181920845,
              "87": 0.005909101295480211
            },
            "worker_cpu_seconds": 7.36,
            "other_busy_fraction_on_worker_cores": 0.022750039987598773,
            "smt_sibling_busy_fraction": 0.004727281036384169
          }
        },
        "eight_fields": {
          "n": 100,
          "fields": 8,
          "mean_ms": 190.40091319999996,
          "median_ms": 190.119345,
          "p95_ms": 192.7494277,
          "mean_round_ci95_ms": [
            190.30029777,
            190.52504607000003
          ],
          "requests_per_second": 5.252075650233803,
          "fields_per_second": 42.016605201870426,
          "round_mean_ms": [
            190.2675625,
            190.39084904999999,
            190.2877574,
            190.4188591,
            190.63953795000003
          ],
          "distinct_prompt_encodings": 3,
          "prompt_token_lengths": [
            191,
            203,
            204
          ],
          "cpu_contention": {
            "interval_seconds": 20.180053025018424,
            "core_busy_fraction": {
              "20": 0.014370626263492448,
              "21": 0.00445984953004938,
              "22": 0.9449925615337965,
              "23": 0.025768019506951977,
              "84": 0.005450927203393688,
              "85": 0.005450927203393688,
              "86": 0.0009910776733443069,
              "87": 0.0039643106933772276
            },
            "worker_cpu_seconds": 19.1,
            "other_busy_fraction_on_worker_cores": 0.010777969697619304,
            "smt_sibling_busy_fraction": 0.0039643106933772276
          }
        }
      },
      "raw_sha256": "2154b112562a9fb31c2b3d25354ae9da758007f507e3c416a670d9a7208cec9f",
      "metadata_sha256": "fad07587b3f4f468fd5b2118d494efe01c0b9fb62b09aafdc5bb7b13fabfd2b1",
      "telemetry_sha256": "3272cbf6b5dbcb4d4720b3e22f7d3d6d1ad7f612d9523dad95d536644d850bdf",
      "finished_at": "2026-10-05T13:52:52.117295+00:00",
      "hardware": "NVIDIA A100 80GB PCIe",
      "adapter": "qwen_constrained_next_token",
      "dtype": "bfloat16",
      "source": "research/latency_v2/results/qwen35_08b/summary.json",
      "protocol": "research/latency_v2/PROTOCOL.md"
    },
    "qwen35_4b": {
      "status": "DONE",
      "model": "qwen35_4b",
      "workloads": {
        "one_field": {
          "n": 100,
          "fields": 1,
          "mean_ms": 47.71434854,
          "median_ms": 47.574983,
          "p95_ms": 48.01797264999999,
          "mean_round_ci95_ms": [
            47.697307519999995,
            47.72579768000001
          ],
          "requests_per_second": 20.958056236724634,
          "fields_per_second": 20.958056236724634,
          "round_mean_ms": [
            47.7142926,
            47.728932900000004,
            47.721092,
            47.725015299999995,
            47.682409899999996
          ],
          "distinct_prompt_encodings": 1,
          "prompt_token_lengths": [
            204
          ],
          "cpu_contention": {
            "interval_seconds": 5.848033714108169,
            "core_busy_fraction": {
              "20": 0.011969835233871438,
              "21": 0.010259858771889802,
              "22": 0.817368748827221,
              "23": 0.07523896432719189,
              "84": 0.006839905847926535,
              "85": 0.008549882309908169,
              "86": 0.006839905847926535,
              "87": 0.005129929385944901
            },
            "worker_cpu_seconds": 4.8,
            "other_busy_fraction_on_worker_cores": 0.023512176352247494,
            "smt_sibling_busy_fraction": 0.006839905847926535
          }
        },
        "three_fields": {
          "n": 100,
          "fields": 3,
          "mean_ms": 136.00702389999998,
          "median_ms": 135.826619,
          "p95_ms": 137.6711327,
          "mean_round_ci95_ms": [
            135.95015594675,
            136.06053734000002
          ],
          "requests_per_second": 7.352561443703497,
          "fields_per_second": 22.05768433111049,
          "round_mean_ms": [
            135.97919085,
            136.04652025000001,
            135.89387960000002,
            136.0923252,
            136.0232036
          ],
          "distinct_prompt_encodings": 3,
          "prompt_token_lengths": [
            191,
            203,
            204
          ],
          "cpu_contention": {
            "interval_seconds": 14.704748215153813,
            "core_busy_fraction": {
              "20": 0.014961153824672868,
              "21": 0.006120472019184355,
              "22": 0.9269114846831418,
              "23": 0.030602360095921776,
              "84": 0.0034002622328801976,
              "85": 0.008840681805488512,
              "86": 0.004760367126032276,
              "87": 0.002720209786304158
            },
            "worker_cpu_seconds": 13.64,
            "other_busy_fraction_on_worker_cores": 0.01275098337330074,
            "smt_sibling_busy_fraction": 0.0049303802376762865
          }
        },
        "eight_fields": {
          "n": 100,
          "fields": 8,
          "mean_ms": 353.50470513000005,
          "median_ms": 353.36601,
          "p95_ms": 354.02027425,
          "mean_round_ci95_ms": [
            353.38103126,
            353.63474926
          ],
          "requests_per_second": 2.8288166620929522,
          "fields_per_second": 22.630533296743618,
          "round_mean_ms": [
            353.42481039999996,
            353.30155264999996,
            353.57568795,
            353.49067765,
            353.730797
          ],
          "distinct_prompt_encodings": 3,
          "prompt_token_lengths": [
            191,
            203,
            204
          ],
          "cpu_contention": {
            "interval_seconds": 36.55779728945345,
            "core_busy_fraction": {
              "20": 0.011488657171398171,
              "21": 0.0027353945646186126,
              "22": 0.968056136418527,
              "23": 0.01750652521355912,
              "84": 0.003282473477542335,
              "85": 0.0057443285856990855,
              "86": 0.00437663130338978,
              "87": 0.004103091846927918
            },
            "worker_cpu_seconds": 35.4,
            "other_busy_fraction_on_worker_cores": 0.0078642593732785,
            "smt_sibling_busy_fraction": 0.00437663130338978
          }
        }
      },
      "raw_sha256": "3cb77930ede774c1abfaaf96716974c3d869fc2aba75fa2b4e9ec37a5de05994",
      "metadata_sha256": "0fa21659f65e9fbc8160ac25ee40bcdbb9a1649b587ddae971902c39fb633066",
      "telemetry_sha256": "5c1068fcfe0219cfe6b55d45c4331d338a8003698129b0a7e3de0db7aa8bc044",
      "finished_at": "2026-10-05T13:54:14.681332+00:00",
      "hardware": "NVIDIA A100 80GB PCIe",
      "adapter": "qwen_constrained_next_token",
      "dtype": "bfloat16",
      "source": "research/latency_v2/results/qwen35_4b/summary.json",
      "protocol": "research/latency_v2/PROTOCOL.md"
    },
    "qwen35_9b": {
      "status": "DONE",
      "model": "qwen35_9b",
      "workloads": {
        "one_field": {
          "n": 100,
          "fields": 1,
          "mean_ms": 54.955074329999995,
          "median_ms": 54.794224,
          "p95_ms": 55.32976304999999,
          "mean_round_ci95_ms": [
            54.934436930000004,
            54.97950221999999
          ],
          "requests_per_second": 18.196681784016796,
          "fields_per_second": 18.196681784016796,
          "round_mean_ms": [
            54.932703700000005,
            54.974037300000006,
            54.996331799999986,
            54.9344784,
            54.93782044999999
          ],
          "distinct_prompt_encodings": 1,
          "prompt_token_lengths": [
            204
          ],
          "cpu_contention": {
            "interval_seconds": 6.62442348478362,
            "core_busy_fraction": {
              "20": 0.0332104371807362,
              "21": 0.030191306527942004,
              "22": 0.8347896254975964,
              "23": 0.0528347864238985,
              "84": 0.0015095653263971001,
              "85": 0.0045286959791913,
              "86": 0.0015095653263971001,
              "87": 0.007547826631985501
            },
            "worker_cpu_seconds": 5.53,
            "other_busy_fraction_on_worker_cores": 0.029059132533144162,
            "smt_sibling_busy_fraction": 0.0037739133159927505
          }
        },
        "three_fields": {
          "n": 100,
          "fields": 3,
          "mean_ms": 161.8254191,
          "median_ms": 164.386068,
          "p95_ms": 166.37131555,
          "mean_round_ci95_ms": [
            158.70499114,
            164.94584705999998
          ],
          "requests_per_second": 6.179498904199037,
          "fields_per_second": 18.538496712597112,
          "round_mean_ms": [
            157.19257660000002,
            165.31960059999997,
            164.6182144,
            164.73590985,
            157.26079405000002
          ],
          "distinct_prompt_encodings": 3,
          "prompt_token_lengths": [
            191,
            203,
            204
          ],
          "cpu_contention": {
            "interval_seconds": 17.392212816979736,
            "core_busy_fraction": {
              "20": 0.03449819791845174,
              "21": 0.010924429340843051,
              "22": 0.18514032882902434,
              "23": 0.7572354443100157,
              "84": 0.0028748498265376453,
              "85": 0.014374249132688225,
              "86": 0.010924429340843051,
              "87": 0.001724909895922587
            },
            "worker_cpu_seconds": 16.24,
            "other_busy_fraction_on_worker_cores": 0.01351179418472695,
            "smt_sibling_busy_fraction": 0.007474609548997877
          }
        },
        "eight_fields": {
          "n": 100,
          "fields": 8,
          "mean_ms": 409.59823540000014,
          "median_ms": 409.3985975,
          "p95_ms": 411.4539793,
          "mean_round_ci95_ms": [
            409.53878034,
            409.65769045999997
          ],
          "requests_per_second": 2.4414167678809293,
          "fields_per_second": 19.531334143047435,
          "round_mean_ms": [
            409.5072949,
            409.63855449999994,
            409.5203787,
            409.65840319999995,
            409.66654570000003
          ],
          "distinct_prompt_encodings": 3,
          "prompt_token_lengths": [
            191,
            203,
            204
          ],
          "cpu_contention": {
            "interval_seconds": 42.269632793962955,
            "core_busy_fraction": {
              "20": 0.10409364144342456,
              "21": 0.0063875643613010525,
              "22": 0.008989905397386666,
              "23": 0.9699634770864561,
              "84": 0.0035486468673894735,
              "85": 0.004731529156519298,
              "86": 0.006860717276952982,
              "87": 0.002129188120433684
            },
            "worker_cpu_seconds": 41.02,
            "other_busy_fraction_on_worker_cores": 0.02974948957161505,
            "smt_sibling_busy_fraction": 0.004317520355323859
          }
        }
      },
      "raw_sha256": "6b724012c0b0709d54ee253718c9bdfa7b779a8e4bcecc748d9e201a3f306705",
      "metadata_sha256": "a1835b89d6acec24710653332e8240176b94d7d5f625d0cf6f79d9d73411227c",
      "telemetry_sha256": "0728fd70c2d7c52cbdf83ec10ba8f86a467f3d8e0d6519648d22cd9002cea7ef",
      "finished_at": "2026-10-05T14:02:01.431179+00:00",
      "hardware": "NVIDIA A100 80GB PCIe",
      "adapter": "qwen35_constrained_next_token",
      "dtype": "bfloat16",
      "source": "research/latency_v2/results/qwen35_9b/summary.json",
      "protocol": "research/latency_v2/PROTOCOL.md"
    },
    "qwen3_8b": {
      "status": "DONE",
      "model": "qwen3_8b",
      "workloads": {
        "one_field": {
          "n": 100,
          "fields": 1,
          "mean_ms": 30.59651152,
          "median_ms": 30.5275265,
          "p95_ms": 31.368234049999998,
          "mean_round_ci95_ms": [
            30.536514099999998,
            30.65650894
          ],
          "requests_per_second": 32.68346456249859,
          "fields_per_second": 32.68346456249859,
          "round_mean_ms": [
            30.6927808,
            30.535401450000002,
            30.67143685,
            30.528829099999996,
            30.554109399999998
          ],
          "distinct_prompt_encodings": 1,
          "prompt_token_lengths": [
            204
          ],
          "cpu_contention": {
            "interval_seconds": 4.212720918934792,
            "core_busy_fraction": {
              "20": 0.04510149227925653,
              "21": 0.011868813757699086,
              "22": 0.7358664529773433,
              "23": 0.11868813757699086,
              "84": 0.002373762751539817,
              "85": 0.007121288254619451,
              "86": 0,
              "87": 0.004747525503079634
            },
            "worker_cpu_seconds": 3.09,
            "other_busy_fraction_on_worker_cores": 0.0445080515913716,
            "smt_sibling_busy_fraction": 0.0035606441273097257
          }
        },
        "three_fields": {
          "n": 100,
          "fields": 3,
          "mean_ms": 90.0443049,
          "median_ms": 89.8626645,
          "p95_ms": 91.34964545,
          "mean_round_ci95_ms": [
            89.86500179,
            90.29550363999999
          ],
          "requests_per_second": 11.105644061671244,
          "fields_per_second": 33.316932185013734,
          "round_mean_ms": [
            89.82844804999999,
            90.10942070000002,
            89.8803672,
            89.90737845,
            90.49591009999999
          ],
          "distinct_prompt_encodings": 3,
          "prompt_token_lengths": [
            191,
            203,
            204
          ],
          "cpu_contention": {
            "interval_seconds": 10.172379445284605,
            "core_busy_fraction": {
              "20": 0.04423743750618711,
              "21": 0.006881379167629107,
              "22": 0.3715944750519717,
              "23": 0.5426459000758952,
              "84": 0.0029491625004124736,
              "85": 0.006881379167629107,
              "86": 0.003932216667216632,
              "87": 0.005898325000824947
            },
            "worker_cpu_seconds": 9.05,
            "other_busy_fraction_on_worker_cores": 0.01892379271098003,
            "smt_sibling_busy_fraction": 0.00491527083402079
          }
        },
        "eight_fields": {
          "n": 100,
          "fields": 8,
          "mean_ms": 239.05327595999995,
          "median_ms": 239.049994,
          "p95_ms": 240.6938699,
          "mean_round_ci95_ms": [
            238.80662573,
            239.37187181
          ],
          "requests_per_second": 4.1831679402181745,
          "fields_per_second": 33.465343521745396,
          "round_mean_ms": [
            238.95150630000003,
            238.66033195,
            238.93088419999998,
            239.12124860000003,
            239.60240875
          ],
          "distinct_prompt_encodings": 3,
          "prompt_token_lengths": [
            191,
            203,
            204
          ],
          "cpu_contention": {
            "interval_seconds": 25.236914718989283,
            "core_busy_fraction": {
              "20": 0.028529636368673967,
              "21": 0.016246042932161565,
              "22": 0.009113633839993073,
              "23": 0.9482141643088445,
              "84": 0.0027737146469544137,
              "85": 0.004754939394778995,
              "86": 0.0011887348486947487,
              "87": 0.005547429293908827
            },
            "worker_cpu_seconds": 23.97,
            "other_busy_fraction_on_worker_cores": 0.013076083335642238,
            "smt_sibling_busy_fraction": 0.003566204546084246
          }
        }
      },
      "raw_sha256": "74ee6d95db0d9493bf2f9f5126651d359ca06ad10b7903ddb2f25983e1a334df",
      "metadata_sha256": "20002d53a0a34bb828f14fd8cca500396eee6ed183a268b62439f0f83c598075",
      "telemetry_sha256": "9cd3b942d2f332b35c82a27915e2fe5df45a4d62249b631018ca59d1f040e050",
      "finished_at": "2026-10-05T13:37:52.896788+00:00",
      "hardware": "NVIDIA A100 80GB PCIe",
      "adapter": "qwen3_constrained_next_token",
      "dtype": "bfloat16",
      "source": "research/latency_v2/results/qwen3_8b/summary.json",
      "protocol": "research/latency_v2/PROTOCOL.md"
    },
    "startlux_4b": {
      "status": "DONE",
      "model": "startlux_4b",
      "workloads": {
        "one_field": {
          "n": 100,
          "fields": 1,
          "mean_ms": 62.980557960000006,
          "median_ms": 62.805239,
          "p95_ms": 64.69975984999999,
          "mean_round_ci95_ms": [
            62.87361813,
            63.08749779
          ],
          "requests_per_second": 15.877915858337055,
          "fields_per_second": 15.877915858337055,
          "round_mean_ms": [
            63.15534015000001,
            62.86954680000001,
            63.0920619,
            62.94268485,
            62.8431561
          ],
          "distinct_prompt_encodings": 1,
          "prompt_token_lengths": [
            160
          ],
          "cpu_contention": {
            "interval_seconds": 7.4792852331884205,
            "core_busy_fraction": {
              "20": 0.8490116103371276,
              "21": 0.07353643869061736,
              "22": 0.021392418528179594,
              "23": 0.010696209264089797,
              "84": 0,
              "85": 0.006685130790056123,
              "86": 0,
              "87": 0.005348104632044898
            },
            "worker_cpu_seconds": 6.33,
            "other_busy_fraction_on_worker_cores": 0.027074779699727287,
            "smt_sibling_busy_fraction": 0.003008308855525255
          }
        },
        "three_fields": {
          "n": 100,
          "fields": 3,
          "mean_ms": 67.17256708,
          "median_ms": 66.796606,
          "p95_ms": 68.866956,
          "mean_round_ci95_ms": [
            66.93917182,
            67.59238325999999
          ],
          "requests_per_second": 14.88702968295134,
          "fields_per_second": 44.661089048854016,
          "round_mean_ms": [
            68.01210569999999,
            66.91737989999999,
            67.02140035000001,
            66.94914985000001,
            66.96279960000001
          ],
          "distinct_prompt_encodings": 3,
          "prompt_token_lengths": [
            151,
            160,
            161
          ],
          "cpu_contention": {
            "interval_seconds": 7.878118847962469,
            "core_busy_fraction": {
              "20": 0.8618808793111962,
              "21": 0.07489097478550893,
              "22": 0.2729077894726173,
              "23": 0.0063466927784329615,
              "84": 0.0025386771113731846,
              "85": 0,
              "86": 0.008885369889806146,
              "87": 0
            },
            "worker_cpu_seconds": 6.75,
            "other_busy_fraction_on_worker_cores": 0.0898057028148264,
            "smt_sibling_busy_fraction": 0.0028560117502948327
          }
        },
        "eight_fields": {
          "n": 100,
          "fields": 8,
          "mean_ms": 152.52662775,
          "median_ms": 151.9165445,
          "p95_ms": 154.63666515,
          "mean_round_ci95_ms": [
            151.78074955999998,
            153.46805280999996
          ],
          "requests_per_second": 6.556232277285105,
          "fields_per_second": 52.44985821828084,
          "round_mean_ms": [
            154.23230934999998,
            152.93171635000002,
            151.71161965,
            151.72301510000003,
            152.0344783
          ],
          "distinct_prompt_encodings": 3,
          "prompt_token_lengths": [
            151,
            160,
            161
          ],
          "cpu_contention": {
            "interval_seconds": 16.56419954728335,
            "core_busy_fraction": {
              "20": 0.025355888692422994,
              "21": 0.024148465421355233,
              "22": 0.3634344045913962,
              "23": 0.9236788023668376,
              "84": 0.0018111349066016423,
              "85": 0.0018111349066016423,
              "86": 0,
              "87": 0
            },
            "worker_cpu_seconds": 15.3,
            "other_busy_fraction_on_worker_cores": 0.10323468967629361,
            "smt_sibling_busy_fraction": 0.0009055674533008212
          }
        }
      },
      "raw_sha256": "490128828336026418d89b8f57051ca8315105feaf52da9300bc7b2dc55fcb45",
      "metadata_sha256": "91406fbaed6f39fc95410831ed8576e9d9ec2d6de7e3179a7ad1224e6599f181",
      "telemetry_sha256": "f2ec4be9aad2c01492b40094f1a9f0518fd4303d3c107cef07f1fd696a4f574c",
      "finished_at": "2026-10-05T13:29:28.737540+00:00",
      "hardware": "NVIDIA A100 80GB PCIe",
      "adapter": "startlux_official_native",
      "dtype": "bfloat16",
      "source": "research/latency_v2/results/startlux_4b/summary.json",
      "protocol": "research/latency_v2/PROTOCOL.md"
    }
  },
  "pendingModels": [],
  "speedExpected": 14,
  "sources": [
    {
      "path": "research/model_expansion_v3/scores.json",
      "sha256": "82f1339f85c595e2ac45218b8eddcaa95f99c6a7207fe522b85ab739c4dc367d",
      "url": "https://github.com/CYMCharming/system1bench/blob/main/research/model_expansion_v3/scores.json"
    },
    {
      "path": "research/startlux_transfer_v1/summary.json",
      "sha256": "01183ad1fe818ae278bc1e369e47eb5d4e562b9a32fd045b32f688d0c5bae0d2",
      "url": "https://github.com/CYMCharming/system1bench/blob/main/research/startlux_transfer_v1/summary.json"
    },
    {
      "path": "research/model_expansion_v3/summary.json",
      "sha256": "12935896cdfb25f4a63e867a43271fa78dc1b7f821d1d5917c12bdea6eb67096",
      "url": "https://github.com/CYMCharming/system1bench/blob/main/research/model_expansion_v3/summary.json"
    },
    {
      "path": "research/latency_v2/results/intern_08b/summary.json",
      "sha256": "8c5db44ef9ed500e03ee2228e22eb36480c2484ded633d0a607d989213ba3cff",
      "url": "https://github.com/CYMCharming/system1bench/blob/main/research/latency_v2/results/intern_08b/summary.json"
    },
    {
      "path": "research/latency_v2/results/intern_2b/summary.json",
      "sha256": "278c198bd81961545fa71b7ec18540c33c9fbe7fae7e8fb145cf5b3023c9f66e",
      "url": "https://github.com/CYMCharming/system1bench/blob/main/research/latency_v2/results/intern_2b/summary.json"
    },
    {
      "path": "research/latency_v2/results/intern_4b/summary.json",
      "sha256": "d12a034f2dfa3b9f658d939150b66efef601e5ebb7c4658580a7b95a993672be",
      "url": "https://github.com/CYMCharming/system1bench/blob/main/research/latency_v2/results/intern_4b/summary.json"
    },
    {
      "path": "research/latency_v2/results/kev_08b/summary.json",
      "sha256": "c50e9230b65a40ac0be9ac50ba58064cfc43bcca8af083bf7b62a38508eb5cdf",
      "url": "https://github.com/CYMCharming/system1bench/blob/main/research/latency_v2/results/kev_08b/summary.json"
    },
    {
      "path": "research/latency_v2/results/kev_4b/summary.json",
      "sha256": "f6e32fcd09aa4351251514fb90479dc32c4c387abd96cb4aba54b5902bdb6d43",
      "url": "https://github.com/CYMCharming/system1bench/blob/main/research/latency_v2/results/kev_4b/summary.json"
    },
    {
      "path": "research/latency_v2/results/kev_9b/summary.json",
      "sha256": "dc0299191db8731fa3510251d5e98768b441292dd4a83d6a82f699ba0183414f",
      "url": "https://github.com/CYMCharming/system1bench/blob/main/research/latency_v2/results/kev_9b/summary.json"
    },
    {
      "path": "research/latency_v2/results/llama32_1b_public/summary.json",
      "sha256": "57c00785185a25a9a9c36532f8121188f71e3869e11706935bf207e9084fd253",
      "url": "https://github.com/CYMCharming/system1bench/blob/main/research/latency_v2/results/llama32_1b_public/summary.json"
    },
    {
      "path": "research/latency_v2/results/llama32_3b_public/summary.json",
      "sha256": "dad4967f24d7f7d3b94b220e89d94235782f08f3c492f06d9f742e34e934301f",
      "url": "https://github.com/CYMCharming/system1bench/blob/main/research/latency_v2/results/llama32_3b_public/summary.json"
    },
    {
      "path": "research/latency_v2/results/nanojev/summary.json",
      "sha256": "466735855b4d701503dcdf1b11fb79750ffb4687e305620026f543072d77954d",
      "url": "https://github.com/CYMCharming/system1bench/blob/main/research/latency_v2/results/nanojev/summary.json"
    },
    {
      "path": "research/latency_v2/results/qwen35_08b/summary.json",
      "sha256": "545a3f483396df706ecb86f34219f2762450c467cee1596d146500602ba2659d",
      "url": "https://github.com/CYMCharming/system1bench/blob/main/research/latency_v2/results/qwen35_08b/summary.json"
    },
    {
      "path": "research/latency_v2/results/qwen35_4b/summary.json",
      "sha256": "403a2fdc5fc003c29a88493087b0e3368af193d4a33e01a547157cafbfde2cc9",
      "url": "https://github.com/CYMCharming/system1bench/blob/main/research/latency_v2/results/qwen35_4b/summary.json"
    },
    {
      "path": "research/latency_v2/results/qwen35_9b/summary.json",
      "sha256": "82dacc794c26329ceea0cc935b601eac57c561eaf6c9ac2194f6b0614a794cdb",
      "url": "https://github.com/CYMCharming/system1bench/blob/main/research/latency_v2/results/qwen35_9b/summary.json"
    },
    {
      "path": "research/latency_v2/results/qwen3_8b/summary.json",
      "sha256": "d818ce876635597efb3fc7dfc2d381fdfb984218d27618b7bda0ba9558822ea2",
      "url": "https://github.com/CYMCharming/system1bench/blob/main/research/latency_v2/results/qwen3_8b/summary.json"
    },
    {
      "path": "research/latency_v2/results/startlux_4b/summary.json",
      "sha256": "43a3540d7160933ea99df04f4d64dc08f3a698cb32eb6742f735167712232d0f",
      "url": "https://github.com/CYMCharming/system1bench/blob/main/research/latency_v2/results/startlux_4b/summary.json"
    }
  ]
}