{
  "schema": "industrial.grounded.confirmation.browser.summary.v1",
  "saved_results_only": true,
  "rank1_verified": false,
  "model_promoted": false,
  "record_disjoint": true,
  "official_full_benchmark": false,
  "counts": {
    "models": 10,
    "spaces": 11,
    "datasets": 9
  },
  "metrics": {
    "base-boolq": {
      "p_yes": {
        "n": 128,
        "accuracy": 0.7734375,
        "brier": 0.14654051861174916,
        "nll": 0.4370580508867321,
        "ece_10bin": 0.05867074569687246
      },
      "calibrated_p_yes": {
        "n": 128,
        "accuracy": 0.7734375,
        "brier": 0.14984039677631145,
        "nll": 0.46472429821150774,
        "ece_10bin": 0.06195897392556616
      }
    },
    "base-strategyqa_grounded": {
      "p_yes": {
        "n": 32,
        "accuracy": 0.59375,
        "brier": 0.2507869347065766,
        "nll": 0.7496980567223643,
        "ece_10bin": 0.21099844155833125
      },
      "calibrated_p_yes": {
        "n": 32,
        "accuracy": 0.59375,
        "brier": 0.23039465352224847,
        "nll": 0.6541927784091557,
        "ece_10bin": 0.06392632206079035
      }
    },
    "adapted-boolq": {
      "p_yes": {
        "n": 128,
        "accuracy": 0.8359375,
        "brier": 0.15255887761710468,
        "nll": 0.8022586615825527,
        "ece_10bin": 0.15056734635422234
      },
      "calibrated_p_yes": {
        "n": 128,
        "accuracy": 0.8359375,
        "brier": 0.11897035253395735,
        "nll": 0.3829296611848455,
        "ece_10bin": 0.07673722436288109
      }
    },
    "adapted-strategyqa_grounded": {
      "p_yes": {
        "n": 32,
        "accuracy": 0.5625,
        "brier": 0.36507526115119016,
        "nll": 1.7967543888702973,
        "ece_10bin": 0.3930756030222255
      },
      "calibrated_p_yes": {
        "n": 32,
        "accuracy": 0.5625,
        "brier": 0.23707129730837412,
        "nll": 0.6698479453842792,
        "ece_10bin": 0.20458811999901858
      }
    }
  },
  "comparisons": {
    "boolq": {
      "accuracy_gain": 0.0625,
      "accuracy_gain_record_bootstrap_95ci": [
        -0.015625,
        0.13300781249999716
      ],
      "calibrated_brier_reduction": 0.030870044242354105,
      "calibrated_brier_reduction_record_bootstrap_95ci": [
        -0.0017965816580639857,
        0.0635079665703132
      ],
      "adapter_only_correct": 16,
      "base_only_correct": 8,
      "references": {
        "constant_half": {
          "n": 128,
          "accuracy": 0.6875,
          "brier": 0.25,
          "nll": 0.6931471805599453,
          "ece_10bin": 0.1875
        },
        "always_yes": {
          "n": 128,
          "accuracy": 0.6875,
          "brier": 0.3125,
          "nll": 10.793617497606524,
          "ece_10bin": 0.3125
        },
        "always_no": {
          "n": 128,
          "accuracy": 0.3125,
          "brier": 0.6875,
          "nll": 23.74540877150109,
          "ece_10bin": 0.6875
        }
      }
    },
    "strategyqa_grounded": {
      "accuracy_gain": -0.03125,
      "accuracy_gain_record_bootstrap_95ci": [
        -0.21875,
        0.15625
      ],
      "calibrated_brier_reduction": -0.006676643786125626,
      "calibrated_brier_reduction_record_bootstrap_95ci": [
        -0.0731238789624646,
        0.05609479560675499
      ],
      "adapter_only_correct": 4,
      "base_only_correct": 5,
      "references": {
        "constant_half": {
          "n": 32,
          "accuracy": 0.5,
          "brier": 0.25,
          "nll": 0.6931471805599453,
          "ece_10bin": 0.0
        },
        "always_yes": {
          "n": 32,
          "accuracy": 0.5,
          "brier": 0.5,
          "nll": 17.26978799617044,
          "ece_10bin": 0.5
        },
        "always_no": {
          "n": 32,
          "accuracy": 0.5,
          "brier": 0.5,
          "nll": 17.269388197455342,
          "ece_10bin": 0.5
        }
      }
    }
  },
  "reload_failures": {
    "base": {
      "old_records": 8,
      "within_original_tolerance": 4,
      "max_abs_delta": 0.012145638465881348,
      "original_atol": 0.002,
      "tolerance_loosened": false
    },
    "adapted": {
      "old_records": 8,
      "within_original_tolerance": 7,
      "max_abs_delta": 0.010440468788146973,
      "original_atol": 0.002,
      "tolerance_loosened": false
    }
  },
  "evaluated_adapter_revision": "208acc543206a9cd2794feba1863d1c186a4a39c",
  "evaluated_adapter_weight_sha256": "827fd0d08c308a89df55343ad7045d9d5b9ac22394e4fa876bdef8c2a001c845",
  "old_reload_current_model_revision": "208acc543206a9cd2794feba1863d1c186a4a39c"
}