{
  "base": {
    "temperature": 0.5555971907594387,
    "checkpoint_sha256": "d9195e330fe72aed659e067010515a6264ad5da4f1462de17f6389afdfbdd84b",
    "fit_split": "calibration",
    "fit_examples": 496,
    "fit_target": "completion tokens and EOS",
    "fit_nll": 0.0004168543947901982,
    "held_out_split": "test",
    "raw": {
      "tokens": 1742,
      "nll": 0.0016934610208629386,
      "brier": 0.001048142620728704,
      "bins": [
        {
          "lower": 0.0,
          "upper": 0.1,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.1,
          "upper": 0.2,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.2,
          "upper": 0.3,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.3,
          "upper": 0.4,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.4,
          "upper": 0.5,
          "count": 1,
          "confidence": 0.4876455638619116,
          "accuracy": 1.0
        },
        {
          "lower": 0.5,
          "upper": 0.6,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.6,
          "upper": 0.7,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.7,
          "upper": 0.8,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.8,
          "upper": 0.9,
          "count": 1,
          "confidence": 0.8619784708210777,
          "accuracy": 0.0
        },
        {
          "lower": 0.9,
          "upper": 1.0,
          "count": 1740,
          "confidence": 0.9998652676266474,
          "accuracy": 1.0
        }
      ]
    },
    "fitted": {
      "tokens": 1742,
      "nll": 0.0021162975244268653,
      "brier": 0.0011304673283358717,
      "bins": [
        {
          "lower": 0.0,
          "upper": 0.1,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.1,
          "upper": 0.2,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.2,
          "upper": 0.3,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.3,
          "upper": 0.4,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.4,
          "upper": 0.5,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.5,
          "upper": 0.6,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.6,
          "upper": 0.7,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.7,
          "upper": 0.8,
          "count": 1,
          "confidence": 0.7243847123902653,
          "accuracy": 1.0
        },
        {
          "lower": 0.8,
          "upper": 0.9,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.9,
          "upper": 1.0,
          "count": 1741,
          "confidence": 0.9999790116405768,
          "accuracy": 0.9994256174612292
        }
      ]
    }
  },
  "first": {
    "temperature": 0.5978385103325439,
    "checkpoint_sha256": "14b2986a1e3b7eca2c663cf9a57dc942dd3bc1e809ff23122f5e4de6e40f1734",
    "fit_split": "calibration",
    "fit_examples": 496,
    "fit_target": "completion tokens and EOS",
    "fit_nll": 0.0055886063590020645,
    "held_out_split": "test",
    "raw": {
      "tokens": 1742,
      "nll": 0.01205654794458907,
      "brier": 0.00571768692915644,
      "bins": [
        {
          "lower": 0.0,
          "upper": 0.1,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.1,
          "upper": 0.2,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.2,
          "upper": 0.3,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.3,
          "upper": 0.4,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.4,
          "upper": 0.5,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.5,
          "upper": 0.6,
          "count": 7,
          "confidence": 0.5565129022171307,
          "accuracy": 0.5714285714285714
        },
        {
          "lower": 0.6,
          "upper": 0.7,
          "count": 3,
          "confidence": 0.646750023085412,
          "accuracy": 0.6666666666666666
        },
        {
          "lower": 0.7,
          "upper": 0.8,
          "count": 9,
          "confidence": 0.757876719508087,
          "accuracy": 0.8888888888888888
        },
        {
          "lower": 0.8,
          "upper": 0.9,
          "count": 15,
          "confidence": 0.8563825171540351,
          "accuracy": 0.8666666666666667
        },
        {
          "lower": 0.9,
          "upper": 1.0,
          "count": 1708,
          "confidence": 0.9969491963476731,
          "accuracy": 1.0
        }
      ]
    },
    "fitted": {
      "tokens": 1742,
      "nll": 0.009037736991765202,
      "brier": 0.005548740752534359,
      "bins": [
        {
          "lower": 0.0,
          "upper": 0.1,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.1,
          "upper": 0.2,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.2,
          "upper": 0.3,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.3,
          "upper": 0.4,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.4,
          "upper": 0.5,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.5,
          "upper": 0.6,
          "count": 2,
          "confidence": 0.5468532348313693,
          "accuracy": 0.0
        },
        {
          "lower": 0.6,
          "upper": 0.7,
          "count": 5,
          "confidence": 0.6419698942701244,
          "accuracy": 0.8
        },
        {
          "lower": 0.7,
          "upper": 0.8,
          "count": 3,
          "confidence": 0.7579455571234979,
          "accuracy": 0.6666666666666666
        },
        {
          "lower": 0.8,
          "upper": 0.9,
          "count": 6,
          "confidence": 0.8509160703391908,
          "accuracy": 0.8333333333333334
        },
        {
          "lower": 0.9,
          "upper": 1.0,
          "count": 1726,
          "confidence": 0.999236666333138,
          "accuracy": 0.9988412514484357
        }
      ]
    }
  },
  "finetuned": {
    "temperature": 0.7162011840238206,
    "checkpoint_sha256": "66f53eaff2a81219b68051e169d90b302bc1bcb019b1db073f71acaaac5413f6",
    "fit_split": "calibration",
    "fit_examples": 496,
    "fit_target": "completion tokens and EOS",
    "fit_nll": 0.00807378044907046,
    "held_out_split": "test",
    "raw": {
      "tokens": 1742,
      "nll": 0.033412698319983794,
      "brier": 0.01661181325702248,
      "bins": [
        {
          "lower": 0.0,
          "upper": 0.1,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.1,
          "upper": 0.2,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.2,
          "upper": 0.3,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.3,
          "upper": 0.4,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.4,
          "upper": 0.5,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.5,
          "upper": 0.6,
          "count": 4,
          "confidence": 0.5530755217198738,
          "accuracy": 0.5
        },
        {
          "lower": 0.6,
          "upper": 0.7,
          "count": 5,
          "confidence": 0.6358517493285775,
          "accuracy": 0.6
        },
        {
          "lower": 0.7,
          "upper": 0.8,
          "count": 6,
          "confidence": 0.7483844614588359,
          "accuracy": 0.6666666666666666
        },
        {
          "lower": 0.8,
          "upper": 0.9,
          "count": 11,
          "confidence": 0.8602777640133329,
          "accuracy": 0.6363636363636364
        },
        {
          "lower": 0.9,
          "upper": 1.0,
          "count": 1716,
          "confidence": 0.9981541930967871,
          "accuracy": 0.9953379953379954
        }
      ]
    },
    "fitted": {
      "tokens": 1742,
      "nll": 0.04144649026913105,
      "brier": 0.01742719555545611,
      "bins": [
        {
          "lower": 0.0,
          "upper": 0.1,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.1,
          "upper": 0.2,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.2,
          "upper": 0.3,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.3,
          "upper": 0.4,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.4,
          "upper": 0.5,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.5,
          "upper": 0.6,
          "count": 2,
          "confidence": 0.5430256311430557,
          "accuracy": 0.5
        },
        {
          "lower": 0.6,
          "upper": 0.7,
          "count": 5,
          "confidence": 0.6574666550175461,
          "accuracy": 0.6
        },
        {
          "lower": 0.7,
          "upper": 0.8,
          "count": 3,
          "confidence": 0.7540733126231268,
          "accuracy": 0.6666666666666666
        },
        {
          "lower": 0.8,
          "upper": 0.9,
          "count": 6,
          "confidence": 0.8406307743475191,
          "accuracy": 0.5
        },
        {
          "lower": 0.9,
          "upper": 1.0,
          "count": 1726,
          "confidence": 0.9991761555685666,
          "accuracy": 0.9936268829663963
        }
      ]
    }
  },
  "rl": {
    "temperature": 1.0505462629120659,
    "checkpoint_sha256": "b06108603c8647b50f2d6e5504d86ad30b43fabeba1fda62ef50fee27cdffb05",
    "fit_split": "calibration",
    "fit_examples": 496,
    "fit_target": "completion tokens and EOS",
    "fit_nll": 0.05948988793912399,
    "held_out_split": "test",
    "raw": {
      "tokens": 1742,
      "nll": 0.0868680727954487,
      "brier": 0.04720341696560769,
      "bins": [
        {
          "lower": 0.0,
          "upper": 0.1,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.1,
          "upper": 0.2,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.2,
          "upper": 0.3,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.3,
          "upper": 0.4,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.4,
          "upper": 0.5,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.5,
          "upper": 0.6,
          "count": 19,
          "confidence": 0.5466860168202169,
          "accuracy": 0.42105263157894735
        },
        {
          "lower": 0.6,
          "upper": 0.7,
          "count": 27,
          "confidence": 0.6622900137641685,
          "accuracy": 0.48148148148148145
        },
        {
          "lower": 0.7,
          "upper": 0.8,
          "count": 21,
          "confidence": 0.7414918053495695,
          "accuracy": 0.6666666666666666
        },
        {
          "lower": 0.8,
          "upper": 0.9,
          "count": 31,
          "confidence": 0.8542910598946853,
          "accuracy": 0.8387096774193549
        },
        {
          "lower": 0.9,
          "upper": 1.0,
          "count": 1644,
          "confidence": 0.9959950442439898,
          "accuracy": 0.9872262773722628
        }
      ]
    },
    "fitted": {
      "tokens": 1742,
      "nll": 0.08520124927022783,
      "brier": 0.046912958976961905,
      "bins": [
        {
          "lower": 0.0,
          "upper": 0.1,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.1,
          "upper": 0.2,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.2,
          "upper": 0.3,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.3,
          "upper": 0.4,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.4,
          "upper": 0.5,
          "count": 0,
          "confidence": null,
          "accuracy": null
        },
        {
          "lower": 0.5,
          "upper": 0.6,
          "count": 19,
          "confidence": 0.5441676076360781,
          "accuracy": 0.42105263157894735
        },
        {
          "lower": 0.6,
          "upper": 0.7,
          "count": 31,
          "confidence": 0.6601128235696831,
          "accuracy": 0.5161290322580645
        },
        {
          "lower": 0.7,
          "upper": 0.8,
          "count": 19,
          "confidence": 0.7443615381882469,
          "accuracy": 0.6842105263157895
        },
        {
          "lower": 0.8,
          "upper": 0.9,
          "count": 33,
          "confidence": 0.8518261869862318,
          "accuracy": 0.8181818181818182
        },
        {
          "lower": 0.9,
          "upper": 1.0,
          "count": 1640,
          "confidence": 0.9954876504804283,
          "accuracy": 0.9878048780487805
        }
      ]
    }
  }
}