{
  "protocol": "07-experiment-protocol.md",
  "seed": 73,
  "source_sha256": "d9195e330fe72aed659e067010515a6264ad5da4f1462de17f6389afdfbdd84b",
  "selected_sha256": "66f53eaff2a81219b68051e169d90b302bc1bcb019b1db073f71acaaac5413f6",
  "initial_formatted_validation": {
    "examples": 1006,
    "correct": 0,
    "accuracy": 0.0,
    "terminated": 271,
    "next_token_loss": 11.830140253976255,
    "answer_nll": 9.287011329902588,
    "mistakes": [
      {
        "prompt": "Q:0+16;A:",
        "expected": "16",
        "generated": "728=",
        "ended": false
      },
      {
        "prompt": "Q:0+22;A:",
        "expected": "22",
        "generated": "738=",
        "ended": false
      },
      {
        "prompt": "Q:0+35;A:",
        "expected": "35",
        "generated": "730=",
        "ended": false
      },
      {
        "prompt": "Q:0+40;A:",
        "expected": "40",
        "generated": "7333",
        "ended": false
      },
      {
        "prompt": "Q:0+59;A:",
        "expected": "59",
        "generated": "1745",
        "ended": false
      },
      {
        "prompt": "Q:0+75;A:",
        "expected": "75",
        "generated": "77=",
        "ended": true
      },
      {
        "prompt": "Q:1+17;A:",
        "expected": "18",
        "generated": "728=",
        "ended": false
      },
      {
        "prompt": "Q:1+20;A:",
        "expected": "21",
        "generated": "7333",
        "ended": false
      },
      {
        "prompt": "Q:1+43;A:",
        "expected": "44",
        "generated": "7410",
        "ended": false
      },
      {
        "prompt": "Q:1+45;A:",
        "expected": "46",
        "generated": "7344",
        "ended": false
      }
    ]
  },
  "selected_validation": {
    "examples": 1006,
    "correct": 993,
    "accuracy": 0.9870775347912525,
    "terminated": 1005,
    "next_token_loss": 10.664878927251344,
    "answer_nll": 0.013468235900648896,
    "mistakes": [
      {
        "prompt": "Q:1+79;A:",
        "expected": "80",
        "generated": "70",
        "ended": true
      },
      {
        "prompt": "Q:5+99;A:",
        "expected": "104",
        "generated": "103",
        "ended": true
      },
      {
        "prompt": "Q:6+98;A:",
        "expected": "104",
        "generated": "103",
        "ended": true
      },
      {
        "prompt": "Q:7+97;A:",
        "expected": "104",
        "generated": "103",
        "ended": true
      },
      {
        "prompt": "Q:8+70;A:",
        "expected": "78",
        "generated": "79",
        "ended": true
      },
      {
        "prompt": "Q:9+81;A:",
        "expected": "90",
        "generated": "80",
        "ended": true
      },
      {
        "prompt": "Q:9+92;A:",
        "expected": "101",
        "generated": "11",
        "ended": true
      },
      {
        "prompt": "Q:59+1;A:",
        "expected": "60",
        "generated": "50",
        "ended": true
      },
      {
        "prompt": "Q:79+1;A:",
        "expected": "80",
        "generated": "70",
        "ended": true
      },
      {
        "prompt": "Q:92+9;A:",
        "expected": "101",
        "generated": "1013",
        "ended": false
      }
    ]
  },
  "history": [
    {
      "epoch": 1,
      "steps": 63,
      "completion_loss": 1.527765610880175,
      "validation": {
        "examples": 1006,
        "correct": 11,
        "accuracy": 0.010934393638170975,
        "terminated": 999,
        "next_token_loss": 9.425520191625598,
        "answer_nll": 1.4194261987054715,
        "mistakes": [
          {
            "prompt": "Q:0+16;A:",
            "expected": "16",
            "generated": "9",
            "ended": true
          },
          {
            "prompt": "Q:0+22;A:",
            "expected": "22",
            "generated": "8",
            "ended": true
          },
          {
            "prompt": "Q:0+35;A:",
            "expected": "35",
            "generated": "9",
            "ended": true
          },
          {
            "prompt": "Q:0+40;A:",
            "expected": "40",
            "generated": "83",
            "ended": true
          },
          {
            "prompt": "Q:0+59;A:",
            "expected": "59",
            "generated": "10",
            "ended": true
          },
          {
            "prompt": "Q:0+75;A:",
            "expected": "75",
            "generated": "9",
            "ended": true
          },
          {
            "prompt": "Q:1+17;A:",
            "expected": "18",
            "generated": "90",
            "ended": true
          },
          {
            "prompt": "Q:1+20;A:",
            "expected": "21",
            "generated": "8",
            "ended": true
          },
          {
            "prompt": "Q:1+43;A:",
            "expected": "44",
            "generated": "84",
            "ended": true
          },
          {
            "prompt": "Q:1+45;A:",
            "expected": "46",
            "generated": "80",
            "ended": true
          }
        ]
      },
      "canonical_validation": {
        "examples": 1006,
        "correct": 1000,
        "accuracy": 0.9940357852882704,
        "terminated": 1006,
        "next_token_loss": 1.4668244476709276,
        "answer_nll": 0.007099811398806697,
        "mistakes": [
          {
            "prompt": "60+1=",
            "expected": "61",
            "generated": "62",
            "ended": true
          },
          {
            "prompt": "97+2=",
            "expected": "99",
            "generated": "100",
            "ended": true
          },
          {
            "prompt": "3+8=",
            "expected": "11",
            "generated": "12",
            "ended": true
          },
          {
            "prompt": "6+9=",
            "expected": "15",
            "generated": "16",
            "ended": true
          },
          {
            "prompt": "8+3=",
            "expected": "11",
            "generated": "12",
            "ended": true
          },
          {
            "prompt": "65+85=",
            "expected": "150",
            "generated": "151",
            "ended": true
          }
        ]
      }
    },
    {
      "epoch": 2,
      "steps": 126,
      "completion_loss": 0.5778569703630839
    },
    {
      "epoch": 3,
      "steps": 189,
      "completion_loss": 0.33876756140381437
    },
    {
      "epoch": 4,
      "steps": 252,
      "completion_loss": 0.22163651905753454
    },
    {
      "epoch": 5,
      "steps": 315,
      "completion_loss": 0.1690951292500955,
      "validation": {
        "examples": 1006,
        "correct": 638,
        "accuracy": 0.6341948310139165,
        "terminated": 1002,
        "next_token_loss": 9.995092977617235,
        "answer_nll": 0.33587731978727825,
        "mistakes": [
          {
            "prompt": "Q:0+16;A:",
            "expected": "16",
            "generated": "36",
            "ended": true
          },
          {
            "prompt": "Q:0+22;A:",
            "expected": "22",
            "generated": "33",
            "ended": true
          },
          {
            "prompt": "Q:0+40;A:",
            "expected": "40",
            "generated": "33",
            "ended": true
          },
          {
            "prompt": "Q:0+75;A:",
            "expected": "75",
            "generated": "74",
            "ended": true
          },
          {
            "prompt": "Q:1+17;A:",
            "expected": "18",
            "generated": "27",
            "ended": true
          },
          {
            "prompt": "Q:1+20;A:",
            "expected": "21",
            "generated": "31",
            "ended": true
          },
          {
            "prompt": "Q:1+43;A:",
            "expected": "44",
            "generated": "36",
            "ended": true
          },
          {
            "prompt": "Q:1+45;A:",
            "expected": "46",
            "generated": "30",
            "ended": true
          },
          {
            "prompt": "Q:1+47;A:",
            "expected": "48",
            "generated": "35",
            "ended": true
          },
          {
            "prompt": "Q:1+52;A:",
            "expected": "53",
            "generated": "36",
            "ended": true
          }
        ]
      },
      "canonical_validation": {
        "examples": 1006,
        "correct": 1001,
        "accuracy": 0.9950298210735586,
        "terminated": 1006,
        "next_token_loss": 1.7702325288298597,
        "answer_nll": 0.009143132737607815,
        "mistakes": [
          {
            "prompt": "3+8=",
            "expected": "11",
            "generated": "12",
            "ended": true
          },
          {
            "prompt": "6+9=",
            "expected": "15",
            "generated": "14",
            "ended": true
          },
          {
            "prompt": "8+3=",
            "expected": "11",
            "generated": "13",
            "ended": true
          },
          {
            "prompt": "9+6=",
            "expected": "15",
            "generated": "16",
            "ended": true
          },
          {
            "prompt": "87+89=",
            "expected": "176",
            "generated": "166",
            "ended": true
          }
        ]
      }
    },
    {
      "epoch": 6,
      "steps": 378,
      "completion_loss": 0.14018472501588472
    },
    {
      "epoch": 7,
      "steps": 441,
      "completion_loss": 0.11621235766120715
    },
    {
      "epoch": 8,
      "steps": 504,
      "completion_loss": 0.10047735822483352
    },
    {
      "epoch": 9,
      "steps": 567,
      "completion_loss": 0.08868616548081514
    },
    {
      "epoch": 10,
      "steps": 630,
      "completion_loss": 0.07728818968572537,
      "validation": {
        "examples": 1006,
        "correct": 793,
        "accuracy": 0.7882703777335984,
        "terminated": 1006,
        "next_token_loss": 11.0228030843974,
        "answer_nll": 0.17760633961196917,
        "mistakes": [
          {
            "prompt": "Q:0+16;A:",
            "expected": "16",
            "generated": "26",
            "ended": true
          },
          {
            "prompt": "Q:0+22;A:",
            "expected": "22",
            "generated": "2",
            "ended": true
          },
          {
            "prompt": "Q:0+40;A:",
            "expected": "40",
            "generated": "25",
            "ended": true
          },
          {
            "prompt": "Q:1+17;A:",
            "expected": "18",
            "generated": "28",
            "ended": true
          },
          {
            "prompt": "Q:1+20;A:",
            "expected": "21",
            "generated": "22",
            "ended": true
          },
          {
            "prompt": "Q:1+43;A:",
            "expected": "44",
            "generated": "35",
            "ended": true
          },
          {
            "prompt": "Q:1+45;A:",
            "expected": "46",
            "generated": "36",
            "ended": true
          },
          {
            "prompt": "Q:1+47;A:",
            "expected": "48",
            "generated": "38",
            "ended": true
          },
          {
            "prompt": "Q:1+52;A:",
            "expected": "53",
            "generated": "25",
            "ended": true
          },
          {
            "prompt": "Q:1+72;A:",
            "expected": "73",
            "generated": "74",
            "ended": true
          }
        ]
      },
      "canonical_validation": {
        "examples": 1006,
        "correct": 999,
        "accuracy": 0.9930417495029821,
        "terminated": 1005,
        "next_token_loss": 1.9175293594217016,
        "answer_nll": 0.007424167771659478,
        "mistakes": [
          {
            "prompt": "2+99=",
            "expected": "101",
            "generated": "1111",
            "ended": false
          },
          {
            "prompt": "92+9=",
            "expected": "101",
            "generated": "102",
            "ended": true
          },
          {
            "prompt": "99+2=",
            "expected": "101",
            "generated": "102",
            "ended": true
          },
          {
            "prompt": "3+8=",
            "expected": "11",
            "generated": "12",
            "ended": true
          },
          {
            "prompt": "6+9=",
            "expected": "15",
            "generated": "14",
            "ended": true
          },
          {
            "prompt": "8+3=",
            "expected": "11",
            "generated": "12",
            "ended": true
          },
          {
            "prompt": "9+6=",
            "expected": "15",
            "generated": "16",
            "ended": true
          }
        ]
      }
    },
    {
      "epoch": 11,
      "steps": 693,
      "completion_loss": 0.06647823451489672
    },
    {
      "epoch": 12,
      "steps": 756,
      "completion_loss": 0.05578287633691499
    },
    {
      "epoch": 13,
      "steps": 819,
      "completion_loss": 0.048298227231153325
    },
    {
      "epoch": 14,
      "steps": 882,
      "completion_loss": 0.0366209562039274
    },
    {
      "epoch": 15,
      "steps": 945,
      "completion_loss": 0.032086668023384224,
      "validation": {
        "examples": 1006,
        "correct": 917,
        "accuracy": 0.911530815109344,
        "terminated": 1004,
        "next_token_loss": 10.82757283966174,
        "answer_nll": 0.07174393151536058,
        "mistakes": [
          {
            "prompt": "Q:0+16;A:",
            "expected": "16",
            "generated": "17",
            "ended": true
          },
          {
            "prompt": "Q:0+40;A:",
            "expected": "40",
            "generated": "30",
            "ended": true
          },
          {
            "prompt": "Q:1+43;A:",
            "expected": "44",
            "generated": "34",
            "ended": true
          },
          {
            "prompt": "Q:1+45;A:",
            "expected": "46",
            "generated": "36",
            "ended": true
          },
          {
            "prompt": "Q:1+79;A:",
            "expected": "80",
            "generated": "70",
            "ended": true
          },
          {
            "prompt": "Q:2+29;A:",
            "expected": "31",
            "generated": "22",
            "ended": true
          },
          {
            "prompt": "Q:2+99;A:",
            "expected": "101",
            "generated": "100",
            "ended": true
          },
          {
            "prompt": "Q:3+14;A:",
            "expected": "17",
            "generated": "26",
            "ended": true
          },
          {
            "prompt": "Q:3+75;A:",
            "expected": "78",
            "generated": "79",
            "ended": true
          },
          {
            "prompt": "Q:4+56;A:",
            "expected": "60",
            "generated": "50",
            "ended": true
          }
        ]
      },
      "canonical_validation": {
        "examples": 1006,
        "correct": 1003,
        "accuracy": 0.9970178926441352,
        "terminated": 1005,
        "next_token_loss": 2.1022932018407077,
        "answer_nll": 0.005917645778943518,
        "mistakes": [
          {
            "prompt": "2+99=",
            "expected": "101",
            "generated": "1111",
            "ended": false
          },
          {
            "prompt": "6+9=",
            "expected": "15",
            "generated": "14",
            "ended": true
          },
          {
            "prompt": "42+19=",
            "expected": "61",
            "generated": "611",
            "ended": true
          }
        ]
      }
    },
    {
      "epoch": 16,
      "steps": 1008,
      "completion_loss": 0.026365159402361258
    },
    {
      "epoch": 17,
      "steps": 1071,
      "completion_loss": 0.021907041554700555
    },
    {
      "epoch": 18,
      "steps": 1134,
      "completion_loss": 0.020510065558961926
    },
    {
      "epoch": 19,
      "steps": 1197,
      "completion_loss": 0.017785167122067513
    },
    {
      "epoch": 20,
      "steps": 1260,
      "completion_loss": 0.012694605086861474,
      "validation": {
        "examples": 1006,
        "correct": 947,
        "accuracy": 0.941351888667992,
        "terminated": 1005,
        "next_token_loss": 11.082646997741188,
        "answer_nll": 0.044868044478901414,
        "mistakes": [
          {
            "prompt": "Q:0+40;A:",
            "expected": "40",
            "generated": "30",
            "ended": true
          },
          {
            "prompt": "Q:1+79;A:",
            "expected": "80",
            "generated": "70",
            "ended": true
          },
          {
            "prompt": "Q:1+93;A:",
            "expected": "94",
            "generated": "95",
            "ended": true
          },
          {
            "prompt": "Q:2+29;A:",
            "expected": "31",
            "generated": "22",
            "ended": true
          },
          {
            "prompt": "Q:2+63;A:",
            "expected": "65",
            "generated": "66",
            "ended": true
          },
          {
            "prompt": "Q:2+97;A:",
            "expected": "99",
            "generated": "100",
            "ended": true
          },
          {
            "prompt": "Q:2+99;A:",
            "expected": "101",
            "generated": "102",
            "ended": true
          },
          {
            "prompt": "Q:3+14;A:",
            "expected": "17",
            "generated": "18",
            "ended": true
          },
          {
            "prompt": "Q:4+56;A:",
            "expected": "60",
            "generated": "51",
            "ended": true
          },
          {
            "prompt": "Q:5+65;A:",
            "expected": "70",
            "generated": "60",
            "ended": true
          }
        ]
      },
      "canonical_validation": {
        "examples": 1006,
        "correct": 1002,
        "accuracy": 0.9960238568588469,
        "terminated": 1004,
        "next_token_loss": 2.137254149765158,
        "answer_nll": 0.00620642124725012,
        "mistakes": [
          {
            "prompt": "2+99=",
            "expected": "101",
            "generated": "1111",
            "ended": false
          },
          {
            "prompt": "3+7=",
            "expected": "10",
            "generated": "1057",
            "ended": false
          },
          {
            "prompt": "6+9=",
            "expected": "15",
            "generated": "14",
            "ended": true
          },
          {
            "prompt": "7+8=",
            "expected": "15",
            "generated": "16",
            "ended": true
          }
        ]
      }
    },
    {
      "epoch": 21,
      "steps": 1323,
      "completion_loss": 0.012670972812226245
    },
    {
      "epoch": 22,
      "steps": 1386,
      "completion_loss": 0.012141972380553265
    },
    {
      "epoch": 23,
      "steps": 1449,
      "completion_loss": 0.011421972002402598
    },
    {
      "epoch": 24,
      "steps": 1512,
      "completion_loss": 0.010395661452649025
    },
    {
      "epoch": 25,
      "steps": 1575,
      "completion_loss": 0.008496210330709398,
      "validation": {
        "examples": 1006,
        "correct": 974,
        "accuracy": 0.9681908548707754,
        "terminated": 1005,
        "next_token_loss": 10.664347476571834,
        "answer_nll": 0.027660536277823073,
        "mistakes": [
          {
            "prompt": "Q:1+79;A:",
            "expected": "80",
            "generated": "70",
            "ended": true
          },
          {
            "prompt": "Q:1+93;A:",
            "expected": "94",
            "generated": "95",
            "ended": true
          },
          {
            "prompt": "Q:2+94;A:",
            "expected": "96",
            "generated": "102",
            "ended": true
          },
          {
            "prompt": "Q:2+97;A:",
            "expected": "99",
            "generated": "100",
            "ended": true
          },
          {
            "prompt": "Q:3+14;A:",
            "expected": "17",
            "generated": "18",
            "ended": true
          },
          {
            "prompt": "Q:4+56;A:",
            "expected": "60",
            "generated": "51",
            "ended": true
          },
          {
            "prompt": "Q:5+56;A:",
            "expected": "61",
            "generated": "62",
            "ended": true
          },
          {
            "prompt": "Q:5+65;A:",
            "expected": "70",
            "generated": "61",
            "ended": true
          },
          {
            "prompt": "Q:5+74;A:",
            "expected": "79",
            "generated": "89",
            "ended": true
          },
          {
            "prompt": "Q:5+99;A:",
            "expected": "104",
            "generated": "105",
            "ended": true
          }
        ]
      },
      "canonical_validation": {
        "examples": 1006,
        "correct": 1004,
        "accuracy": 0.9980119284294234,
        "terminated": 1005,
        "next_token_loss": 2.278216745586705,
        "answer_nll": 0.0031518290440241494,
        "mistakes": [
          {
            "prompt": "2+99=",
            "expected": "101",
            "generated": "1111",
            "ended": false
          },
          {
            "prompt": "6+9=",
            "expected": "15",
            "generated": "14",
            "ended": true
          }
        ]
      }
    },
    {
      "epoch": 26,
      "steps": 1638,
      "completion_loss": 0.0077872321032913154
    },
    {
      "epoch": 27,
      "steps": 1701,
      "completion_loss": 0.007644684895407092
    },
    {
      "epoch": 28,
      "steps": 1764,
      "completion_loss": 0.00797774783215587
    },
    {
      "epoch": 29,
      "steps": 1827,
      "completion_loss": 0.006823385555693825
    },
    {
      "epoch": 30,
      "steps": 1890,
      "completion_loss": 0.007132385103362094,
      "validation": {
        "examples": 1006,
        "correct": 988,
        "accuracy": 0.9821073558648111,
        "terminated": 1005,
        "next_token_loss": 10.612550578237007,
        "answer_nll": 0.018946862464878745,
        "mistakes": [
          {
            "prompt": "Q:1+79;A:",
            "expected": "80",
            "generated": "70",
            "ended": true
          },
          {
            "prompt": "Q:2+97;A:",
            "expected": "99",
            "generated": "100",
            "ended": true
          },
          {
            "prompt": "Q:5+65;A:",
            "expected": "70",
            "generated": "60",
            "ended": true
          },
          {
            "prompt": "Q:7+52;A:",
            "expected": "59",
            "generated": "69",
            "ended": true
          },
          {
            "prompt": "Q:8+20;A:",
            "expected": "28",
            "generated": "38",
            "ended": true
          },
          {
            "prompt": "Q:9+92;A:",
            "expected": "101",
            "generated": "110",
            "ended": true
          },
          {
            "prompt": "Q:19+5;A:",
            "expected": "24",
            "generated": "2",
            "ended": true
          },
          {
            "prompt": "Q:43+1;A:",
            "expected": "44",
            "generated": "45",
            "ended": true
          },
          {
            "prompt": "Q:59+1;A:",
            "expected": "60",
            "generated": "50",
            "ended": true
          },
          {
            "prompt": "Q:79+1;A:",
            "expected": "80",
            "generated": "79",
            "ended": true
          }
        ]
      },
      "canonical_validation": {
        "examples": 1006,
        "correct": 1003,
        "accuracy": 0.9970178926441352,
        "terminated": 1005,
        "next_token_loss": 2.2384983391765854,
        "answer_nll": 0.0038125094225517856,
        "mistakes": [
          {
            "prompt": "7+33=",
            "expected": "40",
            "generated": "30",
            "ended": true
          },
          {
            "prompt": "6+9=",
            "expected": "15",
            "generated": "14",
            "ended": true
          },
          {
            "prompt": "8+3=",
            "expected": "11",
            "generated": "1173",
            "ended": false
          }
        ]
      }
    },
    {
      "epoch": 31,
      "steps": 1953,
      "completion_loss": 0.006831891652904954
    },
    {
      "epoch": 32,
      "steps": 2016,
      "completion_loss": 0.005453315803029758
    },
    {
      "epoch": 33,
      "steps": 2079,
      "completion_loss": 0.005638716303998863
    },
    {
      "epoch": 34,
      "steps": 2142,
      "completion_loss": 0.005177072764064448
    },
    {
      "epoch": 35,
      "steps": 2205,
      "completion_loss": 0.005761288609486252,
      "validation": {
        "examples": 1006,
        "correct": 993,
        "accuracy": 0.9870775347912525,
        "terminated": 1005,
        "next_token_loss": 10.664878927251344,
        "answer_nll": 0.013468235900648896,
        "mistakes": [
          {
            "prompt": "Q:1+79;A:",
            "expected": "80",
            "generated": "70",
            "ended": true
          },
          {
            "prompt": "Q:5+99;A:",
            "expected": "104",
            "generated": "103",
            "ended": true
          },
          {
            "prompt": "Q:6+98;A:",
            "expected": "104",
            "generated": "103",
            "ended": true
          },
          {
            "prompt": "Q:7+97;A:",
            "expected": "104",
            "generated": "103",
            "ended": true
          },
          {
            "prompt": "Q:8+70;A:",
            "expected": "78",
            "generated": "79",
            "ended": true
          },
          {
            "prompt": "Q:9+81;A:",
            "expected": "90",
            "generated": "80",
            "ended": true
          },
          {
            "prompt": "Q:9+92;A:",
            "expected": "101",
            "generated": "11",
            "ended": true
          },
          {
            "prompt": "Q:59+1;A:",
            "expected": "60",
            "generated": "50",
            "ended": true
          },
          {
            "prompt": "Q:79+1;A:",
            "expected": "80",
            "generated": "70",
            "ended": true
          },
          {
            "prompt": "Q:92+9;A:",
            "expected": "101",
            "generated": "1013",
            "ended": false
          }
        ]
      },
      "canonical_validation": {
        "examples": 1006,
        "correct": 1005,
        "accuracy": 0.9990059642147118,
        "terminated": 1006,
        "next_token_loss": 2.4202768796534744,
        "answer_nll": 0.00366585306713063,
        "mistakes": [
          {
            "prompt": "6+9=",
            "expected": "15",
            "generated": "14",
            "ended": true
          }
        ]
      }
    },
    {
      "epoch": 36,
      "steps": 2268,
      "completion_loss": 0.005297642415140133
    },
    {
      "epoch": 37,
      "steps": 2331,
      "completion_loss": 0.005272457337436909
    },
    {
      "epoch": 38,
      "steps": 2394,
      "completion_loss": 0.0046861867614274395
    },
    {
      "epoch": 39,
      "steps": 2457,
      "completion_loss": 0.0036811195635514443
    },
    {
      "epoch": 40,
      "steps": 2520,
      "completion_loss": 0.003812676349852921,
      "validation": {
        "examples": 1006,
        "correct": 989,
        "accuracy": 0.9831013916500994,
        "terminated": 1005,
        "next_token_loss": 10.549113101571834,
        "answer_nll": 0.016007900644894754,
        "mistakes": [
          {
            "prompt": "Q:4+68;A:",
            "expected": "72",
            "generated": "62",
            "ended": true
          },
          {
            "prompt": "Q:5+65;A:",
            "expected": "70",
            "generated": "60",
            "ended": true
          },
          {
            "prompt": "Q:5+68;A:",
            "expected": "73",
            "generated": "63",
            "ended": true
          },
          {
            "prompt": "Q:5+74;A:",
            "expected": "79",
            "generated": "89",
            "ended": true
          },
          {
            "prompt": "Q:6+98;A:",
            "expected": "104",
            "generated": "103",
            "ended": true
          },
          {
            "prompt": "Q:7+33;A:",
            "expected": "40",
            "generated": "30",
            "ended": true
          },
          {
            "prompt": "Q:9+81;A:",
            "expected": "90",
            "generated": "80",
            "ended": true
          },
          {
            "prompt": "Q:9+92;A:",
            "expected": "101",
            "generated": "11",
            "ended": true
          },
          {
            "prompt": "Q:59+1;A:",
            "expected": "60",
            "generated": "50",
            "ended": true
          },
          {
            "prompt": "Q:79+1;A:",
            "expected": "80",
            "generated": "70",
            "ended": true
          }
        ]
      },
      "canonical_validation": {
        "examples": 1006,
        "correct": 1003,
        "accuracy": 0.9970178926441352,
        "terminated": 1006,
        "next_token_loss": 2.275065538518761,
        "answer_nll": 0.003530404804882607,
        "mistakes": [
          {
            "prompt": "6+98=",
            "expected": "104",
            "generated": "103",
            "ended": true
          },
          {
            "prompt": "3+8=",
            "expected": "11",
            "generated": "10",
            "ended": true
          },
          {
            "prompt": "6+9=",
            "expected": "15",
            "generated": "14",
            "ended": true
          }
        ]
      }
    }
  ],
  "elapsed_seconds": 37.87056825007312,
  "training_rows": 15992,
  "maximum_epochs": 40,
  "test_used_for_selection": false
}