{
  "protocol": "four output tokens including EOS; frozen selected policies",
  "policies": {
    "base": {
      "checkpoint_sha256": "d9195e330fe72aed659e067010515a6264ad5da4f1462de17f6389afdfbdd84b",
      "formatted": false,
      "test": {
        "examples": 502,
        "correct": 501,
        "terminated": 502,
        "slices": {
          "carry_True": {
            "examples": 352,
            "correct": 351,
            "terminated": 352
          },
          "carry_False": {
            "examples": 150,
            "correct": 150,
            "terminated": 150
          },
          "digits_1": {
            "examples": 5,
            "correct": 5,
            "terminated": 5
          },
          "digits_2": {
            "examples": 256,
            "correct": 255,
            "terminated": 256
          },
          "digits_3": {
            "examples": 241,
            "correct": 241,
            "terminated": 241
          }
        },
        "mistakes": [
          {
            "id": "add:9+2",
            "prompt": "9+2=",
            "expected": "11",
            "generated": "10",
            "token_ids": [
              20,
              19,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 2
          }
        ]
      },
      "ood": {
        "examples": 497,
        "correct": 0,
        "terminated": 497,
        "slices": {
          "carry_True": {
            "examples": 384,
            "correct": 0,
            "terminated": 384
          },
          "carry_False": {
            "examples": 113,
            "correct": 0,
            "terminated": 113
          },
          "digits_1": {
            "examples": 0,
            "correct": 0,
            "terminated": 0
          },
          "digits_2": {
            "examples": 0,
            "correct": 0,
            "terminated": 0
          },
          "digits_3": {
            "examples": 497,
            "correct": 0,
            "terminated": 497
          }
        },
        "mistakes": [
          {
            "id": "add:0+134",
            "prompt": "0+134=",
            "expected": "134",
            "generated": "34",
            "token_ids": [
              22,
              23,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:1+102",
            "prompt": "1+102=",
            "expected": "103",
            "generated": "1",
            "token_ids": [
              20,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:1+151",
            "prompt": "1+151=",
            "expected": "152",
            "generated": "62",
            "token_ids": [
              25,
              21,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:1+182",
            "prompt": "1+182=",
            "expected": "183",
            "generated": "93",
            "token_ids": [
              28,
              22,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:2+107",
            "prompt": "2+107=",
            "expected": "109",
            "generated": "32",
            "token_ids": [
              22,
              21,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:2+125",
            "prompt": "2+125=",
            "expected": "127",
            "generated": "43",
            "token_ids": [
              23,
              22,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:3+150",
            "prompt": "3+150=",
            "expected": "153",
            "generated": "61",
            "token_ids": [
              25,
              20,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:3+155",
            "prompt": "3+155=",
            "expected": "158",
            "generated": "78",
            "token_ids": [
              26,
              27,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:4+112",
            "prompt": "4+112=",
            "expected": "116",
            "generated": "53",
            "token_ids": [
              24,
              22,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:7+145",
            "prompt": "7+145=",
            "expected": "152",
            "generated": "71",
            "token_ids": [
              26,
              20,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 3
          },
          {
            "id": "add:7+159",
            "prompt": "7+159=",
            "expected": "166",
            "generated": "160",
            "token_ids": [
              20,
              25,
              19,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 3
          },
          {
            "id": "add:7+165",
            "prompt": "7+165=",
            "expected": "172",
            "generated": "92",
            "token_ids": [
              28,
              21,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 3
          }
        ]
      }
    },
    "first": {
      "checkpoint_sha256": "14b2986a1e3b7eca2c663cf9a57dc942dd3bc1e809ff23122f5e4de6e40f1734",
      "formatted": false,
      "test": {
        "examples": 502,
        "correct": 495,
        "terminated": 502,
        "slices": {
          "carry_True": {
            "examples": 352,
            "correct": 349,
            "terminated": 352
          },
          "carry_False": {
            "examples": 150,
            "correct": 146,
            "terminated": 150
          },
          "digits_1": {
            "examples": 5,
            "correct": 4,
            "terminated": 5
          },
          "digits_2": {
            "examples": 256,
            "correct": 251,
            "terminated": 256
          },
          "digits_3": {
            "examples": 241,
            "correct": 240,
            "terminated": 241
          }
        },
        "mistakes": [
          {
            "id": "add:0+8",
            "prompt": "0+8=",
            "expected": "8",
            "generated": "9",
            "token_ids": [
              28,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 1
          },
          {
            "id": "add:9+2",
            "prompt": "9+2=",
            "expected": "11",
            "generated": "10",
            "token_ids": [
              20,
              19,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 2
          },
          {
            "id": "add:7+64",
            "prompt": "7+64=",
            "expected": "71",
            "generated": "712",
            "token_ids": [
              26,
              20,
              21,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 2
          },
          {
            "id": "add:13+86",
            "prompt": "13+86=",
            "expected": "99",
            "generated": "109",
            "token_ids": [
              20,
              19,
              28,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 2
          },
          {
            "id": "add:20+50",
            "prompt": "20+50=",
            "expected": "70",
            "generated": "60",
            "token_ids": [
              25,
              19,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 2
          },
          {
            "id": "add:48+69",
            "prompt": "48+69=",
            "expected": "117",
            "generated": "107",
            "token_ids": [
              20,
              19,
              26,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 3
          },
          {
            "id": "add:81+16",
            "prompt": "81+16=",
            "expected": "97",
            "generated": "107",
            "token_ids": [
              20,
              19,
              26,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 2
          }
        ]
      },
      "ood": {
        "examples": 497,
        "correct": 0,
        "terminated": 497,
        "slices": {
          "carry_True": {
            "examples": 384,
            "correct": 0,
            "terminated": 384
          },
          "carry_False": {
            "examples": 113,
            "correct": 0,
            "terminated": 113
          },
          "digits_1": {
            "examples": 0,
            "correct": 0,
            "terminated": 0
          },
          "digits_2": {
            "examples": 0,
            "correct": 0,
            "terminated": 0
          },
          "digits_3": {
            "examples": 497,
            "correct": 0,
            "terminated": 497
          }
        },
        "mistakes": [
          {
            "id": "add:0+134",
            "prompt": "0+134=",
            "expected": "134",
            "generated": "20",
            "token_ids": [
              21,
              19,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:1+102",
            "prompt": "1+102=",
            "expected": "103",
            "generated": "20",
            "token_ids": [
              21,
              19,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:1+151",
            "prompt": "1+151=",
            "expected": "152",
            "generated": "50",
            "token_ids": [
              24,
              19,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:1+182",
            "prompt": "1+182=",
            "expected": "183",
            "generated": "10",
            "token_ids": [
              20,
              19,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:2+107",
            "prompt": "2+107=",
            "expected": "109",
            "generated": "23",
            "token_ids": [
              21,
              22,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:2+125",
            "prompt": "2+125=",
            "expected": "127",
            "generated": "32",
            "token_ids": [
              22,
              21,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:3+150",
            "prompt": "3+150=",
            "expected": "153",
            "generated": "59",
            "token_ids": [
              24,
              28,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:3+155",
            "prompt": "3+155=",
            "expected": "158",
            "generated": "69",
            "token_ids": [
              25,
              28,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:4+112",
            "prompt": "4+112=",
            "expected": "116",
            "generated": "44",
            "token_ids": [
              23,
              23,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:7+145",
            "prompt": "7+145=",
            "expected": "152",
            "generated": "80",
            "token_ids": [
              27,
              19,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 3
          },
          {
            "id": "add:7+159",
            "prompt": "7+159=",
            "expected": "166",
            "generated": "87",
            "token_ids": [
              27,
              26,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 3
          },
          {
            "id": "add:7+165",
            "prompt": "7+165=",
            "expected": "172",
            "generated": "90",
            "token_ids": [
              28,
              19,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 3
          }
        ]
      }
    },
    "finetuned": {
      "checkpoint_sha256": "66f53eaff2a81219b68051e169d90b302bc1bcb019b1db073f71acaaac5413f6",
      "formatted": true,
      "test": {
        "examples": 502,
        "correct": 486,
        "terminated": 502,
        "slices": {
          "carry_True": {
            "examples": 352,
            "correct": 340,
            "terminated": 352
          },
          "carry_False": {
            "examples": 150,
            "correct": 146,
            "terminated": 150
          },
          "digits_1": {
            "examples": 5,
            "correct": 3,
            "terminated": 5
          },
          "digits_2": {
            "examples": 256,
            "correct": 246,
            "terminated": 256
          },
          "digits_3": {
            "examples": 241,
            "correct": 237,
            "terminated": 241
          }
        },
        "mistakes": [
          {
            "id": "add:0+4",
            "prompt": "Q:0+4;A:",
            "expected": "4",
            "generated": "5",
            "token_ids": [
              24,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 1
          },
          {
            "id": "add:2+9",
            "prompt": "Q:2+9;A:",
            "expected": "11",
            "generated": "10",
            "token_ids": [
              20,
              19,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 2
          },
          {
            "id": "add:4+0",
            "prompt": "Q:4+0;A:",
            "expected": "4",
            "generated": "5",
            "token_ids": [
              24,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 1
          },
          {
            "id": "add:9+2",
            "prompt": "Q:9+2;A:",
            "expected": "11",
            "generated": "10",
            "token_ids": [
              20,
              19,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 2
          },
          {
            "id": "add:4+13",
            "prompt": "Q:4+13;A:",
            "expected": "17",
            "generated": "16",
            "token_ids": [
              20,
              25,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 2
          },
          {
            "id": "add:5+25",
            "prompt": "Q:5+25;A:",
            "expected": "30",
            "generated": "20",
            "token_ids": [
              21,
              19,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 2
          },
          {
            "id": "add:6+99",
            "prompt": "Q:6+99;A:",
            "expected": "105",
            "generated": "103",
            "token_ids": [
              20,
              19,
              22,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 3
          },
          {
            "id": "add:7+96",
            "prompt": "Q:7+96;A:",
            "expected": "103",
            "generated": "102",
            "token_ids": [
              20,
              19,
              21,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 3
          },
          {
            "id": "add:9+20",
            "prompt": "Q:9+20;A:",
            "expected": "29",
            "generated": "39",
            "token_ids": [
              22,
              28,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 2
          },
          {
            "id": "add:9+91",
            "prompt": "Q:9+91;A:",
            "expected": "100",
            "generated": "90",
            "token_ids": [
              28,
              19,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 3
          },
          {
            "id": "add:24+7",
            "prompt": "Q:24+7;A:",
            "expected": "31",
            "generated": "30",
            "token_ids": [
              22,
              19,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 2
          },
          {
            "id": "add:59+8",
            "prompt": "Q:59+8;A:",
            "expected": "67",
            "generated": "68",
            "token_ids": [
              25,
              27,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 2
          }
        ]
      },
      "ood": {
        "examples": 497,
        "correct": 0,
        "terminated": 497,
        "slices": {
          "carry_True": {
            "examples": 384,
            "correct": 0,
            "terminated": 384
          },
          "carry_False": {
            "examples": 113,
            "correct": 0,
            "terminated": 113
          },
          "digits_1": {
            "examples": 0,
            "correct": 0,
            "terminated": 0
          },
          "digits_2": {
            "examples": 0,
            "correct": 0,
            "terminated": 0
          },
          "digits_3": {
            "examples": 497,
            "correct": 0,
            "terminated": 497
          }
        },
        "mistakes": [
          {
            "id": "add:0+134",
            "prompt": "Q:0+134;A:",
            "expected": "134",
            "generated": "5",
            "token_ids": [
              24,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:1+102",
            "prompt": "Q:1+102;A:",
            "expected": "103",
            "generated": "1",
            "token_ids": [
              20,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:1+151",
            "prompt": "Q:1+151;A:",
            "expected": "152",
            "generated": "6",
            "token_ids": [
              25,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:1+182",
            "prompt": "Q:1+182;A:",
            "expected": "183",
            "generated": "8",
            "token_ids": [
              27,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:2+107",
            "prompt": "Q:2+107;A:",
            "expected": "109",
            "generated": "2",
            "token_ids": [
              21,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:2+125",
            "prompt": "Q:2+125;A:",
            "expected": "127",
            "generated": "4",
            "token_ids": [
              23,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:3+150",
            "prompt": "Q:3+150;A:",
            "expected": "153",
            "generated": "6",
            "token_ids": [
              25,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:3+155",
            "prompt": "Q:3+155;A:",
            "expected": "158",
            "generated": "6",
            "token_ids": [
              25,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:4+112",
            "prompt": "Q:4+112;A:",
            "expected": "116",
            "generated": "5",
            "token_ids": [
              24,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:7+145",
            "prompt": "Q:7+145;A:",
            "expected": "152",
            "generated": "9",
            "token_ids": [
              28,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 3
          },
          {
            "id": "add:7+159",
            "prompt": "Q:7+159;A:",
            "expected": "166",
            "generated": "9",
            "token_ids": [
              28,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 3
          },
          {
            "id": "add:7+165",
            "prompt": "Q:7+165;A:",
            "expected": "172",
            "generated": "9",
            "token_ids": [
              28,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 3
          }
        ]
      },
      "canonical_test_regression": {
        "examples": 502,
        "correct": 501,
        "terminated": 502,
        "slices": {
          "carry_True": {
            "examples": 352,
            "correct": 351,
            "terminated": 352
          },
          "carry_False": {
            "examples": 150,
            "correct": 150,
            "terminated": 150
          },
          "digits_1": {
            "examples": 5,
            "correct": 5,
            "terminated": 5
          },
          "digits_2": {
            "examples": 256,
            "correct": 256,
            "terminated": 256
          },
          "digits_3": {
            "examples": 241,
            "correct": 240,
            "terminated": 241
          }
        },
        "mistakes": [
          {
            "id": "add:91+9",
            "prompt": "91+9=",
            "expected": "100",
            "generated": "101",
            "token_ids": [
              20,
              19,
              20,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 3
          }
        ]
      }
    },
    "rl": {
      "checkpoint_sha256": "b06108603c8647b50f2d6e5504d86ad30b43fabeba1fda62ef50fee27cdffb05",
      "formatted": true,
      "test": {
        "examples": 502,
        "correct": 447,
        "terminated": 502,
        "slices": {
          "carry_True": {
            "examples": 352,
            "correct": 307,
            "terminated": 352
          },
          "carry_False": {
            "examples": 150,
            "correct": 140,
            "terminated": 150
          },
          "digits_1": {
            "examples": 5,
            "correct": 4,
            "terminated": 5
          },
          "digits_2": {
            "examples": 256,
            "correct": 219,
            "terminated": 256
          },
          "digits_3": {
            "examples": 241,
            "correct": 224,
            "terminated": 241
          }
        },
        "mistakes": [
          {
            "id": "add:2+9",
            "prompt": "Q:2+9;A:",
            "expected": "11",
            "generated": "10",
            "token_ids": [
              20,
              19,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 2
          },
          {
            "id": "add:3+3",
            "prompt": "Q:3+3;A:",
            "expected": "6",
            "generated": "5",
            "token_ids": [
              24,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 1
          },
          {
            "id": "add:9+2",
            "prompt": "Q:9+2;A:",
            "expected": "11",
            "generated": "10",
            "token_ids": [
              20,
              19,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 2
          },
          {
            "id": "add:1+40",
            "prompt": "Q:1+40;A:",
            "expected": "41",
            "generated": "31",
            "token_ids": [
              22,
              20,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 2
          },
          {
            "id": "add:4+13",
            "prompt": "Q:4+13;A:",
            "expected": "17",
            "generated": "16",
            "token_ids": [
              20,
              25,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 2
          },
          {
            "id": "add:4+92",
            "prompt": "Q:4+92;A:",
            "expected": "96",
            "generated": "95",
            "token_ids": [
              28,
              24,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 2
          },
          {
            "id": "add:5+25",
            "prompt": "Q:5+25;A:",
            "expected": "30",
            "generated": "20",
            "token_ids": [
              21,
              19,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 2
          },
          {
            "id": "add:6+99",
            "prompt": "Q:6+99;A:",
            "expected": "105",
            "generated": "103",
            "token_ids": [
              20,
              19,
              22,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 3
          },
          {
            "id": "add:7+64",
            "prompt": "Q:7+64;A:",
            "expected": "71",
            "generated": "72",
            "token_ids": [
              26,
              21,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 2
          },
          {
            "id": "add:7+96",
            "prompt": "Q:7+96;A:",
            "expected": "103",
            "generated": "93",
            "token_ids": [
              28,
              22,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 3
          },
          {
            "id": "add:9+20",
            "prompt": "Q:9+20;A:",
            "expected": "29",
            "generated": "39",
            "token_ids": [
              22,
              28,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 2
          },
          {
            "id": "add:9+91",
            "prompt": "Q:9+91;A:",
            "expected": "100",
            "generated": "90",
            "token_ids": [
              28,
              19,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 3
          }
        ]
      },
      "ood": {
        "examples": 497,
        "correct": 0,
        "terminated": 497,
        "slices": {
          "carry_True": {
            "examples": 384,
            "correct": 0,
            "terminated": 384
          },
          "carry_False": {
            "examples": 113,
            "correct": 0,
            "terminated": 113
          },
          "digits_1": {
            "examples": 0,
            "correct": 0,
            "terminated": 0
          },
          "digits_2": {
            "examples": 0,
            "correct": 0,
            "terminated": 0
          },
          "digits_3": {
            "examples": 497,
            "correct": 0,
            "terminated": 497
          }
        },
        "mistakes": [
          {
            "id": "add:0+134",
            "prompt": "Q:0+134;A:",
            "expected": "134",
            "generated": "4",
            "token_ids": [
              23,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:1+102",
            "prompt": "Q:1+102;A:",
            "expected": "103",
            "generated": "2",
            "token_ids": [
              21,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:1+151",
            "prompt": "Q:1+151;A:",
            "expected": "152",
            "generated": "5",
            "token_ids": [
              24,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:1+182",
            "prompt": "Q:1+182;A:",
            "expected": "183",
            "generated": "8",
            "token_ids": [
              27,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:2+107",
            "prompt": "Q:2+107;A:",
            "expected": "109",
            "generated": "3",
            "token_ids": [
              22,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:2+125",
            "prompt": "Q:2+125;A:",
            "expected": "127",
            "generated": "4",
            "token_ids": [
              23,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:3+150",
            "prompt": "Q:3+150;A:",
            "expected": "153",
            "generated": "6",
            "token_ids": [
              25,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:3+155",
            "prompt": "Q:3+155;A:",
            "expected": "158",
            "generated": "6",
            "token_ids": [
              25,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:4+112",
            "prompt": "Q:4+112;A:",
            "expected": "116",
            "generated": "1",
            "token_ids": [
              20,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": false,
            "digits": 3
          },
          {
            "id": "add:7+145",
            "prompt": "Q:7+145;A:",
            "expected": "152",
            "generated": "8",
            "token_ids": [
              27,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 3
          },
          {
            "id": "add:7+159",
            "prompt": "Q:7+159;A:",
            "expected": "166",
            "generated": "9",
            "token_ids": [
              28,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 3
          },
          {
            "id": "add:7+165",
            "prompt": "Q:7+165;A:",
            "expected": "172",
            "generated": "8",
            "token_ids": [
              27,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 3
          }
        ]
      },
      "canonical_test_regression": {
        "examples": 502,
        "correct": 499,
        "terminated": 502,
        "slices": {
          "carry_True": {
            "examples": 352,
            "correct": 349,
            "terminated": 352
          },
          "carry_False": {
            "examples": 150,
            "correct": 150,
            "terminated": 150
          },
          "digits_1": {
            "examples": 5,
            "correct": 5,
            "terminated": 5
          },
          "digits_2": {
            "examples": 256,
            "correct": 256,
            "terminated": 256
          },
          "digits_3": {
            "examples": 241,
            "correct": 238,
            "terminated": 241
          }
        },
        "mistakes": [
          {
            "id": "add:91+9",
            "prompt": "91+9=",
            "expected": "100",
            "generated": "101",
            "token_ids": [
              20,
              19,
              20,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 3
          },
          {
            "id": "add:62+51",
            "prompt": "62+51=",
            "expected": "113",
            "generated": "123",
            "token_ids": [
              20,
              21,
              22,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 3
          },
          {
            "id": "add:91+60",
            "prompt": "91+60=",
            "expected": "151",
            "generated": "141",
            "token_ids": [
              20,
              23,
              20,
              2
            ],
            "ended": true,
            "correct": false,
            "carry": true,
            "digits": 3
          }
        ]
      }
    }
  },
  "data_sha256": {
    "train": "ade03e599b88310e67759ff2d7e8b180f756aca2fcc55334722496707657757a",
    "validation": "16a51dd1d096915cead427b797fa68d0f784736365fac9007f27702b94c89f50",
    "calibration": "4873b66b36699779e16d56712343153e395f1f19ae8397c3a9a5e19a8f3cedcb",
    "test": "f08fecf01f4299dc86c466c00452e77c913a781b079bc517718ce15373da9d3c",
    "ood": "d626fcaaf3b193283e15d5d1f3d63e92adfb41e47e34874c3c7d4e5b5a1c0f1e"
  }
}