{
  "status": "complete fixed PPO training; held-out evaluation and calibration recorded",
  "supervised_sha256": "66f53eaff2a81219b68051e169d90b302bc1bcb019b1db073f71acaaac5413f6",
  "policy_sha256": "b06108603c8647b50f2d6e5504d86ad30b43fabeba1fda62ef50fee27cdffb05",
  "feedback_kind": "ai",
  "label_files": [
    {
      "vote_file_sha256": "5a1f45341f751311fd37663dda9fa5300d41594f39511380e316c38193b8116d",
      "votes": 120
    }
  ],
  "binary_counts": {
    "train": 84,
    "validation": 17,
    "test": 19
  },
  "ties": 0,
  "reject_both": 0,
  "reward_validation_loss": 0.6112990379333496,
  "reward_heldout": {
    "examples": 19,
    "correct": 6,
    "score_ties": 0,
    "pair_loss": 0.8244773149490356
  },
  "reward_history": [
    {
      "epoch": 1,
      "training_pair_loss": 0.732446531454722,
      "validation_pair_loss": 0.7354885339736938
    },
    {
      "epoch": 2,
      "training_pair_loss": 0.756649374961853,
      "validation_pair_loss": 0.7196974754333496
    },
    {
      "epoch": 3,
      "training_pair_loss": 0.7157258093357086,
      "validation_pair_loss": 0.7070950269699097
    },
    {
      "epoch": 4,
      "training_pair_loss": 0.6920875112215678,
      "validation_pair_loss": 0.6948986053466797
    },
    {
      "epoch": 5,
      "training_pair_loss": 0.6988517145315806,
      "validation_pair_loss": 0.6838376522064209
    },
    {
      "epoch": 6,
      "training_pair_loss": 0.6973364452521006,
      "validation_pair_loss": 0.6724436283111572
    },
    {
      "epoch": 7,
      "training_pair_loss": 0.6842869222164154,
      "validation_pair_loss": 0.6618057489395142
    },
    {
      "epoch": 8,
      "training_pair_loss": 0.6420570810635885,
      "validation_pair_loss": 0.6531519889831543
    },
    {
      "epoch": 9,
      "training_pair_loss": 0.6653515994548798,
      "validation_pair_loss": 0.6464831829071045
    },
    {
      "epoch": 10,
      "training_pair_loss": 0.6500606934229533,
      "validation_pair_loss": 0.6420393586158752
    },
    {
      "epoch": 11,
      "training_pair_loss": 0.6445839603741964,
      "validation_pair_loss": 0.6402132511138916
    },
    {
      "epoch": 12,
      "training_pair_loss": 0.6730718413988749,
      "validation_pair_loss": 0.6393415331840515
    },
    {
      "epoch": 13,
      "training_pair_loss": 0.6551045179367065,
      "validation_pair_loss": 0.6365976333618164
    },
    {
      "epoch": 14,
      "training_pair_loss": 0.6270968814690908,
      "validation_pair_loss": 0.633887767791748
    },
    {
      "epoch": 15,
      "training_pair_loss": 0.6431255539258321,
      "validation_pair_loss": 0.6330843567848206
    },
    {
      "epoch": 16,
      "training_pair_loss": 0.6090357601642609,
      "validation_pair_loss": 0.6341031193733215
    },
    {
      "epoch": 17,
      "training_pair_loss": 0.6682893832524618,
      "validation_pair_loss": 0.6329171657562256
    },
    {
      "epoch": 18,
      "training_pair_loss": 0.6379861235618591,
      "validation_pair_loss": 0.631280779838562
    },
    {
      "epoch": 19,
      "training_pair_loss": 0.6449649234612783,
      "validation_pair_loss": 0.6288154125213623
    },
    {
      "epoch": 20,
      "training_pair_loss": 0.6422850290934244,
      "validation_pair_loss": 0.6289156675338745
    },
    {
      "epoch": 21,
      "training_pair_loss": 0.6403370797634125,
      "validation_pair_loss": 0.6287137866020203
    },
    {
      "epoch": 22,
      "training_pair_loss": 0.6153814196586609,
      "validation_pair_loss": 0.6279182434082031
    },
    {
      "epoch": 23,
      "training_pair_loss": 0.6324981947739919,
      "validation_pair_loss": 0.6281201243400574
    },
    {
      "epoch": 24,
      "training_pair_loss": 0.6136738955974579,
      "validation_pair_loss": 0.6269075870513916
    },
    {
      "epoch": 25,
      "training_pair_loss": 0.6511994500954946,
      "validation_pair_loss": 0.6264730095863342
    },
    {
      "epoch": 26,
      "training_pair_loss": 0.6255455712477366,
      "validation_pair_loss": 0.6305367350578308
    },
    {
      "epoch": 27,
      "training_pair_loss": 0.637495368719101,
      "validation_pair_loss": 0.6314413547515869
    },
    {
      "epoch": 28,
      "training_pair_loss": 0.6250569224357605,
      "validation_pair_loss": 0.6263542175292969
    },
    {
      "epoch": 29,
      "training_pair_loss": 0.6164625684420267,
      "validation_pair_loss": 0.624565839767456
    },
    {
      "epoch": 30,
      "training_pair_loss": 0.6288534303506216,
      "validation_pair_loss": 0.6203660368919373
    },
    {
      "epoch": 31,
      "training_pair_loss": 0.6276769836743673,
      "validation_pair_loss": 0.6190195679664612
    },
    {
      "epoch": 32,
      "training_pair_loss": 0.6005490024884542,
      "validation_pair_loss": 0.6189035177230835
    },
    {
      "epoch": 33,
      "training_pair_loss": 0.6029541889826456,
      "validation_pair_loss": 0.6175135374069214
    },
    {
      "epoch": 34,
      "training_pair_loss": 0.5936201612154642,
      "validation_pair_loss": 0.6175190806388855
    },
    {
      "epoch": 35,
      "training_pair_loss": 0.6120120485623678,
      "validation_pair_loss": 0.614869236946106
    },
    {
      "epoch": 36,
      "training_pair_loss": 0.581287701924642,
      "validation_pair_loss": 0.615369975566864
    },
    {
      "epoch": 37,
      "training_pair_loss": 0.6012997428576151,
      "validation_pair_loss": 0.6176727414131165
    },
    {
      "epoch": 38,
      "training_pair_loss": 0.5969312290350596,
      "validation_pair_loss": 0.6271389126777649
    },
    {
      "epoch": 39,
      "training_pair_loss": 0.5793554782867432,
      "validation_pair_loss": 0.6312544345855713
    },
    {
      "epoch": 40,
      "training_pair_loss": 0.5831093887488047,
      "validation_pair_loss": 0.6290249824523926
    },
    {
      "epoch": 41,
      "training_pair_loss": 0.6282937824726105,
      "validation_pair_loss": 0.6263436675071716
    },
    {
      "epoch": 42,
      "training_pair_loss": 0.6111309429009756,
      "validation_pair_loss": 0.6211633086204529
    },
    {
      "epoch": 43,
      "training_pair_loss": 0.5983485629161199,
      "validation_pair_loss": 0.6203481554985046
    },
    {
      "epoch": 44,
      "training_pair_loss": 0.6019033342599869,
      "validation_pair_loss": 0.6121244430541992
    },
    {
      "epoch": 45,
      "training_pair_loss": 0.6073039968808492,
      "validation_pair_loss": 0.6112990379333496
    },
    {
      "epoch": 46,
      "training_pair_loss": 0.5883496304353079,
      "validation_pair_loss": 0.6132257580757141
    },
    {
      "epoch": 47,
      "training_pair_loss": 0.5943388938903809,
      "validation_pair_loss": 0.6143561005592346
    },
    {
      "epoch": 48,
      "training_pair_loss": 0.587270716826121,
      "validation_pair_loss": 0.6151614189147949
    },
    {
      "epoch": 49,
      "training_pair_loss": 0.5937922894954681,
      "validation_pair_loss": 0.6185734868049622
    },
    {
      "epoch": 50,
      "training_pair_loss": 0.5825754900773367,
      "validation_pair_loss": 0.6215407252311707
    },
    {
      "epoch": 51,
      "training_pair_loss": 0.5712349812189738,
      "validation_pair_loss": 0.624745786190033
    },
    {
      "epoch": 52,
      "training_pair_loss": 0.5831883450349172,
      "validation_pair_loss": 0.6281951069831848
    },
    {
      "epoch": 53,
      "training_pair_loss": 0.5509530454874039,
      "validation_pair_loss": 0.6335658431053162
    },
    {
      "epoch": 54,
      "training_pair_loss": 0.5623500247796377,
      "validation_pair_loss": 0.6371464133262634
    },
    {
      "epoch": 55,
      "training_pair_loss": 0.5671578894058863,
      "validation_pair_loss": 0.6369690895080566
    },
    {
      "epoch": 56,
      "training_pair_loss": 0.5789371281862259,
      "validation_pair_loss": 0.6422266364097595
    },
    {
      "epoch": 57,
      "training_pair_loss": 0.5815341075261434,
      "validation_pair_loss": 0.6450821161270142
    },
    {
      "epoch": 58,
      "training_pair_loss": 0.5634511262178421,
      "validation_pair_loss": 0.6413621306419373
    },
    {
      "epoch": 59,
      "training_pair_loss": 0.5653099914391836,
      "validation_pair_loss": 0.6409940123558044
    },
    {
      "epoch": 60,
      "training_pair_loss": 0.5636946558952332,
      "validation_pair_loss": 0.6410163640975952
    },
    {
      "epoch": 61,
      "training_pair_loss": 0.5881442874670029,
      "validation_pair_loss": 0.6416041254997253
    },
    {
      "epoch": 62,
      "training_pair_loss": 0.5869395285844803,
      "validation_pair_loss": 0.6463533639907837
    },
    {
      "epoch": 63,
      "training_pair_loss": 0.5578558047612509,
      "validation_pair_loss": 0.6455363035202026
    },
    {
      "epoch": 64,
      "training_pair_loss": 0.5480411797761917,
      "validation_pair_loss": 0.6450743079185486
    },
    {
      "epoch": 65,
      "training_pair_loss": 0.5684447983900706,
      "validation_pair_loss": 0.6424922943115234
    },
    {
      "epoch": 66,
      "training_pair_loss": 0.5442517300446829,
      "validation_pair_loss": 0.636288046836853
    },
    {
      "epoch": 67,
      "training_pair_loss": 0.5471461415290833,
      "validation_pair_loss": 0.6321863532066345
    },
    {
      "epoch": 68,
      "training_pair_loss": 0.5394303699334463,
      "validation_pair_loss": 0.6306481957435608
    },
    {
      "epoch": 69,
      "training_pair_loss": 0.559449682633082,
      "validation_pair_loss": 0.627718448638916
    },
    {
      "epoch": 70,
      "training_pair_loss": 0.56000816822052,
      "validation_pair_loss": 0.6239803433418274
    },
    {
      "epoch": 71,
      "training_pair_loss": 0.570099800825119,
      "validation_pair_loss": 0.6247791647911072
    },
    {
      "epoch": 72,
      "training_pair_loss": 0.5803659955660502,
      "validation_pair_loss": 0.6263472437858582
    },
    {
      "epoch": 73,
      "training_pair_loss": 0.5881955424944559,
      "validation_pair_loss": 0.6313856244087219
    },
    {
      "epoch": 74,
      "training_pair_loss": 0.5955139497915903,
      "validation_pair_loss": 0.6384927034378052
    },
    {
      "epoch": 75,
      "training_pair_loss": 0.5571016768614451,
      "validation_pair_loss": 0.6425955295562744
    },
    {
      "epoch": 76,
      "training_pair_loss": 0.5378590375185013,
      "validation_pair_loss": 0.6432824730873108
    },
    {
      "epoch": 77,
      "training_pair_loss": 0.5422138820091883,
      "validation_pair_loss": 0.638453483581543
    },
    {
      "epoch": 78,
      "training_pair_loss": 0.5310110201438268,
      "validation_pair_loss": 0.6349430084228516
    },
    {
      "epoch": 79,
      "training_pair_loss": 0.5282997240622839,
      "validation_pair_loss": 0.6359946131706238
    },
    {
      "epoch": 80,
      "training_pair_loss": 0.5972785949707031,
      "validation_pair_loss": 0.6296086311340332
    },
    {
      "epoch": 81,
      "training_pair_loss": 0.5603173524141312,
      "validation_pair_loss": 0.6311172842979431
    },
    {
      "epoch": 82,
      "training_pair_loss": 0.5330286026000977,
      "validation_pair_loss": 0.6300211548805237
    },
    {
      "epoch": 83,
      "training_pair_loss": 0.5342774738868078,
      "validation_pair_loss": 0.6257214546203613
    },
    {
      "epoch": 84,
      "training_pair_loss": 0.5615833004315695,
      "validation_pair_loss": 0.626687228679657
    },
    {
      "epoch": 85,
      "training_pair_loss": 0.562104195356369,
      "validation_pair_loss": 0.6331090331077576
    },
    {
      "epoch": 86,
      "training_pair_loss": 0.5265299578507742,
      "validation_pair_loss": 0.6364940404891968
    },
    {
      "epoch": 87,
      "training_pair_loss": 0.5460903743902842,
      "validation_pair_loss": 0.6345710754394531
    },
    {
      "epoch": 88,
      "training_pair_loss": 0.5280517935752869,
      "validation_pair_loss": 0.6359588503837585
    },
    {
      "epoch": 89,
      "training_pair_loss": 0.5389540195465088,
      "validation_pair_loss": 0.6385628581047058
    },
    {
      "epoch": 90,
      "training_pair_loss": 0.5388951748609543,
      "validation_pair_loss": 0.641994833946228
    },
    {
      "epoch": 91,
      "training_pair_loss": 0.5516747633616129,
      "validation_pair_loss": 0.6473745107650757
    },
    {
      "epoch": 92,
      "training_pair_loss": 0.5331861078739166,
      "validation_pair_loss": 0.644588828086853
    },
    {
      "epoch": 93,
      "training_pair_loss": 0.5468221654494604,
      "validation_pair_loss": 0.6430701017379761
    },
    {
      "epoch": 94,
      "training_pair_loss": 0.5657043556372324,
      "validation_pair_loss": 0.641301155090332
    },
    {
      "epoch": 95,
      "training_pair_loss": 0.548022136092186,
      "validation_pair_loss": 0.643640398979187
    },
    {
      "epoch": 96,
      "training_pair_loss": 0.5319892118374506,
      "validation_pair_loss": 0.651678204536438
    },
    {
      "epoch": 97,
      "training_pair_loss": 0.5311064620812734,
      "validation_pair_loss": 0.656090259552002
    },
    {
      "epoch": 98,
      "training_pair_loss": 0.5206660777330399,
      "validation_pair_loss": 0.6569125652313232
    },
    {
      "epoch": 99,
      "training_pair_loss": 0.5465453118085861,
      "validation_pair_loss": 0.6590514183044434
    },
    {
      "epoch": 100,
      "training_pair_loss": 0.5106392055749893,
      "validation_pair_loss": 0.6522769331932068
    }
  ],
  "ppo_history": [
    {
      "update": 1,
      "reward": -0.5108681954443455,
      "reference_kl": 1.674364903215412e-06,
      "clipping_fraction": 0.0,
      "entropy": 0.004939224999927339,
      "mean_output_length": 3.5625,
      "loss": 0.751640260219574
    },
    {
      "update": 2,
      "reward": -0.780377265997231,
      "reference_kl": 2.151297433804567e-05,
      "clipping_fraction": 0.0,
      "entropy": 0.008486695361848423,
      "mean_output_length": 3.5625,
      "loss": 0.7850374579429626
    },
    {
      "update": 3,
      "reward": -0.6156287724152207,
      "reference_kl": 2.0159658797327734e-05,
      "clipping_fraction": 0.0,
      "entropy": 0.004183808808909055,
      "mean_output_length": 3.5625,
      "loss": 0.726406991481781
    },
    {
      "update": 4,
      "reward": -1.2468746472150087,
      "reference_kl": 0.00039175248626977687,
      "clipping_fraction": 0.0,
      "entropy": 0.01851695254853637,
      "mean_output_length": 3.5,
      "loss": 0.9345951974391937
    },
    {
      "update": 5,
      "reward": -0.8691093400120735,
      "reference_kl": 0.00039919165438896654,
      "clipping_fraction": 0.0,
      "entropy": 0.011177323666212857,
      "mean_output_length": 3.375,
      "loss": 0.7923188209533691
    },
    {
      "update": 6,
      "reward": -0.925772595917806,
      "reference_kl": 0.00017236025102625304,
      "clipping_fraction": 0.0,
      "entropy": 0.012692943612456473,
      "mean_output_length": 3.5625,
      "loss": 0.6919785141944885
    },
    {
      "update": 7,
      "reward": -0.998598026111722,
      "reference_kl": 0.00015994403996860663,
      "clipping_fraction": 0.0,
      "entropy": 0.007369631783659619,
      "mean_output_length": 3.375,
      "loss": 0.9512708187103271
    },
    {
      "update": 8,
      "reward": -1.085020998492837,
      "reference_kl": 0.0012294640949945044,
      "clipping_fraction": 0.0,
      "entropy": 0.023383439261812293,
      "mean_output_length": 3.25,
      "loss": 0.7525726854801178
    },
    {
      "update": 9,
      "reward": -0.6852186264004558,
      "reference_kl": 0.0028577010836818187,
      "clipping_fraction": 0.0,
      "entropy": 0.013336772218915825,
      "mean_output_length": 3.625,
      "loss": 0.770026296377182
    },
    {
      "update": 10,
      "reward": -0.9961314350366592,
      "reference_kl": 0.003578736011804365,
      "clipping_fraction": 0.0,
      "entropy": 0.019320370481018995,
      "mean_output_length": 3.4375,
      "loss": 0.4183124005794525
    },
    {
      "update": 11,
      "reward": -1.0614546164870262,
      "reference_kl": 0.0040138234338643955,
      "clipping_fraction": 0.0,
      "entropy": 0.028348457023534727,
      "mean_output_length": 3.5,
      "loss": 0.9838283061981201
    },
    {
      "update": 12,
      "reward": -0.8720440082252026,
      "reference_kl": 0.0005487164499653713,
      "clipping_fraction": 0.0,
      "entropy": 0.008741281291685766,
      "mean_output_length": 3.5625,
      "loss": 0.5938787460327148
    },
    {
      "update": 13,
      "reward": -0.6227740903850645,
      "reference_kl": 0.006117383200686075,
      "clipping_fraction": 0.0,
      "entropy": 0.023686824142959962,
      "mean_output_length": 3.4375,
      "loss": 0.7922053039073944
    },
    {
      "update": 14,
      "reward": -0.5217424388974905,
      "reference_kl": 0.00907253996830982,
      "clipping_fraction": 0.0,
      "entropy": 0.02834496911487112,
      "mean_output_length": 3.5,
      "loss": 0.6277670562267303
    },
    {
      "update": 15,
      "reward": -0.7501392140984535,
      "reference_kl": 0.015457033461588998,
      "clipping_fraction": 0.0,
      "entropy": 0.028226076262788617,
      "mean_output_length": 3.4375,
      "loss": 0.44595009088516235
    },
    {
      "update": 16,
      "reward": -1.070408177562058,
      "reference_kl": 0.00807966220965424,
      "clipping_fraction": 0.0,
      "entropy": 0.029800384375316548,
      "mean_output_length": 3.5,
      "loss": 0.5132794082164764
    },
    {
      "update": 17,
      "reward": -0.8994524800218642,
      "reference_kl": 0.010995444526456366,
      "clipping_fraction": 0.0,
      "entropy": 0.0329230696012103,
      "mean_output_length": 3.4375,
      "loss": 0.28746794164180756
    },
    {
      "update": 18,
      "reward": -0.7377631599083543,
      "reference_kl": 0.011672380188757372,
      "clipping_fraction": 0.0,
      "entropy": 0.029288750792168902,
      "mean_output_length": 3.3125,
      "loss": 0.2204108014702797
    },
    {
      "update": 19,
      "reward": -0.7072968455031514,
      "reference_kl": 0.045260247535405015,
      "clipping_fraction": 0.0,
      "entropy": 0.047399754833726604,
      "mean_output_length": 3.1875,
      "loss": 0.41506819427013397
    },
    {
      "update": 20,
      "reward": -1.063444186700508,
      "reference_kl": 0.012470264245341063,
      "clipping_fraction": 0.0,
      "entropy": 0.027847767705005803,
      "mean_output_length": 3.375,
      "loss": 0.5487436354160309
    },
    {
      "update": 21,
      "reward": -1.031782062491402,
      "reference_kl": 0.016801267226368162,
      "clipping_fraction": 0.0,
      "entropy": 0.030451298317884946,
      "mean_output_length": 3.4375,
      "loss": 0.4993273466825485
    },
    {
      "update": 22,
      "reward": -0.8958384534344077,
      "reference_kl": 0.07484829358888681,
      "clipping_fraction": 0.0,
      "entropy": 0.029098453484428587,
      "mean_output_length": 3.375,
      "loss": 0.2157704383134842
    },
    {
      "update": 23,
      "reward": -1.0086360350251198,
      "reference_kl": 0.048829838661982296,
      "clipping_fraction": 0.0,
      "entropy": 0.052616774309116196,
      "mean_output_length": 3.4375,
      "loss": 0.5758883357048035
    },
    {
      "update": 24,
      "reward": -0.45932026021182537,
      "reference_kl": 0.011158935599442636,
      "clipping_fraction": 0.0,
      "entropy": 0.03148734353326147,
      "mean_output_length": 3.5625,
      "loss": 0.9479656219482422
    },
    {
      "update": 25,
      "reward": -1.1176029592752457,
      "reference_kl": 0.07755367566766706,
      "clipping_fraction": 0.0,
      "entropy": 0.04935442498401419,
      "mean_output_length": 3.375,
      "loss": 0.30655477941036224
    },
    {
      "update": 26,
      "reward": -0.49483723007142544,
      "reference_kl": 0.26542884540985845,
      "clipping_fraction": 0.0,
      "entropy": 0.07561688984398168,
      "mean_output_length": 3.375,
      "loss": 0.526091992855072
    },
    {
      "update": 27,
      "reward": -0.43592351395636797,
      "reference_kl": 0.16980420023251497,
      "clipping_fraction": 0.0,
      "entropy": 0.08481927229895628,
      "mean_output_length": 3.25,
      "loss": 0.3522157818078995
    },
    {
      "update": 28,
      "reward": -1.0321162333711982,
      "reference_kl": 0.33559885481961405,
      "clipping_fraction": 0.009615384615384616,
      "entropy": 0.047653964875059227,
      "mean_output_length": 3.25,
      "loss": 0.3255975693464279
    },
    {
      "update": 29,
      "reward": -0.7504961770027876,
      "reference_kl": 0.2293173699630942,
      "clipping_fraction": 0.0,
      "entropy": 0.04091793332931053,
      "mean_output_length": 3.3125,
      "loss": 0.3558696061372757
    },
    {
      "update": 30,
      "reward": -1.0410967422649264,
      "reference_kl": 0.12988268182261262,
      "clipping_fraction": 0.0,
      "entropy": 0.045297359902918916,
      "mean_output_length": 3.4375,
      "loss": 0.2357354536652565
    }
  ],
  "reward_source": "ai preferences; scalar reward head with frozen supervised encoder",
  "test_used_for_selection": false,
  "policy_preference_evaluation": {
    "finetuned": {
      "examples": 19,
      "preferred_response_ranked_higher": 18,
      "ties": 0,
      "mean_greedy_reward": -1.0226837724055116,
      "samples": [
        {
          "pair_id": "preference-006",
          "prompt": "Q:68+54;A:",
          "generated": "122",
          "token_ids": [
            20,
            21,
            21,
            2
          ],
          "ended": true,
          "frozen_reward": 0.2129228264093399,
          "preferred_log_likelihood_margin": 2.481626512533694
        },
        {
          "pair_id": "preference-009",
          "prompt": "Q:56+47;A:",
          "generated": "103",
          "token_ids": [
            20,
            19,
            22,
            2
          ],
          "ended": true,
          "frozen_reward": -1.334978699684143,
          "preferred_log_likelihood_margin": 1.4033703358727507
        },
        {
          "pair_id": "preference-010",
          "prompt": "Q:29+10;A:",
          "generated": "39",
          "token_ids": [
            22,
            28,
            2
          ],
          "ended": true,
          "frozen_reward": -1.320291519165039,
          "preferred_log_likelihood_margin": 1.1489810021594167
        },
        {
          "pair_id": "preference-012",
          "prompt": "Q:84+55;A:",
          "generated": "139",
          "token_ids": [
            20,
            22,
            28,
            2
          ],
          "ended": true,
          "frozen_reward": -0.6122369766235352,
          "preferred_log_likelihood_margin": 4.176908492401708
        },
        {
          "pair_id": "preference-030",
          "prompt": "Q:17+2;A:",
          "generated": "29",
          "token_ids": [
            21,
            28,
            2
          ],
          "ended": true,
          "frozen_reward": -0.5251916646957397,
          "preferred_log_likelihood_margin": -0.2838256433606148
        },
        {
          "pair_id": "preference-032",
          "prompt": "Q:88+82;A:",
          "generated": "170",
          "token_ids": [
            20,
            26,
            19,
            2
          ],
          "ended": true,
          "frozen_reward": -0.5571931600570679,
          "preferred_log_likelihood_margin": 5.029810880951118
        },
        {
          "pair_id": "preference-033",
          "prompt": "Q:82+0;A:",
          "generated": "82",
          "token_ids": [
            27,
            21,
            2
          ],
          "ended": true,
          "frozen_reward": -1.0850805044174194,
          "preferred_log_likelihood_margin": 8.42288688570261
        },
        {
          "pair_id": "preference-042",
          "prompt": "Q:32+20;A:",
          "generated": "52",
          "token_ids": [
            24,
            21,
            2
          ],
          "ended": true,
          "frozen_reward": -0.9805736541748047,
          "preferred_log_likelihood_margin": 2.5883908276446164
        },
        {
          "pair_id": "preference-044",
          "prompt": "Q:56+43;A:",
          "generated": "99",
          "token_ids": [
            28,
            28,
            2
          ],
          "ended": true,
          "frozen_reward": -0.37128642201423645,
          "preferred_log_likelihood_margin": 8.695032444753451
        },
        {
          "pair_id": "preference-045",
          "prompt": "Q:87+23;A:",
          "generated": "110",
          "token_ids": [
            20,
            20,
            19,
            2
          ],
          "ended": true,
          "frozen_reward": -2.229574680328369,
          "preferred_log_likelihood_margin": 3.384263618856494
        },
        {
          "pair_id": "preference-052",
          "prompt": "Q:89+64;A:",
          "generated": "153",
          "token_ids": [
            20,
            24,
            22,
            2
          ],
          "ended": true,
          "frozen_reward": -1.4618338346481323,
          "preferred_log_likelihood_margin": 2.342289100648486
        },
        {
          "pair_id": "preference-053",
          "prompt": "Q:92+0;A:",
          "generated": "92",
          "token_ids": [
            28,
            21,
            2
          ],
          "ended": true,
          "frozen_reward": 0.5471355319023132,
          "preferred_log_likelihood_margin": 5.426143332617357
        },
        {
          "pair_id": "preference-062",
          "prompt": "Q:56+53;A:",
          "generated": "109",
          "token_ids": [
            20,
            19,
            28,
            2
          ],
          "ended": true,
          "frozen_reward": -1.2782496213912964,
          "preferred_log_likelihood_margin": 1.3602168839424849
        },
        {
          "pair_id": "preference-071",
          "prompt": "Q:77+70;A:",
          "generated": "147",
          "token_ids": [
            20,
            23,
            26,
            2
          ],
          "ended": true,
          "frozen_reward": -2.830505132675171,
          "preferred_log_likelihood_margin": 3.7886049203953007
        },
        {
          "pair_id": "preference-072",
          "prompt": "Q:72+11;A:",
          "generated": "83",
          "token_ids": [
            27,
            22,
            2
          ],
          "ended": true,
          "frozen_reward": -1.697904109954834,
          "preferred_log_likelihood_margin": 4.857989428623114
        },
        {
          "pair_id": "preference-076",
          "prompt": "Q:48+18;A:",
          "generated": "66",
          "token_ids": [
            25,
            25,
            2
          ],
          "ended": true,
          "frozen_reward": -1.3823732137680054,
          "preferred_log_likelihood_margin": 4.837155510962475
        },
        {
          "pair_id": "preference-095",
          "prompt": "Q:59+31;A:",
          "generated": "90",
          "token_ids": [
            28,
            19,
            2
          ],
          "ended": true,
          "frozen_reward": -1.6636220216751099,
          "preferred_log_likelihood_margin": 4.692114959114406
        },
        {
          "pair_id": "preference-098",
          "prompt": "Q:98+97;A:",
          "generated": "195",
          "token_ids": [
            20,
            28,
            24,
            2
          ],
          "ended": true,
          "frozen_reward": 0.04671143367886543,
          "preferred_log_likelihood_margin": 2.5796294667525217
        },
        {
          "pair_id": "preference-106",
          "prompt": "Q:16+1;A:",
          "generated": "17",
          "token_ids": [
            20,
            26,
            2
          ],
          "ended": true,
          "frozen_reward": -0.9068662524223328,
          "preferred_log_likelihood_margin": 1.2222001729533076
        }
      ]
    },
    "rl": {
      "examples": 19,
      "preferred_response_ranked_higher": 18,
      "ties": 0,
      "mean_greedy_reward": -1.0285282085992788,
      "samples": [
        {
          "pair_id": "preference-006",
          "prompt": "Q:68+54;A:",
          "generated": "122",
          "token_ids": [
            20,
            21,
            21,
            2
          ],
          "ended": true,
          "frozen_reward": 0.2129228264093399,
          "preferred_log_likelihood_margin": 1.8931938511232147
        },
        {
          "pair_id": "preference-009",
          "prompt": "Q:56+47;A:",
          "generated": "103",
          "token_ids": [
            20,
            19,
            22,
            2
          ],
          "ended": true,
          "frozen_reward": -1.334978699684143,
          "preferred_log_likelihood_margin": 1.6630686740973033
        },
        {
          "pair_id": "preference-010",
          "prompt": "Q:29+10;A:",
          "generated": "39",
          "token_ids": [
            22,
            28,
            2
          ],
          "ended": true,
          "frozen_reward": -1.320291519165039,
          "preferred_log_likelihood_margin": 1.997965797316283
        },
        {
          "pair_id": "preference-012",
          "prompt": "Q:84+55;A:",
          "generated": "139",
          "token_ids": [
            20,
            22,
            28,
            2
          ],
          "ended": true,
          "frozen_reward": -0.6122369766235352,
          "preferred_log_likelihood_margin": 3.021168579813093
        },
        {
          "pair_id": "preference-030",
          "prompt": "Q:17+2;A:",
          "generated": "29",
          "token_ids": [
            21,
            28,
            2
          ],
          "ended": true,
          "frozen_reward": -0.5251916646957397,
          "preferred_log_likelihood_margin": -1.4432675600983202
        },
        {
          "pair_id": "preference-032",
          "prompt": "Q:88+82;A:",
          "generated": "170",
          "token_ids": [
            20,
            26,
            19,
            2
          ],
          "ended": true,
          "frozen_reward": -0.5571931600570679,
          "preferred_log_likelihood_margin": 5.223650736566924
        },
        {
          "pair_id": "preference-033",
          "prompt": "Q:82+0;A:",
          "generated": "82",
          "token_ids": [
            27,
            21,
            2
          ],
          "ended": true,
          "frozen_reward": -1.0850805044174194,
          "preferred_log_likelihood_margin": 7.956424237694591
        },
        {
          "pair_id": "preference-042",
          "prompt": "Q:32+20;A:",
          "generated": "52",
          "token_ids": [
            24,
            21,
            2
          ],
          "ended": true,
          "frozen_reward": -0.9805736541748047,
          "preferred_log_likelihood_margin": 3.4182960763573647
        },
        {
          "pair_id": "preference-044",
          "prompt": "Q:56+43;A:",
          "generated": "99",
          "token_ids": [
            28,
            28,
            2
          ],
          "ended": true,
          "frozen_reward": -0.37128642201423645,
          "preferred_log_likelihood_margin": 7.612115134543274
        },
        {
          "pair_id": "preference-045",
          "prompt": "Q:87+23;A:",
          "generated": "110",
          "token_ids": [
            20,
            20,
            19,
            2
          ],
          "ended": true,
          "frozen_reward": -2.229574680328369,
          "preferred_log_likelihood_margin": 3.649067346006632
        },
        {
          "pair_id": "preference-052",
          "prompt": "Q:89+64;A:",
          "generated": "153",
          "token_ids": [
            20,
            24,
            22,
            2
          ],
          "ended": true,
          "frozen_reward": -1.4618338346481323,
          "preferred_log_likelihood_margin": 2.0579031127272174
        },
        {
          "pair_id": "preference-053",
          "prompt": "Q:92+0;A:",
          "generated": "92",
          "token_ids": [
            28,
            21,
            2
          ],
          "ended": true,
          "frozen_reward": 0.5471355319023132,
          "preferred_log_likelihood_margin": 5.168432205915451
        },
        {
          "pair_id": "preference-062",
          "prompt": "Q:56+53;A:",
          "generated": "109",
          "token_ids": [
            20,
            19,
            28,
            2
          ],
          "ended": true,
          "frozen_reward": -1.2782496213912964,
          "preferred_log_likelihood_margin": 0.055103570222854614
        },
        {
          "pair_id": "preference-071",
          "prompt": "Q:77+70;A:",
          "generated": "147",
          "token_ids": [
            20,
            23,
            26,
            2
          ],
          "ended": true,
          "frozen_reward": -2.830505132675171,
          "preferred_log_likelihood_margin": 3.7036359355333843
        },
        {
          "pair_id": "preference-072",
          "prompt": "Q:72+11;A:",
          "generated": "83",
          "token_ids": [
            27,
            22,
            2
          ],
          "ended": true,
          "frozen_reward": -1.697904109954834,
          "preferred_log_likelihood_margin": 4.997844000346959
        },
        {
          "pair_id": "preference-076",
          "prompt": "Q:48+18;A:",
          "generated": "56",
          "token_ids": [
            24,
            25,
            2
          ],
          "ended": true,
          "frozen_reward": -1.493417501449585,
          "preferred_log_likelihood_margin": 4.191630989313126
        },
        {
          "pair_id": "preference-095",
          "prompt": "Q:59+31;A:",
          "generated": "90",
          "token_ids": [
            28,
            19,
            2
          ],
          "ended": true,
          "frozen_reward": -1.6636220216751099,
          "preferred_log_likelihood_margin": 4.445088585605845
        },
        {
          "pair_id": "preference-098",
          "prompt": "Q:98+97;A:",
          "generated": "195",
          "token_ids": [
            20,
            28,
            24,
            2
          ],
          "ended": true,
          "frozen_reward": 0.04671143367886543,
          "preferred_log_likelihood_margin": 2.1663909614726435
        },
        {
          "pair_id": "preference-106",
          "prompt": "Q:16+1;A:",
          "generated": "17",
          "token_ids": [
            20,
            26,
            2
          ],
          "ended": true,
          "frozen_reward": -0.9068662524223328,
          "preferred_log_likelihood_margin": 0.051974162459373474
        }
      ]
    }
  },
  "preference_metric": "mean log probability of actual candidate tokens including EOS; AI-labeled held-out groups; descriptive, not a new human study",
  "weight_change": {
    "state_tensors_changed": 29,
    "state_tensors_total": 29,
    "maximum_absolute_change": 0.0006477907299995422,
    "changes_by_state_tensor": {
      "token_embedding.weight": 0.0005865991115570068,
      "position_embedding.weight": 0.0005726460367441177,
      "blocks.0.attention_norm.weight": 0.0005639195442199707,
      "blocks.0.attention_norm.bias": 0.000577148050069809,
      "blocks.0.attention.qkv.weight": 0.0006057601422071457,
      "blocks.0.attention.qkv.bias": 0.0005543902516365051,
      "blocks.0.attention.projection.weight": 0.0006129443645477295,
      "blocks.0.attention.projection.bias": 0.000557534396648407,
      "blocks.0.feed_forward_norm.weight": 0.0005643367767333984,
      "blocks.0.feed_forward_norm.bias": 0.0005525089800357819,
      "blocks.0.feed_forward.expand.weight": 0.0006311554461717606,
      "blocks.0.feed_forward.expand.bias": 0.0005862787365913391,
      "blocks.0.feed_forward.contract.weight": 0.0006455099210143089,
      "blocks.0.feed_forward.contract.bias": 0.0005095470696687698,
      "blocks.1.attention_norm.weight": 0.0005771517753601074,
      "blocks.1.attention_norm.bias": 0.0006098237354308367,
      "blocks.1.attention.qkv.weight": 0.0006133266724646091,
      "blocks.1.attention.qkv.bias": 0.0006084144115447998,
      "blocks.1.attention.projection.weight": 0.0006335526704788208,
      "blocks.1.attention.projection.bias": 0.0004812702536582947,
      "blocks.1.feed_forward_norm.weight": 0.0005930662155151367,
      "blocks.1.feed_forward_norm.bias": 0.0005866549909114838,
      "blocks.1.feed_forward.expand.weight": 0.0006477907299995422,
      "blocks.1.feed_forward.expand.bias": 0.0006169453263282776,
      "blocks.1.feed_forward.contract.weight": 0.0006445646286010742,
      "blocks.1.feed_forward.contract.bias": 0.0005770325660705566,
      "final_norm.weight": 0.0006297826766967773,
      "final_norm.bias": 0.00047616660594940186,
      "lm_head.weight": 0.0005865991115570068
    }
  },
  "configuration": {
    "device": "cpu",
    "torch": "2.14.1",
    "numpy": "2.4.2",
    "python": "3.13.15",
    "cpu_threads": 4,
    "reward_seed": 91,
    "reward_epochs": 100,
    "reward_learning_rate": 0.001,
    "ppo_seed": 101,
    "ppo_updates": 30,
    "optimization_passes_per_update": 2,
    "responses_per_update": 16,
    "policy_learning_rate": 1e-05,
    "policy_clip": 0.2,
    "kl_coefficient": 0.05,
    "kl_placement": "per-token reward penalty and explicit differentiable reference regularizer",
    "value_coefficient": 0.5,
    "entropy_coefficient": 0.001,
    "gradient_norm_clip": 1,
    "discount": 1,
    "test_used_for_selection": false
  }
}