{ "best_metric": null, "best_model_checkpoint": null, "epoch": 0.9991503823279524, "eval_steps": 500, "global_step": 147, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0.01, "grad_norm": 48.577925295489194, "learning_rate": 3.3333333333333334e-08, "logps/chosen": -67.63638305664062, "logps/rejected": -55.72713851928711, "loss": 0.6931, "losses/dpo": 0.6931471824645996, "losses/sft": 0.34192484617233276, "losses/total": 0.6931471824645996, "ref_logps/chosen": -67.63638305664062, "ref_logps/rejected": -55.72713851928711, "rewards/accuracies": 0.0, "rewards/chosen": 0.0, "rewards/margins": 0.0, "rewards/rejected": 0.0, "step": 1 }, { "epoch": 0.01, "grad_norm": 31.47832995229055, "learning_rate": 6.666666666666667e-08, "logps/chosen": -68.23696899414062, "logps/rejected": -57.0877799987793, "loss": 0.6931, "losses/dpo": 0.6931471824645996, "losses/sft": 0.35053497552871704, "losses/total": 0.6931471824645996, "ref_logps/chosen": -68.23696899414062, "ref_logps/rejected": -57.0877799987793, "rewards/accuracies": 0.0, "rewards/chosen": 0.0, "rewards/margins": 0.0, "rewards/rejected": 0.0, "step": 2 }, { "epoch": 0.02, "grad_norm": 50.46042718530731, "learning_rate": 1e-07, "logps/chosen": -63.95356750488281, "logps/rejected": -56.19622802734375, "loss": 0.6913, "losses/dpo": 0.7034662961959839, "losses/sft": 0.34674179553985596, "losses/total": 0.7034662961959839, "ref_logps/chosen": -64.00633239746094, "ref_logps/rejected": -56.198883056640625, "rewards/accuracies": 0.5, "rewards/chosen": 0.005276843905448914, "rewards/margins": 0.005011209286749363, "rewards/rejected": 0.0002656341530382633, "step": 3 }, { "epoch": 0.03, "grad_norm": 32.81062658932443, "learning_rate": 1.3333333333333334e-07, "logps/chosen": -61.69102096557617, "logps/rejected": -54.53285217285156, "loss": 0.6963, "losses/dpo": 0.6837673783302307, "losses/sft": 0.3563593626022339, "losses/total": 0.6837673783302307, "ref_logps/chosen": -61.65943145751953, "ref_logps/rejected": -54.55246353149414, "rewards/accuracies": 0.390625, "rewards/chosen": -0.003159371204674244, "rewards/margins": -0.005120169371366501, "rewards/rejected": 0.0019607983995229006, "step": 4 }, { "epoch": 0.03, "grad_norm": 33.80575482734628, "learning_rate": 1.6666666666666665e-07, "logps/chosen": -66.74890899658203, "logps/rejected": -54.70701599121094, "loss": 0.6925, "losses/dpo": 0.6997189521789551, "losses/sft": 0.3123997151851654, "losses/total": 0.6997189521789551, "ref_logps/chosen": -66.69550323486328, "ref_logps/rejected": -54.629825592041016, "rewards/accuracies": 0.53125, "rewards/chosen": -0.005340744741261005, "rewards/margins": 0.0023781214840710163, "rewards/rejected": -0.00771886482834816, "step": 5 }, { "epoch": 0.04, "grad_norm": 39.07641400536226, "learning_rate": 2e-07, "logps/chosen": -68.85008239746094, "logps/rejected": -58.72641372680664, "loss": 0.6966, "losses/dpo": 0.6637319922447205, "losses/sft": 0.3361469805240631, "losses/total": 0.6637319922447205, "ref_logps/chosen": -68.81605529785156, "ref_logps/rejected": -58.74713134765625, "rewards/accuracies": 0.453125, "rewards/chosen": -0.003402549307793379, "rewards/margins": -0.005474450532346964, "rewards/rejected": 0.0020719014573842287, "step": 6 }, { "epoch": 0.05, "grad_norm": 34.000425983037914, "learning_rate": 2.3333333333333333e-07, "logps/chosen": -61.8620491027832, "logps/rejected": -51.837215423583984, "loss": 0.694, "losses/dpo": 0.6815477013587952, "losses/sft": 0.34513574838638306, "losses/total": 0.6815477013587952, "ref_logps/chosen": -61.80104446411133, "ref_logps/rejected": -51.780853271484375, "rewards/accuracies": 0.515625, "rewards/chosen": -0.006101294886320829, "rewards/margins": -0.0004650282207876444, "rewards/rejected": -0.005636268295347691, "step": 7 }, { "epoch": 0.05, "grad_norm": 39.18548739683539, "learning_rate": 2.6666666666666667e-07, "logps/chosen": -65.5419921875, "logps/rejected": -53.712867736816406, "loss": 0.6864, "losses/dpo": 0.6901150345802307, "losses/sft": 0.3755946159362793, "losses/total": 0.6901150345802307, "ref_logps/chosen": -65.61614227294922, "ref_logps/rejected": -53.64268493652344, "rewards/accuracies": 0.65625, "rewards/chosen": 0.007414784282445908, "rewards/margins": 0.014433156698942184, "rewards/rejected": -0.007018372416496277, "step": 8 }, { "epoch": 0.06, "grad_norm": 34.05787792436153, "learning_rate": 3e-07, "logps/chosen": -66.96398162841797, "logps/rejected": -52.923221588134766, "loss": 0.7011, "losses/dpo": 0.6919645071029663, "losses/sft": 0.34879329800605774, "losses/total": 0.6919645071029663, "ref_logps/chosen": -66.94356536865234, "ref_logps/rejected": -53.04799270629883, "rewards/accuracies": 0.4375, "rewards/chosen": -0.0020427405834198, "rewards/margins": -0.014519838616251945, "rewards/rejected": 0.012477097101509571, "step": 9 }, { "epoch": 0.07, "grad_norm": 32.92454994636796, "learning_rate": 3.333333333333333e-07, "logps/chosen": -68.3951416015625, "logps/rejected": -53.54661178588867, "loss": 0.6903, "losses/dpo": 0.6679682731628418, "losses/sft": 0.3381645083427429, "losses/total": 0.6679682731628418, "ref_logps/chosen": -68.44535827636719, "ref_logps/rejected": -53.52064895629883, "rewards/accuracies": 0.515625, "rewards/chosen": 0.005021924152970314, "rewards/margins": 0.007618834264576435, "rewards/rejected": -0.002596910111606121, "step": 10 }, { "epoch": 0.07, "grad_norm": 28.62792358507012, "learning_rate": 3.666666666666666e-07, "logps/chosen": -58.64153289794922, "logps/rejected": -49.53267288208008, "loss": 0.6999, "losses/dpo": 0.6928383111953735, "losses/sft": 0.34639859199523926, "losses/total": 0.6928383111953735, "ref_logps/chosen": -58.6944465637207, "ref_logps/rejected": -49.710044860839844, "rewards/accuracies": 0.375, "rewards/chosen": 0.005291242152452469, "rewards/margins": -0.012445551343262196, "rewards/rejected": 0.01773679442703724, "step": 11 }, { "epoch": 0.08, "grad_norm": 34.95324550960433, "learning_rate": 4e-07, "logps/chosen": -63.45814514160156, "logps/rejected": -51.141937255859375, "loss": 0.6926, "losses/dpo": 0.6734712719917297, "losses/sft": 0.38542595505714417, "losses/total": 0.6734712719917297, "ref_logps/chosen": -63.53767395019531, "ref_logps/rejected": -51.197601318359375, "rewards/accuracies": 0.5, "rewards/chosen": 0.007953131571412086, "rewards/margins": 0.0023870535660535097, "rewards/rejected": 0.00556607823818922, "step": 12 }, { "epoch": 0.09, "grad_norm": 40.49931666997992, "learning_rate": 4.3333333333333335e-07, "logps/chosen": -72.46746826171875, "logps/rejected": -58.34828186035156, "loss": 0.6943, "losses/dpo": 0.6939187049865723, "losses/sft": 0.3672393262386322, "losses/total": 0.6939187049865723, "ref_logps/chosen": -72.57909393310547, "ref_logps/rejected": -58.46982955932617, "rewards/accuracies": 0.421875, "rewards/chosen": 0.011162188835442066, "rewards/margins": -0.000992664834484458, "rewards/rejected": 0.012154852971434593, "step": 13 }, { "epoch": 0.1, "grad_norm": 35.43728228028493, "learning_rate": 4.6666666666666666e-07, "logps/chosen": -60.627281188964844, "logps/rejected": -52.48558807373047, "loss": 0.6921, "losses/dpo": 0.6789003014564514, "losses/sft": 0.3384055495262146, "losses/total": 0.6789003014564514, "ref_logps/chosen": -60.64947509765625, "ref_logps/rejected": -52.47667694091797, "rewards/accuracies": 0.484375, "rewards/chosen": 0.0022186655551195145, "rewards/margins": 0.003109627403318882, "rewards/rejected": -0.0008909617317840457, "step": 14 }, { "epoch": 0.1, "grad_norm": 37.2571656085439, "learning_rate": 5e-07, "logps/chosen": -64.5811538696289, "logps/rejected": -56.402488708496094, "loss": 0.6889, "losses/dpo": 0.6773965358734131, "losses/sft": 0.32888567447662354, "losses/total": 0.6773965358734131, "ref_logps/chosen": -64.77919006347656, "ref_logps/rejected": -56.505523681640625, "rewards/accuracies": 0.578125, "rewards/chosen": 0.019803302362561226, "rewards/margins": 0.009500503540039062, "rewards/rejected": 0.010302798822522163, "step": 15 }, { "epoch": 0.11, "grad_norm": 37.083526947632535, "learning_rate": 4.999291986732822e-07, "logps/chosen": -67.24109649658203, "logps/rejected": -54.730838775634766, "loss": 0.6882, "losses/dpo": 0.6811863780021667, "losses/sft": 0.32620126008987427, "losses/total": 0.6811863780021667, "ref_logps/chosen": -67.33450317382812, "ref_logps/rejected": -54.712337493896484, "rewards/accuracies": 0.546875, "rewards/chosen": 0.009341094642877579, "rewards/margins": 0.011191939935088158, "rewards/rejected": -0.0018508443608880043, "step": 16 }, { "epoch": 0.12, "grad_norm": 31.937569243375922, "learning_rate": 4.99716834795752e-07, "logps/chosen": -63.45280456542969, "logps/rejected": -52.51921463012695, "loss": 0.6823, "losses/dpo": 0.678799569606781, "losses/sft": 0.3536127507686615, "losses/total": 0.678799569606781, "ref_logps/chosen": -63.669334411621094, "ref_logps/rejected": -52.50813674926758, "rewards/accuracies": 0.65625, "rewards/chosen": 0.021653007715940475, "rewards/margins": 0.022760074585676193, "rewards/rejected": -0.0011070647742599249, "step": 17 }, { "epoch": 0.12, "grad_norm": 40.33340375657743, "learning_rate": 4.993630286525634e-07, "logps/chosen": -67.25548553466797, "logps/rejected": -56.92215347290039, "loss": 0.6874, "losses/dpo": 0.6505956053733826, "losses/sft": 0.3500673770904541, "losses/total": 0.6505956053733826, "ref_logps/chosen": -67.41217041015625, "ref_logps/rejected": -56.942283630371094, "rewards/accuracies": 0.578125, "rewards/chosen": 0.01566872000694275, "rewards/margins": 0.013655520044267178, "rewards/rejected": 0.0020132004283368587, "step": 18 }, { "epoch": 0.13, "grad_norm": 38.0338752718148, "learning_rate": 4.988679806432711e-07, "logps/chosen": -68.8526611328125, "logps/rejected": -57.075347900390625, "loss": 0.6723, "losses/dpo": 0.6660802960395813, "losses/sft": 0.34118345379829407, "losses/total": 0.6660802960395813, "ref_logps/chosen": -69.11880493164062, "ref_logps/rejected": -56.906070709228516, "rewards/accuracies": 0.640625, "rewards/chosen": 0.026613827794790268, "rewards/margins": 0.04354187101125717, "rewards/rejected": -0.016928039491176605, "step": 19 }, { "epoch": 0.14, "grad_norm": 35.61816166775489, "learning_rate": 4.982319711683221e-07, "logps/chosen": -72.06625366210938, "logps/rejected": -59.52400207519531, "loss": 0.6721, "losses/dpo": 0.6615892052650452, "losses/sft": 0.34661561250686646, "losses/total": 0.6615892052650452, "ref_logps/chosen": -72.32228088378906, "ref_logps/rejected": -59.32331848144531, "rewards/accuracies": 0.65625, "rewards/chosen": 0.02560277283191681, "rewards/margins": 0.04567151144146919, "rewards/rejected": -0.02006874606013298, "step": 20 }, { "epoch": 0.14, "grad_norm": 35.39098813520557, "learning_rate": 4.974553604702332e-07, "logps/chosen": -69.02694702148438, "logps/rejected": -56.37590789794922, "loss": 0.6788, "losses/dpo": 0.6626015901565552, "losses/sft": 0.3223264813423157, "losses/total": 0.6626015901565552, "ref_logps/chosen": -69.218994140625, "ref_logps/rejected": -56.257652282714844, "rewards/accuracies": 0.625, "rewards/chosen": 0.01920466497540474, "rewards/margins": 0.031030891463160515, "rewards/rejected": -0.011826226487755775, "step": 21 }, { "epoch": 0.15, "grad_norm": 33.9907845041531, "learning_rate": 4.965385884295466e-07, "logps/chosen": -58.412986755371094, "logps/rejected": -53.20189666748047, "loss": 0.6719, "losses/dpo": 0.6767945289611816, "losses/sft": 0.3387165367603302, "losses/total": 0.6767945289611816, "ref_logps/chosen": -58.631317138671875, "ref_logps/rejected": -52.960777282714844, "rewards/accuracies": 0.640625, "rewards/chosen": 0.021833162754774094, "rewards/margins": 0.045944713056087494, "rewards/rejected": -0.024111557751893997, "step": 22 }, { "epoch": 0.16, "grad_norm": 33.80480054954672, "learning_rate": 4.954821743156767e-07, "logps/chosen": -67.24698638916016, "logps/rejected": -56.11846160888672, "loss": 0.6741, "losses/dpo": 0.6613680124282837, "losses/sft": 0.34880080819129944, "losses/total": 0.6613680124282837, "ref_logps/chosen": -67.44197082519531, "ref_logps/rejected": -55.9046630859375, "rewards/accuracies": 0.609375, "rewards/chosen": 0.019499044865369797, "rewards/margins": 0.040879033505916595, "rewards/rejected": -0.0213799886405468, "step": 23 }, { "epoch": 0.16, "grad_norm": 33.73597666461483, "learning_rate": 4.942867164927899e-07, "logps/chosen": -62.87995910644531, "logps/rejected": -55.92186737060547, "loss": 0.6554, "losses/dpo": 0.6621065139770508, "losses/sft": 0.350619375705719, "losses/total": 0.6621065139770508, "ref_logps/chosen": -63.19189453125, "ref_logps/rejected": -55.41385269165039, "rewards/accuracies": 0.6875, "rewards/chosen": 0.031193653121590614, "rewards/margins": 0.08199521154165268, "rewards/rejected": -0.05080154910683632, "step": 24 }, { "epoch": 0.17, "grad_norm": 31.252590331214783, "learning_rate": 4.929528920808854e-07, "logps/chosen": -70.07772064208984, "logps/rejected": -58.75320816040039, "loss": 0.6566, "losses/dpo": 0.646221935749054, "losses/sft": 0.3677119016647339, "losses/total": 0.646221935749054, "ref_logps/chosen": -70.34449005126953, "ref_logps/rejected": -58.24370574951172, "rewards/accuracies": 0.75, "rewards/chosen": 0.026676896959543228, "rewards/margins": 0.07762698084115982, "rewards/rejected": -0.05095008388161659, "step": 25 }, { "epoch": 0.18, "grad_norm": 31.628164542125273, "learning_rate": 4.91481456572267e-07, "logps/chosen": -63.15786361694336, "logps/rejected": -51.286197662353516, "loss": 0.6378, "losses/dpo": 0.6170041561126709, "losses/sft": 0.36194464564323425, "losses/total": 0.6170041561126709, "ref_logps/chosen": -63.57653045654297, "ref_logps/rejected": -50.502952575683594, "rewards/accuracies": 0.796875, "rewards/chosen": 0.04186610132455826, "rewards/margins": 0.12019069492816925, "rewards/rejected": -0.07832459360361099, "step": 26 }, { "epoch": 0.18, "grad_norm": 29.472445320476258, "learning_rate": 4.898732434036243e-07, "logps/chosen": -60.576107025146484, "logps/rejected": -51.86144256591797, "loss": 0.6304, "losses/dpo": 0.6577066779136658, "losses/sft": 0.31096988916397095, "losses/total": 0.6577066779136658, "ref_logps/chosen": -61.04180145263672, "ref_logps/rejected": -50.955745697021484, "rewards/accuracies": 0.828125, "rewards/chosen": 0.04656900465488434, "rewards/margins": 0.13713900744915009, "rewards/rejected": -0.09057000279426575, "step": 27 }, { "epoch": 0.19, "grad_norm": 34.12892875093247, "learning_rate": 4.881291634839651e-07, "logps/chosen": -71.571044921875, "logps/rejected": -59.838462829589844, "loss": 0.6348, "losses/dpo": 0.6427409052848816, "losses/sft": 0.370533287525177, "losses/total": 0.6427409052848816, "ref_logps/chosen": -71.80653381347656, "ref_logps/rejected": -58.735103607177734, "rewards/accuracies": 0.640625, "rewards/chosen": 0.023549124598503113, "rewards/margins": 0.1338854283094406, "rewards/rejected": -0.1103362962603569, "step": 28 }, { "epoch": 0.2, "grad_norm": 32.265156441586456, "learning_rate": 4.86250204678667e-07, "logps/chosen": -65.21573638916016, "logps/rejected": -56.004364013671875, "loss": 0.6235, "losses/dpo": 0.6029931902885437, "losses/sft": 0.3526380658149719, "losses/total": 0.6029931902885437, "ref_logps/chosen": -65.50960540771484, "ref_logps/rejected": -54.726261138916016, "rewards/accuracies": 0.8125, "rewards/chosen": 0.029387805610895157, "rewards/margins": 0.15719717741012573, "rewards/rejected": -0.12780936062335968, "step": 29 }, { "epoch": 0.2, "grad_norm": 37.3823888174941, "learning_rate": 4.842374312499405e-07, "logps/chosen": -73.87957763671875, "logps/rejected": -60.16520309448242, "loss": 0.5862, "losses/dpo": 0.5859757661819458, "losses/sft": 0.34089845418930054, "losses/total": 0.5859757661819458, "ref_logps/chosen": -74.58474731445312, "ref_logps/rejected": -58.38753890991211, "rewards/accuracies": 0.796875, "rewards/chosen": 0.0705181285738945, "rewards/margins": 0.24828468263149261, "rewards/rejected": -0.1777665764093399, "step": 30 }, { "epoch": 0.21, "grad_norm": 32.144180117174194, "learning_rate": 4.820919832540181e-07, "logps/chosen": -65.9237060546875, "logps/rejected": -57.733497619628906, "loss": 0.6071, "losses/dpo": 0.6222472190856934, "losses/sft": 0.33616167306900024, "losses/total": 0.6222472190856934, "ref_logps/chosen": -65.80138397216797, "ref_logps/rejected": -55.510379791259766, "rewards/accuracies": 0.71875, "rewards/chosen": -0.012232396751642227, "rewards/margins": 0.21007922291755676, "rewards/rejected": -0.22231163084506989, "step": 31 }, { "epoch": 0.22, "grad_norm": 35.501453167718694, "learning_rate": 4.798150758954164e-07, "logps/chosen": -68.75838470458984, "logps/rejected": -60.68508529663086, "loss": 0.5798, "losses/dpo": 0.5868023037910461, "losses/sft": 0.3538406789302826, "losses/total": 0.5868023037910461, "ref_logps/chosen": -68.71626281738281, "ref_logps/rejected": -57.89375305175781, "rewards/accuracies": 0.765625, "rewards/chosen": -0.004212833940982819, "rewards/margins": 0.2749200761318207, "rewards/rejected": -0.2791329324245453, "step": 32 }, { "epoch": 0.22, "grad_norm": 32.699996982131026, "learning_rate": 4.774079988386296e-07, "logps/chosen": -70.96333312988281, "logps/rejected": -61.67507553100586, "loss": 0.5944, "losses/dpo": 0.5362982749938965, "losses/sft": 0.37232011556625366, "losses/total": 0.5362982749938965, "ref_logps/chosen": -70.30509948730469, "ref_logps/rejected": -58.60476303100586, "rewards/accuracies": 0.765625, "rewards/chosen": -0.06582287698984146, "rewards/margins": 0.24120771884918213, "rewards/rejected": -0.3070306181907654, "step": 33 }, { "epoch": 0.23, "grad_norm": 37.488816567711076, "learning_rate": 4.748721154776508e-07, "logps/chosen": -65.87406921386719, "logps/rejected": -59.7636604309082, "loss": 0.5652, "losses/dpo": 0.500527024269104, "losses/sft": 0.3647816479206085, "losses/total": 0.500527024269104, "ref_logps/chosen": -65.68878173828125, "ref_logps/rejected": -56.097267150878906, "rewards/accuracies": 0.734375, "rewards/chosen": -0.018529564142227173, "rewards/margins": 0.3481099605560303, "rewards/rejected": -0.36663955450057983, "step": 34 }, { "epoch": 0.24, "grad_norm": 33.919463466183984, "learning_rate": 4.7220886216373085e-07, "logps/chosen": -67.95443725585938, "logps/rejected": -64.25914001464844, "loss": 0.5351, "losses/dpo": 0.5877989530563354, "losses/sft": 0.3324495255947113, "losses/total": 0.5877989530563354, "ref_logps/chosen": -67.34272766113281, "ref_logps/rejected": -59.69349670410156, "rewards/accuracies": 0.875, "rewards/chosen": -0.06117025017738342, "rewards/margins": 0.3953939974308014, "rewards/rejected": -0.4565642774105072, "step": 35 }, { "epoch": 0.24, "grad_norm": 36.85611869447123, "learning_rate": 4.694197473918139e-07, "logps/chosen": -73.41097259521484, "logps/rejected": -63.35462188720703, "loss": 0.5958, "losses/dpo": 0.5920328497886658, "losses/sft": 0.34827345609664917, "losses/total": 0.5920328497886658, "ref_logps/chosen": -72.06607055664062, "ref_logps/rejected": -59.26370620727539, "rewards/accuracies": 0.671875, "rewards/chosen": -0.1344902515411377, "rewards/margins": 0.2746003270149231, "rewards/rejected": -0.4090905785560608, "step": 36 }, { "epoch": 0.25, "grad_norm": 29.396807219224034, "learning_rate": 4.6650635094610966e-07, "logps/chosen": -69.74252319335938, "logps/rejected": -59.216861724853516, "loss": 0.511, "losses/dpo": 0.4569849967956543, "losses/sft": 0.330870658159256, "losses/total": 0.4569849967956543, "ref_logps/chosen": -68.98128509521484, "ref_logps/rejected": -53.205997467041016, "rewards/accuracies": 0.84375, "rewards/chosen": -0.07612372934818268, "rewards/margins": 0.5249630212783813, "rewards/rejected": -0.6010866761207581, "step": 37 }, { "epoch": 0.26, "grad_norm": 32.158672339711444, "learning_rate": 4.63470323005287e-07, "logps/chosen": -74.17395782470703, "logps/rejected": -65.09367370605469, "loss": 0.4616, "losses/dpo": 0.4395897388458252, "losses/sft": 0.3282167613506317, "losses/total": 0.4395897388458252, "ref_logps/chosen": -72.66117858886719, "ref_logps/rejected": -56.74953079223633, "rewards/accuracies": 0.859375, "rewards/chosen": -0.15127676725387573, "rewards/margins": 0.6831383109092712, "rewards/rejected": -0.8344150185585022, "step": 38 }, { "epoch": 0.27, "grad_norm": 30.39214935377121, "learning_rate": 4.603133832077953e-07, "logps/chosen": -72.17027282714844, "logps/rejected": -66.1070785522461, "loss": 0.5353, "losses/dpo": 0.47761982679367065, "losses/sft": 0.38627445697784424, "losses/total": 0.47761982679367065, "ref_logps/chosen": -69.07978820800781, "ref_logps/rejected": -58.215328216552734, "rewards/accuracies": 0.75, "rewards/chosen": -0.3090481758117676, "rewards/margins": 0.4801270365715027, "rewards/rejected": -0.7891752123832703, "step": 39 }, { "epoch": 0.27, "grad_norm": 29.86640568721065, "learning_rate": 4.5703731967784265e-07, "logps/chosen": -61.24689483642578, "logps/rejected": -59.919918060302734, "loss": 0.5273, "losses/dpo": 0.7497649192810059, "losses/sft": 0.35406211018562317, "losses/total": 0.7497649192810059, "ref_logps/chosen": -57.38946533203125, "ref_logps/rejected": -50.80248260498047, "rewards/accuracies": 0.734375, "rewards/chosen": -0.3857432007789612, "rewards/margins": 0.5260001420974731, "rewards/rejected": -0.9117434024810791, "step": 40 }, { "epoch": 0.28, "grad_norm": 29.500472780356045, "learning_rate": 4.5364398801258394e-07, "logps/chosen": -63.82752990722656, "logps/rejected": -60.21675491333008, "loss": 0.5103, "losses/dpo": 0.5127725601196289, "losses/sft": 0.3456054925918579, "losses/total": 0.5127725601196289, "ref_logps/chosen": -60.51768493652344, "ref_logps/rejected": -50.69208526611328, "rewards/accuracies": 0.84375, "rewards/chosen": -0.3309844732284546, "rewards/margins": 0.6214821934700012, "rewards/rejected": -0.952466607093811, "step": 41 }, { "epoch": 0.29, "grad_norm": 28.303717346192418, "learning_rate": 4.501353102310901e-07, "logps/chosen": -64.48641967773438, "logps/rejected": -60.165767669677734, "loss": 0.5197, "losses/dpo": 0.4626217782497406, "losses/sft": 0.3867911994457245, "losses/total": 0.4626217782497406, "ref_logps/chosen": -60.505496978759766, "ref_logps/rejected": -50.563045501708984, "rewards/accuracies": 0.765625, "rewards/chosen": -0.3980926275253296, "rewards/margins": 0.562179684638977, "rewards/rejected": -0.9602723121643066, "step": 42 }, { "epoch": 0.29, "grad_norm": 26.546916183591545, "learning_rate": 4.4651327368569684e-07, "logps/chosen": -74.27953338623047, "logps/rejected": -66.18184661865234, "loss": 0.478, "losses/dpo": 0.5494570136070251, "losses/sft": 0.3961746394634247, "losses/total": 0.5494570136070251, "ref_logps/chosen": -68.87181854248047, "ref_logps/rejected": -54.055946350097656, "rewards/accuracies": 0.84375, "rewards/chosen": -0.5407715439796448, "rewards/margins": 0.671818733215332, "rewards/rejected": -1.212590217590332, "step": 43 }, { "epoch": 0.3, "grad_norm": 26.197389360540555, "learning_rate": 4.4277992993634686e-07, "logps/chosen": -79.60818481445312, "logps/rejected": -79.91624450683594, "loss": 0.4052, "losses/dpo": 0.4185926914215088, "losses/sft": 0.3548857569694519, "losses/total": 0.4185926914215088, "ref_logps/chosen": -72.74970245361328, "ref_logps/rejected": -62.91980743408203, "rewards/accuracies": 0.84375, "rewards/chosen": -0.6858479976654053, "rewards/margins": 1.0137956142425537, "rewards/rejected": -1.699643611907959, "step": 44 }, { "epoch": 0.31, "grad_norm": 27.708445312624427, "learning_rate": 4.3893739358856455e-07, "logps/chosen": -79.15251159667969, "logps/rejected": -72.04096221923828, "loss": 0.4729, "losses/dpo": 0.4918556213378906, "losses/sft": 0.37935709953308105, "losses/total": 0.4918556213378906, "ref_logps/chosen": -72.23287963867188, "ref_logps/rejected": -56.32197570800781, "rewards/accuracies": 0.71875, "rewards/chosen": -0.6919631958007812, "rewards/margins": 0.8799355626106262, "rewards/rejected": -1.5718988180160522, "step": 45 }, { "epoch": 0.31, "grad_norm": 30.962017558121154, "learning_rate": 4.34987841095721e-07, "logps/chosen": -75.75450134277344, "logps/rejected": -75.48016357421875, "loss": 0.4757, "losses/dpo": 0.47784677147865295, "losses/sft": 0.37939363718032837, "losses/total": 0.47784677147865295, "ref_logps/chosen": -67.21387481689453, "ref_logps/rejected": -58.56239318847656, "rewards/accuracies": 0.84375, "rewards/chosen": -0.8540633320808411, "rewards/margins": 0.8377144932746887, "rewards/rejected": -1.6917778253555298, "step": 46 }, { "epoch": 0.32, "grad_norm": 27.327055413855135, "learning_rate": 4.309335095262675e-07, "logps/chosen": -84.39332580566406, "logps/rejected": -78.34506225585938, "loss": 0.4763, "losses/dpo": 0.3084137439727783, "losses/sft": 0.3708834648132324, "losses/total": 0.3084137439727783, "ref_logps/chosen": -74.23382568359375, "ref_logps/rejected": -59.50791931152344, "rewards/accuracies": 0.671875, "rewards/chosen": -1.0159505605697632, "rewards/margins": 0.8677631616592407, "rewards/rejected": -1.8837138414382935, "step": 47 }, { "epoch": 0.33, "grad_norm": 37.60224677954145, "learning_rate": 4.2677669529663686e-07, "logps/chosen": -85.45326232910156, "logps/rejected": -80.37303161621094, "loss": 0.498, "losses/dpo": 0.5172345638275146, "losses/sft": 0.46193623542785645, "losses/total": 0.5172345638275146, "ref_logps/chosen": -76.32438659667969, "ref_logps/rejected": -63.45566940307617, "rewards/accuracies": 0.734375, "rewards/chosen": -0.9128875732421875, "rewards/margins": 0.7788479328155518, "rewards/rejected": -1.6917355060577393, "step": 48 }, { "epoch": 0.33, "grad_norm": 26.012407404622028, "learning_rate": 4.22519752870528e-07, "logps/chosen": -86.257080078125, "logps/rejected": -79.96172332763672, "loss": 0.4015, "losses/dpo": 0.3850885331630707, "losses/sft": 0.4463062882423401, "losses/total": 0.3850885331630707, "ref_logps/chosen": -77.25395965576172, "ref_logps/rejected": -60.78544998168945, "rewards/accuracies": 0.890625, "rewards/chosen": -0.9003111124038696, "rewards/margins": 1.0173161029815674, "rewards/rejected": -1.9176270961761475, "step": 49 }, { "epoch": 0.34, "grad_norm": 29.5339729693226, "learning_rate": 4.1816509342531317e-07, "logps/chosen": -72.80648803710938, "logps/rejected": -73.68412780761719, "loss": 0.4609, "losses/dpo": 0.5073346495628357, "losses/sft": 0.3690868318080902, "losses/total": 0.5073346495628357, "ref_logps/chosen": -61.75213623046875, "ref_logps/rejected": -53.534000396728516, "rewards/accuracies": 0.765625, "rewards/chosen": -1.1054348945617676, "rewards/margins": 0.9095778465270996, "rewards/rejected": -2.015012741088867, "step": 50 }, { "epoch": 0.35, "grad_norm": 24.67832174233356, "learning_rate": 4.137151834863213e-07, "logps/chosen": -78.76303100585938, "logps/rejected": -81.20369720458984, "loss": 0.3581, "losses/dpo": 0.31153589487075806, "losses/sft": 0.34659337997436523, "losses/total": 0.31153589487075806, "ref_logps/chosen": -67.64138793945312, "ref_logps/rejected": -56.615203857421875, "rewards/accuracies": 0.84375, "rewards/chosen": -1.1121646165847778, "rewards/margins": 1.3466840982437134, "rewards/rejected": -2.458848714828491, "step": 51 }, { "epoch": 0.35, "grad_norm": 25.652161578301136, "learning_rate": 4.0917254352977206e-07, "logps/chosen": -80.27860260009766, "logps/rejected": -78.06929016113281, "loss": 0.4625, "losses/dpo": 0.5235142707824707, "losses/sft": 0.3862667977809906, "losses/total": 0.5235142707824707, "ref_logps/chosen": -65.9322280883789, "ref_logps/rejected": -54.390098571777344, "rewards/accuracies": 0.8125, "rewards/chosen": -1.4346369504928589, "rewards/margins": 0.9332831501960754, "rewards/rejected": -2.367920160293579, "step": 52 }, { "epoch": 0.36, "grad_norm": 31.613726978757875, "learning_rate": 4.045397465551513e-07, "logps/chosen": -84.57078552246094, "logps/rejected": -76.68920135498047, "loss": 0.5117, "losses/dpo": 0.4083520770072937, "losses/sft": 0.32504379749298096, "losses/total": 0.4083520770072937, "ref_logps/chosen": -72.13128662109375, "ref_logps/rejected": -53.99201965332031, "rewards/accuracies": 0.78125, "rewards/chosen": -1.2439489364624023, "rewards/margins": 1.0257703065872192, "rewards/rejected": -2.269719123840332, "step": 53 }, { "epoch": 0.37, "grad_norm": 29.84992723438259, "learning_rate": 3.998194166278367e-07, "logps/chosen": -76.96097564697266, "logps/rejected": -74.58714294433594, "loss": 0.4777, "losses/dpo": 0.4091704785823822, "losses/sft": 0.4479122757911682, "losses/total": 0.4091704785823822, "ref_logps/chosen": -61.228057861328125, "ref_logps/rejected": -50.48232650756836, "rewards/accuracies": 0.71875, "rewards/chosen": -1.573291540145874, "rewards/margins": 0.8371899127960205, "rewards/rejected": -2.4104812145233154, "step": 54 }, { "epoch": 0.37, "grad_norm": 36.88672265696095, "learning_rate": 3.9501422739279953e-07, "logps/chosen": -81.66922760009766, "logps/rejected": -75.91566467285156, "loss": 0.6061, "losses/dpo": 0.5785080194473267, "losses/sft": 0.37718018889427185, "losses/total": 0.5785080194473267, "ref_logps/chosen": -67.54790496826172, "ref_logps/rejected": -57.624385833740234, "rewards/accuracies": 0.640625, "rewards/chosen": -1.4121325016021729, "rewards/margins": 0.4169955253601074, "rewards/rejected": -1.8291279077529907, "step": 55 }, { "epoch": 0.38, "grad_norm": 26.409540611725895, "learning_rate": 3.9012690056022345e-07, "logps/chosen": -88.31976318359375, "logps/rejected": -80.9065170288086, "loss": 0.3836, "losses/dpo": 0.4441046714782715, "losses/sft": 0.4260564148426056, "losses/total": 0.4441046714782715, "ref_logps/chosen": -74.53655242919922, "ref_logps/rejected": -56.45116424560547, "rewards/accuracies": 0.875, "rewards/chosen": -1.3783206939697266, "rewards/margins": 1.0672156810760498, "rewards/rejected": -2.4455361366271973, "step": 56 }, { "epoch": 0.39, "grad_norm": 26.34257247806165, "learning_rate": 3.851602043638994e-07, "logps/chosen": -80.94815826416016, "logps/rejected": -77.87812805175781, "loss": 0.4378, "losses/dpo": 0.38587698340415955, "losses/sft": 0.39670416712760925, "losses/total": 0.38587698340415955, "ref_logps/chosen": -64.29881286621094, "ref_logps/rejected": -51.452911376953125, "rewards/accuracies": 0.8125, "rewards/chosen": -1.6649348735809326, "rewards/margins": 0.9775880575180054, "rewards/rejected": -2.6425228118896484, "step": 57 }, { "epoch": 0.39, "grad_norm": 27.08511242764537, "learning_rate": 3.8011695199326805e-07, "logps/chosen": -80.61515808105469, "logps/rejected": -81.2110366821289, "loss": 0.4141, "losses/dpo": 0.3337501883506775, "losses/sft": 0.3477111756801605, "losses/total": 0.3337501883506775, "ref_logps/chosen": -66.6812744140625, "ref_logps/rejected": -56.64939880371094, "rewards/accuracies": 0.84375, "rewards/chosen": -1.393389344215393, "rewards/margins": 1.062773585319519, "rewards/rejected": -2.456162929534912, "step": 58 }, { "epoch": 0.4, "grad_norm": 29.443828368623393, "learning_rate": 3.75e-07, "logps/chosen": -79.15809631347656, "logps/rejected": -77.99955749511719, "loss": 0.4174, "losses/dpo": 0.2030327320098877, "losses/sft": 0.36094900965690613, "losses/total": 0.2030327320098877, "ref_logps/chosen": -64.46220397949219, "ref_logps/rejected": -51.404056549072266, "rewards/accuracies": 0.75, "rewards/chosen": -1.4695887565612793, "rewards/margins": 1.1899616718292236, "rewards/rejected": -2.659550428390503, "step": 59 }, { "epoch": 0.41, "grad_norm": 31.360451766133608, "learning_rate": 3.698122466800142e-07, "logps/chosen": -83.48946380615234, "logps/rejected": -80.81828308105469, "loss": 0.5084, "losses/dpo": 0.6314846873283386, "losses/sft": 0.4423762559890747, "losses/total": 0.6314846873283386, "ref_logps/chosen": -69.77751159667969, "ref_logps/rejected": -59.079166412353516, "rewards/accuracies": 0.734375, "rewards/chosen": -1.3711949586868286, "rewards/margins": 0.802715539932251, "rewards/rejected": -2.173910617828369, "step": 60 }, { "epoch": 0.41, "grad_norm": 32.77515833972762, "learning_rate": 3.645566304318526e-07, "logps/chosen": -88.54704284667969, "logps/rejected": -83.86685180664062, "loss": 0.4477, "losses/dpo": 0.4280022382736206, "losses/sft": 0.3504616618156433, "losses/total": 0.4280022382736206, "ref_logps/chosen": -75.23544311523438, "ref_logps/rejected": -60.14040756225586, "rewards/accuracies": 0.84375, "rewards/chosen": -1.331160068511963, "rewards/margins": 1.04148530960083, "rewards/rejected": -2.372645378112793, "step": 61 }, { "epoch": 0.42, "grad_norm": 27.946308305497556, "learning_rate": 3.5923612809233984e-07, "logps/chosen": -78.10906982421875, "logps/rejected": -79.12262725830078, "loss": 0.4413, "losses/dpo": 0.4408959746360779, "losses/sft": 0.3510644733905792, "losses/total": 0.4408959746360779, "ref_logps/chosen": -64.80330657958984, "ref_logps/rejected": -56.04015350341797, "rewards/accuracies": 0.828125, "rewards/chosen": -1.3305763006210327, "rewards/margins": 0.9776719212532043, "rewards/rejected": -2.308248281478882, "step": 62 }, { "epoch": 0.43, "grad_norm": 41.17153515866478, "learning_rate": 3.5385375325047163e-07, "logps/chosen": -80.082275390625, "logps/rejected": -79.17640686035156, "loss": 0.4871, "losses/dpo": 0.45268240571022034, "losses/sft": 0.3942197263240814, "losses/total": 0.45268240571022034, "ref_logps/chosen": -65.72177124023438, "ref_logps/rejected": -55.820762634277344, "rewards/accuracies": 0.765625, "rewards/chosen": -1.4360499382019043, "rewards/margins": 0.8995141983032227, "rewards/rejected": -2.335564136505127, "step": 63 }, { "epoch": 0.44, "grad_norm": 26.987470660431804, "learning_rate": 3.4841255454048535e-07, "logps/chosen": -78.40692138671875, "logps/rejected": -81.3203125, "loss": 0.4319, "losses/dpo": 0.3416772782802582, "losses/sft": 0.351028174161911, "losses/total": 0.3416772782802582, "ref_logps/chosen": -66.26089477539062, "ref_logps/rejected": -58.60002136230469, "rewards/accuracies": 0.84375, "rewards/chosen": -1.214601755142212, "rewards/margins": 1.057426929473877, "rewards/rejected": -2.272028923034668, "step": 64 }, { "epoch": 0.44, "grad_norm": 33.31694146367439, "learning_rate": 3.4291561391508185e-07, "logps/chosen": -82.72046661376953, "logps/rejected": -85.01447296142578, "loss": 0.494, "losses/dpo": 0.4605589509010315, "losses/sft": 0.3883049190044403, "losses/total": 0.4605589509010315, "ref_logps/chosen": -67.7024917602539, "ref_logps/rejected": -59.51292419433594, "rewards/accuracies": 0.703125, "rewards/chosen": -1.5017977952957153, "rewards/margins": 1.0483572483062744, "rewards/rejected": -2.5501549243927, "step": 65 }, { "epoch": 0.45, "grad_norm": 30.20641981592105, "learning_rate": 3.373660448997746e-07, "logps/chosen": -78.78214263916016, "logps/rejected": -76.67687225341797, "loss": 0.461, "losses/dpo": 0.3649926483631134, "losses/sft": 0.3876701891422272, "losses/total": 0.3649926483631134, "ref_logps/chosen": -65.60338592529297, "ref_logps/rejected": -53.363441467285156, "rewards/accuracies": 0.765625, "rewards/chosen": -1.3178752660751343, "rewards/margins": 1.0134679079055786, "rewards/rejected": -2.331343173980713, "step": 66 }, { "epoch": 0.46, "grad_norm": 28.499338933696293, "learning_rate": 3.317669908293554e-07, "logps/chosen": -80.8163070678711, "logps/rejected": -78.53517150878906, "loss": 0.4977, "losses/dpo": 0.5431786775588989, "losses/sft": 0.3635212779045105, "losses/total": 0.5431786775588989, "ref_logps/chosen": -67.38165283203125, "ref_logps/rejected": -56.12194061279297, "rewards/accuracies": 0.765625, "rewards/chosen": -1.3434655666351318, "rewards/margins": 0.8978573679924011, "rewards/rejected": -2.2413227558135986, "step": 67 }, { "epoch": 0.46, "grad_norm": 30.8426766786845, "learning_rate": 3.261216230674768e-07, "logps/chosen": -80.0118408203125, "logps/rejected": -78.16852569580078, "loss": 0.4656, "losses/dpo": 0.22820590436458588, "losses/sft": 0.41188591718673706, "losses/total": 0.22820590436458588, "ref_logps/chosen": -67.98802185058594, "ref_logps/rejected": -55.998008728027344, "rewards/accuracies": 0.765625, "rewards/chosen": -1.2023829221725464, "rewards/margins": 1.0146691799163818, "rewards/rejected": -2.2170519828796387, "step": 68 }, { "epoch": 0.47, "grad_norm": 26.8942337262588, "learning_rate": 3.204331392103574e-07, "logps/chosen": -76.27379608154297, "logps/rejected": -72.5108413696289, "loss": 0.3977, "losses/dpo": 0.349028617143631, "losses/sft": 0.36189785599708557, "losses/total": 0.349028617143631, "ref_logps/chosen": -65.81362915039062, "ref_logps/rejected": -51.00428009033203, "rewards/accuracies": 0.890625, "rewards/chosen": -1.0460165739059448, "rewards/margins": 1.1046404838562012, "rewards/rejected": -2.1506569385528564, "step": 69 }, { "epoch": 0.48, "grad_norm": 26.047792222109905, "learning_rate": 3.147047612756302e-07, "logps/chosen": -86.95272827148438, "logps/rejected": -83.82672119140625, "loss": 0.3953, "losses/dpo": 0.46198731660842896, "losses/sft": 0.4407782554626465, "losses/total": 0.46198731660842896, "ref_logps/chosen": -75.7898178100586, "ref_logps/rejected": -60.74498748779297, "rewards/accuracies": 0.8125, "rewards/chosen": -1.1162903308868408, "rewards/margins": 1.1918830871582031, "rewards/rejected": -2.308173418045044, "step": 70 }, { "epoch": 0.48, "grad_norm": 24.670894739162968, "learning_rate": 3.0893973387735683e-07, "logps/chosen": -70.71044921875, "logps/rejected": -71.96138763427734, "loss": 0.3944, "losses/dpo": 0.29774680733680725, "losses/sft": 0.3790546953678131, "losses/total": 0.29774680733680725, "ref_logps/chosen": -61.44975280761719, "ref_logps/rejected": -51.039268493652344, "rewards/accuracies": 0.859375, "rewards/chosen": -0.9260695576667786, "rewards/margins": 1.166142225265503, "rewards/rejected": -2.092211961746216, "step": 71 }, { "epoch": 0.49, "grad_norm": 28.239155495003125, "learning_rate": 3.0314132238824415e-07, "logps/chosen": -81.26232147216797, "logps/rejected": -78.04046630859375, "loss": 0.3912, "losses/dpo": 0.5779224038124084, "losses/sft": 0.3678759038448334, "losses/total": 0.5779224038124084, "ref_logps/chosen": -72.41909790039062, "ref_logps/rejected": -57.58437728881836, "rewards/accuracies": 0.859375, "rewards/chosen": -0.8843216300010681, "rewards/margins": 1.1612873077392578, "rewards/rejected": -2.0456089973449707, "step": 72 }, { "epoch": 0.5, "grad_norm": 23.65680821014391, "learning_rate": 2.9731281109010253e-07, "logps/chosen": -78.29800415039062, "logps/rejected": -76.63243103027344, "loss": 0.3589, "losses/dpo": 0.39184096455574036, "losses/sft": 0.35749632120132446, "losses/total": 0.39184096455574036, "ref_logps/chosen": -70.57292175292969, "ref_logps/rejected": -56.80116653442383, "rewards/accuracies": 0.875, "rewards/chosen": -0.7725088000297546, "rewards/margins": 1.210618257522583, "rewards/rejected": -1.9831271171569824, "step": 73 }, { "epoch": 0.5, "grad_norm": 57.763902094880514, "learning_rate": 2.9145750131359355e-07, "logps/chosen": -71.69022369384766, "logps/rejected": -74.53846740722656, "loss": 0.4711, "losses/dpo": 0.3447693884372711, "losses/sft": 0.3251613676548004, "losses/total": 0.3447693884372711, "ref_logps/chosen": -61.327369689941406, "ref_logps/rejected": -53.874820709228516, "rewards/accuracies": 0.75, "rewards/chosen": -1.036285638809204, "rewards/margins": 1.0300801992416382, "rewards/rejected": -2.0663657188415527, "step": 74 }, { "epoch": 0.51, "grad_norm": 37.61505605094831, "learning_rate": 2.8557870956832133e-07, "logps/chosen": -78.79322052001953, "logps/rejected": -73.63280487060547, "loss": 0.4466, "losses/dpo": 0.4499085545539856, "losses/sft": 0.3389120101928711, "losses/total": 0.4499085545539856, "ref_logps/chosen": -70.82493591308594, "ref_logps/rejected": -56.25859832763672, "rewards/accuracies": 0.8125, "rewards/chosen": -0.7968282699584961, "rewards/margins": 0.9405927658081055, "rewards/rejected": -1.7374211549758911, "step": 75 }, { "epoch": 0.52, "grad_norm": 25.268092842517042, "learning_rate": 2.7967976566432627e-07, "logps/chosen": -79.96376037597656, "logps/rejected": -75.98302459716797, "loss": 0.3581, "losses/dpo": 0.3253936469554901, "losses/sft": 0.3978617191314697, "losses/total": 0.3253936469554901, "ref_logps/chosen": -70.92337036132812, "ref_logps/rejected": -53.73899841308594, "rewards/accuracies": 0.84375, "rewards/chosen": -0.9040391445159912, "rewards/margins": 1.320363163948059, "rewards/rejected": -2.2244021892547607, "step": 76 }, { "epoch": 0.52, "grad_norm": 24.79898935884772, "learning_rate": 2.737640108260456e-07, "logps/chosen": -74.60429382324219, "logps/rejected": -74.33484649658203, "loss": 0.3639, "losses/dpo": 0.19668929278850555, "losses/sft": 0.34319061040878296, "losses/total": 0.19668929278850555, "ref_logps/chosen": -67.25056457519531, "ref_logps/rejected": -54.8765869140625, "rewards/accuracies": 0.921875, "rewards/chosen": -0.7353739142417908, "rewards/margins": 1.2104520797729492, "rewards/rejected": -1.9458260536193848, "step": 77 }, { "epoch": 0.53, "grad_norm": 32.80096075705349, "learning_rate": 2.6783479579980807e-07, "logps/chosen": -73.52947998046875, "logps/rejected": -75.42818450927734, "loss": 0.4477, "losses/dpo": 0.4471086859703064, "losses/sft": 0.3155350685119629, "losses/total": 0.4471086859703064, "ref_logps/chosen": -64.02719116210938, "ref_logps/rejected": -56.57443618774414, "rewards/accuracies": 0.828125, "rewards/chosen": -0.9502288103103638, "rewards/margins": 0.9351459741592407, "rewards/rejected": -1.885374903678894, "step": 78 }, { "epoch": 0.54, "grad_norm": 30.01631729835111, "learning_rate": 2.618954789559356e-07, "logps/chosen": -78.71662902832031, "logps/rejected": -70.70683288574219, "loss": 0.4583, "losses/dpo": 0.4225349426269531, "losses/sft": 0.37885919213294983, "losses/total": 0.4225349426269531, "ref_logps/chosen": -70.06804656982422, "ref_logps/rejected": -53.02828598022461, "rewards/accuracies": 0.78125, "rewards/chosen": -0.8648585677146912, "rewards/margins": 0.9029962420463562, "rewards/rejected": -1.7678548097610474, "step": 79 }, { "epoch": 0.54, "grad_norm": 27.778753710279016, "learning_rate": 2.5594942438652685e-07, "logps/chosen": -74.57829284667969, "logps/rejected": -75.41488647460938, "loss": 0.4252, "losses/dpo": 0.4811527132987976, "losses/sft": 0.3381648361682892, "losses/total": 0.4811527132987976, "ref_logps/chosen": -65.43878173828125, "ref_logps/rejected": -56.21765899658203, "rewards/accuracies": 0.78125, "rewards/chosen": -0.9139504432678223, "rewards/margins": 1.005772352218628, "rewards/rejected": -1.9197226762771606, "step": 80 }, { "epoch": 0.55, "grad_norm": 27.418542939543443, "learning_rate": 2.5e-07, "logps/chosen": -84.89808654785156, "logps/rejected": -77.45452117919922, "loss": 0.457, "losses/dpo": 0.4078216552734375, "losses/sft": 0.3400785028934479, "losses/total": 0.4078216552734375, "ref_logps/chosen": -78.10498046875, "ref_logps/rejected": -61.716976165771484, "rewards/accuracies": 0.78125, "rewards/chosen": -0.6793106198310852, "rewards/margins": 0.8944438695907593, "rewards/rejected": -1.5737544298171997, "step": 81 }, { "epoch": 0.56, "grad_norm": 29.24936699437702, "learning_rate": 2.4405057561347313e-07, "logps/chosen": -71.50326538085938, "logps/rejected": -74.53978729248047, "loss": 0.496, "losses/dpo": 0.3848089277744293, "losses/sft": 0.3731555938720703, "losses/total": 0.3848089277744293, "ref_logps/chosen": -60.67548370361328, "ref_logps/rejected": -55.07884216308594, "rewards/accuracies": 0.75, "rewards/chosen": -1.0827784538269043, "rewards/margins": 0.8633174896240234, "rewards/rejected": -1.9460958242416382, "step": 82 }, { "epoch": 0.56, "grad_norm": 48.23269343517185, "learning_rate": 2.381045210440644e-07, "logps/chosen": -78.87052917480469, "logps/rejected": -73.31388092041016, "loss": 0.4528, "losses/dpo": 0.43188297748565674, "losses/sft": 0.36860212683677673, "losses/total": 0.43188297748565674, "ref_logps/chosen": -70.29788970947266, "ref_logps/rejected": -54.075828552246094, "rewards/accuracies": 0.796875, "rewards/chosen": -0.8572635650634766, "rewards/margins": 1.0665409564971924, "rewards/rejected": -1.923804521560669, "step": 83 }, { "epoch": 0.57, "grad_norm": 36.11750167575466, "learning_rate": 2.321652042001919e-07, "logps/chosen": -75.86955261230469, "logps/rejected": -77.73924255371094, "loss": 0.5125, "losses/dpo": 0.49389997124671936, "losses/sft": 0.4043150544166565, "losses/total": 0.49389997124671936, "ref_logps/chosen": -67.49108123779297, "ref_logps/rejected": -60.198638916015625, "rewards/accuracies": 0.734375, "rewards/chosen": -0.8378474712371826, "rewards/margins": 0.9162120819091797, "rewards/rejected": -1.7540594339370728, "step": 84 }, { "epoch": 0.58, "grad_norm": 44.25422567833337, "learning_rate": 2.2623598917395436e-07, "logps/chosen": -78.2550048828125, "logps/rejected": -69.63221740722656, "loss": 0.4812, "losses/dpo": 0.24741792678833008, "losses/sft": 0.38957494497299194, "losses/total": 0.24741792678833008, "ref_logps/chosen": -69.76954650878906, "ref_logps/rejected": -52.76110076904297, "rewards/accuracies": 0.765625, "rewards/chosen": -0.8485451936721802, "rewards/margins": 0.8385663628578186, "rewards/rejected": -1.6871116161346436, "step": 85 }, { "epoch": 0.58, "grad_norm": 29.36328595493774, "learning_rate": 2.2032023433567376e-07, "logps/chosen": -72.15760040283203, "logps/rejected": -76.89938354492188, "loss": 0.3683, "losses/dpo": 0.3277527093887329, "losses/sft": 0.34014564752578735, "losses/total": 0.3277527093887329, "ref_logps/chosen": -63.951847076416016, "ref_logps/rejected": -56.66627883911133, "rewards/accuracies": 0.875, "rewards/chosen": -0.8205756545066833, "rewards/margins": 1.2027339935302734, "rewards/rejected": -2.0233094692230225, "step": 86 }, { "epoch": 0.59, "grad_norm": 24.453783435169065, "learning_rate": 2.1442129043167873e-07, "logps/chosen": -75.3466796875, "logps/rejected": -76.63810729980469, "loss": 0.352, "losses/dpo": 0.4673713743686676, "losses/sft": 0.3854469656944275, "losses/total": 0.4673713743686676, "ref_logps/chosen": -68.66751861572266, "ref_logps/rejected": -56.724952697753906, "rewards/accuracies": 0.84375, "rewards/chosen": -0.6679168939590454, "rewards/margins": 1.3233978748321533, "rewards/rejected": -1.9913148880004883, "step": 87 }, { "epoch": 0.6, "grad_norm": 28.45352429062705, "learning_rate": 2.085424986864065e-07, "logps/chosen": -80.1172103881836, "logps/rejected": -72.56062316894531, "loss": 0.4422, "losses/dpo": 0.3042376637458801, "losses/sft": 0.28263095021247864, "losses/total": 0.3042376637458801, "ref_logps/chosen": -74.51995849609375, "ref_logps/rejected": -56.53531265258789, "rewards/accuracies": 0.8125, "rewards/chosen": -0.5597256422042847, "rewards/margins": 1.0428049564361572, "rewards/rejected": -1.602530598640442, "step": 88 }, { "epoch": 0.6, "grad_norm": 29.271005488942123, "learning_rate": 2.0268718890989752e-07, "logps/chosen": -71.47293853759766, "logps/rejected": -76.03471374511719, "loss": 0.4194, "losses/dpo": 0.4656490683555603, "losses/sft": 0.3225964307785034, "losses/total": 0.4656490683555603, "ref_logps/chosen": -63.167877197265625, "ref_logps/rejected": -56.3370361328125, "rewards/accuracies": 0.859375, "rewards/chosen": -0.8305058479309082, "rewards/margins": 1.1392616033554077, "rewards/rejected": -1.9697675704956055, "step": 89 }, { "epoch": 0.61, "grad_norm": 35.681125753631115, "learning_rate": 1.968586776117558e-07, "logps/chosen": -79.76475524902344, "logps/rejected": -73.98800659179688, "loss": 0.4659, "losses/dpo": 0.35364383459091187, "losses/sft": 0.3435375690460205, "losses/total": 0.35364383459091187, "ref_logps/chosen": -72.63055419921875, "ref_logps/rejected": -56.81049346923828, "rewards/accuracies": 0.734375, "rewards/chosen": -0.7134197950363159, "rewards/margins": 1.004331350326538, "rewards/rejected": -1.7177510261535645, "step": 90 }, { "epoch": 0.62, "grad_norm": 36.49093259706875, "learning_rate": 1.9106026612264315e-07, "logps/chosen": -78.425537109375, "logps/rejected": -74.1988525390625, "loss": 0.3985, "losses/dpo": 0.19245147705078125, "losses/sft": 0.39025193452835083, "losses/total": 0.19245147705078125, "ref_logps/chosen": -69.4635009765625, "ref_logps/rejected": -53.60189437866211, "rewards/accuracies": 0.8125, "rewards/chosen": -0.8962031006813049, "rewards/margins": 1.1634931564331055, "rewards/rejected": -2.0596961975097656, "step": 91 }, { "epoch": 0.63, "grad_norm": 27.105142818170325, "learning_rate": 1.8529523872436977e-07, "logps/chosen": -75.47040557861328, "logps/rejected": -75.96112060546875, "loss": 0.4417, "losses/dpo": 0.3523982763290405, "losses/sft": 0.3631323575973511, "losses/total": 0.3523982763290405, "ref_logps/chosen": -67.11583709716797, "ref_logps/rejected": -58.03672790527344, "rewards/accuracies": 0.796875, "rewards/chosen": -0.8354578018188477, "rewards/margins": 0.9569823145866394, "rewards/rejected": -1.7924400568008423, "step": 92 }, { "epoch": 0.63, "grad_norm": 38.458971249921234, "learning_rate": 1.7956686078964255e-07, "logps/chosen": -72.20645141601562, "logps/rejected": -72.76055145263672, "loss": 0.4654, "losses/dpo": 0.48378482460975647, "losses/sft": 0.2976914644241333, "losses/total": 0.48378482460975647, "ref_logps/chosen": -64.7495346069336, "ref_logps/rejected": -56.04286193847656, "rewards/accuracies": 0.8125, "rewards/chosen": -0.7456916570663452, "rewards/margins": 0.9260772466659546, "rewards/rejected": -1.671769142150879, "step": 93 }, { "epoch": 0.64, "grad_norm": 21.64298997501433, "learning_rate": 1.7387837693252327e-07, "logps/chosen": -80.37052917480469, "logps/rejected": -79.18669891357422, "loss": 0.3604, "losses/dpo": 0.3508569896221161, "losses/sft": 0.39343804121017456, "losses/total": 0.3508569896221161, "ref_logps/chosen": -72.72434997558594, "ref_logps/rejected": -58.45079803466797, "rewards/accuracies": 0.875, "rewards/chosen": -0.764617919921875, "rewards/margins": 1.3089728355407715, "rewards/rejected": -2.0735907554626465, "step": 94 }, { "epoch": 0.65, "grad_norm": 28.31625318441289, "learning_rate": 1.6823300917064458e-07, "logps/chosen": -74.7480697631836, "logps/rejected": -71.78422546386719, "loss": 0.4383, "losses/dpo": 0.17416918277740479, "losses/sft": 0.3635005056858063, "losses/total": 0.17416918277740479, "ref_logps/chosen": -66.6783447265625, "ref_logps/rejected": -53.81765365600586, "rewards/accuracies": 0.75, "rewards/chosen": -0.8069729804992676, "rewards/margins": 0.9896839261054993, "rewards/rejected": -1.796656847000122, "step": 95 }, { "epoch": 0.65, "grad_norm": 31.43003387376896, "learning_rate": 1.6263395510022543e-07, "logps/chosen": -82.76495361328125, "logps/rejected": -77.74391174316406, "loss": 0.3885, "losses/dpo": 0.40873783826828003, "losses/sft": 0.3334735929965973, "losses/total": 0.40873783826828003, "ref_logps/chosen": -74.447998046875, "ref_logps/rejected": -57.47111511230469, "rewards/accuracies": 0.84375, "rewards/chosen": -0.8316947221755981, "rewards/margins": 1.1955851316452026, "rewards/rejected": -2.0272796154022217, "step": 96 }, { "epoch": 0.66, "grad_norm": 29.307347493310413, "learning_rate": 1.5708438608491815e-07, "logps/chosen": -73.7046890258789, "logps/rejected": -74.98145294189453, "loss": 0.4818, "losses/dpo": 0.3637702465057373, "losses/sft": 0.3260781168937683, "losses/total": 0.3637702465057373, "ref_logps/chosen": -67.44843292236328, "ref_logps/rejected": -59.09331130981445, "rewards/accuracies": 0.765625, "rewards/chosen": -0.625625491142273, "rewards/margins": 0.9631882905960083, "rewards/rejected": -1.5888137817382812, "step": 97 }, { "epoch": 0.67, "grad_norm": 51.123126271441144, "learning_rate": 1.5158744545951465e-07, "logps/chosen": -75.0676040649414, "logps/rejected": -72.10035705566406, "loss": 0.4318, "losses/dpo": 0.29737696051597595, "losses/sft": 0.3478523790836334, "losses/total": 0.29737696051597595, "ref_logps/chosen": -67.30294799804688, "ref_logps/rejected": -53.60346603393555, "rewards/accuracies": 0.8125, "rewards/chosen": -0.7764655947685242, "rewards/margins": 1.073223352432251, "rewards/rejected": -1.84968900680542, "step": 98 }, { "epoch": 0.67, "grad_norm": 26.667202752541407, "learning_rate": 1.461462467495284e-07, "logps/chosen": -75.2646484375, "logps/rejected": -74.6376953125, "loss": 0.3632, "losses/dpo": 0.4617415964603424, "losses/sft": 0.3319079577922821, "losses/total": 0.4617415964603424, "ref_logps/chosen": -67.51861572265625, "ref_logps/rejected": -55.421478271484375, "rewards/accuracies": 0.890625, "rewards/chosen": -0.7746031284332275, "rewards/margins": 1.1470186710357666, "rewards/rejected": -1.9216216802597046, "step": 99 }, { "epoch": 0.68, "grad_norm": 32.0757645021365, "learning_rate": 1.4076387190766014e-07, "logps/chosen": -67.63773345947266, "logps/rejected": -70.43733215332031, "loss": 0.4539, "losses/dpo": 0.4691275358200073, "losses/sft": 0.3476483225822449, "losses/total": 0.4691275358200073, "ref_logps/chosen": -57.368900299072266, "ref_logps/rejected": -50.622802734375, "rewards/accuracies": 0.75, "rewards/chosen": -1.0268828868865967, "rewards/margins": 0.9545705914497375, "rewards/rejected": -1.9814532995224, "step": 100 }, { "epoch": 0.69, "grad_norm": 26.664220237307592, "learning_rate": 1.354433695681474e-07, "logps/chosen": -80.13877868652344, "logps/rejected": -81.61981201171875, "loss": 0.4337, "losses/dpo": 0.4848173260688782, "losses/sft": 0.36751094460487366, "losses/total": 0.4848173260688782, "ref_logps/chosen": -72.32598876953125, "ref_logps/rejected": -63.25422668457031, "rewards/accuracies": 0.765625, "rewards/chosen": -0.7812784910202026, "rewards/margins": 1.0552804470062256, "rewards/rejected": -1.8365588188171387, "step": 101 }, { "epoch": 0.69, "grad_norm": 121.65683713646945, "learning_rate": 1.301877533199859e-07, "logps/chosen": -72.0447006225586, "logps/rejected": -72.38668823242188, "loss": 0.4732, "losses/dpo": 0.3546653389930725, "losses/sft": 0.34984883666038513, "losses/total": 0.3546653389930725, "ref_logps/chosen": -63.54951095581055, "ref_logps/rejected": -53.592952728271484, "rewards/accuracies": 0.734375, "rewards/chosen": -0.8495190739631653, "rewards/margins": 1.0298542976379395, "rewards/rejected": -1.87937331199646, "step": 102 }, { "epoch": 0.7, "grad_norm": 32.59677272775599, "learning_rate": 1.2500000000000005e-07, "logps/chosen": -66.81201934814453, "logps/rejected": -67.73169708251953, "loss": 0.4377, "losses/dpo": 0.4282957911491394, "losses/sft": 0.3223702311515808, "losses/total": 0.4282957911491394, "ref_logps/chosen": -59.40269088745117, "ref_logps/rejected": -50.77289581298828, "rewards/accuracies": 0.84375, "rewards/chosen": -0.7409323453903198, "rewards/margins": 0.9549475908279419, "rewards/rejected": -1.6958800554275513, "step": 103 }, { "epoch": 0.71, "grad_norm": 24.94130531155071, "learning_rate": 1.1988304800673196e-07, "logps/chosen": -69.9290542602539, "logps/rejected": -70.86064147949219, "loss": 0.4243, "losses/dpo": 0.4698595404624939, "losses/sft": 0.37894606590270996, "losses/total": 0.4698595404624939, "ref_logps/chosen": -62.15961456298828, "ref_logps/rejected": -51.80973815917969, "rewards/accuracies": 0.78125, "rewards/chosen": -0.7769437432289124, "rewards/margins": 1.1281468868255615, "rewards/rejected": -1.9050908088684082, "step": 104 }, { "epoch": 0.71, "grad_norm": 29.429373971218872, "learning_rate": 1.1483979563610069e-07, "logps/chosen": -78.253173828125, "logps/rejected": -75.22711181640625, "loss": 0.4335, "losses/dpo": 0.2583315968513489, "losses/sft": 0.35668855905532837, "losses/total": 0.2583315968513489, "ref_logps/chosen": -68.73954010009766, "ref_logps/rejected": -54.715335845947266, "rewards/accuracies": 0.765625, "rewards/chosen": -0.9513629674911499, "rewards/margins": 1.0998142957687378, "rewards/rejected": -2.0511772632598877, "step": 105 }, { "epoch": 0.72, "grad_norm": 27.54291684365646, "learning_rate": 1.0987309943977646e-07, "logps/chosen": -79.83787536621094, "logps/rejected": -78.37857055664062, "loss": 0.3904, "losses/dpo": 0.4721170663833618, "losses/sft": 0.3833865225315094, "losses/total": 0.4721170663833618, "ref_logps/chosen": -71.21440887451172, "ref_logps/rejected": -57.57903289794922, "rewards/accuracies": 0.8125, "rewards/chosen": -0.8623473048210144, "rewards/margins": 1.2176061868667603, "rewards/rejected": -2.07995343208313, "step": 106 }, { "epoch": 0.73, "grad_norm": 28.492473761434397, "learning_rate": 1.0498577260720048e-07, "logps/chosen": -78.24122619628906, "logps/rejected": -77.19696044921875, "loss": 0.4256, "losses/dpo": 0.5148700475692749, "losses/sft": 0.3965683579444885, "losses/total": 0.5148700475692749, "ref_logps/chosen": -72.94232177734375, "ref_logps/rejected": -62.735538482666016, "rewards/accuracies": 0.8125, "rewards/chosen": -0.5298899412155151, "rewards/margins": 0.9162522554397583, "rewards/rejected": -1.4461421966552734, "step": 107 }, { "epoch": 0.73, "grad_norm": 20.982493953158375, "learning_rate": 1.0018058337216326e-07, "logps/chosen": -76.80966186523438, "logps/rejected": -72.25653839111328, "loss": 0.3378, "losses/dpo": 0.3177239000797272, "losses/sft": 0.333109587430954, "losses/total": 0.3177239000797272, "ref_logps/chosen": -69.32987976074219, "ref_logps/rejected": -51.46967697143555, "rewards/accuracies": 0.890625, "rewards/chosen": -0.7479775547981262, "rewards/margins": 1.3307081460952759, "rewards/rejected": -2.078685760498047, "step": 108 }, { "epoch": 0.74, "grad_norm": 64.19848206989985, "learning_rate": 9.546025344484868e-08, "logps/chosen": -80.88861083984375, "logps/rejected": -78.88575744628906, "loss": 0.4023, "losses/dpo": 0.39368999004364014, "losses/sft": 0.34988299012184143, "losses/total": 0.39368999004364014, "ref_logps/chosen": -73.32632446289062, "ref_logps/rejected": -61.007301330566406, "rewards/accuracies": 0.890625, "rewards/chosen": -0.7562285661697388, "rewards/margins": 1.0316171646118164, "rewards/rejected": -1.7878458499908447, "step": 109 }, { "epoch": 0.75, "grad_norm": 23.38411036651133, "learning_rate": 9.082745647022797e-08, "logps/chosen": -81.5500717163086, "logps/rejected": -78.32415771484375, "loss": 0.3158, "losses/dpo": 0.32109877467155457, "losses/sft": 0.4066527783870697, "losses/total": 0.32109877467155457, "ref_logps/chosen": -74.28262329101562, "ref_logps/rejected": -55.555335998535156, "rewards/accuracies": 0.875, "rewards/chosen": -0.7267451882362366, "rewards/margins": 1.5501360893249512, "rewards/rejected": -2.276881217956543, "step": 110 }, { "epoch": 0.75, "grad_norm": 40.260701911712964, "learning_rate": 8.628481651367875e-08, "logps/chosen": -74.5044174194336, "logps/rejected": -76.19696044921875, "loss": 0.3743, "losses/dpo": 0.31068116426467896, "losses/sft": 0.34140655398368835, "losses/total": 0.31068116426467896, "ref_logps/chosen": -64.63957214355469, "ref_logps/rejected": -53.67190170288086, "rewards/accuracies": 0.84375, "rewards/chosen": -0.986484169960022, "rewards/margins": 1.266021728515625, "rewards/rejected": -2.2525057792663574, "step": 111 }, { "epoch": 0.76, "grad_norm": 31.80013523303015, "learning_rate": 8.183490657468686e-08, "logps/chosen": -71.22296142578125, "logps/rejected": -71.00870513916016, "loss": 0.5294, "losses/dpo": 0.4347510039806366, "losses/sft": 0.36885449290275574, "losses/total": 0.4347510039806366, "ref_logps/chosen": -60.00767517089844, "ref_logps/rejected": -51.94877624511719, "rewards/accuracies": 0.71875, "rewards/chosen": -1.1215291023254395, "rewards/margins": 0.7844638228416443, "rewards/rejected": -1.905992865562439, "step": 112 }, { "epoch": 0.77, "grad_norm": 26.736632589832187, "learning_rate": 7.748024712947204e-08, "logps/chosen": -79.0911636352539, "logps/rejected": -78.3819580078125, "loss": 0.3736, "losses/dpo": 0.38549065589904785, "losses/sft": 0.36682289838790894, "losses/total": 0.38549065589904785, "ref_logps/chosen": -71.06074523925781, "ref_logps/rejected": -57.33882141113281, "rewards/accuracies": 0.765625, "rewards/chosen": -0.8030425906181335, "rewards/margins": 1.301270842552185, "rewards/rejected": -2.104313373565674, "step": 113 }, { "epoch": 0.77, "grad_norm": 27.914463891065783, "learning_rate": 7.322330470336313e-08, "logps/chosen": -79.21043395996094, "logps/rejected": -78.50171661376953, "loss": 0.4552, "losses/dpo": 0.4180038869380951, "losses/sft": 0.3502272665500641, "losses/total": 0.4180038869380951, "ref_logps/chosen": -70.78549194335938, "ref_logps/rejected": -59.83561706542969, "rewards/accuracies": 0.78125, "rewards/chosen": -0.8424935340881348, "rewards/margins": 1.024115800857544, "rewards/rejected": -1.8666093349456787, "step": 114 }, { "epoch": 0.78, "grad_norm": 36.14683391187098, "learning_rate": 6.906649047373245e-08, "logps/chosen": -66.70146179199219, "logps/rejected": -67.81217193603516, "loss": 0.3941, "losses/dpo": 0.3595428466796875, "losses/sft": 0.4029202461242676, "losses/total": 0.3595428466796875, "ref_logps/chosen": -57.97339630126953, "ref_logps/rejected": -47.349143981933594, "rewards/accuracies": 0.765625, "rewards/chosen": -0.8728064298629761, "rewards/margins": 1.173496961593628, "rewards/rejected": -2.0463035106658936, "step": 115 }, { "epoch": 0.79, "grad_norm": 34.93282588073872, "learning_rate": 6.501215890427908e-08, "logps/chosen": -74.01496887207031, "logps/rejected": -71.6580810546875, "loss": 0.4737, "losses/dpo": 0.47708314657211304, "losses/sft": 0.3861905336380005, "losses/total": 0.47708314657211304, "ref_logps/chosen": -66.82437896728516, "ref_logps/rejected": -56.22287368774414, "rewards/accuracies": 0.765625, "rewards/chosen": -0.7190604209899902, "rewards/margins": 0.8244602680206299, "rewards/rejected": -1.5435206890106201, "step": 116 }, { "epoch": 0.8, "grad_norm": 33.61435457152851, "learning_rate": 6.106260641143546e-08, "logps/chosen": -79.60284423828125, "logps/rejected": -77.92535400390625, "loss": 0.4682, "losses/dpo": 0.290701687335968, "losses/sft": 0.3620028495788574, "losses/total": 0.290701687335968, "ref_logps/chosen": -70.14583587646484, "ref_logps/rejected": -59.40486526489258, "rewards/accuracies": 0.78125, "rewards/chosen": -0.9457005262374878, "rewards/margins": 0.9063488245010376, "rewards/rejected": -1.8520495891571045, "step": 117 }, { "epoch": 0.8, "grad_norm": 32.27039772238261, "learning_rate": 5.722007006365309e-08, "logps/chosen": -85.39405059814453, "logps/rejected": -77.39488220214844, "loss": 0.4029, "losses/dpo": 0.29392948746681213, "losses/sft": 0.4053553342819214, "losses/total": 0.29392948746681213, "ref_logps/chosen": -75.33457946777344, "ref_logps/rejected": -56.223270416259766, "rewards/accuracies": 0.84375, "rewards/chosen": -1.0059471130371094, "rewards/margins": 1.111214280128479, "rewards/rejected": -2.117161273956299, "step": 118 }, { "epoch": 0.81, "grad_norm": 30.691331571937976, "learning_rate": 5.3486726314303175e-08, "logps/chosen": -76.40483856201172, "logps/rejected": -76.4593734741211, "loss": 0.4526, "losses/dpo": 0.48409292101860046, "losses/sft": 0.3616543412208557, "losses/total": 0.48409292101860046, "ref_logps/chosen": -66.07306671142578, "ref_logps/rejected": -56.153717041015625, "rewards/accuracies": 0.828125, "rewards/chosen": -1.0331764221191406, "rewards/margins": 0.9973889589309692, "rewards/rejected": -2.0305655002593994, "step": 119 }, { "epoch": 0.82, "grad_norm": 29.441590057939685, "learning_rate": 4.986468976890992e-08, "logps/chosen": -75.17059326171875, "logps/rejected": -73.5225830078125, "loss": 0.4307, "losses/dpo": 0.33158645033836365, "losses/sft": 0.3496292531490326, "losses/total": 0.33158645033836365, "ref_logps/chosen": -68.26935577392578, "ref_logps/rejected": -55.77709197998047, "rewards/accuracies": 0.8125, "rewards/chosen": -0.69012451171875, "rewards/margins": 1.0844244956970215, "rewards/rejected": -1.774549126625061, "step": 120 }, { "epoch": 0.82, "grad_norm": 24.37008051594277, "learning_rate": 4.635601198741607e-08, "logps/chosen": -73.0021743774414, "logps/rejected": -74.11600494384766, "loss": 0.3949, "losses/dpo": 0.5414682030677795, "losses/sft": 0.37046751379966736, "losses/total": 0.5414682030677795, "ref_logps/chosen": -64.94493103027344, "ref_logps/rejected": -54.75990676879883, "rewards/accuracies": 0.8125, "rewards/chosen": -0.8057246804237366, "rewards/margins": 1.1298854351043701, "rewards/rejected": -1.935610294342041, "step": 121 }, { "epoch": 0.83, "grad_norm": 26.892127512970333, "learning_rate": 4.296268032215733e-08, "logps/chosen": -75.42649841308594, "logps/rejected": -76.27777099609375, "loss": 0.4499, "losses/dpo": 0.5582671761512756, "losses/sft": 0.3608691692352295, "losses/total": 0.5582671761512756, "ref_logps/chosen": -66.92321014404297, "ref_logps/rejected": -57.275753021240234, "rewards/accuracies": 0.75, "rewards/chosen": -0.8503296971321106, "rewards/margins": 1.0498719215393066, "rewards/rejected": -1.9002017974853516, "step": 122 }, { "epoch": 0.84, "grad_norm": 41.688513331786744, "learning_rate": 3.968661679220467e-08, "logps/chosen": -72.4287338256836, "logps/rejected": -75.88028717041016, "loss": 0.4095, "losses/dpo": 0.6144567131996155, "losses/sft": 0.3270949125289917, "losses/total": 0.6144567131996155, "ref_logps/chosen": -63.542545318603516, "ref_logps/rejected": -54.83192443847656, "rewards/accuracies": 0.84375, "rewards/chosen": -0.8886189460754395, "rewards/margins": 1.2162187099456787, "rewards/rejected": -2.104837656021118, "step": 123 }, { "epoch": 0.84, "grad_norm": 25.469129701195566, "learning_rate": 3.652967699471299e-08, "logps/chosen": -74.50592803955078, "logps/rejected": -72.23234558105469, "loss": 0.3866, "losses/dpo": 0.28718867897987366, "losses/sft": 0.37371402978897095, "losses/total": 0.28718867897987366, "ref_logps/chosen": -65.68087768554688, "ref_logps/rejected": -51.324337005615234, "rewards/accuracies": 0.828125, "rewards/chosen": -0.8825055360794067, "rewards/margins": 1.2082949876785278, "rewards/rejected": -2.0908007621765137, "step": 124 }, { "epoch": 0.85, "grad_norm": 35.98399239694971, "learning_rate": 3.349364905389032e-08, "logps/chosen": -79.32814025878906, "logps/rejected": -75.2787094116211, "loss": 0.3999, "losses/dpo": 0.5298404693603516, "losses/sft": 0.36322808265686035, "losses/total": 0.5298404693603516, "ref_logps/chosen": -72.24817657470703, "ref_logps/rejected": -56.569156646728516, "rewards/accuracies": 0.859375, "rewards/chosen": -0.7079958915710449, "rewards/margins": 1.1629595756530762, "rewards/rejected": -1.8709555864334106, "step": 125 }, { "epoch": 0.86, "grad_norm": 29.48115104276339, "learning_rate": 3.058025260818609e-08, "logps/chosen": -79.290283203125, "logps/rejected": -73.2989730834961, "loss": 0.4999, "losses/dpo": 0.6051620841026306, "losses/sft": 0.35250741243362427, "losses/total": 0.6051620841026306, "ref_logps/chosen": -70.70803833007812, "ref_logps/rejected": -55.791748046875, "rewards/accuracies": 0.75, "rewards/chosen": -0.8582238554954529, "rewards/margins": 0.8924983143806458, "rewards/rejected": -1.7507224082946777, "step": 126 }, { "epoch": 0.86, "grad_norm": 24.758070012254365, "learning_rate": 2.7791137836269158e-08, "logps/chosen": -78.37055969238281, "logps/rejected": -74.77693939208984, "loss": 0.3954, "losses/dpo": 0.3199983537197113, "losses/sft": 0.39879244565963745, "losses/total": 0.3199983537197113, "ref_logps/chosen": -69.10183715820312, "ref_logps/rejected": -54.88367462158203, "rewards/accuracies": 0.78125, "rewards/chosen": -0.9268715381622314, "rewards/margins": 1.062455177307129, "rewards/rejected": -1.98932683467865, "step": 127 }, { "epoch": 0.87, "grad_norm": 20.827088770452676, "learning_rate": 2.512788452234921e-08, "logps/chosen": -70.80789184570312, "logps/rejected": -75.08622741699219, "loss": 0.3325, "losses/dpo": 0.3453621566295624, "losses/sft": 0.3389894664287567, "losses/total": 0.3453621566295624, "ref_logps/chosen": -61.59326934814453, "ref_logps/rejected": -52.41365432739258, "rewards/accuracies": 0.859375, "rewards/chosen": -0.9214622974395752, "rewards/margins": 1.345794677734375, "rewards/rejected": -2.26725697517395, "step": 128 }, { "epoch": 0.88, "grad_norm": 31.45091455444117, "learning_rate": 2.259200116137039e-08, "logps/chosen": -76.39757537841797, "logps/rejected": -77.06990051269531, "loss": 0.4837, "losses/dpo": 0.6226012706756592, "losses/sft": 0.37185585498809814, "losses/total": 0.6226012706756592, "ref_logps/chosen": -66.43987274169922, "ref_logps/rejected": -58.27755355834961, "rewards/accuracies": 0.75, "rewards/chosen": -0.9957698583602905, "rewards/margins": 0.8834652900695801, "rewards/rejected": -1.8792351484298706, "step": 129 }, { "epoch": 0.88, "grad_norm": 25.29491524946383, "learning_rate": 2.0184924104583612e-08, "logps/chosen": -69.09542083740234, "logps/rejected": -73.32018280029297, "loss": 0.4023, "losses/dpo": 0.36410456895828247, "losses/sft": 0.33447620272636414, "losses/total": 0.36410456895828247, "ref_logps/chosen": -59.17249298095703, "ref_logps/rejected": -51.49285125732422, "rewards/accuracies": 0.828125, "rewards/chosen": -0.992293655872345, "rewards/margins": 1.190440058708191, "rewards/rejected": -2.1827335357666016, "step": 130 }, { "epoch": 0.89, "grad_norm": 27.582904411320246, "learning_rate": 1.7908016745981856e-08, "logps/chosen": -86.50100708007812, "logps/rejected": -82.30694580078125, "loss": 0.4445, "losses/dpo": 0.4145382344722748, "losses/sft": 0.346317321062088, "losses/total": 0.4145382344722748, "ref_logps/chosen": -77.55247497558594, "ref_logps/rejected": -61.19115447998047, "rewards/accuracies": 0.78125, "rewards/chosen": -0.8948546648025513, "rewards/margins": 1.2167249917984009, "rewards/rejected": -2.111579418182373, "step": 131 }, { "epoch": 0.9, "grad_norm": 28.77405060036572, "learning_rate": 1.57625687500596e-08, "logps/chosen": -75.88212585449219, "logps/rejected": -74.88499450683594, "loss": 0.5229, "losses/dpo": 0.4454580843448639, "losses/sft": 0.36321359872817993, "losses/total": 0.4454580843448639, "ref_logps/chosen": -67.06256866455078, "ref_logps/rejected": -57.12347412109375, "rewards/accuracies": 0.71875, "rewards/chosen": -0.8819558024406433, "rewards/margins": 0.8941954970359802, "rewards/rejected": -1.776151418685913, "step": 132 }, { "epoch": 0.9, "grad_norm": 27.7470456349191, "learning_rate": 1.3749795321332885e-08, "logps/chosen": -73.93744659423828, "logps/rejected": -76.04200744628906, "loss": 0.3594, "losses/dpo": 0.17011409997940063, "losses/sft": 0.40163373947143555, "losses/total": 0.17011409997940063, "ref_logps/chosen": -64.7332534790039, "ref_logps/rejected": -53.972320556640625, "rewards/accuracies": 0.9375, "rewards/chosen": -0.9204192757606506, "rewards/margins": 1.2865492105484009, "rewards/rejected": -2.206968307495117, "step": 133 }, { "epoch": 0.91, "grad_norm": 36.37444245491115, "learning_rate": 1.1870836516034876e-08, "logps/chosen": -82.49986267089844, "logps/rejected": -78.48277282714844, "loss": 0.481, "losses/dpo": 0.45800262689590454, "losses/sft": 0.4599795341491699, "losses/total": 0.45800262689590454, "ref_logps/chosen": -73.60972595214844, "ref_logps/rejected": -60.565696716308594, "rewards/accuracies": 0.828125, "rewards/chosen": -0.8890135288238525, "rewards/margins": 0.9026941657066345, "rewards/rejected": -1.7917077541351318, "step": 134 }, { "epoch": 0.92, "grad_norm": 85.09720359081324, "learning_rate": 1.0126756596375685e-08, "logps/chosen": -69.55728149414062, "logps/rejected": -67.4758071899414, "loss": 0.4461, "losses/dpo": 0.3670344650745392, "losses/sft": 0.37169790267944336, "losses/total": 0.3670344650745392, "ref_logps/chosen": -60.00959396362305, "ref_logps/rejected": -48.21699905395508, "rewards/accuracies": 0.796875, "rewards/chosen": -0.9547686576843262, "rewards/margins": 0.9711121916770935, "rewards/rejected": -1.9258809089660645, "step": 135 }, { "epoch": 0.92, "grad_norm": 25.805500137707952, "learning_rate": 8.518543427732949e-09, "logps/chosen": -77.5588607788086, "logps/rejected": -72.98957824707031, "loss": 0.3857, "losses/dpo": 0.34388694167137146, "losses/sft": 0.39463409781455994, "losses/total": 0.34388694167137146, "ref_logps/chosen": -71.34376525878906, "ref_logps/rejected": -55.052791595458984, "rewards/accuracies": 0.84375, "rewards/chosen": -0.6215101480484009, "rewards/margins": 1.1721692085266113, "rewards/rejected": -1.7936793565750122, "step": 136 }, { "epoch": 0.93, "grad_norm": 28.925653233926095, "learning_rate": 7.047107919114586e-09, "logps/chosen": -76.60894775390625, "logps/rejected": -67.70733642578125, "loss": 0.5413, "losses/dpo": 0.40706807374954224, "losses/sft": 0.38619473576545715, "losses/total": 0.40706807374954224, "ref_logps/chosen": -65.6761245727539, "ref_logps/rejected": -48.80441665649414, "rewards/accuracies": 0.75, "rewards/chosen": -1.0932811498641968, "rewards/margins": 0.7970114350318909, "rewards/rejected": -1.8902926445007324, "step": 137 }, { "epoch": 0.94, "grad_norm": 28.194863492832845, "learning_rate": 5.7132835072101486e-09, "logps/chosen": -73.64652252197266, "logps/rejected": -75.85942077636719, "loss": 0.3847, "losses/dpo": 0.4270258843898773, "losses/sft": 0.4282100200653076, "losses/total": 0.4270258843898773, "ref_logps/chosen": -65.63545227050781, "ref_logps/rejected": -55.20812225341797, "rewards/accuracies": 0.8125, "rewards/chosen": -0.8011073470115662, "rewards/margins": 1.2640223503112793, "rewards/rejected": -2.0651297569274902, "step": 138 }, { "epoch": 0.94, "grad_norm": 24.471573256004906, "learning_rate": 4.517825684323323e-09, "logps/chosen": -76.01177215576172, "logps/rejected": -72.83094787597656, "loss": 0.3493, "losses/dpo": 0.42532169818878174, "losses/sft": 0.38909339904785156, "losses/total": 0.42532169818878174, "ref_logps/chosen": -69.16812896728516, "ref_logps/rejected": -52.844810485839844, "rewards/accuracies": 0.90625, "rewards/chosen": -0.6843640208244324, "rewards/margins": 1.3142496347427368, "rewards/rejected": -1.9986138343811035, "step": 139 }, { "epoch": 0.95, "grad_norm": 28.104692013897658, "learning_rate": 3.4614115704533766e-09, "logps/chosen": -80.1229019165039, "logps/rejected": -77.62194061279297, "loss": 0.3651, "losses/dpo": 0.3558647334575653, "losses/sft": 0.3259223401546478, "losses/total": 0.3558647334575653, "ref_logps/chosen": -71.98576354980469, "ref_logps/rejected": -57.123634338378906, "rewards/accuracies": 0.84375, "rewards/chosen": -0.8137134313583374, "rewards/margins": 1.2361174821853638, "rewards/rejected": -2.049830913543701, "step": 140 }, { "epoch": 0.96, "grad_norm": 30.06950212059891, "learning_rate": 2.5446395297668287e-09, "logps/chosen": -77.057373046875, "logps/rejected": -77.57244873046875, "loss": 0.3862, "losses/dpo": 0.46156057715415955, "losses/sft": 0.3272428512573242, "losses/total": 0.46156057715415955, "ref_logps/chosen": -69.33092498779297, "ref_logps/rejected": -57.40209197998047, "rewards/accuracies": 0.78125, "rewards/chosen": -0.772644579410553, "rewards/margins": 1.2443914413452148, "rewards/rejected": -2.017035961151123, "step": 141 }, { "epoch": 0.97, "grad_norm": 22.27867702599573, "learning_rate": 1.7680288316779256e-09, "logps/chosen": -76.52747344970703, "logps/rejected": -76.79586029052734, "loss": 0.3613, "losses/dpo": 0.38265928626060486, "losses/sft": 0.330098032951355, "losses/total": 0.38265928626060486, "ref_logps/chosen": -67.75516510009766, "ref_logps/rejected": -55.386016845703125, "rewards/accuracies": 0.859375, "rewards/chosen": -0.8772311806678772, "rewards/margins": 1.263753056526184, "rewards/rejected": -2.140984058380127, "step": 142 }, { "epoch": 0.97, "grad_norm": 33.24181116137987, "learning_rate": 1.1320193567288527e-09, "logps/chosen": -68.91748809814453, "logps/rejected": -68.46461486816406, "loss": 0.4453, "losses/dpo": 0.6369032859802246, "losses/sft": 0.38167503476142883, "losses/total": 0.6369032859802246, "ref_logps/chosen": -59.43241500854492, "ref_logps/rejected": -47.996742248535156, "rewards/accuracies": 0.78125, "rewards/chosen": -0.9485068321228027, "rewards/margins": 1.0982803106307983, "rewards/rejected": -2.0467872619628906, "step": 143 }, { "epoch": 0.98, "grad_norm": 27.26573554059411, "learning_rate": 6.369713474366212e-10, "logps/chosen": -79.28267669677734, "logps/rejected": -79.29057312011719, "loss": 0.4268, "losses/dpo": 0.3871089816093445, "losses/sft": 0.3575269877910614, "losses/total": 0.3871089816093445, "ref_logps/chosen": -70.6344985961914, "ref_logps/rejected": -60.456382751464844, "rewards/accuracies": 0.8125, "rewards/chosen": -0.864818811416626, "rewards/margins": 1.018599510192871, "rewards/rejected": -1.883418321609497, "step": 144 }, { "epoch": 0.99, "grad_norm": 28.26392966654158, "learning_rate": 2.831652042480093e-10, "logps/chosen": -71.9800796508789, "logps/rejected": -73.54934692382812, "loss": 0.4465, "losses/dpo": 0.4069312512874603, "losses/sft": 0.35074400901794434, "losses/total": 0.4069312512874603, "ref_logps/chosen": -62.18163299560547, "ref_logps/rejected": -53.78350067138672, "rewards/accuracies": 0.765625, "rewards/chosen": -0.9798437356948853, "rewards/margins": 0.996740460395813, "rewards/rejected": -1.9765844345092773, "step": 145 }, { "epoch": 0.99, "grad_norm": 23.91111528235258, "learning_rate": 7.080132671774541e-11, "logps/chosen": -75.3148422241211, "logps/rejected": -74.4091567993164, "loss": 0.3508, "losses/dpo": 0.42828118801116943, "losses/sft": 0.41488099098205566, "losses/total": 0.42828118801116943, "ref_logps/chosen": -68.02532196044922, "ref_logps/rejected": -53.932071685791016, "rewards/accuracies": 0.921875, "rewards/chosen": -0.7289534211158752, "rewards/margins": 1.3187549114227295, "rewards/rejected": -2.047708511352539, "step": 146 }, { "epoch": 1.0, "grad_norm": 25.683687754039113, "learning_rate": 0.0, "logps/chosen": -77.05089569091797, "logps/rejected": -74.92003631591797, "loss": 0.4269, "losses/dpo": 0.3264519274234772, "losses/sft": 0.35349562764167786, "losses/total": 0.3264519274234772, "ref_logps/chosen": -69.42140197753906, "ref_logps/rejected": -56.2528190612793, "rewards/accuracies": 0.75, "rewards/chosen": -0.7629485130310059, "rewards/margins": 1.103774070739746, "rewards/rejected": -1.8667224645614624, "step": 147 }, { "epoch": 1.0, "step": 147, "total_flos": 0.0, "train_loss": 0.4901739539337807, "train_runtime": 4483.0286, "train_samples_per_second": 2.102, "train_steps_per_second": 0.033 } ], "logging_steps": 1.0, "max_steps": 147, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 10000, "total_flos": 0.0, "train_batch_size": 1, "trial_name": null, "trial_params": null }