Model save

README.md

@@ -2,13 +2,9 @@
 license: apache-2.0
 base_model: alignment-handbook/zephyr-7b-sft-full
 tags:
-- alignment-handbook
-- generated_from_trainer
 - trl
 - dpo
 - generated_from_trainer
-datasets:
-- HuggingFaceH4/ultrafeedback_binarized
 model-index:
 - name: zephyr-7b-dpo-full
   results: []
@@ -19,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-dpo-full
 
-This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the HuggingFaceH4/ultrafeedback_binarized dataset.
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.5028
-- Rewards/chosen: -0.9469
-- Rewards/rejected: -1.8932
-- Rewards/accuracies: 0.7656
-- Rewards/margins: 0.9463
-- Logps/rejected: -451.4661
-- Logps/chosen: -357.2325
-- Logits/rejected: 1.5731
-- Logits/chosen: 0.6530
+- Loss: 0.5042
+- Rewards/chosen: -1.0500
+- Rewards/rejected: -2.0480
+- Rewards/accuracies: 0.7539
+- Rewards/margins: 0.9980
+- Logps/rejected: -468.1450
+- Logps/chosen: -368.4135
+- Logits/rejected: 2.3821
+- Logits/chosen: 1.6141
 
 ## Model description
 
@@ -66,10 +62,10 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.5545        | 0.21  | 100  | 0.5658          | -0.4953        | -1.1217          | 0.7188             | 0.6264          | -374.3159      | -312.0799    | -1.0287         | -1.3212       |
-| 0.5026        | 0.42  | 200  | 0.5202          | -0.8995        | -1.7718          | 0.7461             | 0.8723          | -439.3264      | -352.4985    | 0.5190          | -0.1773       |
-| 0.5106        | 0.63  | 300  | 0.5104          | -0.7946        | -1.6285          | 0.7656             | 0.8339          | -424.9976      | -342.0043    | 0.9099          | 0.0862        |
-| 0.4859        | 0.84  | 400  | 0.5031          | -0.9777        | -1.9580          | 0.7578             | 0.9803          | -457.9452      | -360.3139    | 1.7438          | 0.7818        |
+| 0.5723        | 0.21  | 100  | 0.5851          | -0.4097        | -0.8752          | 0.7031             | 0.4655          | -350.8695      | -304.3812    | -2.3494         | -2.4070       |
+| 0.5084        | 0.42  | 200  | 0.5251          | -0.9116        | -1.7472          | 0.7422             | 0.8355          | -438.0663      | -354.5790    | 1.3918          | 0.9248        |
+| 0.5059        | 0.63  | 300  | 0.5130          | -0.8646        | -1.7542          | 0.75               | 0.8896          | -438.7735      | -349.8758    | 2.0331          | 1.2558        |
+| 0.4853        | 0.84  | 400  | 0.5050          | -1.0929        | -2.1085          | 0.7539             | 1.0156          | -474.1963      | -372.7067    | 2.5922          | 1.8194        |
 
 
 ### Framework versions

all_results.json

@@ -1,21 +1,21 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": 0.6529867053031921,
-    "eval_logits/rejected": 1.5730761289596558,
-    "eval_logps/chosen": -357.2324523925781,
-    "eval_logps/rejected": -451.466064453125,
-    "eval_loss": 0.5028161406517029,
-    "eval_rewards/accuracies": 0.765625,
-    "eval_rewards/chosen": -0.9468507170677185,
-    "eval_rewards/margins": 0.946345865726471,
-    "eval_rewards/rejected": -1.8931965827941895,
-    "eval_runtime": 86.4354,
+    "eval_logits/chosen": 1.6140714883804321,
+    "eval_logits/rejected": 2.3821487426757812,
+    "eval_logps/chosen": -368.41351318359375,
+    "eval_logps/rejected": -468.14495849609375,
+    "eval_loss": 0.5042223334312439,
+    "eval_rewards/accuracies": 0.75390625,
+    "eval_rewards/chosen": -1.049981713294983,
+    "eval_rewards/margins": 0.9979785680770874,
+    "eval_rewards/rejected": -2.0479602813720703,
+    "eval_runtime": 91.1991,
     "eval_samples": 2000,
-    "eval_samples_per_second": 23.139,
-    "eval_steps_per_second": 0.37,
-    "train_loss": 0.5366686437918052,
-    "train_runtime": 5196.7487,
+    "eval_samples_per_second": 21.93,
+    "eval_steps_per_second": 0.351,
+    "train_loss": 0.5379065808890754,
+    "train_runtime": 5396.8094,
     "train_samples": 61135,
-    "train_samples_per_second": 11.764,
-    "train_steps_per_second": 0.092
+    "train_samples_per_second": 11.328,
+    "train_steps_per_second": 0.089
 }

eval_results.json

@@ -1,16 +1,16 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": 0.6529867053031921,
-    "eval_logits/rejected": 1.5730761289596558,
-    "eval_logps/chosen": -357.2324523925781,
-    "eval_logps/rejected": -451.466064453125,
-    "eval_loss": 0.5028161406517029,
-    "eval_rewards/accuracies": 0.765625,
-    "eval_rewards/chosen": -0.9468507170677185,
-    "eval_rewards/margins": 0.946345865726471,
-    "eval_rewards/rejected": -1.8931965827941895,
-    "eval_runtime": 86.4354,
+    "eval_logits/chosen": 1.6140714883804321,
+    "eval_logits/rejected": 2.3821487426757812,
+    "eval_logps/chosen": -368.41351318359375,
+    "eval_logps/rejected": -468.14495849609375,
+    "eval_loss": 0.5042223334312439,
+    "eval_rewards/accuracies": 0.75390625,
+    "eval_rewards/chosen": -1.049981713294983,
+    "eval_rewards/margins": 0.9979785680770874,
+    "eval_rewards/rejected": -2.0479602813720703,
+    "eval_runtime": 91.1991,
     "eval_samples": 2000,
-    "eval_samples_per_second": 23.139,
-    "eval_steps_per_second": 0.37
+    "eval_samples_per_second": 21.93,
+    "eval_steps_per_second": 0.351
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:17698e797fee795be264ff6fceb1ce5de5e2d3504ccebb6cc762eccf36863396
+oid sha256:74afc0675c24c08eae25d9a1b5d0d849aba3015f65871f579181d4f822dd5216
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d57e247aa8357fee9c5771c9956ae96eea45624a113b40fdba0fcd2b848476f7
+oid sha256:0d7a278fc93c6f6c9f8c55bfe6421851acb60fb90b311cdbd8edf9e9f44858b7
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:daa924d9453a024a65583a098420d94aa3946268cadc893d3a672cde300d7731
+oid sha256:35c86540b037e38a7b9b55a77f9551c811ebda4b3b99b4c31acdf75bffdf9917
 size 4540516344

runs/Jan10_03-25-54_ip-26-0-168-34/events.out.tfevents.1704857214.ip-26-0-168-34.1146299.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bf725a56b70c28826538ef708644d9fb6baeb7dd379efca2b4e66857bdb5aa34
-size 33329
+oid sha256:d16ef4aaa5aa22c0a866161c2f9c5b8fcc1f479c9f913439e27dfe74dd18608d
+size 38121

runs/Jan10_03-25-54_ip-26-0-168-34/events.out.tfevents.1704862702.ip-26-0-168-34.1146299.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3fbb23ad68116119e144362585f232e6defa4b4d47e223c62e5f3615b2d54ba
+size 828

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.5366686437918052,
-    "train_runtime": 5196.7487,
+    "train_loss": 0.5379065808890754,
+    "train_runtime": 5396.8094,
     "train_samples": 61135,
-    "train_samples_per_second": 11.764,
-    "train_steps_per_second": 0.092
+    "train_samples_per_second": 11.328,
+    "train_steps_per_second": 0.089
 }

trainer_state.json

@@ -11,10 +11,10 @@
     {
       "epoch": 0.0,
       "learning_rate": 1.0416666666666666e-08,
-      "logits/chosen": -2.5017967224121094,
-      "logits/rejected": -2.3871021270751953,
-      "logps/chosen": -332.3011474609375,
-      "logps/rejected": -277.1512756347656,
+      "logits/chosen": -2.6023898124694824,
+      "logits/rejected": -2.49088191986084,
+      "logps/chosen": -330.5306396484375,
+      "logps/rejected": -275.0410461425781,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -25,733 +25,733 @@
     {
       "epoch": 0.02,
       "learning_rate": 1.0416666666666667e-07,
-      "logits/chosen": -2.540015459060669,
-      "logits/rejected": -2.500929832458496,
-      "logps/chosen": -247.11024475097656,
-      "logps/rejected": -213.2850341796875,
+      "logits/chosen": -2.624011516571045,
+      "logits/rejected": -2.59273624420166,
+      "logps/chosen": -247.91769409179688,
+      "logps/rejected": -215.07041931152344,
       "loss": 0.6932,
-      "rewards/accuracies": 0.4097222089767456,
-      "rewards/chosen": -0.000517037755344063,
-      "rewards/margins": -0.00017402732919435948,
-      "rewards/rejected": -0.0003430104407016188,
+      "rewards/accuracies": 0.3541666567325592,
+      "rewards/chosen": -0.00047609664034098387,
+      "rewards/margins": -0.0011458636727184057,
+      "rewards/rejected": 0.0006697670323774219,
       "step": 10
     },
     {
       "epoch": 0.04,
       "learning_rate": 2.0833333333333333e-07,
-      "logits/chosen": -2.5355143547058105,
-      "logits/rejected": -2.484562873840332,
-      "logps/chosen": -272.35711669921875,
-      "logps/rejected": -249.6931915283203,
-      "loss": 0.6924,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": 0.0008343200897797942,
-      "rewards/margins": 0.001956597436219454,
-      "rewards/rejected": -0.001122277113609016,
+      "logits/chosen": -2.614908218383789,
+      "logits/rejected": -2.573396682739258,
+      "logps/chosen": -273.2959289550781,
+      "logps/rejected": -251.2639617919922,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": 0.0009359431569464505,
+      "rewards/margins": 0.002007069531828165,
+      "rewards/rejected": -0.0010711264330893755,
       "step": 20
     },
     {
       "epoch": 0.06,
       "learning_rate": 3.1249999999999997e-07,
-      "logits/chosen": -2.6150834560394287,
-      "logits/rejected": -2.542232036590576,
-      "logps/chosen": -283.331298828125,
-      "logps/rejected": -276.0316467285156,
-      "loss": 0.6885,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.008794735185801983,
-      "rewards/margins": 0.006988237611949444,
-      "rewards/rejected": 0.0018064973410218954,
+      "logits/chosen": -2.6856637001037598,
+      "logits/rejected": -2.6220130920410156,
+      "logps/chosen": -284.86114501953125,
+      "logps/rejected": -277.53057861328125,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.00352325732819736,
+      "rewards/margins": 0.007650823798030615,
+      "rewards/rejected": -0.0041275653056800365,
       "step": 30
     },
     {
       "epoch": 0.08,
       "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -2.519700527191162,
-      "logits/rejected": -2.4544477462768555,
-      "logps/chosen": -288.609130859375,
-      "logps/rejected": -265.47576904296875,
+      "logits/chosen": -2.579878807067871,
+      "logits/rejected": -2.5135815143585205,
+      "logps/chosen": -292.1109619140625,
+      "logps/rejected": -274.44683837890625,
       "loss": 0.6756,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": 0.04671400785446167,
-      "rewards/margins": 0.03756200894713402,
-      "rewards/rejected": 0.009151997044682503,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.02378256432712078,
+      "rewards/margins": 0.03553395718336105,
+      "rewards/rejected": -0.011751385405659676,
       "step": 40
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999733114418725e-07,
-      "logits/chosen": -2.4968419075012207,
-      "logits/rejected": -2.4539003372192383,
-      "logps/chosen": -312.46380615234375,
-      "logps/rejected": -309.41632080078125,
+      "logits/chosen": -2.5302300453186035,
+      "logits/rejected": -2.4865477085113525,
+      "logps/chosen": -315.3640441894531,
+      "logps/rejected": -310.5618591308594,
       "loss": 0.6601,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.006021331064403057,
-      "rewards/margins": 0.07272790372371674,
-      "rewards/rejected": -0.07874923944473267,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.014850592240691185,
+      "rewards/margins": 0.06933780014514923,
+      "rewards/rejected": -0.08418838679790497,
       "step": 50
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.990398100856366e-07,
-      "logits/chosen": -2.461306095123291,
-      "logits/rejected": -2.4000632762908936,
-      "logps/chosen": -267.9833679199219,
-      "logps/rejected": -251.8109130859375,
-      "loss": 0.6381,
+      "logits/chosen": -2.461594820022583,
+      "logits/rejected": -2.393406867980957,
+      "logps/chosen": -264.4418640136719,
+      "logps/rejected": -252.02163696289062,
+      "loss": 0.6391,
       "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.10297150909900665,
-      "rewards/margins": 0.12468205392360687,
-      "rewards/rejected": -0.22765357792377472,
+      "rewards/chosen": -0.06258662045001984,
+      "rewards/margins": 0.1386002004146576,
+      "rewards/rejected": -0.20118682086467743,
       "step": 60
     },
     {
       "epoch": 0.15,
       "learning_rate": 4.967775735898179e-07,
-      "logits/chosen": -2.427084445953369,
-      "logits/rejected": -2.366863250732422,
-      "logps/chosen": -298.81085205078125,
-      "logps/rejected": -287.7050476074219,
-      "loss": 0.6177,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.1814979612827301,
-      "rewards/margins": 0.2026226818561554,
-      "rewards/rejected": -0.3841206133365631,
+      "logits/chosen": -2.5176403522491455,
+      "logits/rejected": -2.444599151611328,
+      "logps/chosen": -308.10845947265625,
+      "logps/rejected": -298.1520690917969,
+      "loss": 0.6219,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.26666340231895447,
+      "rewards/margins": 0.21313416957855225,
+      "rewards/rejected": -0.4797976016998291,
       "step": 70
     },
     {
       "epoch": 0.17,
       "learning_rate": 4.931986719649298e-07,
-      "logits/chosen": -1.6477082967758179,
-      "logits/rejected": -1.5141746997833252,
-      "logps/chosen": -328.52178955078125,
-      "logps/rejected": -357.9725646972656,
-      "loss": 0.609,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.5934440493583679,
-      "rewards/margins": 0.24087591469287872,
-      "rewards/rejected": -0.8343199491500854,
+      "logits/chosen": -2.4516353607177734,
+      "logits/rejected": -2.4085216522216797,
+      "logps/chosen": -298.8356018066406,
+      "logps/rejected": -325.5304260253906,
+      "loss": 0.611,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.2911642789840698,
+      "rewards/margins": 0.20117318630218506,
+      "rewards/rejected": -0.49233752489089966,
       "step": 80
     },
     {
       "epoch": 0.19,
       "learning_rate": 4.883222001996351e-07,
-      "logits/chosen": -1.456400990486145,
-      "logits/rejected": -1.1816024780273438,
-      "logps/chosen": -324.58734130859375,
-      "logps/rejected": -347.7338562011719,
-      "loss": 0.5717,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.5771389603614807,
-      "rewards/margins": 0.38918066024780273,
-      "rewards/rejected": -0.966319739818573,
+      "logits/chosen": -2.426361560821533,
+      "logits/rejected": -2.3368563652038574,
+      "logps/chosen": -293.616943359375,
+      "logps/rejected": -308.7396545410156,
+      "loss": 0.5867,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.2579975724220276,
+      "rewards/margins": 0.30983540415763855,
+      "rewards/rejected": -0.5678330063819885,
       "step": 90
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.821741763807186e-07,
-      "logits/chosen": -1.2872960567474365,
-      "logits/rejected": -0.9513088464736938,
-      "logps/chosen": -339.1795959472656,
-      "logps/rejected": -366.32769775390625,
-      "loss": 0.5545,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.5583249926567078,
-      "rewards/margins": 0.47431764006614685,
-      "rewards/rejected": -1.0326426029205322,
+      "logits/chosen": -2.488579034805298,
+      "logits/rejected": -2.3800113201141357,
+      "logps/chosen": -328.0105285644531,
+      "logps/rejected": -337.8644104003906,
+      "loss": 0.5723,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.4366111755371094,
+      "rewards/margins": 0.3044855296611786,
+      "rewards/rejected": -0.7410967350006104,
       "step": 100
     },
     {
       "epoch": 0.21,
-      "eval_logits/chosen": -1.3212240934371948,
-      "eval_logits/rejected": -1.0286760330200195,
-      "eval_logps/chosen": -312.0798645019531,
-      "eval_logps/rejected": -374.31585693359375,
-      "eval_loss": 0.5658453106880188,
-      "eval_rewards/accuracies": 0.71875,
-      "eval_rewards/chosen": -0.49532508850097656,
-      "eval_rewards/margins": 0.6263692378997803,
-      "eval_rewards/rejected": -1.1216944456100464,
-      "eval_runtime": 85.4149,
-      "eval_samples_per_second": 23.415,
-      "eval_steps_per_second": 0.375,
+      "eval_logits/chosen": -2.4070217609405518,
+      "eval_logits/rejected": -2.3494362831115723,
+      "eval_logps/chosen": -304.3812255859375,
+      "eval_logps/rejected": -350.8694763183594,
+      "eval_loss": 0.5851432681083679,
+      "eval_rewards/accuracies": 0.703125,
+      "eval_rewards/chosen": -0.4096587896347046,
+      "eval_rewards/margins": 0.46554654836654663,
+      "eval_rewards/rejected": -0.8752052783966064,
+      "eval_runtime": 91.1907,
+      "eval_samples_per_second": 21.932,
+      "eval_steps_per_second": 0.351,
       "step": 100
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.747874028753375e-07,
-      "logits/chosen": -1.1495229005813599,
-      "logits/rejected": -0.6862327456474304,
-      "logps/chosen": -370.4473571777344,
-      "logps/rejected": -381.8430480957031,
-      "loss": 0.5534,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.503128170967102,
-      "rewards/margins": 0.6164419054985046,
-      "rewards/rejected": -1.1195701360702515,
+      "logits/chosen": -2.0290980339050293,
+      "logits/rejected": -1.8976500034332275,
+      "logps/chosen": -374.5489807128906,
+      "logps/rejected": -375.1778869628906,
+      "loss": 0.5723,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.5513430833816528,
+      "rewards/margins": 0.49042654037475586,
+      "rewards/rejected": -1.0417697429656982,
       "step": 110
     },
     {
       "epoch": 0.25,
       "learning_rate": 4.662012913161997e-07,
-      "logits/chosen": -0.21524420380592346,
-      "logits/rejected": 0.5560011863708496,
-      "logps/chosen": -363.287353515625,
-      "logps/rejected": -385.227783203125,
-      "loss": 0.5308,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.8372129201889038,
-      "rewards/margins": 0.710771918296814,
-      "rewards/rejected": -1.5479847192764282,
+      "logits/chosen": -0.8261772990226746,
+      "logits/rejected": -0.4543725550174713,
+      "logps/chosen": -370.54437255859375,
+      "logps/rejected": -376.8744201660156,
+      "loss": 0.546,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.893993079662323,
+      "rewards/margins": 0.5693421363830566,
+      "rewards/rejected": -1.4633351564407349,
       "step": 120
     },
     {
       "epoch": 0.27,
       "learning_rate": 4.5646165232345103e-07,
-      "logits/chosen": -0.9693559408187866,
-      "logits/rejected": -0.585214376449585,
-      "logps/chosen": -310.8514404296875,
-      "logps/rejected": -407.11956787109375,
-      "loss": 0.5387,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.4881950318813324,
-      "rewards/margins": 0.6856316328048706,
-      "rewards/rejected": -1.1738265752792358,
+      "logits/chosen": -0.5733903050422668,
+      "logits/rejected": -0.41144052147865295,
+      "logps/chosen": -331.88458251953125,
+      "logps/rejected": -418.39404296875,
+      "loss": 0.5492,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.6849642395973206,
+      "rewards/margins": 0.5858219265937805,
+      "rewards/rejected": -1.2707862854003906,
       "step": 130
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.456204510851956e-07,
-      "logits/chosen": -1.0192829370498657,
-      "logits/rejected": -0.2164110690355301,
-      "logps/chosen": -359.496826171875,
-      "logps/rejected": -383.21826171875,
-      "loss": 0.5381,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.6586028337478638,
-      "rewards/margins": 0.6349435448646545,
-      "rewards/rejected": -1.2935463190078735,
+      "logits/chosen": -0.7106949687004089,
+      "logits/rejected": -0.2236645519733429,
+      "logps/chosen": -367.40484619140625,
+      "logps/rejected": -390.296142578125,
+      "loss": 0.5335,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.7277344465255737,
+      "rewards/margins": 0.6220408082008362,
+      "rewards/rejected": -1.3497753143310547,
       "step": 140
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.337355301007335e-07,
-      "logits/chosen": -0.4600129723548889,
-      "logits/rejected": 0.2966030240058899,
-      "logps/chosen": -379.4418640136719,
-      "logps/rejected": -396.3006286621094,
-      "loss": 0.5288,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.9255654215812683,
-      "rewards/margins": 0.6836373805999756,
-      "rewards/rejected": -1.6092027425765991,
+      "logits/chosen": -0.2654598355293274,
+      "logits/rejected": 0.43950486183166504,
+      "logps/chosen": -385.2984924316406,
+      "logps/rejected": -397.6144714355469,
+      "loss": 0.5356,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.9714946746826172,
+      "rewards/margins": 0.61899733543396,
+      "rewards/rejected": -1.5904920101165771,
       "step": 150
     },
     {
       "epoch": 0.33,
       "learning_rate": 4.2087030056579986e-07,
-      "logits/chosen": -0.17871518433094025,
-      "logits/rejected": 0.4601938724517822,
-      "logps/chosen": -382.32659912109375,
-      "logps/rejected": -447.52716064453125,
-      "loss": 0.5063,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -0.9520798921585083,
-      "rewards/margins": 0.810064435005188,
-      "rewards/rejected": -1.7621443271636963,
+      "logits/chosen": 0.1484789103269577,
+      "logits/rejected": 0.8263363838195801,
+      "logps/chosen": -369.7867736816406,
+      "logps/rejected": -436.39373779296875,
+      "loss": 0.5065,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.8267679214477539,
+      "rewards/margins": 0.8252193331718445,
+      "rewards/rejected": -1.6519873142242432,
       "step": 160
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.070934040463998e-07,
-      "logits/chosen": -0.7312272191047668,
-      "logits/rejected": -0.21112406253814697,
-      "logps/chosen": -320.51849365234375,
-      "logps/rejected": -360.88427734375,
-      "loss": 0.5618,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.7297524213790894,
-      "rewards/margins": 0.565701425075531,
-      "rewards/rejected": -1.2954537868499756,
+      "logits/chosen": 0.2387746274471283,
+      "logits/rejected": 0.7541650533676147,
+      "logps/chosen": -330.07525634765625,
+      "logps/rejected": -366.41204833984375,
+      "loss": 0.5659,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.8212235569953918,
+      "rewards/margins": 0.529572606086731,
+      "rewards/rejected": -1.3507962226867676,
       "step": 170
     },
     {
       "epoch": 0.38,
       "learning_rate": 3.9247834624635404e-07,
-      "logits/chosen": -0.5727821588516235,
-      "logits/rejected": -0.19889096915721893,
-      "logps/chosen": -369.8377990722656,
-      "logps/rejected": -436.2527770996094,
-      "loss": 0.5238,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.7431085705757141,
-      "rewards/margins": 0.8097877502441406,
-      "rewards/rejected": -1.55289626121521,
+      "logits/chosen": 0.45646604895591736,
+      "logits/rejected": 0.8084599375724792,
+      "logps/chosen": -366.8728942871094,
+      "logps/rejected": -432.2496032714844,
+      "loss": 0.5249,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.6927820444107056,
+      "rewards/margins": 0.8015207052230835,
+      "rewards/rejected": -1.4943029880523682,
       "step": 180
     },
     {
       "epoch": 0.4,
       "learning_rate": 3.7710310482256523e-07,
-      "logits/chosen": -0.8184123039245605,
-      "logits/rejected": -0.09880775213241577,
-      "logps/chosen": -360.60174560546875,
-      "logps/rejected": -434.35540771484375,
-      "loss": 0.5039,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.7761167287826538,
-      "rewards/margins": 0.8615070581436157,
-      "rewards/rejected": -1.6376237869262695,
+      "logits/chosen": 1.0517617464065552,
+      "logits/rejected": 1.6709725856781006,
+      "logps/chosen": -378.12396240234375,
+      "logps/rejected": -458.1866149902344,
+      "loss": 0.5056,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.9326898455619812,
+      "rewards/margins": 0.9154269099235535,
+      "rewards/rejected": -1.8481168746948242,
       "step": 190
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.610497133404795e-07,
-      "logits/chosen": -0.597638726234436,
-      "logits/rejected": 0.14091506600379944,
-      "logps/chosen": -373.0635681152344,
-      "logps/rejected": -402.6120910644531,
-      "loss": 0.5026,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.908000648021698,
-      "rewards/margins": 0.6652692556381226,
-      "rewards/rejected": -1.5732697248458862,
+      "logits/chosen": 0.9935806393623352,
+      "logits/rejected": 1.650398850440979,
+      "logps/chosen": -391.5450744628906,
+      "logps/rejected": -418.3558654785156,
+      "loss": 0.5084,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.0861790180206299,
+      "rewards/margins": 0.634604275226593,
+      "rewards/rejected": -1.7207832336425781,
       "step": 200
     },
     {
       "epoch": 0.42,
-      "eval_logits/chosen": -0.17727772891521454,
-      "eval_logits/rejected": 0.5189895629882812,
-      "eval_logps/chosen": -352.49847412109375,
-      "eval_logps/rejected": -439.3264465332031,
-      "eval_loss": 0.5201631188392639,
-      "eval_rewards/accuracies": 0.74609375,
-      "eval_rewards/chosen": -0.899510383605957,
-      "eval_rewards/margins": 0.8722902536392212,
-      "eval_rewards/rejected": -1.7718006372451782,
-      "eval_runtime": 85.7368,
-      "eval_samples_per_second": 23.327,
-      "eval_steps_per_second": 0.373,
+      "eval_logits/chosen": 0.9247687458992004,
+      "eval_logits/rejected": 1.3918358087539673,
+      "eval_logps/chosen": -354.5789794921875,
+      "eval_logps/rejected": -438.0662536621094,
+      "eval_loss": 0.5251370072364807,
+      "eval_rewards/accuracies": 0.7421875,
+      "eval_rewards/chosen": -0.9116362929344177,
+      "eval_rewards/margins": 0.8355368375778198,
+      "eval_rewards/rejected": -1.7471731901168823,
+      "eval_runtime": 91.7577,
+      "eval_samples_per_second": 21.797,
+      "eval_steps_per_second": 0.349,
       "step": 200
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.4440382358952115e-07,
-      "logits/chosen": -0.04346243292093277,
-      "logits/rejected": 0.8316682577133179,
-      "logps/chosen": -377.2864990234375,
-      "logps/rejected": -403.12933349609375,
-      "loss": 0.5357,
+      "logits/chosen": 1.0475047826766968,
+      "logits/rejected": 1.849473237991333,
+      "logps/chosen": -367.184814453125,
+      "logps/rejected": -398.2117614746094,
+      "loss": 0.5251,
       "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -1.005774736404419,
-      "rewards/margins": 0.6370893716812134,
-      "rewards/rejected": -1.6428638696670532,
+      "rewards/chosen": -0.8909347653388977,
+      "rewards/margins": 0.6959229707717896,
+      "rewards/rejected": -1.586857557296753,
       "step": 210
     },
     {
       "epoch": 0.46,
       "learning_rate": 3.272542485937368e-07,
-      "logits/chosen": 0.27445563673973083,
-      "logits/rejected": 0.8739471435546875,
-      "logps/chosen": -359.21343994140625,
-      "logps/rejected": -410.8561096191406,
+      "logits/chosen": 1.6884968280792236,
+      "logits/rejected": 2.2008445262908936,
+      "logps/chosen": -353.2514343261719,
+      "logps/rejected": -404.71221923828125,
       "loss": 0.5269,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -0.8274116516113281,
-      "rewards/margins": 0.8387683033943176,
-      "rewards/rejected": -1.6661800146102905,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.7567670345306396,
+      "rewards/margins": 0.8415945768356323,
+      "rewards/rejected": -1.5983617305755615,
       "step": 220
     },
     {
       "epoch": 0.48,
       "learning_rate": 3.096924887558854e-07,
-      "logits/chosen": 0.12390404939651489,
-      "logits/rejected": 0.7874934673309326,
-      "logps/chosen": -343.6993713378906,
-      "logps/rejected": -443.02716064453125,
-      "loss": 0.5192,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -0.7115140557289124,
-      "rewards/margins": 0.9087392091751099,
-      "rewards/rejected": -1.6202532052993774,
+      "logits/chosen": 1.460933804512024,
+      "logits/rejected": 1.9314343929290771,
+      "logps/chosen": -351.2489318847656,
+      "logps/rejected": -453.9790954589844,
+      "loss": 0.519,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.7599745988845825,
+      "rewards/margins": 0.8532025218009949,
+      "rewards/rejected": -1.6131770610809326,
       "step": 230
     },
     {
       "epoch": 0.5,
       "learning_rate": 2.9181224366319943e-07,
-      "logits/chosen": 0.47393637895584106,
-      "logits/rejected": 1.2820873260498047,
-      "logps/chosen": -341.52032470703125,
-      "logps/rejected": -402.3468322753906,
-      "loss": 0.5251,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.9017995595932007,
-      "rewards/margins": 0.7208075523376465,
-      "rewards/rejected": -1.6226072311401367,
+      "logits/chosen": 1.796936273574829,
+      "logits/rejected": 2.389878988265991,
+      "logps/chosen": -351.67498779296875,
+      "logps/rejected": -421.3821716308594,
+      "loss": 0.5261,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.9927783012390137,
+      "rewards/margins": 0.786289632320404,
+      "rewards/rejected": -1.7790677547454834,
       "step": 240
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.7370891215954565e-07,
-      "logits/chosen": 0.6556827425956726,
-      "logits/rejected": 1.5371648073196411,
-      "logps/chosen": -363.3061218261719,
-      "logps/rejected": -423.22216796875,
-      "loss": 0.5168,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.9660114049911499,
-      "rewards/margins": 0.6768711805343628,
-      "rewards/rejected": -1.6428825855255127,
+      "logits/chosen": 1.5744327306747437,
+      "logits/rejected": 2.3407230377197266,
+      "logps/chosen": -358.4691467285156,
+      "logps/rejected": -418.01031494140625,
+      "loss": 0.5134,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.903947651386261,
+      "rewards/margins": 0.6940609216690063,
+      "rewards/rejected": -1.5980085134506226,
       "step": 250
     },
     {
       "epoch": 0.54,
       "learning_rate": 2.55479083351317e-07,
-      "logits/chosen": 1.074449062347412,
-      "logits/rejected": 2.2258691787719727,
-      "logps/chosen": -393.25042724609375,
-      "logps/rejected": -435.5625915527344,
-      "loss": 0.5007,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -1.1183383464813232,
-      "rewards/margins": 0.817892849445343,
-      "rewards/rejected": -1.936231255531311,
+      "logits/chosen": 1.873732566833496,
+      "logits/rejected": 2.9474740028381348,
+      "logps/chosen": -371.85552978515625,
+      "logps/rejected": -420.95904541015625,
+      "loss": 0.4922,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -0.9079627990722656,
+      "rewards/margins": 0.8738547563552856,
+      "rewards/rejected": -1.7818174362182617,
       "step": 260
     },
     {
       "epoch": 0.56,
       "learning_rate": 2.3722002126275822e-07,
-      "logits/chosen": 1.035612940788269,
-      "logits/rejected": 1.9490232467651367,
-      "logps/chosen": -391.06488037109375,
-      "logps/rejected": -470.64599609375,
-      "loss": 0.5064,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -1.1720463037490845,
-      "rewards/margins": 0.9149090051651001,
-      "rewards/rejected": -2.0869553089141846,
+      "logits/chosen": 2.415181875228882,
+      "logits/rejected": 3.162013530731201,
+      "logps/chosen": -388.0815734863281,
+      "logps/rejected": -478.11785888671875,
+      "loss": 0.498,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.128756046295166,
+      "rewards/margins": 1.0180633068084717,
+      "rewards/rejected": -2.146819591522217,
       "step": 270
     },
     {
       "epoch": 0.59,
       "learning_rate": 2.19029145890313e-07,
-      "logits/chosen": 0.5210274457931519,
-      "logits/rejected": 1.496964693069458,
-      "logps/chosen": -369.4839782714844,
-      "logps/rejected": -415.17974853515625,
-      "loss": 0.5189,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.9482043981552124,
-      "rewards/margins": 0.8109124898910522,
-      "rewards/rejected": -1.759116768836975,
+      "logits/chosen": 1.9844467639923096,
+      "logits/rejected": 2.9561781883239746,
+      "logps/chosen": -369.2903747558594,
+      "logps/rejected": -419.6259765625,
+      "loss": 0.5207,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.9253425598144531,
+      "rewards/margins": 0.8587535619735718,
+      "rewards/rejected": -1.784096121788025,
       "step": 280
     },
     {
       "epoch": 0.61,
       "learning_rate": 2.0100351342479216e-07,
-      "logits/chosen": 0.4483126103878021,
-      "logits/rejected": 1.3368585109710693,
-      "logps/chosen": -373.81475830078125,
-      "logps/rejected": -435.36151123046875,
-      "loss": 0.519,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.8730584979057312,
-      "rewards/margins": 0.7535277009010315,
-      "rewards/rejected": -1.6265861988067627,
+      "logits/chosen": 1.8705106973648071,
+      "logits/rejected": 2.6589739322662354,
+      "logps/chosen": -380.0862731933594,
+      "logps/rejected": -439.79168701171875,
+      "loss": 0.515,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.9231119155883789,
+      "rewards/margins": 0.735679030418396,
+      "rewards/rejected": -1.6587913036346436,
       "step": 290
     },
     {
       "epoch": 0.63,
       "learning_rate": 1.8323929841460178e-07,
-      "logits/chosen": 0.27887973189353943,
-      "logits/rejected": 1.0783421993255615,
-      "logps/chosen": -381.5480651855469,
-      "logps/rejected": -453.80242919921875,
-      "loss": 0.5106,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.8568315505981445,
-      "rewards/margins": 0.6526933312416077,
-      "rewards/rejected": -1.5095248222351074,
+      "logits/chosen": 1.3944432735443115,
+      "logits/rejected": 2.3618969917297363,
+      "logps/chosen": -389.6896057128906,
+      "logps/rejected": -470.2090759277344,
+      "loss": 0.5059,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.8919968605041504,
+      "rewards/margins": 0.6746976971626282,
+      "rewards/rejected": -1.5666944980621338,
       "step": 300
     },
     {
       "epoch": 0.63,
-      "eval_logits/chosen": 0.08615332096815109,
-      "eval_logits/rejected": 0.9099248647689819,
-      "eval_logps/chosen": -342.00433349609375,
-      "eval_logps/rejected": -424.99755859375,
-      "eval_loss": 0.5103623270988464,
-      "eval_rewards/accuracies": 0.765625,
-      "eval_rewards/chosen": -0.7945692539215088,
-      "eval_rewards/margins": 0.8339425921440125,
-      "eval_rewards/rejected": -1.628511905670166,
-      "eval_runtime": 86.1066,
-      "eval_samples_per_second": 23.227,
-      "eval_steps_per_second": 0.372,
+      "eval_logits/chosen": 1.2558308839797974,
+      "eval_logits/rejected": 2.033073902130127,
+      "eval_logps/chosen": -349.8758239746094,
+      "eval_logps/rejected": -438.77349853515625,
+      "eval_loss": 0.5130496621131897,
+      "eval_rewards/accuracies": 0.75,
+      "eval_rewards/chosen": -0.8646047711372375,
+      "eval_rewards/margins": 0.8896409273147583,
+      "eval_rewards/rejected": -1.7542455196380615,
+      "eval_runtime": 92.0798,
+      "eval_samples_per_second": 21.72,
+      "eval_steps_per_second": 0.348,
       "step": 300
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.6583128063291573e-07,
-      "logits/chosen": 0.13822266459465027,
-      "logits/rejected": 1.3017876148223877,
-      "logps/chosen": -365.63958740234375,
-      "logps/rejected": -384.9111328125,
-      "loss": 0.4988,
+      "logits/chosen": 1.2974698543548584,
+      "logits/rejected": 2.6388087272644043,
+      "logps/chosen": -382.4002990722656,
+      "logps/rejected": -406.01153564453125,
+      "loss": 0.4978,
       "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.9199568629264832,
-      "rewards/margins": 0.7403375506401062,
-      "rewards/rejected": -1.6602942943572998,
+      "rewards/chosen": -1.0794718265533447,
+      "rewards/margins": 0.7805131673812866,
+      "rewards/rejected": -1.8599849939346313,
       "step": 310
     },
     {
       "epoch": 0.67,
       "learning_rate": 1.488723393865766e-07,
-      "logits/chosen": 0.672313392162323,
-      "logits/rejected": 1.8797632455825806,
-      "logps/chosen": -326.7810974121094,
-      "logps/rejected": -421.84765625,
-      "loss": 0.5073,
+      "logits/chosen": 1.9306262731552124,
+      "logits/rejected": 2.9958901405334473,
+      "logps/chosen": -357.4389953613281,
+      "logps/rejected": -452.7220764160156,
+      "loss": 0.5064,
       "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.7888548374176025,
-      "rewards/margins": 1.0594618320465088,
-      "rewards/rejected": -1.8483164310455322,
+      "rewards/chosen": -1.0856704711914062,
+      "rewards/margins": 1.057279109954834,
+      "rewards/rejected": -2.1429495811462402,
       "step": 320
     },
     {
       "epoch": 0.69,
       "learning_rate": 1.3245295796480788e-07,
-      "logits/chosen": 0.46894121170043945,
-      "logits/rejected": 1.3748019933700562,
-      "logps/chosen": -373.7689208984375,
-      "logps/rejected": -419.54364013671875,
-      "loss": 0.5143,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.8813614845275879,
-      "rewards/margins": 0.7780431509017944,
-      "rewards/rejected": -1.6594045162200928,
+      "logits/chosen": 1.4244121313095093,
+      "logits/rejected": 2.2654335498809814,
+      "logps/chosen": -404.91082763671875,
+      "logps/rejected": -450.8277893066406,
+      "loss": 0.5096,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.1859899759292603,
+      "rewards/margins": 0.7777279019355774,
+      "rewards/rejected": -1.9637176990509033,
       "step": 330
     },
     {
       "epoch": 0.71,
       "learning_rate": 1.1666074087171627e-07,
-      "logits/chosen": 0.8704012036323547,
-      "logits/rejected": 1.6405874490737915,
-      "logps/chosen": -351.2308654785156,
-      "logps/rejected": -407.41064453125,
-      "loss": 0.5135,
+      "logits/chosen": 1.5507278442382812,
+      "logits/rejected": 2.3268961906433105,
+      "logps/chosen": -363.16473388671875,
+      "logps/rejected": -420.6800231933594,
+      "loss": 0.5173,
       "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.8790988922119141,
-      "rewards/margins": 0.7077994346618652,
-      "rewards/rejected": -1.5868984460830688,
+      "rewards/chosen": -1.0014616250991821,
+      "rewards/margins": 0.7089160680770874,
+      "rewards/rejected": -1.7103776931762695,
       "step": 340
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0157994641835734e-07,
-      "logits/chosen": 0.8922025561332703,
-      "logits/rejected": 2.06630277633667,
-      "logps/chosen": -348.2925720214844,
-      "logps/rejected": -419.91363525390625,
-      "loss": 0.4771,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -0.9759310483932495,
-      "rewards/margins": 0.898769736289978,
-      "rewards/rejected": -1.8747007846832275,
+      "logits/chosen": 1.5202906131744385,
+      "logits/rejected": 2.6713767051696777,
+      "logps/chosen": -359.4294128417969,
+      "logps/rejected": -433.394287109375,
+      "loss": 0.4787,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.0805784463882446,
+      "rewards/margins": 0.9193571209907532,
+      "rewards/rejected": -1.999935507774353,
       "step": 350
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.729103716819111e-08,
-      "logits/chosen": 0.996601402759552,
-      "logits/rejected": 2.303062915802002,
-      "logps/chosen": -422.4617614746094,
-      "logps/rejected": -457.77691650390625,
-      "loss": 0.4833,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -1.044933557510376,
-      "rewards/margins": 0.9208766222000122,
-      "rewards/rejected": -1.9658104181289673,
+      "logits/chosen": 1.5974103212356567,
+      "logits/rejected": 3.016284942626953,
+      "logps/chosen": -435.1712951660156,
+      "logps/rejected": -469.9830017089844,
+      "loss": 0.4902,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.1632494926452637,
+      "rewards/margins": 0.9136824607849121,
+      "rewards/rejected": -2.0769317150115967,
       "step": 360
     },
     {
       "epoch": 0.77,
       "learning_rate": 7.387025063449081e-08,
-      "logits/chosen": 1.0122365951538086,
-      "logits/rejected": 2.279644727706909,
-      "logps/chosen": -411.8594665527344,
-      "logps/rejected": -457.96807861328125,
-      "loss": 0.4969,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -1.0128066539764404,
-      "rewards/margins": 1.0020781755447388,
-      "rewards/rejected": -2.0148847103118896,
+      "logits/chosen": 1.7092777490615845,
+      "logits/rejected": 2.965677261352539,
+      "logps/chosen": -423.5621643066406,
+      "logps/rejected": -466.57196044921875,
+      "loss": 0.5002,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -1.1220273971557617,
+      "rewards/margins": 0.9678171277046204,
+      "rewards/rejected": -2.0898444652557373,
       "step": 370
     },
     {
       "epoch": 0.79,
       "learning_rate": 6.138919252022435e-08,
-      "logits/chosen": 0.8450608253479004,
-      "logits/rejected": 1.820166826248169,
-      "logps/chosen": -383.67559814453125,
-      "logps/rejected": -475.4779357910156,
-      "loss": 0.4768,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.0418307781219482,
-      "rewards/margins": 0.9196658134460449,
-      "rewards/rejected": -1.9614967107772827,
+      "logits/chosen": 1.7860336303710938,
+      "logits/rejected": 2.569241523742676,
+      "logps/chosen": -395.4902648925781,
+      "logps/rejected": -483.0901794433594,
+      "loss": 0.4772,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.1515331268310547,
+      "rewards/margins": 0.8898499608039856,
+      "rewards/rejected": -2.0413832664489746,
       "step": 380
     },
     {
       "epoch": 0.82,
       "learning_rate": 4.991445467064689e-08,
-      "logits/chosen": 1.0426132678985596,
-      "logits/rejected": 1.9015953540802002,
-      "logps/chosen": -384.1719970703125,
-      "logps/rejected": -459.9111328125,
-      "loss": 0.5073,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -1.1213358640670776,
-      "rewards/margins": 0.7478152513504028,
-      "rewards/rejected": -1.8691511154174805,
+      "logits/chosen": 2.0826852321624756,
+      "logits/rejected": 2.8060660362243652,
+      "logps/chosen": -398.78375244140625,
+      "logps/rejected": -471.2264099121094,
+      "loss": 0.5066,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.2567694187164307,
+      "rewards/margins": 0.7216086983680725,
+      "rewards/rejected": -1.9783780574798584,
       "step": 390
     },
     {
       "epoch": 0.84,
       "learning_rate": 3.9507259776993954e-08,
-      "logits/chosen": 0.8149229288101196,
-      "logits/rejected": 1.6555734872817993,
-      "logps/chosen": -430.68231201171875,
-      "logps/rejected": -496.66900634765625,
-      "loss": 0.4859,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -1.098642110824585,
-      "rewards/margins": 0.9415690302848816,
-      "rewards/rejected": -2.0402112007141113,
+      "logits/chosen": 1.802354097366333,
+      "logits/rejected": 2.5923492908477783,
+      "logps/chosen": -446.500244140625,
+      "logps/rejected": -510.20269775390625,
+      "loss": 0.4853,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.2238214015960693,
+      "rewards/margins": 0.9289990663528442,
+      "rewards/rejected": -2.152820587158203,
       "step": 400
     },
     {
       "epoch": 0.84,
-      "eval_logits/chosen": 0.7817753553390503,
-      "eval_logits/rejected": 1.7438044548034668,
-      "eval_logps/chosen": -360.3138732910156,
-      "eval_logps/rejected": -457.9451599121094,
-      "eval_loss": 0.5031262040138245,
-      "eval_rewards/accuracies": 0.7578125,
-      "eval_rewards/chosen": -0.9776647090911865,
-      "eval_rewards/margins": 0.9803228974342346,
-      "eval_rewards/rejected": -1.9579875469207764,
-      "eval_runtime": 86.2141,
-      "eval_samples_per_second": 23.198,
-      "eval_steps_per_second": 0.371,
+      "eval_logits/chosen": 1.8194458484649658,
+      "eval_logits/rejected": 2.592175245285034,
+      "eval_logps/chosen": -372.7066650390625,
+      "eval_logps/rejected": -474.1963195800781,
+      "eval_loss": 0.5050143003463745,
+      "eval_rewards/accuracies": 0.75390625,
+      "eval_rewards/chosen": -1.0929131507873535,
+      "eval_rewards/margins": 1.0155609846115112,
+      "eval_rewards/rejected": -2.108474016189575,
+      "eval_runtime": 90.5801,
+      "eval_samples_per_second": 22.08,
+      "eval_steps_per_second": 0.353,
       "step": 400
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.022313472693447e-08,
-      "logits/chosen": 1.0873724222183228,
-      "logits/rejected": 2.1809914112091064,
-      "logps/chosen": -358.1825256347656,
-      "logps/rejected": -440.631591796875,
-      "loss": 0.5114,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.9868854284286499,
-      "rewards/margins": 0.9311205744743347,
-      "rewards/rejected": -1.918006181716919,
+      "logits/chosen": 2.2372403144836426,
+      "logits/rejected": 3.196664333343506,
+      "logps/chosen": -370.81719970703125,
+      "logps/rejected": -452.06549072265625,
+      "loss": 0.5086,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.1016533374786377,
+      "rewards/margins": 0.9261430501937866,
+      "rewards/rejected": -2.0277962684631348,
       "step": 410
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.2111614344599684e-08,
-      "logits/chosen": 0.8678689002990723,
-      "logits/rejected": 1.581181526184082,
-      "logps/chosen": -370.5611877441406,
-      "logps/rejected": -477.7791442871094,
-      "loss": 0.5041,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.9568517804145813,
-      "rewards/margins": 0.967370331287384,
-      "rewards/rejected": -1.9242219924926758,
+      "logits/chosen": 1.831080675125122,
+      "logits/rejected": 2.4410791397094727,
+      "logps/chosen": -385.7922058105469,
+      "logps/rejected": -492.590576171875,
+      "loss": 0.5061,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.084149956703186,
+      "rewards/margins": 0.9615718722343445,
+      "rewards/rejected": -2.0457215309143066,
       "step": 420
     },
     {
       "epoch": 0.9,
       "learning_rate": 1.521597710086439e-08,
-      "logits/chosen": 0.4731677174568176,
-      "logits/rejected": 1.4058442115783691,
-      "logps/chosen": -396.15325927734375,
-      "logps/rejected": -442.45404052734375,
-      "loss": 0.5063,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.994260311126709,
-      "rewards/margins": 0.8869916796684265,
-      "rewards/rejected": -1.8812516927719116,
+      "logits/chosen": 1.4260971546173096,
+      "logits/rejected": 2.3162856101989746,
+      "logps/chosen": -407.1165466308594,
+      "logps/rejected": -454.90374755859375,
+      "loss": 0.5059,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.0966671705245972,
+      "rewards/margins": 0.9018322229385376,
+      "rewards/rejected": -1.9984995126724243,
       "step": 430
     },
     {
       "epoch": 0.92,
       "learning_rate": 9.57301420397924e-09,
-      "logits/chosen": 0.8329499959945679,
-      "logits/rejected": 1.7063286304473877,
-      "logps/chosen": -360.3570861816406,
-      "logps/rejected": -442.45574951171875,
-      "loss": 0.4884,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.0168806314468384,
-      "rewards/margins": 0.8515257835388184,
-      "rewards/rejected": -1.8684062957763672,
+      "logits/chosen": 1.783463716506958,
+      "logits/rejected": 2.5885117053985596,
+      "logps/chosen": -373.5993347167969,
+      "logps/rejected": -458.12091064453125,
+      "loss": 0.487,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.1429402828216553,
+      "rewards/margins": 0.8700854182243347,
+      "rewards/rejected": -2.0130257606506348,
       "step": 440
     },
     {
       "epoch": 0.94,
       "learning_rate": 5.212833302556258e-09,
-      "logits/chosen": 0.8873745799064636,
-      "logits/rejected": 1.8809913396835327,
-      "logps/chosen": -344.71197509765625,
-      "logps/rejected": -413.39471435546875,
-      "loss": 0.5068,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.99580317735672,
-      "rewards/margins": 0.8256435394287109,
-      "rewards/rejected": -1.8214466571807861,
+      "logits/chosen": 1.8070141077041626,
+      "logits/rejected": 2.747885227203369,
+      "logps/chosen": -355.58221435546875,
+      "logps/rejected": -426.42584228515625,
+      "loss": 0.5082,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.0844265222549438,
+      "rewards/margins": 0.8474240303039551,
+      "rewards/rejected": -1.9318506717681885,
       "step": 450
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.158697848236607e-09,
-      "logits/chosen": 1.0402039289474487,
-      "logits/rejected": 2.0626494884490967,
-      "logps/chosen": -350.25115966796875,
-      "logps/rejected": -416.1396484375,
-      "loss": 0.4859,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.070770502090454,
-      "rewards/margins": 0.807015597820282,
-      "rewards/rejected": -1.8777860403060913,
+      "logits/chosen": 2.0278899669647217,
+      "logits/rejected": 3.022653818130493,
+      "logps/chosen": -362.0993347167969,
+      "logps/rejected": -428.6521911621094,
+      "loss": 0.4861,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.1751190423965454,
+      "rewards/margins": 0.813240647315979,
+      "rewards/rejected": -1.9883596897125244,
       "step": 460
     },
     {
       "epoch": 0.98,
       "learning_rate": 4.269029751107489e-10,
-      "logits/chosen": 0.4914283752441406,
-      "logits/rejected": 1.8748416900634766,
-      "logps/chosen": -398.6361083984375,
-      "logps/rejected": -465.1482849121094,
-      "loss": 0.4872,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -1.004058599472046,
-      "rewards/margins": 1.027204155921936,
-      "rewards/rejected": -2.0312628746032715,
+      "logits/chosen": 1.3355131149291992,
+      "logits/rejected": 2.729475736618042,
+      "logps/chosen": -406.28033447265625,
+      "logps/rejected": -480.8604431152344,
+      "loss": 0.4807,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.0706168413162231,
+      "rewards/margins": 1.0933626890182495,
+      "rewards/rejected": -2.1639795303344727,
       "step": 470
     },
     {
       "epoch": 1.0,
       "step": 478,
       "total_flos": 0.0,
-      "train_loss": 0.5366686437918052,
-      "train_runtime": 5196.7487,
-      "train_samples_per_second": 11.764,
-      "train_steps_per_second": 0.092
+      "train_loss": 0.5379065808890754,
+      "train_runtime": 5396.8094,
+      "train_samples_per_second": 11.328,
+      "train_steps_per_second": 0.089
     }
   ],
   "logging_steps": 10,