Model save

README.md

@@ -20,7 +20,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [google/gemma-7b](https://huggingface.co/google/gemma-7b) on the generator dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.6544
+- Loss: 1.6534
 
 ## Model description
 
@@ -57,7 +57,7 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss |
 |:-------------:|:-----:|:----:|:---------------:|
-| 0.8317        | 1.0   | 301  | 1.6544          |
+| 0.8265        | 1.0   | 301  | 1.6534          |
 
 
 ### Framework versions

all_results.json

@@ -1,14 +1,9 @@
 {
     "epoch": 1.0,
-    "eval_loss": 1.6543629169464111,
-    "eval_runtime": 1.2677,
-    "eval_samples": 16,
-    "eval_samples_per_second": 7.888,
-    "eval_steps_per_second": 0.789,
     "total_flos": 9.17802542639874e+17,
-    "train_loss": 2.9563070911901734,
-    "train_runtime": 3725.7135,
+    "train_loss": 2.9511482283918564,
+    "train_runtime": 778.9577,
     "train_samples": 116368,
-    "train_samples_per_second": 10.33,
-    "train_steps_per_second": 0.081
+    "train_samples_per_second": 49.406,
+    "train_steps_per_second": 0.386
 }

runs/Nov18_01-59-33_main-lora-gemma7b-coding-0-0/events.out.tfevents.1731913882.main-lora-gemma7b-coding-0-0.458.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:536b06f752fea660457651812e23328001c4b4f172a1057b994a0f2bb1cbafd8
-size 18466
+oid sha256:0105e1f113726d8e52c40af25e0adf9d33715470ea11320d7896d239fe48e085
+size 19091

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.0,
     "total_flos": 9.17802542639874e+17,
-    "train_loss": 2.9563070911901734,
-    "train_runtime": 3725.7135,
+    "train_loss": 2.9511482283918564,
+    "train_runtime": 778.9577,
     "train_samples": 116368,
-    "train_samples_per_second": 10.33,
-    "train_steps_per_second": 0.081
+    "train_samples_per_second": 49.406,
+    "train_steps_per_second": 0.386
 }

trainer_state.json

@@ -10,447 +10,447 @@
   "log_history": [
     {
       "epoch": 0.0033222591362126247,
-      "grad_norm": 58.848934173583984,
+      "grad_norm": 51.88744354248047,
       "learning_rate": 6.451612903225806e-06,
-      "loss": 20.4567,
+      "loss": 20.0655,
       "step": 1
     },
     {
       "epoch": 0.016611295681063124,
-      "grad_norm": 35.33861541748047,
+      "grad_norm": 38.35562515258789,
       "learning_rate": 3.2258064516129034e-05,
-      "loss": 20.5366,
+      "loss": 20.4442,
       "step": 5
     },
     {
       "epoch": 0.03322259136212625,
-      "grad_norm": 16.172595977783203,
+      "grad_norm": 16.541790008544922,
       "learning_rate": 6.451612903225807e-05,
-      "loss": 17.7569,
+      "loss": 17.9286,
       "step": 10
     },
     {
       "epoch": 0.04983388704318937,
-      "grad_norm": 7.711285591125488,
+      "grad_norm": 7.802853584289551,
       "learning_rate": 9.677419354838711e-05,
-      "loss": 15.7111,
+      "loss": 15.769,
       "step": 15
     },
     {
       "epoch": 0.0664451827242525,
-      "grad_norm": 7.137362957000732,
+      "grad_norm": 7.048831462860107,
       "learning_rate": 0.00012903225806451613,
-      "loss": 14.1919,
+      "loss": 14.2071,
       "step": 20
     },
     {
       "epoch": 0.08305647840531562,
-      "grad_norm": 2.8490428924560547,
+      "grad_norm": 2.842278003692627,
       "learning_rate": 0.00016129032258064516,
-      "loss": 13.0076,
+      "loss": 13.0251,
       "step": 25
     },
     {
       "epoch": 0.09966777408637874,
-      "grad_norm": 3.174508571624756,
+      "grad_norm": 3.1177785396575928,
       "learning_rate": 0.00019354838709677422,
-      "loss": 12.4104,
+      "loss": 12.4164,
       "step": 30
     },
     {
       "epoch": 0.11627906976744186,
-      "grad_norm": 6.637372970581055,
+      "grad_norm": 7.123711109161377,
       "learning_rate": 0.0001998917111338525,
-      "loss": 11.4493,
+      "loss": 11.4556,
       "step": 35
     },
     {
       "epoch": 0.132890365448505,
-      "grad_norm": 12.817553520202637,
+      "grad_norm": 13.379329681396484,
       "learning_rate": 0.00019945218953682734,
-      "loss": 10.3687,
+      "loss": 10.3635,
       "step": 40
     },
     {
       "epoch": 0.14950166112956811,
-      "grad_norm": 17.324180603027344,
+      "grad_norm": 17.68781280517578,
       "learning_rate": 0.00019867615321125795,
-      "loss": 8.29,
+      "loss": 8.2723,
       "step": 45
     },
     {
       "epoch": 0.16611295681063123,
-      "grad_norm": 20.25031852722168,
+      "grad_norm": 18.207794189453125,
       "learning_rate": 0.00019756622801842143,
-      "loss": 5.0469,
+      "loss": 4.9814,
       "step": 50
     },
     {
       "epoch": 0.18272425249169436,
-      "grad_norm": 10.32929801940918,
+      "grad_norm": 23.382278442382812,
       "learning_rate": 0.0001961261695938319,
-      "loss": 2.5342,
+      "loss": 2.4786,
       "step": 55
     },
     {
       "epoch": 0.19933554817275748,
-      "grad_norm": 6.437869071960449,
+      "grad_norm": 4.926826000213623,
       "learning_rate": 0.00019436085063935835,
-      "loss": 1.8296,
+      "loss": 1.8199,
       "step": 60
     },
     {
       "epoch": 0.2159468438538206,
-      "grad_norm": 2.7442312240600586,
+      "grad_norm": 2.2475671768188477,
       "learning_rate": 0.00019227624443554425,
-      "loss": 1.5211,
+      "loss": 1.519,
       "step": 65
     },
     {
       "epoch": 0.23255813953488372,
-      "grad_norm": 1.103617548942566,
+      "grad_norm": 1.4462262392044067,
       "learning_rate": 0.0001898794046299167,
-      "loss": 1.342,
+      "loss": 1.3358,
       "step": 70
     },
     {
       "epoch": 0.24916943521594684,
-      "grad_norm": 0.8008713722229004,
+      "grad_norm": 0.7703434824943542,
       "learning_rate": 0.00018717844136967624,
-      "loss": 1.2224,
+      "loss": 1.2165,
       "step": 75
     },
     {
       "epoch": 0.26578073089701,
-      "grad_norm": 0.9825754761695862,
+      "grad_norm": 0.6548369526863098,
       "learning_rate": 0.00018418249385952575,
-      "loss": 1.1639,
+      "loss": 1.161,
       "step": 80
     },
     {
       "epoch": 0.2823920265780731,
-      "grad_norm": 1.6092805862426758,
+      "grad_norm": 1.4386329650878906,
       "learning_rate": 0.00018090169943749476,
-      "loss": 1.106,
+      "loss": 1.1037,
       "step": 85
     },
     {
       "epoch": 0.29900332225913623,
-      "grad_norm": 0.7608142495155334,
+      "grad_norm": 0.7257230281829834,
       "learning_rate": 0.0001773471592733964,
-      "loss": 1.0649,
+      "loss": 1.0609,
       "step": 90
     },
     {
       "epoch": 0.31561461794019935,
-      "grad_norm": 0.7974920272827148,
+      "grad_norm": 0.8698992729187012,
       "learning_rate": 0.0001735309008059829,
-      "loss": 1.0544,
+      "loss": 1.048,
       "step": 95
     },
     {
       "epoch": 0.33222591362126247,
-      "grad_norm": 0.8733552694320679,
+      "grad_norm": 0.7444359064102173,
       "learning_rate": 0.00016946583704589973,
-      "loss": 1.0418,
+      "loss": 1.034,
       "step": 100
     },
     {
       "epoch": 0.3488372093023256,
-      "grad_norm": 0.8620198965072632,
+      "grad_norm": 1.0887985229492188,
       "learning_rate": 0.00016516572288214552,
-      "loss": 1.0134,
+      "loss": 1.0021,
       "step": 105
     },
     {
       "epoch": 0.3654485049833887,
-      "grad_norm": 0.9430036544799805,
+      "grad_norm": 1.341417670249939,
       "learning_rate": 0.00016064510853988138,
-      "loss": 1.0117,
+      "loss": 1.0024,
       "step": 110
     },
     {
       "epoch": 0.38205980066445183,
-      "grad_norm": 0.9424466490745544,
+      "grad_norm": 0.7008532285690308,
       "learning_rate": 0.0001559192903470747,
-      "loss": 0.9687,
+      "loss": 0.9584,
       "step": 115
     },
     {
       "epoch": 0.39867109634551495,
-      "grad_norm": 0.7452667355537415,
+      "grad_norm": 0.8793571591377258,
       "learning_rate": 0.00015100425897656753,
-      "loss": 0.9678,
+      "loss": 0.9583,
       "step": 120
     },
     {
       "epoch": 0.4152823920265781,
-      "grad_norm": 0.6089577674865723,
+      "grad_norm": 0.7256600856781006,
       "learning_rate": 0.00014591664533870118,
-      "loss": 0.9428,
+      "loss": 0.9338,
       "step": 125
     },
     {
       "epoch": 0.4318936877076412,
-      "grad_norm": 0.5161769390106201,
+      "grad_norm": 0.5686385631561279,
       "learning_rate": 0.00014067366430758004,
-      "loss": 0.9344,
+      "loss": 0.9274,
       "step": 130
     },
     {
       "epoch": 0.4485049833887043,
-      "grad_norm": 1.061056137084961,
+      "grad_norm": 1.2567209005355835,
       "learning_rate": 0.00013529305647138687,
-      "loss": 0.9316,
+      "loss": 0.925,
       "step": 135
     },
     {
       "epoch": 0.46511627906976744,
-      "grad_norm": 1.1914814710617065,
+      "grad_norm": 1.181892991065979,
       "learning_rate": 0.0001297930281038482,
-      "loss": 0.932,
+      "loss": 0.9243,
       "step": 140
     },
     {
       "epoch": 0.48172757475083056,
-      "grad_norm": 1.3082082271575928,
+      "grad_norm": 0.7734186053276062,
       "learning_rate": 0.00012419218955996676,
-      "loss": 0.9225,
+      "loss": 0.9127,
       "step": 145
     },
     {
       "epoch": 0.4983388704318937,
-      "grad_norm": 0.9995535612106323,
+      "grad_norm": 0.9539685845375061,
       "learning_rate": 0.00011850949230447145,
-      "loss": 0.9153,
+      "loss": 0.9106,
       "step": 150
     },
     {
       "epoch": 0.5149501661129569,
-      "grad_norm": 0.5884081125259399,
+      "grad_norm": 0.8648198843002319,
       "learning_rate": 0.00011276416478605949,
-      "loss": 0.9105,
+      "loss": 0.9035,
       "step": 155
     },
     {
       "epoch": 0.53156146179402,
-      "grad_norm": 0.9028487801551819,
+      "grad_norm": 0.7837306261062622,
       "learning_rate": 0.00010697564737441252,
-      "loss": 0.9142,
+      "loss": 0.9096,
       "step": 160
     },
     {
       "epoch": 0.5481727574750831,
-      "grad_norm": 0.9643915891647339,
+      "grad_norm": 0.7229638695716858,
       "learning_rate": 0.00010116352658013973,
-      "loss": 0.8963,
+      "loss": 0.8911,
       "step": 165
     },
     {
       "epoch": 0.5647840531561462,
-      "grad_norm": 0.6744817495346069,
+      "grad_norm": 0.7054806351661682,
       "learning_rate": 9.534746878022534e-05,
-      "loss": 0.9227,
+      "loss": 0.9145,
       "step": 170
     },
     {
       "epoch": 0.5813953488372093,
-      "grad_norm": 0.5441331267356873,
+      "grad_norm": 0.5297299027442932,
       "learning_rate": 8.954715367323468e-05,
-      "loss": 0.8981,
+      "loss": 0.8937,
       "step": 175
     },
     {
       "epoch": 0.5980066445182725,
-      "grad_norm": 0.6811143755912781,
+      "grad_norm": 0.5183060169219971,
       "learning_rate": 8.378220768944327e-05,
-      "loss": 0.8746,
+      "loss": 0.8682,
       "step": 180
     },
     {
       "epoch": 0.6146179401993356,
-      "grad_norm": 0.5517932176589966,
+      "grad_norm": 0.5382773876190186,
       "learning_rate": 7.807213758120966e-05,
-      "loss": 0.8821,
+      "loss": 0.8748,
       "step": 185
     },
     {
       "epoch": 0.6312292358803987,
-      "grad_norm": 1.0475077629089355,
+      "grad_norm": 0.8112709522247314,
       "learning_rate": 7.243626441830009e-05,
-      "loss": 0.8766,
+      "loss": 0.8694,
       "step": 190
     },
     {
       "epoch": 0.6478405315614618,
-      "grad_norm": 0.8785480856895447,
+      "grad_norm": 0.7376631498336792,
       "learning_rate": 6.68936582115042e-05,
-      "loss": 0.8696,
+      "loss": 0.8641,
       "step": 195
     },
     {
       "epoch": 0.6644518272425249,
-      "grad_norm": 0.5514780282974243,
+      "grad_norm": 0.45725372433662415,
       "learning_rate": 6.146307338575519e-05,
-      "loss": 0.8435,
+      "loss": 0.839,
       "step": 200
     },
     {
       "epoch": 0.6810631229235881,
-      "grad_norm": 0.7921276688575745,
+      "grad_norm": 0.6437954306602478,
       "learning_rate": 5.616288532109225e-05,
-      "loss": 0.8613,
+      "loss": 0.8563,
       "step": 205
     },
     {
       "epoch": 0.6976744186046512,
-      "grad_norm": 1.2019799947738647,
+      "grad_norm": 1.122187852859497,
       "learning_rate": 5.101102817619131e-05,
-      "loss": 0.8491,
+      "loss": 0.8448,
       "step": 210
     },
     {
       "epoch": 0.7142857142857143,
-      "grad_norm": 0.5968192219734192,
+      "grad_norm": 0.7291833758354187,
       "learning_rate": 4.6024934204848745e-05,
-      "loss": 0.8399,
+      "loss": 0.8338,
       "step": 215
     },
     {
       "epoch": 0.7308970099667774,
-      "grad_norm": 0.622888445854187,
+      "grad_norm": 0.6493574380874634,
       "learning_rate": 4.12214747707527e-05,
-      "loss": 0.8589,
+      "loss": 0.8546,
       "step": 220
     },
     {
       "epoch": 0.7475083056478405,
-      "grad_norm": 0.6290799379348755,
+      "grad_norm": 0.7318633794784546,
       "learning_rate": 3.661690326012897e-05,
-      "loss": 0.8479,
+      "loss": 0.8432,
       "step": 225
     },
     {
       "epoch": 0.7641196013289037,
-      "grad_norm": 0.5619077086448669,
+      "grad_norm": 0.5700820088386536,
       "learning_rate": 3.222680008542678e-05,
-      "loss": 0.8341,
+      "loss": 0.828,
       "step": 230
     },
     {
       "epoch": 0.7807308970099668,
-      "grad_norm": 0.4804949164390564,
+      "grad_norm": 0.4655066430568695,
       "learning_rate": 2.8066019966134904e-05,
-      "loss": 0.8355,
+      "loss": 0.8318,
       "step": 235
     },
     {
       "epoch": 0.7973421926910299,
-      "grad_norm": 0.8997146487236023,
+      "grad_norm": 0.8703047633171082,
       "learning_rate": 2.4148641665113113e-05,
-      "loss": 0.8305,
+      "loss": 0.8275,
       "step": 240
     },
     {
       "epoch": 0.813953488372093,
-      "grad_norm": 0.62757408618927,
+      "grad_norm": 0.6757218241691589,
       "learning_rate": 2.0487920350515212e-05,
-      "loss": 0.8352,
+      "loss": 0.8317,
       "step": 245
     },
     {
       "epoch": 0.8305647840531561,
-      "grad_norm": 0.6770641207695007,
+      "grad_norm": 0.7051078677177429,
       "learning_rate": 1.7096242744495837e-05,
-      "loss": 0.8365,
+      "loss": 0.8323,
       "step": 250
     },
     {
       "epoch": 0.8471760797342193,
-      "grad_norm": 0.8780685663223267,
+      "grad_norm": 0.9345777630805969,
       "learning_rate": 1.3985085210463477e-05,
-      "loss": 0.8397,
+      "loss": 0.8348,
       "step": 255
     },
     {
       "epoch": 0.8637873754152824,
-      "grad_norm": 0.5401960611343384,
+      "grad_norm": 0.5797870755195618,
       "learning_rate": 1.116497492069961e-05,
-      "loss": 0.8265,
+      "loss": 0.8215,
       "step": 260
     },
     {
       "epoch": 0.8803986710963455,
-      "grad_norm": 0.5332029461860657,
+      "grad_norm": 0.5265169739723206,
       "learning_rate": 8.645454235739903e-06,
-      "loss": 0.8441,
+      "loss": 0.8405,
       "step": 265
     },
     {
       "epoch": 0.8970099667774086,
-      "grad_norm": 0.6243578195571899,
+      "grad_norm": 0.6980829834938049,
       "learning_rate": 6.435048416046863e-06,
-      "loss": 0.8333,
+      "loss": 0.8281,
       "step": 270
     },
     {
       "epoch": 0.9136212624584718,
-      "grad_norm": 0.5947065949440002,
+      "grad_norm": 0.5821875333786011,
       "learning_rate": 4.541236775226809e-06,
-      "loss": 0.8311,
+      "loss": 0.8259,
       "step": 275
     },
     {
       "epoch": 0.9302325581395349,
-      "grad_norm": 0.5467190742492676,
+      "grad_norm": 0.6054457426071167,
       "learning_rate": 2.970427372400353e-06,
-      "loss": 0.8217,
+      "loss": 0.8171,
       "step": 280
     },
     {
       "epoch": 0.946843853820598,
-      "grad_norm": 0.5683417320251465,
+      "grad_norm": 0.5492629408836365,
       "learning_rate": 1.7279353293586765e-06,
-      "loss": 0.8293,
+      "loss": 0.8246,
       "step": 285
     },
     {
       "epoch": 0.9634551495016611,
-      "grad_norm": 0.5739116072654724,
+      "grad_norm": 0.606873631477356,
       "learning_rate": 8.17964845873831e-07,
-      "loss": 0.833,
+      "loss": 0.8269,
       "step": 290
     },
     {
       "epoch": 0.9800664451827242,
-      "grad_norm": 0.6611495614051819,
+      "grad_norm": 0.8362772464752197,
       "learning_rate": 2.4359497401758024e-07,
-      "loss": 0.8317,
+      "loss": 0.8267,
       "step": 295
     },
     {
       "epoch": 0.9966777408637874,
-      "grad_norm": 0.7239180207252502,
+      "grad_norm": 0.8066886067390442,
       "learning_rate": 6.769199623779532e-09,
-      "loss": 0.8317,
+      "loss": 0.8265,
       "step": 300
     },
     {
       "epoch": 1.0,
-      "eval_loss": 1.6543629169464111,
-      "eval_runtime": 1.2337,
-      "eval_samples_per_second": 8.106,
-      "eval_steps_per_second": 0.811,
+      "eval_loss": 1.6534239053726196,
+      "eval_runtime": 0.4546,
+      "eval_samples_per_second": 21.998,
+      "eval_steps_per_second": 2.2,
       "step": 301
     },
     {
       "epoch": 1.0,
       "step": 301,
       "total_flos": 9.17802542639874e+17,
-      "train_loss": 2.9563070911901734,
-      "train_runtime": 3725.7135,
-      "train_samples_per_second": 10.33,
-      "train_steps_per_second": 0.081
+      "train_loss": 2.9511482283918564,
+      "train_runtime": 778.9577,
+      "train_samples_per_second": 49.406,
+      "train_steps_per_second": 0.386
     }
   ],
   "logging_steps": 5,