Model save

README.md

@@ -20,7 +20,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [google/gemma-7b](https://huggingface.co/google/gemma-7b) on the generator dataset.
 It achieves the following results on the evaluation set:
-- Loss: 2.3991
+- Loss: 2.4052
 
 ## Model description
 
@@ -57,7 +57,7 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss |
 |:-------------:|:------:|:----:|:---------------:|
-| 1.2456        | 0.9981 | 261  | 2.3991          |
+| 1.2442        | 0.9981 | 261  | 2.4052          |
 
 
 ### Framework versions

all_results.json

@@ -1,14 +1,9 @@
 {
     "epoch": 0.9980879541108987,
-    "eval_loss": 2.3990681171417236,
-    "eval_runtime": 1.2805,
-    "eval_samples": 15,
-    "eval_samples_per_second": 9.371,
-    "eval_steps_per_second": 0.781,
     "total_flos": 7.958354271676662e+17,
-    "train_loss": 4.106667400776655,
-    "train_runtime": 3226.2655,
+    "train_loss": 4.106453996508514,
+    "train_runtime": 670.8506,
     "train_samples": 111440,
-    "train_samples_per_second": 10.373,
-    "train_steps_per_second": 0.081
+    "train_samples_per_second": 49.884,
+    "train_steps_per_second": 0.389
 }

runs/Nov18_06-39-27_main-lora-gemma7b-closedqa-0-0/events.out.tfevents.1731930676.main-lora-gemma7b-closedqa-0-0.458.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8eb2eb9493bbd0739d7bf3029b4c1e1c6324a110a4538501a2ff1598d5705a14
-size 16782
+oid sha256:045439e960d40b91d793c9c72ae0d9ced6bef45fd60651c284cf5f3f668d5445
+size 17407

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9980879541108987,
     "total_flos": 7.958354271676662e+17,
-    "train_loss": 4.106667400776655,
-    "train_runtime": 3226.2655,
+    "train_loss": 4.106453996508514,
+    "train_runtime": 670.8506,
     "train_samples": 111440,
-    "train_samples_per_second": 10.373,
-    "train_steps_per_second": 0.081
+    "train_samples_per_second": 49.884,
+    "train_steps_per_second": 0.389
 }

trainer_state.json

@@ -10,391 +10,391 @@
   "log_history": [
     {
       "epoch": 0.0038240917782026767,
-      "grad_norm": 109.6296157836914,
+      "grad_norm": 105.78279876708984,
       "learning_rate": 7.4074074074074075e-06,
-      "loss": 28.2707,
+      "loss": 27.5504,
       "step": 1
     },
     {
       "epoch": 0.019120458891013385,
-      "grad_norm": 73.76856231689453,
+      "grad_norm": 89.91433715820312,
       "learning_rate": 3.7037037037037037e-05,
-      "loss": 26.7709,
+      "loss": 26.708,
       "step": 5
     },
     {
       "epoch": 0.03824091778202677,
-      "grad_norm": 27.987791061401367,
+      "grad_norm": 27.077810287475586,
       "learning_rate": 7.407407407407407e-05,
-      "loss": 22.5898,
+      "loss": 22.6974,
       "step": 10
     },
     {
       "epoch": 0.05736137667304015,
-      "grad_norm": 13.016111373901367,
+      "grad_norm": 13.22861385345459,
       "learning_rate": 0.00011111111111111112,
-      "loss": 18.9729,
+      "loss": 19.0567,
       "step": 15
     },
     {
       "epoch": 0.07648183556405354,
-      "grad_norm": 11.862752914428711,
+      "grad_norm": 11.456145286560059,
       "learning_rate": 0.00014814814814814815,
-      "loss": 16.2855,
+      "loss": 16.3135,
       "step": 20
     },
     {
       "epoch": 0.09560229445506692,
-      "grad_norm": 2.9698574542999268,
+      "grad_norm": 3.0151147842407227,
       "learning_rate": 0.0001851851851851852,
-      "loss": 15.0127,
+      "loss": 15.0328,
       "step": 25
     },
     {
       "epoch": 0.1147227533460803,
-      "grad_norm": 4.096982955932617,
+      "grad_norm": 4.0936784744262695,
       "learning_rate": 0.00019991889981715698,
-      "loss": 13.9569,
+      "loss": 13.9759,
       "step": 30
     },
     {
       "epoch": 0.1338432122370937,
-      "grad_norm": 6.29751443862915,
+      "grad_norm": 6.428543567657471,
       "learning_rate": 0.0001994237638847428,
-      "loss": 13.1471,
+      "loss": 13.1664,
       "step": 35
     },
     {
       "epoch": 0.15296367112810708,
-      "grad_norm": 13.995941162109375,
+      "grad_norm": 13.633052825927734,
       "learning_rate": 0.00019848077530122083,
-      "loss": 11.9192,
+      "loss": 11.9549,
       "step": 40
     },
     {
       "epoch": 0.17208413001912046,
-      "grad_norm": 23.425451278686523,
+      "grad_norm": 23.461597442626953,
       "learning_rate": 0.0001970941817426052,
-      "loss": 9.1546,
+      "loss": 9.1656,
       "step": 45
     },
     {
       "epoch": 0.19120458891013384,
-      "grad_norm": 24.60663604736328,
+      "grad_norm": 25.158462524414062,
       "learning_rate": 0.00019527022909596536,
-      "loss": 6.0786,
+      "loss": 6.0581,
       "step": 50
     },
     {
       "epoch": 0.21032504780114722,
-      "grad_norm": 9.548501014709473,
+      "grad_norm": 9.126226425170898,
       "learning_rate": 0.00019301713332493386,
-      "loss": 3.2243,
+      "loss": 3.1728,
       "step": 55
     },
     {
       "epoch": 0.2294455066921606,
-      "grad_norm": 6.405646324157715,
+      "grad_norm": 6.13222074508667,
       "learning_rate": 0.00019034504346103823,
-      "loss": 2.4261,
+      "loss": 2.3894,
       "step": 60
     },
     {
       "epoch": 0.248565965583174,
-      "grad_norm": 2.391706943511963,
+      "grad_norm": 1.9926798343658447,
       "learning_rate": 0.00018726599588756145,
-      "loss": 2.1088,
+      "loss": 2.0757,
       "step": 65
     },
     {
       "epoch": 0.2676864244741874,
-      "grad_norm": 1.3095039129257202,
+      "grad_norm": 1.3959568738937378,
       "learning_rate": 0.00018379386012185814,
-      "loss": 1.8208,
+      "loss": 1.7977,
       "step": 70
     },
     {
       "epoch": 0.28680688336520077,
-      "grad_norm": 1.0937951803207397,
+      "grad_norm": 2.5797815322875977,
       "learning_rate": 0.00017994427634035015,
-      "loss": 1.6814,
+      "loss": 1.6855,
       "step": 75
     },
     {
       "epoch": 0.30592734225621415,
-      "grad_norm": 0.9703987240791321,
+      "grad_norm": 0.8123499155044556,
       "learning_rate": 0.00017573458492761801,
-      "loss": 1.5943,
+      "loss": 1.6023,
       "step": 80
     },
     {
       "epoch": 0.32504780114722753,
-      "grad_norm": 1.7736315727233887,
+      "grad_norm": 1.2287877798080444,
       "learning_rate": 0.00017118374836693406,
-      "loss": 1.5357,
+      "loss": 1.543,
       "step": 85
     },
     {
       "epoch": 0.3441682600382409,
-      "grad_norm": 0.7995043992996216,
+      "grad_norm": 0.7896401882171631,
       "learning_rate": 0.00016631226582407952,
-      "loss": 1.4938,
+      "loss": 1.5,
       "step": 90
     },
     {
       "epoch": 0.3632887189292543,
-      "grad_norm": 0.689298152923584,
+      "grad_norm": 1.0136632919311523,
       "learning_rate": 0.00016114208080920123,
-      "loss": 1.4527,
+      "loss": 1.4581,
       "step": 95
     },
     {
       "epoch": 0.3824091778202677,
-      "grad_norm": 1.3656119108200073,
+      "grad_norm": 1.0349676609039307,
       "learning_rate": 0.00015569648233264394,
-      "loss": 1.4352,
+      "loss": 1.4393,
       "step": 100
     },
     {
       "epoch": 0.40152963671128106,
-      "grad_norm": 0.8344822525978088,
+      "grad_norm": 0.7523437738418579,
       "learning_rate": 0.00015000000000000001,
-      "loss": 1.4039,
+      "loss": 1.4075,
       "step": 105
     },
     {
       "epoch": 0.42065009560229444,
-      "grad_norm": 0.9604928493499756,
+      "grad_norm": 0.8223061561584473,
       "learning_rate": 0.00014407829351891857,
-      "loss": 1.3876,
+      "loss": 1.3899,
       "step": 110
     },
     {
       "epoch": 0.4397705544933078,
-      "grad_norm": 0.645470917224884,
+      "grad_norm": 0.7651774883270264,
       "learning_rate": 0.00013795803711538966,
-      "loss": 1.3706,
+      "loss": 1.3739,
       "step": 115
     },
     {
       "epoch": 0.4588910133843212,
-      "grad_norm": 0.5904386639595032,
+      "grad_norm": 0.6884527802467346,
       "learning_rate": 0.00013166679938014726,
-      "loss": 1.3562,
+      "loss": 1.357,
       "step": 120
     },
     {
       "epoch": 0.4780114722753346,
-      "grad_norm": 0.6904798746109009,
+      "grad_norm": 0.7700805068016052,
       "learning_rate": 0.00012523291908642217,
-      "loss": 1.3502,
+      "loss": 1.3525,
       "step": 125
     },
     {
       "epoch": 0.497131931166348,
-      "grad_norm": 0.8584344387054443,
+      "grad_norm": 1.0306934118270874,
       "learning_rate": 0.00011868537753842051,
-      "loss": 1.3508,
+      "loss": 1.3514,
       "step": 130
     },
     {
       "epoch": 0.5162523900573613,
-      "grad_norm": 1.3949240446090698,
+      "grad_norm": 1.1528677940368652,
       "learning_rate": 0.0001120536680255323,
-      "loss": 1.3284,
+      "loss": 1.3288,
       "step": 135
     },
     {
       "epoch": 0.5353728489483748,
-      "grad_norm": 1.0056356191635132,
+      "grad_norm": 0.7638438940048218,
       "learning_rate": 0.00010536766297031215,
-      "loss": 1.318,
+      "loss": 1.3204,
       "step": 140
     },
     {
       "epoch": 0.5544933078393881,
-      "grad_norm": 0.9676538109779358,
+      "grad_norm": 0.8795316219329834,
       "learning_rate": 9.865747936866027e-05,
-      "loss": 1.3218,
+      "loss": 1.3215,
       "step": 145
     },
     {
       "epoch": 0.5736137667304015,
-      "grad_norm": 0.8652806878089905,
+      "grad_norm": 0.8259925246238708,
       "learning_rate": 9.195334312832742e-05,
-      "loss": 1.3008,
+      "loss": 1.3012,
       "step": 150
     },
     {
       "epoch": 0.5927342256214149,
-      "grad_norm": 1.193677306175232,
+      "grad_norm": 1.0407475233078003,
       "learning_rate": 8.528545291682838e-05,
-      "loss": 1.2873,
+      "loss": 1.2876,
       "step": 155
     },
     {
       "epoch": 0.6118546845124283,
-      "grad_norm": 0.6716729998588562,
+      "grad_norm": 0.7297894358634949,
       "learning_rate": 7.868384413205842e-05,
-      "loss": 1.2834,
+      "loss": 1.2845,
       "step": 160
     },
     {
       "epoch": 0.6309751434034416,
-      "grad_norm": 4.712409019470215,
+      "grad_norm": 2.0282633304595947,
       "learning_rate": 7.217825360835473e-05,
-      "loss": 1.2933,
+      "loss": 1.2929,
       "step": 165
     },
     {
       "epoch": 0.6500956022944551,
-      "grad_norm": 1.2476301193237305,
+      "grad_norm": 1.362356185913086,
       "learning_rate": 6.579798566743314e-05,
-      "loss": 1.2727,
+      "loss": 1.2715,
       "step": 170
     },
     {
       "epoch": 0.6692160611854685,
-      "grad_norm": 0.7000300288200378,
+      "grad_norm": 0.5886189937591553,
       "learning_rate": 5.957178011756952e-05,
-      "loss": 1.2674,
+      "loss": 1.2666,
       "step": 175
     },
     {
       "epoch": 0.6883365200764818,
-      "grad_norm": 0.6005345582962036,
+      "grad_norm": 0.683692991733551,
       "learning_rate": 5.3527682795623146e-05,
-      "loss": 1.2683,
+      "loss": 1.2673,
       "step": 180
     },
     {
       "epoch": 0.7074569789674953,
-      "grad_norm": 0.6676229238510132,
+      "grad_norm": 0.7453979253768921,
       "learning_rate": 4.7692919235042255e-05,
-      "loss": 1.2739,
+      "loss": 1.2728,
       "step": 185
     },
     {
       "epoch": 0.7265774378585086,
-      "grad_norm": 0.5791189670562744,
+      "grad_norm": 0.563890814781189,
       "learning_rate": 4.209377202891212e-05,
-      "loss": 1.2686,
+      "loss": 1.2673,
       "step": 190
     },
     {
       "epoch": 0.745697896749522,
-      "grad_norm": 0.6084746718406677,
+      "grad_norm": 0.6703224182128906,
       "learning_rate": 3.675546244046228e-05,
-      "loss": 1.2485,
+      "loss": 1.2457,
       "step": 195
     },
     {
       "epoch": 0.7648183556405354,
-      "grad_norm": 0.5911340117454529,
+      "grad_norm": 0.6253310441970825,
       "learning_rate": 3.170203679431584e-05,
-      "loss": 1.2727,
+      "loss": 1.2703,
       "step": 200
     },
     {
       "epoch": 0.7839388145315488,
-      "grad_norm": 0.7844299674034119,
+      "grad_norm": 0.8390852212905884,
       "learning_rate": 2.6956258160229695e-05,
-      "loss": 1.2634,
+      "loss": 1.2605,
       "step": 205
     },
     {
       "epoch": 0.8030592734225621,
-      "grad_norm": 0.6864188313484192,
+      "grad_norm": 0.6806175708770752,
       "learning_rate": 2.2539503817234553e-05,
-      "loss": 1.2407,
+      "loss": 1.2402,
       "step": 210
     },
     {
       "epoch": 0.8221797323135756,
-      "grad_norm": 0.7077968716621399,
+      "grad_norm": 0.774019718170166,
       "learning_rate": 1.8471668960045574e-05,
-      "loss": 1.2416,
+      "loss": 1.241,
       "step": 215
     },
     {
       "epoch": 0.8413001912045889,
-      "grad_norm": 0.6579145789146423,
+      "grad_norm": 0.6997012495994568,
       "learning_rate": 1.4771077081496654e-05,
-      "loss": 1.2472,
+      "loss": 1.2451,
       "step": 220
     },
     {
       "epoch": 0.8604206500956023,
-      "grad_norm": 0.8815127015113831,
+      "grad_norm": 0.8751155138015747,
       "learning_rate": 1.1454397434679021e-05,
-      "loss": 1.2361,
+      "loss": 1.2339,
       "step": 225
     },
     {
       "epoch": 0.8795411089866156,
-      "grad_norm": 0.8044998049736023,
+      "grad_norm": 0.7787600159645081,
       "learning_rate": 8.536569946574546e-06,
-      "loss": 1.2467,
+      "loss": 1.2464,
       "step": 230
     },
     {
       "epoch": 0.8986615678776291,
-      "grad_norm": 0.5612942576408386,
+      "grad_norm": 0.5735809803009033,
       "learning_rate": 6.030737921409169e-06,
-      "loss": 1.2457,
+      "loss": 1.2439,
       "step": 235
     },
     {
       "epoch": 0.9177820267686424,
-      "grad_norm": 0.6176737546920776,
+      "grad_norm": 0.6756762862205505,
       "learning_rate": 3.948188836862776e-06,
-      "loss": 1.2367,
+      "loss": 1.2356,
       "step": 240
     },
     {
       "epoch": 0.9369024856596558,
-      "grad_norm": 0.5503106117248535,
+      "grad_norm": 0.573499858379364,
       "learning_rate": 2.2983034998182997e-06,
-      "loss": 1.2424,
+      "loss": 1.2411,
       "step": 245
     },
     {
       "epoch": 0.9560229445506692,
-      "grad_norm": 0.6087459921836853,
+      "grad_norm": 0.6631729006767273,
       "learning_rate": 1.0885137906768372e-06,
-      "loss": 1.2377,
+      "loss": 1.2363,
       "step": 250
     },
     {
       "epoch": 0.9751434034416826,
-      "grad_norm": 0.6323258876800537,
+      "grad_norm": 0.5861126780509949,
       "learning_rate": 3.2426918657900704e-07,
-      "loss": 1.233,
+      "loss": 1.2316,
       "step": 255
     },
     {
       "epoch": 0.994263862332696,
-      "grad_norm": 0.6436792612075806,
+      "grad_norm": 0.6123586297035217,
       "learning_rate": 9.012214327897006e-09,
-      "loss": 1.2456,
+      "loss": 1.2442,
       "step": 260
     },
     {
       "epoch": 0.9980879541108987,
-      "eval_loss": 2.3990681171417236,
-      "eval_runtime": 1.2567,
-      "eval_samples_per_second": 9.549,
-      "eval_steps_per_second": 0.796,
+      "eval_loss": 2.405249834060669,
+      "eval_runtime": 0.4542,
+      "eval_samples_per_second": 26.419,
+      "eval_steps_per_second": 2.202,
       "step": 261
     },
     {
       "epoch": 0.9980879541108987,
       "step": 261,
       "total_flos": 7.958354271676662e+17,
-      "train_loss": 4.106667400776655,
-      "train_runtime": 3226.2655,
-      "train_samples_per_second": 10.373,
-      "train_steps_per_second": 0.081
+      "train_loss": 4.106453996508514,
+      "train_runtime": 670.8506,
+      "train_samples_per_second": 49.884,
+      "train_steps_per_second": 0.389
     }
   ],
   "logging_steps": 5,