Model save

README.md

@@ -0,0 +1,69 @@
+---
+library_name: peft
+license: gemma
+base_model: google/gemma-7b
+tags:
+- trl
+- sft
+- generated_from_trainer
+datasets:
+- generator
+model-index:
+- name: gemma7b-kasa-coding-11-v1
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# gemma7b-kasa-coding-11-v1
+
+This model is a fine-tuned version of [google/gemma-7b](https://huggingface.co/google/gemma-7b) on the generator dataset.
+It achieves the following results on the evaluation set:
+- Loss: 9.9363
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0002
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 128
+- total_eval_batch_size: 64
+- optimizer: Use adamw_torch with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss |
+|:-------------:|:-----:|:----:|:---------------:|
+| 10.3696       | 1.0   | 140  | 9.9363          |
+
+
+### Framework versions
+
+- PEFT 0.13.1.dev0
+- Transformers 4.46.2
+- Pytorch 2.5.1+cu124
+- Datasets 3.1.0
+- Tokenizers 0.20.3

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 4.268850850782249e+17,
+    "train_loss": 14.163321549551828,
+    "train_runtime": 384.2161,
+    "train_samples": 51241,
+    "train_samples_per_second": 46.599,
+    "train_steps_per_second": 0.364
+}

runs/Nov18_16-49-44_bold-food-flourishes-fin-02/events.out.tfevents.1731950056.bold-food-flourishes-fin-02.72110.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b66d68b80a2325a4751cf94d94d2c13cc725dbf083491c749c7a645e9c0a4a06
-size 11737
+oid sha256:08dfef669035b29af38b724a9d0a9860238cf054abca4afa72601b8936919fbd
+size 12362

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 4.268850850782249e+17,
+    "train_loss": 14.163321549551828,
+    "train_runtime": 384.2161,
+    "train_samples": 51241,
+    "train_samples_per_second": 46.599,
+    "train_steps_per_second": 0.364
+}

trainer_state.json

@@ -0,0 +1,253 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 140,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.007142857142857143,
+      "grad_norm": 1138.7742919921875,
+      "learning_rate": 1.4285714285714285e-05,
+      "loss": 48.0816,
+      "step": 1
+    },
+    {
+      "epoch": 0.03571428571428571,
+      "grad_norm": 242.92247009277344,
+      "learning_rate": 7.142857142857143e-05,
+      "loss": 38.2264,
+      "step": 5
+    },
+    {
+      "epoch": 0.07142857142857142,
+      "grad_norm": 99.10001373291016,
+      "learning_rate": 0.00014285714285714287,
+      "loss": 26.8907,
+      "step": 10
+    },
+    {
+      "epoch": 0.10714285714285714,
+      "grad_norm": 51.8281364440918,
+      "learning_rate": 0.00019996891820008164,
+      "loss": 23.0483,
+      "step": 15
+    },
+    {
+      "epoch": 0.14285714285714285,
+      "grad_norm": 16.425758361816406,
+      "learning_rate": 0.00019888308262251285,
+      "loss": 20.2011,
+      "step": 20
+    },
+    {
+      "epoch": 0.17857142857142858,
+      "grad_norm": 8.041358947753906,
+      "learning_rate": 0.0001962624246950012,
+      "loss": 18.1936,
+      "step": 25
+    },
+    {
+      "epoch": 0.21428571428571427,
+      "grad_norm": 5.986516952514648,
+      "learning_rate": 0.00019214762118704076,
+      "loss": 16.4036,
+      "step": 30
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 3.8124237060546875,
+      "learning_rate": 0.00018660254037844388,
+      "loss": 15.1102,
+      "step": 35
+    },
+    {
+      "epoch": 0.2857142857142857,
+      "grad_norm": 3.229938268661499,
+      "learning_rate": 0.00017971325072229226,
+      "loss": 14.0912,
+      "step": 40
+    },
+    {
+      "epoch": 0.32142857142857145,
+      "grad_norm": 3.9065780639648438,
+      "learning_rate": 0.00017158668492597186,
+      "loss": 13.3336,
+      "step": 45
+    },
+    {
+      "epoch": 0.35714285714285715,
+      "grad_norm": 4.85410213470459,
+      "learning_rate": 0.00016234898018587337,
+      "loss": 12.878,
+      "step": 50
+    },
+    {
+      "epoch": 0.39285714285714285,
+      "grad_norm": 4.169612884521484,
+      "learning_rate": 0.0001521435203379498,
+      "loss": 12.3397,
+      "step": 55
+    },
+    {
+      "epoch": 0.42857142857142855,
+      "grad_norm": 2.173008441925049,
+      "learning_rate": 0.00014112871031306119,
+      "loss": 11.9716,
+      "step": 60
+    },
+    {
+      "epoch": 0.4642857142857143,
+      "grad_norm": 2.008362054824829,
+      "learning_rate": 0.00012947551744109043,
+      "loss": 11.6934,
+      "step": 65
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 3.3055272102355957,
+      "learning_rate": 0.00011736481776669306,
+      "loss": 11.4663,
+      "step": 70
+    },
+    {
+      "epoch": 0.5357142857142857,
+      "grad_norm": 1.9420216083526611,
+      "learning_rate": 0.00010498458856606972,
+      "loss": 11.2446,
+      "step": 75
+    },
+    {
+      "epoch": 0.5714285714285714,
+      "grad_norm": 1.257276177406311,
+      "learning_rate": 9.252699064135758e-05,
+      "loss": 11.0546,
+      "step": 80
+    },
+    {
+      "epoch": 0.6071428571428571,
+      "grad_norm": 1.5491865873336792,
+      "learning_rate": 8.018538568006027e-05,
+      "loss": 10.8896,
+      "step": 85
+    },
+    {
+      "epoch": 0.6428571428571429,
+      "grad_norm": 21.172082901000977,
+      "learning_rate": 6.815133497483157e-05,
+      "loss": 10.7927,
+      "step": 90
+    },
+    {
+      "epoch": 0.6785714285714286,
+      "grad_norm": 1.501454472541809,
+      "learning_rate": 5.6611626088244194e-05,
+      "loss": 10.7241,
+      "step": 95
+    },
+    {
+      "epoch": 0.7142857142857143,
+      "grad_norm": 1.7851982116699219,
+      "learning_rate": 4.574537361342407e-05,
+      "loss": 10.5994,
+      "step": 100
+    },
+    {
+      "epoch": 0.75,
+      "grad_norm": 2.065108299255371,
+      "learning_rate": 3.5721239031346066e-05,
+      "loss": 10.5151,
+      "step": 105
+    },
+    {
+      "epoch": 0.7857142857142857,
+      "grad_norm": 1.3877239227294922,
+      "learning_rate": 2.669481281701739e-05,
+      "loss": 10.5179,
+      "step": 110
+    },
+    {
+      "epoch": 0.8214285714285714,
+      "grad_norm": 1.3329989910125732,
+      "learning_rate": 1.880619942841435e-05,
+      "loss": 10.4364,
+      "step": 115
+    },
+    {
+      "epoch": 0.8571428571428571,
+      "grad_norm": 1.767424464225769,
+      "learning_rate": 1.2177842662977135e-05,
+      "loss": 10.4225,
+      "step": 120
+    },
+    {
+      "epoch": 0.8928571428571429,
+      "grad_norm": 1.3661062717437744,
+      "learning_rate": 6.9126251355795864e-06,
+      "loss": 10.4211,
+      "step": 125
+    },
+    {
+      "epoch": 0.9285714285714286,
+      "grad_norm": 1.1075615882873535,
+      "learning_rate": 3.092271377092215e-06,
+      "loss": 10.3691,
+      "step": 130
+    },
+    {
+      "epoch": 0.9642857142857143,
+      "grad_norm": 1.3276323080062866,
+      "learning_rate": 7.760793399827937e-07,
+      "loss": 10.3975,
+      "step": 135
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 1.1502283811569214,
+      "learning_rate": 0.0,
+      "loss": 10.3696,
+      "step": 140
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 9.936275482177734,
+      "eval_runtime": 1.4505,
+      "eval_samples_per_second": 122.712,
+      "eval_steps_per_second": 2.068,
+      "step": 140
+    },
+    {
+      "epoch": 1.0,
+      "step": 140,
+      "total_flos": 4.268850850782249e+17,
+      "train_loss": 14.163321549551828,
+      "train_runtime": 384.2161,
+      "train_samples_per_second": 46.599,
+      "train_steps_per_second": 0.364
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 140,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 4.268850850782249e+17,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}