Model save

README.md

@@ -0,0 +1,69 @@
+---
+library_name: peft
+license: llama3
+base_model: meta-llama/Meta-Llama-3-8B
+tags:
+- trl
+- sft
+- generated_from_trainer
+datasets:
+- generator
+model-index:
+- name: llama3-8b-lora-summarization-11-v1
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# llama3-8b-lora-summarization-11-v1
+
+This model is a fine-tuned version of [meta-llama/Meta-Llama-3-8B](https://huggingface.co/meta-llama/Meta-Llama-3-8B) on the generator dataset.
+It achieves the following results on the evaluation set:
+- Loss: 2.3717
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0002
+- train_batch_size: 16
+- eval_batch_size: 16
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 256
+- total_eval_batch_size: 128
+- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss |
+|:-------------:|:------:|:----:|:---------------:|
+| 1.6835        | 0.9955 | 111  | 2.3717          |
+
+
+### Framework versions
+
+- PEFT 0.13.2
+- Transformers 4.46.2
+- Pytorch 2.3.1+cu121
+- Datasets 3.1.0
+- Tokenizers 0.20.3

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9955156950672646,
+    "total_flos": 6.554327132594504e+17,
+    "train_loss": 1.8387699009061933,
+    "train_runtime": 1914.8973,
+    "train_samples": 129221,
+    "train_samples_per_second": 14.857,
+    "train_steps_per_second": 0.058
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9955156950672646,
+    "total_flos": 6.554327132594504e+17,
+    "train_loss": 1.8387699009061933,
+    "train_runtime": 1914.8973,
+    "train_samples": 129221,
+    "train_samples_per_second": 14.857,
+    "train_steps_per_second": 0.058
+}

trainer_state.json

@@ -0,0 +1,211 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9955156950672646,
+  "eval_steps": 500,
+  "global_step": 111,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.008968609865470852,
+      "grad_norm": 1.1094273328781128,
+      "learning_rate": 1.6666666666666667e-05,
+      "loss": 2.4851,
+      "step": 1
+    },
+    {
+      "epoch": 0.04484304932735426,
+      "grad_norm": 0.9243674278259277,
+      "learning_rate": 8.333333333333334e-05,
+      "loss": 2.4898,
+      "step": 5
+    },
+    {
+      "epoch": 0.08968609865470852,
+      "grad_norm": 0.6846497654914856,
+      "learning_rate": 0.0001666666666666667,
+      "loss": 2.4067,
+      "step": 10
+    },
+    {
+      "epoch": 0.13452914798206278,
+      "grad_norm": 0.8646454811096191,
+      "learning_rate": 0.00019954719225730847,
+      "loss": 2.2437,
+      "step": 15
+    },
+    {
+      "epoch": 0.17937219730941703,
+      "grad_norm": 0.659982442855835,
+      "learning_rate": 0.00019679487013963564,
+      "loss": 2.0573,
+      "step": 20
+    },
+    {
+      "epoch": 0.2242152466367713,
+      "grad_norm": 0.579827070236206,
+      "learning_rate": 0.00019161084574320696,
+      "loss": 1.9415,
+      "step": 25
+    },
+    {
+      "epoch": 0.26905829596412556,
+      "grad_norm": 0.3958710730075836,
+      "learning_rate": 0.00018412535328311814,
+      "loss": 1.868,
+      "step": 30
+    },
+    {
+      "epoch": 0.31390134529147984,
+      "grad_norm": 0.2995990514755249,
+      "learning_rate": 0.0001745264449675755,
+      "loss": 1.814,
+      "step": 35
+    },
+    {
+      "epoch": 0.35874439461883406,
+      "grad_norm": 0.2901516556739807,
+      "learning_rate": 0.00016305526670845226,
+      "loss": 1.7888,
+      "step": 40
+    },
+    {
+      "epoch": 0.40358744394618834,
+      "grad_norm": 0.27101069688796997,
+      "learning_rate": 0.00015000000000000001,
+      "loss": 1.759,
+      "step": 45
+    },
+    {
+      "epoch": 0.4484304932735426,
+      "grad_norm": 0.252288281917572,
+      "learning_rate": 0.00013568862215918717,
+      "loss": 1.7476,
+      "step": 50
+    },
+    {
+      "epoch": 0.49327354260089684,
+      "grad_norm": 0.24804328382015228,
+      "learning_rate": 0.00012048066680651908,
+      "loss": 1.738,
+      "step": 55
+    },
+    {
+      "epoch": 0.5381165919282511,
+      "grad_norm": 0.25213855504989624,
+      "learning_rate": 0.00010475819158237425,
+      "loss": 1.7199,
+      "step": 60
+    },
+    {
+      "epoch": 0.5829596412556054,
+      "grad_norm": 0.25072190165519714,
+      "learning_rate": 8.891618000989891e-05,
+      "loss": 1.7053,
+      "step": 65
+    },
+    {
+      "epoch": 0.6278026905829597,
+      "grad_norm": 0.24229365587234497,
+      "learning_rate": 7.335261863099651e-05,
+      "loss": 1.708,
+      "step": 70
+    },
+    {
+      "epoch": 0.672645739910314,
+      "grad_norm": 0.245011568069458,
+      "learning_rate": 5.845849869981137e-05,
+      "loss": 1.6943,
+      "step": 75
+    },
+    {
+      "epoch": 0.7174887892376681,
+      "grad_norm": 0.25127002596855164,
+      "learning_rate": 4.4607993613388976e-05,
+      "loss": 1.6854,
+      "step": 80
+    },
+    {
+      "epoch": 0.7623318385650224,
+      "grad_norm": 0.25898587703704834,
+      "learning_rate": 3.21490588442868e-05,
+      "loss": 1.6834,
+      "step": 85
+    },
+    {
+      "epoch": 0.8071748878923767,
+      "grad_norm": 0.2714327871799469,
+      "learning_rate": 2.139469052572127e-05,
+      "loss": 1.6902,
+      "step": 90
+    },
+    {
+      "epoch": 0.852017937219731,
+      "grad_norm": 0.22857919335365295,
+      "learning_rate": 1.2615062293021507e-05,
+      "loss": 1.6917,
+      "step": 95
+    },
+    {
+      "epoch": 0.8968609865470852,
+      "grad_norm": 0.2975883185863495,
+      "learning_rate": 6.030737921409169e-06,
+      "loss": 1.6887,
+      "step": 100
+    },
+    {
+      "epoch": 0.9417040358744395,
+      "grad_norm": 0.2528745234012604,
+      "learning_rate": 1.8071302737293295e-06,
+      "loss": 1.6853,
+      "step": 105
+    },
+    {
+      "epoch": 0.9865470852017937,
+      "grad_norm": 0.2566002309322357,
+      "learning_rate": 5.0345761681491746e-08,
+      "loss": 1.6835,
+      "step": 110
+    },
+    {
+      "epoch": 0.9955156950672646,
+      "eval_loss": 2.371652841567993,
+      "eval_runtime": 1.116,
+      "eval_samples_per_second": 18.816,
+      "eval_steps_per_second": 0.896,
+      "step": 111
+    },
+    {
+      "epoch": 0.9955156950672646,
+      "step": 111,
+      "total_flos": 6.554327132594504e+17,
+      "train_loss": 1.8387699009061933,
+      "train_runtime": 1914.8973,
+      "train_samples_per_second": 14.857,
+      "train_steps_per_second": 0.058
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 111,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 6.554327132594504e+17,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}