Model save

README.md

@@ -0,0 +1,69 @@
+---
+library_name: peft
+license: apache-2.0
+base_model: mistralai/Mistral-7B-v0.3
+tags:
+- trl
+- sft
+- generated_from_trainer
+datasets:
+- generator
+model-index:
+- name: mistral7b-lora-classification-11-v1
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# mistral7b-lora-classification-11-v1
+
+This model is a fine-tuned version of [mistralai/Mistral-7B-v0.3](https://huggingface.co/mistralai/Mistral-7B-v0.3) on the generator dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.9204
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0002
+- train_batch_size: 14
+- eval_batch_size: 14
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 224
+- total_eval_batch_size: 112
+- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss |
+|:-------------:|:------:|:----:|:---------------:|
+| 1.7207        | 0.9936 | 78   | 1.9204          |
+
+
+### Framework versions
+
+- PEFT 0.13.2
+- Transformers 4.46.2
+- Pytorch 2.3.1+cu121
+- Datasets 3.1.0
+- Tokenizers 0.20.3

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9936305732484076,
+    "total_flos": 3.820096642099446e+17,
+    "train_loss": 1.8799311350553463,
+    "train_runtime": 1320.5477,
+    "train_samples": 92634,
+    "train_samples_per_second": 13.248,
+    "train_steps_per_second": 0.059
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9936305732484076,
+    "total_flos": 3.820096642099446e+17,
+    "train_loss": 1.8799311350553463,
+    "train_runtime": 1320.5477,
+    "train_samples": 92634,
+    "train_samples_per_second": 13.248,
+    "train_steps_per_second": 0.059
+}

trainer_state.json

@@ -0,0 +1,162 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9936305732484076,
+  "eval_steps": 500,
+  "global_step": 78,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.012738853503184714,
+      "grad_norm": 4.837225437164307,
+      "learning_rate": 2.5e-05,
+      "loss": 2.5854,
+      "step": 1
+    },
+    {
+      "epoch": 0.06369426751592357,
+      "grad_norm": 3.6128976345062256,
+      "learning_rate": 0.000125,
+      "loss": 2.5321,
+      "step": 5
+    },
+    {
+      "epoch": 0.12738853503184713,
+      "grad_norm": 2.7008352279663086,
+      "learning_rate": 0.00019959742939952392,
+      "loss": 2.3462,
+      "step": 10
+    },
+    {
+      "epoch": 0.1910828025477707,
+      "grad_norm": 2.4735682010650635,
+      "learning_rate": 0.00019510565162951537,
+      "loss": 2.1254,
+      "step": 15
+    },
+    {
+      "epoch": 0.25477707006369427,
+      "grad_norm": 1.8116168975830078,
+      "learning_rate": 0.00018584487936018661,
+      "loss": 1.9483,
+      "step": 20
+    },
+    {
+      "epoch": 0.3184713375796178,
+      "grad_norm": 1.1086759567260742,
+      "learning_rate": 0.00017227948638273916,
+      "loss": 1.8338,
+      "step": 25
+    },
+    {
+      "epoch": 0.3821656050955414,
+      "grad_norm": 1.007157802581787,
+      "learning_rate": 0.00015508969814521025,
+      "loss": 1.8085,
+      "step": 30
+    },
+    {
+      "epoch": 0.445859872611465,
+      "grad_norm": 0.9624175429344177,
+      "learning_rate": 0.0001351374824081343,
+      "loss": 1.7777,
+      "step": 35
+    },
+    {
+      "epoch": 0.5095541401273885,
+      "grad_norm": 1.133021593093872,
+      "learning_rate": 0.00011342332658176555,
+      "loss": 1.7611,
+      "step": 40
+    },
+    {
+      "epoch": 0.5732484076433121,
+      "grad_norm": 0.9674370884895325,
+      "learning_rate": 9.103606910965666e-05,
+      "loss": 1.7547,
+      "step": 45
+    },
+    {
+      "epoch": 0.6369426751592356,
+      "grad_norm": 0.9040862321853638,
+      "learning_rate": 6.909830056250527e-05,
+      "loss": 1.7319,
+      "step": 50
+    },
+    {
+      "epoch": 0.7006369426751592,
+      "grad_norm": 0.8982458710670471,
+      "learning_rate": 4.87100722594094e-05,
+      "loss": 1.7384,
+      "step": 55
+    },
+    {
+      "epoch": 0.7643312101910829,
+      "grad_norm": 0.8207002878189087,
+      "learning_rate": 3.089373510131354e-05,
+      "loss": 1.7354,
+      "step": 60
+    },
+    {
+      "epoch": 0.8280254777070064,
+      "grad_norm": 0.8674971461296082,
+      "learning_rate": 1.6542674627869737e-05,
+      "loss": 1.7322,
+      "step": 65
+    },
+    {
+      "epoch": 0.89171974522293,
+      "grad_norm": 0.8455283641815186,
+      "learning_rate": 6.37651293602628e-06,
+      "loss": 1.7258,
+      "step": 70
+    },
+    {
+      "epoch": 0.9554140127388535,
+      "grad_norm": 0.7830201387405396,
+      "learning_rate": 9.0502382320653e-07,
+      "loss": 1.7207,
+      "step": 75
+    },
+    {
+      "epoch": 0.9936305732484076,
+      "eval_loss": 1.9203951358795166,
+      "eval_runtime": 1.4459,
+      "eval_samples_per_second": 2.766,
+      "eval_steps_per_second": 0.692,
+      "step": 78
+    },
+    {
+      "epoch": 0.9936305732484076,
+      "step": 78,
+      "total_flos": 3.820096642099446e+17,
+      "train_loss": 1.8799311350553463,
+      "train_runtime": 1320.5477,
+      "train_samples_per_second": 13.248,
+      "train_steps_per_second": 0.059
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 78,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.820096642099446e+17,
+  "train_batch_size": 14,
+  "trial_name": null,
+  "trial_params": null
+}