cpt core 4

scripts/cpt_core_model_4.py

@@ -33,14 +33,16 @@ model, tokenizer = FastLanguageModel.from_pretrained(
 
 model = FastLanguageModel.get_peft_model(
     model,
-    r=256, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
+    # r=256, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
+    r=16, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
     target_modules=[
         "q_proj", "k_proj", "v_proj", "o_proj",
         "gate_proj",
         "up_proj", "down_proj",
         "embed_tokens", "lm_head",
     ],
-    lora_alpha=32,
+    # lora_alpha=32,
+    lora_alpha=16,
     lora_dropout=0, # Supports any, but = 0 is optimized
     bias="none",    # Supports any, but = "none" is optimized
     # [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes!
@@ -73,6 +75,7 @@ def unlsoth_generator():
 
 
 train_dataset = Dataset.from_generator(unlsoth_generator)
+dataset = train_dataset.train_test_split(test_size=0.01)
 
 #
 # trainer
@@ -86,9 +89,11 @@ from unsloth import UnslothTrainer, UnslothTrainingArguments
 trainer = UnslothTrainer(
     model=model,
     tokenizer=tokenizer,
-    train_dataset=train_dataset,
-    max_seq_length=max_seq_length,
+    # train_dataset=train_dataset,
+    train_dataset=dataset['train'],
+    eval_dataset=dataset['test'],
     dataset_num_proc=32,
+    max_seq_length=max_seq_length,
     max_steps=len(litgpt_streaming_dataset),
     packing=False, # Can make training 5x faster for short sequences.
 
@@ -112,11 +117,16 @@ trainer = UnslothTrainer(
         seed=23,
         output_dir=output_dir,
         report_to='wandb',
+
+        save_steps=100,
         run_name=run_name,
 
         do_eval=True,
-        save_steps=100,
-        eval_steps=100,
+        fp16_full_eval=True,
+        per_device_eval_batch_size=2,
+        eval_accumulation_steps=4,
+        eval_strategy='steps',
+        eval_steps=10,
     ),
 )