cpt core 4

README.md

@@ -400,9 +400,9 @@ litgpt convert_pretrained_checkpoint ../out/pretrain-core-3/final ../out/pretrai
 ```
 
 ```bash
-litgpt convert_from_litgpt ../out/pretrain-core-3/final ../out/pretrain-core-3/hf
-cp ../config-3.json ../out/pretrain-core-3/hf/config.json
-cp -rv ../tokenizer/* ../out/pretrain-core-3/hf
+litgpt convert_from_litgpt ../out/pretrain-core-3/final ../out/cpt-core-pre-4
+cp ../config-4.json ../out/cpt-core-pre-4/config.json
+cp -rv ../tokenizer/* ../out/cpt-core-pre-4
 python -B convert_pth_to_safetensors.py
 ```
 

scripts/cpt_core_model_4.py

@@ -13,7 +13,7 @@ dataset_block_size = 16385
 max_seq_length = 16385
 dtype = torch.bfloat16
 load_in_4bit = False
-model_name = '../out/pretrain-core-3/hf'
+model_name = '../out/cpt-core-pre-4'
 output_dir = '../out/cpt-core-4'
 
 

scripts/requirements-litgpt.in

@@ -16,12 +16,3 @@ torchao
 sophia-opt
 # dolphinflow
 lm_eval[ifeval,math]
-
-
-# unsloth
-unsloth
-xformers
-trl
-peft
-accelerate
-bitsandbytes