Hugging Face's logo

unites-lab
/
pythia-70m-moepp-6e-20k

Model card Files
xet
Community
Pingzhi Li commited on
Commit
80de0a6
·
1 Parent(s): 64ffc85

init

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. global_step0/configs/pythia-70m-moe++-6e.yml +110 -0
  2. global_step0/layer_00-model_00-model_states.pt +3 -0
  3. global_step0/layer_02-model_00-model_states.pt +3 -0
  4. global_step0/layer_03-model_00-model_states.pt +3 -0
  5. global_step0/layer_04-model_00-model_states.pt +3 -0
  6. global_step0/layer_05-model_00-model_states.pt +3 -0
  7. global_step0/layer_06-model_00-model_states.pt +3 -0
  8. global_step0/layer_07-model_00-model_states.pt +3 -0
  9. global_step0/layer_09-model_00-model_states.pt +3 -0
  10. global_step0/layer_10-model_00-model_states.pt +3 -0
  11. global_step0/mp_rank_00_model_states.pt +3 -0
  12. global_step0/zero_pp_rank_0_mp_rank_00_optim_states.pt +3 -0
  13. global_step0/zero_pp_rank_1_mp_rank_00_optim_states.pt +3 -0
  14. global_step0/zero_pp_rank_2_mp_rank_00_optim_states.pt +3 -0
  15. global_step0/zero_pp_rank_3_mp_rank_00_optim_states.pt +3 -0
  16. global_step0/zero_pp_rank_4_mp_rank_00_optim_states.pt +3 -0
  17. global_step0/zero_pp_rank_5_mp_rank_00_optim_states.pt +3 -0
  18. global_step0/zero_pp_rank_6_mp_rank_00_optim_states.pt +3 -0
  19. global_step0/zero_pp_rank_7_mp_rank_00_optim_states.pt +3 -0
  20. global_step1000/configs/pythia-70m-moe++-6e.yml +110 -0
  21. global_step1000/layer_00-model_00-model_states.pt +3 -0
  22. global_step1000/layer_02-model_00-model_states.pt +3 -0
  23. global_step1000/layer_03-model_00-model_states.pt +3 -0
  24. global_step1000/layer_04-model_00-model_states.pt +3 -0
  25. global_step1000/layer_05-model_00-model_states.pt +3 -0
  26. global_step1000/layer_06-model_00-model_states.pt +3 -0
  27. global_step1000/layer_07-model_00-model_states.pt +3 -0
  28. global_step1000/layer_09-model_00-model_states.pt +3 -0
  29. global_step1000/layer_10-model_00-model_states.pt +3 -0
  30. global_step1000/mp_rank_00_model_states.pt +3 -0
  31. global_step1000/zero_pp_rank_0_mp_rank_00_optim_states.pt +3 -0
  32. global_step1000/zero_pp_rank_1_mp_rank_00_optim_states.pt +3 -0
  33. global_step1000/zero_pp_rank_2_mp_rank_00_optim_states.pt +3 -0
  34. global_step1000/zero_pp_rank_3_mp_rank_00_optim_states.pt +3 -0
  35. global_step1000/zero_pp_rank_4_mp_rank_00_optim_states.pt +3 -0
  36. global_step1000/zero_pp_rank_5_mp_rank_00_optim_states.pt +3 -0
  37. global_step1000/zero_pp_rank_6_mp_rank_00_optim_states.pt +3 -0
  38. global_step1000/zero_pp_rank_7_mp_rank_00_optim_states.pt +3 -0
  39. global_step10000/configs/pythia-70m-moe++-6e.yml +110 -0
  40. global_step10000/layer_00-model_00-model_states.pt +3 -0
  41. global_step10000/layer_02-model_00-model_states.pt +3 -0
  42. global_step10000/layer_03-model_00-model_states.pt +3 -0
  43. global_step10000/layer_04-model_00-model_states.pt +3 -0
  44. global_step10000/layer_05-model_00-model_states.pt +3 -0
  45. global_step10000/layer_06-model_00-model_states.pt +3 -0
  46. global_step10000/layer_07-model_00-model_states.pt +3 -0
  47. global_step10000/layer_09-model_00-model_states.pt +3 -0
  48. global_step10000/layer_10-model_00-model_states.pt +3 -0
  49. global_step10000/mp_rank_00_model_states.pt +3 -0
  50. global_step10000/zero_pp_rank_0_mp_rank_00_optim_states.pt +3 -0
global_step0/configs/pythia-70m-moe++-6e.yml ADDED
@@ -0,0 +1,110 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ # parallelism settings
3
+ "pipe-parallel-size": 1,
4
+ "model-parallel-size": 1,
5
+
6
+ # model settings
7
+ "num-layers": 6,
8
+ "hidden-size": 512,
9
+ "num-attention-heads": 8,
10
+ "seq-length": 2048,
11
+ "max-position-embeddings": 2048,
12
+ "pos-emb": "rotary",
13
+ "rotary-pct": 0.25,
14
+ "no-weight-tying": true,
15
+ "gpt-j-residual": true,
16
+ "output-layer-parallelism": "column",
17
+
18
+ "attention-config": [[["moe++"], 6]],
19
+ "attention-num-experts": 6,
20
+
21
+ "scaled-upper-triang-masked-softmax-fusion": true,
22
+ "bias-gelu-fusion": true,
23
+
24
+ # init methods
25
+ "init_method": "small_init",
26
+ "output_layer_init_method": "wang_init",
27
+
28
+ "optimizer": {
29
+ "type": "Adam",
30
+ "params": {
31
+ "lr": 0.001,
32
+ "betas": [0.9, 0.95],
33
+ "eps": 1.0e-8
34
+ }
35
+ },
36
+ "min_lr": 0.0001,
37
+
38
+ "zero_optimization": {
39
+ "stage": 1,
40
+ "allgather_partitions": true,
41
+ "allgather_bucket_size": 500000000,
42
+ "overlap_comm": true,
43
+ "reduce_scatter": true,
44
+ "reduce_bucket_size": 500000000,
45
+ "contiguous_gradients": true,
46
+ "cpu_offload": false
47
+ },
48
+
49
+ "train_micro_batch_size_per_gpu": 32,
50
+ "gradient_accumulation_steps": 4,
51
+ "data-impl": "mmap",
52
+ "num_workers": 4,
53
+
54
+ # activation checkpointing
55
+ "checkpoint-activations": true,
56
+ "checkpoint-num-layers": 1,
57
+ "partition-activations": true,
58
+ "synchronize-each-layer": true,
59
+
60
+ # regularization
61
+ "gradient_clipping": 1.0,
62
+ "weight-decay": 0.1,
63
+ "hidden-dropout": 0,
64
+ "attention-dropout": 0,
65
+
66
+ # precision settings
67
+ "fp16": {
68
+ "fp16": true,
69
+ "enabled": true,
70
+ "loss_scale": 0,
71
+ "loss_scale_window": 1000,
72
+ "initial_scale_power": 12,
73
+ "hysteresis": 2,
74
+ "min_loss_scale": 1
75
+ },
76
+
77
+ # "train-iters": 143000,
78
+ # "lr-decay-iters": 143000,
79
+ "train-iters": 20000,
80
+ "lr-decay-iters": 20000,
81
+ "distributed-backend": "nccl",
82
+ "lr-decay-style": "cosine",
83
+ "warmup": 0.01,
84
+ "checkpoint-factor": 1000,
85
+ "extra-save-iters": [0,2,128,512],
86
+ "eval-interval": 100000,
87
+ "eval-iters": 10,
88
+
89
+ "log-interval": 10,
90
+ "steps_per_print": 10,
91
+ "wall_clock_breakdown": true,
92
+
93
+ # "train-data-paths": [ "/data4/pzli/pythia_pile_idxmaps/pile_0.87_deduped_text_document" ],
94
+ # "valid-data-paths": [ "/data4/pzli/pythia_pile_idxmaps/pile_0.87_deduped_text_document" ],
95
+ # "test-data-paths": [ "/data4/pzli/pythia_pile_idxmaps/pile_0.87_deduped_text_document" ],
96
+ "train-data-paths": [ "../datasets/pythia/pythia_pile_idxmaps/pile_0.87_deduped_text_document" ],
97
+ "valid-data-paths": [ "../datasets/pythia/pythia_pile_idxmaps/pile_0.87_deduped_text_document" ],
98
+ "test-data-paths": [ "../datasets/pythia/pythia_pile_idxmaps/pile_0.87_deduped_text_document" ],
99
+ "save": "checkpoints/pythia-70m-moe++-6e-20k",
100
+
101
+ "tokenizer-type": "HFTokenizer",
102
+ # "vocab-file": "/data2/pzli/pythia/utils/20B_tokenizer.json",
103
+ "vocab-file": "../datasets/pythia/utils/20B_tokenizer.json",
104
+
105
+ "deepspeed_mpi": false,
106
+
107
+ "use_wandb": true,
108
+ "wandb_project": "pythia",
109
+ "wandb_group": "pythia-70m-moe++-4e-20k",
110
+ }
global_step0/layer_00-model_00-model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1ea2f539fc34e02a5784e1ffa4c4ada9a48f1df049dabd5410cdc29517160d44
3
+ size 51512719
global_step0/layer_02-model_00-model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:beb2ae13c645bee68b01889bbd7fc5a5c79ca3e369087fccdce77fb9084b1dc1
3
+ size 8423690
global_step0/layer_03-model_00-model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f18d831059de4364de92f13d041bb936fe9b4149f89863ce6612b92750b1b14f
3
+ size 8423690
global_step0/layer_04-model_00-model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:64bdbcaeacc54f1660a5e87392f02c05d294b2fc14bc598bdc395cebafcbfd70
3
+ size 8423690
global_step0/layer_05-model_00-model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dd55adc84c181c09d944b589da4be1d1ed2beeb1ad7c8e39ac538afc9f8dea1d
3
+ size 8423690
global_step0/layer_06-model_00-model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ed26ba85457ac75e22138eb612f896d243d80c4d0a36f33551a7aa916000a06e
3
+ size 8423690
global_step0/layer_07-model_00-model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7a9d986380bde40812058c545323932a828755bbbbb8841d535471f2379447b9
3
+ size 8423690
global_step0/layer_09-model_00-model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4d61da49f99c1cdc8a883434eaaed658cc6a6da1a86487b497899fafeba334ab
3
+ size 3746
global_step0/layer_10-model_00-model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7705d8d6789854908ac52ceb828ba5ef9d8dcdbd23bc5466aec35b694abeccbe
3
+ size 51512719
global_step0/mp_rank_00_model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:09eeb8a74e71ee1c206aad7a1c27a6eb018accb656f47e09c0ffdbd1d8dbc894
3
+ size 27716
global_step0/zero_pp_rank_0_mp_rank_00_optim_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8c71ab54f522149688075dc7c4e2e7f53766c132fba1f3faee6de1bcf88eb407
3
+ size 115135102
global_step0/zero_pp_rank_1_mp_rank_00_optim_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a190efc946554ecf0a5236d8e9c563a84daf627eae2aac7a05be9e4c66afab2f
3
+ size 115134974
global_step0/zero_pp_rank_2_mp_rank_00_optim_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:41369f203247ab4e649ce3bffb8b4e37d0e55c8de0fd1e1805e8b5f5e9be50de
3
+ size 115135806
global_step0/zero_pp_rank_3_mp_rank_00_optim_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f1cbf7113dba9cf3090c870fffa3f40991068b50496273cf49505031ffb8bba9
3
+ size 115136958
global_step0/zero_pp_rank_4_mp_rank_00_optim_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dea1df6df95ca37c4f366d62a7bbac4662c35def4f275f55dd5ddf756a227f5e
3
+ size 115137662
global_step0/zero_pp_rank_5_mp_rank_00_optim_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0d786660b3bf39a7afcbffb22fd9deb328a7d0e3ee64bad37ab19d353a7c3c77
3
+ size 115135486
global_step0/zero_pp_rank_6_mp_rank_00_optim_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0262c21e6dfdeb891925a4ac438f92e62bbeef4962af9d2d858606b9aa8e3616
3
+ size 115134846
global_step0/zero_pp_rank_7_mp_rank_00_optim_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3c55488e3349b097337efcf01a6353644d7a4c7b14c516bae3e4d2b5e4e69dc0
3
+ size 115134846
global_step1000/configs/pythia-70m-moe++-6e.yml ADDED
@@ -0,0 +1,110 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ # parallelism settings
3
+ "pipe-parallel-size": 1,
4
+ "model-parallel-size": 1,
5
+
6
+ # model settings
7
+ "num-layers": 6,
8
+ "hidden-size": 512,
9
+ "num-attention-heads": 8,
10
+ "seq-length": 2048,
11
+ "max-position-embeddings": 2048,
12
+ "pos-emb": "rotary",
13
+ "rotary-pct": 0.25,
14
+ "no-weight-tying": true,
15
+ "gpt-j-residual": true,
16
+ "output-layer-parallelism": "column",
17
+
18
+ "attention-config": [[["moe++"], 6]],
19
+ "attention-num-experts": 6,
20
+
21
+ "scaled-upper-triang-masked-softmax-fusion": true,
22
+ "bias-gelu-fusion": true,
23
+
24
+ # init methods
25
+ "init_method": "small_init",
26
+ "output_layer_init_method": "wang_init",
27
+
28
+ "optimizer": {
29
+ "type": "Adam",
30
+ "params": {
31
+ "lr": 0.001,
32
+ "betas": [0.9, 0.95],
33
+ "eps": 1.0e-8
34
+ }
35
+ },
36
+ "min_lr": 0.0001,
37
+
38
+ "zero_optimization": {
39
+ "stage": 1,
40
+ "allgather_partitions": true,
41
+ "allgather_bucket_size": 500000000,
42
+ "overlap_comm": true,
43
+ "reduce_scatter": true,
44
+ "reduce_bucket_size": 500000000,
45
+ "contiguous_gradients": true,
46
+ "cpu_offload": false
47
+ },
48
+
49
+ "train_micro_batch_size_per_gpu": 32,
50
+ "gradient_accumulation_steps": 4,
51
+ "data-impl": "mmap",
52
+ "num_workers": 4,
53
+
54
+ # activation checkpointing
55
+ "checkpoint-activations": true,
56
+ "checkpoint-num-layers": 1,
57
+ "partition-activations": true,
58
+ "synchronize-each-layer": true,
59
+
60
+ # regularization
61
+ "gradient_clipping": 1.0,
62
+ "weight-decay": 0.1,
63
+ "hidden-dropout": 0,
64
+ "attention-dropout": 0,
65
+
66
+ # precision settings
67
+ "fp16": {
68
+ "fp16": true,
69
+ "enabled": true,
70
+ "loss_scale": 0,
71
+ "loss_scale_window": 1000,
72
+ "initial_scale_power": 12,
73
+ "hysteresis": 2,
74
+ "min_loss_scale": 1
75
+ },
76
+
77
+ # "train-iters": 143000,
78
+ # "lr-decay-iters": 143000,
79
+ "train-iters": 20000,
80
+ "lr-decay-iters": 20000,
81
+ "distributed-backend": "nccl",
82
+ "lr-decay-style": "cosine",
83
+ "warmup": 0.01,
84
+ "checkpoint-factor": 1000,
85
+ "extra-save-iters": [0,2,128,512],
86
+ "eval-interval": 100000,
87
+ "eval-iters": 10,
88
+
89
+ "log-interval": 10,
90
+ "steps_per_print": 10,
91
+ "wall_clock_breakdown": true,
92
+
93
+ # "train-data-paths": [ "/data4/pzli/pythia_pile_idxmaps/pile_0.87_deduped_text_document" ],
94
+ # "valid-data-paths": [ "/data4/pzli/pythia_pile_idxmaps/pile_0.87_deduped_text_document" ],
95
+ # "test-data-paths": [ "/data4/pzli/pythia_pile_idxmaps/pile_0.87_deduped_text_document" ],
96
+ "train-data-paths": [ "../datasets/pythia/pythia_pile_idxmaps/pile_0.87_deduped_text_document" ],
97
+ "valid-data-paths": [ "../datasets/pythia/pythia_pile_idxmaps/pile_0.87_deduped_text_document" ],
98
+ "test-data-paths": [ "../datasets/pythia/pythia_pile_idxmaps/pile_0.87_deduped_text_document" ],
99
+ "save": "checkpoints/pythia-70m-moe++-6e-20k",
100
+
101
+ "tokenizer-type": "HFTokenizer",
102
+ # "vocab-file": "/data2/pzli/pythia/utils/20B_tokenizer.json",
103
+ "vocab-file": "../datasets/pythia/utils/20B_tokenizer.json",
104
+
105
+ "deepspeed_mpi": false,
106
+
107
+ "use_wandb": true,
108
+ "wandb_project": "pythia",
109
+ "wandb_group": "pythia-70m-moe++-4e-20k",
110
+ }
global_step1000/layer_00-model_00-model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f7a6b3ceeaf54ec234c3ee07911991d3012ab6580354f48c0e7d15bbfe8fac52
3
+ size 51512719
global_step1000/layer_02-model_00-model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a387624c304a759868f19686cf50a63ad0750d3a245a61af57b293bfda8a0e24
3
+ size 8423690
global_step1000/layer_03-model_00-model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bf9e6ce98f194274d50b21d6f15b63f6b5f6955f936048b4e72564cb5a31d3a9
3
+ size 8423690
global_step1000/layer_04-model_00-model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2767ebc8ca16b0d309057721e7f719266f16bba521bc453a41e1ff5292d080df
3
+ size 8423690
global_step1000/layer_05-model_00-model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:54685834fb774cc5f2d2f7d9ab0e14dc0b494c9bea3a00d17f89ddb7e8eed673
3
+ size 8423690
global_step1000/layer_06-model_00-model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f961deeb71c9565b51280eb9b5c00441d07002cb9a7e0efd205591bef6387355
3
+ size 8423690
global_step1000/layer_07-model_00-model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1b9ff512fef9d981038fed1caa6f5cae491b4a3d24c01dcbb4c324c07a27d190
3
+ size 8423690
global_step1000/layer_09-model_00-model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cebb1f2c868d5b22594543039071cc5ecf9c40b2bfb91048f5aeb923cc74c011
3
+ size 3746
global_step1000/layer_10-model_00-model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bf64d8ab4ebcca38824e86f3815dad5b72305f2f8a66b4615b42e066589d5b36
3
+ size 51512719
global_step1000/mp_rank_00_model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c29a4a0d0734618cdaa91594c4ee4a2020afd5299c18f976b0831a3deb52c72b
3
+ size 27716
global_step1000/zero_pp_rank_0_mp_rank_00_optim_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c190a55c52091ba1a86582941911ec7093381a2d15f1687fa08b60c95df53a13
3
+ size 115135102
global_step1000/zero_pp_rank_1_mp_rank_00_optim_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b0d78c02a83d5c9a4b3d8eb239e4825c5ea8b840c32775e555bf952297e0cf94
3
+ size 115134974
global_step1000/zero_pp_rank_2_mp_rank_00_optim_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:047cad1ef39cc2507e42b1c06ed731a9c53752ab99a2200e1061e52e9bf22d8a
3
+ size 115135806
global_step1000/zero_pp_rank_3_mp_rank_00_optim_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a24918c021d38e7dd663111dd4f846cef91f627ca9db75e8a693cd0d500d58cc
3
+ size 115137022
global_step1000/zero_pp_rank_4_mp_rank_00_optim_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0dd5b39866bb9b4ca5c47fd2e46446541e750740e789e2ba1cefaa8db661be78
3
+ size 115137662
global_step1000/zero_pp_rank_5_mp_rank_00_optim_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d7c8ccaf68150a139486f22fc4ca7a76403277622bf3a2a67263df20d89d6489
3
+ size 115135486
global_step1000/zero_pp_rank_6_mp_rank_00_optim_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5be1b730bea572ab5ffb81cbe8c04ed50712afd385150589e0994d8a57fc24df
3
+ size 115134846
global_step1000/zero_pp_rank_7_mp_rank_00_optim_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:622577f964509dd9c8d53ab0f9f1b9840452b1050b0944bf8c158b8ca511c332
3
+ size 115134846
global_step10000/configs/pythia-70m-moe++-6e.yml ADDED
@@ -0,0 +1,110 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ # parallelism settings
3
+ "pipe-parallel-size": 1,
4
+ "model-parallel-size": 1,
5
+
6
+ # model settings
7
+ "num-layers": 6,
8
+ "hidden-size": 512,
9
+ "num-attention-heads": 8,
10
+ "seq-length": 2048,
11
+ "max-position-embeddings": 2048,
12
+ "pos-emb": "rotary",
13
+ "rotary-pct": 0.25,
14
+ "no-weight-tying": true,
15
+ "gpt-j-residual": true,
16
+ "output-layer-parallelism": "column",
17
+
18
+ "attention-config": [[["moe++"], 6]],
19
+ "attention-num-experts": 6,
20
+
21
+ "scaled-upper-triang-masked-softmax-fusion": true,
22
+ "bias-gelu-fusion": true,
23
+
24
+ # init methods
25
+ "init_method": "small_init",
26
+ "output_layer_init_method": "wang_init",
27
+
28
+ "optimizer": {
29
+ "type": "Adam",
30
+ "params": {
31
+ "lr": 0.001,
32
+ "betas": [0.9, 0.95],
33
+ "eps": 1.0e-8
34
+ }
35
+ },
36
+ "min_lr": 0.0001,
37
+
38
+ "zero_optimization": {
39
+ "stage": 1,
40
+ "allgather_partitions": true,
41
+ "allgather_bucket_size": 500000000,
42
+ "overlap_comm": true,
43
+ "reduce_scatter": true,
44
+ "reduce_bucket_size": 500000000,
45
+ "contiguous_gradients": true,
46
+ "cpu_offload": false
47
+ },
48
+
49
+ "train_micro_batch_size_per_gpu": 32,
50
+ "gradient_accumulation_steps": 4,
51
+ "data-impl": "mmap",
52
+ "num_workers": 4,
53
+
54
+ # activation checkpointing
55
+ "checkpoint-activations": true,
56
+ "checkpoint-num-layers": 1,
57
+ "partition-activations": true,
58
+ "synchronize-each-layer": true,
59
+
60
+ # regularization
61
+ "gradient_clipping": 1.0,
62
+ "weight-decay": 0.1,
63
+ "hidden-dropout": 0,
64
+ "attention-dropout": 0,
65
+
66
+ # precision settings
67
+ "fp16": {
68
+ "fp16": true,
69
+ "enabled": true,
70
+ "loss_scale": 0,
71
+ "loss_scale_window": 1000,
72
+ "initial_scale_power": 12,
73
+ "hysteresis": 2,
74
+ "min_loss_scale": 1
75
+ },
76
+
77
+ # "train-iters": 143000,
78
+ # "lr-decay-iters": 143000,
79
+ "train-iters": 20000,
80
+ "lr-decay-iters": 20000,
81
+ "distributed-backend": "nccl",
82
+ "lr-decay-style": "cosine",
83
+ "warmup": 0.01,
84
+ "checkpoint-factor": 1000,
85
+ "extra-save-iters": [0,2,128,512],
86
+ "eval-interval": 100000,
87
+ "eval-iters": 10,
88
+
89
+ "log-interval": 10,
90
+ "steps_per_print": 10,
91
+ "wall_clock_breakdown": true,
92
+
93
+ # "train-data-paths": [ "/data4/pzli/pythia_pile_idxmaps/pile_0.87_deduped_text_document" ],
94
+ # "valid-data-paths": [ "/data4/pzli/pythia_pile_idxmaps/pile_0.87_deduped_text_document" ],
95
+ # "test-data-paths": [ "/data4/pzli/pythia_pile_idxmaps/pile_0.87_deduped_text_document" ],
96
+ "train-data-paths": [ "../datasets/pythia/pythia_pile_idxmaps/pile_0.87_deduped_text_document" ],
97
+ "valid-data-paths": [ "../datasets/pythia/pythia_pile_idxmaps/pile_0.87_deduped_text_document" ],
98
+ "test-data-paths": [ "../datasets/pythia/pythia_pile_idxmaps/pile_0.87_deduped_text_document" ],
99
+ "save": "checkpoints/pythia-70m-moe++-6e-20k",
100
+
101
+ "tokenizer-type": "HFTokenizer",
102
+ # "vocab-file": "/data2/pzli/pythia/utils/20B_tokenizer.json",
103
+ "vocab-file": "../datasets/pythia/utils/20B_tokenizer.json",
104
+
105
+ "deepspeed_mpi": false,
106
+
107
+ "use_wandb": true,
108
+ "wandb_project": "pythia",
109
+ "wandb_group": "pythia-70m-moe++-4e-20k",
110
+ }
global_step10000/layer_00-model_00-model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3b7fc5e39c1d322d92ad878fb5d622a37650b57e1bf6c66d99e13559fc35385a
3
+ size 51512719
global_step10000/layer_02-model_00-model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e136339b18a58a7ccf36d234f52838e2bc4c84470655a58370fc646722c03fa
3
+ size 8423690
global_step10000/layer_03-model_00-model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:74d3fc34cf989bf3461d5ed1fbbd2abc0a2b71b18909433ba039347195e9455a
3
+ size 8423690
global_step10000/layer_04-model_00-model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d0310ecf2c6df0648c447c106de53a802a13846850a5a67445f9f8a4e29dac7e
3
+ size 8423690
global_step10000/layer_05-model_00-model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c3003a045d46a22d07edc43fd49821eee7f6ff98743b7db7db2648e050630372
3
+ size 8423690
global_step10000/layer_06-model_00-model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:687066bd589077dfdad7f50a273081bb1673c72777edd60d54c35c3b70175ccb
3
+ size 8423690
global_step10000/layer_07-model_00-model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e207dcf2c439625f6a6776e16aaabc424350533dc7dd2056ff2f74f5849c6c22
3
+ size 8423690
global_step10000/layer_09-model_00-model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3d82ff096fde2bad6bb6d28f60d82ce758d1866dea225fb6d6f0e9f342623e99
3
+ size 3746
global_step10000/layer_10-model_00-model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:88a1859c5730d0b926739686497f3b92da7712f61435ac4fdb28bccda97a11d9
3
+ size 51512719
global_step10000/mp_rank_00_model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6d5b321fce53e5a43a7b0ab6f6e1c1752dab6a13a1099bf61d5cdf764489f9d6
3
+ size 27716
global_step10000/zero_pp_rank_0_mp_rank_00_optim_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e3ef4f10846cecebf7357d8253382e19e36a1264f141547422307d0bb77494e3
3
+ size 115135102