sravanthib commited on
Commit
b4d3359
·
verified ·
1 Parent(s): 27b760f

Model save

Browse files
Files changed (4) hide show
  1. README.md +68 -0
  2. all_results.json +8 -0
  3. train_results.json +8 -0
  4. trainer_state.json +217 -0
README.md ADDED
@@ -0,0 +1,68 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model: meta-llama/Llama-3.1-8B-Instruct
3
+ library_name: transformers
4
+ model_name: new-Llama3.1-8b-instruct-RL
5
+ tags:
6
+ - generated_from_trainer
7
+ - trl
8
+ - grpo
9
+ licence: license
10
+ ---
11
+
12
+ # Model Card for new-Llama3.1-8b-instruct-RL
13
+
14
+ This model is a fine-tuned version of [meta-llama/Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct).
15
+ It has been trained using [TRL](https://github.com/huggingface/trl).
16
+
17
+ ## Quick start
18
+
19
+ ```python
20
+ from transformers import pipeline
21
+
22
+ question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
23
+ generator = pipeline("text-generation", model="sravanthib/new-Llama3.1-8b-instruct-RL", device="cuda")
24
+ output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
25
+ print(output["generated_text"])
26
+ ```
27
+
28
+ ## Training procedure
29
+
30
+ [<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/golden-goose/huggingface/runs/k9bt3x5h)
31
+
32
+
33
+ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).
34
+
35
+ ### Framework versions
36
+
37
+ - TRL: 0.16.0.dev0
38
+ - Transformers: 4.49.0
39
+ - Pytorch: 2.5.1
40
+ - Datasets: 3.3.2
41
+ - Tokenizers: 0.21.0
42
+
43
+ ## Citations
44
+
45
+ Cite GRPO as:
46
+
47
+ ```bibtex
48
+ @article{zhihong2024deepseekmath,
49
+ title = {{DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models}},
50
+ author = {Zhihong Shao and Peiyi Wang and Qihao Zhu and Runxin Xu and Junxiao Song and Mingchuan Zhang and Y. K. Li and Y. Wu and Daya Guo},
51
+ year = 2024,
52
+ eprint = {arXiv:2402.03300},
53
+ }
54
+
55
+ ```
56
+
57
+ Cite TRL as:
58
+
59
+ ```bibtex
60
+ @misc{vonwerra2022trl,
61
+ title = {{TRL: Transformer Reinforcement Learning}},
62
+ author = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallouédec},
63
+ year = 2020,
64
+ journal = {GitHub repository},
65
+ publisher = {GitHub},
66
+ howpublished = {\url{https://github.com/huggingface/trl}}
67
+ }
68
+ ```
all_results.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "total_flos": 0.0,
3
+ "train_loss": 0.009057209056256146,
4
+ "train_runtime": 7830.5569,
5
+ "train_samples": 7500,
6
+ "train_samples_per_second": 0.958,
7
+ "train_steps_per_second": 0.007
8
+ }
train_results.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "total_flos": 0.0,
3
+ "train_loss": 0.009057209056256146,
4
+ "train_runtime": 7830.5569,
5
+ "train_samples": 7500,
6
+ "train_samples_per_second": 0.958,
7
+ "train_steps_per_second": 0.007
8
+ }
trainer_state.json ADDED
@@ -0,0 +1,217 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 0.9893390191897654,
5
+ "eval_steps": 100,
6
+ "global_step": 58,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "clip_ratio": 0.0,
13
+ "completion_length": 559.5167694091797,
14
+ "epoch": 0.017057569296375266,
15
+ "grad_norm": 0.012841759249567986,
16
+ "kl": 0.0,
17
+ "learning_rate": 5e-07,
18
+ "loss": 0.0087,
19
+ "reward": 0.11495536100119352,
20
+ "reward_std": 0.195150856859982,
21
+ "rewards/accuracy_reward": 0.11495536100119352,
22
+ "rewards/format_reward": 0.0,
23
+ "step": 1
24
+ },
25
+ {
26
+ "clip_ratio": 0.0,
27
+ "completion_length": 555.0773105621338,
28
+ "epoch": 0.08528784648187633,
29
+ "grad_norm": 0.014003963209688663,
30
+ "kl": 0.0001754462718963623,
31
+ "learning_rate": 2.5e-06,
32
+ "loss": 0.0047,
33
+ "reward": 0.11049107764847577,
34
+ "reward_std": 0.19013801333494484,
35
+ "rewards/accuracy_reward": 0.11049107764847577,
36
+ "rewards/format_reward": 0.0,
37
+ "step": 5
38
+ },
39
+ {
40
+ "clip_ratio": 0.0,
41
+ "completion_length": 538.5312759399415,
42
+ "epoch": 0.17057569296375266,
43
+ "grad_norm": 0.01436670869588852,
44
+ "kl": 0.00021240711212158203,
45
+ "learning_rate": 2.956412726139078e-06,
46
+ "loss": 0.0048,
47
+ "reward": 0.13459821995347737,
48
+ "reward_std": 0.20578723903745413,
49
+ "rewards/accuracy_reward": 0.13459821995347737,
50
+ "rewards/format_reward": 0.0,
51
+ "step": 10
52
+ },
53
+ {
54
+ "clip_ratio": 0.0,
55
+ "completion_length": 534.3341758728027,
56
+ "epoch": 0.255863539445629,
57
+ "grad_norm": 0.014759319834411144,
58
+ "kl": 0.00021679401397705077,
59
+ "learning_rate": 2.7836719084521715e-06,
60
+ "loss": 0.0129,
61
+ "reward": 0.11629464784637093,
62
+ "reward_std": 0.19606330525130033,
63
+ "rewards/accuracy_reward": 0.11629464784637093,
64
+ "rewards/format_reward": 0.0,
65
+ "step": 15
66
+ },
67
+ {
68
+ "clip_ratio": 0.0,
69
+ "completion_length": 532.0986877441406,
70
+ "epoch": 0.3411513859275053,
71
+ "grad_norm": 0.01396097894757986,
72
+ "kl": 0.00022270679473876954,
73
+ "learning_rate": 2.4946839873611927e-06,
74
+ "loss": 0.0112,
75
+ "reward": 0.1265625054948032,
76
+ "reward_std": 0.2010766550898552,
77
+ "rewards/accuracy_reward": 0.1265625054948032,
78
+ "rewards/format_reward": 0.0,
79
+ "step": 20
80
+ },
81
+ {
82
+ "clip_ratio": 0.0,
83
+ "completion_length": 551.1631965637207,
84
+ "epoch": 0.42643923240938164,
85
+ "grad_norm": 0.014589853584766388,
86
+ "kl": 0.00024003982543945311,
87
+ "learning_rate": 2.1156192081791355e-06,
88
+ "loss": 0.0043,
89
+ "reward": 0.1200892923399806,
90
+ "reward_std": 0.19121555145829916,
91
+ "rewards/accuracy_reward": 0.1200892923399806,
92
+ "rewards/format_reward": 0.0,
93
+ "step": 25
94
+ },
95
+ {
96
+ "clip_ratio": 0.0,
97
+ "completion_length": 567.554711151123,
98
+ "epoch": 0.511727078891258,
99
+ "grad_norm": 0.013035917654633522,
100
+ "kl": 0.00024027824401855468,
101
+ "learning_rate": 1.6808050203829845e-06,
102
+ "loss": 0.0111,
103
+ "reward": 0.11250000512227416,
104
+ "reward_std": 0.19112712442874907,
105
+ "rewards/accuracy_reward": 0.11250000512227416,
106
+ "rewards/format_reward": 0.0,
107
+ "step": 30
108
+ },
109
+ {
110
+ "clip_ratio": 0.0,
111
+ "completion_length": 555.3761436462403,
112
+ "epoch": 0.5970149253731343,
113
+ "grad_norm": 0.014955726452171803,
114
+ "kl": 0.0002460002899169922,
115
+ "learning_rate": 1.2296174432791415e-06,
116
+ "loss": 0.0133,
117
+ "reward": 0.11696429131552577,
118
+ "reward_std": 0.1858937400393188,
119
+ "rewards/accuracy_reward": 0.11696429131552577,
120
+ "rewards/format_reward": 0.0,
121
+ "step": 35
122
+ },
123
+ {
124
+ "clip_ratio": 0.0,
125
+ "completion_length": 547.1129707336426,
126
+ "epoch": 0.6823027718550106,
127
+ "grad_norm": 0.014341042377054691,
128
+ "kl": 0.00026259422302246096,
129
+ "learning_rate": 8.029152419343472e-07,
130
+ "loss": 0.0045,
131
+ "reward": 0.1279017912223935,
132
+ "reward_std": 0.19265865609049798,
133
+ "rewards/accuracy_reward": 0.1279017912223935,
134
+ "rewards/format_reward": 0.0,
135
+ "step": 40
136
+ },
137
+ {
138
+ "clip_ratio": 0.0,
139
+ "completion_length": 559.8629776000977,
140
+ "epoch": 0.767590618336887,
141
+ "grad_norm": 0.014500826597213745,
142
+ "kl": 0.0002582550048828125,
143
+ "learning_rate": 4.3933982822017883e-07,
144
+ "loss": 0.0107,
145
+ "reward": 0.12209822023287416,
146
+ "reward_std": 0.19157440066337586,
147
+ "rewards/accuracy_reward": 0.12209822023287416,
148
+ "rewards/format_reward": 0.0,
149
+ "step": 45
150
+ },
151
+ {
152
+ "clip_ratio": 0.0,
153
+ "completion_length": 566.6609634399414,
154
+ "epoch": 0.8528784648187633,
155
+ "grad_norm": 0.014964189380407333,
156
+ "kl": 0.00025937557220458987,
157
+ "learning_rate": 1.718159615201853e-07,
158
+ "loss": 0.0148,
159
+ "reward": 0.12544643357396126,
160
+ "reward_std": 0.20125691760331393,
161
+ "rewards/accuracy_reward": 0.12544643357396126,
162
+ "rewards/format_reward": 0.0,
163
+ "step": 50
164
+ },
165
+ {
166
+ "clip_ratio": 0.0,
167
+ "completion_length": 555.6741317749023,
168
+ "epoch": 0.9381663113006397,
169
+ "grad_norm": 0.012106225825846195,
170
+ "kl": 0.0002725839614868164,
171
+ "learning_rate": 2.4570139579284723e-08,
172
+ "loss": 0.0091,
173
+ "reward": 0.12388393525034189,
174
+ "reward_std": 0.1997589457780123,
175
+ "rewards/accuracy_reward": 0.12388393525034189,
176
+ "rewards/format_reward": 0.0,
177
+ "step": 55
178
+ },
179
+ {
180
+ "clip_ratio": 0.0,
181
+ "completion_length": 555.804059346517,
182
+ "epoch": 0.9893390191897654,
183
+ "kl": 0.0002773602803548177,
184
+ "reward": 0.13988095987588167,
185
+ "reward_std": 0.21711470931768417,
186
+ "rewards/accuracy_reward": 0.13988095987588167,
187
+ "rewards/format_reward": 0.0,
188
+ "step": 58,
189
+ "total_flos": 0.0,
190
+ "train_loss": 0.009057209056256146,
191
+ "train_runtime": 7830.5569,
192
+ "train_samples_per_second": 0.958,
193
+ "train_steps_per_second": 0.007
194
+ }
195
+ ],
196
+ "logging_steps": 5,
197
+ "max_steps": 58,
198
+ "num_input_tokens_seen": 0,
199
+ "num_train_epochs": 1,
200
+ "save_steps": 5,
201
+ "stateful_callbacks": {
202
+ "TrainerControl": {
203
+ "args": {
204
+ "should_epoch_stop": false,
205
+ "should_evaluate": false,
206
+ "should_log": false,
207
+ "should_save": true,
208
+ "should_training_stop": true
209
+ },
210
+ "attributes": {}
211
+ }
212
+ },
213
+ "total_flos": 0.0,
214
+ "train_batch_size": 16,
215
+ "trial_name": null,
216
+ "trial_params": null
217
+ }