ChrisMcCormick commited on
Commit
b4e4358
·
verified ·
1 Parent(s): cc9aef8

Upload deepseek-tiny-v0.1 model weights and documentation

Browse files
README.md CHANGED
@@ -1,3 +1,98 @@
1
- ---
2
- license: mit
3
- ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ language:
4
+ - en
5
+ datasets:
6
+ - wikitext
7
+ - glue
8
+ pipeline_tag: text-generation
9
+ tags:
10
+ - transformer
11
+ - attention
12
+ - mla
13
+ - research
14
+
15
+ ---
16
+
17
+ # Deepseek Tiny V0.1
18
+
19
+ 6-layer DeepSeek-V3 with Multihead Latent Attention (MLA) trained for research on shared subspaces in Transformer attention mechanisms.
20
+
21
+ ## Model Description
22
+
23
+ - **Model Type**: Transformer Decoder (DeepSeek-V3 based)
24
+ - **Architecture**: 6-layer decoder with Mixture of Experts
25
+ - **Parameters**: 16.26M
26
+ - **Hidden Size**: 256
27
+ - **Attention Heads**: 8
28
+ - **Head Dimension**: 32
29
+ - **Sequence Length**: 1,024 tokens
30
+ - **Query Latent Dimension**: 96
31
+ - **Key-Value Latent Dimension**: 64
32
+
33
+
34
+ ## Performance
35
+
36
+ - **SST-2 Accuracy**: 87.96%
37
+ - **WikiText-103 Perplexity**: 28.89
38
+
39
+ ## Research Context
40
+
41
+ This model is part of the [shared-subspaces](https://github.com/chrisjmccormick/shared-subspaces) research project investigating the impact of shared output latent spaces in Transformer attention mechanisms.
42
+
43
+
44
+
45
+
46
+
47
+
48
+
49
+
50
+ ## Usage
51
+
52
+ ```python
53
+ import torch
54
+ from transformers import DeepseekV3ForCausalLM, AutoTokenizer
55
+
56
+ # Load model and tokenizer
57
+ model = DeepseekV3ForCausalLM.from_pretrained("ChrisMcCormick/deepseek-tiny-v0.1")
58
+ tokenizer = AutoTokenizer.from_pretrained("ChrisMcCormick/deepseek-tiny-v0.1")
59
+
60
+
61
+
62
+ # Generate text
63
+ inputs = tokenizer("The future of AI is", return_tensors="pt")
64
+ outputs = model.generate(**inputs, max_length=50, temperature=0.7)
65
+ print(tokenizer.decode(outputs[0], skip_special_tokens=True))
66
+ ```
67
+
68
+ ## Training Details
69
+
70
+ - **Pre-training Dataset**: WikiText-103
71
+ - **Fine-tuning Dataset**: SST-2 (GLUE)
72
+ - **Optimizer**: AdamW
73
+ - **Learning Rate**: 5e-4 (pre-training), 5e-5 (fine-tuning)
74
+ - **Weight Decay**: 0.01 (pre-training), 0.05 (fine-tuning)
75
+ - **Precision**: bfloat16
76
+ - **Compilation**: torch.compile with inductor backend
77
+ - **Training Steps**: 12,500 (pre-training), 1,500 (fine-tuning)
78
+
79
+ ## Limitations
80
+
81
+ - Small scale model (16M parameters) intended for research purposes
82
+ - Trained on limited data compared to production models
83
+ - May require custom loading code for output subspace variants
84
+
85
+ ## Citation
86
+
87
+ ```bibtex
88
+ @misc{mccormick2025sharedsubspaces,
89
+ title={Shared Subspaces in Transformer Attention: Investigating Output Latent Spaces},
90
+ author={McCormick, Chris},
91
+ year={2025},
92
+ howpublished={\url{https://github.com/chrisjmccormick/shared-subspaces}}
93
+ }
94
+ ```
95
+
96
+ ## License
97
+
98
+ Apache 2.0
config.json ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "DeepseekV3ForCausalLM"
4
+ ],
5
+ "attention_backend": "flash_attention_2",
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 50256,
9
+ "classifier_dropout": null,
10
+ "dtype": "float32",
11
+ "eos_token_id": 50256,
12
+ "ep_size": 1,
13
+ "first_k_dense_replace": 1,
14
+ "head_dim": 32,
15
+ "hidden_act": "silu",
16
+ "hidden_dropout_prob": 0.1,
17
+ "hidden_size": 256,
18
+ "initializer_range": 0.02,
19
+ "intermediate_size": 1024,
20
+ "kv_lora_rank": 64,
21
+ "max_position_embeddings": 1024,
22
+ "model_type": "deepseek_v3",
23
+ "moe_intermediate_size": 128,
24
+ "moe_layer_freq": 2,
25
+ "n_group": 1,
26
+ "n_routed_experts": 4,
27
+ "n_shared_experts": 1,
28
+ "norm_topk_prob": true,
29
+ "num_attention_heads": 8,
30
+ "num_experts_per_tok": 2,
31
+ "num_hidden_layers": 6,
32
+ "num_key_value_heads": 8,
33
+ "num_nextn_predict_layers": 1,
34
+ "pad_token_id": 50256,
35
+ "pretraining_tp": 1,
36
+ "q_lora_rank": 96,
37
+ "qk_head_dim": 32,
38
+ "qk_nope_head_dim": 0,
39
+ "qk_rope_head_dim": 32,
40
+ "rms_norm_eps": 1e-06,
41
+ "rope_interleave": true,
42
+ "rope_scaling": null,
43
+ "rope_theta": 10000.0,
44
+ "routed_scaling_factor": 1,
45
+ "scoring_func": "softmax",
46
+ "tie_word_embeddings": true,
47
+ "topk_group": 1,
48
+ "topk_method": "softmax_aux",
49
+ "transformers_version": "4.56.0",
50
+ "use_cache": false,
51
+ "v_head_dim": 32,
52
+ "vocab_size": 50257
53
+ }
example_usage.py ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """
3
+ Example usage for deepseek-tiny-v0.1
4
+ """
5
+
6
+ import torch
7
+ from transformers import DeepseekV3ForCausalLM, AutoTokenizer
8
+
9
+ def main():
10
+ # Load model and tokenizer
11
+ print("Loading model...")
12
+ model = DeepseekV3ForCausalLM.from_pretrained("ChrisMcCormick/deepseek-tiny-v0.1")
13
+ tokenizer = AutoTokenizer.from_pretrained("ChrisMcCormick/deepseek-tiny-v0.1")
14
+
15
+
16
+
17
+ # Set to evaluation mode
18
+ model.eval()
19
+
20
+ # Example prompts
21
+ prompts = [
22
+ "The future of artificial intelligence is",
23
+ "In a world where technology advances rapidly,",
24
+ "The most important discovery in science was",
25
+ ]
26
+
27
+ print("\nGenerating text...")
28
+ for prompt in prompts:
29
+ inputs = tokenizer(prompt, return_tensors="pt")
30
+
31
+ with torch.no_grad():
32
+ outputs = model.generate(
33
+ **inputs,
34
+ max_length=50,
35
+ temperature=0.7,
36
+ do_sample=True,
37
+ pad_token_id=tokenizer.eos_token_id
38
+ )
39
+
40
+ generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
41
+ print(f"Prompt: {prompt}")
42
+ print(f"Generated: {generated_text}")
43
+ print("-" * 50)
44
+
45
+ if __name__ == "__main__":
46
+ main()
generation_config.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 50256,
4
+ "eos_token_id": [
5
+ 50256
6
+ ],
7
+ "pad_token_id": 50256,
8
+ "transformers_version": "4.56.0",
9
+ "use_cache": false
10
+ }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e33b8f75205b4b7721663e348a51e10c000dd3309ad1b2ce59b79990cfd448a7
3
+ size 68229024
special_tokens_map.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "<|endoftext|>",
3
+ "eos_token": "<|endoftext|>",
4
+ "pad_token": "<|endoftext|>",
5
+ "unk_token": "<|endoftext|>"
6
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "added_tokens_decoder": {
4
+ "50256": {
5
+ "content": "<|endoftext|>",
6
+ "lstrip": false,
7
+ "normalized": true,
8
+ "rstrip": false,
9
+ "single_word": false,
10
+ "special": true
11
+ }
12
+ },
13
+ "bos_token": "<|endoftext|>",
14
+ "clean_up_tokenization_spaces": false,
15
+ "eos_token": "<|endoftext|>",
16
+ "extra_special_tokens": {},
17
+ "model_max_length": 1024,
18
+ "pad_token": "<|endoftext|>",
19
+ "tokenizer_class": "GPT2Tokenizer",
20
+ "unk_token": "<|endoftext|>"
21
+ }
vocab.json ADDED
The diff for this file is too large to render. See raw diff