Update config.json
Browse filesThis PR addresses a bug that would prevent flash attention 2 from running with granite-speech-8b using HF transformers. The same bug was not present for the 2b version.
Upon closer inspection the line " "_attn_implementation_autoset": true "was not present in config.json (but was present in the 2b version). After adding this line FA2 appears to be functional again.
- config.json +1 -0
config.json
CHANGED
@@ -23,6 +23,7 @@
|
|
23 |
"initializer_range": 0.02,
|
24 |
"model_type": "granite_speech",
|
25 |
"projector_config": {
|
|
|
26 |
"attention_probs_dropout_prob": 0.1,
|
27 |
"cross_attention_frequency": 1,
|
28 |
"encoder_hidden_size": 1024,
|
|
|
23 |
"initializer_range": 0.02,
|
24 |
"model_type": "granite_speech",
|
25 |
"projector_config": {
|
26 |
+
"_attn_implementation_autoset": true
|
27 |
"attention_probs_dropout_prob": 0.1,
|
28 |
"cross_attention_frequency": 1,
|
29 |
"encoder_hidden_size": 1024,
|