Upload folder using huggingface_hub
Browse files
custom_generate/generate.py
CHANGED
@@ -17,6 +17,15 @@ from transformers import DynamicCache, GenerationConfig
|
|
17 |
from typing import Any, Dict, List, Optional, Tuple
|
18 |
|
19 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
20 |
class LagKVCache(DynamicCache):
|
21 |
"""
|
22 |
A KV compression algorithm that as described in the [LagKV paper](https://arxiv.org/abs/2504.04704).
|
|
|
17 |
from typing import Any, Dict, List, Optional, Tuple
|
18 |
|
19 |
|
20 |
+
UNSUPPORTED_GENERATION_ARGS = [
|
21 |
+
"cache_implementation", # cache-related arguments, here we always use SinkCache
|
22 |
+
"cache_config",
|
23 |
+
"return_legacy_cache",
|
24 |
+
"num_beams", # beam search (and cousin techniques) are not supported
|
25 |
+
"compile_config", # SinkCache doesn't support torch.compile
|
26 |
+
"assistant_model", # it also doesn't support speculative decoding
|
27 |
+
]
|
28 |
+
|
29 |
class LagKVCache(DynamicCache):
|
30 |
"""
|
31 |
A KV compression algorithm that as described in the [LagKV paper](https://arxiv.org/abs/2504.04704).
|