kyleliang commited on
Commit
dafad47
·
verified ·
1 Parent(s): d9addd5

Upload folder using huggingface_hub

Browse files
Files changed (1) hide show
  1. custom_generate/generate.py +9 -0
custom_generate/generate.py CHANGED
@@ -17,6 +17,15 @@ from transformers import DynamicCache, GenerationConfig
17
  from typing import Any, Dict, List, Optional, Tuple
18
 
19
 
 
 
 
 
 
 
 
 
 
20
  class LagKVCache(DynamicCache):
21
  """
22
  A KV compression algorithm that as described in the [LagKV paper](https://arxiv.org/abs/2504.04704).
 
17
  from typing import Any, Dict, List, Optional, Tuple
18
 
19
 
20
+ UNSUPPORTED_GENERATION_ARGS = [
21
+ "cache_implementation", # cache-related arguments, here we always use SinkCache
22
+ "cache_config",
23
+ "return_legacy_cache",
24
+ "num_beams", # beam search (and cousin techniques) are not supported
25
+ "compile_config", # SinkCache doesn't support torch.compile
26
+ "assistant_model", # it also doesn't support speculative decoding
27
+ ]
28
+
29
  class LagKVCache(DynamicCache):
30
  """
31
  A KV compression algorithm that as described in the [LagKV paper](https://arxiv.org/abs/2504.04704).