import{s as ta,o as sa,n as hs}from"../chunks/scheduler.25b97de1.js";import{S as aa,i as la,g as o,s as l,r,A as na,h as i,f as s,c as n,j as ps,u as h,x as c,k as ea,y as J,a,v as m,d,t as u,w as y}from"../chunks/index.d9030fc9.js";import{T as rs}from"../chunks/Tip.baa67368.js";import{C as b}from"../chunks/CodeBlock.e6cd0d95.js";import{H as j,E as oa}from"../chunks/EditOnGithub.91d95064.js";function ia(U){let p,T="One important concept you need to know when writing your own generation loop, is <code>cache_position</code>. In case you want to reuse an already filled Cache object by calling <code>forward()</code>, you have to pass in a valid <code>cache_position</code> which will indicate the positions of inputs in the sequence. Note that <code>cache_position</code> is not affected by padding, and always adds one more position for each token. For example, if key/value cache contains 10 tokens (no matter how many of it is a pad token), the cache position for the next token should be <code>torch.tensor([10])</code>.";return{c(){p=o("p"),p.innerHTML=T},l(M){p=i(M,"P",{"data-svelte-h":!0}),c(p)!=="svelte-fyl20k"&&(p.innerHTML=T)},m(M,f){a(M,p,f)},p:hs,d(M){M&&s(p)}}}function ca(U){let p,T="Cache quantization can be detrimental in terms of latency if the context length is short and there is enough GPU VRAM available to run without cache quantization. It is recommended to seek balance between memory efficiency and latency.";return{c(){p=o("p"),p.textContent=T},l(M){p=i(M,"P",{"data-svelte-h":!0}),c(p)!=="svelte-46etli"&&(p.textContent=T)},m(M,f){a(M,p,f)},p:hs,d(M){M&&s(p)}}}function pa(U){let p,T="Cache offloading requires a CUDA GPU and can be slower than dynamic KV cache. Use it if you are getting CUDA out of memory errors.";return{c(){p=o("p"),p.textContent=T},l(M){p=i(M,"P",{"data-svelte-h":!0}),c(p)!=="svelte-82c9ia"&&(p.textContent=T)},m(M,f){a(M,p,f)},p:hs,d(M){M&&s(p)}}}function ra(U){let p,T,M,f,Z,Ee,v,ms=`Efficient caching is crucial for optimizing the performance of models in various generative tasks,
including text generation, translation, summarization and other transformer-based applications.
Effective caching helps reduce computation time and improve response rates, especially in real-time or resource-intensive applications.`,$e,B,ds=`Transformers support various caching methods, leveraging “Cache” classes to abstract and manage the caching logic.
This document outlines best practices for using these classes to maximize performance and efficiency.
Check out all the available <code>Cache</code> classes in the <a href="./internal/generation_utils">API documentation</a>.`,Se,W,qe,G,us="Imagine you’re having a conversation with someone, and instead of remembering what was said previously, you have to start from scratch every time you respond. This would be slow and inefficient, right? In the world of Transformer models, a similar concept applies, and that’s where Caching keys and values come into play. From now on, I’ll refer to the concept as KV Cache.",Ae,X,ys="KV cache is needed to optimize the generation in autoregressive models, where the model predicts text token by token. This process can be slow since the model can generate only one token at a time, and each new prediction is dependent on the previous context. That means, to predict token number 1000 in the generation, you need information from the previous 999 tokens, which comes in the form of some matrix multiplications across the representations of those tokens. But to predict token number 1001, you also need the same information from the first 999 tokens, plus additional information from token number 1000. That is where key-value cache is used to optimize the sequential generation process by storing previous calculations to reuse in subsequent tokens, so they don’t need to be computed again.",Le,V,Ms="More concretely, key-value cache acts as a memory bank for these generative models, where the model stores key-value pairs derived from self-attention layers for previously processed tokens. By storing this information, the model can avoid redundant computations and instead retrieve keys and values of previous tokens from the cache. Note that caching can be used only in inference and should be disabled when training, otherwise it might cause unexpected errors.",De,g,Ye,gs="<em>For the Curious Minds Who Like to Dive Deep</em>",ts,_,ss,Qe,ws="When utilizing a cache object in the input, the Attention module performs several critical steps to integrate past and present information seamlessly.",as,xe,js="The Attention module concatenates the current key-values with the past key-values stored in the cache. This results in attention weights of shape <code>(new_tokens_length, past_kv_length + new_tokens_length)</code>. Essentially, the past and current key-values are combined to compute attention scores, ensuring that the model considers both previous context and new input. The concatenated key-values are used to compute the attention scores resulting in attention weights of shape <code>(new_tokens_length, past_kv_length + new_tokens_length)</code>.",ls,ze,Js="Therefore, when iteratively calling <code>forward()</code> instead of the <code>generate()</code> method, it’s crucial to ensure that the attention mask shape matches the combined length of past and current key-values. The attention mask should have the shape <code>(batch_size, past_kv_length + new_tokens_length)</code>. This is usually handled internally when you call <code>generate()</code> method. If you want to implement your own generation loop with Cache classes, take this into consideration and prepare the attention mask to hold values to current and past tokens.",ns,k,os,Fe,Ts="See an example below for how to implement your own generation loop.",is,R,cs,N,Ke,Y,Pe,Q,bs=`In 🤗 Transformers, we support various Cache types to optimize the performance across different models and tasks. By default, all models generate with caching,
with the <a href="/docs/transformers/main/en/internal/generation_utils#transformers.DynamicCache">~DynamicCache</a> class being the default cache for most models. It allows us to dynamically grow cache size, by saving more and more keys and values as we generate. If for some reason you don’t want to use caches, you can pass <code>use_cache=False</code> into the <code>generate()</code> method.`,Oe,x,fs="Refer to the table below to see the difference between cache types and choose the one that suits best for your use-case. Models for which initialization is recommended should be initialized before calling the model and passed to model as a kwarg. In all other cases you can simply define desired <code>cache_implementation</code> and we take care of the rest for you.",et,z,Us="<thead><tr><th>Cache Type</th> <th>Memory Efficient</th> <th>Supports torch.compile()</th> <th>Initialization Recommended</th> <th>Latency</th> <th>Long Context Generation</th></tr></thead> <tbody><tr><td>Dynamic Cache</td> <td>No</td> <td>No</td> <td>No</td> <td>Mid</td> <td>No</td></tr> <tr><td>Static Cache</td> <td>No</td> <td>Yes</td> <td>Yes</td> <td>High</td> <td>No</td></tr> <tr><td>Offloaded Cache</td> <td>Yes</td> <td>No</td> <td>No</td> <td>Low</td> <td>Yes</td></tr> <tr><td>Offloaded Static Cache</td> <td>No</td> <td>Yes</td> <td>Yes</td> <td>High</td> <td>Yes</td></tr> <tr><td>Quantized Cache</td> <td>Yes</td> <td>No</td> <td>No</td> <td>Low</td> <td>Yes</td></tr> <tr><td>Sliding Window Cache</td> <td>No</td> <td>Yes</td> <td>Yes</td> <td>High</td> <td>No</td></tr> <tr><td>Sink Cache</td> <td>Yes</td> <td>No</td> <td>Yes</td> <td>Mid</td> <td>Yes</td></tr></tbody>",tt,F,ks='These cache classes can be set with a <code>cache_implementation</code> argument when generating. To learn about the available options for the cache_implementation flag, please refer to the <a href="./main_classes/text_generation#transformers.GenerationConfig">API Documentation</a>. Now, let’s explore each cache type in detail and see how to use them. Note that the below examples are for decoder-only Tranformer-based models. We also support [“Model-Specific Cache”] classes for models such as Mamba or Jamba, keep reading for more details.',st,H,at,E,Cs=`The key and value cache can occupy a large portion of memory, becoming a <a href="https://huggingface.co/blog/llama31#inference-memory-requirements" rel="nofollow">bottleneck for long-context generation</a>, especially for Large Language Models.
Quantizing the cache when using <code>generate()</code> can significantly reduce memory requirements at the cost of speed.`,lt,$,Is='KV Cache quantization in <code>transformers</code> is largely inspired by the paper <a href="https://arxiv.org/abs/2402.02750" rel="nofollow">“KIVI: A Tuning-Free Asymmetric 2bit Quantization for KV Cache”</a> and currently supports <a href="/docs/transformers/main/en/internal/generation_utils#transformers.QuantoQuantizedCache">~QuantoQuantizedCache</a> and <a href="/docs/transformers/main/en/internal/generation_utils#transformers.HQQQuantizedCache">~HQQQuantizedCache</a> classes. For more information on the inner workings see the paper.',nt,S,Zs=`To enable quantization of the key-value cache, one needs to indicate <code>cache_implementation=&quot;quantized&quot;</code> in the <code>generation_config</code>.
Quantization related arguments should be passed to the <code>generation_config</code> either as a <code>dict</code> or an instance of a <a href="/docs/transformers/main/en/internal/generation_utils#transformers.QuantizedCacheConfig">~QuantizedCacheConfig</a> class.
One has to indicate which quantization backend to use in the <a href="/docs/transformers/main/en/internal/generation_utils#transformers.QuantizedCacheConfig">~QuantizedCacheConfig</a>, the default is <code>quanto</code>.`,ot,q,vs="It is recommended to set <code>axis-key/axis-value</code> parameters in the cache config to <code>0</code> if you’re using the <code>quanto</code> backend and to <code>1</code> if you’re using the <code>HQQ</code> backend. For other config values, please use the defaults unless you’re running out of memory. In that case, you may consider decreasing the residual length.",it,C,ct,A,pt,L,rt,D,Bs=`Similarly to KV cache quantization, <a href="/docs/transformers/main/en/internal/generation_utils#transformers.OffloadedCache">~OffloadedCache</a> strategy aims to reduce GPU VRAM usage.
It does so by moving the KV cache for most layers to the CPU.
As the model’s <code>forward()</code> method iterates over the layers, this strategy maintains the current layer cache on the GPU.
At the same time it asynchronously prefetches the next layer cache as well as sending the previous layer cache back to the CPU.
Unlike KV cache quantization, this strategy always produces the same result as the default KV cache implementation.
Thus, it can serve as a drop-in replacement or a fallback for it.`,ht,K,Ws=`Depending on your model and the characteristics of your generation task (size of context, number of generated tokens, number of beams, etc.)
you may notice a small degradation in generation throughput compared to the default KV cache implementation.`,mt,P,Gs=`To enable KV cache offloading, pass <code>cache_implementation=&quot;offloaded&quot;</code> in the <code>generation_config</code> or directly to the <code>generate()</code> call.
Use <code>cache_implementation=&quot;offloaded_static&quot;</code> for an offloaded static cache (see also <a href="#offloaded-static-cache">Offloaded Static Cache</a> below).`,dt,O,ut,I,yt,ee,Xs="The example below shows how KV cache offloading can be used as a fallback strategy.",Mt,te,gt,se,Vs="On a GPU with 50 GB of RAM, running this code will print",wt,ae,jt,le,_s="before successfully generating 40 beams.",Jt,ne,Tt,oe,Rs=`Since the “DynamicCache” dynamically grows with each generation step, it prevents you from taking advantage of JIT optimizations. The <a href="/docs/transformers/main/en/internal/generation_utils#transformers.StaticCache">~StaticCache</a> pre-allocates
a specific maximum size for the keys and values, allowing you to generate up to the maximum length without having to modify cache size. Check the below usage example.`,bt,ie,Ns='For more examples with Static Cache and JIT compilation, take a look at <a href="./llm_optims#static-kv-cache-and-torchcompile">StaticCache &amp; torchcompile</a>',ft,ce,Ut,pe,kt,re,Ys=`Like <a href="/docs/transformers/main/en/internal/generation_utils#transformers.OffloadedCache">~OffloadedCache</a> exists for offloading a “DynamicCache”, there is also an offloaded static cache. It fully supports
JIT optimizations. Just pass <code>cache_implementation=&quot;offloaded_static&quot;</code> in the <code>generation_config</code> or directly to the <code>generate()</code> call.
This will use the <a href="/docs/transformers/main/en/internal/generation_utils#transformers.OffloadedStaticCache">~OffloadedStaticCache</a> implementation instead.`,Ct,he,It,me,Qs="Cache offloading requires a CUDA GPU.",Zt,de,vt,ue,xs="As the name suggests, this cache type implements a sliding window over previous keys and values, retaining only the last <code>sliding_window</code> tokens. It should be used with models like Mistral that support sliding window attention. Additionally, similar to Static Cache, this one is JIT-friendly and can be used with the same compile tecniques as Static Cache.",Bt,ye,zs="Note that you can use this cache only for models that support sliding window, e.g. Mistral models.",Wt,Me,Gt,ge,Xt,we,Fs='Sink Cache was introduced in <a href="https://arxiv.org/abs/2309.17453" rel="nofollow">“Efficient Streaming Language Models with Attention Sinks”</a>. It allows you to generate long sequences of text (“infinite length” according to the paper) without any fine-tuning. That is achieved by smart handling of previous keys and values, specifically it retains a few initial tokens from the sequence, called “sink tokens”. This is based on the observation that these initial tokens attract a significant portion of attention scores during the generation process. Tokens that come after “sink tokens” are discarded on a sliding windowed basis, keeping only the latest <code>window_size</code> tokens. By keeping these initial tokens as “attention sinks,” the model maintains stable performance even when dealing with very long texts, thus discarding most of the previous knowledge.',Vt,je,Hs="Unlike other cache classes, this one can’t be used directly by indicating a <code>cache_implementation</code>. You have to initialize the Cache before calling on <code>generate()</code> as follows.",_t,Je,Rt,Te,Nt,be,Es='The <a href="/docs/transformers/main/en/internal/generation_utils#transformers.EncoderDecoderCache">~EncoderDecoderCache</a> is a wrapper designed to handle the caching needs of encoder-decoder models. This cache type is specifically built to manage both self-attention and cross-attention caches, ensuring storage and retrieval of past key/values required for these complex models. Cool thing about Encoder-Decoder Cache is that you can set different cache types for the encoder and for the decoder, depending on your use case. Currently this cache is only supported in <a href="./model_doc/whisper">Whisper</a> models but we will be adding more models soon.',Yt,fe,$s="In terms of usage, there is nothing special to be done and calling <code>generate()</code> or <code>forward()</code> will handle everything for you.",Qt,Ue,xt,ke,Ss='Some models require storing previous keys, values, or states in a specific way, and the above cache classes cannot be used. For such cases, we have several specialized cache classes that are designed for specific models. These models only accept their own dedicated cache classes and do not support using any other cache types. Some examples include <a href="/docs/transformers/main/en/internal/generation_utils#transformers.HybridCache">~HybridCache</a> for <a href="./model_doc/gemma2">Gemma2</a> series models or <a href="/docs/transformers/main/en/internal/generation_utils#transformers.MambaCache">~MambaCache</a> for <a href="./model_doc/mamba">Mamba</a> architecture models.',zt,Ce,Ft,Ie,qs="We have seen how to use each of the cache types when generating. What if you want to use cache in iterative generation setting, for example in applications like chatbots, where interactions involve multiple turns and continuous back-and-forth exchanges. Iterative generation with cache allows these systems to handle ongoing conversations effectively without reprocessing the entire context at each step. But there are some tips that you should know before you start implementing:",Ht,Ze,As='The general format when doing iterative generation is as below. First you have to initialize an empty cache of the type you want, and you can start feeding in new prompts iteratively. Keeping track of dialogues history and formatting can be done with chat templates, read more on that in <a href="./chat_templating">chat_templating</a>',Et,ve,Ls="In case you are using Sink Cache, you have to crop your inputs to that maximum length because Sink Cache can generate text longer than its maximum window size, but it expects the first input to not exceed the maximum cache length.",$t,Be,St,We,qt,Ge,Ds="Sometimes you would want to first fill-in cache object with key/values for certain prefix prompt and re-use it several times to generate different sequences from it. In that case you can construct a <code>Cache</code> object that will hold the instruction prompt, and re-use it several times with different text sequences.",At,Xe,Lt,Ve,Dt,_e,Ks=`Prior to the introduction of the <code>Cache</code> object, the cache of LLMs used to be a tuple of tuples of tensors. The legacy
format has a dynamic size, growing as we generate text — very similar to <code>DynamicCache</code>. If your project depend on
this legacy format, you can seamlessly convert it to a <code>DynamicCache</code> and back.`,Kt,Re,Pt,Ne,Ot,He,es;return Z=new j({props:{title:"Best Practices for Generation with Cache",local:"best-practices-for-generation-with-cache",headingTag:"h1"}}),W=new j({props:{title:"What is Cache and why we should care?",local:"what-is-cache-and-why-we-should-care",headingTag:"h2"}}),_=new j({props:{title:"Under the Hood: How Cache Object Works in Attention Mechanism",local:"under-the-hood-how-cache-object-works-in-attention-mechanism",headingTag:"h3"}}),k=new rs({props:{warning:!0,$$slots:{default:[ia]},$$scope:{ctx:U}}}),R=new b({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwRHluYW1pY0NhY2hlJTBBJTBBbW9kZWxfaWQlMjAlM0QlMjAlMjJUaW55TGxhbWElMkZUaW55TGxhbWEtMS4xQi1DaGF0LXYxLjAlMjIlMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZChtb2RlbF9pZCUyQyUyMHRvcmNoX2R0eXBlJTNEdG9yY2guYmZsb2F0MTYlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiklMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChtb2RlbF9pZCklMEElMEFwYXN0X2tleV92YWx1ZXMlMjAlM0QlMjBEeW5hbWljQ2FjaGUoKSUwQW1lc3NhZ2VzJTIwJTNEJTIwJTVCJTdCJTIycm9sZSUyMiUzQSUyMCUyMnVzZXIlMjIlMkMlMjAlMjJjb250ZW50JTIyJTNBJTIwJTIySGVsbG8lMkMlMjB3aGF0J3MlMjB5b3VyJTIwbmFtZS4lMjIlN0QlNUQlMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIuYXBwbHlfY2hhdF90ZW1wbGF0ZShtZXNzYWdlcyUyQyUyMGFkZF9nZW5lcmF0aW9uX3Byb21wdCUzRFRydWUlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyJTJDJTIwcmV0dXJuX2RpY3QlM0RUcnVlKS50byhtb2RlbC5kZXZpY2UpJTBBJTBBZ2VuZXJhdGVkX2lkcyUyMCUzRCUyMGlucHV0cy5pbnB1dF9pZHMlMEFjYWNoZV9wb3NpdGlvbiUyMCUzRCUyMHRvcmNoLmFyYW5nZShpbnB1dHMuaW5wdXRfaWRzLnNoYXBlJTVCMSU1RCUyQyUyMGR0eXBlJTNEdG9yY2guaW50NjQlMkMlMjBkZXZpY2UlM0Rtb2RlbC5kZXZpY2UpJTBBbWF4X25ld190b2tlbnMlMjAlM0QlMjAxMCUwQSUwQWZvciUyMF8lMjBpbiUyMHJhbmdlKG1heF9uZXdfdG9rZW5zKSUzQSUwQSUyMCUyMCUyMCUyMG91dHB1dHMlMjAlM0QlMjBtb2RlbCgqKmlucHV0cyUyQyUyMGNhY2hlX3Bvc2l0aW9uJTNEY2FjaGVfcG9zaXRpb24lMkMlMjBwYXN0X2tleV92YWx1ZXMlM0RwYXN0X2tleV92YWx1ZXMlMkMlMjB1c2VfY2FjaGUlM0RUcnVlKSUwQSUyMCUyMCUyMCUyMCUyMyUyMEdyZWVkaWx5JTIwc2FtcGxlJTIwb25lJTIwbmV4dCUyMHRva2VuJTBBJTIwJTIwJTIwJTIwbmV4dF90b2tlbl9pZHMlMjAlM0QlMjBvdXRwdXRzLmxvZ2l0cyU1QiUzQSUyQyUyMC0xJTNBJTVELmFyZ21heCgtMSklMEElMjAlMjAlMjAlMjBnZW5lcmF0ZWRfaWRzJTIwJTNEJTIwdG9yY2guY2F0KCU1QmdlbmVyYXRlZF9pZHMlMkMlMjBuZXh0X3Rva2VuX2lkcyU1RCUyQyUyMGRpbSUzRC0xKSUwQSUyMCUyMCUyMCUyMCUyMyUyMFByZXBhcmUlMjBpbnB1dHMlMjBmb3IlMjB0aGUlMjBuZXh0JTIwZ2VuZXJhdGlvbiUyMHN0ZXAlMjBieSUyMGxlYWF2aW5nJTIwdW5wcm9jZXNzZWQlMjB0b2tlbnMlMkMlMjBpbiUyMG91ciUyMGNhc2UlMjB3ZSUyMGhhdmUlMjBvbmx5JTIwb25lJTIwbmV3JTIwdG9rZW4lMEElMjAlMjAlMjAlMjAlMjMlMjBhbmQlMjBleHBhbmRpbmclMjBhdHRuJTIwbWFzayUyMGZvciUyMHRoZSUyMG5ldyUyMHRva2VuJTJDJTIwYXMlMjBleHBsYWluZWQlMjBhYm92ZSUwQSUyMCUyMCUyMCUyMGF0dGVudGlvbl9tYXNrJTIwJTNEJTIwaW5wdXRzJTVCJTIyYXR0ZW50aW9uX21hc2slMjIlNUQlMEElMjAlMjAlMjAlMjBhdHRlbnRpb25fbWFzayUyMCUzRCUyMHRvcmNoLmNhdCglNUJhdHRlbnRpb25fbWFzayUyQyUyMGF0dGVudGlvbl9tYXNrLm5ld19vbmVzKChhdHRlbnRpb25fbWFzay5zaGFwZSU1QjAlNUQlMkMlMjAxKSklNUQlMkMlMjBkaW0lM0QtMSklMEElMjAlMjAlMjAlMjBpbnB1dHMlMjAlM0QlMjAlN0IlMjJpbnB1dF9pZHMlMjIlM0ElMjBuZXh0X3Rva2VuX2lkcyUyQyUyMCUyMmF0dGVudGlvbl9tYXNrJTIyJTNBJTIwYXR0ZW50aW9uX21hc2slN0QlMEElMjAlMjAlMjAlMjBjYWNoZV9wb3NpdGlvbiUyMCUzRCUyMGNhY2hlX3Bvc2l0aW9uJTVCLTElM0ElNUQlMjAlMkIlMjAxJTIwJTIzJTIwYWRkJTIwb25lJTIwbW9yZSUyMHBvc2l0aW9uJTIwZm9yJTIwdGhlJTIwbmV4dCUyMHRva2VuJTBBJTBBcHJpbnQodG9rZW5pemVyLmJhdGNoX2RlY29kZShnZW5lcmF0ZWRfaWRzJTJDJTIwc2tpcF9zcGVjaWFsX3Rva2VucyUzRFRydWUpJTVCMCU1RCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM, DynamicCache

<span class="hljs-meta">&gt;&gt;&gt; </span>model_id = <span class="hljs-string">&quot;TinyLlama/TinyLlama-1.1B-Chat-v1.0&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.bfloat16, device_map=<span class="hljs-string">&quot;auto&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(model_id)

<span class="hljs-meta">&gt;&gt;&gt; </span>past_key_values = DynamicCache()
<span class="hljs-meta">&gt;&gt;&gt; </span>messages = [{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;Hello, what&#x27;s your name.&quot;</span>}]
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer.apply_chat_template(messages, add_generation_prompt=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>, return_dict=<span class="hljs-literal">True</span>).to(model.device)

<span class="hljs-meta">&gt;&gt;&gt; </span>generated_ids = inputs.input_ids
<span class="hljs-meta">&gt;&gt;&gt; </span>cache_position = torch.arange(inputs.input_ids.shape[<span class="hljs-number">1</span>], dtype=torch.int64, device=model.device)
<span class="hljs-meta">&gt;&gt;&gt; </span>max_new_tokens = <span class="hljs-number">10</span>

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(max_new_tokens):
<span class="hljs-meta">... </span>    outputs = model(**inputs, cache_position=cache_position, past_key_values=past_key_values, use_cache=<span class="hljs-literal">True</span>)
<span class="hljs-meta">... </span>    <span class="hljs-comment"># Greedily sample one next token</span>
<span class="hljs-meta">... </span>    next_token_ids = outputs.logits[:, -<span class="hljs-number">1</span>:].argmax(-<span class="hljs-number">1</span>)
<span class="hljs-meta">... </span>    generated_ids = torch.cat([generated_ids, next_token_ids], dim=-<span class="hljs-number">1</span>)
...
<span class="hljs-meta">... </span>    <span class="hljs-comment"># Prepare inputs for the next generation step by leaaving unprocessed tokens, in our case we have only one new token</span>
<span class="hljs-meta">... </span>    <span class="hljs-comment"># and expanding attn mask for the new token, as explained above</span>
<span class="hljs-meta">... </span>    attention_mask = inputs[<span class="hljs-string">&quot;attention_mask&quot;</span>]
<span class="hljs-meta">... </span>    attention_mask = torch.cat([attention_mask, attention_mask.new_ones((attention_mask.shape[<span class="hljs-number">0</span>], <span class="hljs-number">1</span>))], dim=-<span class="hljs-number">1</span>)
<span class="hljs-meta">... </span>    inputs = {<span class="hljs-string">&quot;input_ids&quot;</span>: next_token_ids, <span class="hljs-string">&quot;attention_mask&quot;</span>: attention_mask}
<span class="hljs-meta">... </span>    cache_position = cache_position[-<span class="hljs-number">1</span>:] + <span class="hljs-number">1</span> <span class="hljs-comment"># add one more position for the next token</span>

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(tokenizer.batch_decode(generated_ids, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>])`,wrap:!1}}),N=new b({props:{code:"JTNDJTdDdXNlciU3QyUzRSUwQUhlbGxvJTJDJTIwd2hhdCdzJTIweW91ciUyMG5hbWUuJTIwJTBBJTNDJTdDYXNzaXN0YW50JTdDJTNFJTBBTXklMjBuYW1lJTIwaXMlMjBTYXJhaC4lMjAlMEElM0MlN0M=",highlighted:`&lt;|user|&gt;
Hello, what&#x27;s your name. 
&lt;|assistant|&gt;
My name is Sarah. 
&lt;|`,wrap:!1}}),Y=new j({props:{title:"Generate with Cache",local:"generate-with-cache",headingTag:"h2"}}),H=new j({props:{title:"Quantized Cache",local:"quantized-cache",headingTag:"h3"}}),C=new rs({props:{warning:!0,$$slots:{default:[ca]},$$scope:{ctx:U}}}),A=new b({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyVGlueUxsYW1hJTJGVGlueUxsYW1hLTEuMUItQ2hhdC12MS4wJTIyKSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMlRpbnlMbGFtYSUyRlRpbnlMbGFtYS0xLjFCLUNoYXQtdjEuMCUyMiUyQyUyMHRvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNiUyQyUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyKSUwQWlucHV0cyUyMCUzRCUyMHRva2VuaXplciglMjJJJTIwbGlrZSUyMHJvY2slMjBtdXNpYyUyMGJlY2F1c2UlMjIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS50byhtb2RlbC5kZXZpY2UpJTBBJTBBb3V0JTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMlMkMlMjBkb19zYW1wbGUlM0RGYWxzZSUyQyUyMG1heF9uZXdfdG9rZW5zJTNEMjAlMkMlMjBjYWNoZV9pbXBsZW1lbnRhdGlvbiUzRCUyMnF1YW50aXplZCUyMiUyQyUyMGNhY2hlX2NvbmZpZyUzRCU3QiUyMm5iaXRzJTIyJTNBJTIwNCUyQyUyMCUyMmJhY2tlbmQlMjIlM0ElMjAlMjJxdWFudG8lMjIlN0QpJTBBcHJpbnQodG9rZW5pemVyLmJhdGNoX2RlY29kZShvdXQlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSklNUIwJTVEKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;TinyLlama/TinyLlama-1.1B-Chat-v1.0&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;TinyLlama/TinyLlama-1.1B-Chat-v1.0&quot;</span>, torch_dtype=torch.float16, device_map=<span class="hljs-string">&quot;auto&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(<span class="hljs-string">&quot;I like rock music because&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device)

<span class="hljs-meta">&gt;&gt;&gt; </span>out = model.generate(**inputs, do_sample=<span class="hljs-literal">False</span>, max_new_tokens=<span class="hljs-number">20</span>, cache_implementation=<span class="hljs-string">&quot;quantized&quot;</span>, cache_config={<span class="hljs-string">&quot;nbits&quot;</span>: <span class="hljs-number">4</span>, <span class="hljs-string">&quot;backend&quot;</span>: <span class="hljs-string">&quot;quanto&quot;</span>})
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(tokenizer.batch_decode(out, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>])
I like rock music because it<span class="hljs-string">&#x27;s a great way to express myself. I like the way it makes me feel, the</span>`,wrap:!1}}),L=new j({props:{title:"Offloaded Cache",local:"offloaded-cache",headingTag:"h3"}}),O=new b({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBY2twdCUyMCUzRCUyMCUyMm1pY3Jvc29mdCUyRlBoaS0zLW1pbmktNGstaW5zdHJ1Y3QlMjIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChja3B0KSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKGNrcHQlMkMlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiklMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoJTIyRnVuJTIwZmFjdCUzQSUyMFRoZSUyMHNob3J0ZXN0JTIyJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikudG8obW9kZWwuZGV2aWNlKSUwQSUwQW91dCUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqaW5wdXRzJTJDJTIwZG9fc2FtcGxlJTNERmFsc2UlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDIzJTJDJTIwY2FjaGVfaW1wbGVtZW50YXRpb24lM0QlMjJvZmZsb2FkZWQlMjIpJTBBcHJpbnQodG9rZW5pemVyLmJhdGNoX2RlY29kZShvdXQlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSklNUIwJTVEKSUwQSUwQW91dCUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqaW5wdXRzJTJDJTIwZG9fc2FtcGxlJTNERmFsc2UlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDIzKSUwQXByaW50KHRva2VuaXplci5iYXRjaF9kZWNvZGUob3V0JTJDJTIwc2tpcF9zcGVjaWFsX3Rva2VucyUzRFRydWUpJTVCMCU1RCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM
<span class="hljs-meta">&gt;&gt;&gt; </span>ckpt = <span class="hljs-string">&quot;microsoft/Phi-3-mini-4k-instruct&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(ckpt)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(ckpt, torch_dtype=torch.float16, device_map=<span class="hljs-string">&quot;auto&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(<span class="hljs-string">&quot;Fun fact: The shortest&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device)

<span class="hljs-meta">&gt;&gt;&gt; </span>out = model.generate(**inputs, do_sample=<span class="hljs-literal">False</span>, max_new_tokens=<span class="hljs-number">23</span>, cache_implementation=<span class="hljs-string">&quot;offloaded&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(tokenizer.batch_decode(out, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>])
Fun fact: The shortest war <span class="hljs-keyword">in</span> history was between Britain <span class="hljs-keyword">and</span> Zanzibar on August <span class="hljs-number">27</span>, <span class="hljs-number">1896.</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>out = model.generate(**inputs, do_sample=<span class="hljs-literal">False</span>, max_new_tokens=<span class="hljs-number">23</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(tokenizer.batch_decode(out, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>])
Fun fact: The shortest war <span class="hljs-keyword">in</span> history was between Britain <span class="hljs-keyword">and</span> Zanzibar on August <span class="hljs-number">27</span>, <span class="hljs-number">1896.</span>`,wrap:!1}}),I=new rs({props:{warning:!0,$$slots:{default:[pa]},$$scope:{ctx:U}}}),te=new b({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBZGVmJTIwcmVzaWxpZW50X2dlbmVyYXRlKG1vZGVsJTJDJTIwKmFyZ3MlMkMlMjAqKmt3YXJncyklM0ElMEElMjAlMjAlMjAlMjBvb20lMjAlM0QlMjBGYWxzZSUwQSUyMCUyMCUyMCUyMHRyeSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldHVybiUyMG1vZGVsLmdlbmVyYXRlKCphcmdzJTJDJTIwKiprd2FyZ3MpJTBBJTIwJTIwJTIwJTIwZXhjZXB0JTIwdG9yY2guY3VkYS5PdXRPZk1lbW9yeUVycm9yJTIwYXMlMjBlJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcHJpbnQoZSklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcmludCglMjJyZXRyeWluZyUyMHdpdGglMjBjYWNoZV9pbXBsZW1lbnRhdGlvbiUzRCdvZmZsb2FkZWQnJTIyKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG9vbSUyMCUzRCUyMFRydWUlMEElMjAlMjAlMjAlMjBpZiUyMG9vbSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHRvcmNoLmN1ZGEuZW1wdHlfY2FjaGUoKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGt3YXJncyU1QiUyMmNhY2hlX2ltcGxlbWVudGF0aW9uJTIyJTVEJTIwJTNEJTIwJTIyb2ZmbG9hZGVkJTIyJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV0dXJuJTIwbW9kZWwuZ2VuZXJhdGUoKmFyZ3MlMkMlMjAqKmt3YXJncyklMEFja3B0JTIwJTNEJTIwJTIybWljcm9zb2Z0JTJGUGhpLTMtbWluaS00ay1pbnN0cnVjdCUyMiUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKGNrcHQpJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoY2twdCUyQyUyMHRvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNikudG8oJTIyY3VkYSUzQTAlMjIpJTBBcHJvbXB0JTIwJTNEJTIwJTVCJTIyb2theSUyMCUyMioxMDAwJTIwJTJCJTIwJTIyRnVuJTIwZmFjdCUzQSUyMFRoZSUyMG1vc3QlMjIlNUQlMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIocHJvbXB0JTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikudG8obW9kZWwuZGV2aWNlKSUwQWJlYW1zJTIwJTNEJTIwJTdCJTIwJTIybnVtX2JlYW1zJTIyJTNBJTIwNDAlMkMlMjAlMjJudW1fYmVhbV9ncm91cHMlMjIlM0ElMjA0MCUyQyUyMCUyMm51bV9yZXR1cm5fc2VxdWVuY2VzJTIyJTNBJTIwNDAlMkMlMjAlMjJkaXZlcnNpdHlfcGVuYWx0eSUyMiUzQSUyMDEuMCUyQyUyMCUyMm1heF9uZXdfdG9rZW5zJTIyJTNBJTIwMjMlMkMlMjAlMjJlYXJseV9zdG9wcGluZyUyMiUzQSUyMFRydWUlMkMlMjAlN0QlMEFvdXQlMjAlM0QlMjByZXNpbGllbnRfZ2VuZXJhdGUobW9kZWwlMkMlMjAqKmlucHV0cyUyQyUyMCoqYmVhbXMpJTBBcmVzcG9uc2VzJTIwJTNEJTIwdG9rZW5pemVyLmJhdGNoX2RlY29kZShvdXQlNUIlM0ElMkMtMjglM0ElNUQlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">resilient_generate</span>(<span class="hljs-params">model, *args, **kwargs</span>):
<span class="hljs-meta">... </span>    oom = <span class="hljs-literal">False</span>
<span class="hljs-meta">... </span>    <span class="hljs-keyword">try</span>:
<span class="hljs-meta">... </span>        <span class="hljs-keyword">return</span> model.generate(*args, **kwargs)
<span class="hljs-meta">... </span>    <span class="hljs-keyword">except</span> torch.cuda.OutOfMemoryError <span class="hljs-keyword">as</span> e:
<span class="hljs-meta">... </span>        <span class="hljs-built_in">print</span>(e)
<span class="hljs-meta">... </span>        <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;retrying with cache_implementation=&#x27;offloaded&#x27;&quot;</span>)
<span class="hljs-meta">... </span>        oom = <span class="hljs-literal">True</span>
<span class="hljs-meta">... </span>    <span class="hljs-keyword">if</span> oom:
<span class="hljs-meta">... </span>        torch.cuda.empty_cache()
<span class="hljs-meta">... </span>        kwargs[<span class="hljs-string">&quot;cache_implementation&quot;</span>] = <span class="hljs-string">&quot;offloaded&quot;</span>
<span class="hljs-meta">... </span>        <span class="hljs-keyword">return</span> model.generate(*args, **kwargs)
...
...
<span class="hljs-meta">&gt;&gt;&gt; </span>ckpt = <span class="hljs-string">&quot;microsoft/Phi-3-mini-4k-instruct&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(ckpt)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(ckpt, torch_dtype=torch.float16).to(<span class="hljs-string">&quot;cuda:0&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>prompt = [<span class="hljs-string">&quot;okay &quot;</span>*<span class="hljs-number">1000</span> + <span class="hljs-string">&quot;Fun fact: The most&quot;</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device)
<span class="hljs-meta">&gt;&gt;&gt; </span>beams = { <span class="hljs-string">&quot;num_beams&quot;</span>: <span class="hljs-number">40</span>, <span class="hljs-string">&quot;num_beam_groups&quot;</span>: <span class="hljs-number">40</span>, <span class="hljs-string">&quot;num_return_sequences&quot;</span>: <span class="hljs-number">40</span>, <span class="hljs-string">&quot;diversity_penalty&quot;</span>: <span class="hljs-number">1.0</span>, <span class="hljs-string">&quot;max_new_tokens&quot;</span>: <span class="hljs-number">23</span>, <span class="hljs-string">&quot;early_stopping&quot;</span>: <span class="hljs-literal">True</span>, }
<span class="hljs-meta">&gt;&gt;&gt; </span>out = resilient_generate(model, **inputs, **beams)
<span class="hljs-meta">&gt;&gt;&gt; </span>responses = tokenizer.batch_decode(out[:,-<span class="hljs-number">28</span>:], skip_special_tokens=<span class="hljs-literal">True</span>)`,wrap:!1}}),ae=new b({props:{code:"Q1VEQSUyMG91dCUyMG9mJTIwbWVtb3J5LiUyMFRyaWVkJTIwdG8lMjBhbGxvY2F0ZSUyMDQuODMlMjBHaUIuJTIwR1BVJTBBcmV0cnlpbmclMjB3aXRoJTIwY2FjaGVfaW1wbGVtZW50YXRpb24lM0Qnb2ZmbG9hZGVkJw==",highlighted:`CUDA <span class="hljs-keyword">out</span> <span class="hljs-keyword">of</span> memory. Tried <span class="hljs-keyword">to</span> allocate <span class="hljs-number">4.83</span> GiB. GPU
retrying <span class="hljs-keyword">with</span> cache_implementation=<span class="hljs-string">&#x27;offloaded&#x27;</span>`,wrap:!1}}),ne=new j({props:{title:"Static Cache",local:"static-cache",headingTag:"h3"}}),ce=new b({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyVGlueUxsYW1hJTJGVGlueUxsYW1hLTEuMUItQ2hhdC12MS4wJTIyKSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMlRpbnlMbGFtYSUyRlRpbnlMbGFtYS0xLjFCLUNoYXQtdjEuMCUyMiUyQyUyMHRvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNiUyQyUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyKSUwQWlucHV0cyUyMCUzRCUyMHRva2VuaXplciglMjJIZWxsbyUyQyUyMG15JTIwbmFtZSUyMGlzJTIyJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikudG8obW9kZWwuZGV2aWNlKSUwQSUwQSUyMyUyMHNpbXBseSUyMHBhc3MlMjB0aGUlMjBjYWNoZSUyMGltcGxlbWVudGF0aW9uJTNEJTIyc3RhdGljJTIyJTBBb3V0JTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMlMkMlMjBkb19zYW1wbGUlM0RGYWxzZSUyQyUyMG1heF9uZXdfdG9rZW5zJTNEMjAlMkMlMjBjYWNoZV9pbXBsZW1lbnRhdGlvbiUzRCUyMnN0YXRpYyUyMiklMEF0b2tlbml6ZXIuYmF0Y2hfZGVjb2RlKG91dCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSU1QjAlNUQ=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;TinyLlama/TinyLlama-1.1B-Chat-v1.0&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;TinyLlama/TinyLlama-1.1B-Chat-v1.0&quot;</span>, torch_dtype=torch.float16, device_map=<span class="hljs-string">&quot;auto&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(<span class="hljs-string">&quot;Hello, my name is&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># simply pass the cache implementation=&quot;static&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>out = model.generate(**inputs, do_sample=<span class="hljs-literal">False</span>, max_new_tokens=<span class="hljs-number">20</span>, cache_implementation=<span class="hljs-string">&quot;static&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.batch_decode(out, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]
<span class="hljs-string">&quot;Hello, my name is [Your Name] and I am a [Your Position] at [Your Company]. I am writing&quot;</span>`,wrap:!1}}),pe=new j({props:{title:"Offloaded Static Cache",local:"offloaded-static-cache",headingTag:"h2"}}),he=new b({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIybWV0YS1sbGFtYSUyRkxsYW1hLTItN2ItY2hhdC1oZiUyMiklMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJtZXRhLWxsYW1hJTJGTGxhbWEtMi03Yi1jaGF0LWhmJTIyJTJDJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIpJTBBaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKCUyMkhlbGxvJTJDJTIwbXklMjBuYW1lJTIwaXMlMjIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS50byhtb2RlbC5kZXZpY2UpJTBBJTBBJTIzJTIwc2ltcGx5JTIwcGFzcyUyMHRoZSUyMGNhY2hlJTIwaW1wbGVtZW50YXRpb24lM0QlMjJvZmZsb2FkZWRfc3RhdGljJTIyJTBBb3V0JTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMlMkMlMjBkb19zYW1wbGUlM0RGYWxzZSUyQyUyMG1heF9uZXdfdG9rZW5zJTNEMjAlMkMlMjBjYWNoZV9pbXBsZW1lbnRhdGlvbiUzRCUyMm9mZmxvYWRlZF9zdGF0aWMlMjIpJTBBdG9rZW5pemVyLmJhdGNoX2RlY29kZShvdXQlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSklNUIwJTVE",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-2-7b-chat-hf&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-2-7b-chat-hf&quot;</span>, torch_dtype=torch.float16, device_map=<span class="hljs-string">&quot;auto&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(<span class="hljs-string">&quot;Hello, my name is&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># simply pass the cache implementation=&quot;offloaded_static&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>out = model.generate(**inputs, do_sample=<span class="hljs-literal">False</span>, max_new_tokens=<span class="hljs-number">20</span>, cache_implementation=<span class="hljs-string">&quot;offloaded_static&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.batch_decode(out, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]
<span class="hljs-string">&quot;Hello, my name is [Your Name], and I am a [Your Profession] with [Number of Years] of&quot;</span>`,wrap:!1}}),de=new j({props:{title:"Sliding Window Cache",local:"sliding-window-cache",headingTag:"h3"}}),Me=new b({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwU2lua0NhY2hlJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIydGVrbml1bSUyRk9wZW5IZXJtZXMtMi41LU1pc3RyYWwtN0IlMjIpJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTIydGVrbml1bSUyRk9wZW5IZXJtZXMtMi41LU1pc3RyYWwtN0IlMjIlMkMlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiklMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoJTIyWWVzdGVyZGF5JTIwSSUyMHdhcyUyMG9uJTIwYSUyMHJvY2slMjBjb25jZXJ0JTIwYW5kLiUyMiUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnRvKG1vZGVsLmRldmljZSklMEElMEElMjMlMjBjYW4lMjBiZSUyMHVzZWQlMjBieSUyMHBhc3NpbmclMjBpbiUyMGNhY2hlJTIwaW1wbGVtZW50YXRpb24lMEFvdXQlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSgqKmlucHV0cyUyQyUyMGRvX3NhbXBsZSUzREZhbHNlJTJDJTIwbWF4X25ld190b2tlbnMlM0QzMCUyQyUyMGNhY2hlX2ltcGxlbWVudGF0aW9uJTNEJTIyc2xpZGluZ193aW5kb3clMjIpJTBBdG9rZW5pemVyLmJhdGNoX2RlY29kZShvdXQlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSklNUIwJTVE",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM, SinkCache

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;teknium/OpenHermes-2.5-Mistral-7B&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;teknium/OpenHermes-2.5-Mistral-7B&quot;</span>, torch_dtype=torch.float16, device_map=<span class="hljs-string">&quot;auto&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(<span class="hljs-string">&quot;Yesterday I was on a rock concert and.&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># can be used by passing in cache implementation</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>out = model.generate(**inputs, do_sample=<span class="hljs-literal">False</span>, max_new_tokens=<span class="hljs-number">30</span>, cache_implementation=<span class="hljs-string">&quot;sliding_window&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.batch_decode(out, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]
<span class="hljs-string">&quot;Yesterday I was on a rock concert and. I was so excited to see my favorite band perform live. I was so happy that I could hardly contain myself. I was jumping up and down and&quot;</span>`,wrap:!1}}),ge=new j({props:{title:"Sink Cache",local:"sink-cache",headingTag:"h3"}}),Je=new b({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwU2lua0NhY2hlJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyVGlueUxsYW1hJTJGVGlueUxsYW1hLTEuMUItQ2hhdC12MS4wJTIyKSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMlRpbnlMbGFtYSUyRlRpbnlMbGFtYS0xLjFCLUNoYXQtdjEuMCUyMiUyQyUyMHRvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNiUyQyUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyKSUwQWlucHV0cyUyMCUzRCUyMHRva2VuaXplciglMjJUaGlzJTIwaXMlMjBhJTIwbG9uZyUyMHN0b3J5JTIwYWJvdXQlMjB1bmljb3JucyUyQyUyMGZhaXJpZXMlMjBhbmQlMjBtYWdpYy4lMjIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS50byhtb2RlbC5kZXZpY2UpJTBBJTBBJTIzJTIwZ2V0JTIwb3VyJTIwY2FjaGUlMkMlMjBzcGVjaWZ5JTIwbnVtYmVyJTIwb2YlMjBzaW5rJTIwdG9rZW5zJTIwYW5kJTIwd2luZG93JTIwc2l6ZSUwQSUyMyUyME5vdGUlMjB0aGF0JTIwd2luZG93JTIwc2l6ZSUyMGFscmVhZHklMjBpbmNsdWRlcyUyMHNpbmslMjB0b2tlbnMlMkMlMjBzbyUyMGhhcyUyMHRvJTIwYmUlMjBsYXJnZXIlMEFwYXN0X2tleV92YWx1ZXMlMjAlM0QlMjBTaW5rQ2FjaGUod2luZG93X2xlbmd0aCUzRDI1NiUyQyUyMG51bV9zaW5rX3Rva2VucyUzRDQpJTBBb3V0JTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMlMkMlMjBkb19zYW1wbGUlM0RGYWxzZSUyQyUyMG1heF9uZXdfdG9rZW5zJTNEMzAlMkMlMjBwYXN0X2tleV92YWx1ZXMlM0RwYXN0X2tleV92YWx1ZXMpJTBBdG9rZW5pemVyLmJhdGNoX2RlY29kZShvdXQlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSklNUIwJTVE",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM, SinkCache

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;TinyLlama/TinyLlama-1.1B-Chat-v1.0&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;TinyLlama/TinyLlama-1.1B-Chat-v1.0&quot;</span>, torch_dtype=torch.float16, device_map=<span class="hljs-string">&quot;auto&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(<span class="hljs-string">&quot;This is a long story about unicorns, fairies and magic.&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># get our cache, specify number of sink tokens and window size</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Note that window size already includes sink tokens, so has to be larger</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>past_key_values = SinkCache(window_length=<span class="hljs-number">256</span>, num_sink_tokens=<span class="hljs-number">4</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>out = model.generate(**inputs, do_sample=<span class="hljs-literal">False</span>, max_new_tokens=<span class="hljs-number">30</span>, past_key_values=past_key_values)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.batch_decode(out, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]
<span class="hljs-string">&quot;This is a long story about unicorns, fairies and magic. It is a story about a young girl named Lily who discovers that she has the power to control the elements. She learns that she can&quot;</span>`,wrap:!1}}),Te=new j({props:{title:"Encoder-Decoder Cache",local:"encoder-decoder-cache",headingTag:"h3"}}),Ue=new j({props:{title:"Model-specific Cache Classes",local:"model-specific-cache-classes",headingTag:"h3"}}),Ce=new j({props:{title:"Iterative Generation with Cache",local:"iterative-generation-with-cache",headingTag:"h2"}}),Be=new b({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQ0F1dG9Nb2RlbEZvckNhdXNhbExNJTBBZnJvbSUyMHRyYW5zZm9ybWVycy5jYWNoZV91dGlscyUyMGltcG9ydCUyMCglMEElMjAlMjAlMjBEeW5hbWljQ2FjaGUlMkMlMEElMjAlMjAlMjBTaW5rQ2FjaGUlMkMlMEElMjAlMjAlMjBTdGF0aWNDYWNoZSUyQyUwQSUyMCUyMCUyMFNsaWRpbmdXaW5kb3dDYWNoZSUyQyUwQSUyMCUyMCUyMFF1YW50b1F1YW50aXplZENhY2hlJTJDJTBBJTIwJTIwJTIwUXVhbnRpemVkQ2FjaGVDb25maWclMkMlMEEpJTBBJTBBbW9kZWxfaWQlMjAlM0QlMjAlMjJUaW55TGxhbWElMkZUaW55TGxhbWEtMS4xQi1DaGF0LXYxLjAlMjIlMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZChtb2RlbF9pZCUyQyUyMHRvcmNoX2R0eXBlJTNEdG9yY2guYmZsb2F0MTYlMkMlMjBkZXZpY2VfbWFwJTNEJ2F1dG8nKSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkKSUwQSUwQXVzZXJfcHJvbXB0cyUyMCUzRCUyMCU1QiUyMkhlbGxvJTJDJTIwd2hhdCdzJTIweW91ciUyMG5hbWUlM0YlMjIlMkMlMjAlMjJCdHclMkMlMjB5ZXN0ZXJkYXklMjBJJTIwd2FzJTIwb24lMjBhJTIwcm9jayUyMGNvbmNlcnQuJTIyJTVEJTBBJTBBcGFzdF9rZXlfdmFsdWVzJTIwJTNEJTIwRHluYW1pY0NhY2hlKCklMEFtYXhfY2FjaGVfbGVuZ3RoJTIwJTNEJTIwcGFzdF9rZXlfdmFsdWVzLmdldF9tYXhfY2FjaGVfc2hhcGUoKSUwQSUwQW1lc3NhZ2VzJTIwJTNEJTIwJTVCJTVEJTBBZm9yJTIwcHJvbXB0JTIwaW4lMjB1c2VyX3Byb21wdHMlM0ElMEElMjAlMjAlMjAlMjBtZXNzYWdlcy5hcHBlbmQoJTdCJTIycm9sZSUyMiUzQSUyMCUyMnVzZXIlMjIlMkMlMjAlMjJjb250ZW50JTIyJTNBJTIwcHJvbXB0JTdEKSUwQSUyMCUyMCUyMCUyMGlucHV0cyUyMCUzRCUyMHRva2VuaXplci5hcHBseV9jaGF0X3RlbXBsYXRlKG1lc3NhZ2VzJTJDJTIwYWRkX2dlbmVyYXRpb25fcHJvbXB0JTNEVHJ1ZSUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIlMkMlMjByZXR1cm5fZGljdCUzRFRydWUpLnRvKG1vZGVsLmRldmljZSklMEElMjAlMjAlMjAlMjBpZiUyMGlzaW5zdGFuY2UocGFzdF9rZXlfdmFsdWVzJTJDJTIwU2lua0NhY2hlKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlucHV0cyUyMCUzRCUyMCU3QmslM0ElMjB2JTVCJTNBJTJDJTIwLW1heF9jYWNoZV9sZW5ndGglM0ElNUQlMjBmb3IlMjBrJTJDJTIwdiUyMGluJTIwaW5wdXRzLml0ZW1zKCklN0QlMEElMjAlMjAlMjAlMjBpbnB1dF9sZW5ndGglMjAlM0QlMjBpbnB1dHMlNUIlMjJpbnB1dF9pZHMlMjIlNUQuc2hhcGUlNUIxJTVEJTBBJTIwJTIwJTIwJTIwb3V0cHV0cyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqaW5wdXRzJTJDJTIwZG9fc2FtcGxlJTNERmFsc2UlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDI1NiUyQyUyMHBhc3Rfa2V5X3ZhbHVlcyUzRHBhc3Rfa2V5X3ZhbHVlcyklMEElMjAlMjAlMjAlMjBjb21wbGV0aW9uJTIwJTNEJTIwdG9rZW5pemVyLmRlY29kZShvdXRwdXRzJTVCMCUyQyUyMGlucHV0X2xlbmd0aCUzQSUyMCU1RCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSUwQSUyMCUyMCUyMCUyMG1lc3NhZ2VzLmFwcGVuZCglN0IlMjJyb2xlJTIyJTNBJTIwJTIyYXNzaXN0YW50JTIyJTJDJTIwJTIyY29udGVudCUyMiUzQSUyMGNvbXBsZXRpb24lN0QpJTBB",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer,AutoModelForCausalLM
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers.cache_utils <span class="hljs-keyword">import</span> (
<span class="hljs-meta">... </span>   DynamicCache,
<span class="hljs-meta">... </span>   SinkCache,
<span class="hljs-meta">... </span>   StaticCache,
<span class="hljs-meta">... </span>   SlidingWindowCache,
<span class="hljs-meta">... </span>   QuantoQuantizedCache,
<span class="hljs-meta">... </span>   QuantizedCacheConfig,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>model_id = <span class="hljs-string">&quot;TinyLlama/TinyLlama-1.1B-Chat-v1.0&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.bfloat16, device_map=<span class="hljs-string">&#x27;auto&#x27;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(model_id)

<span class="hljs-meta">&gt;&gt;&gt; </span>user_prompts = [<span class="hljs-string">&quot;Hello, what&#x27;s your name?&quot;</span>, <span class="hljs-string">&quot;Btw, yesterday I was on a rock concert.&quot;</span>]

<span class="hljs-meta">&gt;&gt;&gt; </span>past_key_values = DynamicCache()
<span class="hljs-meta">&gt;&gt;&gt; </span>max_cache_length = past_key_values.get_max_cache_shape()

<span class="hljs-meta">&gt;&gt;&gt; </span>messages = []
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> prompt <span class="hljs-keyword">in</span> user_prompts:
<span class="hljs-meta">... </span>    messages.append({<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: prompt})
<span class="hljs-meta">... </span>    inputs = tokenizer.apply_chat_template(messages, add_generation_prompt=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>, return_dict=<span class="hljs-literal">True</span>).to(model.device)
<span class="hljs-meta">... </span>    <span class="hljs-keyword">if</span> <span class="hljs-built_in">isinstance</span>(past_key_values, SinkCache):
<span class="hljs-meta">... </span>        inputs = {k: v[:, -max_cache_length:] <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> inputs.items()}
...
<span class="hljs-meta">... </span>    input_length = inputs[<span class="hljs-string">&quot;input_ids&quot;</span>].shape[<span class="hljs-number">1</span>]
...
<span class="hljs-meta">... </span>    outputs = model.generate(**inputs, do_sample=<span class="hljs-literal">False</span>, max_new_tokens=<span class="hljs-number">256</span>, past_key_values=past_key_values)
<span class="hljs-meta">... </span>    completion = tokenizer.decode(outputs[<span class="hljs-number">0</span>, input_length: ], skip_special_tokens=<span class="hljs-literal">True</span>)
<span class="hljs-meta">... </span>    messages.append({<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;assistant&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: completion})

<span class="hljs-built_in">print</span>(messages)
[{<span class="hljs-string">&#x27;role&#x27;</span>: <span class="hljs-string">&#x27;user&#x27;</span>, <span class="hljs-string">&#x27;content&#x27;</span>: <span class="hljs-string">&quot;Hello, what&#x27;s your name?&quot;</span>}, {<span class="hljs-string">&#x27;role&#x27;</span>: <span class="hljs-string">&#x27;assistant&#x27;</span>, <span class="hljs-string">&#x27;content&#x27;</span>: <span class="hljs-string">&quot;Hello, I&#x27;m AI.&quot;</span>}, {<span class="hljs-string">&#x27;role&#x27;</span>: <span class="hljs-string">&#x27;user&#x27;</span>, <span class="hljs-string">&#x27;content&#x27;</span>: <span class="hljs-string">&#x27;Btw, yesterday I was on a rock concert.&#x27;</span>}, {<span class="hljs-string">&#x27;role&#x27;</span>: <span class="hljs-string">&#x27;assistant&#x27;</span>, <span class="hljs-string">&#x27;content&#x27;</span>: <span class="hljs-string">&quot;I&#x27;m sorry to hear that you were on a rock concert yesterday. It sounds like a fun experience, but I&#x27;m not capable of experiencing music or concerts. However, I can provide you with some information about rock music and its history. Rock music emerged in the 1950s and 1960s in the United States and Britain, and it quickly gained popularity around the world. Some of the most famous rock bands of all time include The Beatles, The Rolling Stones, Led Zeppelin, and Pink Floyd. Rock music has a distinct sound and style, with elements of blues, country, and folk music. It often features guitar solos, heavy bass lines, and drums. Rock music has had a significant impact on popular culture, influencing genres such as punk rock, heavy metal, and alternative rock.&quot;</span>}]`,wrap:!1}}),We=new j({props:{title:"Re-use Cache to continue generation",local:"re-use-cache-to-continue-generation",headingTag:"h2"}}),Xe=new b({props:{code:"aW1wb3J0JTIwY29weSUwQWltcG9ydCUyMHRvcmNoJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUyQyUyMER5bmFtaWNDYWNoZSUyQyUyMFN0YXRpY0NhY2hlJTBBZnJvbSUyMGFjY2VsZXJhdGUudGVzdF91dGlscy50ZXN0aW5nJTIwaW1wb3J0JTIwZ2V0X2JhY2tlbmQlMEElMEFERVZJQ0UlMkMlMjBfJTJDJTIwXyUyMCUzRCUyMGdldF9iYWNrZW5kKCklMjAlMjMlMjBhdXRvbWF0aWNhbGx5JTIwZGV0ZWN0cyUyMHRoZSUyMHVuZGVybHlpbmclMjBkZXZpY2UlMjB0eXBlJTIwKENVREElMkMlMjBDUFUlMkMlMjBYUFUlMkMlMjBNUFMlMkMlMjBldGMuKSUwQW1vZGVsX2lkJTIwJTNEJTIwJTIyVGlueUxsYW1hJTJGVGlueUxsYW1hLTEuMUItQ2hhdC12MS4wJTIyJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQlMkMlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2JTJDJTIwZGV2aWNlX21hcCUzRERFVklDRSklMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChtb2RlbF9pZCklMEElMEElMjMlMjBJbml0JTIwU3RhdGljQ2FjaGUlMjB3aXRoJTIwYmlnJTIwZW5vdWdoJTIwbWF4LWxlbmd0aCUyMCgxMDI0JTIwdG9rZW5zJTIwZm9yJTIwdGhlJTIwYmVsb3clMjBleGFtcGxlKSUwQSUyMyUyMFlvdSUyMGNhbiUyMGFsc28lMjBpbml0JTIwYSUyMER5bmFtaWNDYWNoZSUyQyUyMGlmJTIwdGhhdCUyMHN1aXRzJTIweW91JTIwYmV0dGVyJTBBcHJvbXB0X2NhY2hlJTIwJTNEJTIwU3RhdGljQ2FjaGUoY29uZmlnJTNEbW9kZWwuY29uZmlnJTJDJTIwbWF4X2JhdGNoX3NpemUlM0QxJTJDJTIwbWF4X2NhY2hlX2xlbiUzRDEwMjQlMkMlMjBkZXZpY2UlM0RERVZJQ0UlMkMlMjBkdHlwZSUzRHRvcmNoLmJmbG9hdDE2KSUwQSUwQUlOSVRJQUxfUFJPTVBUJTIwJTNEJTIwJTIyWW91JTIwYXJlJTIwYSUyMGhlbHBmdWwlMjBhc3Npc3RhbnQuJTIwJTIyJTBBaW5wdXRzX2luaXRpYWxfcHJvbXB0JTIwJTNEJTIwdG9rZW5pemVyKElOSVRJQUxfUFJPTVBUJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikudG8oREVWSUNFKSUwQSUyMyUyMFRoaXMlMjBpcyUyMHRoZSUyMGNvbW1vbiUyMHByb21wdCUyMGNhY2hlZCUyQyUyMHdlJTIwbmVlZCUyMHRvJTIwcnVuJTIwZm9yd2FyZCUyMHdpdGhvdXQlMjBncmFkJTIwdG8lMjBiZSUyMGFiZWwlMjB0byUyMGNvcHklMEF3aXRoJTIwdG9yY2gubm9fZ3JhZCgpJTNBJTBBJTIwJTIwJTIwJTIwJTIwcHJvbXB0X2NhY2hlJTIwJTNEJTIwbW9kZWwoKippbnB1dHNfaW5pdGlhbF9wcm9tcHQlMkMlMjBwYXN0X2tleV92YWx1ZXMlMjAlM0QlMjBwcm9tcHRfY2FjaGUpLnBhc3Rfa2V5X3ZhbHVlcyUwQSUwQXByb21wdHMlMjAlM0QlMjAlNUIlMjJIZWxwJTIwbWUlMjB0byUyMHdyaXRlJTIwYSUyMGJsb2dwb3N0JTIwYWJvdXQlMjB0cmF2ZWxsaW5nLiUyMiUyQyUyMCUyMldoYXQlMjBpcyUyMHRoZSUyMGNhcGl0YWwlMjBvZiUyMEZyYW5jZSUzRiUyMiU1RCUwQXJlc3BvbnNlcyUyMCUzRCUyMCU1QiU1RCUwQWZvciUyMHByb21wdCUyMGluJTIwcHJvbXB0cyUzQSUwQSUyMCUyMCUyMCUyMG5ld19pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoSU5JVElBTF9QUk9NUFQlMjAlMkIlMjBwcm9tcHQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS50byhERVZJQ0UpJTBBJTIwJTIwJTIwJTIwcGFzdF9rZXlfdmFsdWVzJTIwJTNEJTIwY29weS5kZWVwY29weShwcm9tcHRfY2FjaGUpJTBBJTIwJTIwJTIwJTIwb3V0cHV0cyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqbmV3X2lucHV0cyUyQyUyMHBhc3Rfa2V5X3ZhbHVlcyUzRHBhc3Rfa2V5X3ZhbHVlcyUyQ21heF9uZXdfdG9rZW5zJTNEMjApJTBBJTIwJTIwJTIwJTIwcmVzcG9uc2UlMjAlM0QlMjB0b2tlbml6ZXIuYmF0Y2hfZGVjb2RlKG91dHB1dHMpJTVCMCU1RCUwQSUyMCUyMCUyMCUyMHJlc3BvbnNlcy5hcHBlbmQocmVzcG9uc2UpJTBBJTBBcHJpbnQocmVzcG9uc2VzKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> copy
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, DynamicCache, StaticCache
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> accelerate.test_utils.testing <span class="hljs-keyword">import</span> get_backend

<span class="hljs-meta">&gt;&gt;&gt; </span>DEVICE, _, _ = get_backend() <span class="hljs-comment"># automatically detects the underlying device type (CUDA, CPU, XPU, MPS, etc.)</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model_id = <span class="hljs-string">&quot;TinyLlama/TinyLlama-1.1B-Chat-v1.0&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.bfloat16, device_map=DEVICE)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(model_id)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Init StaticCache with big enough max-length (1024 tokens for the below example)</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># You can also init a DynamicCache, if that suits you better</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>prompt_cache = StaticCache(config=model.config, max_batch_size=<span class="hljs-number">1</span>, max_cache_len=<span class="hljs-number">1024</span>, device=DEVICE, dtype=torch.bfloat16)

<span class="hljs-meta">&gt;&gt;&gt; </span>INITIAL_PROMPT = <span class="hljs-string">&quot;You are a helpful assistant. &quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs_initial_prompt = tokenizer(INITIAL_PROMPT, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(DEVICE)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># This is the common prompt cached, we need to run forward without grad to be abel to copy</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">with</span> torch.no_grad():
<span class="hljs-meta">... </span>     prompt_cache = model(**inputs_initial_prompt, past_key_values = prompt_cache).past_key_values

<span class="hljs-meta">&gt;&gt;&gt; </span>prompts = [<span class="hljs-string">&quot;Help me to write a blogpost about travelling.&quot;</span>, <span class="hljs-string">&quot;What is the capital of France?&quot;</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>responses = []
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> prompt <span class="hljs-keyword">in</span> prompts:
<span class="hljs-meta">... </span>    new_inputs = tokenizer(INITIAL_PROMPT + prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(DEVICE)
<span class="hljs-meta">... </span>    past_key_values = copy.deepcopy(prompt_cache)
<span class="hljs-meta">... </span>    outputs = model.generate(**new_inputs, past_key_values=past_key_values,max_new_tokens=<span class="hljs-number">20</span>)
<span class="hljs-meta">... </span>    response = tokenizer.batch_decode(outputs)[<span class="hljs-number">0</span>]
<span class="hljs-meta">... </span>    responses.append(response)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(responses)
[<span class="hljs-string">&#x27;&lt;s&gt; You are a helpful assistant. Help me to write a blogpost about travelling.  I am excited to share my experiences with you.  I have been traveling for the past&#x27;</span>, <span class="hljs-string">&#x27;&lt;s&gt; You are a helpful assistant. What is the capital of France? \\n\\nAnswer: Paris is the capital of France.&lt;/s&gt;&#x27;</span>]`,wrap:!1}}),Ve=new j({props:{title:"Legacy cache format",local:"legacy-cache-format",headingTag:"h2"}}),Re=new b({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwRHluYW1pY0NhY2hlJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyVGlueUxsYW1hJTJGVGlueUxsYW1hLTEuMUItQ2hhdC12MS4wJTIyKSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMlRpbnlMbGFtYSUyRlRpbnlMbGFtYS0xLjFCLUNoYXQtdjEuMCUyMiUyQyUyMHRvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNiUyQyUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyKSUwQWlucHV0cyUyMCUzRCUyMHRva2VuaXplciglMjJIZWxsbyUyQyUyMG15JTIwbmFtZSUyMGlzJTIyJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikudG8obW9kZWwuZGV2aWNlKSUwQSUwQSUyMyUyMCU2MHJldHVybl9kaWN0X2luX2dlbmVyYXRlJTNEVHJ1ZSU2MCUyMGlzJTIwcmVxdWlyZWQlMjB0byUyMHJldHVybiUyMHRoZSUyMGNhY2hlLiUyMCU2MHJldHVybl9sZWdhY3lfY2FjaGUlNjAlMjBmb3JjZXMlMjB0aGUlMjByZXR1cm5lZCUyMGNhY2hlJTBBJTIzJTIwdG8lMjBiZSUyMG9mJTIwdGhlJTIwbGVnYWN5JTIwdHlwZSUwQWdlbmVyYXRpb25fb3V0cHV0cyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqaW5wdXRzJTJDJTIwcmV0dXJuX2RpY3RfaW5fZ2VuZXJhdGUlM0RUcnVlJTJDJTIwcmV0dXJuX2xlZ2FjeV9jYWNoZSUzRFRydWUlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDUpJTBBJTBBJTIzJTIwV2UlMjBjYW4lMjBjb252ZXJ0JTIwYSUyMGxlZ2FjeSUyMGNhY2hlJTIwdG8lMjBhJTIwRHluYW1pY0NhY2hlJTIwLS0lMjBhbmQlMjB0aGUlMjBvdGhlciUyMHdheSUyMGFyb3VuZC4lMjBUaGlzJTIwaXMlMjBoZWxwZnVsJTIwaWYlMjB5b3UlMjBoYXZlJTIwY3VzdG9tJTBBJTIzJTIwbG9naWMlMjB0byUyMG1hbmlwdWxhdGUlMjBhJTIwY2FjaGUlMjBpbiUyMGElMjBzcGVjaWZpYyUyMGZvcm1hdC4lMEFjYWNoZSUyMCUzRCUyMER5bmFtaWNDYWNoZS5mcm9tX2xlZ2FjeV9jYWNoZShnZW5lcmF0aW9uX291dHB1dHMucGFzdF9rZXlfdmFsdWVzKSUwQWxlZ2FjeV9mb3JtYXRfY2FjaGUlMjAlM0QlMjBjYWNoZS50b19sZWdhY3lfY2FjaGUoKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM, DynamicCache

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;TinyLlama/TinyLlama-1.1B-Chat-v1.0&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;TinyLlama/TinyLlama-1.1B-Chat-v1.0&quot;</span>, torch_dtype=torch.float16, device_map=<span class="hljs-string">&quot;auto&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(<span class="hljs-string">&quot;Hello, my name is&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># \`return_dict_in_generate=True\` is required to return the cache. \`return_legacy_cache\` forces the returned cache</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># to be of the legacy type</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>generation_outputs = model.generate(**inputs, return_dict_in_generate=<span class="hljs-literal">True</span>, return_legacy_cache=<span class="hljs-literal">True</span>, max_new_tokens=<span class="hljs-number">5</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># We can convert a legacy cache to a DynamicCache -- and the other way around. This is helpful if you have custom</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># logic to manipulate a cache in a specific format.</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>cache = DynamicCache.from_legacy_cache(generation_outputs.past_key_values)
<span class="hljs-meta">&gt;&gt;&gt; </span>legacy_format_cache = cache.to_legacy_cache()`,wrap:!1}}),Ne=new oa({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/kv_cache.md"}}),{c(){p=o("meta"),T=l(),M=o("p"),f=l(),r(Z.$$.fragment),Ee=l(),v=o("p"),v.textContent=ms,$e=l(),B=o("p"),B.innerHTML=ds,Se=l(),r(W.$$.fragment),qe=l(),G=o("p"),G.textContent=us,Ae=l(),X=o("p"),X.textContent=ys,Le=l(),V=o("p"),V.textContent=Ms,De=l(),g=o("details"),Ye=o("summary"),Ye.innerHTML=gs,ts=l(),r(_.$$.fragment),ss=l(),Qe=o("p"),Qe.textContent=ws,as=l(),xe=o("p"),xe.innerHTML=js,ls=l(),ze=o("p"),ze.innerHTML=Js,ns=l(),r(k.$$.fragment),os=l(),Fe=o("p"),Fe.textContent=Ts,is=l(),r(R.$$.fragment),cs=l(),r(N.$$.fragment),Ke=l(),r(Y.$$.fragment),Pe=l(),Q=o("p"),Q.innerHTML=bs,Oe=l(),x=o("p"),x.innerHTML=fs,et=l(),z=o("table"),z.innerHTML=Us,tt=l(),F=o("p"),F.innerHTML=ks,st=l(),r(H.$$.fragment),at=l(),E=o("p"),E.innerHTML=Cs,lt=l(),$=o("p"),$.innerHTML=Is,nt=l(),S=o("p"),S.innerHTML=Zs,ot=l(),q=o("p"),q.innerHTML=vs,it=l(),r(C.$$.fragment),ct=l(),r(A.$$.fragment),pt=l(),r(L.$$.fragment),rt=l(),D=o("p"),D.innerHTML=Bs,ht=l(),K=o("p"),K.textContent=Ws,mt=l(),P=o("p"),P.innerHTML=Gs,dt=l(),r(O.$$.fragment),ut=l(),r(I.$$.fragment),yt=l(),ee=o("p"),ee.textContent=Xs,Mt=l(),r(te.$$.fragment),gt=l(),se=o("p"),se.textContent=Vs,wt=l(),r(ae.$$.fragment),jt=l(),le=o("p"),le.textContent=_s,Jt=l(),r(ne.$$.fragment),Tt=l(),oe=o("p"),oe.innerHTML=Rs,bt=l(),ie=o("p"),ie.innerHTML=Ns,ft=l(),r(ce.$$.fragment),Ut=l(),r(pe.$$.fragment),kt=l(),re=o("p"),re.innerHTML=Ys,Ct=l(),r(he.$$.fragment),It=l(),me=o("p"),me.textContent=Qs,Zt=l(),r(de.$$.fragment),vt=l(),ue=o("p"),ue.innerHTML=xs,Bt=l(),ye=o("p"),ye.textContent=zs,Wt=l(),r(Me.$$.fragment),Gt=l(),r(ge.$$.fragment),Xt=l(),we=o("p"),we.innerHTML=Fs,Vt=l(),je=o("p"),je.innerHTML=Hs,_t=l(),r(Je.$$.fragment),Rt=l(),r(Te.$$.fragment),Nt=l(),be=o("p"),be.innerHTML=Es,Yt=l(),fe=o("p"),fe.innerHTML=$s,Qt=l(),r(Ue.$$.fragment),xt=l(),ke=o("p"),ke.innerHTML=Ss,zt=l(),r(Ce.$$.fragment),Ft=l(),Ie=o("p"),Ie.textContent=qs,Ht=l(),Ze=o("p"),Ze.innerHTML=As,Et=l(),ve=o("p"),ve.textContent=Ls,$t=l(),r(Be.$$.fragment),St=l(),r(We.$$.fragment),qt=l(),Ge=o("p"),Ge.innerHTML=Ds,At=l(),r(Xe.$$.fragment),Lt=l(),r(Ve.$$.fragment),Dt=l(),_e=o("p"),_e.innerHTML=Ks,Kt=l(),r(Re.$$.fragment),Pt=l(),r(Ne.$$.fragment),Ot=l(),He=o("p"),this.h()},l(e){const t=na("svelte-u9bgzb",document.head);p=i(t,"META",{name:!0,content:!0}),t.forEach(s),T=n(e),M=i(e,"P",{}),ps(M).forEach(s),f=n(e),h(Z.$$.fragment,e),Ee=n(e),v=i(e,"P",{"data-svelte-h":!0}),c(v)!=="svelte-ivdsz6"&&(v.textContent=ms),$e=n(e),B=i(e,"P",{"data-svelte-h":!0}),c(B)!=="svelte-1ru3n2s"&&(B.innerHTML=ds),Se=n(e),h(W.$$.fragment,e),qe=n(e),G=i(e,"P",{"data-svelte-h":!0}),c(G)!=="svelte-pgg2dh"&&(G.textContent=us),Ae=n(e),X=i(e,"P",{"data-svelte-h":!0}),c(X)!=="svelte-1pcidqj"&&(X.textContent=ys),Le=n(e),V=i(e,"P",{"data-svelte-h":!0}),c(V)!=="svelte-19ueo1l"&&(V.textContent=Ms),De=n(e),g=i(e,"DETAILS",{});var w=ps(g);Ye=i(w,"SUMMARY",{"data-svelte-h":!0}),c(Ye)!=="svelte-8dl4q2"&&(Ye.innerHTML=gs),ts=n(w),h(_.$$.fragment,w),ss=n(w),Qe=i(w,"P",{"data-svelte-h":!0}),c(Qe)!=="svelte-1e25at2"&&(Qe.textContent=ws),as=n(w),xe=i(w,"P",{"data-svelte-h":!0}),c(xe)!=="svelte-1l5g8w7"&&(xe.innerHTML=js),ls=n(w),ze=i(w,"P",{"data-svelte-h":!0}),c(ze)!=="svelte-8mjlg7"&&(ze.innerHTML=Js),ns=n(w),h(k.$$.fragment,w),os=n(w),Fe=i(w,"P",{"data-svelte-h":!0}),c(Fe)!=="svelte-181iwu3"&&(Fe.textContent=Ts),is=n(w),h(R.$$.fragment,w),cs=n(w),h(N.$$.fragment,w),w.forEach(s),Ke=n(e),h(Y.$$.fragment,e),Pe=n(e),Q=i(e,"P",{"data-svelte-h":!0}),c(Q)!=="svelte-1sp8xhf"&&(Q.innerHTML=bs),Oe=n(e),x=i(e,"P",{"data-svelte-h":!0}),c(x)!=="svelte-ez9zp4"&&(x.innerHTML=fs),et=n(e),z=i(e,"TABLE",{"data-svelte-h":!0}),c(z)!=="svelte-yhqrld"&&(z.innerHTML=Us),tt=n(e),F=i(e,"P",{"data-svelte-h":!0}),c(F)!=="svelte-b5pfm0"&&(F.innerHTML=ks),st=n(e),h(H.$$.fragment,e),at=n(e),E=i(e,"P",{"data-svelte-h":!0}),c(E)!=="svelte-1yajrsk"&&(E.innerHTML=Cs),lt=n(e),$=i(e,"P",{"data-svelte-h":!0}),c($)!=="svelte-1so9rnx"&&($.innerHTML=Is),nt=n(e),S=i(e,"P",{"data-svelte-h":!0}),c(S)!=="svelte-gznk4o"&&(S.innerHTML=Zs),ot=n(e),q=i(e,"P",{"data-svelte-h":!0}),c(q)!=="svelte-1ke22x8"&&(q.innerHTML=vs),it=n(e),h(C.$$.fragment,e),ct=n(e),h(A.$$.fragment,e),pt=n(e),h(L.$$.fragment,e),rt=n(e),D=i(e,"P",{"data-svelte-h":!0}),c(D)!=="svelte-uj0xty"&&(D.innerHTML=Bs),ht=n(e),K=i(e,"P",{"data-svelte-h":!0}),c(K)!=="svelte-1x4pr6r"&&(K.textContent=Ws),mt=n(e),P=i(e,"P",{"data-svelte-h":!0}),c(P)!=="svelte-6aujat"&&(P.innerHTML=Gs),dt=n(e),h(O.$$.fragment,e),ut=n(e),h(I.$$.fragment,e),yt=n(e),ee=i(e,"P",{"data-svelte-h":!0}),c(ee)!=="svelte-f3m4wg"&&(ee.textContent=Xs),Mt=n(e),h(te.$$.fragment,e),gt=n(e),se=i(e,"P",{"data-svelte-h":!0}),c(se)!=="svelte-opu10i"&&(se.textContent=Vs),wt=n(e),h(ae.$$.fragment,e),jt=n(e),le=i(e,"P",{"data-svelte-h":!0}),c(le)!=="svelte-83p2wm"&&(le.textContent=_s),Jt=n(e),h(ne.$$.fragment,e),Tt=n(e),oe=i(e,"P",{"data-svelte-h":!0}),c(oe)!=="svelte-1js08qo"&&(oe.innerHTML=Rs),bt=n(e),ie=i(e,"P",{"data-svelte-h":!0}),c(ie)!=="svelte-hxs9qp"&&(ie.innerHTML=Ns),ft=n(e),h(ce.$$.fragment,e),Ut=n(e),h(pe.$$.fragment,e),kt=n(e),re=i(e,"P",{"data-svelte-h":!0}),c(re)!=="svelte-1hku8s1"&&(re.innerHTML=Ys),Ct=n(e),h(he.$$.fragment,e),It=n(e),me=i(e,"P",{"data-svelte-h":!0}),c(me)!=="svelte-zdzdtx"&&(me.textContent=Qs),Zt=n(e),h(de.$$.fragment,e),vt=n(e),ue=i(e,"P",{"data-svelte-h":!0}),c(ue)!=="svelte-krlgtc"&&(ue.innerHTML=xs),Bt=n(e),ye=i(e,"P",{"data-svelte-h":!0}),c(ye)!=="svelte-14gr42g"&&(ye.textContent=zs),Wt=n(e),h(Me.$$.fragment,e),Gt=n(e),h(ge.$$.fragment,e),Xt=n(e),we=i(e,"P",{"data-svelte-h":!0}),c(we)!=="svelte-128onc7"&&(we.innerHTML=Fs),Vt=n(e),je=i(e,"P",{"data-svelte-h":!0}),c(je)!=="svelte-1vy8lmv"&&(je.innerHTML=Hs),_t=n(e),h(Je.$$.fragment,e),Rt=n(e),h(Te.$$.fragment,e),Nt=n(e),be=i(e,"P",{"data-svelte-h":!0}),c(be)!=="svelte-19fdrce"&&(be.innerHTML=Es),Yt=n(e),fe=i(e,"P",{"data-svelte-h":!0}),c(fe)!=="svelte-159fa3b"&&(fe.innerHTML=$s),Qt=n(e),h(Ue.$$.fragment,e),xt=n(e),ke=i(e,"P",{"data-svelte-h":!0}),c(ke)!=="svelte-1p1qecq"&&(ke.innerHTML=Ss),zt=n(e),h(Ce.$$.fragment,e),Ft=n(e),Ie=i(e,"P",{"data-svelte-h":!0}),c(Ie)!=="svelte-au9ho2"&&(Ie.textContent=qs),Ht=n(e),Ze=i(e,"P",{"data-svelte-h":!0}),c(Ze)!=="svelte-108v17k"&&(Ze.innerHTML=As),Et=n(e),ve=i(e,"P",{"data-svelte-h":!0}),c(ve)!=="svelte-1yn0icb"&&(ve.textContent=Ls),$t=n(e),h(Be.$$.fragment,e),St=n(e),h(We.$$.fragment,e),qt=n(e),Ge=i(e,"P",{"data-svelte-h":!0}),c(Ge)!=="svelte-1v9i2j4"&&(Ge.innerHTML=Ds),At=n(e),h(Xe.$$.fragment,e),Lt=n(e),h(Ve.$$.fragment,e),Dt=n(e),_e=i(e,"P",{"data-svelte-h":!0}),c(_e)!=="svelte-wtnkn"&&(_e.innerHTML=Ks),Kt=n(e),h(Re.$$.fragment,e),Pt=n(e),h(Ne.$$.fragment,e),Ot=n(e),He=i(e,"P",{}),ps(He).forEach(s),this.h()},h(){ea(p,"name","hf:doc:metadata"),ea(p,"content",ha)},m(e,t){J(document.head,p),a(e,T,t),a(e,M,t),a(e,f,t),m(Z,e,t),a(e,Ee,t),a(e,v,t),a(e,$e,t),a(e,B,t),a(e,Se,t),m(W,e,t),a(e,qe,t),a(e,G,t),a(e,Ae,t),a(e,X,t),a(e,Le,t),a(e,V,t),a(e,De,t),a(e,g,t),J(g,Ye),J(g,ts),m(_,g,null),J(g,ss),J(g,Qe),J(g,as),J(g,xe),J(g,ls),J(g,ze),J(g,ns),m(k,g,null),J(g,os),J(g,Fe),J(g,is),m(R,g,null),J(g,cs),m(N,g,null),a(e,Ke,t),m(Y,e,t),a(e,Pe,t),a(e,Q,t),a(e,Oe,t),a(e,x,t),a(e,et,t),a(e,z,t),a(e,tt,t),a(e,F,t),a(e,st,t),m(H,e,t),a(e,at,t),a(e,E,t),a(e,lt,t),a(e,$,t),a(e,nt,t),a(e,S,t),a(e,ot,t),a(e,q,t),a(e,it,t),m(C,e,t),a(e,ct,t),m(A,e,t),a(e,pt,t),m(L,e,t),a(e,rt,t),a(e,D,t),a(e,ht,t),a(e,K,t),a(e,mt,t),a(e,P,t),a(e,dt,t),m(O,e,t),a(e,ut,t),m(I,e,t),a(e,yt,t),a(e,ee,t),a(e,Mt,t),m(te,e,t),a(e,gt,t),a(e,se,t),a(e,wt,t),m(ae,e,t),a(e,jt,t),a(e,le,t),a(e,Jt,t),m(ne,e,t),a(e,Tt,t),a(e,oe,t),a(e,bt,t),a(e,ie,t),a(e,ft,t),m(ce,e,t),a(e,Ut,t),m(pe,e,t),a(e,kt,t),a(e,re,t),a(e,Ct,t),m(he,e,t),a(e,It,t),a(e,me,t),a(e,Zt,t),m(de,e,t),a(e,vt,t),a(e,ue,t),a(e,Bt,t),a(e,ye,t),a(e,Wt,t),m(Me,e,t),a(e,Gt,t),m(ge,e,t),a(e,Xt,t),a(e,we,t),a(e,Vt,t),a(e,je,t),a(e,_t,t),m(Je,e,t),a(e,Rt,t),m(Te,e,t),a(e,Nt,t),a(e,be,t),a(e,Yt,t),a(e,fe,t),a(e,Qt,t),m(Ue,e,t),a(e,xt,t),a(e,ke,t),a(e,zt,t),m(Ce,e,t),a(e,Ft,t),a(e,Ie,t),a(e,Ht,t),a(e,Ze,t),a(e,Et,t),a(e,ve,t),a(e,$t,t),m(Be,e,t),a(e,St,t),m(We,e,t),a(e,qt,t),a(e,Ge,t),a(e,At,t),m(Xe,e,t),a(e,Lt,t),m(Ve,e,t),a(e,Dt,t),a(e,_e,t),a(e,Kt,t),m(Re,e,t),a(e,Pt,t),m(Ne,e,t),a(e,Ot,t),a(e,He,t),es=!0},p(e,[t]){const w={};t&2&&(w.$$scope={dirty:t,ctx:e}),k.$set(w);const Ps={};t&2&&(Ps.$$scope={dirty:t,ctx:e}),C.$set(Ps);const Os={};t&2&&(Os.$$scope={dirty:t,ctx:e}),I.$set(Os)},i(e){es||(d(Z.$$.fragment,e),d(W.$$.fragment,e),d(_.$$.fragment,e),d(k.$$.fragment,e),d(R.$$.fragment,e),d(N.$$.fragment,e),d(Y.$$.fragment,e),d(H.$$.fragment,e),d(C.$$.fragment,e),d(A.$$.fragment,e),d(L.$$.fragment,e),d(O.$$.fragment,e),d(I.$$.fragment,e),d(te.$$.fragment,e),d(ae.$$.fragment,e),d(ne.$$.fragment,e),d(ce.$$.fragment,e),d(pe.$$.fragment,e),d(he.$$.fragment,e),d(de.$$.fragment,e),d(Me.$$.fragment,e),d(ge.$$.fragment,e),d(Je.$$.fragment,e),d(Te.$$.fragment,e),d(Ue.$$.fragment,e),d(Ce.$$.fragment,e),d(Be.$$.fragment,e),d(We.$$.fragment,e),d(Xe.$$.fragment,e),d(Ve.$$.fragment,e),d(Re.$$.fragment,e),d(Ne.$$.fragment,e),es=!0)},o(e){u(Z.$$.fragment,e),u(W.$$.fragment,e),u(_.$$.fragment,e),u(k.$$.fragment,e),u(R.$$.fragment,e),u(N.$$.fragment,e),u(Y.$$.fragment,e),u(H.$$.fragment,e),u(C.$$.fragment,e),u(A.$$.fragment,e),u(L.$$.fragment,e),u(O.$$.fragment,e),u(I.$$.fragment,e),u(te.$$.fragment,e),u(ae.$$.fragment,e),u(ne.$$.fragment,e),u(ce.$$.fragment,e),u(pe.$$.fragment,e),u(he.$$.fragment,e),u(de.$$.fragment,e),u(Me.$$.fragment,e),u(ge.$$.fragment,e),u(Je.$$.fragment,e),u(Te.$$.fragment,e),u(Ue.$$.fragment,e),u(Ce.$$.fragment,e),u(Be.$$.fragment,e),u(We.$$.fragment,e),u(Xe.$$.fragment,e),u(Ve.$$.fragment,e),u(Re.$$.fragment,e),u(Ne.$$.fragment,e),es=!1},d(e){e&&(s(T),s(M),s(f),s(Ee),s(v),s($e),s(B),s(Se),s(qe),s(G),s(Ae),s(X),s(Le),s(V),s(De),s(g),s(Ke),s(Pe),s(Q),s(Oe),s(x),s(et),s(z),s(tt),s(F),s(st),s(at),s(E),s(lt),s($),s(nt),s(S),s(ot),s(q),s(it),s(ct),s(pt),s(rt),s(D),s(ht),s(K),s(mt),s(P),s(dt),s(ut),s(yt),s(ee),s(Mt),s(gt),s(se),s(wt),s(jt),s(le),s(Jt),s(Tt),s(oe),s(bt),s(ie),s(ft),s(Ut),s(kt),s(re),s(Ct),s(It),s(me),s(Zt),s(vt),s(ue),s(Bt),s(ye),s(Wt),s(Gt),s(Xt),s(we),s(Vt),s(je),s(_t),s(Rt),s(Nt),s(be),s(Yt),s(fe),s(Qt),s(xt),s(ke),s(zt),s(Ft),s(Ie),s(Ht),s(Ze),s(Et),s(ve),s($t),s(St),s(qt),s(Ge),s(At),s(Lt),s(Dt),s(_e),s(Kt),s(Pt),s(Ot),s(He)),s(p),y(Z,e),y(W,e),y(_),y(k),y(R),y(N),y(Y,e),y(H,e),y(C,e),y(A,e),y(L,e),y(O,e),y(I,e),y(te,e),y(ae,e),y(ne,e),y(ce,e),y(pe,e),y(he,e),y(de,e),y(Me,e),y(ge,e),y(Je,e),y(Te,e),y(Ue,e),y(Ce,e),y(Be,e),y(We,e),y(Xe,e),y(Ve,e),y(Re,e),y(Ne,e)}}}const ha='{"title":"Best Practices for Generation with Cache","local":"best-practices-for-generation-with-cache","sections":[{"title":"What is Cache and why we should care?","local":"what-is-cache-and-why-we-should-care","sections":[{"title":"Under the Hood: How Cache Object Works in Attention Mechanism","local":"under-the-hood-how-cache-object-works-in-attention-mechanism","sections":[],"depth":3}],"depth":2},{"title":"Generate with Cache","local":"generate-with-cache","sections":[{"title":"Quantized Cache","local":"quantized-cache","sections":[],"depth":3},{"title":"Offloaded Cache","local":"offloaded-cache","sections":[],"depth":3},{"title":"Static Cache","local":"static-cache","sections":[],"depth":3}],"depth":2},{"title":"Offloaded Static Cache","local":"offloaded-static-cache","sections":[{"title":"Sliding Window Cache","local":"sliding-window-cache","sections":[],"depth":3},{"title":"Sink Cache","local":"sink-cache","sections":[],"depth":3},{"title":"Encoder-Decoder Cache","local":"encoder-decoder-cache","sections":[],"depth":3},{"title":"Model-specific Cache Classes","local":"model-specific-cache-classes","sections":[],"depth":3}],"depth":2},{"title":"Iterative Generation with Cache","local":"iterative-generation-with-cache","sections":[],"depth":2},{"title":"Re-use Cache to continue generation","local":"re-use-cache-to-continue-generation","sections":[],"depth":2},{"title":"Legacy cache format","local":"legacy-cache-format","sections":[],"depth":2}],"depth":1}';function ma(U){return sa(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class wa extends aa{constructor(p){super(),la(this,p,ma,ra,ta,{})}}export{wa as component};
