import{s as St,f as qt,o as Lt,n as W}from"../chunks/scheduler.9bc65507.js";import{S as At,i as Dt,g as u,s as c,r as h,A as Pt,h as M,f as a,c as m,j as ht,u as y,x as U,k as ge,y as Kt,a as n,v as f,d as J,t as T,w as b}from"../chunks/index.707bf1b6.js";import{T as je}from"../chunks/Tip.c2ecdbf4.js";import{C as _}from"../chunks/CodeBlock.54a9f38d.js";import{H as E,E as Ot}from"../chunks/EditOnGithub.922df6ba.js";import{H as yt,a as $e}from"../chunks/HfOption.6d864328.js";function es(j){let t,r='Hugging Face also provides <a href="https://hf.co/docs/text-generation-inference" rel="nofollow">Text Generation Inference (TGI)</a>, a library dedicated to deploying and serving highly optimized LLMs for inference. It includes more optimization features not included in Transformers, such as continuous batching for increasing throughput and tensor parallelism for multi-GPU inference.';return{c(){t=u("p"),t.innerHTML=r},l(l){t=M(l,"P",{"data-svelte-h":!0}),U(t)!=="svelte-1wl2u2g"&&(t.innerHTML=r)},m(l,p){n(l,t,p)},p:W,d(l){l&&a(t)}}}function ts(j){let t,r='Currently, only <a href="./model_doc/cohere">Command R</a>, <a href="./model_doc/gemma">Gemma</a> and <a href="./model_doc/llama2">Llama</a> models support static kv-cache and torch.compile.';return{c(){t=u("p"),t.innerHTML=r},l(l){t=M(l,"P",{"data-svelte-h":!0}),U(t)!=="svelte-16ohxue"&&(t.innerHTML=r)},m(l,p){n(l,t,p)},p:W,d(l){l&&a(t)}}}function ss(j){let t,r="Access the model’s <code>generation_config</code> attribute and set the <code>cache_implementation</code> to “static”.",l,p,o,i,g="Call torch.compile on the model to compile the forward pass with the static kv-cache.",$,Z,v;return p=new _({props:{code:"bW9kZWwuZ2VuZXJhdGlvbl9jb25maWcuY2FjaGVfaW1wbGVtZW50YXRpb24lMjAlM0QlMjAlMjJzdGF0aWMlMjI=",highlighted:'model.generation_config.cache_implementation = <span class="hljs-string">&quot;static&quot;</span>',wrap:!1}}),Z=new _({props:{code:"Y29tcGlsZWRfbW9kZWwlMjAlM0QlMjB0b3JjaC5jb21waWxlKG1vZGVsJTJDJTIwbW9kZSUzRCUyMnJlZHVjZS1vdmVyaGVhZCUyMiUyQyUyMGZ1bGxncmFwaCUzRFRydWUpJTBBaW5wdXRfdGV4dCUyMCUzRCUyMCUyMlRoZSUyMHRoZW9yeSUyMG9mJTIwc3BlY2lhbCUyMHJlbGF0aXZpdHklMjBzdGF0ZXMlMjAlMjIlMEFpbnB1dF9pZHMlMjAlM0QlMjB0b2tlbml6ZXIoaW5wdXRfdGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnRvKCUyMmN1ZGElMjIpJTBBJTBBb3V0cHV0cyUyMCUzRCUyMGNvbXBpbGVkX21vZGVsLmdlbmVyYXRlKCoqaW5wdXRfaWRzKSUwQXRva2VuaXplci5iYXRjaF9kZWNvZGUob3V0cHV0cyUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSUwQSU1QidUaGUlMjB0aGVvcnklMjBvZiUyMHNwZWNpYWwlMjByZWxhdGl2aXR5JTIwc3RhdGVzJTIwMS4lMjBUaGUlMjBzcGVlZCUyMG9mJTIwbGlnaHQlMjBpcyUyMGNvbnN0YW50JTIwaW4lMjBhbGwlMjBpbmVydGlhbCUyMHJlZmVyZW5jZSclNUQ=",highlighted:`compiled_model = torch.<span class="hljs-built_in">compile</span>(model, mode=<span class="hljs-string">&quot;reduce-overhead&quot;</span>, fullgraph=<span class="hljs-literal">True</span>)
input_text = <span class="hljs-string">&quot;The theory of special relativity states &quot;</span>
input_ids = tokenizer(input_text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)

outputs = compiled_model.generate(**input_ids)
tokenizer.batch_decode(outputs, skip_special_tokens=<span class="hljs-literal">True</span>)
[<span class="hljs-string">&#x27;The theory of special relativity states 1. The speed of light is constant in all inertial reference&#x27;</span>]`,wrap:!1}}),{c(){t=u("p"),t.innerHTML=r,l=c(),h(p.$$.fragment),o=c(),i=u("p"),i.textContent=g,$=c(),h(Z.$$.fragment)},l(w){t=M(w,"P",{"data-svelte-h":!0}),U(t)!=="svelte-oou7hc"&&(t.innerHTML=r),l=m(w),y(p.$$.fragment,w),o=m(w),i=M(w,"P",{"data-svelte-h":!0}),U(i)!=="svelte-pgmdb4"&&(i.textContent=g),$=m(w),y(Z.$$.fragment,w)},m(w,k){n(w,t,k),n(w,l,k),f(p,w,k),n(w,o,k),n(w,i,k),n(w,$,k),f(Z,w,k),v=!0},p:W,i(w){v||(J(p.$$.fragment,w),J(Z.$$.fragment,w),v=!0)},o(w){T(p.$$.fragment,w),T(Z.$$.fragment,w),v=!1},d(w){w&&(a(t),a(l),a(o),a(i),a($)),b(p,w),b(Z,w)}}}function ls(j){let t,r="The <code>_setup_cache</code> method is an internal and private method that is still under development. This means it may not be backward compatible and the API design may change in the future.";return{c(){t=u("p"),t.innerHTML=r},l(l){t=M(l,"P",{"data-svelte-h":!0}),U(t)!=="svelte-ftnkie"&&(t.innerHTML=r)},m(l,p){n(l,t,p)},p:W,d(l){l&&a(t)}}}function as(j){let t,r,l,p='The <code>_setup_cache</code> method doesn’t support <a href="/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate">generate()</a> yet, so this method is a bit more involved. You’ll need to write your own function to decode the next token given the current token and position and cache position of previously generated tokens.',o,i,g,$,Z="There are a few important things you must do to enable static kv-cache and torch.compile with the <code>_setup_cache</code> method:",v,w,k='<li><p>Access the model’s <code>_setup_cache</code> method and pass it the <a href="/docs/transformers/main/en/internal/generation_utils#transformers.StaticCache">StaticCache</a> class. This is a more flexible method because it allows you to configure parameters like the maximum batch size and sequence length.</p></li> <li><p>Call torch.compile on the model to compile the forward pass with the static kv-cache.</p></li> <li><p>Set <code>enable_math=True</code> in the <a href="https://pytorch.org/docs/master/generated/torch.nn.functional.scaled_dot_product_attention.html" rel="nofollow">torch.backends.cuda.sdp_kernel</a> context manager to enable the native PyTorch C++ implementation of scaled dot product attention to speed up inference even more.</p></li>',I,C,B;return t=new je({props:{warning:!0,$$slots:{default:[ls]},$$scope:{ctx:j}}}),i=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMExsYW1hVG9rZW5pemVyJTJDJTIwTGxhbWFGb3JDYXVzYWxMTSUyQyUyMFN0YXRpY0NhY2hlJTJDJTIwbG9nZ2luZyUwQWZyb20lMjB0cmFuc2Zvcm1lcnMudGVzdGluZ191dGlscyUyMGltcG9ydCUyMENhcHR1cmVMb2dnZXIlMEFpbXBvcnQlMjB0b3JjaCUwQSUwQXByb21wdHMlMjAlM0QlMjAlNUIlMEElMjAlMjAlMjAlMjAlMjJTaW1wbHklMjBwdXQlMkMlMjB0aGUlMjB0aGVvcnklMjBvZiUyMHJlbGF0aXZpdHklMjBzdGF0ZXMlMjB0aGF0JTIwJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIyTXklMjBmYXZvcml0ZSUyMGFsbCUyMHRpbWUlMjBmYXZvcml0ZSUyMGNvbmRpbWVudCUyMGlzJTIwa2V0Y2h1cC4lMjIlMkMlMEElNUQlMEElMEFOVU1fVE9LRU5TX1RPX0dFTkVSQVRFJTIwJTNEJTIwNDAlMEF0b3JjaF9kZXZpY2UlMjAlM0QlMjAlMjJjdWRhJTIyJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwTGxhbWFUb2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMm1ldGEtbGxhbWElMkZMbGFtYS0yLTdiLWhmJTIyJTJDJTIwcGFkX3Rva2VuJTNEJTIyJTNDJTJGcyUzRSUyMiUyQyUyMHBhZGRpbmdfc2lkZSUzRCUyMnJpZ2h0JTIyKSUwQW1vZGVsJTIwJTNEJTIwTGxhbWFGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTIybWV0YS1sbGFtYSUyRkxsYW1hLTItN2ItaGYlMjIlMkMlMjBkZXZpY2VfbWFwJTNEJTIyc2VxdWVudGlhbCUyMiklMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIocHJvbXB0cyUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIlMkMlMjBwYWRkaW5nJTNEVHJ1ZSkudG8obW9kZWwuZGV2aWNlKSUwQSUwQWRlZiUyMGRlY29kZV9vbmVfdG9rZW5zKG1vZGVsJTJDJTIwY3VyX3Rva2VuJTJDJTIwaW5wdXRfcG9zJTJDJTIwY2FjaGVfcG9zaXRpb24pJTNBJTBBJTIwJTIwJTIwJTIwbG9naXRzJTIwJTNEJTIwbW9kZWwoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY3VyX3Rva2VuJTJDJTIwcG9zaXRpb25faWRzJTNEaW5wdXRfcG9zJTJDJTIwY2FjaGVfcG9zaXRpb24lM0RjYWNoZV9wb3NpdGlvbiUyQyUyMHJldHVybl9kaWN0JTNERmFsc2UlMkMlMjB1c2VfY2FjaGUlM0RUcnVlJTBBJTIwJTIwJTIwJTIwKSU1QjAlNUQlMEElMjAlMjAlMjAlMjBuZXdfdG9rZW4lMjAlM0QlMjB0b3JjaC5hcmdtYXgobG9naXRzJTVCJTNBJTJDJTIwLTElNUQlMkMlMjBkaW0lM0QtMSklNUIlM0ElMkMlMjBOb25lJTVEJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwbmV3X3Rva2Vu",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> LlamaTokenizer, LlamaForCausalLM, StaticCache, logging
<span class="hljs-keyword">from</span> transformers.testing_utils <span class="hljs-keyword">import</span> CaptureLogger
<span class="hljs-keyword">import</span> torch

prompts = [
    <span class="hljs-string">&quot;Simply put, the theory of relativity states that &quot;</span>,
    <span class="hljs-string">&quot;My favorite all time favorite condiment is ketchup.&quot;</span>,
]

NUM_TOKENS_TO_GENERATE = <span class="hljs-number">40</span>
torch_device = <span class="hljs-string">&quot;cuda&quot;</span>

tokenizer = LlamaTokenizer.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-2-7b-hf&quot;</span>, pad_token=<span class="hljs-string">&quot;&lt;/s&gt;&quot;</span>, padding_side=<span class="hljs-string">&quot;right&quot;</span>)
model = LlamaForCausalLM.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-2-7b-hf&quot;</span>, device_map=<span class="hljs-string">&quot;sequential&quot;</span>)
inputs = tokenizer(prompts, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>, padding=<span class="hljs-literal">True</span>).to(model.device)

<span class="hljs-keyword">def</span> <span class="hljs-title function_">decode_one_tokens</span>(<span class="hljs-params">model, cur_token, input_pos, cache_position</span>):
    logits = model(
        cur_token, position_ids=input_pos, cache_position=cache_position, return_dict=<span class="hljs-literal">False</span>, use_cache=<span class="hljs-literal">True</span>
    )[<span class="hljs-number">0</span>]
    new_token = torch.argmax(logits[:, -<span class="hljs-number">1</span>], dim=-<span class="hljs-number">1</span>)[:, <span class="hljs-literal">None</span>]
    <span class="hljs-keyword">return</span> new_token`,wrap:!1}}),C=new _({props:{code:"YmF0Y2hfc2l6ZSUyQyUyMHNlcV9sZW5ndGglMjAlM0QlMjBpbnB1dHMlNUIlMjJpbnB1dF9pZHMlMjIlNUQuc2hhcGUlMEF3aXRoJTIwdG9yY2gubm9fZ3JhZCgpJTNBJTBBJTIwJTIwJTIwJTIwJTIwbW9kZWwuX3NldHVwX2NhY2hlKFN0YXRpY0NhY2hlJTJDJTIwMiUyQyUyMG1heF9jYWNoZV9sZW4lM0Q0MDk2KSUwQSUyMCUyMCUyMCUyMCUyMGNhY2hlX3Bvc2l0aW9uJTIwJTNEJTIwdG9yY2guYXJhbmdlKHNlcV9sZW5ndGglMkMlMjBkZXZpY2UlM0R0b3JjaF9kZXZpY2UpJTBBJTIwJTIwJTIwJTIwJTIwZ2VuZXJhdGVkX2lkcyUyMCUzRCUyMHRvcmNoLnplcm9zKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGJhdGNoX3NpemUlMkMlMjBzZXFfbGVuZ3RoJTIwJTJCJTIwTlVNX1RPS0VOU19UT19HRU5FUkFURSUyMCUyQiUyMDElMkMlMjBkdHlwZSUzRHRvcmNoLmludCUyQyUyMGRldmljZSUzRHRvcmNoX2RldmljZSUwQSUyMCUyMCUyMCUyMCUyMCklMEElMjAlMjAlMjAlMjAlMjBnZW5lcmF0ZWRfaWRzJTVCJTNBJTJDJTIwY2FjaGVfcG9zaXRpb24lNUQlMjAlM0QlMjBpbnB1dHMlNUIlMjJpbnB1dF9pZHMlMjIlNUQudG8odG9yY2hfZGV2aWNlKS50byh0b3JjaC5pbnQpJTBBJTBBJTIwJTIwJTIwJTIwJTIwbG9naXRzJTIwJTNEJTIwbW9kZWwoKippbnB1dHMlMkMlMjBjYWNoZV9wb3NpdGlvbiUzRGNhY2hlX3Bvc2l0aW9uJTJDJTIwcmV0dXJuX2RpY3QlM0RGYWxzZSUyQyUyMHVzZV9jYWNoZSUzRFRydWUpJTVCMCU1RCUwQSUyMCUyMCUyMCUyMCUyMG5leHRfdG9rZW4lMjAlM0QlMjB0b3JjaC5hcmdtYXgobG9naXRzJTVCJTNBJTJDJTIwLTElNUQlMkMlMjBkaW0lM0QtMSklNUIlM0ElMkMlMjBOb25lJTVEJTBBJTIwJTIwJTIwJTIwJTIwZ2VuZXJhdGVkX2lkcyU1QiUzQSUyQyUyMHNlcV9sZW5ndGglNUQlMjAlM0QlMjBuZXh0X3Rva2VuJTVCJTNBJTJDJTIwMCU1RCUwQSUwQSUyMCUyMCUyMCUyMCUyMGRlY29kZV9vbmVfdG9rZW5zJTIwJTNEJTIwdG9yY2guY29tcGlsZShkZWNvZGVfb25lX3Rva2VucyUyQyUyMG1vZGUlM0QlMjJyZWR1Y2Utb3ZlcmhlYWQlMjIlMkMlMjBmdWxsZ3JhcGglM0RUcnVlKSUwQSUyMCUyMCUyMCUyMCUyMGNhY2hlX3Bvc2l0aW9uJTIwJTNEJTIwdG9yY2gudGVuc29yKCU1QnNlcV9sZW5ndGglMjAlMkIlMjAxJTVEJTJDJTIwZGV2aWNlJTNEdG9yY2hfZGV2aWNlKSUwQSUyMCUyMCUyMCUyMCUyMGZvciUyMF8lMjBpbiUyMHJhbmdlKDElMkMlMjBOVU1fVE9LRU5TX1RPX0dFTkVSQVRFKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHdpdGglMjB0b3JjaC5iYWNrZW5kcy5jdWRhLnNkcF9rZXJuZWwoZW5hYmxlX2ZsYXNoJTNERmFsc2UlMkMlMjBlbmFibGVfbWVtX2VmZmljaWVudCUzREZhbHNlJTJDJTIwZW5hYmxlX21hdGglM0RUcnVlKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG5leHRfdG9rZW4lMjAlM0QlMjBkZWNvZGVfb25lX3Rva2Vucyhtb2RlbCUyQyUyMG5leHRfdG9rZW4uY2xvbmUoKSUyQyUyME5vbmUlMkMlMjBjYWNoZV9wb3NpdGlvbiklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBnZW5lcmF0ZWRfaWRzJTVCJTNBJTJDJTIwY2FjaGVfcG9zaXRpb24lNUQlMjAlM0QlMjBuZXh0X3Rva2VuLmludCgpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY2FjaGVfcG9zaXRpb24lMjAlMkIlM0QlMjAxJTBBJTBBdGV4dCUyMCUzRCUyMHRva2VuaXplci5iYXRjaF9kZWNvZGUoZ2VuZXJhdGVkX2lkcyUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSUwQXRleHQlMEElNUInU2ltcGx5JTIwcHV0JTJDJTIwdGhlJTIwdGhlb3J5JTIwb2YlMjByZWxhdGl2aXR5JTIwc3RhdGVzJTIwdGhhdCUyMDEpJTIwdGhlJTIwc3BlZWQlMjBvZiUyMGxpZ2h0JTIwaXMlMjBjb25zdGFudCUyQyUyMDIpJTIwdGhlJTIwc3BlZWQlMjBvZiUyMGxpZ2h0JTIwaXMlMjB0aGUlMjBzYW1lJTIwZm9yJTIwYWxsJTIwb2JzZXJ2ZXJzJTJDJTIwYW5kJTIwMyklMjB0aGUlMjBsYXdzJTIwb2YlMjBwaHlzaWNzJTIwYXJlJTIwdGhlJTIwc2FtZSUyMGZvciUyMGFsbCUyMG9ic2VydmVycy4nJTJDJTBBJTIwJ015JTIwZmF2b3JpdGUlMjBhbGwlMjB0aW1lJTIwZmF2b3JpdGUlMjBjb25kaW1lbnQlMjBpcyUyMGtldGNodXAuJTIwSSUyMGxvdmUlMjBpdCUyMG9uJTIwZXZlcnl0aGluZy4lMjBJJTIwbG92ZSUyMGl0JTIwb24lMjBteSUyMGVnZ3MlMkMlMjBteSUyMGZyaWVzJTJDJTIwbXklMjBjaGlja2VuJTJDJTIwbXklMjBidXJnZXJzJTJDJTIwbXklMjBob3QlMjBkb2dzJTJDJTIwbXklMjBzYW5kd2ljaGVzJTJDJTIwbXklMjBzYWxhZHMlMkMlMjBteSUyMHAnJTVE",highlighted:`batch_size, seq_length = inputs[<span class="hljs-string">&quot;input_ids&quot;</span>].shape
<span class="hljs-keyword">with</span> torch.no_grad():
     model._setup_cache(StaticCache, <span class="hljs-number">2</span>, max_cache_len=<span class="hljs-number">4096</span>)
     cache_position = torch.arange(seq_length, device=torch_device)
     generated_ids = torch.zeros(
         batch_size, seq_length + NUM_TOKENS_TO_GENERATE + <span class="hljs-number">1</span>, dtype=torch.<span class="hljs-built_in">int</span>, device=torch_device
     )
     generated_ids[:, cache_position] = inputs[<span class="hljs-string">&quot;input_ids&quot;</span>].to(torch_device).to(torch.<span class="hljs-built_in">int</span>)

     logits = model(**inputs, cache_position=cache_position, return_dict=<span class="hljs-literal">False</span>, use_cache=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]
     next_token = torch.argmax(logits[:, -<span class="hljs-number">1</span>], dim=-<span class="hljs-number">1</span>)[:, <span class="hljs-literal">None</span>]
     generated_ids[:, seq_length] = next_token[:, <span class="hljs-number">0</span>]

     decode_one_tokens = torch.<span class="hljs-built_in">compile</span>(decode_one_tokens, mode=<span class="hljs-string">&quot;reduce-overhead&quot;</span>, fullgraph=<span class="hljs-literal">True</span>)
     cache_position = torch.tensor([seq_length + <span class="hljs-number">1</span>], device=torch_device)
     <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">1</span>, NUM_TOKENS_TO_GENERATE):
         <span class="hljs-keyword">with</span> torch.backends.cuda.sdp_kernel(enable_flash=<span class="hljs-literal">False</span>, enable_mem_efficient=<span class="hljs-literal">False</span>, enable_math=<span class="hljs-literal">True</span>):
             next_token = decode_one_tokens(model, next_token.clone(), <span class="hljs-literal">None</span>, cache_position)
             generated_ids[:, cache_position] = next_token.<span class="hljs-built_in">int</span>()
         cache_position += <span class="hljs-number">1</span>

text = tokenizer.batch_decode(generated_ids, skip_special_tokens=<span class="hljs-literal">True</span>)
text
[<span class="hljs-string">&#x27;Simply put, the theory of relativity states that 1) the speed of light is constant, 2) the speed of light is the same for all observers, and 3) the laws of physics are the same for all observers.&#x27;</span>,
 <span class="hljs-string">&#x27;My favorite all time favorite condiment is ketchup. I love it on everything. I love it on my eggs, my fries, my chicken, my burgers, my hot dogs, my sandwiches, my salads, my p&#x27;</span>]`,wrap:!1}}),{c(){h(t.$$.fragment),r=c(),l=u("p"),l.innerHTML=p,o=c(),h(i.$$.fragment),g=c(),$=u("p"),$.innerHTML=Z,v=c(),w=u("ol"),w.innerHTML=k,I=c(),h(C.$$.fragment)},l(d){y(t.$$.fragment,d),r=m(d),l=M(d,"P",{"data-svelte-h":!0}),U(l)!=="svelte-ubvdtc"&&(l.innerHTML=p),o=m(d),y(i.$$.fragment,d),g=m(d),$=M(d,"P",{"data-svelte-h":!0}),U($)!=="svelte-ksgcud"&&($.innerHTML=Z),v=m(d),w=M(d,"OL",{"data-svelte-h":!0}),U(w)!=="svelte-1x5y2ry"&&(w.innerHTML=k),I=m(d),y(C.$$.fragment,d)},m(d,G){f(t,d,G),n(d,r,G),n(d,l,G),n(d,o,G),f(i,d,G),n(d,g,G),n(d,$,G),n(d,v,G),n(d,w,G),n(d,I,G),f(C,d,G),B=!0},p(d,G){const R={};G&2&&(R.$$scope={dirty:G,ctx:d}),t.$set(R)},i(d){B||(J(t.$$.fragment,d),J(i.$$.fragment,d),J(C.$$.fragment,d),B=!0)},o(d){T(t.$$.fragment,d),T(i.$$.fragment,d),T(C.$$.fragment,d),B=!1},d(d){d&&(a(r),a(l),a(o),a(g),a($),a(v),a(w),a(I)),b(t,d),b(i,d),b(C,d)}}}function ns(j){let t,r,l,p;return t=new $e({props:{id:"static-kv",option:"generation_config",$$slots:{default:[ss]},$$scope:{ctx:j}}}),l=new $e({props:{id:"static-kv",option:"setup_cache",$$slots:{default:[as]},$$scope:{ctx:j}}}),{c(){h(t.$$.fragment),r=c(),h(l.$$.fragment)},l(o){y(t.$$.fragment,o),r=m(o),y(l.$$.fragment,o)},m(o,i){f(t,o,i),n(o,r,i),f(l,o,i),p=!0},p(o,i){const g={};i&2&&(g.$$scope={dirty:i,ctx:o}),t.$set(g);const $={};i&2&&($.$$scope={dirty:i,ctx:o}),l.$set($)},i(o){p||(J(t.$$.fragment,o),J(l.$$.fragment,o),p=!0)},o(o){T(t.$$.fragment,o),T(l.$$.fragment,o),p=!1},d(o){o&&a(r),b(t,o),b(l,o)}}}function os(j){let t,r='For a more in-depth explanation, take a look at the <a href="https://hf.co/blog/assisted-generation" rel="nofollow">Assisted Generation: a new direction toward low-latency text generation</a> blog post!';return{c(){t=u("p"),t.innerHTML=r},l(l){t=M(l,"P",{"data-svelte-h":!0}),U(t)!=="svelte-112w74b"&&(t.innerHTML=r)},m(l,p){n(l,t,p)},p:W,d(l){l&&a(t)}}}function is(j){let t,r="Speculative decoding is only supported for the greedy search and sampling decoding strategies, and it also doesn’t support batched inputs.";return{c(){t=u("p"),t.textContent=r},l(l){t=M(l,"P",{"data-svelte-h":!0}),U(t)!=="svelte-eq929k"&&(t.textContent=r)},m(l,p){n(l,t,p)},p:W,d(l){l&&a(t)}}}function rs(j){let t,r;return t=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQWltcG9ydCUyMHRvcmNoJTBBJTBBZGV2aWNlJTIwJTNEJTIwJTIyY3VkYSUyMiUyMGlmJTIwdG9yY2guY3VkYS5pc19hdmFpbGFibGUoKSUyMGVsc2UlMjAlMjJjcHUlMjIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJmYWNlYm9vayUyRm9wdC0xLjNiJTIyKSUwQWlucHV0cyUyMCUzRCUyMHRva2VuaXplciglMjJFaW5zdGVpbidzJTIwdGhlb3J5JTIwb2YlMjByZWxhdGl2aXR5JTIwc3RhdGVzJTIyJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikudG8oZGV2aWNlKSUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMmZhY2Vib29rJTJGb3B0LTEuM2IlMjIpLnRvKGRldmljZSklMEFhc3Npc3RhbnRfbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTIyZmFjZWJvb2slMkZvcHQtMTI1bSUyMikudG8oZGV2aWNlKSUwQW91dHB1dHMlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSgqKmlucHV0cyUyQyUyMGFzc2lzdGFudF9tb2RlbCUzRGFzc2lzdGFudF9tb2RlbCklMEF0b2tlbml6ZXIuYmF0Y2hfZGVjb2RlKG91dHB1dHMlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSklMEElNUIlMjJFaW5zdGVpbidzJTIwdGhlb3J5JTIwb2YlMjByZWxhdGl2aXR5JTIwc3RhdGVzJTIwdGhhdCUyMHRoZSUyMHNwZWVkJTIwb2YlMjBsaWdodCUyMGlzJTIwY29uc3RhbnQuJTIwJTIwJTIwJTIwJTIyJTVE",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer
<span class="hljs-keyword">import</span> torch

device = <span class="hljs-string">&quot;cuda&quot;</span> <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> <span class="hljs-string">&quot;cpu&quot;</span>

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;facebook/opt-1.3b&quot;</span>)
inputs = tokenizer(<span class="hljs-string">&quot;Einstein&#x27;s theory of relativity states&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(device)

model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;facebook/opt-1.3b&quot;</span>).to(device)
assistant_model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;facebook/opt-125m&quot;</span>).to(device)
outputs = model.generate(**inputs, assistant_model=assistant_model)
tokenizer.batch_decode(outputs, skip_special_tokens=<span class="hljs-literal">True</span>)
[<span class="hljs-string">&quot;Einstein&#x27;s theory of relativity states that the speed of light is constant.    &quot;</span>]`,wrap:!1}}),{c(){h(t.$$.fragment)},l(l){y(t.$$.fragment,l)},m(l,p){f(t,l,p),r=!0},p:W,i(l){r||(J(t.$$.fragment,l),r=!0)},o(l){T(t.$$.fragment,l),r=!1},d(l){b(t,l)}}}function ps(j){let t,r='For speculative sampling decoding, add the <code>do_sample</code> and <code>temperature</code> parameters to the <a href="/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate">generate()</a> method in addition to the assistant model.',l,p,o;return p=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQWltcG9ydCUyMHRvcmNoJTBBJTBBZGV2aWNlJTIwJTNEJTIwJTIyY3VkYSUyMiUyMGlmJTIwdG9yY2guY3VkYS5pc19hdmFpbGFibGUoKSUyMGVsc2UlMjAlMjJjcHUlMjIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJmYWNlYm9vayUyRm9wdC0xLjNiJTIyKSUwQWlucHV0cyUyMCUzRCUyMHRva2VuaXplciglMjJFaW5zdGVpbidzJTIwdGhlb3J5JTIwb2YlMjByZWxhdGl2aXR5JTIwc3RhdGVzJTIyJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikudG8oZGV2aWNlKSUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMmZhY2Vib29rJTJGb3B0LTEuM2IlMjIpLnRvKGRldmljZSklMEFhc3Npc3RhbnRfbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTIyZmFjZWJvb2slMkZvcHQtMTI1bSUyMikudG8oZGV2aWNlKSUwQW91dHB1dHMlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSgqKmlucHV0cyUyQyUyMGFzc2lzdGFudF9tb2RlbCUzRGFzc2lzdGFudF9tb2RlbCUyQyUyMGRvX3NhbXBsZSUzRFRydWUlMkMlMjB0ZW1wZXJhdHVyZSUzRDAuNyklMEFwcmludCh0b2tlbml6ZXIuYmF0Y2hfZGVjb2RlKG91dHB1dHMlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSkpJTBBJTVCJTIyRWluc3RlaW4ncyUyMHRoZW9yeSUyMG9mJTIwcmVsYXRpdml0eSUyMHN0YXRlcyUyMHRoYXQlMjBtb3Rpb24lMjBpbiUyMHRoZSUyMHVuaXZlcnNlJTIwaXMlMjBub3QlMjBhJTIwc3RyYWlnaHQlMjBsaW5lLiU1Q24lMjIlNUQ=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer
<span class="hljs-keyword">import</span> torch

device = <span class="hljs-string">&quot;cuda&quot;</span> <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> <span class="hljs-string">&quot;cpu&quot;</span>

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;facebook/opt-1.3b&quot;</span>)
inputs = tokenizer(<span class="hljs-string">&quot;Einstein&#x27;s theory of relativity states&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(device)

model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;facebook/opt-1.3b&quot;</span>).to(device)
assistant_model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;facebook/opt-125m&quot;</span>).to(device)
outputs = model.generate(**inputs, assistant_model=assistant_model, do_sample=<span class="hljs-literal">True</span>, temperature=<span class="hljs-number">0.7</span>)
<span class="hljs-built_in">print</span>(tokenizer.batch_decode(outputs, skip_special_tokens=<span class="hljs-literal">True</span>))
[<span class="hljs-string">&quot;Einstein&#x27;s theory of relativity states that motion in the universe is not a straight line.\\n&quot;</span>]`,wrap:!1}}),{c(){t=u("p"),t.innerHTML=r,l=c(),h(p.$$.fragment)},l(i){t=M(i,"P",{"data-svelte-h":!0}),U(t)!=="svelte-1nmf1gl"&&(t.innerHTML=r),l=m(i),y(p.$$.fragment,i)},m(i,g){n(i,t,g),n(i,l,g),f(p,i,g),o=!0},p:W,i(i){o||(J(p.$$.fragment,i),o=!0)},o(i){T(p.$$.fragment,i),o=!1},d(i){i&&(a(t),a(l)),b(p,i)}}}function cs(j){let t,r,l,p;return t=new $e({props:{id:"spec-decoding",option:"greedy search",$$slots:{default:[rs]},$$scope:{ctx:j}}}),l=new $e({props:{id:"spec-decoding",option:"sampling",$$slots:{default:[ps]},$$scope:{ctx:j}}}),{c(){h(t.$$.fragment),r=c(),h(l.$$.fragment)},l(o){y(t.$$.fragment,o),r=m(o),y(l.$$.fragment,o)},m(o,i){f(t,o,i),n(o,r,i),f(l,o,i),p=!0},p(o,i){const g={};i&2&&(g.$$scope={dirty:i,ctx:o}),t.$set(g);const $={};i&2&&($.$$scope={dirty:i,ctx:o}),l.$set($)},i(o){p||(J(t.$$.fragment,o),J(l.$$.fragment,o),p=!0)},o(o){T(t.$$.fragment,o),T(l.$$.fragment,o),p=!1},d(o){o&&a(r),b(t,o),b(l,o)}}}function ms(j){let t,r;return t=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQWltcG9ydCUyMHRvcmNoJTBBJTBBZGV2aWNlJTIwJTNEJTIwJTIyY3VkYSUyMiUyMGlmJTIwdG9yY2guY3VkYS5pc19hdmFpbGFibGUoKSUyMGVsc2UlMjAlMjJjcHUlMjIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJmYWNlYm9vayUyRm9wdC0xLjNiJTIyKSUwQWlucHV0cyUyMCUzRCUyMHRva2VuaXplciglMjJUaGUlMjBzZWNvbmQlMjBsYXclMjBvZiUyMHRoZXJtb2R5bmFtaWNzJTIwc3RhdGVzJTIyJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikudG8oZGV2aWNlKSUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMmZhY2Vib29rJTJGb3B0LTEuM2IlMjIpLnRvKGRldmljZSklMEFhc3Npc3RhbnRfbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTIyZmFjZWJvb2slMkZvcHQtMTI1bSUyMikudG8oZGV2aWNlKSUwQW91dHB1dHMlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSgqKmlucHV0cyUyQyUyMHByb21wdF9sb29rdXBfbnVtX3Rva2VucyUzRDMpJTBBcHJpbnQodG9rZW5pemVyLmJhdGNoX2RlY29kZShvdXRwdXRzJTJDJTIwc2tpcF9zcGVjaWFsX3Rva2VucyUzRFRydWUpKSUwQSU1QidUaGUlMjBzZWNvbmQlMjBsYXclMjBvZiUyMHRoZXJtb2R5bmFtaWNzJTIwc3RhdGVzJTIwdGhhdCUyMGVudHJvcHklMjBpbmNyZWFzZXMlMjB3aXRoJTIwdGVtcGVyYXR1cmUuJTIwJTIwJTIwJTIwJTIwJTIwJyU1RA==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer
<span class="hljs-keyword">import</span> torch

device = <span class="hljs-string">&quot;cuda&quot;</span> <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> <span class="hljs-string">&quot;cpu&quot;</span>

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;facebook/opt-1.3b&quot;</span>)
inputs = tokenizer(<span class="hljs-string">&quot;The second law of thermodynamics states&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(device)

model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;facebook/opt-1.3b&quot;</span>).to(device)
assistant_model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;facebook/opt-125m&quot;</span>).to(device)
outputs = model.generate(**inputs, prompt_lookup_num_tokens=<span class="hljs-number">3</span>)
<span class="hljs-built_in">print</span>(tokenizer.batch_decode(outputs, skip_special_tokens=<span class="hljs-literal">True</span>))
[<span class="hljs-string">&#x27;The second law of thermodynamics states that entropy increases with temperature.      &#x27;</span>]`,wrap:!1}}),{c(){h(t.$$.fragment)},l(l){y(t.$$.fragment,l)},m(l,p){f(t,l,p),r=!0},p:W,i(l){r||(J(t.$$.fragment,l),r=!0)},o(l){T(t.$$.fragment,l),r=!1},d(l){b(t,l)}}}function ds(j){let t,r='For prompt lookup decoding with sampling, add the <code>do_sample</code> and <code>temperature</code> parameters to the <a href="/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate">generate()</a> method.',l,p,o;return p=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQWltcG9ydCUyMHRvcmNoJTBBJTBBZGV2aWNlJTIwJTNEJTIwJTIyY3VkYSUyMiUyMGlmJTIwdG9yY2guY3VkYS5pc19hdmFpbGFibGUoKSUyMGVsc2UlMjAlMjJjcHUlMjIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJmYWNlYm9vayUyRm9wdC0xLjNiJTIyKSUwQWlucHV0cyUyMCUzRCUyMHRva2VuaXplciglMjJUaGUlMjBzZWNvbmQlMjBsYXclMjBvZiUyMHRoZXJtb2R5bmFtaWNzJTIwc3RhdGVzJTIyJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikudG8oZGV2aWNlKSUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMmZhY2Vib29rJTJGb3B0LTEuM2IlMjIpLnRvKGRldmljZSklMEFvdXRwdXRzJTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMlMkMlMjBwcm9tcHRfbG9va3VwX251bV90b2tlbnMlM0QzJTJDJTIwZG9fc2FtcGxlJTNEVHJ1ZSUyQyUyMHRlbXBlcmF0dXJlJTNEMC43KSUwQXByaW50KHRva2VuaXplci5iYXRjaF9kZWNvZGUob3V0cHV0cyUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSklMEElNUIlMjJUaGUlMjBzZWNvbmQlMjBsYXclMjBvZiUyMHRoZXJtb2R5bmFtaWNzJTIwc3RhdGVzJTIwdGhhdCUyMGVuZXJneSUyMGNhbm5vdCUyMGJlJTIwY3JlYXRlZCUyMG5vciUyMGRlc3Ryb3llZC4lMjBJdCdzJTIwbm90JTIwYSUyMiU1RA==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer
<span class="hljs-keyword">import</span> torch

device = <span class="hljs-string">&quot;cuda&quot;</span> <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> <span class="hljs-string">&quot;cpu&quot;</span>

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;facebook/opt-1.3b&quot;</span>)
inputs = tokenizer(<span class="hljs-string">&quot;The second law of thermodynamics states&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(device)

model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;facebook/opt-1.3b&quot;</span>).to(device)
outputs = model.generate(**inputs, prompt_lookup_num_tokens=<span class="hljs-number">3</span>, do_sample=<span class="hljs-literal">True</span>, temperature=<span class="hljs-number">0.7</span>)
<span class="hljs-built_in">print</span>(tokenizer.batch_decode(outputs, skip_special_tokens=<span class="hljs-literal">True</span>))
[<span class="hljs-string">&quot;The second law of thermodynamics states that energy cannot be created nor destroyed. It&#x27;s not a&quot;</span>]`,wrap:!1}}),{c(){t=u("p"),t.innerHTML=r,l=c(),h(p.$$.fragment)},l(i){t=M(i,"P",{"data-svelte-h":!0}),U(t)!=="svelte-arx1ua"&&(t.innerHTML=r),l=m(i),y(p.$$.fragment,i)},m(i,g){n(i,t,g),n(i,l,g),f(p,i,g),o=!0},p:W,i(i){o||(J(p.$$.fragment,i),o=!0)},o(i){T(p.$$.fragment,i),o=!1},d(i){i&&(a(t),a(l)),b(p,i)}}}function us(j){let t,r,l,p;return t=new $e({props:{id:"pld",option:"greedy decoding",$$slots:{default:[ms]},$$scope:{ctx:j}}}),l=new $e({props:{id:"pld",option:"sampling",$$slots:{default:[ds]},$$scope:{ctx:j}}}),{c(){h(t.$$.fragment),r=c(),h(l.$$.fragment)},l(o){y(t.$$.fragment,o),r=m(o),y(l.$$.fragment,o)},m(o,i){f(t,o,i),n(o,r,i),f(l,o,i),p=!0},p(o,i){const g={};i&2&&(g.$$scope={dirty:i,ctx:o}),t.$set(g);const $={};i&2&&($.$$scope={dirty:i,ctx:o}),l.$set($)},i(o){p||(J(t.$$.fragment,o),J(l.$$.fragment,o),p=!0)},o(o){T(t.$$.fragment,o),T(l.$$.fragment,o),p=!1},d(o){o&&a(r),b(t,o),b(l,o)}}}function Ms(j){let t,r="SDPA supports FlashAttention-2 as long as you have the latest PyTorch version installed.";return{c(){t=u("p"),t.textContent=r},l(l){t=M(l,"P",{"data-svelte-h":!0}),U(t)!=="svelte-12yxz1f"&&(t.textContent=r)},m(l,p){n(l,t,p)},p:W,d(l){l&&a(t)}}}function hs(j){let t,r='There are many quantization libraries (see the <a href="./quantization">Quantization</a> guide for more details) available, such as Quanto, AQLM, AWQ, and AutoGPTQ. Feel free to try them out and see which one works best for your use case. We also recommend reading the <a href="https://hf.co/blog/overview-quantization-transformers" rel="nofollow">Overview of natively supported quantization schemes in 🤗 Transformers</a> blog post which compares AutoGPTQ and bitsandbytes.';return{c(){t=u("p"),t.innerHTML=r},l(l){t=M(l,"P",{"data-svelte-h":!0}),U(t)!=="svelte-1b7gbou"&&(t.innerHTML=r)},m(l,p){n(l,t,p)},p:W,d(l){l&&a(t)}}}function ys(j){let t,r,l,p,o,i,g,$="Large language models (LLMs) have pushed text generation applications, such as chat and code completion models, to the next level by producing text that displays a high level of understanding and fluency. But what makes LLMs so powerful - namely their size - also presents challenges for inference.",Z,v,w="Basic inference is slow because LLMs have to be called repeatedly to generate the next token. The input sequence increases as generation progresses, which takes longer and longer for the LLM to process. LLMs also have billions of parameters, making it a challenge to store and handle all those weights in memory.",k,I,C="This guide will show you how to use the optimization techniques available in Transformers to accelerate LLM inference.",B,d,G,R,ve,S,ft="During decoding, a LLM computes the key-value (kv) values for each input token and since it is autoregressive, it computes the same kv values each time because the generated output becomes part of the input now. This is not very efficient because you’re recomputing the same kv values each time.",Ze,q,Jt='To optimize this, you can use a kv-cache to store the past keys and values instead of recomputing them each time. However, since the kv-cache grows with each generation step and is dynamic, it prevents you from taking advantage of <a href="./perf_torch_compile">torch.compile</a>, a powerful optimization tool that fuses PyTorch code into fast and optimized kernels.',ke,L,Tt="The <em>static kv-cache</em> solves this issue by pre-allocating the kv-cache size to a maximum value which allows you to combine it with torch.compile for up to a 4x speed up.",_e,X,Ie,A,bt='For this example, let’s load the <a href="https://hf.co/google/gemma-2b" rel="nofollow">Gemma</a> model.',We,D,Ce,P,wt='There are two ways you can configure the model to use a static kv-cache. For a 7B model on an A100, both methods get a 4x speed up in the forward pass. Your speed up may vary depending on the model size (larger models have a smaller speed up) and hardware. If you’re using the <a href="/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate">generate()</a> method, the speed up is ~3x. The forward pass (which still gets 4x speed up) is only a part of the whole <a href="/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate">generate()</a> code.',Re,z,Ve,K,Be,N,Xe,O,Ut="Another issue with autoregression is that for each input token you need to load the model weights each time during the forward pass. This is slow and cumbersome for LLMs which have billions of parameters. Speculative decoding alleviates this slowdown by using a second smaller and faster assistant model to generate candidate tokens that are verified by the larger LLM in a single forward pass. If the verified tokens are correct, the LLM essentially gets them for “free” without having to generate them itself. There is no degradation in accuracy because the verification forward pass ensures the same outputs are generated as if the LLM had generated them on its own.",ze,ee,jt="To get the largest speed up, the assistant model should be a lot smaller than the LLM so that it can generate tokens quickly. The assistant and LLM model must also share the same tokenizer to avoid re-encoding and decoding tokens.",Ne,F,Fe,te,gt='Enable speculative decoding by loading an assistant model and passing it to the <a href="/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate">generate()</a> method.',Qe,Q,Ye,se,He,le,$t="Prompt lookup decoding is a variant of speculative decoding that is also compatible with greedy search and sampling. Prompt lookup works especially well for input-grounded tasks - such as summarization - where there is often overlapping words between the prompt and output. These overlapping n-grams are used as the LLM candidate tokens.",xe,ae,Gt='To enable prompt lookup decoding, specify the number of tokens that should be overlapping in the <code>prompt_lookup_num_tokens</code> parameter. Then you can pass this parameter to the <a href="/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate">generate()</a> method.',Ee,Y,Se,ne,qe,oe,vt="A known issue with transformer models is that the self-attention mechanism grows quadratically in compute and memory with the number of input tokens. This limitation is only magnified in LLMs which handles much longer sequences. To address this, try FlashAttention2 or PyTorch’s scaled dot product attention (SDPA), which are more memory efficient attention implementations and can accelerate inference.",Le,ie,Ae,re,Zt='FlashAttention and <a href="./perf_infer_gpu_one#flashattention-2">FlashAttention-2</a> break up the attention computation into smaller chunks and reduces the number of intermediate read/write operations to GPU memory to speed up inference. FlashAttention-2 improves on the original FlashAttention algorithm by also parallelizing over sequence length dimension and better partitioning work on the hardware to reduce synchronization and communication overhead.',De,pe,kt='To use FlashAttention-2, set <code>attn_implementation=&quot;flash_attention_2&quot;</code> in the <a href="/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method.',Pe,ce,Ke,me,Oe,de,_t="Scaled dot product attention (SDPA) is automatically enabled in PyTorch 2.0 and it supports FlashAttention, xFormers, and PyTorch’s C++ implementation. SDPA chooses the most performant attention algorithm if you’re using a CUDA backend. For other backends, SDPA defaults to the PyTorch C++ implementation.",et,H,tt,ue,It='Use the <a href="https://pytorch.org/docs/master/generated/torch.nn.functional.scaled_dot_product_attention.html" rel="nofollow">torch.backends.cuda.sdp_kernel</a> context manager to explicitly enable or disable any of the three attention algorithms. For example, set <code>enable_flash=True</code> to enable FlashAttention.',st,Me,lt,he,at,ye,Wt="Quantization reduces the size of the LLM weights by storing them in a lower precision. This translates to lower memory usage and makes loading LLMs for inference more accessible if you’re constrained by your GPUs memory. If you aren’t limited by your GPU, you don’t necessarily need to quantize your model because it can incur a small latency cost (except for AWQ and fused AWQ modules) due to the extra step required to quantize and dequantize the weights.",nt,x,ot,fe,Ct='Use the Model Memory Calculator below to estimate and compare how much memory is required to load a model. For example, try estimating how much memory it costs to load <a href="https://huggingface.co/mistralai/Mistral-7B-v0.1" rel="nofollow">Mistral-7B-v0.1</a>.',it,V,Rt,rt,Je,Vt='To load Mistral-7B-v0.1 in half-precision, set the <code>torch_dtype</code> parameter in the <a href="/docs/transformers/main/en/model_doc/auto#transformers.AutoModel.from_pretrained">from_pretrained()</a> method to <code>torch.bfloat16</code>. This requires 13.74GB of memory.',pt,Te,ct,be,Bt='To load a quantized model (8-bit or 4-bit) for inference, try <a href="https://hf.co/docs/bitsandbytes" rel="nofollow">bitsandbytes</a> and set the <code>load_in_4bit</code> or <code>load_in_8bit</code> parameters to <code>True</code>. Loading the model in 8-bits only requires 6.87 GB of memory.',mt,we,dt,Ue,ut,Ge,Mt;return o=new E({props:{title:"LLM inference optimization",local:"llm-inference-optimization",headingTag:"h1"}}),d=new je({props:{warning:!1,$$slots:{default:[es]},$$scope:{ctx:j}}}),R=new E({props:{title:"Static kv-cache and torch.compile",local:"static-kv-cache-and-torchcompile",headingTag:"h2"}}),X=new je({props:{warning:!0,$$slots:{default:[ts]},$$scope:{ctx:j}}}),D=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZSUyRmdlbW1hLTJiJTIyKSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmdvb2dsZSUyRmdlbW1hLTJiJTIyJTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMEEp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;google/gemma-2b&quot;</span>)
model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;google/gemma-2b&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>
)`,wrap:!1}}),z=new yt({props:{id:"static-kv",options:["generation_config","setup_cache"],$$slots:{default:[ns]},$$scope:{ctx:j}}}),K=new E({props:{title:"Speculative decoding",local:"speculative-decoding",headingTag:"h2"}}),N=new je({props:{warning:!1,$$slots:{default:[os]},$$scope:{ctx:j}}}),F=new je({props:{warning:!0,$$slots:{default:[is]},$$scope:{ctx:j}}}),Q=new yt({props:{id:"spec-decoding",options:["greedy search","sampling"],$$slots:{default:[cs]},$$scope:{ctx:j}}}),se=new E({props:{title:"Prompt lookup decoding",local:"prompt-lookup-decoding",headingTag:"h3"}}),Y=new yt({props:{id:"pld",options:["greedy decoding","sampling"],$$slots:{default:[us]},$$scope:{ctx:j}}}),ne=new E({props:{title:"Attention optimizations",local:"attention-optimizations",headingTag:"h2"}}),ie=new E({props:{title:"FlashAttention-2",local:"flashattention-2",headingTag:"h3"}}),ce=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBcXVhbnRfY29uZmlnJTIwJTNEJTIwQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fOGJpdCUzRFRydWUpJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyZ29vZ2xlJTJGZ2VtbWEtMmIlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRfY29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5iZmxvYXQxNiUyQyUwQSUyMCUyMCUyMCUyMGF0dG5faW1wbGVtZW50YXRpb24lM0QlMjJmbGFzaF9hdHRlbnRpb25fMiUyMiUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, BitsAndBytesConfig

quant_config = BitsAndBytesConfig(load_in_8bit=<span class="hljs-literal">True</span>)
model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;google/gemma-2b&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.bfloat16,
    attn_implementation=<span class="hljs-string">&quot;flash_attention_2&quot;</span>,
)`,wrap:!1}}),me=new E({props:{title:"PyTorch scaled dot product attention",local:"pytorch-scaled-dot-product-attention",headingTag:"h3"}}),H=new je({props:{warning:!1,$$slots:{default:[Ms]},$$scope:{ctx:j}}}),Me=new _({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJnb29nbGUlMkZnZW1tYS0yYiUyMiUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEdG9yY2guYmZsb2F0MTYlMkMlMEEpJTBBJTBBd2l0aCUyMHRvcmNoLmJhY2tlbmRzLmN1ZGEuc2RwX2tlcm5lbChlbmFibGVfZmxhc2glM0RUcnVlJTJDJTIwZW5hYmxlX21hdGglM0RGYWxzZSUyQyUyMGVuYWJsZV9tZW1fZWZmaWNpZW50JTNERmFsc2UpJTNBJTBBJTIwJTIwJTIwJTIwb3V0cHV0cyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqaW5wdXRzKQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;google/gemma-2b&quot;</span>,
    torch_dtype=torch.bfloat16,
)

<span class="hljs-keyword">with</span> torch.backends.cuda.sdp_kernel(enable_flash=<span class="hljs-literal">True</span>, enable_math=<span class="hljs-literal">False</span>, enable_mem_efficient=<span class="hljs-literal">False</span>):
    outputs = model.generate(**inputs)`,wrap:!1}}),he=new E({props:{title:"Quantization",local:"quantization",headingTag:"h2"}}),x=new je({props:{warning:!1,$$slots:{default:[hs]},$$scope:{ctx:j}}}),Te=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUwQWltcG9ydCUyMHRvcmNoJTBBJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIybWlzdHJhbGFpJTJGTWlzdHJhbC03Qi12MC4xJTIyJTJDJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5iZmxvYXQxNiUyQyUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM
<span class="hljs-keyword">import</span> torch

model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;mistralai/Mistral-7B-v0.1&quot;</span>, torch_dtype=torch.bfloat16, device_map=<span class="hljs-string">&quot;auto&quot;</span>,
)`,wrap:!1}}),we=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUyQyUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQWltcG9ydCUyMHRvcmNoJTBBJTBBcXVhbnRfY29uZmlnJTIwJTNEJTIwQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fOGJpdCUzRFRydWUpJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIybWlzdHJhbGFpJTJGTWlzdHJhbC03Qi12MC4xJTIyJTJDJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50X2NvbmZpZyUyQyUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
<span class="hljs-keyword">import</span> torch

quant_config = BitsAndBytesConfig(load_in_8bit=<span class="hljs-literal">True</span>)
model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;mistralai/Mistral-7B-v0.1&quot;</span>, quantization_config=quant_config, device_map=<span class="hljs-string">&quot;auto&quot;</span>
)`,wrap:!1}}),Ue=new Ot({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/llm_optims.md"}}),{c(){t=u("meta"),r=c(),l=u("p"),p=c(),h(o.$$.fragment),i=c(),g=u("p"),g.textContent=$,Z=c(),v=u("p"),v.textContent=w,k=c(),I=u("p"),I.textContent=C,B=c(),h(d.$$.fragment),G=c(),h(R.$$.fragment),ve=c(),S=u("p"),S.textContent=ft,Ze=c(),q=u("p"),q.innerHTML=Jt,ke=c(),L=u("p"),L.innerHTML=Tt,_e=c(),h(X.$$.fragment),Ie=c(),A=u("p"),A.innerHTML=bt,We=c(),h(D.$$.fragment),Ce=c(),P=u("p"),P.innerHTML=wt,Re=c(),h(z.$$.fragment),Ve=c(),h(K.$$.fragment),Be=c(),h(N.$$.fragment),Xe=c(),O=u("p"),O.textContent=Ut,ze=c(),ee=u("p"),ee.textContent=jt,Ne=c(),h(F.$$.fragment),Fe=c(),te=u("p"),te.innerHTML=gt,Qe=c(),h(Q.$$.fragment),Ye=c(),h(se.$$.fragment),He=c(),le=u("p"),le.textContent=$t,xe=c(),ae=u("p"),ae.innerHTML=Gt,Ee=c(),h(Y.$$.fragment),Se=c(),h(ne.$$.fragment),qe=c(),oe=u("p"),oe.textContent=vt,Le=c(),h(ie.$$.fragment),Ae=c(),re=u("p"),re.innerHTML=Zt,De=c(),pe=u("p"),pe.innerHTML=kt,Pe=c(),h(ce.$$.fragment),Ke=c(),h(me.$$.fragment),Oe=c(),de=u("p"),de.textContent=_t,et=c(),h(H.$$.fragment),tt=c(),ue=u("p"),ue.innerHTML=It,st=c(),h(Me.$$.fragment),lt=c(),h(he.$$.fragment),at=c(),ye=u("p"),ye.textContent=Wt,nt=c(),h(x.$$.fragment),ot=c(),fe=u("p"),fe.innerHTML=Ct,it=c(),V=u("iframe"),rt=c(),Je=u("p"),Je.innerHTML=Vt,pt=c(),h(Te.$$.fragment),ct=c(),be=u("p"),be.innerHTML=Bt,mt=c(),h(we.$$.fragment),dt=c(),h(Ue.$$.fragment),ut=c(),Ge=u("p"),this.h()},l(e){const s=Pt("svelte-u9bgzb",document.head);t=M(s,"META",{name:!0,content:!0}),s.forEach(a),r=m(e),l=M(e,"P",{}),ht(l).forEach(a),p=m(e),y(o.$$.fragment,e),i=m(e),g=M(e,"P",{"data-svelte-h":!0}),U(g)!=="svelte-1kjw8nj"&&(g.textContent=$),Z=m(e),v=M(e,"P",{"data-svelte-h":!0}),U(v)!=="svelte-1l3iyos"&&(v.textContent=w),k=m(e),I=M(e,"P",{"data-svelte-h":!0}),U(I)!=="svelte-1wb7yan"&&(I.textContent=C),B=m(e),y(d.$$.fragment,e),G=m(e),y(R.$$.fragment,e),ve=m(e),S=M(e,"P",{"data-svelte-h":!0}),U(S)!=="svelte-bq37ce"&&(S.textContent=ft),Ze=m(e),q=M(e,"P",{"data-svelte-h":!0}),U(q)!=="svelte-1mrwa3s"&&(q.innerHTML=Jt),ke=m(e),L=M(e,"P",{"data-svelte-h":!0}),U(L)!=="svelte-yamton"&&(L.innerHTML=Tt),_e=m(e),y(X.$$.fragment,e),Ie=m(e),A=M(e,"P",{"data-svelte-h":!0}),U(A)!=="svelte-45dili"&&(A.innerHTML=bt),We=m(e),y(D.$$.fragment,e),Ce=m(e),P=M(e,"P",{"data-svelte-h":!0}),U(P)!=="svelte-1ndigja"&&(P.innerHTML=wt),Re=m(e),y(z.$$.fragment,e),Ve=m(e),y(K.$$.fragment,e),Be=m(e),y(N.$$.fragment,e),Xe=m(e),O=M(e,"P",{"data-svelte-h":!0}),U(O)!=="svelte-18z9bzk"&&(O.textContent=Ut),ze=m(e),ee=M(e,"P",{"data-svelte-h":!0}),U(ee)!=="svelte-1nm1j4o"&&(ee.textContent=jt),Ne=m(e),y(F.$$.fragment,e),Fe=m(e),te=M(e,"P",{"data-svelte-h":!0}),U(te)!=="svelte-4l211j"&&(te.innerHTML=gt),Qe=m(e),y(Q.$$.fragment,e),Ye=m(e),y(se.$$.fragment,e),He=m(e),le=M(e,"P",{"data-svelte-h":!0}),U(le)!=="svelte-123yl06"&&(le.textContent=$t),xe=m(e),ae=M(e,"P",{"data-svelte-h":!0}),U(ae)!=="svelte-1xnaq70"&&(ae.innerHTML=Gt),Ee=m(e),y(Y.$$.fragment,e),Se=m(e),y(ne.$$.fragment,e),qe=m(e),oe=M(e,"P",{"data-svelte-h":!0}),U(oe)!=="svelte-zxt0p7"&&(oe.textContent=vt),Le=m(e),y(ie.$$.fragment,e),Ae=m(e),re=M(e,"P",{"data-svelte-h":!0}),U(re)!=="svelte-pa091d"&&(re.innerHTML=Zt),De=m(e),pe=M(e,"P",{"data-svelte-h":!0}),U(pe)!=="svelte-1ym14uu"&&(pe.innerHTML=kt),Pe=m(e),y(ce.$$.fragment,e),Ke=m(e),y(me.$$.fragment,e),Oe=m(e),de=M(e,"P",{"data-svelte-h":!0}),U(de)!=="svelte-16n93px"&&(de.textContent=_t),et=m(e),y(H.$$.fragment,e),tt=m(e),ue=M(e,"P",{"data-svelte-h":!0}),U(ue)!=="svelte-cbkj1o"&&(ue.innerHTML=It),st=m(e),y(Me.$$.fragment,e),lt=m(e),y(he.$$.fragment,e),at=m(e),ye=M(e,"P",{"data-svelte-h":!0}),U(ye)!=="svelte-x4c749"&&(ye.textContent=Wt),nt=m(e),y(x.$$.fragment,e),ot=m(e),fe=M(e,"P",{"data-svelte-h":!0}),U(fe)!=="svelte-1i2aaqz"&&(fe.innerHTML=Ct),it=m(e),V=M(e,"IFRAME",{src:!0,frameborder:!0,width:!0,height:!0}),ht(V).forEach(a),rt=m(e),Je=M(e,"P",{"data-svelte-h":!0}),U(Je)!=="svelte-l6iyvl"&&(Je.innerHTML=Vt),pt=m(e),y(Te.$$.fragment,e),ct=m(e),be=M(e,"P",{"data-svelte-h":!0}),U(be)!=="svelte-a32zwr"&&(be.innerHTML=Bt),mt=m(e),y(we.$$.fragment,e),dt=m(e),y(Ue.$$.fragment,e),ut=m(e),Ge=M(e,"P",{}),ht(Ge).forEach(a),this.h()},h(){ge(t,"name","hf:doc:metadata"),ge(t,"content",fs),qt(V.src,Rt="https://hf-accelerate-model-memory-usage.hf.space")||ge(V,"src",Rt),ge(V,"frameborder","0"),ge(V,"width","850"),ge(V,"height","450")},m(e,s){Kt(document.head,t),n(e,r,s),n(e,l,s),n(e,p,s),f(o,e,s),n(e,i,s),n(e,g,s),n(e,Z,s),n(e,v,s),n(e,k,s),n(e,I,s),n(e,B,s),f(d,e,s),n(e,G,s),f(R,e,s),n(e,ve,s),n(e,S,s),n(e,Ze,s),n(e,q,s),n(e,ke,s),n(e,L,s),n(e,_e,s),f(X,e,s),n(e,Ie,s),n(e,A,s),n(e,We,s),f(D,e,s),n(e,Ce,s),n(e,P,s),n(e,Re,s),f(z,e,s),n(e,Ve,s),f(K,e,s),n(e,Be,s),f(N,e,s),n(e,Xe,s),n(e,O,s),n(e,ze,s),n(e,ee,s),n(e,Ne,s),f(F,e,s),n(e,Fe,s),n(e,te,s),n(e,Qe,s),f(Q,e,s),n(e,Ye,s),f(se,e,s),n(e,He,s),n(e,le,s),n(e,xe,s),n(e,ae,s),n(e,Ee,s),f(Y,e,s),n(e,Se,s),f(ne,e,s),n(e,qe,s),n(e,oe,s),n(e,Le,s),f(ie,e,s),n(e,Ae,s),n(e,re,s),n(e,De,s),n(e,pe,s),n(e,Pe,s),f(ce,e,s),n(e,Ke,s),f(me,e,s),n(e,Oe,s),n(e,de,s),n(e,et,s),f(H,e,s),n(e,tt,s),n(e,ue,s),n(e,st,s),f(Me,e,s),n(e,lt,s),f(he,e,s),n(e,at,s),n(e,ye,s),n(e,nt,s),f(x,e,s),n(e,ot,s),n(e,fe,s),n(e,it,s),n(e,V,s),n(e,rt,s),n(e,Je,s),n(e,pt,s),f(Te,e,s),n(e,ct,s),n(e,be,s),n(e,mt,s),f(we,e,s),n(e,dt,s),f(Ue,e,s),n(e,ut,s),n(e,Ge,s),Mt=!0},p(e,[s]){const Xt={};s&2&&(Xt.$$scope={dirty:s,ctx:e}),d.$set(Xt);const zt={};s&2&&(zt.$$scope={dirty:s,ctx:e}),X.$set(zt);const Nt={};s&2&&(Nt.$$scope={dirty:s,ctx:e}),z.$set(Nt);const Ft={};s&2&&(Ft.$$scope={dirty:s,ctx:e}),N.$set(Ft);const Qt={};s&2&&(Qt.$$scope={dirty:s,ctx:e}),F.$set(Qt);const Yt={};s&2&&(Yt.$$scope={dirty:s,ctx:e}),Q.$set(Yt);const Ht={};s&2&&(Ht.$$scope={dirty:s,ctx:e}),Y.$set(Ht);const xt={};s&2&&(xt.$$scope={dirty:s,ctx:e}),H.$set(xt);const Et={};s&2&&(Et.$$scope={dirty:s,ctx:e}),x.$set(Et)},i(e){Mt||(J(o.$$.fragment,e),J(d.$$.fragment,e),J(R.$$.fragment,e),J(X.$$.fragment,e),J(D.$$.fragment,e),J(z.$$.fragment,e),J(K.$$.fragment,e),J(N.$$.fragment,e),J(F.$$.fragment,e),J(Q.$$.fragment,e),J(se.$$.fragment,e),J(Y.$$.fragment,e),J(ne.$$.fragment,e),J(ie.$$.fragment,e),J(ce.$$.fragment,e),J(me.$$.fragment,e),J(H.$$.fragment,e),J(Me.$$.fragment,e),J(he.$$.fragment,e),J(x.$$.fragment,e),J(Te.$$.fragment,e),J(we.$$.fragment,e),J(Ue.$$.fragment,e),Mt=!0)},o(e){T(o.$$.fragment,e),T(d.$$.fragment,e),T(R.$$.fragment,e),T(X.$$.fragment,e),T(D.$$.fragment,e),T(z.$$.fragment,e),T(K.$$.fragment,e),T(N.$$.fragment,e),T(F.$$.fragment,e),T(Q.$$.fragment,e),T(se.$$.fragment,e),T(Y.$$.fragment,e),T(ne.$$.fragment,e),T(ie.$$.fragment,e),T(ce.$$.fragment,e),T(me.$$.fragment,e),T(H.$$.fragment,e),T(Me.$$.fragment,e),T(he.$$.fragment,e),T(x.$$.fragment,e),T(Te.$$.fragment,e),T(we.$$.fragment,e),T(Ue.$$.fragment,e),Mt=!1},d(e){e&&(a(r),a(l),a(p),a(i),a(g),a(Z),a(v),a(k),a(I),a(B),a(G),a(ve),a(S),a(Ze),a(q),a(ke),a(L),a(_e),a(Ie),a(A),a(We),a(Ce),a(P),a(Re),a(Ve),a(Be),a(Xe),a(O),a(ze),a(ee),a(Ne),a(Fe),a(te),a(Qe),a(Ye),a(He),a(le),a(xe),a(ae),a(Ee),a(Se),a(qe),a(oe),a(Le),a(Ae),a(re),a(De),a(pe),a(Pe),a(Ke),a(Oe),a(de),a(et),a(tt),a(ue),a(st),a(lt),a(at),a(ye),a(nt),a(ot),a(fe),a(it),a(V),a(rt),a(Je),a(pt),a(ct),a(be),a(mt),a(dt),a(ut),a(Ge)),a(t),b(o,e),b(d,e),b(R,e),b(X,e),b(D,e),b(z,e),b(K,e),b(N,e),b(F,e),b(Q,e),b(se,e),b(Y,e),b(ne,e),b(ie,e),b(ce,e),b(me,e),b(H,e),b(Me,e),b(he,e),b(x,e),b(Te,e),b(we,e),b(Ue,e)}}}const fs='{"title":"LLM inference optimization","local":"llm-inference-optimization","sections":[{"title":"Static kv-cache and torch.compile","local":"static-kv-cache-and-torchcompile","sections":[],"depth":2},{"title":"Speculative decoding","local":"speculative-decoding","sections":[{"title":"Prompt lookup decoding","local":"prompt-lookup-decoding","sections":[],"depth":3}],"depth":2},{"title":"Attention optimizations","local":"attention-optimizations","sections":[{"title":"FlashAttention-2","local":"flashattention-2","sections":[],"depth":3},{"title":"PyTorch scaled dot product attention","local":"pytorch-scaled-dot-product-attention","sections":[],"depth":3}],"depth":2},{"title":"Quantization","local":"quantization","sections":[],"depth":2}],"depth":1}';function Js(j){return Lt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class $s extends At{constructor(t){super(),Dt(this,t,Js,ys,St,{})}}export{$s as component};
