import{s as nt,o as at,n as lt}from"../chunks/scheduler.25b97de1.js";import{S as st,i as it,g as p,s,r as Z,A as ot,h as m,f as n,c as i,j as tt,u as V,x as _,k as et,y as pt,a,v as F,d as C,t as G,w as X}from"../chunks/index.d9030fc9.js";import{T as mt}from"../chunks/Tip.baa67368.js";import{C as I}from"../chunks/CodeBlock.e6cd0d95.js";import{H as rt,E as ut}from"../chunks/EditOnGithub.91d95064.js";function ct(k){let l,u="You need a GPU with compute capability>=9 (e.g. H100)";return{c(){l=p("p"),l.textContent=u},l(o){l=m(o,"P",{"data-svelte-h":!0}),_(l)!=="svelte-uga55r"&&(l.textContent=u)},m(o,$){a(o,l,$)},p:lt,d(o){o&&n(l)}}}function ft(k){let l,u,o,$,c,J,f,N="With FBGEMM FP8 quantization method, you can quantize your model in FP8 (W8A8):",W,d,L="<li>the weights will be quantized in 8bit (FP8) per channel</li> <li>the activation will be quantized in 8bit (FP8) per token</li>",R,h,S='It relies on the <a href="https://github.com/pytorch/FBGEMM" rel="nofollow">FBGEMM</a> library which provides efficient low-precision general matrix multiplication for small batch sizes and support for accuracy-loss minimizing techniques such as row-wise quantization and outlier-aware quantization.',q,r,z,M,A="Before you begin, make sure the following libraries are installed with their latest version:",B,b,E,y,K='If you are having issues with fbgemm-gpu and torch library, you might need to install the nightly release. You can follow the instruction <a href="https://pytorch.org/FBGEMM/fbgemm_gpu-development/InstallationInstructions.html#fbgemm-gpu-install-libraries:~:text=found%20here.-,Install%20the%20FBGEMM_GPU%20Package,-Install%20through%20PyTorch" rel="nofollow">here</a>',x,w,j,U,D="A quantized model can be saved via “saved_pretrained” and be reused again via the “from_pretrained”.",Y,g,P,v,Q,T,H;return c=new rt({props:{title:"FBGEMM FP8",local:"fbgemm-fp8",headingTag:"h1"}}),r=new mt({props:{warning:!1,$$slots:{default:[ct]},$$scope:{ctx:k}}}),b=new I({props:{code:"cGlwJTIwaW5zdGFsbCUyMC0tdXBncmFkZSUyMGFjY2VsZXJhdGUlMjBmYmdlbW0tZ3B1JTIwdG9yY2g=",highlighted:"pip install --upgrade accelerate fbgemm-gpu torch",wrap:!1}}),w=new I({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEZiZ2VtbUZwOENvbmZpZyUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQW1vZGVsX25hbWUlMjAlM0QlMjAlMjJtZXRhLWxsYW1hJTJGTWV0YS1MbGFtYS0zLThCJTIyJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEZiZ2VtbUZwOENvbmZpZygpJTBBcXVhbnRpemVkX21vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX25hbWUlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudGl6YXRpb25fY29uZmlnKSUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX25hbWUpJTBBaW5wdXRfdGV4dCUyMCUzRCUyMCUyMldoYXQlMjBhcmUlMjB3ZSUyMGhhdmluZyUyMGZvciUyMGRpbm5lciUzRiUyMiUwQWlucHV0X2lkcyUyMCUzRCUyMHRva2VuaXplcihpbnB1dF90ZXh0JTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikudG8oJTIyY3VkYSUyMiklMEElMEFvdXRwdXQlMjAlM0QlMjBxdWFudGl6ZWRfbW9kZWwuZ2VuZXJhdGUoKippbnB1dF9pZHMlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDEwKSUwQXByaW50KHRva2VuaXplci5kZWNvZGUob3V0cHV0JTVCMCU1RCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> FbgemmFp8Config, AutoModelForCausalLM, AutoTokenizer

model_name = <span class="hljs-string">&quot;meta-llama/Meta-Llama-3-8B&quot;</span>
quantization_config = FbgemmFp8Config()
quantized_model = AutoModelForCausalLM.from_pretrained(model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, quantization_config=quantization_config)

tokenizer = AutoTokenizer.from_pretrained(model_name)
input_text = <span class="hljs-string">&quot;What are we having for dinner?&quot;</span>
input_ids = tokenizer(input_text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)

output = quantized_model.generate(**input_ids, max_new_tokens=<span class="hljs-number">10</span>)
<span class="hljs-built_in">print</span>(tokenizer.decode(output[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>))`,wrap:!1}}),g=new I({props:{code:"cXVhbnRfcGF0aCUyMCUzRCUyMCUyMiUyRnBhdGglMkZ0byUyRnNhdmUlMkZxdWFudGl6ZWQlMkZtb2RlbCUyMiUwQW1vZGVsLnNhdmVfcHJldHJhaW5lZChxdWFudF9wYXRoKSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKHF1YW50X3BhdGglMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMik=",highlighted:`quant_path = <span class="hljs-string">&quot;/path/to/save/quantized/model&quot;</span>
model.save_pretrained(quant_path)
model = AutoModelForCausalLM.from_pretrained(quant_path, device_map=<span class="hljs-string">&quot;auto&quot;</span>)`,wrap:!1}}),v=new ut({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/quantization/fbgemm_fp8.md"}}),{c(){l=p("meta"),u=s(),o=p("p"),$=s(),Z(c.$$.fragment),J=s(),f=p("p"),f.textContent=N,W=s(),d=p("ul"),d.innerHTML=L,R=s(),h=p("p"),h.innerHTML=S,q=s(),Z(r.$$.fragment),z=s(),M=p("p"),M.textContent=A,B=s(),Z(b.$$.fragment),E=s(),y=p("p"),y.innerHTML=K,x=s(),Z(w.$$.fragment),j=s(),U=p("p"),U.textContent=D,Y=s(),Z(g.$$.fragment),P=s(),Z(v.$$.fragment),Q=s(),T=p("p"),this.h()},l(t){const e=ot("svelte-u9bgzb",document.head);l=m(e,"META",{name:!0,content:!0}),e.forEach(n),u=i(t),o=m(t,"P",{}),tt(o).forEach(n),$=i(t),V(c.$$.fragment,t),J=i(t),f=m(t,"P",{"data-svelte-h":!0}),_(f)!=="svelte-1ippvwg"&&(f.textContent=N),W=i(t),d=m(t,"UL",{"data-svelte-h":!0}),_(d)!=="svelte-c7lsip"&&(d.innerHTML=L),R=i(t),h=m(t,"P",{"data-svelte-h":!0}),_(h)!=="svelte-p8lslh"&&(h.innerHTML=S),q=i(t),V(r.$$.fragment,t),z=i(t),M=m(t,"P",{"data-svelte-h":!0}),_(M)!=="svelte-1wrsrbm"&&(M.textContent=A),B=i(t),V(b.$$.fragment,t),E=i(t),y=m(t,"P",{"data-svelte-h":!0}),_(y)!=="svelte-ygzl1b"&&(y.innerHTML=K),x=i(t),V(w.$$.fragment,t),j=i(t),U=m(t,"P",{"data-svelte-h":!0}),_(U)!=="svelte-b8ansb"&&(U.textContent=D),Y=i(t),V(g.$$.fragment,t),P=i(t),V(v.$$.fragment,t),Q=i(t),T=m(t,"P",{}),tt(T).forEach(n),this.h()},h(){et(l,"name","hf:doc:metadata"),et(l,"content",dt)},m(t,e){pt(document.head,l),a(t,u,e),a(t,o,e),a(t,$,e),F(c,t,e),a(t,J,e),a(t,f,e),a(t,W,e),a(t,d,e),a(t,R,e),a(t,h,e),a(t,q,e),F(r,t,e),a(t,z,e),a(t,M,e),a(t,B,e),F(b,t,e),a(t,E,e),a(t,y,e),a(t,x,e),F(w,t,e),a(t,j,e),a(t,U,e),a(t,Y,e),F(g,t,e),a(t,P,e),F(v,t,e),a(t,Q,e),a(t,T,e),H=!0},p(t,[e]){const O={};e&2&&(O.$$scope={dirty:e,ctx:t}),r.$set(O)},i(t){H||(C(c.$$.fragment,t),C(r.$$.fragment,t),C(b.$$.fragment,t),C(w.$$.fragment,t),C(g.$$.fragment,t),C(v.$$.fragment,t),H=!0)},o(t){G(c.$$.fragment,t),G(r.$$.fragment,t),G(b.$$.fragment,t),G(w.$$.fragment,t),G(g.$$.fragment,t),G(v.$$.fragment,t),H=!1},d(t){t&&(n(u),n(o),n($),n(J),n(f),n(W),n(d),n(R),n(h),n(q),n(z),n(M),n(B),n(E),n(y),n(x),n(j),n(U),n(Y),n(P),n(Q),n(T)),n(l),X(c,t),X(r,t),X(b,t),X(w,t),X(g,t),X(v,t)}}}const dt='{"title":"FBGEMM FP8","local":"fbgemm-fp8","sections":[],"depth":1}';function ht(k){return at(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class gt extends st{constructor(l){super(),it(this,l,ht,ft,nt,{})}}export{gt as component};
