import{s as Z,n as z,o as B}from"../chunks/scheduler.25b97de1.js";import{S as j,i as E,g as f,s as r,r as S,A as Y,h as d,f as a,c as o,j as x,u as Q,x as R,k as F,y as P,a as s,v as g,d as V,t as J,w as q}from"../chunks/index.d9030fc9.js";import{C as H}from"../chunks/CodeBlock.e6cd0d95.js";import{H as W,E as A}from"../chunks/EditOnGithub.91d95064.js";function N(k){let n,y,h,T,l,U,i,L='<a href="https://github.com/Vahe1994/SpQR" rel="nofollow">SpQR</a> quantization algorithm involves a 16x16 tiled bi-level group 3-bit quantization structure, with sparse outliers as detailed in <a href="https://arxiv.org/abs/2306.03078" rel="nofollow">SpQR: A Sparse-Quantized Representation for Near-Lossless LLM Weight Compression</a>.',b,m,X='To SpQR-quantize a model, refer to the <a href="https://github.com/Vahe1994/SpQR" rel="nofollow">Vahe1994/SpQR</a> repository.',_,p,G='Load a pre-SpQR-quantized model in <a href="/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a>.',v,u,C,c,w,M,$;return l=new W({props:{title:"SpQR",local:"spqr",headingTag:"h1"}}),u=new H({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUwQWltcG9ydCUyMHRvcmNoJTBBJTBBcXVhbnRpemVkX21vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmVsdmlyY3JuJTJGTGxhbWEtMi03Yi1TUFFSLTNCaXQtMTZ4MTYtcmVkX3BhamFtYS1oZiUyMiUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEdG9yY2guaGFsZiUyQyUwQSUyMCUyMCUyMCUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTBBKSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMmVsdmlyY3JuJTJGTGxhbWEtMi03Yi1TUFFSLTNCaXQtMTZ4MTYtcmVkX3BhamFtYS1oZiUyMik=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM
<span class="hljs-keyword">import</span> torch

quantized_model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;elvircrn/Llama-2-7b-SPQR-3Bit-16x16-red_pajama-hf&quot;</span>,
    torch_dtype=torch.half,
    device_map=<span class="hljs-string">&quot;auto&quot;</span>
)
tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;elvircrn/Llama-2-7b-SPQR-3Bit-16x16-red_pajama-hf&quot;</span>)`,wrap:!1}}),c=new A({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/quantization/spqr.md"}}),{c(){n=f("meta"),y=r(),h=f("p"),T=r(),S(l.$$.fragment),U=r(),i=f("p"),i.innerHTML=L,b=r(),m=f("p"),m.innerHTML=X,_=r(),p=f("p"),p.innerHTML=G,v=r(),S(u.$$.fragment),C=r(),S(c.$$.fragment),w=r(),M=f("p"),this.h()},l(t){const e=Y("svelte-u9bgzb",document.head);n=d(e,"META",{name:!0,content:!0}),e.forEach(a),y=o(t),h=d(t,"P",{}),x(h).forEach(a),T=o(t),Q(l.$$.fragment,t),U=o(t),i=d(t,"P",{"data-svelte-h":!0}),R(i)!=="svelte-rwflg"&&(i.innerHTML=L),b=o(t),m=d(t,"P",{"data-svelte-h":!0}),R(m)!=="svelte-159tqjq"&&(m.innerHTML=X),_=o(t),p=d(t,"P",{"data-svelte-h":!0}),R(p)!=="svelte-l69gi3"&&(p.innerHTML=G),v=o(t),Q(u.$$.fragment,t),C=o(t),Q(c.$$.fragment,t),w=o(t),M=d(t,"P",{}),x(M).forEach(a),this.h()},h(){F(n,"name","hf:doc:metadata"),F(n,"content",I)},m(t,e){P(document.head,n),s(t,y,e),s(t,h,e),s(t,T,e),g(l,t,e),s(t,U,e),s(t,i,e),s(t,b,e),s(t,m,e),s(t,_,e),s(t,p,e),s(t,v,e),g(u,t,e),s(t,C,e),g(c,t,e),s(t,w,e),s(t,M,e),$=!0},p:z,i(t){$||(V(l.$$.fragment,t),V(u.$$.fragment,t),V(c.$$.fragment,t),$=!0)},o(t){J(l.$$.fragment,t),J(u.$$.fragment,t),J(c.$$.fragment,t),$=!1},d(t){t&&(a(y),a(h),a(T),a(U),a(i),a(b),a(m),a(_),a(p),a(v),a(C),a(w),a(M)),a(n),q(l,t),q(u,t),q(c,t)}}}const I='{"title":"SpQR","local":"spqr","sections":[],"depth":1}';function K(k){return B(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class at extends j{constructor(n){super(),E(this,n,K,N,Z,{})}}export{at as component};
