import{s as rt,n as dt,o as pt}from"../chunks/scheduler.01eeda35.js";import{S as mt,i as ft,g as s,s as l,r as C,A as ut,h as i,f as n,c as o,j as ot,u as v,x as d,k as st,y as ht,a,v as L,d as _,t as T,w as A}from"../chunks/index.6dd51b66.js";import{C as it}from"../chunks/CodeBlock.19ec9b8c.js";import{H as K,E as ct}from"../chunks/index.58fe8f9d.js";function Mt(I){let r,H,Q,R,p,P,m,V='Additive Quantization of Language Models (<a href="https://arxiv.org/abs/2401.06118" rel="nofollow">AQLM</a>) quantizes multiple weights together and takes advantage of interdependencies between them. AQLM represents groups of 8-16 weights as a sum of multiple vector codes.',q,f,D='AQLM also supports fine-tuning with <a href="https://huggingface.co/docs/peft/package_reference/lora" rel="nofollow">LoRA</a> with the <a href="https://huggingface.co/docs/peft" rel="nofollow">PEFT</a> library, and is fully compatible with <a href="https://pytorch.org/tutorials/intermediate/torch_compile_tutorial.html" rel="nofollow">torch.compile</a> for even faster inference and training.',z,u,O="Run the command below to install the AQLM library with kernel support for both GPU and CPU inference and training. AQLM only works with Python 3.10+.",N,h,X,c,tt='Load an AQLM-quantized model with <a href="/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a>.',j,M,E,b,G,y,et="AQLM quantization setups vary mainly in the number of codebooks used, as well as codebook sizes in bits. The most popular setups and supported inference kernels are shown below.",J,$,nt="<thead><tr><th>Kernel</th> <th>Number of codebooks</th> <th>Codebook size, bits</th> <th>Notation</th> <th>Accuracy</th> <th>Speedup</th> <th>Fast GPU inference</th> <th>Fast CPU inference</th></tr></thead> <tbody><tr><td>Triton</td> <td>K</td> <td>N</td> <td>KxN</td> <td>-</td> <td>Up to ~0.7x</td> <td>✅</td> <td>❌</td></tr> <tr><td>CUDA</td> <td>1</td> <td>16</td> <td>1x16</td> <td>Best</td> <td>Up to ~1.3x</td> <td>✅</td> <td>❌</td></tr> <tr><td>CUDA</td> <td>2</td> <td>8</td> <td>2x8</td> <td>OK</td> <td>Up to ~3.0x</td> <td>✅</td> <td>❌</td></tr> <tr><td>Numba</td> <td>K</td> <td>8</td> <td>Kx8</td> <td>Good</td> <td>Up to ~4.0x</td> <td>❌</td> <td>✅</td></tr></tbody>",B,g,F,U,at='Run the AQLM demo <a href="https://colab.research.google.com/drive/1-xZmBRXT5Fm3Ghn4Mwa2KRypORXb855X?usp=sharing" rel="nofollow">notebook</a> for more examples of how to quantize a model, push a quantized model to the Hub, and more.',S,w,lt='For more example demo notebooks, visit the AQLM <a href="https://github.com/Vahe1994/AQLM" rel="nofollow">repository</a>.',Z,x,W,k,Y;return p=new K({props:{title:"AQLM",local:"aqlm",headingTag:"h1"}}),h=new it({props:{code:"cGlwJTIwaW5zdGFsbCUyMGFxbG0lNUJncHUlMkNjcHUlNUQ=",highlighted:"pip install aqlm[gpu,cpu]",wrap:!1}}),M=new it({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUwQSUwQXF1YW50aXplZF9tb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJJU1RBLURBU0xhYiUyRk1peHRyYWwtOHg3Yi1BUUxNLTJCaXQtMXgxNi1oZiUyMiUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEJTIyYXV0byUyMiUyQyUyMCUwQSUyMCUyMCUyMCUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM

quantized_model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;ISTA-DASLab/Mixtral-8x7b-AQLM-2Bit-1x16-hf&quot;</span>,
    torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>, 
    device_map=<span class="hljs-string">&quot;auto&quot;</span>
)`,wrap:!1}}),b=new K({props:{title:"Configurations",local:"configurations",headingTag:"h2"}}),g=new K({props:{title:"Resources",local:"resources",headingTag:"h2"}}),x=new ct({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/quantization/aqlm.md"}}),{c(){r=s("meta"),H=l(),Q=s("p"),R=l(),C(p.$$.fragment),P=l(),m=s("p"),m.innerHTML=V,q=l(),f=s("p"),f.innerHTML=D,z=l(),u=s("p"),u.textContent=O,N=l(),C(h.$$.fragment),X=l(),c=s("p"),c.innerHTML=tt,j=l(),C(M.$$.fragment),E=l(),C(b.$$.fragment),G=l(),y=s("p"),y.textContent=et,J=l(),$=s("table"),$.innerHTML=nt,B=l(),C(g.$$.fragment),F=l(),U=s("p"),U.innerHTML=at,S=l(),w=s("p"),w.innerHTML=lt,Z=l(),C(x.$$.fragment),W=l(),k=s("p"),this.h()},l(t){const e=ut("svelte-u9bgzb",document.head);r=i(e,"META",{name:!0,content:!0}),e.forEach(n),H=o(t),Q=i(t,"P",{}),ot(Q).forEach(n),R=o(t),v(p.$$.fragment,t),P=o(t),m=i(t,"P",{"data-svelte-h":!0}),d(m)!=="svelte-1n18v94"&&(m.innerHTML=V),q=o(t),f=i(t,"P",{"data-svelte-h":!0}),d(f)!=="svelte-xbaobp"&&(f.innerHTML=D),z=o(t),u=i(t,"P",{"data-svelte-h":!0}),d(u)!=="svelte-vyhcfw"&&(u.textContent=O),N=o(t),v(h.$$.fragment,t),X=o(t),c=i(t,"P",{"data-svelte-h":!0}),d(c)!=="svelte-1ffuixl"&&(c.innerHTML=tt),j=o(t),v(M.$$.fragment,t),E=o(t),v(b.$$.fragment,t),G=o(t),y=i(t,"P",{"data-svelte-h":!0}),d(y)!=="svelte-c1htvs"&&(y.textContent=et),J=o(t),$=i(t,"TABLE",{"data-svelte-h":!0}),d($)!=="svelte-mgp7ro"&&($.innerHTML=nt),B=o(t),v(g.$$.fragment,t),F=o(t),U=i(t,"P",{"data-svelte-h":!0}),d(U)!=="svelte-b17hb8"&&(U.innerHTML=at),S=o(t),w=i(t,"P",{"data-svelte-h":!0}),d(w)!=="svelte-1rn08ds"&&(w.innerHTML=lt),Z=o(t),v(x.$$.fragment,t),W=o(t),k=i(t,"P",{}),ot(k).forEach(n),this.h()},h(){st(r,"name","hf:doc:metadata"),st(r,"content",bt)},m(t,e){ht(document.head,r),a(t,H,e),a(t,Q,e),a(t,R,e),L(p,t,e),a(t,P,e),a(t,m,e),a(t,q,e),a(t,f,e),a(t,z,e),a(t,u,e),a(t,N,e),L(h,t,e),a(t,X,e),a(t,c,e),a(t,j,e),L(M,t,e),a(t,E,e),L(b,t,e),a(t,G,e),a(t,y,e),a(t,J,e),a(t,$,e),a(t,B,e),L(g,t,e),a(t,F,e),a(t,U,e),a(t,S,e),a(t,w,e),a(t,Z,e),L(x,t,e),a(t,W,e),a(t,k,e),Y=!0},p:dt,i(t){Y||(_(p.$$.fragment,t),_(h.$$.fragment,t),_(M.$$.fragment,t),_(b.$$.fragment,t),_(g.$$.fragment,t),_(x.$$.fragment,t),Y=!0)},o(t){T(p.$$.fragment,t),T(h.$$.fragment,t),T(M.$$.fragment,t),T(b.$$.fragment,t),T(g.$$.fragment,t),T(x.$$.fragment,t),Y=!1},d(t){t&&(n(H),n(Q),n(R),n(P),n(m),n(q),n(f),n(z),n(u),n(N),n(X),n(c),n(j),n(E),n(G),n(y),n(J),n($),n(B),n(F),n(U),n(S),n(w),n(Z),n(W),n(k)),n(r),A(p,t),A(h,t),A(M,t),A(b,t),A(g,t),A(x,t)}}}const bt='{"title":"AQLM","local":"aqlm","sections":[{"title":"Configurations","local":"configurations","sections":[],"depth":2},{"title":"Resources","local":"resources","sections":[],"depth":2}],"depth":1}';function yt(I){return pt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class xt extends mt{constructor(r){super(),ft(this,r,yt,Mt,rt,{})}}export{xt as component};
