import{s as Fe,o as Ee,n as Be}from"../chunks/scheduler.8c3d61f6.js";import{S as Qe,i as Ye,g as o,s as l,r as M,A as Le,h as p,f as n,c as a,j as He,u as w,x as u,k as Ae,y as We,a as i,v as b,d as T,t as y,w as J}from"../chunks/index.da70eac4.js";import{T as Ge}from"../chunks/Tip.1d9b8c37.js";import{C as be}from"../chunks/CodeBlock.a9c4becf.js";import{H as Te,E as Pe}from"../chunks/index.ef90fe87.js";function Ve(L){let s,d='Interested in adding a new quantization method to Diffusers? Refer to the <a href="https://huggingface.co/docs/transformers/main/en/quantization/contribute" rel="nofollow">Contribute new quantization method guide</a> to learn more about adding a new quantization method.';return{c(){s=o("p"),s.innerHTML=d},l(r){s=p(r,"P",{"data-svelte-h":!0}),u(s)!=="svelte-1rkietk"&&(s.innerHTML=d)},m(r,f){i(r,s,f)},p:Be,d(r){r&&n(s)}}}function Ne(L){let s,d="If you are new to the quantization field, we recommend you to check out these beginner-friendly courses about quantization in collaboration with DeepLearning.AI:",r,f,h='<li><a href="https://www.deeplearning.ai/short-courses/quantization-fundamentals-with-hugging-face/" rel="nofollow">Quantization Fundamentals with Hugging Face</a></li> <li><a href="https://www.deeplearning.ai/short-courses/quantization-in-depth/" rel="nofollow">Quantization in Depth</a></li>';return{c(){s=o("p"),s.textContent=d,r=l(),f=o("ul"),f.innerHTML=h},l(m){s=p(m,"P",{"data-svelte-h":!0}),u(s)!=="svelte-lvs4zq"&&(s.textContent=d),r=a(m),f=p(m,"UL",{"data-svelte-h":!0}),u(f)!=="svelte-tsydsg"&&(f.innerHTML=h)},m(m,c){i(m,s,c),i(m,r,c),i(m,f,c)},p:Be,d(m){m&&(n(s),n(r),n(f))}}}function Se(L){let s,d,r,f,h,m,c,ye="Quantization techniques focus on representing data with less information while also trying to not lose too much accuracy. This often means converting a data type to represent the same information with fewer bits. For example, if your model weights are stored as 32-bit floating points and they’re quantized to 16-bit floating points, this halves the model size which makes it easier to store and reduces memory-usage. Lower precision can also speedup inference because it takes less time to perform calculations with fewer bits.",P,j,V,g,N,q,S,_,Je="Diffusers currently supports the following quantization methods.",D,$,je='<li><a href="./bitsandbytes">BitsandBytes</a></li> <li><a href="./torchao">TorchAO</a></li> <li><a href="./gguf">GGUF</a></li> <li><a href="./quanto.md">Quanto</a></li>',K,v,ge='<a href="https://huggingface.co/docs/transformers/main/en/quantization/overview#when-to-use-what" rel="nofollow">This resource</a> provides a good overview of the pros and cons of different quantization techniques.',O,z,ee,I,qe=`Diffusers allows users to directly initialize pipelines from checkpoints that may contain quantized models (<a href="https://huggingface.co/hf-internal-testing/flux.1-dev-nf4-pkg" rel="nofollow">example</a>). However, users may want to apply
quantization on-the-fly when initializing a pipeline from a pre-trained and non-quantized checkpoint. You can
do this with <a href="/docs/diffusers/main/en/api/quantization#diffusers.quantizers.PipelineQuantizationConfig">PipelineQuantizationConfig</a>.`,te,U,_e="Start by defining a <code>PipelineQuantizationConfig</code>:",ne,C,ie,Z,$e='Then pass it to <a href="/docs/diffusers/main/en/api/pipelines/overview#diffusers.DiffusionPipeline.from_pretrained">from_pretrained()</a> and run inference:',le,k,ae,X,ve=`This method allows for more granular control over the quantization specifications of individual
model-level components of a pipeline. It also allows for different quantization backends for
different components. In the above example, you used a combination of Quanto and BitsandBytes. However,
one caveat of this method is that users need to know which components come from <code>transformers</code> to be able
to import the right quantization config class.`,se,x,ze=`The other method is simpler in terms of experience but is
less-flexible. Start by defining a <code>PipelineQuantizationConfig</code> but in a different way:`,oe,R,pe,H,Ie='This <code>pipeline_quant_config</code> can now be passed to <a href="/docs/diffusers/main/en/api/pipelines/overview#diffusers.DiffusionPipeline.from_pretrained">from_pretrained()</a> similar to the above example.',ue,A,Ue=`In this case, <code>quant_kwargs</code> will be used to initialize the quantization specifications
of the respective quantization configuration class of <code>quant_backend</code>. <code>components_to_quantize</code>
is used to denote the components that will be quantized. For most pipelines, you would want to
keep <code>transformer</code> in the list as that is often the most compute and memory intensive.`,re,G,Ce=`The config below will work for most diffusion pipelines that have a <code>transformer</code> component present.
In most case, you will want to quantize the <code>transformer</code> component as that is often the most compute-
intensive part of a diffusion pipeline.`,fe,B,me,F,Ze="Below is a list of the supported quantization backends available in both <code>diffusers</code> and <code>transformers</code>:",ce,E,ke="<li><code>bitsandbytes_4bit</code></li> <li><code>bitsandbytes_8bit</code></li> <li><code>gguf</code></li> <li><code>quanto</code></li> <li><code>torchao</code></li>",de,Q,Xe=`Diffusion pipelines can have multiple text encoders. <a href="/docs/diffusers/main/en/api/pipelines/flux#diffusers.FluxPipeline">FluxPipeline</a> has two, for example. It’s
recommended to quantize the text encoders that are memory-intensive. Some examples include T5,
Llama, Gemma, etc. In the above example, you quantized the T5 model of <a href="/docs/diffusers/main/en/api/pipelines/flux#diffusers.FluxPipeline">FluxPipeline</a> through
<code>text_encoder_2</code> while keeping the CLIP model intact (accessible through <code>text_encoder</code>).`,he,Y,Me,W,we;return h=new Te({props:{title:"Quantization",local:"quantization",headingTag:"h1"}}),j=new Ge({props:{$$slots:{default:[Ve]},$$scope:{ctx:L}}}),g=new Ge({props:{$$slots:{default:[Ne]},$$scope:{ctx:L}}}),q=new Te({props:{title:"When to use what?",local:"when-to-use-what",headingTag:"h2"}}),z=new Te({props:{title:"Pipeline-level quantization",local:"pipeline-level-quantization",headingTag:"h2"}}),C=new be({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwRGlmZnVzaW9uUGlwZWxpbmUlMEFmcm9tJTIwZGlmZnVzZXJzLnF1YW50aXplcnMucXVhbnRpemF0aW9uX2NvbmZpZyUyMGltcG9ydCUyMFF1YW50b0NvbmZpZyUwQWZyb20lMjBkaWZmdXNlcnMucXVhbnRpemVycyUyMGltcG9ydCUyMFBpcGVsaW5lUXVhbnRpemF0aW9uQ29uZmlnJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQXBpcGVsaW5lX3F1YW50X2NvbmZpZyUyMCUzRCUyMFBpcGVsaW5lUXVhbnRpemF0aW9uQ29uZmlnKCUwQSUyMCUyMCUyMCUyMHF1YW50X21hcHBpbmclM0QlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJ0cmFuc2Zvcm1lciUyMiUzQSUyMFF1YW50b0NvbmZpZyh3ZWlnaHRzX2R0eXBlJTNEJTIyaW50OCUyMiklMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJ0ZXh0X2VuY29kZXJfMiUyMiUzQSUyMEJpdHNBbmRCeXRlc0NvbmZpZyglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsb2FkX2luXzRiaXQlM0RUcnVlJTJDJTIwY29tcHV0ZV9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKSUyQyUwQSUyMCUyMCUyMCUyMCU3RCUwQSk=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> DiffusionPipeline
<span class="hljs-keyword">from</span> diffusers.quantizers.quantization_config <span class="hljs-keyword">import</span> QuantoConfig
<span class="hljs-keyword">from</span> diffusers.quantizers <span class="hljs-keyword">import</span> PipelineQuantizationConfig
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig

pipeline_quant_config = PipelineQuantizationConfig(
    quant_mapping={
        <span class="hljs-string">&quot;transformer&quot;</span>: QuantoConfig(weights_dtype=<span class="hljs-string">&quot;int8&quot;</span>),
        <span class="hljs-string">&quot;text_encoder_2&quot;</span>: BitsAndBytesConfig(
            load_in_4bit=<span class="hljs-literal">True</span>, compute_dtype=torch.bfloat16
        ),
    }
)`,wrap:!1}}),k=new be({props:{code:"cGlwZSUyMCUzRCUyMERpZmZ1c2lvblBpcGVsaW5lLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJibGFjay1mb3Jlc3QtbGFicyUyRkZMVVguMS1kZXYlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcGlwZWxpbmVfcXVhbnRfY29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5iZmxvYXQxNiUyQyUwQSkudG8oJTIyY3VkYSUyMiklMEElMEFpbWFnZSUyMCUzRCUyMHBpcGUoJTIycGhvdG8lMjBvZiUyMGElMjBjdXRlJTIwZG9nJTIyKS5pbWFnZXMlNUIwJTVE",highlighted:`pipe = DiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    quantization_config=pipeline_quant_config,
    torch_dtype=torch.bfloat16,
).to(<span class="hljs-string">&quot;cuda&quot;</span>)

image = pipe(<span class="hljs-string">&quot;photo of a cute dog&quot;</span>).images[<span class="hljs-number">0</span>]`,wrap:!1}}),R=new be({props:{code:"cGlwZWxpbmVfcXVhbnRfY29uZmlnJTIwJTNEJTIwUGlwZWxpbmVRdWFudGl6YXRpb25Db25maWcoJTBBJTIwJTIwJTIwJTIwcXVhbnRfYmFja2VuZCUzRCUyMmJpdHNhbmRieXRlc180Yml0JTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRfa3dhcmdzJTNEJTdCJTIybG9hZF9pbl80Yml0JTIyJTNBJTIwVHJ1ZSUyQyUyMCUyMmJuYl80Yml0X3F1YW50X3R5cGUlMjIlM0ElMjAlMjJuZjQlMjIlMkMlMjAlMjJibmJfNGJpdF9jb21wdXRlX2R0eXBlJTIyJTNBJTIwdG9yY2guYmZsb2F0MTYlN0QlMkMlMEElMjAlMjAlMjAlMjBjb21wb25lbnRzX3RvX3F1YW50aXplJTNEJTVCJTIydHJhbnNmb3JtZXIlMjIlMkMlMjAlMjJ0ZXh0X2VuY29kZXJfMiUyMiU1RCUyQyUwQSk=",highlighted:`pipeline_quant_config = PipelineQuantizationConfig(
    quant_backend=<span class="hljs-string">&quot;bitsandbytes_4bit&quot;</span>,
    quant_kwargs={<span class="hljs-string">&quot;load_in_4bit&quot;</span>: <span class="hljs-literal">True</span>, <span class="hljs-string">&quot;bnb_4bit_quant_type&quot;</span>: <span class="hljs-string">&quot;nf4&quot;</span>, <span class="hljs-string">&quot;bnb_4bit_compute_dtype&quot;</span>: torch.bfloat16},
    components_to_quantize=[<span class="hljs-string">&quot;transformer&quot;</span>, <span class="hljs-string">&quot;text_encoder_2&quot;</span>],
)`,wrap:!1}}),B=new be({props:{code:"cGlwZWxpbmVfcXVhbnRfY29uZmlnJTIwJTNEJTIwUGlwZWxpbmVRdWFudGl6YXRpb25Db25maWcoJTBBJTIwJTIwJTIwJTIwcXVhbnRfYmFja2VuZCUzRCUyMmJpdHNhbmRieXRlc180Yml0JTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRfa3dhcmdzJTNEJTdCJTIybG9hZF9pbl80Yml0JTIyJTNBJTIwVHJ1ZSUyQyUyMCUyMmJuYl80Yml0X3F1YW50X3R5cGUlMjIlM0ElMjAlMjJuZjQlMjIlMkMlMjAlMjJibmJfNGJpdF9jb21wdXRlX2R0eXBlJTIyJTNBJTIwdG9yY2guYmZsb2F0MTYlN0QlMkMlMEElMjAlMjAlMjAlMjBjb21wb25lbnRzX3RvX3F1YW50aXplJTNEJTVCJTIydHJhbnNmb3JtZXIlMjIlNUQlMkMlMEEp",highlighted:`pipeline_quant_config = PipelineQuantizationConfig(
    quant_backend=<span class="hljs-string">&quot;bitsandbytes_4bit&quot;</span>,
    quant_kwargs={<span class="hljs-string">&quot;load_in_4bit&quot;</span>: <span class="hljs-literal">True</span>, <span class="hljs-string">&quot;bnb_4bit_quant_type&quot;</span>: <span class="hljs-string">&quot;nf4&quot;</span>, <span class="hljs-string">&quot;bnb_4bit_compute_dtype&quot;</span>: torch.bfloat16},
    components_to_quantize=[<span class="hljs-string">&quot;transformer&quot;</span>],
)`,wrap:!1}}),Y=new Pe({props:{source:"https://github.com/huggingface/diffusers/blob/main/docs/source/en/quantization/overview.md"}}),{c(){s=o("meta"),d=l(),r=o("p"),f=l(),M(h.$$.fragment),m=l(),c=o("p"),c.textContent=ye,P=l(),M(j.$$.fragment),V=l(),M(g.$$.fragment),N=l(),M(q.$$.fragment),S=l(),_=o("p"),_.textContent=Je,D=l(),$=o("ul"),$.innerHTML=je,K=l(),v=o("p"),v.innerHTML=ge,O=l(),M(z.$$.fragment),ee=l(),I=o("p"),I.innerHTML=qe,te=l(),U=o("p"),U.innerHTML=_e,ne=l(),M(C.$$.fragment),ie=l(),Z=o("p"),Z.innerHTML=$e,le=l(),M(k.$$.fragment),ae=l(),X=o("p"),X.innerHTML=ve,se=l(),x=o("p"),x.innerHTML=ze,oe=l(),M(R.$$.fragment),pe=l(),H=o("p"),H.innerHTML=Ie,ue=l(),A=o("p"),A.innerHTML=Ue,re=l(),G=o("p"),G.innerHTML=Ce,fe=l(),M(B.$$.fragment),me=l(),F=o("p"),F.innerHTML=Ze,ce=l(),E=o("ul"),E.innerHTML=ke,de=l(),Q=o("p"),Q.innerHTML=Xe,he=l(),M(Y.$$.fragment),Me=l(),W=o("p"),this.h()},l(e){const t=Le("svelte-u9bgzb",document.head);s=p(t,"META",{name:!0,content:!0}),t.forEach(n),d=a(e),r=p(e,"P",{}),He(r).forEach(n),f=a(e),w(h.$$.fragment,e),m=a(e),c=p(e,"P",{"data-svelte-h":!0}),u(c)!=="svelte-1euzkei"&&(c.textContent=ye),P=a(e),w(j.$$.fragment,e),V=a(e),w(g.$$.fragment,e),N=a(e),w(q.$$.fragment,e),S=a(e),_=p(e,"P",{"data-svelte-h":!0}),u(_)!=="svelte-2vpsnq"&&(_.textContent=Je),D=a(e),$=p(e,"UL",{"data-svelte-h":!0}),u($)!=="svelte-ct2mst"&&($.innerHTML=je),K=a(e),v=p(e,"P",{"data-svelte-h":!0}),u(v)!=="svelte-199sve8"&&(v.innerHTML=ge),O=a(e),w(z.$$.fragment,e),ee=a(e),I=p(e,"P",{"data-svelte-h":!0}),u(I)!=="svelte-mbbluf"&&(I.innerHTML=qe),te=a(e),U=p(e,"P",{"data-svelte-h":!0}),u(U)!=="svelte-lqe7ww"&&(U.innerHTML=_e),ne=a(e),w(C.$$.fragment,e),ie=a(e),Z=p(e,"P",{"data-svelte-h":!0}),u(Z)!=="svelte-1k5x077"&&(Z.innerHTML=$e),le=a(e),w(k.$$.fragment,e),ae=a(e),X=p(e,"P",{"data-svelte-h":!0}),u(X)!=="svelte-80ipkg"&&(X.innerHTML=ve),se=a(e),x=p(e,"P",{"data-svelte-h":!0}),u(x)!=="svelte-1e367nm"&&(x.innerHTML=ze),oe=a(e),w(R.$$.fragment,e),pe=a(e),H=p(e,"P",{"data-svelte-h":!0}),u(H)!=="svelte-18r02ua"&&(H.innerHTML=Ie),ue=a(e),A=p(e,"P",{"data-svelte-h":!0}),u(A)!=="svelte-172kpb7"&&(A.innerHTML=Ue),re=a(e),G=p(e,"P",{"data-svelte-h":!0}),u(G)!=="svelte-1x2kk7q"&&(G.innerHTML=Ce),fe=a(e),w(B.$$.fragment,e),me=a(e),F=p(e,"P",{"data-svelte-h":!0}),u(F)!=="svelte-11jz92r"&&(F.innerHTML=Ze),ce=a(e),E=p(e,"UL",{"data-svelte-h":!0}),u(E)!=="svelte-1wzcgf3"&&(E.innerHTML=ke),de=a(e),Q=p(e,"P",{"data-svelte-h":!0}),u(Q)!=="svelte-176iuvs"&&(Q.innerHTML=Xe),he=a(e),w(Y.$$.fragment,e),Me=a(e),W=p(e,"P",{}),He(W).forEach(n),this.h()},h(){Ae(s,"name","hf:doc:metadata"),Ae(s,"content",De)},m(e,t){We(document.head,s),i(e,d,t),i(e,r,t),i(e,f,t),b(h,e,t),i(e,m,t),i(e,c,t),i(e,P,t),b(j,e,t),i(e,V,t),b(g,e,t),i(e,N,t),b(q,e,t),i(e,S,t),i(e,_,t),i(e,D,t),i(e,$,t),i(e,K,t),i(e,v,t),i(e,O,t),b(z,e,t),i(e,ee,t),i(e,I,t),i(e,te,t),i(e,U,t),i(e,ne,t),b(C,e,t),i(e,ie,t),i(e,Z,t),i(e,le,t),b(k,e,t),i(e,ae,t),i(e,X,t),i(e,se,t),i(e,x,t),i(e,oe,t),b(R,e,t),i(e,pe,t),i(e,H,t),i(e,ue,t),i(e,A,t),i(e,re,t),i(e,G,t),i(e,fe,t),b(B,e,t),i(e,me,t),i(e,F,t),i(e,ce,t),i(e,E,t),i(e,de,t),i(e,Q,t),i(e,he,t),b(Y,e,t),i(e,Me,t),i(e,W,t),we=!0},p(e,[t]){const xe={};t&2&&(xe.$$scope={dirty:t,ctx:e}),j.$set(xe);const Re={};t&2&&(Re.$$scope={dirty:t,ctx:e}),g.$set(Re)},i(e){we||(T(h.$$.fragment,e),T(j.$$.fragment,e),T(g.$$.fragment,e),T(q.$$.fragment,e),T(z.$$.fragment,e),T(C.$$.fragment,e),T(k.$$.fragment,e),T(R.$$.fragment,e),T(B.$$.fragment,e),T(Y.$$.fragment,e),we=!0)},o(e){y(h.$$.fragment,e),y(j.$$.fragment,e),y(g.$$.fragment,e),y(q.$$.fragment,e),y(z.$$.fragment,e),y(C.$$.fragment,e),y(k.$$.fragment,e),y(R.$$.fragment,e),y(B.$$.fragment,e),y(Y.$$.fragment,e),we=!1},d(e){e&&(n(d),n(r),n(f),n(m),n(c),n(P),n(V),n(N),n(S),n(_),n(D),n($),n(K),n(v),n(O),n(ee),n(I),n(te),n(U),n(ne),n(ie),n(Z),n(le),n(ae),n(X),n(se),n(x),n(oe),n(pe),n(H),n(ue),n(A),n(re),n(G),n(fe),n(me),n(F),n(ce),n(E),n(de),n(Q),n(he),n(Me),n(W)),n(s),J(h,e),J(j,e),J(g,e),J(q,e),J(z,e),J(C,e),J(k,e),J(R,e),J(B,e),J(Y,e)}}}const De='{"title":"Quantization","local":"quantization","sections":[{"title":"When to use what?","local":"when-to-use-what","sections":[],"depth":2},{"title":"Pipeline-level quantization","local":"pipeline-level-quantization","sections":[],"depth":2}],"depth":1}';function Ke(L){return Ee(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class lt extends Qe{constructor(s){super(),Ye(this,s,Ke,Se,Fe,{})}}export{lt as component};
