import{s as Jt,o as Tt,n as be}from"../chunks/scheduler.8c3d61f6.js";import{S as Ut,i as $t,g as J,s as i,r as d,A as wt,h as T,f as s,c as a,j as bt,u,x as U,k as Mt,y as Zt,a as n,v as y,d as c,t as b,w as M}from"../chunks/index.da70eac4.js";import{T as Ke}from"../chunks/Tip.1d9b8c37.js";import{C as z}from"../chunks/CodeBlock.00a903b3.js";import{H as j,E as Ct}from"../chunks/EditOnGithub.1e64e623.js";import{H as Wt,a as ht}from"../chunks/HfOption.c1483eb1.js";function gt(B){let l,$="Quantizing a model in 8-bit halves the memory-usage:",r,p,f,h,_="By default, all the other modules such as <code>torch.nn.LayerNorm</code> are converted to <code>torch.float16</code>. You can change the data type of these modules with the <code>torch_dtype</code> parameter if you want:",R,w,W,C,k='Once a model is quantized, you can push the model to the Hub with the <a href="/docs/diffusers/main/en/api/pipelines/overview#diffusers.utils.PushToHubMixin.push_to_hub">push_to_hub()</a> method. The quantization <code>config.json</code> file is pushed first, followed by the quantized model weights.',g,m,Z;return p=new z({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEZsdXhUcmFuc2Zvcm1lcjJETW9kZWwlMkMlMjBCaXRzQW5kQnl0ZXNDb25maWclMEElMEFxdWFudGl6YXRpb25fY29uZmlnJTIwJTNEJTIwQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fOGJpdCUzRFRydWUpJTBBJTBBbW9kZWxfOGJpdCUyMCUzRCUyMEZsdXhUcmFuc2Zvcm1lcjJETW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmJsYWNrLWZvcmVzdC1sYWJzJTJGRkxVWC4xLWRldiUyMiUyQyUyMCUwQSUyMCUyMCUyMCUyMHN1YmZvbGRlciUzRCUyMnRyYW5zZm9ybWVyJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWclMEEp",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> FluxTransformer2DModel, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_8bit=<span class="hljs-literal">True</span>)

model_8bit = FluxTransformer2DModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>, 
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quantization_config
)`,wrap:!1}}),w=new z({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEZsdXhUcmFuc2Zvcm1lcjJETW9kZWwlMkMlMjBCaXRzQW5kQnl0ZXNDb25maWclMEElMEFxdWFudGl6YXRpb25fY29uZmlnJTIwJTNEJTIwQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fOGJpdCUzRFRydWUpJTBBJTBBbW9kZWxfOGJpdCUyMCUzRCUyMEZsdXhUcmFuc2Zvcm1lcjJETW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmJsYWNrLWZvcmVzdC1sYWJzJTJGRkxVWC4xLWRldiUyMiUyQyUyMCUwQSUyMCUyMCUyMCUyMHN1YmZvbGRlciUzRCUyMnRyYW5zZm9ybWVyJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWclMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MzIlMEEpJTBBbW9kZWxfOGJpdC50cmFuc2Zvcm1lcl9ibG9ja3MubGF5ZXJzJTVCLTElNUQubm9ybTIud2VpZ2h0LmR0eXBl",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> FluxTransformer2DModel, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_8bit=<span class="hljs-literal">True</span>)

model_8bit = FluxTransformer2DModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>, 
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quantization_config,
    torch_dtype=torch.float32
)
model_8bit.transformer_blocks.layers[-<span class="hljs-number">1</span>].norm2.weight.dtype`,wrap:!1}}),m=new z({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEZsdXhUcmFuc2Zvcm1lcjJETW9kZWwlMkMlMjBCaXRzQW5kQnl0ZXNDb25maWclMEElMEFxdWFudGl6YXRpb25fY29uZmlnJTIwJTNEJTIwQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fOGJpdCUzRFRydWUpJTBBJTBBbW9kZWxfOGJpdCUyMCUzRCUyMEZsdXhUcmFuc2Zvcm1lcjJETW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmJsYWNrLWZvcmVzdC1sYWJzJTJGRkxVWC4xLWRldiUyMiUyQyUyMCUwQSUyMCUyMCUyMCUyMHN1YmZvbGRlciUzRCUyMnRyYW5zZm9ybWVyJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWclMEEp",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> FluxTransformer2DModel, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_8bit=<span class="hljs-literal">True</span>)

model_8bit = FluxTransformer2DModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>, 
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quantization_config
)`,wrap:!1}}),{c(){l=J("p"),l.textContent=$,r=i(),d(p.$$.fragment),f=i(),h=J("p"),h.innerHTML=_,R=i(),d(w.$$.fragment),W=i(),C=J("p"),C.innerHTML=k,g=i(),d(m.$$.fragment)},l(o){l=T(o,"P",{"data-svelte-h":!0}),U(l)!=="svelte-4djpqq"&&(l.textContent=$),r=a(o),u(p.$$.fragment,o),f=a(o),h=T(o,"P",{"data-svelte-h":!0}),U(h)!=="svelte-kfzkum"&&(h.innerHTML=_),R=a(o),u(w.$$.fragment,o),W=a(o),C=T(o,"P",{"data-svelte-h":!0}),U(C)!=="svelte-s1v45k"&&(C.innerHTML=k),g=a(o),u(m.$$.fragment,o)},m(o,v){n(o,l,v),n(o,r,v),y(p,o,v),n(o,f,v),n(o,h,v),n(o,R,v),y(w,o,v),n(o,W,v),n(o,C,v),n(o,g,v),y(m,o,v),Z=!0},p:be,i(o){Z||(c(p.$$.fragment,o),c(w.$$.fragment,o),c(m.$$.fragment,o),Z=!0)},o(o){b(p.$$.fragment,o),b(w.$$.fragment,o),b(m.$$.fragment,o),Z=!1},d(o){o&&(s(l),s(r),s(f),s(h),s(R),s(W),s(C),s(g)),M(p,o),M(w,o),M(m,o)}}}function _t(B){let l,$="Quantizing a model in 4-bit reduces your memory-usage by 4x:",r,p,f,h,_="By default, all the other modules such as <code>torch.nn.LayerNorm</code> are converted to <code>torch.float16</code>. You can change the data type of these modules with the <code>torch_dtype</code> parameter if you want:",R,w,W,C,k='Call <a href="/docs/diffusers/main/en/api/pipelines/overview#diffusers.utils.PushToHubMixin.push_to_hub">push_to_hub()</a> after loading it in 4-bit precision. You can also save the serialized 4-bit models locally with <a href="/docs/diffusers/main/en/api/models/overview#diffusers.ModelMixin.save_pretrained">save_pretrained()</a>.',g;return p=new z({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEZsdXhUcmFuc2Zvcm1lcjJETW9kZWwlMkMlMjBCaXRzQW5kQnl0ZXNDb25maWclMEElMEFxdWFudGl6YXRpb25fY29uZmlnJTIwJTNEJTIwQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fNGJpdCUzRFRydWUpJTBBJTBBbW9kZWxfNGJpdCUyMCUzRCUyMEZsdXhUcmFuc2Zvcm1lcjJETW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmJsYWNrLWZvcmVzdC1sYWJzJTJGRkxVWC4xLWRldiUyMiUyQyUyMCUwQSUyMCUyMCUyMCUyMHN1YmZvbGRlciUzRCUyMnRyYW5zZm9ybWVyJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWclMEEp",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> FluxTransformer2DModel, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>)

model_4bit = FluxTransformer2DModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>, 
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quantization_config
)`,wrap:!1}}),w=new z({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEZsdXhUcmFuc2Zvcm1lcjJETW9kZWwlMkMlMjBCaXRzQW5kQnl0ZXNDb25maWclMEElMEFxdWFudGl6YXRpb25fY29uZmlnJTIwJTNEJTIwQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fNGJpdCUzRFRydWUpJTBBJTBBbW9kZWxfNGJpdCUyMCUzRCUyMEZsdXhUcmFuc2Zvcm1lcjJETW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmJsYWNrLWZvcmVzdC1sYWJzJTJGRkxVWC4xLWRldiUyMiUyQyUyMCUwQSUyMCUyMCUyMCUyMHN1YmZvbGRlciUzRCUyMnRyYW5zZm9ybWVyJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWclMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MzIlMEEpJTBBbW9kZWxfNGJpdC50cmFuc2Zvcm1lcl9ibG9ja3MubGF5ZXJzJTVCLTElNUQubm9ybTIud2VpZ2h0LmR0eXBl",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> FluxTransformer2DModel, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>)

model_4bit = FluxTransformer2DModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>, 
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quantization_config,
    torch_dtype=torch.float32
)
model_4bit.transformer_blocks.layers[-<span class="hljs-number">1</span>].norm2.weight.dtype`,wrap:!1}}),{c(){l=J("p"),l.textContent=$,r=i(),d(p.$$.fragment),f=i(),h=J("p"),h.innerHTML=_,R=i(),d(w.$$.fragment),W=i(),C=J("p"),C.innerHTML=k},l(m){l=T(m,"P",{"data-svelte-h":!0}),U(l)!=="svelte-i07c71"&&(l.textContent=$),r=a(m),u(p.$$.fragment,m),f=a(m),h=T(m,"P",{"data-svelte-h":!0}),U(h)!=="svelte-kfzkum"&&(h.innerHTML=_),R=a(m),u(w.$$.fragment,m),W=a(m),C=T(m,"P",{"data-svelte-h":!0}),U(C)!=="svelte-1feojqt"&&(C.innerHTML=k)},m(m,Z){n(m,l,Z),n(m,r,Z),y(p,m,Z),n(m,f,Z),n(m,h,Z),n(m,R,Z),y(w,m,Z),n(m,W,Z),n(m,C,Z),g=!0},p:be,i(m){g||(c(p.$$.fragment,m),c(w.$$.fragment,m),g=!0)},o(m){b(p.$$.fragment,m),b(w.$$.fragment,m),g=!1},d(m){m&&(s(l),s(r),s(f),s(h),s(R),s(W),s(C)),M(p,m),M(w,m)}}}function vt(B){let l,$,r,p;return l=new ht({props:{id:"bnb",option:"8-bit",$$slots:{default:[gt]},$$scope:{ctx:B}}}),r=new ht({props:{id:"bnb",option:"4-bit",$$slots:{default:[_t]},$$scope:{ctx:B}}}),{c(){d(l.$$.fragment),$=i(),d(r.$$.fragment)},l(f){u(l.$$.fragment,f),$=a(f),u(r.$$.fragment,f)},m(f,h){y(l,f,h),n(f,$,h),y(r,f,h),p=!0},p(f,h){const _={};h&2&&(_.$$scope={dirty:h,ctx:f}),l.$set(_);const R={};h&2&&(R.$$scope={dirty:h,ctx:f}),r.$set(R)},i(f){p||(c(l.$$.fragment,f),c(r.$$.fragment,f),p=!0)},o(f){b(l.$$.fragment,f),b(r.$$.fragment,f),p=!1},d(f){f&&s($),M(l,f),M(r,f)}}}function Rt(B){let l,$="Training with 8-bit and 4-bit weights are only supported for training <em>extra</em> parameters.";return{c(){l=J("p"),l.innerHTML=$},l(r){l=T(r,"P",{"data-svelte-h":!0}),U(l)!=="svelte-of9sym"&&(l.innerHTML=$)},m(r,p){n(r,l,p)},p:be,d(r){r&&s(l)}}}function Bt(B){let l,$='Learn more about the details of 8-bit quantization in this <a href="https://huggingface.co/blog/hf-bitsandbytes-integration" rel="nofollow">blog post</a>!';return{c(){l=J("p"),l.innerHTML=$},l(r){l=T(r,"P",{"data-svelte-h":!0}),U(l)!=="svelte-1bb05fp"&&(l.innerHTML=$)},m(r,p){n(r,l,p)},p:be,d(r){r&&s(l)}}}function zt(B){let l,$='Learn more about its details in this <a href="https://huggingface.co/blog/4bit-transformers-bitsandbytes" rel="nofollow">blog post</a>.';return{c(){l=J("p"),l.innerHTML=$},l(r){l=T(r,"P",{"data-svelte-h":!0}),U(l)!=="svelte-kpdzjq"&&(l.innerHTML=$)},m(r,p){n(r,l,p)},p:be,d(r){r&&s(l)}}}function kt(B){let l,$,r,p,f,h,_,R='<a href="https://huggingface.co/docs/bitsandbytes/index" rel="nofollow">bitsandbytes</a> is the easiest option for quantizing a model to 8 and 4-bit. 8-bit quantization multiplies outliers in fp16 with non-outliers in int8, converts the non-outlier values back to fp16, and then adds them together to return the weights in fp16. This reduces the degradative effect outlier values have on a model’s performance.',w,W,C='4-bit quantization compresses a model even further, and it is commonly used with <a href="https://hf.co/papers/2305.14314" rel="nofollow">QLoRA</a> to finetune quantized LLMs.',k,g,m="To use bitsandbytes, make sure you have the following libraries installed:",Z,o,v,X,Oe='Now you can quantize a model by passing a <a href="/docs/diffusers/main/en/api/quantization#diffusers.BitsAndBytesConfig">BitsAndBytesConfig</a> to <a href="/docs/diffusers/main/en/api/models/overview#diffusers.ModelMixin.from_pretrained">from_pretrained()</a>. This works for any model in any modality, as long as it supports loading with <a href="https://hf.co/docs/accelerate/index" rel="nofollow">Accelerate</a> and contains <code>torch.nn.Linear</code> layers.',Me,F,he,G,Je,Q,et="Check your memory footprint with the <code>get_memory_footprint</code> method:",Te,E,Ue,x,tt='Quantized models can be loaded from the <a href="/docs/diffusers/main/en/api/models/overview#diffusers.ModelMixin.from_pretrained">from_pretrained()</a> method without needing to specify the <code>quantization_config</code> parameters:',$e,I,we,N,Ze,q,Ce,Y,st="This section explores some of the specific features of 8-bit models, such as outlier thresholds and skipping module conversion.",We,H,ge,L,nt="An “outlier” is a hidden state value greater than a certain threshold, and these values are computed in fp16. While the values are usually normally distributed ([-3.5, 3.5]), this distribution can be very different for large models ([-60, 6] or [6, 60]). 8-bit quantization works well for values ~5, but beyond that, there is a significant performance penalty. A good default threshold value is 6, but a lower threshold may be needed for more unstable models (small models or finetuning).",_e,A,lt='To find the best threshold for your model, we recommend experimenting with the <code>llm_int8_threshold</code> parameter in <a href="/docs/diffusers/main/en/api/quantization#diffusers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',ve,S,Re,D,Be,P,it='For some models, you don’t need to quantize every module to 8-bit which can actually cause instability. For example, for diffusion models like <a href="../api/pipelines/stable_diffusion/stable_diffusion_3">Stable Diffusion 3</a>, the <code>proj_out</code> module can be skipped using the <code>llm_int8_skip_modules</code> parameter in <a href="/docs/diffusers/main/en/api/quantization#diffusers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',ze,K,ke,O,je,V,Fe,ee,at="This section explores some of the specific features of 4-bit models, such as changing the compute data type, using the Normal Float 4 (NF4) data type, and using nested quantization.",Ge,te,qe,se,ot='To speedup computation, you can change the data type from float32 (the default value) to bf16 using the <code>bnb_4bit_compute_dtype</code> parameter in <a href="/docs/diffusers/main/en/api/quantization#diffusers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',Ve,ne,Xe,le,Qe,ie,rt='NF4 is a 4-bit data type from the <a href="https://hf.co/papers/2305.14314" rel="nofollow">QLoRA</a> paper, adapted for weights initialized from a normal distribution. You should use NF4 for training 4-bit base models. This can be configured with the <code>bnb_4bit_quant_type</code> parameter in the <a href="/docs/diffusers/main/en/api/quantization#diffusers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',Ee,ae,xe,oe,mt="For inference, the <code>bnb_4bit_quant_type</code> does not have a huge impact on performance. However, to remain consistent with the model weights, you should use the <code>bnb_4bit_compute_dtype</code> and <code>torch_dtype</code> values.",Ie,re,Ne,me,ft="Nested quantization is a technique that can save additional memory at no additional performance cost. This feature performs a second quantization of the already quantized weights to save an additional 0.4 bits/parameter.",Ye,fe,He,pe,Le,de,pt="Once quantized, you can dequantize the model to the original precision but this might result in a small quality loss of the model. Make sure you have enough GPU RAM to fit the dequantized model.",Ae,ue,Se,ye,De,ce,Pe;return f=new j({props:{title:"bitsandbytes",local:"bitsandbytes",headingTag:"h1"}}),o=new z({props:{code:"cGlwJTIwaW5zdGFsbCUyMGRpZmZ1c2VycyUyMHRyYW5zZm9ybWVycyUyMGFjY2VsZXJhdGUlMjBiaXRzYW5kYnl0ZXMlMjAtVQ==",highlighted:"pip install diffusers transformers accelerate bitsandbytes -U",wrap:!1}}),F=new Wt({props:{id:"bnb",options:["8-bit","4-bit"],$$slots:{default:[vt]},$$scope:{ctx:B}}}),G=new Ke({props:{warning:!0,$$slots:{default:[Rt]},$$scope:{ctx:B}}}),E=new z({props:{code:"cHJpbnQobW9kZWwuZ2V0X21lbW9yeV9mb290cHJpbnQoKSk=",highlighted:'<span class="hljs-built_in">print</span>(model.get_memory_footprint())',wrap:!1}}),I=new z({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEZsdXhUcmFuc2Zvcm1lcjJETW9kZWwlMkMlMjBCaXRzQW5kQnl0ZXNDb25maWclMEElMEFxdWFudGl6YXRpb25fY29uZmlnJTIwJTNEJTIwQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fNGJpdCUzRFRydWUpJTBBJTBBbW9kZWxfNGJpdCUyMCUzRCUyMEZsdXhUcmFuc2Zvcm1lcjJETW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMnNheWFrcGF1bCUyRmZsdXguMS1kZXYtbmY0LXBrZyUyMiUyQyUyMHN1YmZvbGRlciUzRCUyMnRyYW5zZm9ybWVyJTIyJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> FluxTransformer2DModel, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>)

model_4bit = FluxTransformer2DModel.from_pretrained(
    <span class="hljs-string">&quot;sayakpaul/flux.1-dev-nf4-pkg&quot;</span>, subfolder=<span class="hljs-string">&quot;transformer&quot;</span>
)`,wrap:!1}}),N=new j({props:{title:"8-bit (LLM.int8() algorithm)",local:"8-bit-llmint8-algorithm",headingTag:"h2"}}),q=new Ke({props:{$$slots:{default:[Bt]},$$scope:{ctx:B}}}),H=new j({props:{title:"Outlier threshold",local:"outlier-threshold",headingTag:"h3"}}),S=new z({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEZsdXhUcmFuc2Zvcm1lcjJETW9kZWwlMkMlMjBCaXRzQW5kQnl0ZXNDb25maWclMEElMEFxdWFudGl6YXRpb25fY29uZmlnJTIwJTNEJTIwQml0c0FuZEJ5dGVzQ29uZmlnKCUwQSUyMCUyMCUyMCUyMGxvYWRfaW5fOGJpdCUzRFRydWUlMkMlMjBsbG1faW50OF90aHJlc2hvbGQlM0QxMCUyQyUwQSklMEElMEFtb2RlbF84Yml0JTIwJTNEJTIwRmx1eFRyYW5zZm9ybWVyMkRNb2RlbC5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyYmxhY2stZm9yZXN0LWxhYnMlMkZGTFVYLjEtZGV2JTIyJTJDJTBBJTIwJTIwJTIwJTIwc3ViZm9sZGVyJTNEJTIydHJhbnNmb3JtZXIlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRpemF0aW9uX2NvbmZpZyUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> FluxTransformer2DModel, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(
    load_in_8bit=<span class="hljs-literal">True</span>, llm_int8_threshold=<span class="hljs-number">10</span>,
)

model_8bit = FluxTransformer2DModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quantization_config,
)`,wrap:!1}}),D=new j({props:{title:"Skip module conversion",local:"skip-module-conversion",headingTag:"h3"}}),K=new z({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMFNEM1RyYW5zZm9ybWVyMkRNb2RlbCUyQyUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl84Yml0JTNEVHJ1ZSUyQyUyMGxsbV9pbnQ4X3NraXBfbW9kdWxlcyUzRCU1QiUyMnByb2pfb3V0JTIyJTVEJTJDJTBBKSUwQSUwQW1vZGVsXzhiaXQlMjAlM0QlMjBTRDNUcmFuc2Zvcm1lcjJETW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMnN0YWJpbGl0eWFpJTJGc3RhYmxlLWRpZmZ1c2lvbi0zLW1lZGl1bS1kaWZmdXNlcnMlMjIlMkMlMEElMjAlMjAlMjAlMjBzdWJmb2xkZXIlM0QlMjJ0cmFuc2Zvcm1lciUyMiUyQyUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudGl6YXRpb25fY29uZmlnJTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> SD3Transformer2DModel, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(
    load_in_8bit=<span class="hljs-literal">True</span>, llm_int8_skip_modules=[<span class="hljs-string">&quot;proj_out&quot;</span>],
)

model_8bit = SD3Transformer2DModel.from_pretrained(
    <span class="hljs-string">&quot;stabilityai/stable-diffusion-3-medium-diffusers&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quantization_config,
)`,wrap:!1}}),O=new j({props:{title:"4-bit (QLoRA algorithm)",local:"4-bit-qlora-algorithm",headingTag:"h2"}}),V=new Ke({props:{$$slots:{default:[zt]},$$scope:{ctx:B}}}),te=new j({props:{title:"Compute data type",local:"compute-data-type",headingTag:"h3"}}),ne=new z({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyhsb2FkX2luXzRiaXQlM0RUcnVlJTJDJTIwYm5iXzRiaXRfY29tcHV0ZV9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2KQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>, bnb_4bit_compute_dtype=torch.bfloat16)`,wrap:!1}}),le=new j({props:{title:"Normal Float 4 (NF4)",local:"normal-float-4-nf4",headingTag:"h3"}}),ae=new z({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQW5mNF9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl80Yml0JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3F1YW50X3R5cGUlM0QlMjJuZjQlMjIlMkMlMEEpJTBBJTBBbW9kZWxfbmY0JTIwJTNEJTIwU0QzVHJhbnNmb3JtZXIyRE1vZGVsLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJzdGFiaWxpdHlhaSUyRnN0YWJsZS1kaWZmdXNpb24tMy1tZWRpdW0tZGlmZnVzZXJzJTIyJTJDJTBBJTIwJTIwJTIwJTIwc3ViZm9sZGVyJTNEJTIydHJhbnNmb3JtZXIlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEbmY0X2NvbmZpZyUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> BitsAndBytesConfig

nf4_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_quant_type=<span class="hljs-string">&quot;nf4&quot;</span>,
)

model_nf4 = SD3Transformer2DModel.from_pretrained(
    <span class="hljs-string">&quot;stabilityai/stable-diffusion-3-medium-diffusers&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=nf4_config,
)`,wrap:!1}}),re=new j({props:{title:"Nested quantization",local:"nested-quantization",headingTag:"h3"}}),fe=new z({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQWRvdWJsZV9xdWFudF9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl80Yml0JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3VzZV9kb3VibGVfcXVhbnQlM0RUcnVlJTJDJTBBKSUwQSUwQWRvdWJsZV9xdWFudF9tb2RlbCUyMCUzRCUyMFNEM1RyYW5zZm9ybWVyMkRNb2RlbC5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyc3RhYmlsaXR5YWklMkZzdGFibGUtZGlmZnVzaW9uLTMtbWVkaXVtLWRpZmZ1c2VycyUyMiUyQyUwQSUyMCUyMCUyMCUyMHN1YmZvbGRlciUzRCUyMnRyYW5zZm9ybWVyJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRGRvdWJsZV9xdWFudF9jb25maWclMkMlMEEp",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> BitsAndBytesConfig

double_quant_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_use_double_quant=<span class="hljs-literal">True</span>,
)

double_quant_model = SD3Transformer2DModel.from_pretrained(
    <span class="hljs-string">&quot;stabilityai/stable-diffusion-3-medium-diffusers&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=double_quant_config,
)`,wrap:!1}}),pe=new j({props:{title:"Dequantizing bitsandbytes models",local:"dequantizing-bitsandbytes-models",headingTag:"h2"}}),ue=new z({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQWRvdWJsZV9xdWFudF9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl80Yml0JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3VzZV9kb3VibGVfcXVhbnQlM0RUcnVlJTJDJTBBKSUwQSUwQWRvdWJsZV9xdWFudF9tb2RlbCUyMCUzRCUyMFNEM1RyYW5zZm9ybWVyMkRNb2RlbC5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyc3RhYmlsaXR5YWklMkZzdGFibGUtZGlmZnVzaW9uLTMtbWVkaXVtLWRpZmZ1c2VycyUyMiUyQyUwQSUyMCUyMCUyMCUyMHN1YmZvbGRlciUzRCUyMnRyYW5zZm9ybWVyJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRGRvdWJsZV9xdWFudF9jb25maWclMkMlMEEpJTBBbW9kZWwuZGVxdWFudGl6ZSgp",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> BitsAndBytesConfig

double_quant_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_use_double_quant=<span class="hljs-literal">True</span>,
)

double_quant_model = SD3Transformer2DModel.from_pretrained(
    <span class="hljs-string">&quot;stabilityai/stable-diffusion-3-medium-diffusers&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=double_quant_config,
)
model.dequantize()`,wrap:!1}}),ye=new Ct({props:{source:"https://github.com/huggingface/diffusers/blob/main/docs/source/en/quantization/bitsandbytes.md"}}),{c(){l=J("meta"),$=i(),r=J("p"),p=i(),d(f.$$.fragment),h=i(),_=J("p"),_.innerHTML=R,w=i(),W=J("p"),W.innerHTML=C,k=i(),g=J("p"),g.textContent=m,Z=i(),d(o.$$.fragment),v=i(),X=J("p"),X.innerHTML=Oe,Me=i(),d(F.$$.fragment),he=i(),d(G.$$.fragment),Je=i(),Q=J("p"),Q.innerHTML=et,Te=i(),d(E.$$.fragment),Ue=i(),x=J("p"),x.innerHTML=tt,$e=i(),d(I.$$.fragment),we=i(),d(N.$$.fragment),Ze=i(),d(q.$$.fragment),Ce=i(),Y=J("p"),Y.textContent=st,We=i(),d(H.$$.fragment),ge=i(),L=J("p"),L.textContent=nt,_e=i(),A=J("p"),A.innerHTML=lt,ve=i(),d(S.$$.fragment),Re=i(),d(D.$$.fragment),Be=i(),P=J("p"),P.innerHTML=it,ze=i(),d(K.$$.fragment),ke=i(),d(O.$$.fragment),je=i(),d(V.$$.fragment),Fe=i(),ee=J("p"),ee.textContent=at,Ge=i(),d(te.$$.fragment),qe=i(),se=J("p"),se.innerHTML=ot,Ve=i(),d(ne.$$.fragment),Xe=i(),d(le.$$.fragment),Qe=i(),ie=J("p"),ie.innerHTML=rt,Ee=i(),d(ae.$$.fragment),xe=i(),oe=J("p"),oe.innerHTML=mt,Ie=i(),d(re.$$.fragment),Ne=i(),me=J("p"),me.textContent=ft,Ye=i(),d(fe.$$.fragment),He=i(),d(pe.$$.fragment),Le=i(),de=J("p"),de.textContent=pt,Ae=i(),d(ue.$$.fragment),Se=i(),d(ye.$$.fragment),De=i(),ce=J("p"),this.h()},l(e){const t=wt("svelte-u9bgzb",document.head);l=T(t,"META",{name:!0,content:!0}),t.forEach(s),$=a(e),r=T(e,"P",{}),bt(r).forEach(s),p=a(e),u(f.$$.fragment,e),h=a(e),_=T(e,"P",{"data-svelte-h":!0}),U(_)!=="svelte-1m50ob7"&&(_.innerHTML=R),w=a(e),W=T(e,"P",{"data-svelte-h":!0}),U(W)!=="svelte-11pyf03"&&(W.innerHTML=C),k=a(e),g=T(e,"P",{"data-svelte-h":!0}),U(g)!=="svelte-gf36q7"&&(g.textContent=m),Z=a(e),u(o.$$.fragment,e),v=a(e),X=T(e,"P",{"data-svelte-h":!0}),U(X)!=="svelte-17fqaf"&&(X.innerHTML=Oe),Me=a(e),u(F.$$.fragment,e),he=a(e),u(G.$$.fragment,e),Je=a(e),Q=T(e,"P",{"data-svelte-h":!0}),U(Q)!=="svelte-139tok6"&&(Q.innerHTML=et),Te=a(e),u(E.$$.fragment,e),Ue=a(e),x=T(e,"P",{"data-svelte-h":!0}),U(x)!=="svelte-10wvd51"&&(x.innerHTML=tt),$e=a(e),u(I.$$.fragment,e),we=a(e),u(N.$$.fragment,e),Ze=a(e),u(q.$$.fragment,e),Ce=a(e),Y=T(e,"P",{"data-svelte-h":!0}),U(Y)!=="svelte-1myadau"&&(Y.textContent=st),We=a(e),u(H.$$.fragment,e),ge=a(e),L=T(e,"P",{"data-svelte-h":!0}),U(L)!=="svelte-ur5rgd"&&(L.textContent=nt),_e=a(e),A=T(e,"P",{"data-svelte-h":!0}),U(A)!=="svelte-1pta66b"&&(A.innerHTML=lt),ve=a(e),u(S.$$.fragment,e),Re=a(e),u(D.$$.fragment,e),Be=a(e),P=T(e,"P",{"data-svelte-h":!0}),U(P)!=="svelte-1rff4ou"&&(P.innerHTML=it),ze=a(e),u(K.$$.fragment,e),ke=a(e),u(O.$$.fragment,e),je=a(e),u(V.$$.fragment,e),Fe=a(e),ee=T(e,"P",{"data-svelte-h":!0}),U(ee)!=="svelte-7ob7j"&&(ee.textContent=at),Ge=a(e),u(te.$$.fragment,e),qe=a(e),se=T(e,"P",{"data-svelte-h":!0}),U(se)!=="svelte-19knmj2"&&(se.innerHTML=ot),Ve=a(e),u(ne.$$.fragment,e),Xe=a(e),u(le.$$.fragment,e),Qe=a(e),ie=T(e,"P",{"data-svelte-h":!0}),U(ie)!=="svelte-6pxdrm"&&(ie.innerHTML=rt),Ee=a(e),u(ae.$$.fragment,e),xe=a(e),oe=T(e,"P",{"data-svelte-h":!0}),U(oe)!=="svelte-1qoc2ct"&&(oe.innerHTML=mt),Ie=a(e),u(re.$$.fragment,e),Ne=a(e),me=T(e,"P",{"data-svelte-h":!0}),U(me)!=="svelte-ep1hhf"&&(me.textContent=ft),Ye=a(e),u(fe.$$.fragment,e),He=a(e),u(pe.$$.fragment,e),Le=a(e),de=T(e,"P",{"data-svelte-h":!0}),U(de)!=="svelte-1gygypd"&&(de.textContent=pt),Ae=a(e),u(ue.$$.fragment,e),Se=a(e),u(ye.$$.fragment,e),De=a(e),ce=T(e,"P",{}),bt(ce).forEach(s),this.h()},h(){Mt(l,"name","hf:doc:metadata"),Mt(l,"content",jt)},m(e,t){Zt(document.head,l),n(e,$,t),n(e,r,t),n(e,p,t),y(f,e,t),n(e,h,t),n(e,_,t),n(e,w,t),n(e,W,t),n(e,k,t),n(e,g,t),n(e,Z,t),y(o,e,t),n(e,v,t),n(e,X,t),n(e,Me,t),y(F,e,t),n(e,he,t),y(G,e,t),n(e,Je,t),n(e,Q,t),n(e,Te,t),y(E,e,t),n(e,Ue,t),n(e,x,t),n(e,$e,t),y(I,e,t),n(e,we,t),y(N,e,t),n(e,Ze,t),y(q,e,t),n(e,Ce,t),n(e,Y,t),n(e,We,t),y(H,e,t),n(e,ge,t),n(e,L,t),n(e,_e,t),n(e,A,t),n(e,ve,t),y(S,e,t),n(e,Re,t),y(D,e,t),n(e,Be,t),n(e,P,t),n(e,ze,t),y(K,e,t),n(e,ke,t),y(O,e,t),n(e,je,t),y(V,e,t),n(e,Fe,t),n(e,ee,t),n(e,Ge,t),y(te,e,t),n(e,qe,t),n(e,se,t),n(e,Ve,t),y(ne,e,t),n(e,Xe,t),y(le,e,t),n(e,Qe,t),n(e,ie,t),n(e,Ee,t),y(ae,e,t),n(e,xe,t),n(e,oe,t),n(e,Ie,t),y(re,e,t),n(e,Ne,t),n(e,me,t),n(e,Ye,t),y(fe,e,t),n(e,He,t),y(pe,e,t),n(e,Le,t),n(e,de,t),n(e,Ae,t),y(ue,e,t),n(e,Se,t),y(ye,e,t),n(e,De,t),n(e,ce,t),Pe=!0},p(e,[t]){const dt={};t&2&&(dt.$$scope={dirty:t,ctx:e}),F.$set(dt);const ut={};t&2&&(ut.$$scope={dirty:t,ctx:e}),G.$set(ut);const yt={};t&2&&(yt.$$scope={dirty:t,ctx:e}),q.$set(yt);const ct={};t&2&&(ct.$$scope={dirty:t,ctx:e}),V.$set(ct)},i(e){Pe||(c(f.$$.fragment,e),c(o.$$.fragment,e),c(F.$$.fragment,e),c(G.$$.fragment,e),c(E.$$.fragment,e),c(I.$$.fragment,e),c(N.$$.fragment,e),c(q.$$.fragment,e),c(H.$$.fragment,e),c(S.$$.fragment,e),c(D.$$.fragment,e),c(K.$$.fragment,e),c(O.$$.fragment,e),c(V.$$.fragment,e),c(te.$$.fragment,e),c(ne.$$.fragment,e),c(le.$$.fragment,e),c(ae.$$.fragment,e),c(re.$$.fragment,e),c(fe.$$.fragment,e),c(pe.$$.fragment,e),c(ue.$$.fragment,e),c(ye.$$.fragment,e),Pe=!0)},o(e){b(f.$$.fragment,e),b(o.$$.fragment,e),b(F.$$.fragment,e),b(G.$$.fragment,e),b(E.$$.fragment,e),b(I.$$.fragment,e),b(N.$$.fragment,e),b(q.$$.fragment,e),b(H.$$.fragment,e),b(S.$$.fragment,e),b(D.$$.fragment,e),b(K.$$.fragment,e),b(O.$$.fragment,e),b(V.$$.fragment,e),b(te.$$.fragment,e),b(ne.$$.fragment,e),b(le.$$.fragment,e),b(ae.$$.fragment,e),b(re.$$.fragment,e),b(fe.$$.fragment,e),b(pe.$$.fragment,e),b(ue.$$.fragment,e),b(ye.$$.fragment,e),Pe=!1},d(e){e&&(s($),s(r),s(p),s(h),s(_),s(w),s(W),s(k),s(g),s(Z),s(v),s(X),s(Me),s(he),s(Je),s(Q),s(Te),s(Ue),s(x),s($e),s(we),s(Ze),s(Ce),s(Y),s(We),s(ge),s(L),s(_e),s(A),s(ve),s(Re),s(Be),s(P),s(ze),s(ke),s(je),s(Fe),s(ee),s(Ge),s(qe),s(se),s(Ve),s(Xe),s(Qe),s(ie),s(Ee),s(xe),s(oe),s(Ie),s(Ne),s(me),s(Ye),s(He),s(Le),s(de),s(Ae),s(Se),s(De),s(ce)),s(l),M(f,e),M(o,e),M(F,e),M(G,e),M(E,e),M(I,e),M(N,e),M(q,e),M(H,e),M(S,e),M(D,e),M(K,e),M(O,e),M(V,e),M(te,e),M(ne,e),M(le,e),M(ae,e),M(re,e),M(fe,e),M(pe,e),M(ue,e),M(ye,e)}}}const jt='{"title":"bitsandbytes","local":"bitsandbytes","sections":[{"title":"8-bit (LLM.int8() algorithm)","local":"8-bit-llmint8-algorithm","sections":[{"title":"Outlier threshold","local":"outlier-threshold","sections":[],"depth":3},{"title":"Skip module conversion","local":"skip-module-conversion","sections":[],"depth":3}],"depth":2},{"title":"4-bit (QLoRA algorithm)","local":"4-bit-qlora-algorithm","sections":[{"title":"Compute data type","local":"compute-data-type","sections":[],"depth":3},{"title":"Normal Float 4 (NF4)","local":"normal-float-4-nf4","sections":[],"depth":3},{"title":"Nested quantization","local":"nested-quantization","sections":[],"depth":3}],"depth":2},{"title":"Dequantizing bitsandbytes models","local":"dequantizing-bitsandbytes-models","sections":[],"depth":2}],"depth":1}';function Ft(B){return Tt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class xt extends Ut{constructor(l){super(),$t(this,l,Ft,kt,Jt,{})}}export{xt as component};
