import{s as qt,o as It,n as Q}from"../chunks/scheduler.25b97de1.js";import{S as Ht,i as Lt,g,s as o,r as b,A as xt,h as J,f as l,c as r,j as Vt,u as c,x as w,k as Et,y as Yt,a as n,v as y,d as h,t as M,w as $}from"../chunks/index.d9030fc9.js";import{T as ct}from"../chunks/Tip.baa67368.js";import{C as W}from"../chunks/CodeBlock.e6cd0d95.js";import{H as R,E as St}from"../chunks/EditOnGithub.91d95064.js";import{H as At,a as yt}from"../chunks/HfOption.1e589c90.js";function Pt(U){let s,f;return s=new W({props:{code:"cGlwJTIwaW5zdGFsbCUyMHRyYW5zZm9ybWVycyUyMGFjY2VsZXJhdGUlMjBiaXRzYW5kYnl0ZXMlM0UwLjM3LjA=",highlighted:"pip install transformers accelerate bitsandbytes&gt;0.37.0",wrap:!1}}),{c(){b(s.$$.fragment)},l(a){c(s.$$.fragment,a)},m(a,m){y(s,a,m),f=!0},p:Q,i(a){f||(h(s.$$.fragment,a),f=!0)},o(a){M(s.$$.fragment,a),f=!1},d(a){$(s,a)}}}function Dt(U){let s,f;return s=new W({props:{code:"cGlwJTIwaW5zdGFsbCUyMGJpdHNhbmRieXRlcyUzRSUzRDAuMzkuMCUwQXBpcCUyMGluc3RhbGwlMjAtLXVwZ3JhZGUlMjBhY2NlbGVyYXRlJTIwdHJhbnNmb3JtZXJz",highlighted:`pip install bitsandbytes&gt;=0.39.0
pip install --upgrade accelerate transformers`,wrap:!1}}),{c(){b(s.$$.fragment)},l(a){c(s.$$.fragment,a)},m(a,m){y(s,a,m),f=!0},p:Q,i(a){f||(h(s.$$.fragment,a),f=!0)},o(a){M(s.$$.fragment,a),f=!1},d(a){$(s,a)}}}function Kt(U){let s,f,a,m;return s=new yt({props:{id:"bnb",option:"8-bit",$$slots:{default:[Pt]},$$scope:{ctx:U}}}),a=new yt({props:{id:"bnb",option:"4-bit",$$slots:{default:[Dt]},$$scope:{ctx:U}}}),{c(){b(s.$$.fragment),f=o(),b(a.$$.fragment)},l(i){c(s.$$.fragment,i),f=r(i),c(a.$$.fragment,i)},m(i,p){y(s,i,p),n(i,f,p),y(a,i,p),m=!0},p(i,p){const T={};p&2&&(T.$$scope={dirty:p,ctx:i}),s.$set(T);const Z={};p&2&&(Z.$$scope={dirty:p,ctx:i}),a.$set(Z)},i(i){m||(h(s.$$.fragment,i),h(a.$$.fragment,i),m=!0)},o(i){M(s.$$.fragment,i),M(a.$$.fragment,i),m=!1},d(i){i&&l(f),$(s,i),$(a,i)}}}function Ot(U){let s,f='bitsandbytes is being refactored to support multiple backends beyond CUDA. Currently, ROCm (AMD GPU) and Intel CPU implementations are mature, with Intel XPU in progress and Apple Silicon support expected by Q4/Q1. For installation instructions and the latest backend updates, visit <a href="https://huggingface.co/docs/bitsandbytes/main/en/installation#multi-backend" rel="nofollow">this link</a>.',a,m,i='We value your feedback to help identify bugs before the full release! Check out <a href="https://huggingface.co/docs/bitsandbytes/main/en/non_cuda_backends" rel="nofollow">these docs</a> for more details and feedback links.';return{c(){s=g("p"),s.innerHTML=f,a=o(),m=g("p"),m.innerHTML=i},l(p){s=J(p,"P",{"data-svelte-h":!0}),w(s)!=="svelte-nz63kd"&&(s.innerHTML=f),a=r(p),m=J(p,"P",{"data-svelte-h":!0}),w(m)!=="svelte-aze40l"&&(m.innerHTML=i)},m(p,T){n(p,s,T),n(p,a,T),n(p,m,T)},p:Q,d(p){p&&(l(s),l(a),l(m))}}}function el(U){let s,f="Quantizing a model in 8-bit halves the memory-usage, and for large models, set <code>device_map=&quot;auto&quot;</code> to efficiently use the GPUs available:",a,m,i,p,T="By default, all the other modules such as <code>torch.nn.LayerNorm</code> are converted to <code>torch.float16</code>. You can change the data type of these modules with the <code>torch_dtype</code> parameter if you want:",Z,v,k,j,z='Once a model is quantized to 8-bit, you can’t push the quantized weights to the Hub unless you’re using the latest version of Transformers and bitsandbytes. If you have the latest versions, then you can push the 8-bit model to the Hub with the <a href="/docs/transformers/main/en/main_classes/model#transformers.utils.PushToHubMixin.push_to_hub">push_to_hub()</a> method. The quantization config.json file is pushed first, followed by the quantized model weights.',B,d,_;return m=new W({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyhsb2FkX2luXzhiaXQlM0RUcnVlKSUwQSUwQW1vZGVsXzhiaXQlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyYmlnc2NpZW5jZSUyRmJsb29tLTFiNyUyMiUyQyUyMCUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudGl6YXRpb25fY29uZmlnJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_8bit=<span class="hljs-literal">True</span>)

model_8bit = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>, 
    quantization_config=quantization_config
)`,wrap:!1}}),v=new W({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMkMlMjBCaXRzQW5kQnl0ZXNDb25maWclMEElMEFxdWFudGl6YXRpb25fY29uZmlnJTIwJTNEJTIwQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fOGJpdCUzRFRydWUpJTBBJTBBbW9kZWxfOGJpdCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJmYWNlYm9vayUyRm9wdC0zNTBtJTIyJTJDJTIwJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWclMkMlMjAlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MzIlMEEpJTBBbW9kZWxfOGJpdC5tb2RlbC5kZWNvZGVyLmxheWVycyU1Qi0xJTVELmZpbmFsX2xheWVyX25vcm0ud2VpZ2h0LmR0eXBl",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_8bit=<span class="hljs-literal">True</span>)

model_8bit = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;facebook/opt-350m&quot;</span>, 
    quantization_config=quantization_config, 
    torch_dtype=torch.float32
)
model_8bit.model.decoder.layers[-<span class="hljs-number">1</span>].final_layer_norm.weight.dtype`,wrap:!1}}),d=new W({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUyQyUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcobG9hZF9pbl84Yml0JTNEVHJ1ZSklMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJiaWdzY2llbmNlJTJGYmxvb20tNTYwbSUyMiUyQyUyMCUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudGl6YXRpb25fY29uZmlnJTBBKSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMmJpZ3NjaWVuY2UlMkZibG9vbS01NjBtJTIyKSUwQSUwQW1vZGVsLnB1c2hfdG9faHViKCUyMmJsb29tLTU2MG0tOGJpdCUyMik=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_8bit=<span class="hljs-literal">True</span>)

model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;bigscience/bloom-560m&quot;</span>, 
    quantization_config=quantization_config
)
tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bigscience/bloom-560m&quot;</span>)

model.push_to_hub(<span class="hljs-string">&quot;bloom-560m-8bit&quot;</span>)`,wrap:!1}}),{c(){s=g("p"),s.innerHTML=f,a=o(),b(m.$$.fragment),i=o(),p=g("p"),p.innerHTML=T,Z=o(),b(v.$$.fragment),k=o(),j=g("p"),j.innerHTML=z,B=o(),b(d.$$.fragment)},l(u){s=J(u,"P",{"data-svelte-h":!0}),w(s)!=="svelte-1dz8xeq"&&(s.innerHTML=f),a=r(u),c(m.$$.fragment,u),i=r(u),p=J(u,"P",{"data-svelte-h":!0}),w(p)!=="svelte-kfzkum"&&(p.innerHTML=T),Z=r(u),c(v.$$.fragment,u),k=r(u),j=J(u,"P",{"data-svelte-h":!0}),w(j)!=="svelte-951rn"&&(j.innerHTML=z),B=r(u),c(d.$$.fragment,u)},m(u,C){n(u,s,C),n(u,a,C),y(m,u,C),n(u,i,C),n(u,p,C),n(u,Z,C),y(v,u,C),n(u,k,C),n(u,j,C),n(u,B,C),y(d,u,C),_=!0},p:Q,i(u){_||(h(m.$$.fragment,u),h(v.$$.fragment,u),h(d.$$.fragment,u),_=!0)},o(u){M(m.$$.fragment,u),M(v.$$.fragment,u),M(d.$$.fragment,u),_=!1},d(u){u&&(l(s),l(a),l(i),l(p),l(Z),l(k),l(j),l(B)),$(m,u),$(v,u),$(d,u)}}}function tl(U){let s,f="Quantizing a model in 4-bit reduces your memory-usage by 4x, and for large models, set <code>device_map=&quot;auto&quot;</code> to efficiently use the GPUs available:",a,m,i,p,T="By default, all the other modules such as <code>torch.nn.LayerNorm</code> are converted to <code>torch.float16</code>. You can change the data type of these modules with the <code>torch_dtype</code> parameter if you want:",Z,v,k,j,z="If you have <code>bitsandbytes&gt;=0.41.3</code>, you can serialize 4-bit models and push them on Hugging Face Hub. Simply call <code>model.push_to_hub()</code> after loading it in 4-bit precision. You can also save the serialized 4-bit models locally with <code>model.save_pretrained()</code> command.",B;return m=new W({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyhsb2FkX2luXzRiaXQlM0RUcnVlKSUwQSUwQW1vZGVsXzRiaXQlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyYmlnc2NpZW5jZSUyRmJsb29tLTFiNyUyMiUyQyUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudGl6YXRpb25fY29uZmlnJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>)

model_4bit = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>,
    quantization_config=quantization_config
)`,wrap:!1}}),v=new W({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMkMlMjBCaXRzQW5kQnl0ZXNDb25maWclMEElMEFxdWFudGl6YXRpb25fY29uZmlnJTIwJTNEJTIwQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fNGJpdCUzRFRydWUpJTBBJTBBbW9kZWxfNGJpdCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJmYWNlYm9vayUyRm9wdC0zNTBtJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWclMkMlMjAlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MzIlMEEpJTBBbW9kZWxfNGJpdC5tb2RlbC5kZWNvZGVyLmxheWVycyU1Qi0xJTVELmZpbmFsX2xheWVyX25vcm0ud2VpZ2h0LmR0eXBl",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>)

model_4bit = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;facebook/opt-350m&quot;</span>,
    quantization_config=quantization_config, 
    torch_dtype=torch.float32
)
model_4bit.model.decoder.layers[-<span class="hljs-number">1</span>].final_layer_norm.weight.dtype`,wrap:!1}}),{c(){s=g("p"),s.innerHTML=f,a=o(),b(m.$$.fragment),i=o(),p=g("p"),p.innerHTML=T,Z=o(),b(v.$$.fragment),k=o(),j=g("p"),j.innerHTML=z},l(d){s=J(d,"P",{"data-svelte-h":!0}),w(s)!=="svelte-1b6jp1p"&&(s.innerHTML=f),a=r(d),c(m.$$.fragment,d),i=r(d),p=J(d,"P",{"data-svelte-h":!0}),w(p)!=="svelte-kfzkum"&&(p.innerHTML=T),Z=r(d),c(v.$$.fragment,d),k=r(d),j=J(d,"P",{"data-svelte-h":!0}),w(j)!=="svelte-1pmx2wb"&&(j.innerHTML=z)},m(d,_){n(d,s,_),n(d,a,_),y(m,d,_),n(d,i,_),n(d,p,_),n(d,Z,_),y(v,d,_),n(d,k,_),n(d,j,_),B=!0},p:Q,i(d){B||(h(m.$$.fragment,d),h(v.$$.fragment,d),B=!0)},o(d){M(m.$$.fragment,d),M(v.$$.fragment,d),B=!1},d(d){d&&(l(s),l(a),l(i),l(p),l(Z),l(k),l(j)),$(m,d),$(v,d)}}}function ll(U){let s,f,a,m;return s=new yt({props:{id:"bnb",option:"8-bit",$$slots:{default:[el]},$$scope:{ctx:U}}}),a=new yt({props:{id:"bnb",option:"4-bit",$$slots:{default:[tl]},$$scope:{ctx:U}}}),{c(){b(s.$$.fragment),f=o(),b(a.$$.fragment)},l(i){c(s.$$.fragment,i),f=r(i),c(a.$$.fragment,i)},m(i,p){y(s,i,p),n(i,f,p),y(a,i,p),m=!0},p(i,p){const T={};p&2&&(T.$$scope={dirty:p,ctx:i}),s.$set(T);const Z={};p&2&&(Z.$$scope={dirty:p,ctx:i}),a.$set(Z)},i(i){m||(h(s.$$.fragment,i),h(a.$$.fragment,i),m=!0)},o(i){M(s.$$.fragment,i),M(a.$$.fragment,i),m=!1},d(i){i&&l(f),$(s,i),$(a,i)}}}function nl(U){let s,f="Training with 8-bit and 4-bit weights are only supported for training <em>extra</em> parameters.";return{c(){s=g("p"),s.innerHTML=f},l(a){s=J(a,"P",{"data-svelte-h":!0}),w(s)!=="svelte-of9sym"&&(s.innerHTML=f)},m(a,m){n(a,s,m)},p:Q,d(a){a&&l(s)}}}function sl(U){let s,f='Learn more about the details of 8-bit quantization in this <a href="https://huggingface.co/blog/hf-bitsandbytes-integration" rel="nofollow">blog post</a>!';return{c(){s=g("p"),s.innerHTML=f},l(a){s=J(a,"P",{"data-svelte-h":!0}),w(s)!=="svelte-1bb05fp"&&(s.innerHTML=f)},m(a,m){n(a,s,m)},p:Q,d(a){a&&l(s)}}}function al(U){let s,f='Try 4-bit quantization in this <a href="https://colab.research.google.com/drive/1ge2F1QSK8Q7h0hn3YKuBCOAS0bK8E0wf" rel="nofollow">notebook</a> and learn more about it’s details in this <a href="https://huggingface.co/blog/4bit-transformers-bitsandbytes" rel="nofollow">blog post</a>.';return{c(){s=g("p"),s.innerHTML=f},l(a){s=J(a,"P",{"data-svelte-h":!0}),w(s)!=="svelte-rme94c"&&(s.innerHTML=f)},m(a,m){n(a,s,m)},p:Q,d(a){a&&l(s)}}}function il(U){let s,f,a,m,i,p,T,Z='<a href="https://github.com/TimDettmers/bitsandbytes" rel="nofollow">bitsandbytes</a> is the easiest option for quantizing a model to 8 and 4-bit. 8-bit quantization multiplies outliers in fp16 with non-outliers in int8, converts the non-outlier values back to fp16, and then adds them together to return the weights in fp16. This reduces the degradative effect outlier values have on a model’s performance. 4-bit quantization compresses a model even further, and it is commonly used with <a href="https://hf.co/papers/2305.14314" rel="nofollow">QLoRA</a> to finetune quantized LLMs.',v,k,j="To use bitsandbytes, make sure you have the following libraries installed:",z,B,d,_,u,C,ht='Now you can quantize a model by passing a <code>BitsAndBytesConfig</code> to <a href="/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method. This works for any model in any modality, as long as it supports loading with Accelerate and contains <code>torch.nn.Linear</code> layers.',Ue,X,ve,G,Ce,V,Mt="You can check your memory footprint with the <code>get_memory_footprint</code> method:",Ze,E,je,A,$t='Quantized models can be loaded from the <a href="/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method without needing to specify the <code>load_in_8bit</code> or <code>load_in_4bit</code> parameters:',Be,q,We,I,ke,N,Re,H,gt="This section explores some of the specific features of 8-bit models, such as offloading, outlier thresholds, skipping module conversion, and finetuning.",ze,L,Xe,x,Jt='8-bit models can offload weights between the CPU and GPU to support fitting very large models into memory. The weights dispatched to the CPU are actually stored in <strong>float32</strong>, and aren’t converted to 8-bit. For example, to enable offloading for the <a href="https://huggingface.co/bigscience/bloom-1b7" rel="nofollow">bigscience/bloom-1b7</a> model, start by creating a <a href="/docs/transformers/main/en/main_classes/quantization#transformers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',Ge,Y,Ne,S,wt="Design a custom device map to fit everything on your GPU except for the <code>lm_head</code>, which you’ll dispatch to the CPU:",Fe,P,Qe,D,Tt="Now load your model with the custom <code>device_map</code> and <code>quantization_config</code>:",Ve,K,Ee,O,Ae,ee,_t="An “outlier” is a hidden state value greater than a certain threshold, and these values are computed in fp16. While the values are usually normally distributed ([-3.5, 3.5]), this distribution can be very different for large models ([-60, 6] or [6, 60]). 8-bit quantization works well for values ~5, but beyond that, there is a significant performance penalty. A good default threshold value is 6, but a lower threshold may be needed for more unstable models (small models or finetuning).",qe,te,Ut='To find the best threshold for your model, we recommend experimenting with the <code>llm_int8_threshold</code> parameter in <a href="/docs/transformers/main/en/main_classes/quantization#transformers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',Ie,le,He,ne,Le,se,vt='For some models, like <a href="model_doc/jukebox">Jukebox</a>, you don’t need to quantize every module to 8-bit which can actually cause instability. With Jukebox, there are several <code>lm_head</code> modules that should be skipped using the <code>llm_int8_skip_modules</code> parameter in <a href="/docs/transformers/main/en/main_classes/quantization#transformers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',xe,ae,Ye,ie,Se,oe,Ct='With the <a href="https://github.com/huggingface/peft" rel="nofollow">PEFT</a> library, you can finetune large models like <a href="https://huggingface.co/google/flan-t5-large" rel="nofollow">flan-t5-large</a> and <a href="https://huggingface.co/facebook/opt-6.7b" rel="nofollow">facebook/opt-6.7b</a> with 8-bit quantization. You don’t need to pass the <code>device_map</code> parameter for training because it’ll automatically load your model on a GPU. However, you can still customize the device map with the <code>device_map</code> parameter if you want to (<code>device_map=&quot;auto&quot;</code> should only be used for inference).',Pe,re,De,F,Ke,me,Zt="This section explores some of the specific features of 4-bit models, such as changing the compute data type, using the Normal Float 4 (NF4) data type, and using nested quantization.",Oe,pe,et,fe,jt='To speedup computation, you can change the data type from float32 (the default value) to bf16 using the <code>bnb_4bit_compute_dtype</code> parameter in <a href="/docs/transformers/main/en/main_classes/quantization#transformers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',tt,de,lt,ue,nt,be,Bt='NF4 is a 4-bit data type from the <a href="https://hf.co/papers/2305.14314" rel="nofollow">QLoRA</a> paper, adapted for weights initialized from a normal distribution. You should use NF4 for training 4-bit base models. This can be configured with the <code>bnb_4bit_quant_type</code> parameter in the <a href="/docs/transformers/main/en/main_classes/quantization#transformers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',st,ce,at,ye,Wt="For inference, the <code>bnb_4bit_quant_type</code> does not have a huge impact on performance. However, to remain consistent with the model weights, you should use the <code>bnb_4bit_compute_dtype</code> and <code>torch_dtype</code> values.",it,he,ot,Me,kt='Nested quantization is a technique that can save additional memory at no additional performance cost. This feature performs a second quantization of the already quantized weights to save an additional 0.4 bits/parameter. For example, with nested quantization, you can finetune a <a href="https://huggingface.co/meta-llama/Llama-2-13b" rel="nofollow">Llama-13b</a> model on a 16GB NVIDIA T4 GPU with a sequence length of 1024, a batch size of 1, and enabling gradient accumulation with 4 steps.',rt,$e,mt,ge,pt,Je,Rt="Once quantized, you can dequantize the model to the original precision but this might result in a small quality loss of the model. Make sure you have enough GPU RAM to fit the dequantized model.",ft,we,dt,Te,ut,_e,bt;return i=new R({props:{title:"bitsandbytes",local:"bitsandbytes",headingTag:"h1"}}),B=new At({props:{id:"bnb",options:["8-bit","4-bit"],$$slots:{default:[Kt]},$$scope:{ctx:U}}}),_=new ct({props:{$$slots:{default:[Ot]},$$scope:{ctx:U}}}),X=new At({props:{id:"bnb",options:["8-bit","4-bit"],$$slots:{default:[ll]},$$scope:{ctx:U}}}),G=new ct({props:{warning:!0,$$slots:{default:[nl]},$$scope:{ctx:U}}}),E=new W({props:{code:"cHJpbnQobW9kZWwuZ2V0X21lbW9yeV9mb290cHJpbnQoKSk=",highlighted:'<span class="hljs-built_in">print</span>(model.get_memory_footprint())',wrap:!1}}),q=new W({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMiU3QnlvdXJfdXNlcm5hbWUlN0QlMkZibG9vbS01NjBtLThiaXQlMjIlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMik=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;{your_username}/bloom-560m-8bit&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>)`,wrap:!1}}),I=new R({props:{title:"8-bit (LLM.int8() algorithm)",local:"8-bit-llmint8-algorithm",headingTag:"h2"}}),N=new ct({props:{$$slots:{default:[sl]},$$scope:{ctx:U}}}),L=new R({props:{title:"Offloading",local:"offloading",headingTag:"h3"}}),Y=new W({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyhsbG1faW50OF9lbmFibGVfZnAzMl9jcHVfb2ZmbG9hZCUzRFRydWUp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(llm_int8_enable_fp32_cpu_offload=<span class="hljs-literal">True</span>)`,wrap:!1}}),P=new W({props:{code:"ZGV2aWNlX21hcCUyMCUzRCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMnRyYW5zZm9ybWVyLndvcmRfZW1iZWRkaW5ncyUyMiUzQSUyMDAlMkMlMEElMjAlMjAlMjAlMjAlMjJ0cmFuc2Zvcm1lci53b3JkX2VtYmVkZGluZ3NfbGF5ZXJub3JtJTIyJTNBJTIwMCUyQyUwQSUyMCUyMCUyMCUyMCUyMmxtX2hlYWQlMjIlM0ElMjAlMjJjcHUlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJ0cmFuc2Zvcm1lci5oJTIyJTNBJTIwMCUyQyUwQSUyMCUyMCUyMCUyMCUyMnRyYW5zZm9ybWVyLmxuX2YlMjIlM0ElMjAwJTJDJTBBJTdE",highlighted:`device_map = {
    <span class="hljs-string">&quot;transformer.word_embeddings&quot;</span>: <span class="hljs-number">0</span>,
    <span class="hljs-string">&quot;transformer.word_embeddings_layernorm&quot;</span>: <span class="hljs-number">0</span>,
    <span class="hljs-string">&quot;lm_head&quot;</span>: <span class="hljs-string">&quot;cpu&quot;</span>,
    <span class="hljs-string">&quot;transformer.h&quot;</span>: <span class="hljs-number">0</span>,
    <span class="hljs-string">&quot;transformer.ln_f&quot;</span>: <span class="hljs-number">0</span>,
}`,wrap:!1}}),K=new W({props:{code:"bW9kZWxfOGJpdCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJiaWdzY2llbmNlJTJGYmxvb20tMWI3JTIyJTJDJTBBJTIwJTIwJTIwJTIwZGV2aWNlX21hcCUzRGRldmljZV9tYXAlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRpemF0aW9uX2NvbmZpZyUyQyUwQSk=",highlighted:`model_8bit = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>,
    device_map=device_map,
    quantization_config=quantization_config,
)`,wrap:!1}}),O=new R({props:{title:"Outlier threshold",local:"outlier-threshold",headingTag:"h3"}}),le=new W({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBbW9kZWxfaWQlMjAlM0QlMjAlMjJiaWdzY2llbmNlJTJGYmxvb20tMWI3JTIyJTBBJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyglMEElMjAlMjAlMjAlMjBsbG1faW50OF90aHJlc2hvbGQlM0QxMCUyQyUwQSklMEElMEFtb2RlbF84Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMG1vZGVsX2lkJTJDJTBBJTIwJTIwJTIwJTIwZGV2aWNlX21hcCUzRGRldmljZV9tYXAlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRpemF0aW9uX2NvbmZpZyUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, BitsAndBytesConfig

model_id = <span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>

quantization_config = BitsAndBytesConfig(
    llm_int8_threshold=<span class="hljs-number">10</span>,
)

model_8bit = AutoModelForCausalLM.from_pretrained(
    model_id,
    device_map=device_map,
    quantization_config=quantization_config,
)`,wrap:!1}}),ne=new R({props:{title:"Skip module conversion",local:"skip-module-conversion",headingTag:"h3"}}),ae=new W({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUyQyUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQW1vZGVsX2lkJTIwJTNEJTIwJTIyYmlnc2NpZW5jZSUyRmJsb29tLTFiNyUyMiUwQSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbGxtX2ludDhfc2tpcF9tb2R1bGVzJTNEJTVCJTIybG1faGVhZCUyMiU1RCUyQyUwQSklMEElMEFtb2RlbF84Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMG1vZGVsX2lkJTJDJTBBJTIwJTIwJTIwJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRpemF0aW9uX2NvbmZpZyUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig

model_id = <span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>

quantization_config = BitsAndBytesConfig(
    llm_int8_skip_modules=[<span class="hljs-string">&quot;lm_head&quot;</span>],
)

model_8bit = AutoModelForCausalLM.from_pretrained(
    model_id,
    device_map=<span class="hljs-string">&quot;auto&quot;</span>,
    quantization_config=quantization_config,
)`,wrap:!1}}),ie=new R({props:{title:"Finetuning",local:"finetuning",headingTag:"h3"}}),re=new R({props:{title:"4-bit (QLoRA algorithm)",local:"4-bit-qlora-algorithm",headingTag:"h2"}}),F=new ct({props:{$$slots:{default:[al]},$$scope:{ctx:U}}}),pe=new R({props:{title:"Compute data type",local:"compute-data-type",headingTag:"h3"}}),de=new W({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyhsb2FkX2luXzRiaXQlM0RUcnVlJTJDJTIwYm5iXzRiaXRfY29tcHV0ZV9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2KQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>, bnb_4bit_compute_dtype=torch.bfloat16)`,wrap:!1}}),ue=new R({props:{title:"Normal Float 4 (NF4)",local:"normal-float-4-nf4",headingTag:"h3"}}),ce=new W({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQW5mNF9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl80Yml0JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3F1YW50X3R5cGUlM0QlMjJuZjQlMjIlMkMlMEEpJTBBJTBBbW9kZWxfbmY0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkJTJDJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRG5mNF9jb25maWcp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig

nf4_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_quant_type=<span class="hljs-string">&quot;nf4&quot;</span>,
)

model_nf4 = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=nf4_config)`,wrap:!1}}),he=new R({props:{title:"Nested quantization",local:"nested-quantization",headingTag:"h3"}}),$e=new W({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQWRvdWJsZV9xdWFudF9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl80Yml0JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3VzZV9kb3VibGVfcXVhbnQlM0RUcnVlJTJDJTBBKSUwQSUwQW1vZGVsX2RvdWJsZV9xdWFudCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJtZXRhLWxsYW1hJTJGTGxhbWEtMi0xM2IlMjIlMkMlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEZG91YmxlX3F1YW50X2NvbmZpZyk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig

double_quant_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_use_double_quant=<span class="hljs-literal">True</span>,
)

model_double_quant = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-2-13b&quot;</span>, quantization_config=double_quant_config)`,wrap:!1}}),ge=new R({props:{title:"Dequantizing bitsandbytes models",local:"dequantizing-bitsandbytes-models",headingTag:"h2"}}),we=new W({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQml0c0FuZEJ5dGVzQ29uZmlnJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQW1vZGVsX2lkJTIwJTNEJTIwJTIyZmFjZWJvb2slMkZvcHQtMTI1bSUyMiUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkJTJDJTIwQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fNGJpdCUzRFRydWUpKSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkKSUwQSUwQW1vZGVsLmRlcXVhbnRpemUoKSUwQSUwQXRleHQlMjAlM0QlMjB0b2tlbml6ZXIoJTIySGVsbG8lMjBteSUyMG5hbWUlMjBpcyUyMiUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnRvKDApJTBBJTBBb3V0JTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKip0ZXh0KSUwQXByaW50KHRva2VuaXplci5kZWNvZGUob3V0JTVCMCU1RCkp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, BitsAndBytesConfig, AutoTokenizer

model_id = <span class="hljs-string">&quot;facebook/opt-125m&quot;</span>

model = AutoModelForCausalLM.from_pretrained(model_id, BitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>))
tokenizer = AutoTokenizer.from_pretrained(model_id)

model.dequantize()

text = tokenizer(<span class="hljs-string">&quot;Hello my name is&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-number">0</span>)

out = model.generate(**text)
<span class="hljs-built_in">print</span>(tokenizer.decode(out[<span class="hljs-number">0</span>]))`,wrap:!1}}),Te=new St({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/quantization/bitsandbytes.md"}}),{c(){s=g("meta"),f=o(),a=g("p"),m=o(),b(i.$$.fragment),p=o(),T=g("p"),T.innerHTML=Z,v=o(),k=g("p"),k.textContent=j,z=o(),b(B.$$.fragment),d=o(),b(_.$$.fragment),u=o(),C=g("p"),C.innerHTML=ht,Ue=o(),b(X.$$.fragment),ve=o(),b(G.$$.fragment),Ce=o(),V=g("p"),V.innerHTML=Mt,Ze=o(),b(E.$$.fragment),je=o(),A=g("p"),A.innerHTML=$t,Be=o(),b(q.$$.fragment),We=o(),b(I.$$.fragment),ke=o(),b(N.$$.fragment),Re=o(),H=g("p"),H.textContent=gt,ze=o(),b(L.$$.fragment),Xe=o(),x=g("p"),x.innerHTML=Jt,Ge=o(),b(Y.$$.fragment),Ne=o(),S=g("p"),S.innerHTML=wt,Fe=o(),b(P.$$.fragment),Qe=o(),D=g("p"),D.innerHTML=Tt,Ve=o(),b(K.$$.fragment),Ee=o(),b(O.$$.fragment),Ae=o(),ee=g("p"),ee.textContent=_t,qe=o(),te=g("p"),te.innerHTML=Ut,Ie=o(),b(le.$$.fragment),He=o(),b(ne.$$.fragment),Le=o(),se=g("p"),se.innerHTML=vt,xe=o(),b(ae.$$.fragment),Ye=o(),b(ie.$$.fragment),Se=o(),oe=g("p"),oe.innerHTML=Ct,Pe=o(),b(re.$$.fragment),De=o(),b(F.$$.fragment),Ke=o(),me=g("p"),me.textContent=Zt,Oe=o(),b(pe.$$.fragment),et=o(),fe=g("p"),fe.innerHTML=jt,tt=o(),b(de.$$.fragment),lt=o(),b(ue.$$.fragment),nt=o(),be=g("p"),be.innerHTML=Bt,st=o(),b(ce.$$.fragment),at=o(),ye=g("p"),ye.innerHTML=Wt,it=o(),b(he.$$.fragment),ot=o(),Me=g("p"),Me.innerHTML=kt,rt=o(),b($e.$$.fragment),mt=o(),b(ge.$$.fragment),pt=o(),Je=g("p"),Je.textContent=Rt,ft=o(),b(we.$$.fragment),dt=o(),b(Te.$$.fragment),ut=o(),_e=g("p"),this.h()},l(e){const t=xt("svelte-u9bgzb",document.head);s=J(t,"META",{name:!0,content:!0}),t.forEach(l),f=r(e),a=J(e,"P",{}),Vt(a).forEach(l),m=r(e),c(i.$$.fragment,e),p=r(e),T=J(e,"P",{"data-svelte-h":!0}),w(T)!=="svelte-5a92xi"&&(T.innerHTML=Z),v=r(e),k=J(e,"P",{"data-svelte-h":!0}),w(k)!=="svelte-gf36q7"&&(k.textContent=j),z=r(e),c(B.$$.fragment,e),d=r(e),c(_.$$.fragment,e),u=r(e),C=J(e,"P",{"data-svelte-h":!0}),w(C)!=="svelte-1hwnf1e"&&(C.innerHTML=ht),Ue=r(e),c(X.$$.fragment,e),ve=r(e),c(G.$$.fragment,e),Ce=r(e),V=J(e,"P",{"data-svelte-h":!0}),w(V)!=="svelte-1bxp667"&&(V.innerHTML=Mt),Ze=r(e),c(E.$$.fragment,e),je=r(e),A=J(e,"P",{"data-svelte-h":!0}),w(A)!=="svelte-17ps4nd"&&(A.innerHTML=$t),Be=r(e),c(q.$$.fragment,e),We=r(e),c(I.$$.fragment,e),ke=r(e),c(N.$$.fragment,e),Re=r(e),H=J(e,"P",{"data-svelte-h":!0}),w(H)!=="svelte-1yx1x2g"&&(H.textContent=gt),ze=r(e),c(L.$$.fragment,e),Xe=r(e),x=J(e,"P",{"data-svelte-h":!0}),w(x)!=="svelte-1v0jh01"&&(x.innerHTML=Jt),Ge=r(e),c(Y.$$.fragment,e),Ne=r(e),S=J(e,"P",{"data-svelte-h":!0}),w(S)!=="svelte-15oy7am"&&(S.innerHTML=wt),Fe=r(e),c(P.$$.fragment,e),Qe=r(e),D=J(e,"P",{"data-svelte-h":!0}),w(D)!=="svelte-k1g8c2"&&(D.innerHTML=Tt),Ve=r(e),c(K.$$.fragment,e),Ee=r(e),c(O.$$.fragment,e),Ae=r(e),ee=J(e,"P",{"data-svelte-h":!0}),w(ee)!=="svelte-ur5rgd"&&(ee.textContent=_t),qe=r(e),te=J(e,"P",{"data-svelte-h":!0}),w(te)!=="svelte-v5sezx"&&(te.innerHTML=Ut),Ie=r(e),c(le.$$.fragment,e),He=r(e),c(ne.$$.fragment,e),Le=r(e),se=J(e,"P",{"data-svelte-h":!0}),w(se)!=="svelte-1ba1s3u"&&(se.innerHTML=vt),xe=r(e),c(ae.$$.fragment,e),Ye=r(e),c(ie.$$.fragment,e),Se=r(e),oe=J(e,"P",{"data-svelte-h":!0}),w(oe)!=="svelte-4nvx4q"&&(oe.innerHTML=Ct),Pe=r(e),c(re.$$.fragment,e),De=r(e),c(F.$$.fragment,e),Ke=r(e),me=J(e,"P",{"data-svelte-h":!0}),w(me)!=="svelte-7ob7j"&&(me.textContent=Zt),Oe=r(e),c(pe.$$.fragment,e),et=r(e),fe=J(e,"P",{"data-svelte-h":!0}),w(fe)!=="svelte-15j4whk"&&(fe.innerHTML=jt),tt=r(e),c(de.$$.fragment,e),lt=r(e),c(ue.$$.fragment,e),nt=r(e),be=J(e,"P",{"data-svelte-h":!0}),w(be)!=="svelte-7qr54g"&&(be.innerHTML=Bt),st=r(e),c(ce.$$.fragment,e),at=r(e),ye=J(e,"P",{"data-svelte-h":!0}),w(ye)!=="svelte-1qoc2ct"&&(ye.innerHTML=Wt),it=r(e),c(he.$$.fragment,e),ot=r(e),Me=J(e,"P",{"data-svelte-h":!0}),w(Me)!=="svelte-obxwut"&&(Me.innerHTML=kt),rt=r(e),c($e.$$.fragment,e),mt=r(e),c(ge.$$.fragment,e),pt=r(e),Je=J(e,"P",{"data-svelte-h":!0}),w(Je)!=="svelte-1gygypd"&&(Je.textContent=Rt),ft=r(e),c(we.$$.fragment,e),dt=r(e),c(Te.$$.fragment,e),ut=r(e),_e=J(e,"P",{}),Vt(_e).forEach(l),this.h()},h(){Et(s,"name","hf:doc:metadata"),Et(s,"content",ol)},m(e,t){Yt(document.head,s),n(e,f,t),n(e,a,t),n(e,m,t),y(i,e,t),n(e,p,t),n(e,T,t),n(e,v,t),n(e,k,t),n(e,z,t),y(B,e,t),n(e,d,t),y(_,e,t),n(e,u,t),n(e,C,t),n(e,Ue,t),y(X,e,t),n(e,ve,t),y(G,e,t),n(e,Ce,t),n(e,V,t),n(e,Ze,t),y(E,e,t),n(e,je,t),n(e,A,t),n(e,Be,t),y(q,e,t),n(e,We,t),y(I,e,t),n(e,ke,t),y(N,e,t),n(e,Re,t),n(e,H,t),n(e,ze,t),y(L,e,t),n(e,Xe,t),n(e,x,t),n(e,Ge,t),y(Y,e,t),n(e,Ne,t),n(e,S,t),n(e,Fe,t),y(P,e,t),n(e,Qe,t),n(e,D,t),n(e,Ve,t),y(K,e,t),n(e,Ee,t),y(O,e,t),n(e,Ae,t),n(e,ee,t),n(e,qe,t),n(e,te,t),n(e,Ie,t),y(le,e,t),n(e,He,t),y(ne,e,t),n(e,Le,t),n(e,se,t),n(e,xe,t),y(ae,e,t),n(e,Ye,t),y(ie,e,t),n(e,Se,t),n(e,oe,t),n(e,Pe,t),y(re,e,t),n(e,De,t),y(F,e,t),n(e,Ke,t),n(e,me,t),n(e,Oe,t),y(pe,e,t),n(e,et,t),n(e,fe,t),n(e,tt,t),y(de,e,t),n(e,lt,t),y(ue,e,t),n(e,nt,t),n(e,be,t),n(e,st,t),y(ce,e,t),n(e,at,t),n(e,ye,t),n(e,it,t),y(he,e,t),n(e,ot,t),n(e,Me,t),n(e,rt,t),y($e,e,t),n(e,mt,t),y(ge,e,t),n(e,pt,t),n(e,Je,t),n(e,ft,t),y(we,e,t),n(e,dt,t),y(Te,e,t),n(e,ut,t),n(e,_e,t),bt=!0},p(e,[t]){const zt={};t&2&&(zt.$$scope={dirty:t,ctx:e}),B.$set(zt);const Xt={};t&2&&(Xt.$$scope={dirty:t,ctx:e}),_.$set(Xt);const Gt={};t&2&&(Gt.$$scope={dirty:t,ctx:e}),X.$set(Gt);const Nt={};t&2&&(Nt.$$scope={dirty:t,ctx:e}),G.$set(Nt);const Ft={};t&2&&(Ft.$$scope={dirty:t,ctx:e}),N.$set(Ft);const Qt={};t&2&&(Qt.$$scope={dirty:t,ctx:e}),F.$set(Qt)},i(e){bt||(h(i.$$.fragment,e),h(B.$$.fragment,e),h(_.$$.fragment,e),h(X.$$.fragment,e),h(G.$$.fragment,e),h(E.$$.fragment,e),h(q.$$.fragment,e),h(I.$$.fragment,e),h(N.$$.fragment,e),h(L.$$.fragment,e),h(Y.$$.fragment,e),h(P.$$.fragment,e),h(K.$$.fragment,e),h(O.$$.fragment,e),h(le.$$.fragment,e),h(ne.$$.fragment,e),h(ae.$$.fragment,e),h(ie.$$.fragment,e),h(re.$$.fragment,e),h(F.$$.fragment,e),h(pe.$$.fragment,e),h(de.$$.fragment,e),h(ue.$$.fragment,e),h(ce.$$.fragment,e),h(he.$$.fragment,e),h($e.$$.fragment,e),h(ge.$$.fragment,e),h(we.$$.fragment,e),h(Te.$$.fragment,e),bt=!0)},o(e){M(i.$$.fragment,e),M(B.$$.fragment,e),M(_.$$.fragment,e),M(X.$$.fragment,e),M(G.$$.fragment,e),M(E.$$.fragment,e),M(q.$$.fragment,e),M(I.$$.fragment,e),M(N.$$.fragment,e),M(L.$$.fragment,e),M(Y.$$.fragment,e),M(P.$$.fragment,e),M(K.$$.fragment,e),M(O.$$.fragment,e),M(le.$$.fragment,e),M(ne.$$.fragment,e),M(ae.$$.fragment,e),M(ie.$$.fragment,e),M(re.$$.fragment,e),M(F.$$.fragment,e),M(pe.$$.fragment,e),M(de.$$.fragment,e),M(ue.$$.fragment,e),M(ce.$$.fragment,e),M(he.$$.fragment,e),M($e.$$.fragment,e),M(ge.$$.fragment,e),M(we.$$.fragment,e),M(Te.$$.fragment,e),bt=!1},d(e){e&&(l(f),l(a),l(m),l(p),l(T),l(v),l(k),l(z),l(d),l(u),l(C),l(Ue),l(ve),l(Ce),l(V),l(Ze),l(je),l(A),l(Be),l(We),l(ke),l(Re),l(H),l(ze),l(Xe),l(x),l(Ge),l(Ne),l(S),l(Fe),l(Qe),l(D),l(Ve),l(Ee),l(Ae),l(ee),l(qe),l(te),l(Ie),l(He),l(Le),l(se),l(xe),l(Ye),l(Se),l(oe),l(Pe),l(De),l(Ke),l(me),l(Oe),l(et),l(fe),l(tt),l(lt),l(nt),l(be),l(st),l(at),l(ye),l(it),l(ot),l(Me),l(rt),l(mt),l(pt),l(Je),l(ft),l(dt),l(ut),l(_e)),l(s),$(i,e),$(B,e),$(_,e),$(X,e),$(G,e),$(E,e),$(q,e),$(I,e),$(N,e),$(L,e),$(Y,e),$(P,e),$(K,e),$(O,e),$(le,e),$(ne,e),$(ae,e),$(ie,e),$(re,e),$(F,e),$(pe,e),$(de,e),$(ue,e),$(ce,e),$(he,e),$($e,e),$(ge,e),$(we,e),$(Te,e)}}}const ol='{"title":"bitsandbytes","local":"bitsandbytes","sections":[{"title":"8-bit (LLM.int8() algorithm)","local":"8-bit-llmint8-algorithm","sections":[{"title":"Offloading","local":"offloading","sections":[],"depth":3},{"title":"Outlier threshold","local":"outlier-threshold","sections":[],"depth":3},{"title":"Skip module conversion","local":"skip-module-conversion","sections":[],"depth":3},{"title":"Finetuning","local":"finetuning","sections":[],"depth":3}],"depth":2},{"title":"4-bit (QLoRA algorithm)","local":"4-bit-qlora-algorithm","sections":[{"title":"Compute data type","local":"compute-data-type","sections":[],"depth":3},{"title":"Normal Float 4 (NF4)","local":"normal-float-4-nf4","sections":[],"depth":3},{"title":"Nested quantization","local":"nested-quantization","sections":[],"depth":3}],"depth":2},{"title":"Dequantizing bitsandbytes models","local":"dequantizing-bitsandbytes-models","sections":[],"depth":2}],"depth":1}';function rl(U){return It(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class cl extends Ht{constructor(s){super(),Lt(this,s,rl,il,qt,{})}}export{cl as component};
