import{s as _l,o as Zl,n as xe}from"../chunks/scheduler.18a86fab.js";import{S as jl,i as Bl,g as r,s as a,r as d,A as Rl,h as m,f as l,c as s,j as He,u as p,x as h,k as Ve,y as j,a as n,v as u,d as f,t as b,w as c,m as vl,n as Cl,l as k}from"../chunks/index.98837b22.js";import{T as Nl}from"../chunks/Tip.77304350.js";import{C as _}from"../chunks/CodeBlock.8d0c2e8a.js";import{H as Z}from"../chunks/Heading.7e7e0c5b.js";import{H as Xl,a as Ul}from"../chunks/HfOption.6641485e.js";import{E as Wl}from"../chunks/index.92f616ba.js";function zl(X){let i,$,y,U,o,v='By default, all other modules such as <a href="https://pytorch.org/docs/stable/generated/torch.nn.LayerNorm.html" rel="nofollow">torch.nn.LayerNorm</a> are set to the default torch dtype. You can change the data type of these modules with the <code>torch_dtype</code> parameter. Setting <code>torch_dtype=&quot;auto&quot;</code> loads the model in the data type defined in a model’s <code>config.json</code> file.',C,J,B,w,W='Once a model is quantized to 8-bit, you can’t push the quantized weights to the Hub unless you’re using the latest version of Transformers and bitsandbytes. If you have the latest versions, then you can push the 8-bit model to the Hub with <a href="/docs/transformers/main/en/main_classes/model#transformers.utils.PushToHubMixin.push_to_hub">push_to_hub()</a>. The quantization config.json file is pushed first, followed by the quantized model weights.',R,T,N;return y=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyhsb2FkX2luXzhiaXQlM0RUcnVlKSUwQSUwQW1vZGVsXzhiaXQlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyYmlnc2NpZW5jZSUyRmJsb29tLTFiNyUyMiUyQyUyMCUwQSUyMCUyMCUyMCUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWclMEEp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_8bit=<span class="hljs-literal">True</span>)

model_8bit = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>, 
    device_map=<span class="hljs-string">&quot;auto&quot;</span>,
    quantization_config=quantization_config
)`,wrap:!1}}),J=new _({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMkMlMjBCaXRzQW5kQnl0ZXNDb25maWclMEElMEFxdWFudGl6YXRpb25fY29uZmlnJTIwJTNEJTIwQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fOGJpdCUzRFRydWUpJTBBJTBBbW9kZWxfOGJpdCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJmYWNlYm9vayUyRm9wdC0zNTBtJTIyJTJDJTIwJTBBJTIwJTIwJTIwJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRpemF0aW9uX2NvbmZpZyUyQyUyMCUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEJTIyYXV0byUyMiUwQSklMEFtb2RlbF84Yml0Lm1vZGVsLmRlY29kZXIubGF5ZXJzJTVCLTElNUQuZmluYWxfbGF5ZXJfbm9ybS53ZWlnaHQuZHR5cGU=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_8bit=<span class="hljs-literal">True</span>)

model_8bit = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;facebook/opt-350m&quot;</span>, 
    device_map=<span class="hljs-string">&quot;auto&quot;</span>,
    quantization_config=quantization_config, 
    torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>
)
model_8bit.model.decoder.layers[-<span class="hljs-number">1</span>].final_layer_norm.weight.dtype`,wrap:!1}}),T=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUyQyUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcobG9hZF9pbl84Yml0JTNEVHJ1ZSklMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJiaWdzY2llbmNlJTJGYmxvb20tNTYwbSUyMiUyQyUyMCUwQSUyMCUyMCUyMCUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWclMEEpJTBBJTBBbW9kZWwucHVzaF90b19odWIoJTIyYmxvb20tNTYwbS04Yml0JTIyKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_8bit=<span class="hljs-literal">True</span>)

model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;bigscience/bloom-560m&quot;</span>, 
    device_map=<span class="hljs-string">&quot;auto&quot;</span>,
    quantization_config=quantization_config
)

model.push_to_hub(<span class="hljs-string">&quot;bloom-560m-8bit&quot;</span>)`,wrap:!1}}),{c(){i=r("div"),$=vl('Quantizing a model in 8-bit halves the memory-usage, and for large models, set `device_map="auto"` to efficiently distribute the weights across all available GPUs.\n\n	'),d(y.$$.fragment),U=a(),o=r("p"),o.innerHTML=v,C=a(),d(J.$$.fragment),B=a(),w=r("p"),w.innerHTML=W,R=a(),d(T.$$.fragment),this.h()},l(g){i=m(g,"DIV",{class:!0,style:!0});var M=He(i);$=Cl(M,'Quantizing a model in 8-bit halves the memory-usage, and for large models, set `device_map="auto"` to efficiently distribute the weights across all available GPUs.\n\n	'),p(y.$$.fragment,M),U=s(M),o=m(M,"P",{"data-svelte-h":!0}),h(o)!=="svelte-110db1o"&&(o.innerHTML=v),C=s(M),p(J.$$.fragment,M),B=s(M),w=m(M,"P",{"data-svelte-h":!0}),h(w)!=="svelte-13qaumd"&&(w.innerHTML=W),R=s(M),p(T.$$.fragment,M),M.forEach(l),this.h()},h(){Ve(i,"class","bnb-container"),k(i,"border","1px solid #ddd"),k(i,"border-radius","8px"),k(i,"padding","20px"),k(i,"margin","20px 0")},m(g,M){n(g,i,M),j(i,$),u(y,i,null),j(i,U),j(i,o),j(i,C),u(J,i,null),j(i,B),j(i,w),j(i,R),u(T,i,null),N=!0},p:xe,i(g){N||(f(y.$$.fragment,g),f(J.$$.fragment,g),f(T.$$.fragment,g),N=!0)},o(g){b(y.$$.fragment,g),b(J.$$.fragment,g),b(T.$$.fragment,g),N=!1},d(g){g&&l(i),c(y),c(J),c(T)}}}function ql(X){let i,$,y,U,o,v='By default, all other modules such as <a href="https://pytorch.org/docs/stable/generated/torch.nn.LayerNorm.html" rel="nofollow">torch.nn.LayerNorm</a> are converted to <code>torch.float16</code>. You can change the data type of these modules with the <code>torch_dtype</code> parameter.. Setting <code>torch_dtype=&quot;auto&quot;</code> loads the model in the data type defined in a model’s <code>config.json</code> file.',C,J,B,w,W='Make sure you have the latest bitsandbytes version so you can serialize 4-bit models and push them to the Hub with <a href="/docs/transformers/main/en/main_classes/model#transformers.utils.PushToHubMixin.push_to_hub">push_to_hub()</a>. Use <a href="/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.save_pretrained">save_pretrained()</a> to save the 4-bit model locally.',R,T,N;return y=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyhsb2FkX2luXzRiaXQlM0RUcnVlKSUwQSUwQW1vZGVsXzRiaXQlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyYmlnc2NpZW5jZSUyRmJsb29tLTFiNyUyMiUyQyUwQSUyMCUyMCUyMCUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWclMEEp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>)

model_4bit = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>,
    device_map=<span class="hljs-string">&quot;auto&quot;</span>,
    quantization_config=quantization_config
)`,wrap:!1}}),J=new _({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMkMlMjBCaXRzQW5kQnl0ZXNDb25maWclMEElMEFxdWFudGl6YXRpb25fY29uZmlnJTIwJTNEJTIwQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fNGJpdCUzRFRydWUpJTBBJTBBbW9kZWxfNGJpdCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJmYWNlYm9vayUyRm9wdC0zNTBtJTIyJTJDJTBBJTIwJTIwJTIwJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRpemF0aW9uX2NvbmZpZyUyQyUyMCUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEJTIyYXV0byUyMiUwQSklMEFtb2RlbF80Yml0Lm1vZGVsLmRlY29kZXIubGF5ZXJzJTVCLTElNUQuZmluYWxfbGF5ZXJfbm9ybS53ZWlnaHQuZHR5cGU=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>)

model_4bit = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;facebook/opt-350m&quot;</span>,
    device_map=<span class="hljs-string">&quot;auto&quot;</span>,
    quantization_config=quantization_config, 
    torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>
)
model_4bit.model.decoder.layers[-<span class="hljs-number">1</span>].final_layer_norm.weight.dtype`,wrap:!1}}),T=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUyQyUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcobG9hZF9pbl80Yml0JTNEVHJ1ZSklMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJiaWdzY2llbmNlJTJGYmxvb20tNTYwbSUyMiUyQyUyMCUwQSUyMCUyMCUyMCUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWclMEEpJTBBJTBBbW9kZWwucHVzaF90b19odWIoJTIyYmxvb20tNTYwbS00Yml0JTIyKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>)

model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;bigscience/bloom-560m&quot;</span>, 
    device_map=<span class="hljs-string">&quot;auto&quot;</span>,
    quantization_config=quantization_config
)

model.push_to_hub(<span class="hljs-string">&quot;bloom-560m-4bit&quot;</span>)`,wrap:!1}}),{c(){i=r("div"),$=vl('Quantizing a model in 4-bit reduces your memory-usage by 4x, and for large models, set `device_map="auto"` to efficiently distribute the weights across all available GPUs.\n\n	'),d(y.$$.fragment),U=a(),o=r("p"),o.innerHTML=v,C=a(),d(J.$$.fragment),B=a(),w=r("p"),w.innerHTML=W,R=a(),d(T.$$.fragment),this.h()},l(g){i=m(g,"DIV",{class:!0,style:!0});var M=He(i);$=Cl(M,'Quantizing a model in 4-bit reduces your memory-usage by 4x, and for large models, set `device_map="auto"` to efficiently distribute the weights across all available GPUs.\n\n	'),p(y.$$.fragment,M),U=s(M),o=m(M,"P",{"data-svelte-h":!0}),h(o)!=="svelte-5p58r6"&&(o.innerHTML=v),C=s(M),p(J.$$.fragment,M),B=s(M),w=m(M,"P",{"data-svelte-h":!0}),h(w)!=="svelte-1flcsk7"&&(w.innerHTML=W),R=s(M),p(T.$$.fragment,M),M.forEach(l),this.h()},h(){Ve(i,"class","bnb-container"),k(i,"border","1px solid #ddd"),k(i,"border-radius","8px"),k(i,"padding","20px"),k(i,"margin","20px 0")},m(g,M){n(g,i,M),j(i,$),u(y,i,null),j(i,U),j(i,o),j(i,C),u(J,i,null),j(i,B),j(i,w),j(i,R),u(T,i,null),N=!0},p:xe,i(g){N||(f(y.$$.fragment,g),f(J.$$.fragment,g),f(T.$$.fragment,g),N=!0)},o(g){b(y.$$.fragment,g),b(J.$$.fragment,g),b(T.$$.fragment,g),N=!1},d(g){g&&l(i),c(y),c(J),c(T)}}}function kl(X){let i,$,y,U;return i=new Ul({props:{id:"bnb",option:"8-bit",$$slots:{default:[zl]},$$scope:{ctx:X}}}),y=new Ul({props:{id:"bnb",option:"4-bit",$$slots:{default:[ql]},$$scope:{ctx:X}}}),{c(){d(i.$$.fragment),$=a(),d(y.$$.fragment)},l(o){p(i.$$.fragment,o),$=s(o),p(y.$$.fragment,o)},m(o,v){u(i,o,v),n(o,$,v),u(y,o,v),U=!0},p(o,v){const C={};v&2&&(C.$$scope={dirty:v,ctx:o}),i.$set(C);const J={};v&2&&(J.$$scope={dirty:v,ctx:o}),y.$set(J)},i(o){U||(f(i.$$.fragment,o),f(y.$$.fragment,o),U=!0)},o(o){b(i.$$.fragment,o),b(y.$$.fragment,o),U=!1},d(o){o&&l($),c(i,o),c(y,o)}}}function El(X){let i,$="8 and 4-bit training is only supported for training <em>extra</em> parameters.";return{c(){i=r("p"),i.innerHTML=$},l(y){i=m(y,"P",{"data-svelte-h":!0}),h(i)!=="svelte-14e2693"&&(i.innerHTML=$)},m(y,U){n(y,i,U)},p:xe,d(y){y&&l(i)}}}function Gl(X){let i,$,y,U,o,v,C,J='The <a href="https://github.com/bitsandbytes-foundation/bitsandbytes" rel="nofollow">bitsandbytes</a> library provides quantization tools for LLMs through a lightweight Python wrapper around CUDA functions. It enables working with large models using limited computational resources by reducing their memory footprint.',B,w,W="At its core, bitsandbytes provides:",R,T,N="<li><strong>Quantized Linear Layers</strong>: <code>Linear8bitLt</code> and <code>Linear4bit</code> layers that replace standard PyTorch linear layers with memory-efficient quantized alternatives</li> <li><strong>Optimized Optimizers</strong>: 8-bit versions of common optimizers through its <code>optim</code> module, enabling training of large models with reduced memory requirements</li> <li><strong>Matrix Multiplication</strong>: Optimized matrix multiplication operations that leverage the quantized format</li>",g,M,Ye="bitsandbytes offers two main quantization features:",Ft,E,Se='<li><p><strong>LLM.int8()</strong> - An 8-bit quantization method that makes inference more accessible without significant performance degradation. Unlike naive quantization, <a href="https://hf.co/papers/2208.07339" rel="nofollow">LLM.int8()</a> dynamically preserves higher precision for critical computations, preventing information loss in sensitive parts of the model.</p></li> <li><p><strong>QLoRA</strong> - A 4-bit quantization technique that compresses models even further while maintaining trainability by inserting a small set of trainable low-rank adaptation (LoRA) weights.</p></li>',At,G,Pe='<p><strong>Note:</strong> For a user-friendly quantization experience, you can use the <code>bitsandbytes</code> <a href="https://huggingface.co/spaces/bnb-community/bnb-my-repo" rel="nofollow">community space</a>.</p>',Qt,F,De="Run the command below to install bitsandbytes.",It,A,Lt,Q,Oe='To compile from source, follow the instructions in the <a href="https://huggingface.co/docs/bitsandbytes/main/en/installation" rel="nofollow">bitsandbytes installation guide</a>.',Ht,I,Vt,L,Ke='bitsandbytes is currently only supported on CUDA GPUs for CUDA versions 11.0 - 12.8. However, there’s an ongoing multi-backend effort under development, which is currently in alpha. If you’re interested in providing feedback or testing, check out the <a href="https://github.com/bitsandbytes-foundation/bitsandbytes" rel="nofollow">bitsandbytes repository</a> for more information.',xt,H,Yt,V,tl="<thead><tr><th>Feature</th> <th>Minimum Hardware Requirement</th></tr></thead> <tbody><tr><td>8-bit optimizers</td> <td>NVIDIA Maxwell (GTX 900 series, TITAN X, M40) or newer GPUs *</td></tr> <tr><td>LLM.int8()</td> <td>NVIDIA Turing (RTX 20 series, T4) or newer GPUs</td></tr> <tr><td>NF4/FP4 quantization</td> <td>NVIDIA Maxwell (GTX 900 series, TITAN X, M40) or newer GPUs *</td></tr></tbody>",St,x,Pt,Y,el="<thead><tr><th>Backend</th> <th>Supported Versions</th> <th>Python versions</th> <th>Architecture Support</th> <th>Status</th></tr></thead> <tbody><tr><td>AMD ROCm</td> <td>6.1+</td> <td>3.10+</td> <td>minimum CDNA - gfx90a, RDNA - gfx1100</td> <td>Alpha</td></tr> <tr><td>Apple Silicon (MPS)</td> <td>WIP</td> <td>3.10+</td> <td>M1/M2 chips</td> <td>Planned</td></tr> <tr><td>Intel CPU</td> <td>v2.4.0+ (ipex)</td> <td>3.10+</td> <td>Intel CPU</td> <td>Alpha</td></tr> <tr><td>Intel GPU</td> <td>v2.4.0+ (ipex)</td> <td>3.10+</td> <td>Intel GPU</td> <td>Experimental</td></tr> <tr><td>Ascend NPU</td> <td>2.1.0+ (torch_npu)</td> <td>3.10+</td> <td>Ascend NPU</td> <td>Experimental</td></tr></tbody>",Dt,S,ll='<p><strong>Note:</strong> Bitsandbytes is moving away from the multi-backend approach towards using <a href="https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html" rel="nofollow">Pytorch Custom Operators</a>, as the main mechanism for supporting new hardware, and dispatching to the correct backend.</p>',Ot,P,Kt,D,nl='Quantize a model by passing a <a href="/docs/transformers/main/en/main_classes/quantization#transformers.BitsAndBytesConfig">BitsAndBytesConfig</a> to <a href="/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a>. This works for any model in any modality, as long as it supports <a href="https://huggingface.co/docs/accelerate/index" rel="nofollow">Accelerate</a> and contains <a href="https://pytorch.org/docs/stable/generated/torch.nn.Linear.html" rel="nofollow">torch.nn.Linear</a> layers.',te,z,ee,q,le,O,al="Check your memory footprint with <code>get_memory_footprint</code>.",ne,K,ae,tt,sl='Load quantized models with <a href="/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> without a <code>quantization_config</code>.',se,et,ie,lt,oe,nt,il="This section explores some of the specific features of 8-bit quantization, such as offloading, outlier thresholds, skipping module conversion, and finetuning.",re,at,me,st,ol='8-bit models can offload weights between the CPU and GPU to fit very large models into memory. The weights dispatched to the CPU are stored in <strong>float32</strong> and aren’t converted to 8-bit. For example, enable offloading for <a href="https://huggingface.co/bigscience/bloom-1b7" rel="nofollow">bigscience/bloom-1b7</a> through <a href="/docs/transformers/main/en/main_classes/quantization#transformers.BitsAndBytesConfig">BitsAndBytesConfig</a>.',de,it,pe,ot,rl="Design a custom device map to fit everything on your GPU except for the <code>lm_head</code>, which is dispatched to the CPU.",ue,rt,fe,mt,ml="Now load your model with the custom <code>device_map</code> and <code>quantization_config</code>.",be,dt,ce,pt,ye,ut,dl="An “outlier” is a hidden state value greater than a certain threshold, and these values are computed in fp16. While the values are usually normally distributed ([-3.5, 3.5]), this distribution can be very different for large models ([-60, 6] or [6, 60]). 8-bit quantization works well for values ~5, but beyond that, there is a significant performance penalty. A good default threshold value is 6, but a lower threshold may be needed for more unstable models (small models or finetuning).",he,ft,pl='To find the best threshold for your model, experiment with the <code>llm_int8_threshold</code> parameter in <a href="/docs/transformers/main/en/main_classes/quantization#transformers.BitsAndBytesConfig">BitsAndBytesConfig</a>. For example, setting the threshold to <code>0.0</code> significantly speeds up inference at the potential cost of some accuracy loss.',Me,bt,ge,ct,Te,yt,ul='For some models, like <a href="model_doc/jukebox">Jukebox</a>, you don’t need to quantize every module to 8-bit because it can actually cause instability. With Jukebox, there are several <code>lm_head</code> modules that should be skipped using the <code>llm_int8_skip_modules</code> parameter in <a href="/docs/transformers/main/en/main_classes/quantization#transformers.BitsAndBytesConfig">BitsAndBytesConfig</a>.',Je,ht,we,Mt,$e,gt,fl='The <a href="https://github.com/huggingface/peft" rel="nofollow">PEFT</a> library supports fine-tuning large models like <a href="https://huggingface.co/google/flan-t5-large" rel="nofollow">flan-t5-large</a> and <a href="https://huggingface.co/facebook/opt-6.7b" rel="nofollow">facebook/opt-6.7b</a> with 8-bit quantization. You don’t need to pass the <code>device_map</code> parameter for training because it automatically loads your model on a GPU. However, you can still customize the device map with the <code>device_map</code> parameter (<code>device_map=&quot;auto&quot;</code> should only be used for inference).',Ue,Tt,ve,Jt,bl="This section explores some of the specific features of 4-bit quantization, such as changing the compute data type, the Normal Float 4 (NF4) data type, and nested quantization.",Ce,wt,_e,$t,cl='Change the data type from float32 (the default value) to bf16 in <a href="/docs/transformers/main/en/main_classes/quantization#transformers.BitsAndBytesConfig">BitsAndBytesConfig</a> to speedup computation.',Ze,Ut,je,vt,Be,Ct,yl='NF4 is a 4-bit data type from the <a href="https://hf.co/papers/2305.14314" rel="nofollow">QLoRA</a> paper, adapted for weights initialized from a normal distribution. You should use NF4 for training 4-bit base models.',Re,_t,Ne,Zt,hl="For inference, the <code>bnb_4bit_quant_type</code> does not have a huge impact on performance. However, to remain consistent with the model weights, you should use the <code>bnb_4bit_compute_dtype</code> and <code>torch_dtype</code> values.",Xe,jt,We,Bt,Ml='Nested quantization can save additional memory at no additional performance cost. This feature performs a second quantization of the already quantized weights to save an additional 0.4 bits/parameter. For example, with nested quantization, you can finetune a <a href="https://huggingface.co/meta-llama/Llama-2-13b" rel="nofollow">Llama-13b</a> model on a 16GB NVIDIA T4 GPU with a sequence length of 1024, a batch size of 1, and enable gradient accumulation with 4 steps.',ze,Rt,qe,Nt,ke,Xt,gl='Once quantized, you can <a href="/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.dequantize">dequantize()</a> a model to the original precision but this may result in some quality loss. Make sure you have enough GPU memory to fit the dequantized model.',Ee,Wt,Ge,zt,Fe,qt,Tl='Learn more about the details of 8-bit quantization in <a href="https://huggingface.co/blog/hf-bitsandbytes-integration" rel="nofollow">A Gentle Introduction to 8-bit Matrix Multiplication for transformers at scale using Hugging Face Transformers, Accelerate and bitsandbytes</a>.',Ae,kt,Jl='Try 4-bit quantization in this <a href="https://colab.research.google.com/drive/1ge2F1QSK8Q7h0hn3YKuBCOAS0bK8E0wf" rel="nofollow">notebook</a> and learn more about it’s details in <a href="https://huggingface.co/blog/4bit-transformers-bitsandbytes" rel="nofollow">Making LLMs even more accessible with bitsandbytes, 4-bit quantization and QLoRA</a>.',Qe,Et,Ie,Gt,Le;return o=new Z({props:{title:"Bitsandbytes",local:"bitsandbytes",headingTag:"h1"}}),A=new _({props:{code:"cGlwJTIwaW5zdGFsbCUyMC0tdXBncmFkZSUyMHRyYW5zZm9ybWVycyUyMGFjY2VsZXJhdGUlMjBiaXRzYW5kYnl0ZXM=",highlighted:"pip install --upgrade transformers accelerate bitsandbytes",wrap:!1}}),I=new Z({props:{title:"Hardware Compatibility",local:"hardware-compatibility",headingTag:"h2"}}),H=new Z({props:{title:"CUDA",local:"cuda",headingTag:"h3"}}),x=new Z({props:{title:"Multi-backend",local:"multi-backend",headingTag:"h3"}}),P=new Z({props:{title:"Quantization Examples",local:"quantization-examples",headingTag:"h2"}}),z=new Xl({props:{id:"bnb",options:["8-bit","4-bit"],$$slots:{default:[kl]},$$scope:{ctx:X}}}),q=new Nl({props:{warning:!0,$$slots:{default:[El]},$$scope:{ctx:X}}}),K=new _({props:{code:"cHJpbnQobW9kZWwuZ2V0X21lbW9yeV9mb290cHJpbnQoKSk=",highlighted:'<span class="hljs-built_in">print</span>(model.get_memory_footprint())',wrap:!1}}),et=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMiU3QnlvdXJfdXNlcm5hbWUlN0QlMkZibG9vbS01NjBtLThiaXQlMjIlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMik=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;{your_username}/bloom-560m-8bit&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>)`,wrap:!1}}),lt=new Z({props:{title:"LLM.int8",local:"llmint8",headingTag:"h2"}}),at=new Z({props:{title:"Offloading",local:"offloading",headingTag:"h3"}}),it=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyhsbG1faW50OF9lbmFibGVfZnAzMl9jcHVfb2ZmbG9hZCUzRFRydWUp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(llm_int8_enable_fp32_cpu_offload=<span class="hljs-literal">True</span>)`,wrap:!1}}),rt=new _({props:{code:"ZGV2aWNlX21hcCUyMCUzRCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMnRyYW5zZm9ybWVyLndvcmRfZW1iZWRkaW5ncyUyMiUzQSUyMDAlMkMlMEElMjAlMjAlMjAlMjAlMjJ0cmFuc2Zvcm1lci53b3JkX2VtYmVkZGluZ3NfbGF5ZXJub3JtJTIyJTNBJTIwMCUyQyUwQSUyMCUyMCUyMCUyMCUyMmxtX2hlYWQlMjIlM0ElMjAlMjJjcHUlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJ0cmFuc2Zvcm1lci5oJTIyJTNBJTIwMCUyQyUwQSUyMCUyMCUyMCUyMCUyMnRyYW5zZm9ybWVyLmxuX2YlMjIlM0ElMjAwJTJDJTBBJTdE",highlighted:`device_map = {
    <span class="hljs-string">&quot;transformer.word_embeddings&quot;</span>: <span class="hljs-number">0</span>,
    <span class="hljs-string">&quot;transformer.word_embeddings_layernorm&quot;</span>: <span class="hljs-number">0</span>,
    <span class="hljs-string">&quot;lm_head&quot;</span>: <span class="hljs-string">&quot;cpu&quot;</span>,
    <span class="hljs-string">&quot;transformer.h&quot;</span>: <span class="hljs-number">0</span>,
    <span class="hljs-string">&quot;transformer.ln_f&quot;</span>: <span class="hljs-number">0</span>,
}`,wrap:!1}}),dt=new _({props:{code:"bW9kZWxfOGJpdCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJiaWdzY2llbmNlJTJGYmxvb20tMWI3JTIyJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0QlMjJhdXRvJTIyJTJDJTBBJTIwJTIwJTIwJTIwZGV2aWNlX21hcCUzRGRldmljZV9tYXAlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRpemF0aW9uX2NvbmZpZyUyQyUwQSk=",highlighted:`model_8bit = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>,
    torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>,
    device_map=device_map,
    quantization_config=quantization_config,
)`,wrap:!1}}),pt=new Z({props:{title:"Outlier threshold",local:"outlier-threshold",headingTag:"h3"}}),bt=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBbW9kZWxfaWQlMjAlM0QlMjAlMjJiaWdzY2llbmNlJTJGYmxvb20tMWI3JTIyJTBBJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyglMEElMjAlMjAlMjAlMjBsbG1faW50OF90aHJlc2hvbGQlM0QwLjAlMkMlMEElMjAlMjAlMjAlMjBsbG1faW50OF9lbmFibGVfZnAzMl9jcHVfb2ZmbG9hZCUzRFRydWUlMEEpJTBBJTBBbW9kZWxfOGJpdCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjBtb2RlbF9pZCUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMGRldmljZV9tYXAlM0RkZXZpY2VfbWFwJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWclMkMlMEEp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, BitsAndBytesConfig

model_id = <span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>

quantization_config = BitsAndBytesConfig(
    llm_int8_threshold=<span class="hljs-number">0.0</span>,
    llm_int8_enable_fp32_cpu_offload=<span class="hljs-literal">True</span>
)

model_8bit = AutoModelForCausalLM.from_pretrained(
    model_id,
    torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>,
    device_map=device_map,
    quantization_config=quantization_config,
)`,wrap:!1}}),ct=new Z({props:{title:"Skip module conversion",local:"skip-module-conversion",headingTag:"h3"}}),ht=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUyQyUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQW1vZGVsX2lkJTIwJTNEJTIwJTIyYmlnc2NpZW5jZSUyRmJsb29tLTFiNyUyMiUwQSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbGxtX2ludDhfc2tpcF9tb2R1bGVzJTNEJTVCJTIybG1faGVhZCUyMiU1RCUyQyUwQSklMEElMEFtb2RlbF84Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMG1vZGVsX2lkJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0QlMjJhdXRvJTIyJTJDJTBBJTIwJTIwJTIwJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRpemF0aW9uX2NvbmZpZyUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig

model_id = <span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>

quantization_config = BitsAndBytesConfig(
    llm_int8_skip_modules=[<span class="hljs-string">&quot;lm_head&quot;</span>],
)

model_8bit = AutoModelForCausalLM.from_pretrained(
    model_id,
    torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>,
    device_map=<span class="hljs-string">&quot;auto&quot;</span>,
    quantization_config=quantization_config,
)`,wrap:!1}}),Mt=new Z({props:{title:"Finetuning",local:"finetuning",headingTag:"h3"}}),Tt=new Z({props:{title:"QLoRA",local:"qlora",headingTag:"h2"}}),wt=new Z({props:{title:"Compute data type",local:"compute-data-type",headingTag:"h3"}}),Ut=new _({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyhsb2FkX2luXzRiaXQlM0RUcnVlJTJDJTIwYm5iXzRiaXRfY29tcHV0ZV9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2KQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>, bnb_4bit_compute_dtype=torch.bfloat16)`,wrap:!1}}),vt=new Z({props:{title:"Normal Float 4 (NF4)",local:"normal-float-4-nf4",headingTag:"h3"}}),_t=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQW5mNF9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl80Yml0JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3F1YW50X3R5cGUlM0QlMjJuZjQlMjIlMkMlMEEpJTBBJTBBbW9kZWxfbmY0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkJTJDJTIwdG9yY2hfZHR5cGUlM0QlMjJhdXRvJTIyJTJDJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRG5mNF9jb25maWcp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig

nf4_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_quant_type=<span class="hljs-string">&quot;nf4&quot;</span>,
)

model_nf4 = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>, quantization_config=nf4_config)`,wrap:!1}}),jt=new Z({props:{title:"Nested quantization",local:"nested-quantization",headingTag:"h3"}}),Rt=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQWRvdWJsZV9xdWFudF9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl80Yml0JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3VzZV9kb3VibGVfcXVhbnQlM0RUcnVlJTJDJTBBKSUwQSUwQW1vZGVsX2RvdWJsZV9xdWFudCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJtZXRhLWxsYW1hJTJGTGxhbWEtMi0xM2ItY2hhdC1oZiUyMiUyQyUyMHRvcmNoX2R0eXBlJTNEJTIyYXV0byUyMiUyQyUyMHF1YW50aXphdGlvbl9jb25maWclM0Rkb3VibGVfcXVhbnRfY29uZmlnKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig

double_quant_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_use_double_quant=<span class="hljs-literal">True</span>,
)

model_double_quant = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-2-13b-chat-hf&quot;</span>, torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>, quantization_config=double_quant_config)`,wrap:!1}}),Nt=new Z({props:{title:"Dequantizing bitsandbytes models",local:"dequantizing-bitsandbytes-models",headingTag:"h2"}}),Wt=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQml0c0FuZEJ5dGVzQ29uZmlnJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMmZhY2Vib29rJTJGb3B0LTEyNW0lMjIlMkMlMjBCaXRzQW5kQnl0ZXNDb25maWcobG9hZF9pbl80Yml0JTNEVHJ1ZSkpJTBBbW9kZWwuZGVxdWFudGl6ZSgp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, BitsAndBytesConfig, AutoTokenizer

model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;facebook/opt-125m&quot;</span>, BitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>))
model.dequantize()`,wrap:!1}}),zt=new Z({props:{title:"Resources",local:"resources",headingTag:"h2"}}),Et=new Wl({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/quantization/bitsandbytes.md"}}),{c(){i=r("meta"),$=a(),y=r("p"),U=a(),d(o.$$.fragment),v=a(),C=r("p"),C.innerHTML=J,B=a(),w=r("p"),w.textContent=W,R=a(),T=r("ul"),T.innerHTML=N,g=a(),M=r("p"),M.textContent=Ye,Ft=a(),E=r("ol"),E.innerHTML=Se,At=a(),G=r("blockquote"),G.innerHTML=Pe,Qt=a(),F=r("p"),F.textContent=De,It=a(),d(A.$$.fragment),Lt=a(),Q=r("p"),Q.innerHTML=Oe,Ht=a(),d(I.$$.fragment),Vt=a(),L=r("p"),L.innerHTML=Ke,xt=a(),d(H.$$.fragment),Yt=a(),V=r("table"),V.innerHTML=tl,St=a(),d(x.$$.fragment),Pt=a(),Y=r("table"),Y.innerHTML=el,Dt=a(),S=r("blockquote"),S.innerHTML=ll,Ot=a(),d(P.$$.fragment),Kt=a(),D=r("p"),D.innerHTML=nl,te=a(),d(z.$$.fragment),ee=a(),d(q.$$.fragment),le=a(),O=r("p"),O.innerHTML=al,ne=a(),d(K.$$.fragment),ae=a(),tt=r("p"),tt.innerHTML=sl,se=a(),d(et.$$.fragment),ie=a(),d(lt.$$.fragment),oe=a(),nt=r("p"),nt.textContent=il,re=a(),d(at.$$.fragment),me=a(),st=r("p"),st.innerHTML=ol,de=a(),d(it.$$.fragment),pe=a(),ot=r("p"),ot.innerHTML=rl,ue=a(),d(rt.$$.fragment),fe=a(),mt=r("p"),mt.innerHTML=ml,be=a(),d(dt.$$.fragment),ce=a(),d(pt.$$.fragment),ye=a(),ut=r("p"),ut.textContent=dl,he=a(),ft=r("p"),ft.innerHTML=pl,Me=a(),d(bt.$$.fragment),ge=a(),d(ct.$$.fragment),Te=a(),yt=r("p"),yt.innerHTML=ul,Je=a(),d(ht.$$.fragment),we=a(),d(Mt.$$.fragment),$e=a(),gt=r("p"),gt.innerHTML=fl,Ue=a(),d(Tt.$$.fragment),ve=a(),Jt=r("p"),Jt.textContent=bl,Ce=a(),d(wt.$$.fragment),_e=a(),$t=r("p"),$t.innerHTML=cl,Ze=a(),d(Ut.$$.fragment),je=a(),d(vt.$$.fragment),Be=a(),Ct=r("p"),Ct.innerHTML=yl,Re=a(),d(_t.$$.fragment),Ne=a(),Zt=r("p"),Zt.innerHTML=hl,Xe=a(),d(jt.$$.fragment),We=a(),Bt=r("p"),Bt.innerHTML=Ml,ze=a(),d(Rt.$$.fragment),qe=a(),d(Nt.$$.fragment),ke=a(),Xt=r("p"),Xt.innerHTML=gl,Ee=a(),d(Wt.$$.fragment),Ge=a(),d(zt.$$.fragment),Fe=a(),qt=r("p"),qt.innerHTML=Tl,Ae=a(),kt=r("p"),kt.innerHTML=Jl,Qe=a(),d(Et.$$.fragment),Ie=a(),Gt=r("p"),this.h()},l(t){const e=Rl("svelte-u9bgzb",document.head);i=m(e,"META",{name:!0,content:!0}),e.forEach(l),$=s(t),y=m(t,"P",{}),He(y).forEach(l),U=s(t),p(o.$$.fragment,t),v=s(t),C=m(t,"P",{"data-svelte-h":!0}),h(C)!=="svelte-cljyyt"&&(C.innerHTML=J),B=s(t),w=m(t,"P",{"data-svelte-h":!0}),h(w)!=="svelte-2si3vm"&&(w.textContent=W),R=s(t),T=m(t,"UL",{"data-svelte-h":!0}),h(T)!=="svelte-1ffoq2t"&&(T.innerHTML=N),g=s(t),M=m(t,"P",{"data-svelte-h":!0}),h(M)!=="svelte-1tdjs4o"&&(M.textContent=Ye),Ft=s(t),E=m(t,"OL",{"data-svelte-h":!0}),h(E)!=="svelte-1quaasy"&&(E.innerHTML=Se),At=s(t),G=m(t,"BLOCKQUOTE",{"data-svelte-h":!0}),h(G)!=="svelte-18yyxaf"&&(G.innerHTML=Pe),Qt=s(t),F=m(t,"P",{"data-svelte-h":!0}),h(F)!=="svelte-nn1zee"&&(F.textContent=De),It=s(t),p(A.$$.fragment,t),Lt=s(t),Q=m(t,"P",{"data-svelte-h":!0}),h(Q)!=="svelte-yeot1a"&&(Q.innerHTML=Oe),Ht=s(t),p(I.$$.fragment,t),Vt=s(t),L=m(t,"P",{"data-svelte-h":!0}),h(L)!=="svelte-15q4hgs"&&(L.innerHTML=Ke),xt=s(t),p(H.$$.fragment,t),Yt=s(t),V=m(t,"TABLE",{"data-svelte-h":!0}),h(V)!=="svelte-13d7wmf"&&(V.innerHTML=tl),St=s(t),p(x.$$.fragment,t),Pt=s(t),Y=m(t,"TABLE",{"data-svelte-h":!0}),h(Y)!=="svelte-1bw3j6b"&&(Y.innerHTML=el),Dt=s(t),S=m(t,"BLOCKQUOTE",{"data-svelte-h":!0}),h(S)!=="svelte-zrh954"&&(S.innerHTML=ll),Ot=s(t),p(P.$$.fragment,t),Kt=s(t),D=m(t,"P",{"data-svelte-h":!0}),h(D)!=="svelte-nvcemd"&&(D.innerHTML=nl),te=s(t),p(z.$$.fragment,t),ee=s(t),p(q.$$.fragment,t),le=s(t),O=m(t,"P",{"data-svelte-h":!0}),h(O)!=="svelte-3u2pq"&&(O.innerHTML=al),ne=s(t),p(K.$$.fragment,t),ae=s(t),tt=m(t,"P",{"data-svelte-h":!0}),h(tt)!=="svelte-5ab1q1"&&(tt.innerHTML=sl),se=s(t),p(et.$$.fragment,t),ie=s(t),p(lt.$$.fragment,t),oe=s(t),nt=m(t,"P",{"data-svelte-h":!0}),h(nt)!=="svelte-9650w3"&&(nt.textContent=il),re=s(t),p(at.$$.fragment,t),me=s(t),st=m(t,"P",{"data-svelte-h":!0}),h(st)!=="svelte-1jt3bby"&&(st.innerHTML=ol),de=s(t),p(it.$$.fragment,t),pe=s(t),ot=m(t,"P",{"data-svelte-h":!0}),h(ot)!=="svelte-80v3vx"&&(ot.innerHTML=rl),ue=s(t),p(rt.$$.fragment,t),fe=s(t),mt=m(t,"P",{"data-svelte-h":!0}),h(mt)!=="svelte-1oaka12"&&(mt.innerHTML=ml),be=s(t),p(dt.$$.fragment,t),ce=s(t),p(pt.$$.fragment,t),ye=s(t),ut=m(t,"P",{"data-svelte-h":!0}),h(ut)!=="svelte-ur5rgd"&&(ut.textContent=dl),he=s(t),ft=m(t,"P",{"data-svelte-h":!0}),h(ft)!=="svelte-1jcweb0"&&(ft.innerHTML=pl),Me=s(t),p(bt.$$.fragment,t),ge=s(t),p(ct.$$.fragment,t),Te=s(t),yt=m(t,"P",{"data-svelte-h":!0}),h(yt)!=="svelte-1f7c9ag"&&(yt.innerHTML=ul),Je=s(t),p(ht.$$.fragment,t),we=s(t),p(Mt.$$.fragment,t),$e=s(t),gt=m(t,"P",{"data-svelte-h":!0}),h(gt)!=="svelte-18qfkz8"&&(gt.innerHTML=fl),Ue=s(t),p(Tt.$$.fragment,t),ve=s(t),Jt=m(t,"P",{"data-svelte-h":!0}),h(Jt)!=="svelte-1b1i0z0"&&(Jt.textContent=bl),Ce=s(t),p(wt.$$.fragment,t),_e=s(t),$t=m(t,"P",{"data-svelte-h":!0}),h($t)!=="svelte-cv16qb"&&($t.innerHTML=cl),Ze=s(t),p(Ut.$$.fragment,t),je=s(t),p(vt.$$.fragment,t),Be=s(t),Ct=m(t,"P",{"data-svelte-h":!0}),h(Ct)!=="svelte-vfa0sb"&&(Ct.innerHTML=yl),Re=s(t),p(_t.$$.fragment,t),Ne=s(t),Zt=m(t,"P",{"data-svelte-h":!0}),h(Zt)!=="svelte-1qoc2ct"&&(Zt.innerHTML=hl),Xe=s(t),p(jt.$$.fragment,t),We=s(t),Bt=m(t,"P",{"data-svelte-h":!0}),h(Bt)!=="svelte-1jt2w56"&&(Bt.innerHTML=Ml),ze=s(t),p(Rt.$$.fragment,t),qe=s(t),p(Nt.$$.fragment,t),ke=s(t),Xt=m(t,"P",{"data-svelte-h":!0}),h(Xt)!=="svelte-192l2ej"&&(Xt.innerHTML=gl),Ee=s(t),p(Wt.$$.fragment,t),Ge=s(t),p(zt.$$.fragment,t),Fe=s(t),qt=m(t,"P",{"data-svelte-h":!0}),h(qt)!=="svelte-km4860"&&(qt.innerHTML=Tl),Ae=s(t),kt=m(t,"P",{"data-svelte-h":!0}),h(kt)!=="svelte-96sk4r"&&(kt.innerHTML=Jl),Qe=s(t),p(Et.$$.fragment,t),Ie=s(t),Gt=m(t,"P",{}),He(Gt).forEach(l),this.h()},h(){Ve(i,"name","hf:doc:metadata"),Ve(i,"content",Fl)},m(t,e){j(document.head,i),n(t,$,e),n(t,y,e),n(t,U,e),u(o,t,e),n(t,v,e),n(t,C,e),n(t,B,e),n(t,w,e),n(t,R,e),n(t,T,e),n(t,g,e),n(t,M,e),n(t,Ft,e),n(t,E,e),n(t,At,e),n(t,G,e),n(t,Qt,e),n(t,F,e),n(t,It,e),u(A,t,e),n(t,Lt,e),n(t,Q,e),n(t,Ht,e),u(I,t,e),n(t,Vt,e),n(t,L,e),n(t,xt,e),u(H,t,e),n(t,Yt,e),n(t,V,e),n(t,St,e),u(x,t,e),n(t,Pt,e),n(t,Y,e),n(t,Dt,e),n(t,S,e),n(t,Ot,e),u(P,t,e),n(t,Kt,e),n(t,D,e),n(t,te,e),u(z,t,e),n(t,ee,e),u(q,t,e),n(t,le,e),n(t,O,e),n(t,ne,e),u(K,t,e),n(t,ae,e),n(t,tt,e),n(t,se,e),u(et,t,e),n(t,ie,e),u(lt,t,e),n(t,oe,e),n(t,nt,e),n(t,re,e),u(at,t,e),n(t,me,e),n(t,st,e),n(t,de,e),u(it,t,e),n(t,pe,e),n(t,ot,e),n(t,ue,e),u(rt,t,e),n(t,fe,e),n(t,mt,e),n(t,be,e),u(dt,t,e),n(t,ce,e),u(pt,t,e),n(t,ye,e),n(t,ut,e),n(t,he,e),n(t,ft,e),n(t,Me,e),u(bt,t,e),n(t,ge,e),u(ct,t,e),n(t,Te,e),n(t,yt,e),n(t,Je,e),u(ht,t,e),n(t,we,e),u(Mt,t,e),n(t,$e,e),n(t,gt,e),n(t,Ue,e),u(Tt,t,e),n(t,ve,e),n(t,Jt,e),n(t,Ce,e),u(wt,t,e),n(t,_e,e),n(t,$t,e),n(t,Ze,e),u(Ut,t,e),n(t,je,e),u(vt,t,e),n(t,Be,e),n(t,Ct,e),n(t,Re,e),u(_t,t,e),n(t,Ne,e),n(t,Zt,e),n(t,Xe,e),u(jt,t,e),n(t,We,e),n(t,Bt,e),n(t,ze,e),u(Rt,t,e),n(t,qe,e),u(Nt,t,e),n(t,ke,e),n(t,Xt,e),n(t,Ee,e),u(Wt,t,e),n(t,Ge,e),u(zt,t,e),n(t,Fe,e),n(t,qt,e),n(t,Ae,e),n(t,kt,e),n(t,Qe,e),u(Et,t,e),n(t,Ie,e),n(t,Gt,e),Le=!0},p(t,[e]){const wl={};e&2&&(wl.$$scope={dirty:e,ctx:t}),z.$set(wl);const $l={};e&2&&($l.$$scope={dirty:e,ctx:t}),q.$set($l)},i(t){Le||(f(o.$$.fragment,t),f(A.$$.fragment,t),f(I.$$.fragment,t),f(H.$$.fragment,t),f(x.$$.fragment,t),f(P.$$.fragment,t),f(z.$$.fragment,t),f(q.$$.fragment,t),f(K.$$.fragment,t),f(et.$$.fragment,t),f(lt.$$.fragment,t),f(at.$$.fragment,t),f(it.$$.fragment,t),f(rt.$$.fragment,t),f(dt.$$.fragment,t),f(pt.$$.fragment,t),f(bt.$$.fragment,t),f(ct.$$.fragment,t),f(ht.$$.fragment,t),f(Mt.$$.fragment,t),f(Tt.$$.fragment,t),f(wt.$$.fragment,t),f(Ut.$$.fragment,t),f(vt.$$.fragment,t),f(_t.$$.fragment,t),f(jt.$$.fragment,t),f(Rt.$$.fragment,t),f(Nt.$$.fragment,t),f(Wt.$$.fragment,t),f(zt.$$.fragment,t),f(Et.$$.fragment,t),Le=!0)},o(t){b(o.$$.fragment,t),b(A.$$.fragment,t),b(I.$$.fragment,t),b(H.$$.fragment,t),b(x.$$.fragment,t),b(P.$$.fragment,t),b(z.$$.fragment,t),b(q.$$.fragment,t),b(K.$$.fragment,t),b(et.$$.fragment,t),b(lt.$$.fragment,t),b(at.$$.fragment,t),b(it.$$.fragment,t),b(rt.$$.fragment,t),b(dt.$$.fragment,t),b(pt.$$.fragment,t),b(bt.$$.fragment,t),b(ct.$$.fragment,t),b(ht.$$.fragment,t),b(Mt.$$.fragment,t),b(Tt.$$.fragment,t),b(wt.$$.fragment,t),b(Ut.$$.fragment,t),b(vt.$$.fragment,t),b(_t.$$.fragment,t),b(jt.$$.fragment,t),b(Rt.$$.fragment,t),b(Nt.$$.fragment,t),b(Wt.$$.fragment,t),b(zt.$$.fragment,t),b(Et.$$.fragment,t),Le=!1},d(t){t&&(l($),l(y),l(U),l(v),l(C),l(B),l(w),l(R),l(T),l(g),l(M),l(Ft),l(E),l(At),l(G),l(Qt),l(F),l(It),l(Lt),l(Q),l(Ht),l(Vt),l(L),l(xt),l(Yt),l(V),l(St),l(Pt),l(Y),l(Dt),l(S),l(Ot),l(Kt),l(D),l(te),l(ee),l(le),l(O),l(ne),l(ae),l(tt),l(se),l(ie),l(oe),l(nt),l(re),l(me),l(st),l(de),l(pe),l(ot),l(ue),l(fe),l(mt),l(be),l(ce),l(ye),l(ut),l(he),l(ft),l(Me),l(ge),l(Te),l(yt),l(Je),l(we),l($e),l(gt),l(Ue),l(ve),l(Jt),l(Ce),l(_e),l($t),l(Ze),l(je),l(Be),l(Ct),l(Re),l(Ne),l(Zt),l(Xe),l(We),l(Bt),l(ze),l(qe),l(ke),l(Xt),l(Ee),l(Ge),l(Fe),l(qt),l(Ae),l(kt),l(Qe),l(Ie),l(Gt)),l(i),c(o,t),c(A,t),c(I,t),c(H,t),c(x,t),c(P,t),c(z,t),c(q,t),c(K,t),c(et,t),c(lt,t),c(at,t),c(it,t),c(rt,t),c(dt,t),c(pt,t),c(bt,t),c(ct,t),c(ht,t),c(Mt,t),c(Tt,t),c(wt,t),c(Ut,t),c(vt,t),c(_t,t),c(jt,t),c(Rt,t),c(Nt,t),c(Wt,t),c(zt,t),c(Et,t)}}}const Fl='{"title":"Bitsandbytes","local":"bitsandbytes","sections":[{"title":"Hardware Compatibility","local":"hardware-compatibility","sections":[{"title":"CUDA","local":"cuda","sections":[],"depth":3},{"title":"Multi-backend","local":"multi-backend","sections":[],"depth":3}],"depth":2},{"title":"Quantization Examples","local":"quantization-examples","sections":[],"depth":2},{"title":"LLM.int8","local":"llmint8","sections":[{"title":"Offloading","local":"offloading","sections":[],"depth":3},{"title":"Outlier threshold","local":"outlier-threshold","sections":[],"depth":3},{"title":"Skip module conversion","local":"skip-module-conversion","sections":[],"depth":3},{"title":"Finetuning","local":"finetuning","sections":[],"depth":3}],"depth":2},{"title":"QLoRA","local":"qlora","sections":[{"title":"Compute data type","local":"compute-data-type","sections":[],"depth":3},{"title":"Normal Float 4 (NF4)","local":"normal-float-4-nf4","sections":[],"depth":3},{"title":"Nested quantization","local":"nested-quantization","sections":[],"depth":3}],"depth":2},{"title":"Dequantizing bitsandbytes models","local":"dequantizing-bitsandbytes-models","sections":[],"depth":2},{"title":"Resources","local":"resources","sections":[],"depth":2}],"depth":1}';function Al(X){return Zl(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Sl extends jl{constructor(i){super(),Bl(this,i,Al,Gl,_l,{})}}export{Sl as component};
