import{s as ee,o as te,n as ae}from"../chunks/scheduler.01eeda35.js";import{S as le,i as ne,g as i,s as o,r as _,A as oe,h as m,f as a,c as s,j as K,u as J,x as w,k as N,y as se,a as l,v as j,d as k,t as Z,w as L}from"../chunks/index.6dd51b66.js";import{T as re}from"../chunks/Tip.de9bae2b.js";import{C as O}from"../chunks/CodeBlock.19ec9b8c.js";import{H as ie,E as me}from"../chunks/index.58fe8f9d.js";function fe(U){let n,c='Models that support GGUF include Llama, Mistral, Qwen2, Qwen2Moe, Phi3, Bloom, Falcon, StableLM, GPT2, Starcoder2, and <a href="https://github.com/huggingface/transformers/blob/main/src/transformers/integrations/ggml.py" rel="nofollow">more</a>';return{c(){n=i("p"),n.innerHTML=c},l(r){n=m(r,"P",{"data-svelte-h":!0}),w(n)!=="svelte-6rmteu"&&(n.innerHTML=c)},m(r,$){l(r,n,$)},p:ae,d(r){r&&a(n)}}}function pe(U){let n,c,r,$,d,F,u,P='<a href="https://github.com/ggerganov/ggml/blob/master/docs/gguf.md" rel="nofollow">GGUF</a> is a file format used to store models for inference with <a href="https://github.com/ggerganov/ggml" rel="nofollow">GGML</a>, a fast and lightweight inference framework written in C and C++. GGUF is a single-file format containing the model metadata and tensors.',H,f,X='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/gguf-spec.png"/>',C,h,A='The GGUF format also supports many quantized data types (refer to <a href="https://hf.co/docs/hub/en/gguf#quantization-types" rel="nofollow">quantization type table</a> for a complete list of supported quantization types) which saves a significant amount of memory, making inference with large models like Whisper and Llama feasible on local and edge devices.',W,g,Y="Transformers supports loading models stored in the GGUF format for further training or finetuning. The GGUF checkpoint is <strong>dequantized to fp32</strong> where the full model weights are available and compatible with PyTorch.",x,p,B,b,q='Add the <code>gguf_file</code> parameter to <a href="/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> to specify the GGUF file to load.',R,M,E,T,S='Once you’re done tinkering with the model, save and convert it back to the GGUF format with the <a href="https://github.com/ggerganov/llama.cpp/blob/master/convert_hf_to_gguf.py" rel="nofollow">convert-hf-to-gguf.py</a> script.',I,y,Q,G,V,v,z;return d=new ie({props:{title:"GGUF",local:"gguf",headingTag:"h1"}}),p=new re({props:{warning:!1,$$slots:{default:[fe]},$$scope:{ctx:U}}}),M=new O({props:{code:"JTIzJTIwcGlwJTIwaW5zdGFsbCUyMGdndWYlMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBbW9kZWxfaWQlMjAlM0QlMjAlMjJUaGVCbG9rZSUyRlRpbnlMbGFtYS0xLjFCLUNoYXQtdjEuMC1HR1VGJTIyJTBBZmlsZW5hbWUlMjAlM0QlMjAlMjJ0aW55bGxhbWEtMS4xYi1jaGF0LXYxLjAuUTZfSy5nZ3VmJTIyJTBBJTBBdG9yY2hfZHR5cGUlMjAlM0QlMjB0b3JjaC5mbG9hdDMyJTIwJTIzJTIwY291bGQlMjBiZSUyMHRvcmNoLmZsb2F0MTYlMjBvciUyMHRvcmNoLmJmbG9hdDE2JTIwdG9vJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQlMkMlMjBnZ3VmX2ZpbGUlM0RmaWxlbmFtZSklMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZChtb2RlbF9pZCUyQyUyMGdndWZfZmlsZSUzRGZpbGVuYW1lJTJDJTIwdG9yY2hfZHR5cGUlM0R0b3JjaF9kdHlwZSk=",highlighted:`<span class="hljs-comment"># pip install gguf</span>
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM

model_id = <span class="hljs-string">&quot;TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF&quot;</span>
filename = <span class="hljs-string">&quot;tinyllama-1.1b-chat-v1.0.Q6_K.gguf&quot;</span>

torch_dtype = torch.float32 <span class="hljs-comment"># could be torch.float16 or torch.bfloat16 too</span>
tokenizer = AutoTokenizer.from_pretrained(model_id, gguf_file=filename)
model = AutoModelForCausalLM.from_pretrained(model_id, gguf_file=filename, torch_dtype=torch_dtype)`,wrap:!1}}),y=new O({props:{code:"dG9rZW5pemVyLnNhdmVfcHJldHJhaW5lZCglMjJkaXJlY3RvcnklMjIpJTBBbW9kZWwuc2F2ZV9wcmV0cmFpbmVkKCUyMmRpcmVjdG9yeSUyMiklMEElMEEhcHl0aG9uJTIwJTI0JTdCcGF0aF90b19sbGFtYV9jcHAlN0QlMkZjb252ZXJ0LWhmLXRvLWdndWYucHklMjAlMjQlN0JkaXJlY3RvcnklN0Q=",highlighted:`tokenizer.save_pretrained(<span class="hljs-string">&quot;directory&quot;</span>)
model.save_pretrained(<span class="hljs-string">&quot;directory&quot;</span>)

!python \${path_to_llama_cpp}/convert-hf-to-gguf.py \${directory}`,wrap:!1}}),G=new me({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/gguf.md"}}),{c(){n=i("meta"),c=o(),r=i("p"),$=o(),_(d.$$.fragment),F=o(),u=i("p"),u.innerHTML=P,H=o(),f=i("div"),f.innerHTML=X,C=o(),h=i("p"),h.innerHTML=A,W=o(),g=i("p"),g.innerHTML=Y,x=o(),_(p.$$.fragment),B=o(),b=i("p"),b.innerHTML=q,R=o(),_(M.$$.fragment),E=o(),T=i("p"),T.innerHTML=S,I=o(),_(y.$$.fragment),Q=o(),_(G.$$.fragment),V=o(),v=i("p"),this.h()},l(e){const t=oe("svelte-u9bgzb",document.head);n=m(t,"META",{name:!0,content:!0}),t.forEach(a),c=s(e),r=m(e,"P",{}),K(r).forEach(a),$=s(e),J(d.$$.fragment,e),F=s(e),u=m(e,"P",{"data-svelte-h":!0}),w(u)!=="svelte-1j8wxcx"&&(u.innerHTML=P),H=s(e),f=m(e,"DIV",{class:!0,"data-svelte-h":!0}),w(f)!=="svelte-1kvm71c"&&(f.innerHTML=X),C=s(e),h=m(e,"P",{"data-svelte-h":!0}),w(h)!=="svelte-ddcg59"&&(h.innerHTML=A),W=s(e),g=m(e,"P",{"data-svelte-h":!0}),w(g)!=="svelte-1xvohjx"&&(g.innerHTML=Y),x=s(e),J(p.$$.fragment,e),B=s(e),b=m(e,"P",{"data-svelte-h":!0}),w(b)!=="svelte-4zldii"&&(b.innerHTML=q),R=s(e),J(M.$$.fragment,e),E=s(e),T=m(e,"P",{"data-svelte-h":!0}),w(T)!=="svelte-hgylw2"&&(T.innerHTML=S),I=s(e),J(y.$$.fragment,e),Q=s(e),J(G.$$.fragment,e),V=s(e),v=m(e,"P",{}),K(v).forEach(a),this.h()},h(){N(n,"name","hf:doc:metadata"),N(n,"content",ce),N(f,"class","flex justify-center")},m(e,t){se(document.head,n),l(e,c,t),l(e,r,t),l(e,$,t),j(d,e,t),l(e,F,t),l(e,u,t),l(e,H,t),l(e,f,t),l(e,C,t),l(e,h,t),l(e,W,t),l(e,g,t),l(e,x,t),j(p,e,t),l(e,B,t),l(e,b,t),l(e,R,t),j(M,e,t),l(e,E,t),l(e,T,t),l(e,I,t),j(y,e,t),l(e,Q,t),j(G,e,t),l(e,V,t),l(e,v,t),z=!0},p(e,[t]){const D={};t&2&&(D.$$scope={dirty:t,ctx:e}),p.$set(D)},i(e){z||(k(d.$$.fragment,e),k(p.$$.fragment,e),k(M.$$.fragment,e),k(y.$$.fragment,e),k(G.$$.fragment,e),z=!0)},o(e){Z(d.$$.fragment,e),Z(p.$$.fragment,e),Z(M.$$.fragment,e),Z(y.$$.fragment,e),Z(G.$$.fragment,e),z=!1},d(e){e&&(a(c),a(r),a($),a(F),a(u),a(H),a(f),a(C),a(h),a(W),a(g),a(x),a(B),a(b),a(R),a(E),a(T),a(I),a(Q),a(V),a(v)),a(n),L(d,e),L(p,e),L(M,e),L(y,e),L(G,e)}}}const ce='{"title":"GGUF","local":"gguf","sections":[],"depth":1}';function de(U){return te(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Te extends le{constructor(n){super(),ne(this,n,de,pe,ee,{})}}export{Te as component};
