import{s as J,n as B,o as X}from"../chunks/scheduler.25b97de1.js";import{S as Y,i as K,g as u,s as o,r as g,A as Q,h as k,f as a,c as l,j as A,u as y,x as R,k as q,y as V,a as n,v as b,d as M,t as _,w as T}from"../chunks/index.d9030fc9.js";import{C as N}from"../chunks/CodeBlock.e6cd0d95.js";import{H as G,E as O}from"../chunks/EditOnGithub.91d95064.js";function D(W){let i,E,$,U,s,v,r,F=`Support for tiktoken model files is seamlessly integrated in 🤗 transformers when loading models
<code>from_pretrained</code> with a <code>tokenizer.model</code> tiktoken file on the Hub, which is automatically converted into our
<a href="https://huggingface.co/docs/transformers/main/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast" rel="nofollow">fast tokenizer</a>.`,z,m,H,d,I="<li>gpt2</li> <li>llama3</li>",x,f,C,p,S=`In order to load <code>tiktoken</code> files in <code>transformers</code>, ensure that the <code>tokenizer.model</code> file is a tiktoken file and it
will automatically be loaded when loading <code>from_pretrained</code>. Here is how one would load a tokenizer and a model, which
can be loaded from the exact same file:`,L,c,Z,h,j,w,P;return s=new G({props:{title:"Tiktoken and interaction with Transformers",local:"tiktoken-and-interaction-with-transformers",headingTag:"h1"}}),m=new G({props:{title:"Known models that were released with a tiktoken.model :",local:"known-models-that-were-released-with-a-tiktokenmodel-",headingTag:"h3"}}),f=new G({props:{title:"Example usage",local:"example-usage",headingTag:"h2"}}),c=new N({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEFtb2RlbF9pZCUyMCUzRCUyMCUyMm1ldGEtbGxhbWElMkZNZXRhLUxsYW1hLTMtOEItSW5zdHJ1Y3QlMjIlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChtb2RlbF9pZCUyQyUyMHN1YmZvbGRlciUzRCUyMm9yaWdpbmFsJTIyKSUyMA==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

model_id = <span class="hljs-string">&quot;meta-llama/Meta-Llama-3-8B-Instruct&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_id, subfolder=<span class="hljs-string">&quot;original&quot;</span>) `,wrap:!1}}),h=new O({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/tiktoken.md"}}),{c(){i=u("meta"),E=o(),$=u("p"),U=o(),g(s.$$.fragment),v=o(),r=u("p"),r.innerHTML=F,z=o(),g(m.$$.fragment),H=o(),d=u("ul"),d.innerHTML=I,x=o(),g(f.$$.fragment),C=o(),p=u("p"),p.innerHTML=S,L=o(),g(c.$$.fragment),Z=o(),g(h.$$.fragment),j=o(),w=u("p"),this.h()},l(e){const t=Q("svelte-u9bgzb",document.head);i=k(t,"META",{name:!0,content:!0}),t.forEach(a),E=l(e),$=k(e,"P",{}),A($).forEach(a),U=l(e),y(s.$$.fragment,e),v=l(e),r=k(e,"P",{"data-svelte-h":!0}),R(r)!=="svelte-zb5ptn"&&(r.innerHTML=F),z=l(e),y(m.$$.fragment,e),H=l(e),d=k(e,"UL",{"data-svelte-h":!0}),R(d)!=="svelte-tocjh9"&&(d.innerHTML=I),x=l(e),y(f.$$.fragment,e),C=l(e),p=k(e,"P",{"data-svelte-h":!0}),R(p)!=="svelte-1k7duvc"&&(p.innerHTML=S),L=l(e),y(c.$$.fragment,e),Z=l(e),y(h.$$.fragment,e),j=l(e),w=k(e,"P",{}),A(w).forEach(a),this.h()},h(){q(i,"name","hf:doc:metadata"),q(i,"content",ee)},m(e,t){V(document.head,i),n(e,E,t),n(e,$,t),n(e,U,t),b(s,e,t),n(e,v,t),n(e,r,t),n(e,z,t),b(m,e,t),n(e,H,t),n(e,d,t),n(e,x,t),b(f,e,t),n(e,C,t),n(e,p,t),n(e,L,t),b(c,e,t),n(e,Z,t),b(h,e,t),n(e,j,t),n(e,w,t),P=!0},p:B,i(e){P||(M(s.$$.fragment,e),M(m.$$.fragment,e),M(f.$$.fragment,e),M(c.$$.fragment,e),M(h.$$.fragment,e),P=!0)},o(e){_(s.$$.fragment,e),_(m.$$.fragment,e),_(f.$$.fragment,e),_(c.$$.fragment,e),_(h.$$.fragment,e),P=!1},d(e){e&&(a(E),a($),a(U),a(v),a(r),a(z),a(H),a(d),a(x),a(C),a(p),a(L),a(Z),a(j),a(w)),a(i),T(s,e),T(m,e),T(f,e),T(c,e),T(h,e)}}}const ee='{"title":"Tiktoken and interaction with Transformers","local":"tiktoken-and-interaction-with-transformers","sections":[{"title":"Known models that were released with a tiktoken.model :","local":"known-models-that-were-released-with-a-tiktokenmodel-","sections":[],"depth":3},{"title":"Example usage","local":"example-usage","sections":[],"depth":2}],"depth":1}';function te(W){return X(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ie extends Y{constructor(i){super(),K(this,i,te,D,J,{})}}export{ie as component};
