import{s as le,n as re,o as se}from"../chunks/scheduler.25b97de1.js";import{S as me,i as de,g as r,s as o,r as m,A as pe,h as s,f as n,c as i,j as oe,u as d,x as J,k as ie,y as fe,a,v as p,d as f,t as c,w as k}from"../chunks/index.d9030fc9.js";import{C as N}from"../chunks/CodeBlock.e6cd0d95.js";import{H as Y,E as ce}from"../chunks/EditOnGithub.91d95064.js";function ke(K){let l,H,U,B,h,E,u,O=`Support for tiktoken model files is seamlessly integrated in 🤗 transformers when loading models
<code>from_pretrained</code> with a <code>tokenizer.model</code> tiktoken file on the Hub, which is automatically converted into our
<a href="https://huggingface.co/docs/transformers/main/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast" rel="nofollow">fast tokenizer</a>.`,W,M,C,b,D="<li>gpt2</li> <li>llama3</li>",G,$,I,w,ee=`In order to load <code>tiktoken</code> files in <code>transformers</code>, ensure that the <code>tokenizer.model</code> file is a tiktoken file and it
will automatically be loaded when loading <code>from_pretrained</code>. Here is how one would load a tokenizer and a model, which
can be loaded from the exact same file:`,L,g,X,y,x,T,te='The <code>tokenizer.model</code> file contains no information about additional tokens or pattern strings. If these are important, convert the tokenizer to <code>tokenizer.json</code>, the appropriate format for <a href="/docs/transformers/main/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a>.',F,j,ne='Generate the <code>tokenizer.model</code> file with <a href="https://github.com/openai/tiktoken/blob/63527649963def8c759b0f91f2eb69a40934e468/tiktoken/registry.py#L63" rel="nofollow">tiktoken.get_encoding</a> and then convert it to <code>tokenizer.json</code> with <code>convert_tiktoken_to_fast</code>.',P,v,V,_,ae='The resulting <code>tokenizer.json</code> file is saved to the specified directory and can be loaded with <a href="/docs/transformers/main/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a>.',q,Z,Q,z,A,R,S;return h=new Y({props:{title:"Tiktoken and interaction with Transformers",local:"tiktoken-and-interaction-with-transformers",headingTag:"h1"}}),M=new Y({props:{title:"Known models that were released with a tiktoken.model :",local:"known-models-that-were-released-with-a-tiktokenmodel-",headingTag:"h3"}}),$=new Y({props:{title:"Example usage",local:"example-usage",headingTag:"h2"}}),g=new N({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEFtb2RlbF9pZCUyMCUzRCUyMCUyMm1ldGEtbGxhbWElMkZNZXRhLUxsYW1hLTMtOEItSW5zdHJ1Y3QlMjIlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChtb2RlbF9pZCUyQyUyMHN1YmZvbGRlciUzRCUyMm9yaWdpbmFsJTIyKSUyMA==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

model_id = <span class="hljs-string">&quot;meta-llama/Meta-Llama-3-8B-Instruct&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_id, subfolder=<span class="hljs-string">&quot;original&quot;</span>) `,wrap:!1}}),y=new Y({props:{title:"Create tiktoken tokenizer",local:"create-tiktoken-tokenizer",headingTag:"h2"}}),v=new N({props:{code:"JTBBZnJvbSUyMHRyYW5zZm9ybWVycy5pbnRlZ3JhdGlvbnMudGlrdG9rZW4lMjBpbXBvcnQlMjBjb252ZXJ0X3Rpa3Rva2VuX3RvX2Zhc3QlMEFmcm9tJTIwdGlrdG9rZW4lMjBpbXBvcnQlMjBnZXRfZW5jb2RpbmclMEElMEElMjMlMjBZb3UlMjBjYW4lMjBsb2FkJTIweW91ciUyMGN1c3RvbSUyMGVuY29kaW5nJTIwb3IlMjB0aGUlMjBvbmUlMjBwcm92aWRlZCUyMGJ5JTIwT3BlbkFJJTBBZW5jb2RpbmclMjAlM0QlMjBnZXRfZW5jb2RpbmcoJTIyZ3B0MiUyMiklMEFjb252ZXJ0X3Rpa3Rva2VuX3RvX2Zhc3QoZW5jb2RpbmclMkMlMjAlMjJjb25maWclMkZzYXZlJTJGZGlyJTIyKQ==",highlighted:`
<span class="hljs-keyword">from</span> transformers.integrations.tiktoken <span class="hljs-keyword">import</span> convert_tiktoken_to_fast
<span class="hljs-keyword">from</span> tiktoken <span class="hljs-keyword">import</span> get_encoding

<span class="hljs-comment"># You can load your custom encoding or the one provided by OpenAI</span>
encoding = get_encoding(<span class="hljs-string">&quot;gpt2&quot;</span>)
convert_tiktoken_to_fast(encoding, <span class="hljs-string">&quot;config/save/dir&quot;</span>)`,wrap:!1}}),Z=new N({props:{code:"dG9rZW5pemVyJTIwJTNEJTIwUHJlVHJhaW5lZFRva2VuaXplckZhc3QuZnJvbV9wcmV0cmFpbmVkKCUyMmNvbmZpZyUyRnNhdmUlMkZkaXIlMjIp",highlighted:'tokenizer = PreTrainedTokenizerFast.from_pretrained(<span class="hljs-string">&quot;config/save/dir&quot;</span>)',wrap:!1}}),z=new ce({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/tiktoken.md"}}),{c(){l=r("meta"),H=o(),U=r("p"),B=o(),m(h.$$.fragment),E=o(),u=r("p"),u.innerHTML=O,W=o(),m(M.$$.fragment),C=o(),b=r("ul"),b.innerHTML=D,G=o(),m($.$$.fragment),I=o(),w=r("p"),w.innerHTML=ee,L=o(),m(g.$$.fragment),X=o(),m(y.$$.fragment),x=o(),T=r("p"),T.innerHTML=te,F=o(),j=r("p"),j.innerHTML=ne,P=o(),m(v.$$.fragment),V=o(),_=r("p"),_.innerHTML=ae,q=o(),m(Z.$$.fragment),Q=o(),m(z.$$.fragment),A=o(),R=r("p"),this.h()},l(e){const t=pe("svelte-u9bgzb",document.head);l=s(t,"META",{name:!0,content:!0}),t.forEach(n),H=i(e),U=s(e,"P",{}),oe(U).forEach(n),B=i(e),d(h.$$.fragment,e),E=i(e),u=s(e,"P",{"data-svelte-h":!0}),J(u)!=="svelte-zb5ptn"&&(u.innerHTML=O),W=i(e),d(M.$$.fragment,e),C=i(e),b=s(e,"UL",{"data-svelte-h":!0}),J(b)!=="svelte-tocjh9"&&(b.innerHTML=D),G=i(e),d($.$$.fragment,e),I=i(e),w=s(e,"P",{"data-svelte-h":!0}),J(w)!=="svelte-1k7duvc"&&(w.innerHTML=ee),L=i(e),d(g.$$.fragment,e),X=i(e),d(y.$$.fragment,e),x=i(e),T=s(e,"P",{"data-svelte-h":!0}),J(T)!=="svelte-12ktbv1"&&(T.innerHTML=te),F=i(e),j=s(e,"P",{"data-svelte-h":!0}),J(j)!=="svelte-1bw63x8"&&(j.innerHTML=ne),P=i(e),d(v.$$.fragment,e),V=i(e),_=s(e,"P",{"data-svelte-h":!0}),J(_)!=="svelte-66oz31"&&(_.innerHTML=ae),q=i(e),d(Z.$$.fragment,e),Q=i(e),d(z.$$.fragment,e),A=i(e),R=s(e,"P",{}),oe(R).forEach(n),this.h()},h(){ie(l,"name","hf:doc:metadata"),ie(l,"content",he)},m(e,t){fe(document.head,l),a(e,H,t),a(e,U,t),a(e,B,t),p(h,e,t),a(e,E,t),a(e,u,t),a(e,W,t),p(M,e,t),a(e,C,t),a(e,b,t),a(e,G,t),p($,e,t),a(e,I,t),a(e,w,t),a(e,L,t),p(g,e,t),a(e,X,t),p(y,e,t),a(e,x,t),a(e,T,t),a(e,F,t),a(e,j,t),a(e,P,t),p(v,e,t),a(e,V,t),a(e,_,t),a(e,q,t),p(Z,e,t),a(e,Q,t),p(z,e,t),a(e,A,t),a(e,R,t),S=!0},p:re,i(e){S||(f(h.$$.fragment,e),f(M.$$.fragment,e),f($.$$.fragment,e),f(g.$$.fragment,e),f(y.$$.fragment,e),f(v.$$.fragment,e),f(Z.$$.fragment,e),f(z.$$.fragment,e),S=!0)},o(e){c(h.$$.fragment,e),c(M.$$.fragment,e),c($.$$.fragment,e),c(g.$$.fragment,e),c(y.$$.fragment,e),c(v.$$.fragment,e),c(Z.$$.fragment,e),c(z.$$.fragment,e),S=!1},d(e){e&&(n(H),n(U),n(B),n(E),n(u),n(W),n(C),n(b),n(G),n(I),n(w),n(L),n(X),n(x),n(T),n(F),n(j),n(P),n(V),n(_),n(q),n(Q),n(A),n(R)),n(l),k(h,e),k(M,e),k($,e),k(g,e),k(y,e),k(v,e),k(Z,e),k(z,e)}}}const he='{"title":"Tiktoken and interaction with Transformers","local":"tiktoken-and-interaction-with-transformers","sections":[{"title":"Known models that were released with a tiktoken.model :","local":"known-models-that-were-released-with-a-tiktokenmodel-","sections":[],"depth":3},{"title":"Example usage","local":"example-usage","sections":[],"depth":2},{"title":"Create tiktoken tokenizer","local":"create-tiktoken-tokenizer","sections":[],"depth":2}],"depth":1}';function ue(K){return se(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ge extends me{constructor(l){super(),de(this,l,ue,ke,le,{})}}export{ge as component};
