import{s as Tt,n as bt,o as vt}from"../chunks/scheduler.25b97de1.js";import{S as xt,i as $t,g as r,s as o,r as p,A as wt,h as a,f as n,c as s,j as T,u as f,x as m,k as b,y as t,a as d,v as _,d as g,t as u,w as h}from"../chunks/index.d9030fc9.js";import{D as $}from"../chunks/Docstring.ffac8efa.js";import{H as Ie,E as zt}from"../chunks/EditOnGithub.91d95064.js";function Mt(tt){let w,_e,pe,ge,C,ue,E,he,I,nt=`The myt5 model was proposed in <a href="https://arxiv.org/pdf/2403.10691.pdf" rel="nofollow">MYTE: Morphology-Driven Byte Encoding for Better and Fairer Multilingual Language Modeling</a> by Tomasz Limisiewicz, Terra Blevins, Hila Gonen, Orevaoghene Ahia, and Luke Zettlemoyer.
MyT5 (<strong>My</strong>te <strong>T5</strong>) is a multilingual language model based on T5 architecture.
The model uses a <strong>m</strong>orphologically-driven <strong>byte</strong> (<strong>MYTE</strong>) representation described in our paper.
<strong>MYTE</strong> uses codepoints corresponding to morphemes in contrast to characters used in UTF-8 encoding.
As a pre-requisite, we used unsupervised morphological segmentation (<a href="https://aclanthology.org/E14-2006.pdf" rel="nofollow">Morfessor</a>) to obtain morpheme inventories for 99 languages.
However, the morphological segmentation step is not needed when using the pre-defined morpheme inventory from the hub (see: <a href="https://huggingface.co/Tomlim/myt5-base" rel="nofollow">Tomli/myt5-base</a>).`,ke,P,ot="The abstract from the paper is the following:",ye,H,st="<em>A major consideration in multilingual language modeling is how to best represent languages with diverse vocabularies and scripts. Although contemporary text encoding methods cover most of the world’s writing systems, they exhibit bias towards the high-resource languages of the Global West. As a result, texts of underrepresented languages tend to be segmented into long sequences of linguistically meaningless units. To address the disparities, we introduce a new paradigm that encodes the same information with segments of consistent size across diverse languages. Our encoding convention (MYTE) is based on morphemes, as their inventories are more balanced across languages than characters, which are used in previous methods. We show that MYTE produces shorter encodings for all 99 analyzed languages, with the most notable improvements for non-European languages and non-Latin scripts. This, in turn, improves multilingual LM performance and diminishes the perplexity gap throughout diverse languages.</em>",Te,A,rt=`This model was contributed by <a href="https://huggingface.co/Tomlim" rel="nofollow">Tomasz Limisiewicz</a>.
The original code can be found <a href="https://github.com/tomlimi/MYTE" rel="nofollow">here</a>.`,be,O,ve,l,V,Pe,Z,at="Construct a MyT5 tokenizer.",He,J,it=`This tokenizer inherits from <a href="/docs/transformers/main/en/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> which contains most of the main methods. Users should refer to
this superclass for more information regarding those methods.`,Ae,v,B,Oe,K,dt=`Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
adding special tokens. A sequence has the following format:`,Ve,Q,lt="<li>single sequence: <code>X &lt;/s&gt;</code></li> <li>pair of sequences: <code>A &lt;/s&gt; B &lt;/s&gt;</code></li>",Be,z,N,Ne,ee,ct=`Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
special tokens using the tokenizer <code>prepare_for_model</code> method.`,je,M,j,Fe,te,mt=`Create a mask from the two sequences passed to be used in a sequence-pair classification task. MyT5 does not
make use of token type ids, therefore a list of zeros is returned.`,Ue,ne,F,xe,U,$e,c,Y,Ye,oe,pt="Construct a MyT5 tokenizer.",We,se,ft=`This tokenizer inherits from <a href="/docs/transformers/main/en/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> which contains most of the main methods. Users should refer to
this superclass for more information regarding those methods.`,Ge,x,W,Re,re,_t=`Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
adding special tokens. A sequence has the following format:`,Se,ae,gt="<li>single sequence: <code>X &lt;/s&gt;</code></li> <li>pair of sequences: <code>A &lt;/s&gt; B &lt;/s&gt;</code></li>",Xe,L,G,Ze,ie,ut="Converts a sequence of tokens (string) in a single string.",Je,q,R,Ke,de,ht=`Create a mask from the two sequences passed to be used in a sequence-pair classification task. MyT5 does not
make use of token type ids, therefore a list of zeros is returned.`,Qe,D,S,et,le,kt=`Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
special tokens using the tokenizer <code>prepare_for_model</code> method.`,we,X,ze,fe,Me;return C=new Ie({props:{title:"myt5",local:"myt5",headingTag:"h1"}}),E=new Ie({props:{title:"Overview",local:"overview",headingTag:"h2"}}),O=new Ie({props:{title:"MyT5Tokenizer",local:"transformers.MyT5Tokenizer",headingTag:"h2"}}),V=new $({props:{name:"class transformers.MyT5Tokenizer",anchor:"transformers.MyT5Tokenizer",parameters:[{name:"vocab_file",val:""},{name:"eos_token",val:" = '</s>'"},{name:"unk_token",val:" = '<unk>'"},{name:"pad_token",val:" = '<pad>'"},{name:"extra_ids",val:" = 125"},{name:"additional_special_tokens",val:" = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.MyT5Tokenizer.vocab_file",description:"<strong>vocab_file</strong> (<code>str</code>) &#x2014; The file containing the byte rewriting rules.",name:"vocab_file"},{anchor:"transformers.MyT5Tokenizer.eos_token",description:`<strong>eos_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;/s&gt;&quot;</code>) &#x2014;
The end of sequence token.`,name:"eos_token"},{anchor:"transformers.MyT5Tokenizer.unk_token",description:`<strong>unk_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;unk&gt;&quot;</code>) &#x2014;
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
token instead.`,name:"unk_token"},{anchor:"transformers.MyT5Tokenizer.pad_token",description:`<strong>pad_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;pad&gt;&quot;</code>) &#x2014;
The token used for padding, for example when batching sequences of different lengths.`,name:"pad_token"},{anchor:"transformers.MyT5Tokenizer.extra_ids",description:`<strong>extra_ids</strong> (<code>int</code>, <em>optional</em>, defaults to 125) &#x2014;
Add a number of extra ids added to the end of the vocabulary for use as sentinels. These tokens are
accessible as &#x201C;<extra<em>id{%d}&gt;&#x201D; where &#x201D;{%d}&#x201D; is a number between 0 and extra_ids-1. Extra tokens are
indexed from the end of the vocabulary up to beginning (&#x201C;<extra_id_0>&#x201D; is the last token in the vocabulary
like in ByT5 preprocessing see
<a href="https://github.com/google-research/text-to-text-transfer-transformer/blob/9fd7b14a769417be33bc6c850f9598764913c833/t5/data/preprocessors.py#L2117" rel="nofollow">here</a>).</extra_id_0></extra<em>`,name:"extra_ids"},{anchor:"transformers.MyT5Tokenizer.additional_special_tokens",description:`<strong>additional_special_tokens</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
Additional special tokens used by the tokenizer.`,name:"additional_special_tokens"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/myt5/tokenization_myt5.py#L135"}}),B=new $({props:{name:"build_inputs_with_special_tokens",anchor:"transformers.MyT5Tokenizer.build_inputs_with_special_tokens",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.MyT5Tokenizer.build_inputs_with_special_tokens.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs to which the special tokens will be added.`,name:"token_ids_0"},{anchor:"transformers.MyT5Tokenizer.build_inputs_with_special_tokens.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/myt5/tokenization_myt5.py#L284",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>List of <a href="../glossary#input-ids">input IDs</a> with the appropriate special tokens.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>List[int]</code></p>
`}}),N=new $({props:{name:"get_special_tokens_mask",anchor:"transformers.MyT5Tokenizer.get_special_tokens_mask",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"},{name:"already_has_special_tokens",val:": bool = False"}],parametersDescription:[{anchor:"transformers.MyT5Tokenizer.get_special_tokens_mask.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs.`,name:"token_ids_0"},{anchor:"transformers.MyT5Tokenizer.get_special_tokens_mask.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"},{anchor:"transformers.MyT5Tokenizer.get_special_tokens_mask.already_has_special_tokens",description:`<strong>already_has_special_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not the token list is already formatted with special tokens for the model.`,name:"already_has_special_tokens"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/myt5/tokenization_myt5.py#L222",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>List[int]</code></p>
`}}),j=new $({props:{name:"create_token_type_ids_from_sequences",anchor:"transformers.MyT5Tokenizer.create_token_type_ids_from_sequences",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.MyT5Tokenizer.create_token_type_ids_from_sequences.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs.`,name:"token_ids_0"},{anchor:"transformers.MyT5Tokenizer.create_token_type_ids_from_sequences.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/myt5/tokenization_myt5.py#L261",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>List of zeros.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>List[int]</code></p>
`}}),F=new $({props:{name:"save_vocabulary",anchor:"transformers.MyT5Tokenizer.save_vocabulary",parameters:[{name:"save_directory",val:": str"},{name:"filename_prefix",val:": typing.Optional[str] = None"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/myt5/tokenization_myt5.py#L368"}}),U=new Ie({props:{title:"MyT5Tokenizer",local:"transformers.MyT5Tokenizer",headingTag:"h2"}}),Y=new $({props:{name:"class transformers.MyT5Tokenizer",anchor:"transformers.MyT5Tokenizer",parameters:[{name:"vocab_file",val:""},{name:"eos_token",val:" = '</s>'"},{name:"unk_token",val:" = '<unk>'"},{name:"pad_token",val:" = '<pad>'"},{name:"extra_ids",val:" = 125"},{name:"additional_special_tokens",val:" = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.MyT5Tokenizer.vocab_file",description:"<strong>vocab_file</strong> (<code>str</code>) &#x2014; The file containing the byte rewriting rules.",name:"vocab_file"},{anchor:"transformers.MyT5Tokenizer.eos_token",description:`<strong>eos_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;/s&gt;&quot;</code>) &#x2014;
The end of sequence token.`,name:"eos_token"},{anchor:"transformers.MyT5Tokenizer.unk_token",description:`<strong>unk_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;unk&gt;&quot;</code>) &#x2014;
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
token instead.`,name:"unk_token"},{anchor:"transformers.MyT5Tokenizer.pad_token",description:`<strong>pad_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;pad&gt;&quot;</code>) &#x2014;
The token used for padding, for example when batching sequences of different lengths.`,name:"pad_token"},{anchor:"transformers.MyT5Tokenizer.extra_ids",description:`<strong>extra_ids</strong> (<code>int</code>, <em>optional</em>, defaults to 125) &#x2014;
Add a number of extra ids added to the end of the vocabulary for use as sentinels. These tokens are
accessible as &#x201C;<extra<em>id{%d}&gt;&#x201D; where &#x201D;{%d}&#x201D; is a number between 0 and extra_ids-1. Extra tokens are
indexed from the end of the vocabulary up to beginning (&#x201C;<extra_id_0>&#x201D; is the last token in the vocabulary
like in ByT5 preprocessing see
<a href="https://github.com/google-research/text-to-text-transfer-transformer/blob/9fd7b14a769417be33bc6c850f9598764913c833/t5/data/preprocessors.py#L2117" rel="nofollow">here</a>).</extra_id_0></extra<em>`,name:"extra_ids"},{anchor:"transformers.MyT5Tokenizer.additional_special_tokens",description:`<strong>additional_special_tokens</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
Additional special tokens used by the tokenizer.`,name:"additional_special_tokens"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/myt5/tokenization_myt5.py#L135"}}),W=new $({props:{name:"build_inputs_with_special_tokens",anchor:"transformers.MyT5Tokenizer.build_inputs_with_special_tokens",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.MyT5Tokenizer.build_inputs_with_special_tokens.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs to which the special tokens will be added.`,name:"token_ids_0"},{anchor:"transformers.MyT5Tokenizer.build_inputs_with_special_tokens.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/myt5/tokenization_myt5.py#L284",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>List of <a href="../glossary#input-ids">input IDs</a> with the appropriate special tokens.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>List[int]</code></p>
`}}),G=new $({props:{name:"convert_tokens_to_string",anchor:"transformers.MyT5Tokenizer.convert_tokens_to_string",parameters:[{name:"tokens",val:""}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/myt5/tokenization_myt5.py#L345"}}),R=new $({props:{name:"create_token_type_ids_from_sequences",anchor:"transformers.MyT5Tokenizer.create_token_type_ids_from_sequences",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.MyT5Tokenizer.create_token_type_ids_from_sequences.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs.`,name:"token_ids_0"},{anchor:"transformers.MyT5Tokenizer.create_token_type_ids_from_sequences.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/myt5/tokenization_myt5.py#L261",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>List of zeros.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>List[int]</code></p>
`}}),S=new $({props:{name:"get_special_tokens_mask",anchor:"transformers.MyT5Tokenizer.get_special_tokens_mask",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"},{name:"already_has_special_tokens",val:": bool = False"}],parametersDescription:[{anchor:"transformers.MyT5Tokenizer.get_special_tokens_mask.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs.`,name:"token_ids_0"},{anchor:"transformers.MyT5Tokenizer.get_special_tokens_mask.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"},{anchor:"transformers.MyT5Tokenizer.get_special_tokens_mask.already_has_special_tokens",description:`<strong>already_has_special_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not the token list is already formatted with special tokens for the model.`,name:"already_has_special_tokens"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/myt5/tokenization_myt5.py#L222",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>List[int]</code></p>
`}}),X=new zt({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/model_doc/myt5.md"}}),{c(){w=r("meta"),_e=o(),pe=r("p"),ge=o(),p(C.$$.fragment),ue=o(),p(E.$$.fragment),he=o(),I=r("p"),I.innerHTML=nt,ke=o(),P=r("p"),P.textContent=ot,ye=o(),H=r("p"),H.innerHTML=st,Te=o(),A=r("p"),A.innerHTML=rt,be=o(),p(O.$$.fragment),ve=o(),l=r("div"),p(V.$$.fragment),Pe=o(),Z=r("p"),Z.textContent=at,He=o(),J=r("p"),J.innerHTML=it,Ae=o(),v=r("div"),p(B.$$.fragment),Oe=o(),K=r("p"),K.textContent=dt,Ve=o(),Q=r("ul"),Q.innerHTML=lt,Be=o(),z=r("div"),p(N.$$.fragment),Ne=o(),ee=r("p"),ee.innerHTML=ct,je=o(),M=r("div"),p(j.$$.fragment),Fe=o(),te=r("p"),te.textContent=mt,Ue=o(),ne=r("div"),p(F.$$.fragment),xe=o(),p(U.$$.fragment),$e=o(),c=r("div"),p(Y.$$.fragment),Ye=o(),oe=r("p"),oe.textContent=pt,We=o(),se=r("p"),se.innerHTML=ft,Ge=o(),x=r("div"),p(W.$$.fragment),Re=o(),re=r("p"),re.textContent=_t,Se=o(),ae=r("ul"),ae.innerHTML=gt,Xe=o(),L=r("div"),p(G.$$.fragment),Ze=o(),ie=r("p"),ie.textContent=ut,Je=o(),q=r("div"),p(R.$$.fragment),Ke=o(),de=r("p"),de.textContent=ht,Qe=o(),D=r("div"),p(S.$$.fragment),et=o(),le=r("p"),le.innerHTML=kt,we=o(),p(X.$$.fragment),ze=o(),fe=r("p"),this.h()},l(e){const i=wt("svelte-u9bgzb",document.head);w=a(i,"META",{name:!0,content:!0}),i.forEach(n),_e=s(e),pe=a(e,"P",{}),T(pe).forEach(n),ge=s(e),f(C.$$.fragment,e),ue=s(e),f(E.$$.fragment,e),he=s(e),I=a(e,"P",{"data-svelte-h":!0}),m(I)!=="svelte-z94w65"&&(I.innerHTML=nt),ke=s(e),P=a(e,"P",{"data-svelte-h":!0}),m(P)!=="svelte-vfdo9a"&&(P.textContent=ot),ye=s(e),H=a(e,"P",{"data-svelte-h":!0}),m(H)!=="svelte-5zkjjh"&&(H.innerHTML=st),Te=s(e),A=a(e,"P",{"data-svelte-h":!0}),m(A)!=="svelte-1rzej88"&&(A.innerHTML=rt),be=s(e),f(O.$$.fragment,e),ve=s(e),l=a(e,"DIV",{class:!0});var k=T(l);f(V.$$.fragment,k),Pe=s(k),Z=a(k,"P",{"data-svelte-h":!0}),m(Z)!=="svelte-1sq3vog"&&(Z.textContent=at),He=s(k),J=a(k,"P",{"data-svelte-h":!0}),m(J)!=="svelte-1c3t5ty"&&(J.innerHTML=it),Ae=s(k),v=a(k,"DIV",{class:!0});var ce=T(v);f(B.$$.fragment,ce),Oe=s(ce),K=a(ce,"P",{"data-svelte-h":!0}),m(K)!=="svelte-1wjq39d"&&(K.textContent=dt),Ve=s(ce),Q=a(ce,"UL",{"data-svelte-h":!0}),m(Q)!=="svelte-8gh3n2"&&(Q.innerHTML=lt),ce.forEach(n),Be=s(k),z=a(k,"DIV",{class:!0});var Le=T(z);f(N.$$.fragment,Le),Ne=s(Le),ee=a(Le,"P",{"data-svelte-h":!0}),m(ee)!=="svelte-1f4f5kp"&&(ee.innerHTML=ct),Le.forEach(n),je=s(k),M=a(k,"DIV",{class:!0});var qe=T(M);f(j.$$.fragment,qe),Fe=s(qe),te=a(qe,"P",{"data-svelte-h":!0}),m(te)!=="svelte-1uh0qbi"&&(te.textContent=mt),qe.forEach(n),Ue=s(k),ne=a(k,"DIV",{class:!0});var yt=T(ne);f(F.$$.fragment,yt),yt.forEach(n),k.forEach(n),xe=s(e),f(U.$$.fragment,e),$e=s(e),c=a(e,"DIV",{class:!0});var y=T(c);f(Y.$$.fragment,y),Ye=s(y),oe=a(y,"P",{"data-svelte-h":!0}),m(oe)!=="svelte-1sq3vog"&&(oe.textContent=pt),We=s(y),se=a(y,"P",{"data-svelte-h":!0}),m(se)!=="svelte-1c3t5ty"&&(se.innerHTML=ft),Ge=s(y),x=a(y,"DIV",{class:!0});var me=T(x);f(W.$$.fragment,me),Re=s(me),re=a(me,"P",{"data-svelte-h":!0}),m(re)!=="svelte-1wjq39d"&&(re.textContent=_t),Se=s(me),ae=a(me,"UL",{"data-svelte-h":!0}),m(ae)!=="svelte-8gh3n2"&&(ae.innerHTML=gt),me.forEach(n),Xe=s(y),L=a(y,"DIV",{class:!0});var De=T(L);f(G.$$.fragment,De),Ze=s(De),ie=a(De,"P",{"data-svelte-h":!0}),m(ie)!=="svelte-b3k2yi"&&(ie.textContent=ut),De.forEach(n),Je=s(y),q=a(y,"DIV",{class:!0});var Ce=T(q);f(R.$$.fragment,Ce),Ke=s(Ce),de=a(Ce,"P",{"data-svelte-h":!0}),m(de)!=="svelte-1uh0qbi"&&(de.textContent=ht),Ce.forEach(n),Qe=s(y),D=a(y,"DIV",{class:!0});var Ee=T(D);f(S.$$.fragment,Ee),et=s(Ee),le=a(Ee,"P",{"data-svelte-h":!0}),m(le)!=="svelte-1f4f5kp"&&(le.innerHTML=kt),Ee.forEach(n),y.forEach(n),we=s(e),f(X.$$.fragment,e),ze=s(e),fe=a(e,"P",{}),T(fe).forEach(n),this.h()},h(){b(w,"name","hf:doc:metadata"),b(w,"content",Lt),b(v,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),b(z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),b(M,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),b(ne,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),b(l,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),b(x,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),b(L,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),b(q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),b(D,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),b(c,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,i){t(document.head,w),d(e,_e,i),d(e,pe,i),d(e,ge,i),_(C,e,i),d(e,ue,i),_(E,e,i),d(e,he,i),d(e,I,i),d(e,ke,i),d(e,P,i),d(e,ye,i),d(e,H,i),d(e,Te,i),d(e,A,i),d(e,be,i),_(O,e,i),d(e,ve,i),d(e,l,i),_(V,l,null),t(l,Pe),t(l,Z),t(l,He),t(l,J),t(l,Ae),t(l,v),_(B,v,null),t(v,Oe),t(v,K),t(v,Ve),t(v,Q),t(l,Be),t(l,z),_(N,z,null),t(z,Ne),t(z,ee),t(l,je),t(l,M),_(j,M,null),t(M,Fe),t(M,te),t(l,Ue),t(l,ne),_(F,ne,null),d(e,xe,i),_(U,e,i),d(e,$e,i),d(e,c,i),_(Y,c,null),t(c,Ye),t(c,oe),t(c,We),t(c,se),t(c,Ge),t(c,x),_(W,x,null),t(x,Re),t(x,re),t(x,Se),t(x,ae),t(c,Xe),t(c,L),_(G,L,null),t(L,Ze),t(L,ie),t(c,Je),t(c,q),_(R,q,null),t(q,Ke),t(q,de),t(c,Qe),t(c,D),_(S,D,null),t(D,et),t(D,le),d(e,we,i),_(X,e,i),d(e,ze,i),d(e,fe,i),Me=!0},p:bt,i(e){Me||(g(C.$$.fragment,e),g(E.$$.fragment,e),g(O.$$.fragment,e),g(V.$$.fragment,e),g(B.$$.fragment,e),g(N.$$.fragment,e),g(j.$$.fragment,e),g(F.$$.fragment,e),g(U.$$.fragment,e),g(Y.$$.fragment,e),g(W.$$.fragment,e),g(G.$$.fragment,e),g(R.$$.fragment,e),g(S.$$.fragment,e),g(X.$$.fragment,e),Me=!0)},o(e){u(C.$$.fragment,e),u(E.$$.fragment,e),u(O.$$.fragment,e),u(V.$$.fragment,e),u(B.$$.fragment,e),u(N.$$.fragment,e),u(j.$$.fragment,e),u(F.$$.fragment,e),u(U.$$.fragment,e),u(Y.$$.fragment,e),u(W.$$.fragment,e),u(G.$$.fragment,e),u(R.$$.fragment,e),u(S.$$.fragment,e),u(X.$$.fragment,e),Me=!1},d(e){e&&(n(_e),n(pe),n(ge),n(ue),n(he),n(I),n(ke),n(P),n(ye),n(H),n(Te),n(A),n(be),n(ve),n(l),n(xe),n($e),n(c),n(we),n(ze),n(fe)),n(w),h(C,e),h(E,e),h(O,e),h(V),h(B),h(N),h(j),h(F),h(U,e),h(Y),h(W),h(G),h(R),h(S),h(X,e)}}}const Lt='{"title":"myt5","local":"myt5","sections":[{"title":"Overview","local":"overview","sections":[],"depth":2},{"title":"MyT5Tokenizer","local":"transformers.MyT5Tokenizer","sections":[],"depth":2},{"title":"MyT5Tokenizer","local":"transformers.MyT5Tokenizer","sections":[],"depth":2}],"depth":1}';function qt(tt){return vt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Pt extends xt{constructor(w){super(),$t(this,w,qt,Mt,Tt,{})}}export{Pt as component};
