import{s as dt,o as pt,n as ct}from"../chunks/scheduler.18a86fab.js";import{S as gt,i as ft,g as a,s as o,r as g,A as ut,h as i,f as n,c as r,j as v,u as f,x as c,k as T,y as s,a as m,v as u,d as h,t as k,w as b}from"../chunks/index.98837b22.js";import{T as ht}from"../chunks/Tip.77304350.js";import{D as B}from"../chunks/Docstring.6758ea23.js";import{C as kt}from"../chunks/CodeBlock.8d0c2e8a.js";import{H as Be}from"../chunks/Heading.7e7e0c5b.js";import{E as bt}from"../chunks/index.92f616ba.js";function wt(de){let p,I=`This implementation is the same as BERT, except for tokenization method. Refer to <a href="bert">BERT documentation</a> for
API reference information.`;return{c(){p=a("p"),p.innerHTML=I},l(w){p=i(w,"P",{"data-svelte-h":!0}),c(p)!=="svelte-wqppg4"&&(p.innerHTML=I)},m(w,O){m(w,p,O)},p:ct,d(w){w&&n(p)}}}function Tt(de){let p,I,w,O,q,pe,y,De=`<img alt="PyTorch" src="https://img.shields.io/badge/PyTorch-DE3412?style=flat&amp;logo=pytorch&amp;logoColor=white"/> <img alt="TensorFlow" src="https://img.shields.io/badge/TensorFlow-FF6F00?style=flat&amp;logo=tensorflow&amp;logoColor=white"/> <img alt="Flax" src="https://img.shields.io/badge/Flax-29a79b.svg?style=flat&amp;logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAC0AAAAtCAMAAAANxBKoAAAC7lBMVEUAAADg5vYHPVgAoJH+/v76+v39/f9JbLP///9+AIgAnY3///+mcqzt8fXy9fgkXa3Ax9709fr+///9/f8qXq49qp5AaLGMwrv8/P0eW60VWawxYq8yqJzG2dytt9Wyu9elzci519Lf3O3S2efY3OrY0+Xp7PT///////+dqNCexMc6Z7AGpJeGvbenstPZ5ejQ1OfJzOLa7ejh4+/r8fT29vpccbklWK8PVa0AS6ghW63O498vYa+lsdKz1NDRt9Kw1c672tbD3tnAxt7R6OHp5vDe7OrDyuDn6vLl6/EAQKak0MgATakkppo3ZK/Bz9y8w9yzu9jey97axdvHzeG21NHH4trTwthKZrVGZLSUSpuPQJiGAI+GAI8SWKydycLL4d7f2OTi1+S9xNzL0ePT6OLGzeEAo5U0qJw/aLEAo5JFa7JBabEAp5Y4qZ2QxLyKmsm3kL2xoMOehrRNb7RIbbOZgrGre68AUqwAqZqNN5aKJ5N/lMq+qsd8kMa4pcWzh7muhLMEV69juq2kbKqgUaOTR5uMMZWLLZSGAI5VAIdEAH+ovNDHuNCnxcy3qcaYx8K8msGplrx+wLahjbYdXrV6vbMvYK9DrZ8QrZ8tqJuFms+Sos6sw8ecy8RffsNVeMCvmb43aLltv7Q4Y7EZWK4QWa1gt6meZKUdr6GOAZVeA4xPAISyveLUwtivxtKTpNJ2jcqfvcltiMiwwcfAoMVxhL+Kx7xjdrqTe60tsaNQs6KaRKACrJ6UTZwkqpqTL5pkHY4AloSgsd2ptNXPvNOOncuxxsqFl8lmg8apt8FJcr9EbryGxLqlkrkrY7dRa7ZGZLQ5t6iXUZ6PPpgVpZeJCJFKAIGareTa0+KJod3H0deY2M+esM25usmYu8d2zsJOdcBVvrCLbqcAOaaHaKQAMaScWqKBXqCXMJ2RHpiLF5NmJZAdAHN2kta11dKu1M+DkcZLdb+Mcql3TppyRJdzQ5ZtNZNlIY+DF4+voCOQAAAAZ3RSTlMABAT+MEEJ/RH+/TP+Zlv+pUo6Ifz8+fco/fz6+evr39S9nJmOilQaF/7+/f38+smmoYp6b1T+/v7++vj189zU0tDJxsGzsrKSfv34+Pf27dDOysG9t6+n/vv6+vr59uzr1tG+tZ6Qg9Ym3QAABR5JREFUSMeNlVVUG1EQhpcuxEspXqS0SKEtxQp1d3d332STTRpIQhIISQgJhODu7lAoDoUCpe7u7u7+1puGpqnCPOyZvffbOXPm/PsP9JfQgyCC+tmTABTOcbxDz/heENS7/1F+9nhvkHePG0wNDLbGWwdXL+rbLWvpmZHXD8+gMfBjTh+aSe6Gnn7lwQIOTR0c8wfX3PWgv7avbdKwf/ZoBp1Gp/PvuvXW3vw5ib7emnTW4OR+3D4jB9vjNJ/7gNvfWWeH/TO/JyYrsiKCRjVEZA3UB+96kON+DxOQ/NLE8PE5iUYgIXjFnCOlxEQMaSGVxjg4gxOnEycGz8bptuNjVx08LscIgrzH3umcn+KKtiBIyvzOO2O99aAdR8cF19oZalnCtvREUw79tCd5sow1g1UKM6kXqUx4T8wsi3sTjJ3yzDmmhenLXLpo8u45eG5y4Vvbk6kkC4LLtJMowkSQxmk4ggVJEG+7c6QpHT8vvW9X7/o7+3ELmiJi2mEzZJiz8cT6TBlanBk70cB5GGIGC1gRDdZ00yADLW1FL6gqhtvNXNG5S9gdSrk4M1qu7JAsmYshzDS4peoMrU/gT7qQdqYGZaYhxZmVbGJAm/CS/HloWyhRUlknQ9KYcExTwS80d3VNOxUZJpITYyspl0LbhArhpZCD9cRWEQuhYkNGMHToQ/2Cs6swJlb39CsllxdXX6IUKh/H5jbnSsPKjgmoaFQ1f8wRLR0UnGE/RcDEjj2jXG1WVTwUs8+zxfcrVO+vSsuOpVKxCfYZiQ0/aPKuxQbQ8lIz+DClxC8u+snlcJ7Yr1z1JPqUH0V+GDXbOwAib931Y4Imaq0NTIXPXY+N5L18GJ37SVWu+hwXff8l72Ds9XuwYIBaXPq6Shm4l+Vl/5QiOlV+uTk6YR9PxKsI9xNJny31ygK1e+nIRC1N97EGkFPI+jCpiHe5PCEy7oWqWSwRrpOvhFzcbTWMbm3ZJAOn1rUKpYIt/lDhW/5RHHteeWFN60qo98YJuoq1nK3uW5AabyspC1BcIEpOhft+SZAShYoLSvnmSfnYADUERP5jJn2h5XtsgCRuhYQqAvwTwn33+YWEKUI72HX5AtfSAZDe8F2DtPPm77afhl0EkthzuCQU0BWApgQIH9+KB0JhopMM7bJrdTRoleM2JAVNMyPF+wdoaz+XJpGoVAQ7WXUkcV7gT3oUZyi/ISIJAVKhgNp+4b4veCFhYVJw4locdSjZCp9cPUhLF9EZ3KKzURepMEtCDPP3VcWFx4UIiZIklIpFNfHpdEafIF2aRmOcrUmjohbT2WUllbmRvgfbythbQO3222fpDJoufaQPncYYuqoGtUEsCJZL6/3PR5b4syeSjZMQG/T2maGANlXT2v8S4AULWaUkCxfLyW8iW4kdka+nEMjxpL2NCwsYNBp+Q61PF43zyDg9Bm9+3NNySn78jMZUUkumqE4Gp7JmFOdP1vc8PpRrzj9+wPinCy8K1PiJ4aYbnTYpCCbDkBSbzhu2QJ1Gd82t8jI8TH51+OzvXoWbnXUOBkNW+0mWFwGcGOUVpU81/n3TOHb5oMt2FgYGjzau0Nif0Ss7Q3XB33hjjQHjHA5E5aOyIQc8CBrLdQSs3j92VG+3nNEjbkbdbBr9zm04ruvw37vh0QKOdeGIkckc80fX3KH/h7PT4BOjgCty8VZ5ux1MoO5Cf5naca2LAsEgehI+drX8o/0Nu+W0m6K/I9gGPd/dfx/EN/wN62AhsBWuAAAAAElFTkSuQmCC
"/>`,ce,U,ge,E,We='The BERTweet model was proposed in <a href="https://www.aclweb.org/anthology/2020.emnlp-demos.2.pdf" rel="nofollow">BERTweet: A pre-trained language model for English Tweets</a> by Dat Quoc Nguyen, Thanh Vu, Anh Tuan Nguyen.',fe,L,Oe="The abstract from the paper is the following:",ue,Z,Ye=`<em>We present BERTweet, the first public large-scale pre-trained language model for English Tweets. Our BERTweet, having
the same architecture as BERT-base (Devlin et al., 2019), is trained using the RoBERTa pre-training procedure (Liu et
al., 2019). Experiments show that BERTweet outperforms strong baselines RoBERTa-base and XLM-R-base (Conneau et al.,
2020), producing better performance results than the previous state-of-the-art models on three Tweet NLP tasks:
Part-of-speech tagging, Named-entity recognition and text classification.</em>`,he,j,Ke='This model was contributed by <a href="https://huggingface.co/dqnguyen" rel="nofollow">dqnguyen</a>. The original code can be found <a href="https://github.com/VinAIResearch/BERTweet" rel="nofollow">here</a>.',ke,R,be,V,we,J,Te,P,_e,l,G,Ie,Y,et="Constructs a BERTweet tokenizer, using Byte-Pair-Encoding.",qe,K,tt=`This tokenizer inherits from <a href="/docs/transformers/main/en/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> which contains most of the main methods. Users should refer to
this superclass for more information regarding those methods.`,Ue,x,N,Ee,ee,nt="Loads a pre-existing dictionary from a text file and adds its symbols to this instance.",Le,_,F,Ze,te,st=`Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
adding special tokens. A BERTweet sequence has the following format:`,je,ne,ot="<li>single sequence: <code>&lt;s&gt; X &lt;/s&gt;</code></li> <li>pair of sequences: <code>&lt;s&gt; A &lt;/s&gt;&lt;/s&gt; B &lt;/s&gt;</code></li>",Re,M,Q,Ve,se,rt="Converts a sequence of tokens (string) in a single string.",Pe,z,H,Ge,oe,at=`Create a mask from the two sequences passed to be used in a sequence-pair classification task. BERTweet does
not make use of token type ids, therefore a list of zeros is returned.`,Ne,$,S,Fe,re,it=`Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
special tokens using the tokenizer <code>prepare_for_model</code> method.`,Qe,A,X,He,ae,lt="Normalize tokens in a Tweet",Se,C,D,Xe,ie,mt="Normalize a raw Tweet",ve,W,ye,me,Je;return q=new Be({props:{title:"BERTweet",local:"bertweet",headingTag:"h1"}}),U=new Be({props:{title:"Overview",local:"overview",headingTag:"h2"}}),R=new Be({props:{title:"Usage example",local:"usage-example",headingTag:"h2"}}),V=new kt({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQWJlcnR3ZWV0JTIwJTNEJTIwQXV0b01vZGVsLmZyb21fcHJldHJhaW5lZCglMjJ2aW5haSUyRmJlcnR3ZWV0LWJhc2UlMjIpJTBBJTBBJTIzJTIwRm9yJTIwdHJhbnNmb3JtZXJzJTIwdjQueCUyQiUzQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMnZpbmFpJTJGYmVydHdlZXQtYmFzZSUyMiUyQyUyMHVzZV9mYXN0JTNERmFsc2UpJTBBJTBBJTIzJTIwRm9yJTIwdHJhbnNmb3JtZXJzJTIwdjMueCUzQSUwQSUyMyUyMHRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMnZpbmFpJTJGYmVydHdlZXQtYmFzZSUyMiklMEElMEElMjMlMjBJTlBVVCUyMFRXRUVUJTIwSVMlMjBBTFJFQURZJTIwTk9STUFMSVpFRCElMEFsaW5lJTIwJTNEJTIwJTIyU0MlMjBoYXMlMjBmaXJzdCUyMHR3byUyMHByZXN1bXB0aXZlJTIwY2FzZXMlMjBvZiUyMGNvcm9uYXZpcnVzJTIwJTJDJTIwREhFQyUyMGNvbmZpcm1zJTIwSFRUUFVSTCUyMHZpYSUyMCU0MFVTRVIlMjAlM0FjcnklM0ElMjIlMEElMEFpbnB1dF9pZHMlMjAlM0QlMjB0b3JjaC50ZW5zb3IoJTVCdG9rZW5pemVyLmVuY29kZShsaW5lKSU1RCklMEElMEF3aXRoJTIwdG9yY2gubm9fZ3JhZCgpJTNBJTBBJTIwJTIwJTIwJTIwZmVhdHVyZXMlMjAlM0QlMjBiZXJ0d2VldChpbnB1dF9pZHMpJTIwJTIwJTIzJTIwTW9kZWxzJTIwb3V0cHV0cyUyMGFyZSUyMG5vdyUyMHR1cGxlcyUwQSUwQSUyMyUyMFdpdGglMjBUZW5zb3JGbG93JTIwMi4wJTJCJTNBJTBBJTIzJTIwZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRGQXV0b01vZGVsJTBBJTIzJTIwYmVydHdlZXQlMjAlM0QlMjBURkF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQoJTIydmluYWklMkZiZXJ0d2VldC1iYXNlJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModel, AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>bertweet = AutoModel.from_pretrained(<span class="hljs-string">&quot;vinai/bertweet-base&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># For transformers v4.x+:</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;vinai/bertweet-base&quot;</span>, use_fast=<span class="hljs-literal">False</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># For transformers v3.x:</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># tokenizer = AutoTokenizer.from_pretrained(&quot;vinai/bertweet-base&quot;)</span>

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># INPUT TWEET IS ALREADY NORMALIZED!</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>line = <span class="hljs-string">&quot;SC has first two presumptive cases of coronavirus , DHEC confirms HTTPURL via @USER :cry:&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>input_ids = torch.tensor([tokenizer.encode(line)])

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">with</span> torch.no_grad():
<span class="hljs-meta">... </span>    features = bertweet(input_ids)  <span class="hljs-comment"># Models outputs are now tuples</span>

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># With TensorFlow 2.0+:</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># from transformers import TFAutoModel</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># bertweet = TFAutoModel.from_pretrained(&quot;vinai/bertweet-base&quot;)</span>`,wrap:!1}}),J=new ht({props:{$$slots:{default:[wt]},$$scope:{ctx:de}}}),P=new Be({props:{title:"BertweetTokenizer",local:"transformers.BertweetTokenizer",headingTag:"h2"}}),G=new B({props:{name:"class transformers.BertweetTokenizer",anchor:"transformers.BertweetTokenizer",parameters:[{name:"vocab_file",val:""},{name:"merges_file",val:""},{name:"normalization",val:" = False"},{name:"bos_token",val:" = '<s>'"},{name:"eos_token",val:" = '</s>'"},{name:"sep_token",val:" = '</s>'"},{name:"cls_token",val:" = '<s>'"},{name:"unk_token",val:" = '<unk>'"},{name:"pad_token",val:" = '<pad>'"},{name:"mask_token",val:" = '<mask>'"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.BertweetTokenizer.vocab_file",description:`<strong>vocab_file</strong> (<code>str</code>) &#x2014;
Path to the vocabulary file.`,name:"vocab_file"},{anchor:"transformers.BertweetTokenizer.merges_file",description:`<strong>merges_file</strong> (<code>str</code>) &#x2014;
Path to the merges file.`,name:"merges_file"},{anchor:"transformers.BertweetTokenizer.normalization",description:`<strong>normalization</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to apply a normalization preprocess.`,name:"normalization"},{anchor:"transformers.BertweetTokenizer.bos_token",description:`<strong>bos_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;s&gt;&quot;</code>) &#x2014;
The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.</p>
<div class="course-tip  bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400">
						
<p>When building a sequence using special tokens, this is not the token that is used for the beginning of
sequence. The token used is the <code>cls_token</code>.</p>

					</div>`,name:"bos_token"},{anchor:"transformers.BertweetTokenizer.eos_token",description:`<strong>eos_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;/s&gt;&quot;</code>) &#x2014;
The end of sequence token.</p>
<div class="course-tip  bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400">
						
<p>When building a sequence using special tokens, this is not the token that is used for the end of sequence.
The token used is the <code>sep_token</code>.</p>

					</div>`,name:"eos_token"},{anchor:"transformers.BertweetTokenizer.sep_token",description:`<strong>sep_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;/s&gt;&quot;</code>) &#x2014;
The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
sequence classification or for a text and a question for question answering. It is also used as the last
token of a sequence built with special tokens.`,name:"sep_token"},{anchor:"transformers.BertweetTokenizer.cls_token",description:`<strong>cls_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;s&gt;&quot;</code>) &#x2014;
The classifier token which is used when doing sequence classification (classification of the whole sequence
instead of per-token classification). It is the first token of the sequence when built with special tokens.`,name:"cls_token"},{anchor:"transformers.BertweetTokenizer.unk_token",description:`<strong>unk_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;unk&gt;&quot;</code>) &#x2014;
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
token instead.`,name:"unk_token"},{anchor:"transformers.BertweetTokenizer.pad_token",description:`<strong>pad_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;pad&gt;&quot;</code>) &#x2014;
The token used for padding, for example when batching sequences of different lengths.`,name:"pad_token"},{anchor:"transformers.BertweetTokenizer.mask_token",description:`<strong>mask_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;mask&gt;&quot;</code>) &#x2014;
The token used for masking values. This is the token used when training this model with masked language
modeling. This is the token which the model will try to predict.`,name:"mask_token"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/bertweet/tokenization_bertweet.py#L54"}}),N=new B({props:{name:"add_from_file",anchor:"transformers.BertweetTokenizer.add_from_file",parameters:[{name:"f",val:""}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/bertweet/tokenization_bertweet.py#L402"}}),F=new B({props:{name:"build_inputs_with_special_tokens",anchor:"transformers.BertweetTokenizer.build_inputs_with_special_tokens",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.BertweetTokenizer.build_inputs_with_special_tokens.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs to which the special tokens will be added.`,name:"token_ids_0"},{anchor:"transformers.BertweetTokenizer.build_inputs_with_special_tokens.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/bertweet/tokenization_bertweet.py#L167",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>List of <a href="../glossary#input-ids">input IDs</a> with the appropriate special tokens.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>List[int]</code></p>
`}}),Q=new B({props:{name:"convert_tokens_to_string",anchor:"transformers.BertweetTokenizer.convert_tokens_to_string",parameters:[{name:"tokens",val:""}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/bertweet/tokenization_bertweet.py#L368"}}),H=new B({props:{name:"create_token_type_ids_from_sequences",anchor:"transformers.BertweetTokenizer.create_token_type_ids_from_sequences",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.BertweetTokenizer.create_token_type_ids_from_sequences.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs.`,name:"token_ids_0"},{anchor:"transformers.BertweetTokenizer.create_token_type_ids_from_sequences.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/bertweet/tokenization_bertweet.py#L221",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>List of zeros.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>List[int]</code></p>
`}}),S=new B({props:{name:"get_special_tokens_mask",anchor:"transformers.BertweetTokenizer.get_special_tokens_mask",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"},{name:"already_has_special_tokens",val:": bool = False"}],parametersDescription:[{anchor:"transformers.BertweetTokenizer.get_special_tokens_mask.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs.`,name:"token_ids_0"},{anchor:"transformers.BertweetTokenizer.get_special_tokens_mask.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"},{anchor:"transformers.BertweetTokenizer.get_special_tokens_mask.already_has_special_tokens",description:`<strong>already_has_special_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not the token list is already formatted with special tokens for the model.`,name:"already_has_special_tokens"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/bertweet/tokenization_bertweet.py#L193",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>List[int]</code></p>
`}}),X=new B({props:{name:"normalizeToken",anchor:"transformers.BertweetTokenizer.normalizeToken",parameters:[{name:"token",val:""}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/bertweet/tokenization_bertweet.py#L341"}}),D=new B({props:{name:"normalizeTweet",anchor:"transformers.BertweetTokenizer.normalizeTweet",parameters:[{name:"tweet",val:""}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/bertweet/tokenization_bertweet.py#L307"}}),W=new bt({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/model_doc/bertweet.md"}}),{c(){p=a("meta"),I=o(),w=a("p"),O=o(),g(q.$$.fragment),pe=o(),y=a("div"),y.innerHTML=De,ce=o(),g(U.$$.fragment),ge=o(),E=a("p"),E.innerHTML=We,fe=o(),L=a("p"),L.textContent=Oe,ue=o(),Z=a("p"),Z.innerHTML=Ye,he=o(),j=a("p"),j.innerHTML=Ke,ke=o(),g(R.$$.fragment),be=o(),g(V.$$.fragment),we=o(),g(J.$$.fragment),Te=o(),g(P.$$.fragment),_e=o(),l=a("div"),g(G.$$.fragment),Ie=o(),Y=a("p"),Y.textContent=et,qe=o(),K=a("p"),K.innerHTML=tt,Ue=o(),x=a("div"),g(N.$$.fragment),Ee=o(),ee=a("p"),ee.textContent=nt,Le=o(),_=a("div"),g(F.$$.fragment),Ze=o(),te=a("p"),te.textContent=st,je=o(),ne=a("ul"),ne.innerHTML=ot,Re=o(),M=a("div"),g(Q.$$.fragment),Ve=o(),se=a("p"),se.textContent=rt,Pe=o(),z=a("div"),g(H.$$.fragment),Ge=o(),oe=a("p"),oe.textContent=at,Ne=o(),$=a("div"),g(S.$$.fragment),Fe=o(),re=a("p"),re.innerHTML=it,Qe=o(),A=a("div"),g(X.$$.fragment),He=o(),ae=a("p"),ae.textContent=lt,Se=o(),C=a("div"),g(D.$$.fragment),Xe=o(),ie=a("p"),ie.textContent=mt,ve=o(),g(W.$$.fragment),ye=o(),me=a("p"),this.h()},l(e){const t=ut("svelte-u9bgzb",document.head);p=i(t,"META",{name:!0,content:!0}),t.forEach(n),I=r(e),w=i(e,"P",{}),v(w).forEach(n),O=r(e),f(q.$$.fragment,e),pe=r(e),y=i(e,"DIV",{class:!0,"data-svelte-h":!0}),c(y)!=="svelte-1px1zvi"&&(y.innerHTML=De),ce=r(e),f(U.$$.fragment,e),ge=r(e),E=i(e,"P",{"data-svelte-h":!0}),c(E)!=="svelte-1tvfkok"&&(E.innerHTML=We),fe=r(e),L=i(e,"P",{"data-svelte-h":!0}),c(L)!=="svelte-vfdo9a"&&(L.textContent=Oe),ue=r(e),Z=i(e,"P",{"data-svelte-h":!0}),c(Z)!=="svelte-eh682o"&&(Z.innerHTML=Ye),he=r(e),j=i(e,"P",{"data-svelte-h":!0}),c(j)!=="svelte-fl8xrj"&&(j.innerHTML=Ke),ke=r(e),f(R.$$.fragment,e),be=r(e),f(V.$$.fragment,e),we=r(e),f(J.$$.fragment,e),Te=r(e),f(P.$$.fragment,e),_e=r(e),l=i(e,"DIV",{class:!0});var d=v(l);f(G.$$.fragment,d),Ie=r(d),Y=i(d,"P",{"data-svelte-h":!0}),c(Y)!=="svelte-b8riyv"&&(Y.textContent=et),qe=r(d),K=i(d,"P",{"data-svelte-h":!0}),c(K)!=="svelte-1c3t5ty"&&(K.innerHTML=tt),Ue=r(d),x=i(d,"DIV",{class:!0});var xe=v(x);f(N.$$.fragment,xe),Ee=r(xe),ee=i(xe,"P",{"data-svelte-h":!0}),c(ee)!=="svelte-ooaeix"&&(ee.textContent=nt),xe.forEach(n),Le=r(d),_=i(d,"DIV",{class:!0});var le=v(_);f(F.$$.fragment,le),Ze=r(le),te=i(le,"P",{"data-svelte-h":!0}),c(te)!=="svelte-zjm6uf"&&(te.textContent=st),je=r(le),ne=i(le,"UL",{"data-svelte-h":!0}),c(ne)!=="svelte-rq8uot"&&(ne.innerHTML=ot),le.forEach(n),Re=r(d),M=i(d,"DIV",{class:!0});var Me=v(M);f(Q.$$.fragment,Me),Ve=r(Me),se=i(Me,"P",{"data-svelte-h":!0}),c(se)!=="svelte-b3k2yi"&&(se.textContent=rt),Me.forEach(n),Pe=r(d),z=i(d,"DIV",{class:!0});var ze=v(z);f(H.$$.fragment,ze),Ge=r(ze),oe=i(ze,"P",{"data-svelte-h":!0}),c(oe)!=="svelte-vpfvn5"&&(oe.textContent=at),ze.forEach(n),Ne=r(d),$=i(d,"DIV",{class:!0});var $e=v($);f(S.$$.fragment,$e),Fe=r($e),re=i($e,"P",{"data-svelte-h":!0}),c(re)!=="svelte-1f4f5kp"&&(re.innerHTML=it),$e.forEach(n),Qe=r(d),A=i(d,"DIV",{class:!0});var Ae=v(A);f(X.$$.fragment,Ae),He=r(Ae),ae=i(Ae,"P",{"data-svelte-h":!0}),c(ae)!=="svelte-1jdrmaw"&&(ae.textContent=lt),Ae.forEach(n),Se=r(d),C=i(d,"DIV",{class:!0});var Ce=v(C);f(D.$$.fragment,Ce),Xe=r(Ce),ie=i(Ce,"P",{"data-svelte-h":!0}),c(ie)!=="svelte-15su17z"&&(ie.textContent=mt),Ce.forEach(n),d.forEach(n),ve=r(e),f(W.$$.fragment,e),ye=r(e),me=i(e,"P",{}),v(me).forEach(n),this.h()},h(){T(p,"name","hf:doc:metadata"),T(p,"content",_t),T(y,"class","flex flex-wrap space-x-1"),T(x,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(_,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(M,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T($,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(A,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(C,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),T(l,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,t){s(document.head,p),m(e,I,t),m(e,w,t),m(e,O,t),u(q,e,t),m(e,pe,t),m(e,y,t),m(e,ce,t),u(U,e,t),m(e,ge,t),m(e,E,t),m(e,fe,t),m(e,L,t),m(e,ue,t),m(e,Z,t),m(e,he,t),m(e,j,t),m(e,ke,t),u(R,e,t),m(e,be,t),u(V,e,t),m(e,we,t),u(J,e,t),m(e,Te,t),u(P,e,t),m(e,_e,t),m(e,l,t),u(G,l,null),s(l,Ie),s(l,Y),s(l,qe),s(l,K),s(l,Ue),s(l,x),u(N,x,null),s(x,Ee),s(x,ee),s(l,Le),s(l,_),u(F,_,null),s(_,Ze),s(_,te),s(_,je),s(_,ne),s(l,Re),s(l,M),u(Q,M,null),s(M,Ve),s(M,se),s(l,Pe),s(l,z),u(H,z,null),s(z,Ge),s(z,oe),s(l,Ne),s(l,$),u(S,$,null),s($,Fe),s($,re),s(l,Qe),s(l,A),u(X,A,null),s(A,He),s(A,ae),s(l,Se),s(l,C),u(D,C,null),s(C,Xe),s(C,ie),m(e,ve,t),u(W,e,t),m(e,ye,t),m(e,me,t),Je=!0},p(e,[t]){const d={};t&2&&(d.$$scope={dirty:t,ctx:e}),J.$set(d)},i(e){Je||(h(q.$$.fragment,e),h(U.$$.fragment,e),h(R.$$.fragment,e),h(V.$$.fragment,e),h(J.$$.fragment,e),h(P.$$.fragment,e),h(G.$$.fragment,e),h(N.$$.fragment,e),h(F.$$.fragment,e),h(Q.$$.fragment,e),h(H.$$.fragment,e),h(S.$$.fragment,e),h(X.$$.fragment,e),h(D.$$.fragment,e),h(W.$$.fragment,e),Je=!0)},o(e){k(q.$$.fragment,e),k(U.$$.fragment,e),k(R.$$.fragment,e),k(V.$$.fragment,e),k(J.$$.fragment,e),k(P.$$.fragment,e),k(G.$$.fragment,e),k(N.$$.fragment,e),k(F.$$.fragment,e),k(Q.$$.fragment,e),k(H.$$.fragment,e),k(S.$$.fragment,e),k(X.$$.fragment,e),k(D.$$.fragment,e),k(W.$$.fragment,e),Je=!1},d(e){e&&(n(I),n(w),n(O),n(pe),n(y),n(ce),n(ge),n(E),n(fe),n(L),n(ue),n(Z),n(he),n(j),n(ke),n(be),n(we),n(Te),n(_e),n(l),n(ve),n(ye),n(me)),n(p),b(q,e),b(U,e),b(R,e),b(V,e),b(J,e),b(P,e),b(G),b(N),b(F),b(Q),b(H),b(S),b(X),b(D),b(W,e)}}}const _t='{"title":"BERTweet","local":"bertweet","sections":[{"title":"Overview","local":"overview","sections":[],"depth":2},{"title":"Usage example","local":"usage-example","sections":[],"depth":2},{"title":"BertweetTokenizer","local":"transformers.BertweetTokenizer","sections":[],"depth":2}],"depth":1}';function vt(de){return pt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ct extends gt{constructor(p){super(),ft(this,p,vt,Tt,dt,{})}}export{Ct as component};
