import{s as rl,o as ol,n as is}from"../chunks/scheduler.9bc65507.js";import{S as il,i as cl,g as b,s as o,r as M,A as ml,h as w,f as l,c as i,j as ll,u as j,x as T,k as tl,y as hl,a as e,v as d,d as u,t as f,w as y,m as nl,n as pl}from"../chunks/index.707bf1b6.js";import{T as Za}from"../chunks/Tip.c2ecdbf4.js";import{Y as el}from"../chunks/Youtube.e1129c6f.js";import{C as _}from"../chunks/CodeBlock.54a9f38d.js";import{D as Ml}from"../chunks/DocNotebookDropdown.3e6b3817.js";import{F as Ba,M as Ls}from"../chunks/Markdown.8ab98a13.js";import{H as qs}from"../chunks/Heading.342b1fa6.js";function jl(k){let t,c,a='<a href="../model_doc/albert">ALBERT</a>, <a href="../model_doc/bert">BERT</a>, <a href="../model_doc/big_bird">BigBird</a>, <a href="../model_doc/biogpt">BioGpt</a>, <a href="../model_doc/bloom">BLOOM</a>, <a href="../model_doc/bros">BROS</a>, <a href="../model_doc/camembert">CamemBERT</a>, <a href="../model_doc/canine">CANINE</a>, <a href="../model_doc/convbert">ConvBERT</a>, <a href="../model_doc/data2vec-text">Data2VecText</a>, <a href="../model_doc/deberta">DeBERTa</a>, <a href="../model_doc/deberta-v2">DeBERTa-v2</a>, <a href="../model_doc/distilbert">DistilBERT</a>, <a href="../model_doc/electra">ELECTRA</a>, <a href="../model_doc/ernie">ERNIE</a>, <a href="../model_doc/ernie_m">ErnieM</a>, <a href="../model_doc/esm">ESM</a>, <a href="../model_doc/falcon">Falcon</a>, <a href="../model_doc/flaubert">FlauBERT</a>, <a href="../model_doc/fnet">FNet</a>, <a href="../model_doc/funnel">Funnel Transformer</a>, <a href="../model_doc/gpt-sw3">GPT-Sw3</a>, <a href="../model_doc/gpt2">OpenAI GPT-2</a>, <a href="../model_doc/gpt_bigcode">GPTBigCode</a>, <a href="../model_doc/gpt_neo">GPT Neo</a>, <a href="../model_doc/gpt_neox">GPT NeoX</a>, <a href="../model_doc/ibert">I-BERT</a>, <a href="../model_doc/layoutlm">LayoutLM</a>, <a href="../model_doc/layoutlmv2">LayoutLMv2</a>, <a href="../model_doc/layoutlmv3">LayoutLMv3</a>, <a href="../model_doc/lilt">LiLT</a>, <a href="../model_doc/longformer">Longformer</a>, <a href="../model_doc/luke">LUKE</a>, <a href="../model_doc/markuplm">MarkupLM</a>, <a href="../model_doc/mega">MEGA</a>, <a href="../model_doc/megatron-bert">Megatron-BERT</a>, <a href="../model_doc/mobilebert">MobileBERT</a>, <a href="../model_doc/mpnet">MPNet</a>, <a href="../model_doc/mpt">MPT</a>, <a href="../model_doc/mra">MRA</a>, <a href="../model_doc/nezha">Nezha</a>, <a href="../model_doc/nystromformer">Nyströmformer</a>, <a href="../model_doc/qdqbert">QDQBert</a>, <a href="../model_doc/rembert">RemBERT</a>, <a href="../model_doc/roberta">RoBERTa</a>, <a href="../model_doc/roberta-prelayernorm">RoBERTa-PreLayerNorm</a>, <a href="../model_doc/roc_bert">RoCBert</a>, <a href="../model_doc/roformer">RoFormer</a>, <a href="../model_doc/squeezebert">SqueezeBERT</a>, <a href="../model_doc/xlm">XLM</a>, <a href="../model_doc/xlm-roberta">XLM-RoBERTa</a>, <a href="../model_doc/xlm-roberta-xl">XLM-RoBERTa-XL</a>, <a href="../model_doc/xlnet">XLNet</a>, <a href="../model_doc/xmod">X-MOD</a>, <a href="../model_doc/yoso">YOSO</a>';return{c(){t=nl(`The task illustrated in this tutorial is supported by the following model architectures:

`),c=b("p"),c.innerHTML=a},l(m){t=pl(m,`The task illustrated in this tutorial is supported by the following model architectures:

`),c=w(m,"P",{"data-svelte-h":!0}),T(c)!=="svelte-1kspzor"&&(c.innerHTML=a)},m(m,g){e(m,t,g),e(m,c,g)},p:is,d(m){m&&(l(t),l(c))}}}function dl(k){let t,c;return t=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMERhdGFDb2xsYXRvckZvclRva2VuQ2xhc3NpZmljYXRpb24lMEElMEFkYXRhX2NvbGxhdG9yJTIwJTNEJTIwRGF0YUNvbGxhdG9yRm9yVG9rZW5DbGFzc2lmaWNhdGlvbih0b2tlbml6ZXIlM0R0b2tlbml6ZXIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer)`,wrap:!1}}),{c(){M(t.$$.fragment)},l(a){j(t.$$.fragment,a)},m(a,m){d(t,a,m),c=!0},p:is,i(a){c||(u(t.$$.fragment,a),c=!0)},o(a){f(t.$$.fragment,a),c=!1},d(a){y(t,a)}}}function ul(k){let t,c;return t=new Ls({props:{$$slots:{default:[dl]},$$scope:{ctx:k}}}),{c(){M(t.$$.fragment)},l(a){j(t.$$.fragment,a)},m(a,m){d(t,a,m),c=!0},p(a,m){const g={};m&2&&(g.$$scope={dirty:m,ctx:a}),t.$set(g)},i(a){c||(u(t.$$.fragment,a),c=!0)},o(a){f(t.$$.fragment,a),c=!1},d(a){y(t,a)}}}function fl(k){let t,c;return t=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMERhdGFDb2xsYXRvckZvclRva2VuQ2xhc3NpZmljYXRpb24lMEElMEFkYXRhX2NvbGxhdG9yJTIwJTNEJTIwRGF0YUNvbGxhdG9yRm9yVG9rZW5DbGFzc2lmaWNhdGlvbih0b2tlbml6ZXIlM0R0b2tlbml6ZXIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnRmJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`,wrap:!1}}),{c(){M(t.$$.fragment)},l(a){j(t.$$.fragment,a)},m(a,m){d(t,a,m),c=!0},p:is,i(a){c||(u(t.$$.fragment,a),c=!0)},o(a){f(t.$$.fragment,a),c=!1},d(a){y(t,a)}}}function yl(k){let t,c;return t=new Ls({props:{$$slots:{default:[fl]},$$scope:{ctx:k}}}),{c(){M(t.$$.fragment)},l(a){j(t.$$.fragment,a)},m(a,m){d(t,a,m),c=!0},p(a,m){const g={};m&2&&(g.$$scope={dirty:m,ctx:a}),t.$set(g)},i(a){c||(u(t.$$.fragment,a),c=!0)},o(a){f(t.$$.fragment,a),c=!1},d(a){y(t,a)}}}function gl(k){let t,c='If you aren’t familiar with finetuning a model with the <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a>, take a look at the basic tutorial <a href="../training#train-with-pytorch-trainer">here</a>!';return{c(){t=b("p"),t.innerHTML=c},l(a){t=w(a,"P",{"data-svelte-h":!0}),T(t)!=="svelte-15s4um0"&&(t.innerHTML=c)},m(a,m){e(a,t,m)},p:is,d(a){a&&l(t)}}}function bl(k){let t,c,a,m='You’re ready to start training your model now! Load DistilBERT with <a href="/docs/transformers/main/en/model_doc/auto#transformers.AutoModelForTokenClassification">AutoModelForTokenClassification</a> along with the number of expected labels, and the label mappings:',g,C,A,Z,I="At this point, only three steps remain:",B,x,X='<li>Define your training hyperparameters in <a href="/docs/transformers/main/en/main_classes/trainer#transformers.TrainingArguments">TrainingArguments</a>. The only required parameter is <code>output_dir</code> which specifies where to save your model. You’ll push this model to the Hub by setting <code>push_to_hub=True</code> (you need to be signed in to Hugging Face to upload your model). At the end of each epoch, the <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a> will evaluate the seqeval scores and save the training checkpoint.</li> <li>Pass the training arguments to <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a> along with the model, dataset, tokenizer, data collator, and <code>compute_metrics</code> function.</li> <li>Call <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer.train">train()</a> to finetune your model.</li>',G,$,R,r,U='Once training is completed, share your model to the Hub with the <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer.push_to_hub">push_to_hub()</a> method so everyone can use your model:',V,E,W;return t=new Za({props:{$$slots:{default:[gl]},$$scope:{ctx:k}}}),C=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvclRva2VuQ2xhc3NpZmljYXRpb24lMkMlMjBUcmFpbmluZ0FyZ3VtZW50cyUyQyUyMFRyYWluZXIlMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvclRva2VuQ2xhc3NpZmljYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmRpc3RpbGJlcnQtYmFzZS11bmNhc2VkJTIyJTJDJTIwbnVtX2xhYmVscyUzRDEzJTJDJTIwaWQybGFiZWwlM0RpZDJsYWJlbCUyQyUyMGxhYmVsMmlkJTNEbGFiZWwyaWQlMEEp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForTokenClassification, TrainingArguments, Trainer

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForTokenClassification.from_pretrained(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>, num_labels=<span class="hljs-number">13</span>, id2label=id2label, label2id=label2id
<span class="hljs-meta">... </span>)`,wrap:!1}}),$=new _({props:{code:"dHJhaW5pbmdfYXJncyUyMCUzRCUyMFRyYWluaW5nQXJndW1lbnRzKCUwQSUyMCUyMCUyMCUyMG91dHB1dF9kaXIlM0QlMjJteV9hd2Vzb21lX3dudXRfbW9kZWwlMjIlMkMlMEElMjAlMjAlMjAlMjBsZWFybmluZ19yYXRlJTNEMmUtNSUyQyUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUzRDE2JTJDJTBBJTIwJTIwJTIwJTIwcGVyX2RldmljZV9ldmFsX2JhdGNoX3NpemUlM0QxNiUyQyUwQSUyMCUyMCUyMCUyMG51bV90cmFpbl9lcG9jaHMlM0QyJTJDJTBBJTIwJTIwJTIwJTIwd2VpZ2h0X2RlY2F5JTNEMC4wMSUyQyUwQSUyMCUyMCUyMCUyMGV2YWx1YXRpb25fc3RyYXRlZ3klM0QlMjJlcG9jaCUyMiUyQyUwQSUyMCUyMCUyMCUyMHNhdmVfc3RyYXRlZ3klM0QlMjJlcG9jaCUyMiUyQyUwQSUyMCUyMCUyMCUyMGxvYWRfYmVzdF9tb2RlbF9hdF9lbmQlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwcHVzaF90b19odWIlM0RUcnVlJTJDJTBBKSUwQSUwQXRyYWluZXIlMjAlM0QlMjBUcmFpbmVyKCUwQSUyMCUyMCUyMCUyMG1vZGVsJTNEbW9kZWwlMkMlMEElMjAlMjAlMjAlMjBhcmdzJTNEdHJhaW5pbmdfYXJncyUyQyUwQSUyMCUyMCUyMCUyMHRyYWluX2RhdGFzZXQlM0R0b2tlbml6ZWRfd251dCU1QiUyMnRyYWluJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwZXZhbF9kYXRhc2V0JTNEdG9rZW5pemVkX3dudXQlNUIlMjJ0ZXN0JTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwdG9rZW5pemVyJTNEdG9rZW5pemVyJTJDJTBBJTIwJTIwJTIwJTIwZGF0YV9jb2xsYXRvciUzRGRhdGFfY29sbGF0b3IlMkMlMEElMjAlMjAlMjAlMjBjb21wdXRlX21ldHJpY3MlM0Rjb21wdXRlX21ldHJpY3MlMkMlMEEpJTBBJTBBdHJhaW5lci50cmFpbigp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>training_args = TrainingArguments(
<span class="hljs-meta">... </span>    output_dir=<span class="hljs-string">&quot;my_awesome_wnut_model&quot;</span>,
<span class="hljs-meta">... </span>    learning_rate=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    per_device_train_batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    per_device_eval_batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    num_train_epochs=<span class="hljs-number">2</span>,
<span class="hljs-meta">... </span>    weight_decay=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>    evaluation_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
<span class="hljs-meta">... </span>    save_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
<span class="hljs-meta">... </span>    load_best_model_at_end=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    push_to_hub=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = Trainer(
<span class="hljs-meta">... </span>    model=model,
<span class="hljs-meta">... </span>    args=training_args,
<span class="hljs-meta">... </span>    train_dataset=tokenized_wnut[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    eval_dataset=tokenized_wnut[<span class="hljs-string">&quot;test&quot;</span>],
<span class="hljs-meta">... </span>    tokenizer=tokenizer,
<span class="hljs-meta">... </span>    data_collator=data_collator,
<span class="hljs-meta">... </span>    compute_metrics=compute_metrics,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.train()`,wrap:!1}}),E=new _({props:{code:"dHJhaW5lci5wdXNoX3RvX2h1Yigp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.push_to_hub()',wrap:!1}}),{c(){M(t.$$.fragment),c=o(),a=b("p"),a.innerHTML=m,g=o(),M(C.$$.fragment),A=o(),Z=b("p"),Z.textContent=I,B=o(),x=b("ol"),x.innerHTML=X,G=o(),M($.$$.fragment),R=o(),r=b("p"),r.innerHTML=U,V=o(),M(E.$$.fragment)},l(h){j(t.$$.fragment,h),c=i(h),a=w(h,"P",{"data-svelte-h":!0}),T(a)!=="svelte-1u5l9g0"&&(a.innerHTML=m),g=i(h),j(C.$$.fragment,h),A=i(h),Z=w(h,"P",{"data-svelte-h":!0}),T(Z)!=="svelte-l42k0i"&&(Z.textContent=I),B=i(h),x=w(h,"OL",{"data-svelte-h":!0}),T(x)!=="svelte-e23su2"&&(x.innerHTML=X),G=i(h),j($.$$.fragment,h),R=i(h),r=w(h,"P",{"data-svelte-h":!0}),T(r)!=="svelte-1v13hlo"&&(r.innerHTML=U),V=i(h),j(E.$$.fragment,h)},m(h,v){d(t,h,v),e(h,c,v),e(h,a,v),e(h,g,v),d(C,h,v),e(h,A,v),e(h,Z,v),e(h,B,v),e(h,x,v),e(h,G,v),d($,h,v),e(h,R,v),e(h,r,v),e(h,V,v),d(E,h,v),W=!0},p(h,v){const F={};v&2&&(F.$$scope={dirty:v,ctx:h}),t.$set(F)},i(h){W||(u(t.$$.fragment,h),u(C.$$.fragment,h),u($.$$.fragment,h),u(E.$$.fragment,h),W=!0)},o(h){f(t.$$.fragment,h),f(C.$$.fragment,h),f($.$$.fragment,h),f(E.$$.fragment,h),W=!1},d(h){h&&(l(c),l(a),l(g),l(A),l(Z),l(B),l(x),l(G),l(R),l(r),l(V)),y(t,h),y(C,h),y($,h),y(E,h)}}}function wl(k){let t,c;return t=new Ls({props:{$$slots:{default:[bl]},$$scope:{ctx:k}}}),{c(){M(t.$$.fragment)},l(a){j(t.$$.fragment,a)},m(a,m){d(t,a,m),c=!0},p(a,m){const g={};m&2&&(g.$$scope={dirty:m,ctx:a}),t.$set(g)},i(a){c||(u(t.$$.fragment,a),c=!0)},o(a){f(t.$$.fragment,a),c=!1},d(a){y(t,a)}}}function Tl(k){let t,c='If you aren’t familiar with finetuning a model with Keras, take a look at the basic tutorial <a href="../training#train-a-tensorflow-model-with-keras">here</a>!';return{c(){t=b("p"),t.innerHTML=c},l(a){t=w(a,"P",{"data-svelte-h":!0}),T(t)!=="svelte-1rd4nl8"&&(t.innerHTML=c)},m(a,m){e(a,t,m)},p:is,d(a){a&&l(t)}}}function Jl(k){let t,c,a,m,g,C='Then you can load DistilBERT with <a href="/docs/transformers/main/en/model_doc/auto#transformers.TFAutoModelForTokenClassification">TFAutoModelForTokenClassification</a> along with the number of expected labels, and the label mappings:',A,Z,I,B,x='Convert your datasets to the <code>tf.data.Dataset</code> format with <a href="/docs/transformers/main/en/main_classes/model#transformers.TFPreTrainedModel.prepare_tf_dataset">prepare_tf_dataset()</a>:',X,G,$,R,r='Configure the model for training with <a href="https://keras.io/api/models/model_training_apis/#compile-method" rel="nofollow"><code>compile</code></a>. Note that Transformers models all have a default task-relevant loss function, so you don’t need to specify one unless you want to:',U,V,E,W,h='The last two things to setup before you start training is to compute the seqeval scores from the predictions, and provide a way to push your model to the Hub. Both are done by using <a href="../main_classes/keras_callbacks">Keras callbacks</a>.',v,F,cs='Pass your <code>compute_metrics</code> function to <a href="/docs/transformers/main/en/main_classes/keras_callbacks#transformers.KerasMetricCallback">KerasMetricCallback</a>:',H,z,N,as,ms='Specify where to push your model and tokenizer in the <a href="/docs/transformers/main/en/main_classes/keras_callbacks#transformers.PushToHubCallback">PushToHubCallback</a>:',Q,Y,q,L,ls="Then bundle your callbacks together:",hs,S,D,P,ts='Finally, you’re ready to start training your model! Call <a href="https://keras.io/api/models/model_training_apis/#fit-method" rel="nofollow"><code>fit</code></a> with your training and validation datasets, the number of epochs, and your callbacks to finetune the model:',Ms,K,O,ss,es="Once training is completed, your model is automatically uploaded to the Hub so everyone can use it!",js;return t=new Za({props:{$$slots:{default:[Tl]},$$scope:{ctx:k}}}),a=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMGNyZWF0ZV9vcHRpbWl6ZXIlMEElMEFiYXRjaF9zaXplJTIwJTNEJTIwMTYlMEFudW1fdHJhaW5fZXBvY2hzJTIwJTNEJTIwMyUwQW51bV90cmFpbl9zdGVwcyUyMCUzRCUyMChsZW4odG9rZW5pemVkX3dudXQlNUIlMjJ0cmFpbiUyMiU1RCklMjAlMkYlMkYlMjBiYXRjaF9zaXplKSUyMColMjBudW1fdHJhaW5fZXBvY2hzJTBBb3B0aW1pemVyJTJDJTIwbHJfc2NoZWR1bGUlMjAlM0QlMjBjcmVhdGVfb3B0aW1pemVyKCUwQSUyMCUyMCUyMCUyMGluaXRfbHIlM0QyZS01JTJDJTBBJTIwJTIwJTIwJTIwbnVtX3RyYWluX3N0ZXBzJTNEbnVtX3RyYWluX3N0ZXBzJTJDJTBBJTIwJTIwJTIwJTIwd2VpZ2h0X2RlY2F5X3JhdGUlM0QwLjAxJTJDJTBBJTIwJTIwJTIwJTIwbnVtX3dhcm11cF9zdGVwcyUzRDAlMkMlMEEp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer

<span class="hljs-meta">&gt;&gt;&gt; </span>batch_size = <span class="hljs-number">16</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>num_train_epochs = <span class="hljs-number">3</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>num_train_steps = (<span class="hljs-built_in">len</span>(tokenized_wnut[<span class="hljs-string">&quot;train&quot;</span>]) // batch_size) * num_train_epochs
<span class="hljs-meta">&gt;&gt;&gt; </span>optimizer, lr_schedule = create_optimizer(
<span class="hljs-meta">... </span>    init_lr=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    num_train_steps=num_train_steps,
<span class="hljs-meta">... </span>    weight_decay_rate=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>    num_warmup_steps=<span class="hljs-number">0</span>,
<span class="hljs-meta">... </span>)`,wrap:!1}}),Z=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRGQXV0b01vZGVsRm9yVG9rZW5DbGFzc2lmaWNhdGlvbiUwQSUwQW1vZGVsJTIwJTNEJTIwVEZBdXRvTW9kZWxGb3JUb2tlbkNsYXNzaWZpY2F0aW9uLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJkaXN0aWxiZXJ0LWJhc2UtdW5jYXNlZCUyMiUyQyUyMG51bV9sYWJlbHMlM0QxMyUyQyUyMGlkMmxhYmVsJTNEaWQybGFiZWwlMkMlMjBsYWJlbDJpZCUzRGxhYmVsMmlkJTBBKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForTokenClassification.from_pretrained(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>, num_labels=<span class="hljs-number">13</span>, id2label=id2label, label2id=label2id
<span class="hljs-meta">... </span>)`,wrap:!1}}),G=new _({props:{code:"dGZfdHJhaW5fc2V0JTIwJTNEJTIwbW9kZWwucHJlcGFyZV90Zl9kYXRhc2V0KCUwQSUyMCUyMCUyMCUyMHRva2VuaXplZF93bnV0JTVCJTIydHJhaW4lMjIlNUQlMkMlMEElMjAlMjAlMjAlMjBzaHVmZmxlJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJhdGNoX3NpemUlM0QxNiUyQyUwQSUyMCUyMCUyMCUyMGNvbGxhdGVfZm4lM0RkYXRhX2NvbGxhdG9yJTJDJTBBKSUwQSUwQXRmX3ZhbGlkYXRpb25fc2V0JTIwJTNEJTIwbW9kZWwucHJlcGFyZV90Zl9kYXRhc2V0KCUwQSUyMCUyMCUyMCUyMHRva2VuaXplZF93bnV0JTVCJTIydmFsaWRhdGlvbiUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMHNodWZmbGUlM0RGYWxzZSUyQyUwQSUyMCUyMCUyMCUyMGJhdGNoX3NpemUlM0QxNiUyQyUwQSUyMCUyMCUyMCUyMGNvbGxhdGVfZm4lM0RkYXRhX2NvbGxhdG9yJTJDJTBBKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tf_train_set = model.prepare_tf_dataset(
<span class="hljs-meta">... </span>    tokenized_wnut[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>tf_validation_set = model.prepare_tf_dataset(
<span class="hljs-meta">... </span>    tokenized_wnut[<span class="hljs-string">&quot;validation&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">False</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)`,wrap:!1}}),V=new _({props:{code:"aW1wb3J0JTIwdGVuc29yZmxvdyUyMGFzJTIwdGYlMEElMEFtb2RlbC5jb21waWxlKG9wdGltaXplciUzRG9wdGltaXplciklMjAlMjAlMjMlMjBObyUyMGxvc3MlMjBhcmd1bWVudCE=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

<span class="hljs-meta">&gt;&gt;&gt; </span>model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)  <span class="hljs-comment"># No loss argument!</span>`,wrap:!1}}),z=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5rZXJhc19jYWxsYmFja3MlMjBpbXBvcnQlMjBLZXJhc01ldHJpY0NhbGxiYWNrJTBBJTBBbWV0cmljX2NhbGxiYWNrJTIwJTNEJTIwS2VyYXNNZXRyaWNDYWxsYmFjayhtZXRyaWNfZm4lM0Rjb21wdXRlX21ldHJpY3MlMkMlMjBldmFsX2RhdGFzZXQlM0R0Zl92YWxpZGF0aW9uX3NldCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers.keras_callbacks <span class="hljs-keyword">import</span> KerasMetricCallback

<span class="hljs-meta">&gt;&gt;&gt; </span>metric_callback = KerasMetricCallback(metric_fn=compute_metrics, eval_dataset=tf_validation_set)`,wrap:!1}}),Y=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5rZXJhc19jYWxsYmFja3MlMjBpbXBvcnQlMjBQdXNoVG9IdWJDYWxsYmFjayUwQSUwQXB1c2hfdG9faHViX2NhbGxiYWNrJTIwJTNEJTIwUHVzaFRvSHViQ2FsbGJhY2soJTBBJTIwJTIwJTIwJTIwb3V0cHV0X2RpciUzRCUyMm15X2F3ZXNvbWVfd251dF9tb2RlbCUyMiUyQyUwQSUyMCUyMCUyMCUyMHRva2VuaXplciUzRHRva2VuaXplciUyQyUwQSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers.keras_callbacks <span class="hljs-keyword">import</span> PushToHubCallback

<span class="hljs-meta">&gt;&gt;&gt; </span>push_to_hub_callback = PushToHubCallback(
<span class="hljs-meta">... </span>    output_dir=<span class="hljs-string">&quot;my_awesome_wnut_model&quot;</span>,
<span class="hljs-meta">... </span>    tokenizer=tokenizer,
<span class="hljs-meta">... </span>)`,wrap:!1}}),S=new _({props:{code:"Y2FsbGJhY2tzJTIwJTNEJTIwJTVCbWV0cmljX2NhbGxiYWNrJTJDJTIwcHVzaF90b19odWJfY2FsbGJhY2slNUQ=",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>callbacks = [metric_callback, push_to_hub_callback]',wrap:!1}}),K=new _({props:{code:"bW9kZWwuZml0KHglM0R0Zl90cmFpbl9zZXQlMkMlMjB2YWxpZGF0aW9uX2RhdGElM0R0Zl92YWxpZGF0aW9uX3NldCUyQyUyMGVwb2NocyUzRDMlMkMlMjBjYWxsYmFja3MlM0RjYWxsYmFja3Mp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>model.fit(x=tf_train_set, validation_data=tf_validation_set, epochs=<span class="hljs-number">3</span>, callbacks=callbacks)',wrap:!1}}),{c(){M(t.$$.fragment),c=nl(`
To finetune a model in TensorFlow, start by setting up an optimizer function, learning rate schedule, and some training hyperparameters:

	`),M(a.$$.fragment),m=o(),g=b("p"),g.innerHTML=C,A=o(),M(Z.$$.fragment),I=o(),B=b("p"),B.innerHTML=x,X=o(),M(G.$$.fragment),$=o(),R=b("p"),R.innerHTML=r,U=o(),M(V.$$.fragment),E=o(),W=b("p"),W.innerHTML=h,v=o(),F=b("p"),F.innerHTML=cs,H=o(),M(z.$$.fragment),N=o(),as=b("p"),as.innerHTML=ms,Q=o(),M(Y.$$.fragment),q=o(),L=b("p"),L.textContent=ls,hs=o(),M(S.$$.fragment),D=o(),P=b("p"),P.innerHTML=ts,Ms=o(),M(K.$$.fragment),O=o(),ss=b("p"),ss.textContent=es},l(p){j(t.$$.fragment,p),c=pl(p,`
To finetune a model in TensorFlow, start by setting up an optimizer function, learning rate schedule, and some training hyperparameters:

	`),j(a.$$.fragment,p),m=i(p),g=w(p,"P",{"data-svelte-h":!0}),T(g)!=="svelte-1p9qjdy"&&(g.innerHTML=C),A=i(p),j(Z.$$.fragment,p),I=i(p),B=w(p,"P",{"data-svelte-h":!0}),T(B)!=="svelte-9ymftz"&&(B.innerHTML=x),X=i(p),j(G.$$.fragment,p),$=i(p),R=w(p,"P",{"data-svelte-h":!0}),T(R)!=="svelte-17cxx5e"&&(R.innerHTML=r),U=i(p),j(V.$$.fragment,p),E=i(p),W=w(p,"P",{"data-svelte-h":!0}),T(W)!=="svelte-nf8aa0"&&(W.innerHTML=h),v=i(p),F=w(p,"P",{"data-svelte-h":!0}),T(F)!=="svelte-bi2rpv"&&(F.innerHTML=cs),H=i(p),j(z.$$.fragment,p),N=i(p),as=w(p,"P",{"data-svelte-h":!0}),T(as)!=="svelte-1b3skyn"&&(as.innerHTML=ms),Q=i(p),j(Y.$$.fragment,p),q=i(p),L=w(p,"P",{"data-svelte-h":!0}),T(L)!=="svelte-1lw9xm8"&&(L.textContent=ls),hs=i(p),j(S.$$.fragment,p),D=i(p),P=w(p,"P",{"data-svelte-h":!0}),T(P)!=="svelte-1hrpv1v"&&(P.innerHTML=ts),Ms=i(p),j(K.$$.fragment,p),O=i(p),ss=w(p,"P",{"data-svelte-h":!0}),T(ss)!=="svelte-2s71om"&&(ss.textContent=es)},m(p,J){d(t,p,J),e(p,c,J),d(a,p,J),e(p,m,J),e(p,g,J),e(p,A,J),d(Z,p,J),e(p,I,J),e(p,B,J),e(p,X,J),d(G,p,J),e(p,$,J),e(p,R,J),e(p,U,J),d(V,p,J),e(p,E,J),e(p,W,J),e(p,v,J),e(p,F,J),e(p,H,J),d(z,p,J),e(p,N,J),e(p,as,J),e(p,Q,J),d(Y,p,J),e(p,q,J),e(p,L,J),e(p,hs,J),d(S,p,J),e(p,D,J),e(p,P,J),e(p,Ms,J),d(K,p,J),e(p,O,J),e(p,ss,J),js=!0},p(p,J){const ds={};J&2&&(ds.$$scope={dirty:J,ctx:p}),t.$set(ds)},i(p){js||(u(t.$$.fragment,p),u(a.$$.fragment,p),u(Z.$$.fragment,p),u(G.$$.fragment,p),u(V.$$.fragment,p),u(z.$$.fragment,p),u(Y.$$.fragment,p),u(S.$$.fragment,p),u(K.$$.fragment,p),js=!0)},o(p){f(t.$$.fragment,p),f(a.$$.fragment,p),f(Z.$$.fragment,p),f(G.$$.fragment,p),f(V.$$.fragment,p),f(z.$$.fragment,p),f(Y.$$.fragment,p),f(S.$$.fragment,p),f(K.$$.fragment,p),js=!1},d(p){p&&(l(c),l(m),l(g),l(A),l(I),l(B),l(X),l($),l(R),l(U),l(E),l(W),l(v),l(F),l(H),l(N),l(as),l(Q),l(q),l(L),l(hs),l(D),l(P),l(Ms),l(O),l(ss)),y(t,p),y(a,p),y(Z,p),y(G,p),y(V,p),y(z,p),y(Y,p),y(S,p),y(K,p)}}}function Ul(k){let t,c;return t=new Ls({props:{$$slots:{default:[Jl]},$$scope:{ctx:k}}}),{c(){M(t.$$.fragment)},l(a){j(t.$$.fragment,a)},m(a,m){d(t,a,m),c=!0},p(a,m){const g={};m&2&&(g.$$scope={dirty:m,ctx:a}),t.$set(g)},i(a){c||(u(t.$$.fragment,a),c=!0)},o(a){f(t.$$.fragment,a),c=!1},d(a){y(t,a)}}}function $l(k){let t,c=`For a more in-depth example of how to finetune a model for token classification, take a look at the corresponding
<a href="https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/token_classification.ipynb" rel="nofollow">PyTorch notebook</a>
or <a href="https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/token_classification-tf.ipynb" rel="nofollow">TensorFlow notebook</a>.`;return{c(){t=b("p"),t.innerHTML=c},l(a){t=w(a,"P",{"data-svelte-h":!0}),T(t)!=="svelte-j7j3qo"&&(t.innerHTML=c)},m(a,m){e(a,t,m)},p:is,d(a){a&&l(t)}}}function xl(k){let t,c="Tokenize the text and return PyTorch tensors:",a,m,g,C,A="Pass your inputs to the model and return the <code>logits</code>:",Z,I,B,x,X="Get the class with the highest probability, and use the model’s <code>id2label</code> mapping to convert it to a text label:",G,$,R;return m=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJzdGV2aGxpdSUyRm15X2F3ZXNvbWVfd251dF9tb2RlbCUyMiklMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIodGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;stevhliu/my_awesome_wnut_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)`,wrap:!1}}),I=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvclRva2VuQ2xhc3NpZmljYXRpb24lMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvclRva2VuQ2xhc3NpZmljYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUyMnN0ZXZobGl1JTJGbXlfYXdlc29tZV93bnV0X21vZGVsJTIyKSUwQXdpdGglMjB0b3JjaC5ub19ncmFkKCklM0ElMEElMjAlMjAlMjAlMjBsb2dpdHMlMjAlM0QlMjBtb2RlbCgqKmlucHV0cykubG9naXRz",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForTokenClassification.from_pretrained(<span class="hljs-string">&quot;stevhliu/my_awesome_wnut_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">with</span> torch.no_grad():
<span class="hljs-meta">... </span>    logits = model(**inputs).logits`,wrap:!1}}),$=new _({props:{code:"cHJlZGljdGlvbnMlMjAlM0QlMjB0b3JjaC5hcmdtYXgobG9naXRzJTJDJTIwZGltJTNEMiklMEFwcmVkaWN0ZWRfdG9rZW5fY2xhc3MlMjAlM0QlMjAlNUJtb2RlbC5jb25maWcuaWQybGFiZWwlNUJ0Lml0ZW0oKSU1RCUyMGZvciUyMHQlMjBpbiUyMHByZWRpY3Rpb25zJTVCMCU1RCU1RCUwQXByZWRpY3RlZF90b2tlbl9jbGFzcw==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>predictions = torch.argmax(logits, dim=<span class="hljs-number">2</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>predicted_token_class = [model.config.id2label[t.item()] <span class="hljs-keyword">for</span> t <span class="hljs-keyword">in</span> predictions[<span class="hljs-number">0</span>]]
<span class="hljs-meta">&gt;&gt;&gt; </span>predicted_token_class
[<span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;B-location&#x27;</span>,
 <span class="hljs-string">&#x27;I-location&#x27;</span>,
 <span class="hljs-string">&#x27;B-group&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;B-location&#x27;</span>,
 <span class="hljs-string">&#x27;B-location&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>]`,wrap:!1}}),{c(){t=b("p"),t.textContent=c,a=o(),M(m.$$.fragment),g=o(),C=b("p"),C.innerHTML=A,Z=o(),M(I.$$.fragment),B=o(),x=b("p"),x.innerHTML=X,G=o(),M($.$$.fragment)},l(r){t=w(r,"P",{"data-svelte-h":!0}),T(t)!=="svelte-1qcz1wr"&&(t.textContent=c),a=i(r),j(m.$$.fragment,r),g=i(r),C=w(r,"P",{"data-svelte-h":!0}),T(C)!=="svelte-f3g043"&&(C.innerHTML=A),Z=i(r),j(I.$$.fragment,r),B=i(r),x=w(r,"P",{"data-svelte-h":!0}),T(x)!=="svelte-6mgrol"&&(x.innerHTML=X),G=i(r),j($.$$.fragment,r)},m(r,U){e(r,t,U),e(r,a,U),d(m,r,U),e(r,g,U),e(r,C,U),e(r,Z,U),d(I,r,U),e(r,B,U),e(r,x,U),e(r,G,U),d($,r,U),R=!0},p:is,i(r){R||(u(m.$$.fragment,r),u(I.$$.fragment,r),u($.$$.fragment,r),R=!0)},o(r){f(m.$$.fragment,r),f(I.$$.fragment,r),f($.$$.fragment,r),R=!1},d(r){r&&(l(t),l(a),l(g),l(C),l(Z),l(B),l(x),l(G)),y(m,r),y(I,r),y($,r)}}}function _l(k){let t,c;return t=new Ls({props:{$$slots:{default:[xl]},$$scope:{ctx:k}}}),{c(){M(t.$$.fragment)},l(a){j(t.$$.fragment,a)},m(a,m){d(t,a,m),c=!0},p(a,m){const g={};m&2&&(g.$$scope={dirty:m,ctx:a}),t.$set(g)},i(a){c||(u(t.$$.fragment,a),c=!0)},o(a){f(t.$$.fragment,a),c=!1},d(a){y(t,a)}}}function kl(k){let t,c="Tokenize the text and return TensorFlow tensors:",a,m,g,C,A="Pass your inputs to the model and return the <code>logits</code>:",Z,I,B,x,X="Get the class with the highest probability, and use the model’s <code>id2label</code> mapping to convert it to a text label:",G,$,R;return m=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJzdGV2aGxpdSUyRm15X2F3ZXNvbWVfd251dF9tb2RlbCUyMiklMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIodGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIydGYlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;stevhliu/my_awesome_wnut_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(text, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`,wrap:!1}}),I=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRGQXV0b01vZGVsRm9yVG9rZW5DbGFzc2lmaWNhdGlvbiUwQSUwQW1vZGVsJTIwJTNEJTIwVEZBdXRvTW9kZWxGb3JUb2tlbkNsYXNzaWZpY2F0aW9uLmZyb21fcHJldHJhaW5lZCglMjJzdGV2aGxpdSUyRm15X2F3ZXNvbWVfd251dF9tb2RlbCUyMiklMEFsb2dpdHMlMjAlM0QlMjBtb2RlbCgqKmlucHV0cykubG9naXRz",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForTokenClassification.from_pretrained(<span class="hljs-string">&quot;stevhliu/my_awesome_wnut_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>logits = model(**inputs).logits`,wrap:!1}}),$=new _({props:{code:"cHJlZGljdGVkX3Rva2VuX2NsYXNzX2lkcyUyMCUzRCUyMHRmLm1hdGguYXJnbWF4KGxvZ2l0cyUyQyUyMGF4aXMlM0QtMSklMEFwcmVkaWN0ZWRfdG9rZW5fY2xhc3MlMjAlM0QlMjAlNUJtb2RlbC5jb25maWcuaWQybGFiZWwlNUJ0JTVEJTIwZm9yJTIwdCUyMGluJTIwcHJlZGljdGVkX3Rva2VuX2NsYXNzX2lkcyU1QjAlNUQubnVtcHkoKS50b2xpc3QoKSU1RCUwQXByZWRpY3RlZF90b2tlbl9jbGFzcw==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>predicted_token_class_ids = tf.math.argmax(logits, axis=-<span class="hljs-number">1</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>predicted_token_class = [model.config.id2label[t] <span class="hljs-keyword">for</span> t <span class="hljs-keyword">in</span> predicted_token_class_ids[<span class="hljs-number">0</span>].numpy().tolist()]
<span class="hljs-meta">&gt;&gt;&gt; </span>predicted_token_class
[<span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;B-location&#x27;</span>,
 <span class="hljs-string">&#x27;I-location&#x27;</span>,
 <span class="hljs-string">&#x27;B-group&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;B-location&#x27;</span>,
 <span class="hljs-string">&#x27;B-location&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>]`,wrap:!1}}),{c(){t=b("p"),t.textContent=c,a=o(),M(m.$$.fragment),g=o(),C=b("p"),C.innerHTML=A,Z=o(),M(I.$$.fragment),B=o(),x=b("p"),x.innerHTML=X,G=o(),M($.$$.fragment)},l(r){t=w(r,"P",{"data-svelte-h":!0}),T(t)!=="svelte-s1qr7b"&&(t.textContent=c),a=i(r),j(m.$$.fragment,r),g=i(r),C=w(r,"P",{"data-svelte-h":!0}),T(C)!=="svelte-f3g043"&&(C.innerHTML=A),Z=i(r),j(I.$$.fragment,r),B=i(r),x=w(r,"P",{"data-svelte-h":!0}),T(x)!=="svelte-6mgrol"&&(x.innerHTML=X),G=i(r),j($.$$.fragment,r)},m(r,U){e(r,t,U),e(r,a,U),d(m,r,U),e(r,g,U),e(r,C,U),e(r,Z,U),d(I,r,U),e(r,B,U),e(r,x,U),e(r,G,U),d($,r,U),R=!0},p:is,i(r){R||(u(m.$$.fragment,r),u(I.$$.fragment,r),u($.$$.fragment,r),R=!0)},o(r){f(m.$$.fragment,r),f(I.$$.fragment,r),f($.$$.fragment,r),R=!1},d(r){r&&(l(t),l(a),l(g),l(C),l(Z),l(B),l(x),l(G)),y(m,r),y(I,r),y($,r)}}}function Cl(k){let t,c;return t=new Ls({props:{$$slots:{default:[kl]},$$scope:{ctx:k}}}),{c(){M(t.$$.fragment)},l(a){j(t.$$.fragment,a)},m(a,m){d(t,a,m),c=!0},p(a,m){const g={};m&2&&(g.$$scope={dirty:m,ctx:a}),t.$set(g)},i(a){c||(u(t.$$.fragment,a),c=!0)},o(a){f(t.$$.fragment,a),c=!1},d(a){y(t,a)}}}function Il(k){let t,c,a,m,g,C,A,Z,I,B,x,X="Token classification assigns a label to individual tokens in a sentence. One of the most common token classification tasks is Named Entity Recognition (NER). NER attempts to find a label for each entity in a sentence, such as a person, location, or organization.",G,$,R="This guide will show you how to:",r,U,V='<li>Finetune <a href="https://huggingface.co/distilbert-base-uncased" rel="nofollow">DistilBERT</a> on the <a href="https://huggingface.co/datasets/wnut_17" rel="nofollow">WNUT 17</a> dataset to detect new entities.</li> <li>Use your finetuned model for inference.</li>',E,W,h,v,F="Before you begin, make sure you have all the necessary libraries installed:",cs,H,z,N,as="We encourage you to login to your Hugging Face account so you can upload and share your model with the community. When prompted, enter your token to login:",ms,Q,Y,q,L,ls,hs="Start by loading the WNUT 17 dataset from the 🤗 Datasets library:",S,D,P,ts,Ms="Then take a look at an example:",K,O,ss,es,js="Each number in <code>ner_tags</code> represents an entity. Convert the numbers to their label names to find out what the entities are:",p,J,ds,us,Ga="The letter that prefixes each <code>ner_tag</code> indicates the token position of the entity:",Ds,fs,Ra=`<li><code>B-</code> indicates the beginning of an entity.</li> <li><code>I-</code> indicates a token is contained inside the same entity (for example, the <code>State</code> token is a part of an entity like
<code>Empire State Building</code>).</li> <li><code>0</code> indicates the token doesn’t correspond to any entity.</li>`,Ps,ys,Ks,gs,Os,bs,Aa="The next step is to load a DistilBERT tokenizer to preprocess the <code>tokens</code> field:",sa,ws,aa,Ts,Wa="As you saw in the example <code>tokens</code> field above, it looks like the input has already been tokenized. But the input actually hasn’t been tokenized yet and you’ll need to set <code>is_split_into_words=True</code> to tokenize the words into subwords. For example:",la,Js,ta,Us,Ea="However, this adds some special tokens <code>[CLS]</code> and <code>[SEP]</code> and the subword tokenization creates a mismatch between the input and labels. A single word corresponding to a single label may now be split into two subwords. You’ll need to realign the tokens and labels by:",ea,$s,Xa='<li>Mapping all tokens to their corresponding word with the <a href="https://huggingface.co/docs/transformers/main_classes/tokenizer#transformers.BatchEncoding.word_ids" rel="nofollow"><code>word_ids</code></a> method.</li> <li>Assigning the label <code>-100</code> to the special tokens <code>[CLS]</code> and <code>[SEP]</code> so they’re ignored by the PyTorch loss function (see <a href="https://pytorch.org/docs/stable/generated/torch.nn.CrossEntropyLoss.html" rel="nofollow">CrossEntropyLoss</a>).</li> <li>Only labeling the first token of a given word. Assign <code>-100</code> to other subtokens from the same word.</li>',na,xs,Va="Here is how you can create a function to realign the tokens and labels, and truncate sequences to be no longer than DistilBERT’s maximum input length:",pa,_s,ra,ks,Fa='To apply the preprocessing function over the entire dataset, use 🤗 Datasets <a href="https://huggingface.co/docs/datasets/main/en/package_reference/main_classes#datasets.Dataset.map" rel="nofollow">map</a> function. You can speed up the <code>map</code> function by setting <code>batched=True</code> to process multiple elements of the dataset at once:',oa,Cs,ia,Is,Ha='Now create a batch of examples using <a href="/docs/transformers/main/en/main_classes/data_collator#transformers.DataCollatorWithPadding">DataCollatorWithPadding</a>. It’s more efficient to <em>dynamically pad</em> the sentences to the longest length in a batch during collation, instead of padding the whole dataset to the maximum length.',ca,ns,ma,vs,ha,Zs,za='Including a metric during training is often helpful for evaluating your model’s performance. You can quickly load a evaluation method with the 🤗 <a href="https://huggingface.co/docs/evaluate/index" rel="nofollow">Evaluate</a> library. For this task, load the <a href="https://huggingface.co/spaces/evaluate-metric/seqeval" rel="nofollow">seqeval</a> framework (see the 🤗 Evaluate <a href="https://huggingface.co/docs/evaluate/a_quick_tour" rel="nofollow">quick tour</a> to learn more about how to load and compute a metric). Seqeval actually produces several scores: precision, recall, F1, and accuracy.',Ma,Bs,ja,Gs,Na='Get the NER labels first, and then create a function that passes your true predictions and true labels to <a href="https://huggingface.co/docs/evaluate/main/en/package_reference/main_classes#evaluate.EvaluationModule.compute" rel="nofollow">compute</a> to calculate the scores:',da,Rs,ua,As,Qa="Your <code>compute_metrics</code> function is ready to go now, and you’ll return to it when you setup your training.",fa,Ws,ya,Es,Ya="Before you start training your model, create a map of the expected ids to their labels with <code>id2label</code> and <code>label2id</code>:",ga,Xs,ba,ps,wa,rs,Ta,Vs,Ja,Fs,qa="Great, now that you’ve finetuned a model, you can use it for inference!",Ua,Hs,La="Grab some text you’d like to run inference on:",$a,zs,xa,Ns,Sa='The simplest way to try out your finetuned model for inference is to use it in a <a href="/docs/transformers/main/en/main_classes/pipelines#transformers.pipeline">pipeline()</a>. Instantiate a <code>pipeline</code> for NER with your model, and pass your text to it:',_a,Qs,ka,Ys,Da="You can also manually replicate the results of the <code>pipeline</code> if you’d like:",Ca,os,Ia,Ss,va;return g=new qs({props:{title:"Token classification",local:"token-classification",headingTag:"h1"}}),A=new Ml({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Mixed",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/token_classification.ipynb"},{label:"PyTorch",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/pytorch/token_classification.ipynb"},{label:"TensorFlow",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/tensorflow/token_classification.ipynb"},{label:"Mixed",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/token_classification.ipynb"},{label:"PyTorch",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/pytorch/token_classification.ipynb"},{label:"TensorFlow",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/tensorflow/token_classification.ipynb"}]}}),I=new el({props:{id:"wVHdVlPScxA"}}),W=new Za({props:{$$slots:{default:[jl]},$$scope:{ctx:k}}}),H=new _({props:{code:"cGlwJTIwaW5zdGFsbCUyMHRyYW5zZm9ybWVycyUyMGRhdGFzZXRzJTIwZXZhbHVhdGUlMjBzZXFldmFs",highlighted:"pip install transformers datasets evaluate seqeval",wrap:!1}}),Q=new _({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMG5vdGVib29rX2xvZ2luJTBBJTBBbm90ZWJvb2tfbG9naW4oKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

<span class="hljs-meta">&gt;&gt;&gt; </span>notebook_login()`,wrap:!1}}),q=new qs({props:{title:"Load WNUT 17 dataset",local:"load-wnut-17-dataset",headingTag:"h2"}}),D=new _({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBd251dCUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJ3bnV0XzE3JTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>wnut = load_dataset(<span class="hljs-string">&quot;wnut_17&quot;</span>)`,wrap:!1}}),O=new _({props:{code:"d251dCU1QiUyMnRyYWluJTIyJTVEJTVCMCU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>wnut[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;id&#x27;</span>: <span class="hljs-string">&#x27;0&#x27;</span>,
 <span class="hljs-string">&#x27;ner_tags&#x27;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">7</span>, <span class="hljs-number">8</span>, <span class="hljs-number">8</span>, <span class="hljs-number">0</span>, <span class="hljs-number">7</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
 <span class="hljs-string">&#x27;tokens&#x27;</span>: [<span class="hljs-string">&#x27;@paulwalk&#x27;</span>, <span class="hljs-string">&#x27;It&#x27;</span>, <span class="hljs-string">&quot;&#x27;s&quot;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;view&#x27;</span>, <span class="hljs-string">&#x27;from&#x27;</span>, <span class="hljs-string">&#x27;where&#x27;</span>, <span class="hljs-string">&#x27;I&#x27;</span>, <span class="hljs-string">&quot;&#x27;m&quot;</span>, <span class="hljs-string">&#x27;living&#x27;</span>, <span class="hljs-string">&#x27;for&#x27;</span>, <span class="hljs-string">&#x27;two&#x27;</span>, <span class="hljs-string">&#x27;weeks&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;Empire&#x27;</span>, <span class="hljs-string">&#x27;State&#x27;</span>, <span class="hljs-string">&#x27;Building&#x27;</span>, <span class="hljs-string">&#x27;=&#x27;</span>, <span class="hljs-string">&#x27;ESB&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;Pretty&#x27;</span>, <span class="hljs-string">&#x27;bad&#x27;</span>, <span class="hljs-string">&#x27;storm&#x27;</span>, <span class="hljs-string">&#x27;here&#x27;</span>, <span class="hljs-string">&#x27;last&#x27;</span>, <span class="hljs-string">&#x27;evening&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>]
}`,wrap:!1}}),J=new _({props:{code:"bGFiZWxfbGlzdCUyMCUzRCUyMHdudXQlNUIlMjJ0cmFpbiUyMiU1RC5mZWF0dXJlcyU1QmYlMjJuZXJfdGFncyUyMiU1RC5mZWF0dXJlLm5hbWVzJTBBbGFiZWxfbGlzdA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>label_list = wnut[<span class="hljs-string">&quot;train&quot;</span>].features[<span class="hljs-string">f&quot;ner_tags&quot;</span>].feature.names
<span class="hljs-meta">&gt;&gt;&gt; </span>label_list
[
    <span class="hljs-string">&quot;O&quot;</span>,
    <span class="hljs-string">&quot;B-corporation&quot;</span>,
    <span class="hljs-string">&quot;I-corporation&quot;</span>,
    <span class="hljs-string">&quot;B-creative-work&quot;</span>,
    <span class="hljs-string">&quot;I-creative-work&quot;</span>,
    <span class="hljs-string">&quot;B-group&quot;</span>,
    <span class="hljs-string">&quot;I-group&quot;</span>,
    <span class="hljs-string">&quot;B-location&quot;</span>,
    <span class="hljs-string">&quot;I-location&quot;</span>,
    <span class="hljs-string">&quot;B-person&quot;</span>,
    <span class="hljs-string">&quot;I-person&quot;</span>,
    <span class="hljs-string">&quot;B-product&quot;</span>,
    <span class="hljs-string">&quot;I-product&quot;</span>,
]`,wrap:!1}}),ys=new qs({props:{title:"Preprocess",local:"preprocess",headingTag:"h2"}}),gs=new el({props:{id:"iY2AZYdZAr0"}}),ws=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJkaXN0aWxiZXJ0LWJhc2UtdW5jYXNlZCUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>)`,wrap:!1}}),Js=new _({props:{code:"ZXhhbXBsZSUyMCUzRCUyMHdudXQlNUIlMjJ0cmFpbiUyMiU1RCU1QjAlNUQlMEF0b2tlbml6ZWRfaW5wdXQlMjAlM0QlMjB0b2tlbml6ZXIoZXhhbXBsZSU1QiUyMnRva2VucyUyMiU1RCUyQyUyMGlzX3NwbGl0X2ludG9fd29yZHMlM0RUcnVlKSUwQXRva2VucyUyMCUzRCUyMHRva2VuaXplci5jb252ZXJ0X2lkc190b190b2tlbnModG9rZW5pemVkX2lucHV0JTVCJTIyaW5wdXRfaWRzJTIyJTVEKSUwQXRva2Vucw==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>example = wnut[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_input = tokenizer(example[<span class="hljs-string">&quot;tokens&quot;</span>], is_split_into_words=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokens = tokenizer.convert_ids_to_tokens(tokenized_input[<span class="hljs-string">&quot;input_ids&quot;</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>tokens
[<span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;@&#x27;</span>, <span class="hljs-string">&#x27;paul&#x27;</span>, <span class="hljs-string">&#x27;##walk&#x27;</span>, <span class="hljs-string">&#x27;it&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;view&#x27;</span>, <span class="hljs-string">&#x27;from&#x27;</span>, <span class="hljs-string">&#x27;where&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;m&#x27;</span>, <span class="hljs-string">&#x27;living&#x27;</span>, <span class="hljs-string">&#x27;for&#x27;</span>, <span class="hljs-string">&#x27;two&#x27;</span>, <span class="hljs-string">&#x27;weeks&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;empire&#x27;</span>, <span class="hljs-string">&#x27;state&#x27;</span>, <span class="hljs-string">&#x27;building&#x27;</span>, <span class="hljs-string">&#x27;=&#x27;</span>, <span class="hljs-string">&#x27;es&#x27;</span>, <span class="hljs-string">&#x27;##b&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;pretty&#x27;</span>, <span class="hljs-string">&#x27;bad&#x27;</span>, <span class="hljs-string">&#x27;storm&#x27;</span>, <span class="hljs-string">&#x27;here&#x27;</span>, <span class="hljs-string">&#x27;last&#x27;</span>, <span class="hljs-string">&#x27;evening&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>]`,wrap:!1}}),_s=new _({props:{code:"ZGVmJTIwdG9rZW5pemVfYW5kX2FsaWduX2xhYmVscyhleGFtcGxlcyklM0ElMEElMjAlMjAlMjAlMjB0b2tlbml6ZWRfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKGV4YW1wbGVzJTVCJTIydG9rZW5zJTIyJTVEJTJDJTIwdHJ1bmNhdGlvbiUzRFRydWUlMkMlMjBpc19zcGxpdF9pbnRvX3dvcmRzJTNEVHJ1ZSklMEElMEElMjAlMjAlMjAlMjBsYWJlbHMlMjAlM0QlMjAlNUIlNUQlMEElMjAlMjAlMjAlMjBmb3IlMjBpJTJDJTIwbGFiZWwlMjBpbiUyMGVudW1lcmF0ZShleGFtcGxlcyU1QmYlMjJuZXJfdGFncyUyMiU1RCklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB3b3JkX2lkcyUyMCUzRCUyMHRva2VuaXplZF9pbnB1dHMud29yZF9pZHMoYmF0Y2hfaW5kZXglM0RpKSUyMCUyMCUyMyUyME1hcCUyMHRva2VucyUyMHRvJTIwdGhlaXIlMjByZXNwZWN0aXZlJTIwd29yZC4lMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcmV2aW91c193b3JkX2lkeCUyMCUzRCUyME5vbmUlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsYWJlbF9pZHMlMjAlM0QlMjAlNUIlNUQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmb3IlMjB3b3JkX2lkeCUyMGluJTIwd29yZF9pZHMlM0ElMjAlMjAlMjMlMjBTZXQlMjB0aGUlMjBzcGVjaWFsJTIwdG9rZW5zJTIwdG8lMjAtMTAwLiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlmJTIwd29yZF9pZHglMjBpcyUyME5vbmUlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsYWJlbF9pZHMuYXBwZW5kKC0xMDApJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZWxpZiUyMHdvcmRfaWR4JTIwISUzRCUyMHByZXZpb3VzX3dvcmRfaWR4JTNBJTIwJTIwJTIzJTIwT25seSUyMGxhYmVsJTIwdGhlJTIwZmlyc3QlMjB0b2tlbiUyMG9mJTIwYSUyMGdpdmVuJTIwd29yZC4lMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsYWJlbF9pZHMuYXBwZW5kKGxhYmVsJTVCd29yZF9pZHglNUQpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZWxzZSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGxhYmVsX2lkcy5hcHBlbmQoLTEwMCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcmV2aW91c193b3JkX2lkeCUyMCUzRCUyMHdvcmRfaWR4JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbGFiZWxzLmFwcGVuZChsYWJlbF9pZHMpJTBBJTBBJTIwJTIwJTIwJTIwdG9rZW5pemVkX2lucHV0cyU1QiUyMmxhYmVscyUyMiU1RCUyMCUzRCUyMGxhYmVscyUwQSUyMCUyMCUyMCUyMHJldHVybiUyMHRva2VuaXplZF9pbnB1dHM=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize_and_align_labels</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    tokenized_inputs = tokenizer(examples[<span class="hljs-string">&quot;tokens&quot;</span>], truncation=<span class="hljs-literal">True</span>, is_split_into_words=<span class="hljs-literal">True</span>)

<span class="hljs-meta">... </span>    labels = []
<span class="hljs-meta">... </span>    <span class="hljs-keyword">for</span> i, label <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(examples[<span class="hljs-string">f&quot;ner_tags&quot;</span>]):
<span class="hljs-meta">... </span>        word_ids = tokenized_inputs.word_ids(batch_index=i)  <span class="hljs-comment"># Map tokens to their respective word.</span>
<span class="hljs-meta">... </span>        previous_word_idx = <span class="hljs-literal">None</span>
<span class="hljs-meta">... </span>        label_ids = []
<span class="hljs-meta">... </span>        <span class="hljs-keyword">for</span> word_idx <span class="hljs-keyword">in</span> word_ids:  <span class="hljs-comment"># Set the special tokens to -100.</span>
<span class="hljs-meta">... </span>            <span class="hljs-keyword">if</span> word_idx <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>:
<span class="hljs-meta">... </span>                label_ids.append(-<span class="hljs-number">100</span>)
<span class="hljs-meta">... </span>            <span class="hljs-keyword">elif</span> word_idx != previous_word_idx:  <span class="hljs-comment"># Only label the first token of a given word.</span>
<span class="hljs-meta">... </span>                label_ids.append(label[word_idx])
<span class="hljs-meta">... </span>            <span class="hljs-keyword">else</span>:
<span class="hljs-meta">... </span>                label_ids.append(-<span class="hljs-number">100</span>)
<span class="hljs-meta">... </span>            previous_word_idx = word_idx
<span class="hljs-meta">... </span>        labels.append(label_ids)

<span class="hljs-meta">... </span>    tokenized_inputs[<span class="hljs-string">&quot;labels&quot;</span>] = labels
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> tokenized_inputs`,wrap:!1}}),Cs=new _({props:{code:"dG9rZW5pemVkX3dudXQlMjAlM0QlMjB3bnV0Lm1hcCh0b2tlbml6ZV9hbmRfYWxpZ25fbGFiZWxzJTJDJTIwYmF0Y2hlZCUzRFRydWUp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_wnut = wnut.<span class="hljs-built_in">map</span>(tokenize_and_align_labels, batched=<span class="hljs-literal">True</span>)',wrap:!1}}),ns=new Ba({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[yl],pytorch:[ul]},$$scope:{ctx:k}}}),vs=new qs({props:{title:"Evaluate",local:"evaluate",headingTag:"h2"}}),Bs=new _({props:{code:"aW1wb3J0JTIwZXZhbHVhdGUlMEElMEFzZXFldmFsJTIwJTNEJTIwZXZhbHVhdGUubG9hZCglMjJzZXFldmFsJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> evaluate

<span class="hljs-meta">&gt;&gt;&gt; </span>seqeval = evaluate.load(<span class="hljs-string">&quot;seqeval&quot;</span>)`,wrap:!1}}),Rs=new _({props:{code:"aW1wb3J0JTIwbnVtcHklMjBhcyUyMG5wJTBBJTBBbGFiZWxzJTIwJTNEJTIwJTVCbGFiZWxfbGlzdCU1QmklNUQlMjBmb3IlMjBpJTIwaW4lMjBleGFtcGxlJTVCZiUyMm5lcl90YWdzJTIyJTVEJTVEJTBBJTBBJTBBZGVmJTIwY29tcHV0ZV9tZXRyaWNzKHApJTNBJTBBJTIwJTIwJTIwJTIwcHJlZGljdGlvbnMlMkMlMjBsYWJlbHMlMjAlM0QlMjBwJTBBJTIwJTIwJTIwJTIwcHJlZGljdGlvbnMlMjAlM0QlMjBucC5hcmdtYXgocHJlZGljdGlvbnMlMkMlMjBheGlzJTNEMiklMEElMEElMjAlMjAlMjAlMjB0cnVlX3ByZWRpY3Rpb25zJTIwJTNEJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTVCbGFiZWxfbGlzdCU1QnAlNUQlMjBmb3IlMjAocCUyQyUyMGwpJTIwaW4lMjB6aXAocHJlZGljdGlvbiUyQyUyMGxhYmVsKSUyMGlmJTIwbCUyMCElM0QlMjAtMTAwJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZm9yJTIwcHJlZGljdGlvbiUyQyUyMGxhYmVsJTIwaW4lMjB6aXAocHJlZGljdGlvbnMlMkMlMjBsYWJlbHMpJTBBJTIwJTIwJTIwJTIwJTVEJTBBJTIwJTIwJTIwJTIwdHJ1ZV9sYWJlbHMlMjAlM0QlMjAlNUIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlNUJsYWJlbF9saXN0JTVCbCU1RCUyMGZvciUyMChwJTJDJTIwbCklMjBpbiUyMHppcChwcmVkaWN0aW9uJTJDJTIwbGFiZWwpJTIwaWYlMjBsJTIwISUzRCUyMC0xMDAlNUQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmb3IlMjBwcmVkaWN0aW9uJTJDJTIwbGFiZWwlMjBpbiUyMHppcChwcmVkaWN0aW9ucyUyQyUyMGxhYmVscyklMEElMjAlMjAlMjAlMjAlNUQlMEElMEElMjAlMjAlMjAlMjByZXN1bHRzJTIwJTNEJTIwc2VxZXZhbC5jb21wdXRlKHByZWRpY3Rpb25zJTNEdHJ1ZV9wcmVkaWN0aW9ucyUyQyUyMHJlZmVyZW5jZXMlM0R0cnVlX2xhYmVscyklMEElMjAlMjAlMjAlMjByZXR1cm4lMjAlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJwcmVjaXNpb24lMjIlM0ElMjByZXN1bHRzJTVCJTIyb3ZlcmFsbF9wcmVjaXNpb24lMjIlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJyZWNhbGwlMjIlM0ElMjByZXN1bHRzJTVCJTIyb3ZlcmFsbF9yZWNhbGwlMjIlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJmMSUyMiUzQSUyMHJlc3VsdHMlNUIlMjJvdmVyYWxsX2YxJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyYWNjdXJhY3klMjIlM0ElMjByZXN1bHRzJTVCJTIyb3ZlcmFsbF9hY2N1cmFjeSUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMCU3RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np

<span class="hljs-meta">&gt;&gt;&gt; </span>labels = [label_list[i] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> example[<span class="hljs-string">f&quot;ner_tags&quot;</span>]]


<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_metrics</span>(<span class="hljs-params">p</span>):
<span class="hljs-meta">... </span>    predictions, labels = p
<span class="hljs-meta">... </span>    predictions = np.argmax(predictions, axis=<span class="hljs-number">2</span>)

<span class="hljs-meta">... </span>    true_predictions = [
<span class="hljs-meta">... </span>        [label_list[p] <span class="hljs-keyword">for</span> (p, l) <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(prediction, label) <span class="hljs-keyword">if</span> l != -<span class="hljs-number">100</span>]
<span class="hljs-meta">... </span>        <span class="hljs-keyword">for</span> prediction, label <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(predictions, labels)
<span class="hljs-meta">... </span>    ]
<span class="hljs-meta">... </span>    true_labels = [
<span class="hljs-meta">... </span>        [label_list[l] <span class="hljs-keyword">for</span> (p, l) <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(prediction, label) <span class="hljs-keyword">if</span> l != -<span class="hljs-number">100</span>]
<span class="hljs-meta">... </span>        <span class="hljs-keyword">for</span> prediction, label <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(predictions, labels)
<span class="hljs-meta">... </span>    ]

<span class="hljs-meta">... </span>    results = seqeval.compute(predictions=true_predictions, references=true_labels)
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> {
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;precision&quot;</span>: results[<span class="hljs-string">&quot;overall_precision&quot;</span>],
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;recall&quot;</span>: results[<span class="hljs-string">&quot;overall_recall&quot;</span>],
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;f1&quot;</span>: results[<span class="hljs-string">&quot;overall_f1&quot;</span>],
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;accuracy&quot;</span>: results[<span class="hljs-string">&quot;overall_accuracy&quot;</span>],
<span class="hljs-meta">... </span>    }`,wrap:!1}}),Ws=new qs({props:{title:"Train",local:"train",headingTag:"h2"}}),Xs=new _({props:{code:"aWQybGFiZWwlMjAlM0QlMjAlN0IlMEElMjAlMjAlMjAlMjAwJTNBJTIwJTIyTyUyMiUyQyUwQSUyMCUyMCUyMCUyMDElM0ElMjAlMjJCLWNvcnBvcmF0aW9uJTIyJTJDJTBBJTIwJTIwJTIwJTIwMiUzQSUyMCUyMkktY29ycG9yYXRpb24lMjIlMkMlMEElMjAlMjAlMjAlMjAzJTNBJTIwJTIyQi1jcmVhdGl2ZS13b3JrJTIyJTJDJTBBJTIwJTIwJTIwJTIwNCUzQSUyMCUyMkktY3JlYXRpdmUtd29yayUyMiUyQyUwQSUyMCUyMCUyMCUyMDUlM0ElMjAlMjJCLWdyb3VwJTIyJTJDJTBBJTIwJTIwJTIwJTIwNiUzQSUyMCUyMkktZ3JvdXAlMjIlMkMlMEElMjAlMjAlMjAlMjA3JTNBJTIwJTIyQi1sb2NhdGlvbiUyMiUyQyUwQSUyMCUyMCUyMCUyMDglM0ElMjAlMjJJLWxvY2F0aW9uJTIyJTJDJTBBJTIwJTIwJTIwJTIwOSUzQSUyMCUyMkItcGVyc29uJTIyJTJDJTBBJTIwJTIwJTIwJTIwMTAlM0ElMjAlMjJJLXBlcnNvbiUyMiUyQyUwQSUyMCUyMCUyMCUyMDExJTNBJTIwJTIyQi1wcm9kdWN0JTIyJTJDJTBBJTIwJTIwJTIwJTIwMTIlM0ElMjAlMjJJLXByb2R1Y3QlMjIlMkMlMEElN0QlMEFsYWJlbDJpZCUyMCUzRCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMk8lMjIlM0ElMjAwJTJDJTBBJTIwJTIwJTIwJTIwJTIyQi1jb3Jwb3JhdGlvbiUyMiUzQSUyMDElMkMlMEElMjAlMjAlMjAlMjAlMjJJLWNvcnBvcmF0aW9uJTIyJTNBJTIwMiUyQyUwQSUyMCUyMCUyMCUyMCUyMkItY3JlYXRpdmUtd29yayUyMiUzQSUyMDMlMkMlMEElMjAlMjAlMjAlMjAlMjJJLWNyZWF0aXZlLXdvcmslMjIlM0ElMjA0JTJDJTBBJTIwJTIwJTIwJTIwJTIyQi1ncm91cCUyMiUzQSUyMDUlMkMlMEElMjAlMjAlMjAlMjAlMjJJLWdyb3VwJTIyJTNBJTIwNiUyQyUwQSUyMCUyMCUyMCUyMCUyMkItbG9jYXRpb24lMjIlM0ElMjA3JTJDJTBBJTIwJTIwJTIwJTIwJTIySS1sb2NhdGlvbiUyMiUzQSUyMDglMkMlMEElMjAlMjAlMjAlMjAlMjJCLXBlcnNvbiUyMiUzQSUyMDklMkMlMEElMjAlMjAlMjAlMjAlMjJJLXBlcnNvbiUyMiUzQSUyMDEwJTJDJTBBJTIwJTIwJTIwJTIwJTIyQi1wcm9kdWN0JTIyJTNBJTIwMTElMkMlMEElMjAlMjAlMjAlMjAlMjJJLXByb2R1Y3QlMjIlM0ElMjAxMiUyQyUwQSU3RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>id2label = {
<span class="hljs-meta">... </span>    <span class="hljs-number">0</span>: <span class="hljs-string">&quot;O&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">1</span>: <span class="hljs-string">&quot;B-corporation&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">2</span>: <span class="hljs-string">&quot;I-corporation&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">3</span>: <span class="hljs-string">&quot;B-creative-work&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">4</span>: <span class="hljs-string">&quot;I-creative-work&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">5</span>: <span class="hljs-string">&quot;B-group&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">6</span>: <span class="hljs-string">&quot;I-group&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">7</span>: <span class="hljs-string">&quot;B-location&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">8</span>: <span class="hljs-string">&quot;I-location&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">9</span>: <span class="hljs-string">&quot;B-person&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">10</span>: <span class="hljs-string">&quot;I-person&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">11</span>: <span class="hljs-string">&quot;B-product&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">12</span>: <span class="hljs-string">&quot;I-product&quot;</span>,
<span class="hljs-meta">... </span>}
<span class="hljs-meta">&gt;&gt;&gt; </span>label2id = {
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;O&quot;</span>: <span class="hljs-number">0</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;B-corporation&quot;</span>: <span class="hljs-number">1</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;I-corporation&quot;</span>: <span class="hljs-number">2</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;B-creative-work&quot;</span>: <span class="hljs-number">3</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;I-creative-work&quot;</span>: <span class="hljs-number">4</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;B-group&quot;</span>: <span class="hljs-number">5</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;I-group&quot;</span>: <span class="hljs-number">6</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;B-location&quot;</span>: <span class="hljs-number">7</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;I-location&quot;</span>: <span class="hljs-number">8</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;B-person&quot;</span>: <span class="hljs-number">9</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;I-person&quot;</span>: <span class="hljs-number">10</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;B-product&quot;</span>: <span class="hljs-number">11</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;I-product&quot;</span>: <span class="hljs-number">12</span>,
<span class="hljs-meta">... </span>}`,wrap:!1}}),ps=new Ba({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[Ul],pytorch:[wl]},$$scope:{ctx:k}}}),rs=new Za({props:{$$slots:{default:[$l]},$$scope:{ctx:k}}}),Vs=new qs({props:{title:"Inference",local:"inference",headingTag:"h2"}}),zs=new _({props:{code:"dGV4dCUyMCUzRCUyMCUyMlRoZSUyMEdvbGRlbiUyMFN0YXRlJTIwV2FycmlvcnMlMjBhcmUlMjBhbiUyMEFtZXJpY2FuJTIwcHJvZmVzc2lvbmFsJTIwYmFza2V0YmFsbCUyMHRlYW0lMjBiYXNlZCUyMGluJTIwU2FuJTIwRnJhbmNpc2NvLiUyMg==",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>text = <span class="hljs-string">&quot;The Golden State Warriors are an American professional basketball team based in San Francisco.&quot;</span>',wrap:!1}}),Qs=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMHBpcGVsaW5lJTBBJTBBY2xhc3NpZmllciUyMCUzRCUyMHBpcGVsaW5lKCUyMm5lciUyMiUyQyUyMG1vZGVsJTNEJTIyc3RldmhsaXUlMkZteV9hd2Vzb21lX3dudXRfbW9kZWwlMjIpJTBBY2xhc3NpZmllcih0ZXh0KQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

<span class="hljs-meta">&gt;&gt;&gt; </span>classifier = pipeline(<span class="hljs-string">&quot;ner&quot;</span>, model=<span class="hljs-string">&quot;stevhliu/my_awesome_wnut_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>classifier(text)
[{<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;B-location&#x27;</span>,
  <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.42658573</span>,
  <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">2</span>,
  <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;golden&#x27;</span>,
  <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">4</span>,
  <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">10</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-location&#x27;</span>,
  <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.35856336</span>,
  <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">3</span>,
  <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;state&#x27;</span>,
  <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">11</span>,
  <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">16</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;B-group&#x27;</span>,
  <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.3064001</span>,
  <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">4</span>,
  <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;warriors&#x27;</span>,
  <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">17</span>,
  <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">25</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;B-location&#x27;</span>,
  <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.65523505</span>,
  <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">13</span>,
  <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;san&#x27;</span>,
  <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">80</span>,
  <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">83</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;B-location&#x27;</span>,
  <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.4668663</span>,
  <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">14</span>,
  <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;francisco&#x27;</span>,
  <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">84</span>,
  <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">93</span>}]`,wrap:!1}}),os=new Ba({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[Cl],pytorch:[_l]},$$scope:{ctx:k}}}),{c(){t=b("meta"),c=o(),a=b("p"),m=o(),M(g.$$.fragment),C=o(),M(A.$$.fragment),Z=o(),M(I.$$.fragment),B=o(),x=b("p"),x.textContent=X,G=o(),$=b("p"),$.textContent=R,r=o(),U=b("ol"),U.innerHTML=V,E=o(),M(W.$$.fragment),h=o(),v=b("p"),v.textContent=F,cs=o(),M(H.$$.fragment),z=o(),N=b("p"),N.textContent=as,ms=o(),M(Q.$$.fragment),Y=o(),M(q.$$.fragment),L=o(),ls=b("p"),ls.textContent=hs,S=o(),M(D.$$.fragment),P=o(),ts=b("p"),ts.textContent=Ms,K=o(),M(O.$$.fragment),ss=o(),es=b("p"),es.innerHTML=js,p=o(),M(J.$$.fragment),ds=o(),us=b("p"),us.innerHTML=Ga,Ds=o(),fs=b("ul"),fs.innerHTML=Ra,Ps=o(),M(ys.$$.fragment),Ks=o(),M(gs.$$.fragment),Os=o(),bs=b("p"),bs.innerHTML=Aa,sa=o(),M(ws.$$.fragment),aa=o(),Ts=b("p"),Ts.innerHTML=Wa,la=o(),M(Js.$$.fragment),ta=o(),Us=b("p"),Us.innerHTML=Ea,ea=o(),$s=b("ol"),$s.innerHTML=Xa,na=o(),xs=b("p"),xs.textContent=Va,pa=o(),M(_s.$$.fragment),ra=o(),ks=b("p"),ks.innerHTML=Fa,oa=o(),M(Cs.$$.fragment),ia=o(),Is=b("p"),Is.innerHTML=Ha,ca=o(),M(ns.$$.fragment),ma=o(),M(vs.$$.fragment),ha=o(),Zs=b("p"),Zs.innerHTML=za,Ma=o(),M(Bs.$$.fragment),ja=o(),Gs=b("p"),Gs.innerHTML=Na,da=o(),M(Rs.$$.fragment),ua=o(),As=b("p"),As.innerHTML=Qa,fa=o(),M(Ws.$$.fragment),ya=o(),Es=b("p"),Es.innerHTML=Ya,ga=o(),M(Xs.$$.fragment),ba=o(),M(ps.$$.fragment),wa=o(),M(rs.$$.fragment),Ta=o(),M(Vs.$$.fragment),Ja=o(),Fs=b("p"),Fs.textContent=qa,Ua=o(),Hs=b("p"),Hs.textContent=La,$a=o(),M(zs.$$.fragment),xa=o(),Ns=b("p"),Ns.innerHTML=Sa,_a=o(),M(Qs.$$.fragment),ka=o(),Ys=b("p"),Ys.innerHTML=Da,Ca=o(),M(os.$$.fragment),Ia=o(),Ss=b("p"),this.h()},l(s){const n=ml("svelte-u9bgzb",document.head);t=w(n,"META",{name:!0,content:!0}),n.forEach(l),c=i(s),a=w(s,"P",{}),ll(a).forEach(l),m=i(s),j(g.$$.fragment,s),C=i(s),j(A.$$.fragment,s),Z=i(s),j(I.$$.fragment,s),B=i(s),x=w(s,"P",{"data-svelte-h":!0}),T(x)!=="svelte-o0annf"&&(x.textContent=X),G=i(s),$=w(s,"P",{"data-svelte-h":!0}),T($)!=="svelte-1aff4p7"&&($.textContent=R),r=i(s),U=w(s,"OL",{"data-svelte-h":!0}),T(U)!=="svelte-hnqwcf"&&(U.innerHTML=V),E=i(s),j(W.$$.fragment,s),h=i(s),v=w(s,"P",{"data-svelte-h":!0}),T(v)!=="svelte-1c9nexd"&&(v.textContent=F),cs=i(s),j(H.$$.fragment,s),z=i(s),N=w(s,"P",{"data-svelte-h":!0}),T(N)!=="svelte-k76o1m"&&(N.textContent=as),ms=i(s),j(Q.$$.fragment,s),Y=i(s),j(q.$$.fragment,s),L=i(s),ls=w(s,"P",{"data-svelte-h":!0}),T(ls)!=="svelte-tluco0"&&(ls.textContent=hs),S=i(s),j(D.$$.fragment,s),P=i(s),ts=w(s,"P",{"data-svelte-h":!0}),T(ts)!=="svelte-1m91ua0"&&(ts.textContent=Ms),K=i(s),j(O.$$.fragment,s),ss=i(s),es=w(s,"P",{"data-svelte-h":!0}),T(es)!=="svelte-1684qxr"&&(es.innerHTML=js),p=i(s),j(J.$$.fragment,s),ds=i(s),us=w(s,"P",{"data-svelte-h":!0}),T(us)!=="svelte-b2md1a"&&(us.innerHTML=Ga),Ds=i(s),fs=w(s,"UL",{"data-svelte-h":!0}),T(fs)!=="svelte-13punrg"&&(fs.innerHTML=Ra),Ps=i(s),j(ys.$$.fragment,s),Ks=i(s),j(gs.$$.fragment,s),Os=i(s),bs=w(s,"P",{"data-svelte-h":!0}),T(bs)!=="svelte-1pk56gi"&&(bs.innerHTML=Aa),sa=i(s),j(ws.$$.fragment,s),aa=i(s),Ts=w(s,"P",{"data-svelte-h":!0}),T(Ts)!=="svelte-el75ld"&&(Ts.innerHTML=Wa),la=i(s),j(Js.$$.fragment,s),ta=i(s),Us=w(s,"P",{"data-svelte-h":!0}),T(Us)!=="svelte-1lf9iv0"&&(Us.innerHTML=Ea),ea=i(s),$s=w(s,"OL",{"data-svelte-h":!0}),T($s)!=="svelte-1heekds"&&($s.innerHTML=Xa),na=i(s),xs=w(s,"P",{"data-svelte-h":!0}),T(xs)!=="svelte-yddais"&&(xs.textContent=Va),pa=i(s),j(_s.$$.fragment,s),ra=i(s),ks=w(s,"P",{"data-svelte-h":!0}),T(ks)!=="svelte-1fasmjz"&&(ks.innerHTML=Fa),oa=i(s),j(Cs.$$.fragment,s),ia=i(s),Is=w(s,"P",{"data-svelte-h":!0}),T(Is)!=="svelte-pjl5y5"&&(Is.innerHTML=Ha),ca=i(s),j(ns.$$.fragment,s),ma=i(s),j(vs.$$.fragment,s),ha=i(s),Zs=w(s,"P",{"data-svelte-h":!0}),T(Zs)!=="svelte-434hvn"&&(Zs.innerHTML=za),Ma=i(s),j(Bs.$$.fragment,s),ja=i(s),Gs=w(s,"P",{"data-svelte-h":!0}),T(Gs)!=="svelte-np7vj3"&&(Gs.innerHTML=Na),da=i(s),j(Rs.$$.fragment,s),ua=i(s),As=w(s,"P",{"data-svelte-h":!0}),T(As)!=="svelte-183aynn"&&(As.innerHTML=Qa),fa=i(s),j(Ws.$$.fragment,s),ya=i(s),Es=w(s,"P",{"data-svelte-h":!0}),T(Es)!=="svelte-18c6io4"&&(Es.innerHTML=Ya),ga=i(s),j(Xs.$$.fragment,s),ba=i(s),j(ps.$$.fragment,s),wa=i(s),j(rs.$$.fragment,s),Ta=i(s),j(Vs.$$.fragment,s),Ja=i(s),Fs=w(s,"P",{"data-svelte-h":!0}),T(Fs)!=="svelte-633ppb"&&(Fs.textContent=qa),Ua=i(s),Hs=w(s,"P",{"data-svelte-h":!0}),T(Hs)!=="svelte-o1jbfg"&&(Hs.textContent=La),$a=i(s),j(zs.$$.fragment,s),xa=i(s),Ns=w(s,"P",{"data-svelte-h":!0}),T(Ns)!=="svelte-ap335r"&&(Ns.innerHTML=Sa),_a=i(s),j(Qs.$$.fragment,s),ka=i(s),Ys=w(s,"P",{"data-svelte-h":!0}),T(Ys)!=="svelte-1njl8vm"&&(Ys.innerHTML=Da),Ca=i(s),j(os.$$.fragment,s),Ia=i(s),Ss=w(s,"P",{}),ll(Ss).forEach(l),this.h()},h(){tl(t,"name","hf:doc:metadata"),tl(t,"content",vl)},m(s,n){hl(document.head,t),e(s,c,n),e(s,a,n),e(s,m,n),d(g,s,n),e(s,C,n),d(A,s,n),e(s,Z,n),d(I,s,n),e(s,B,n),e(s,x,n),e(s,G,n),e(s,$,n),e(s,r,n),e(s,U,n),e(s,E,n),d(W,s,n),e(s,h,n),e(s,v,n),e(s,cs,n),d(H,s,n),e(s,z,n),e(s,N,n),e(s,ms,n),d(Q,s,n),e(s,Y,n),d(q,s,n),e(s,L,n),e(s,ls,n),e(s,S,n),d(D,s,n),e(s,P,n),e(s,ts,n),e(s,K,n),d(O,s,n),e(s,ss,n),e(s,es,n),e(s,p,n),d(J,s,n),e(s,ds,n),e(s,us,n),e(s,Ds,n),e(s,fs,n),e(s,Ps,n),d(ys,s,n),e(s,Ks,n),d(gs,s,n),e(s,Os,n),e(s,bs,n),e(s,sa,n),d(ws,s,n),e(s,aa,n),e(s,Ts,n),e(s,la,n),d(Js,s,n),e(s,ta,n),e(s,Us,n),e(s,ea,n),e(s,$s,n),e(s,na,n),e(s,xs,n),e(s,pa,n),d(_s,s,n),e(s,ra,n),e(s,ks,n),e(s,oa,n),d(Cs,s,n),e(s,ia,n),e(s,Is,n),e(s,ca,n),d(ns,s,n),e(s,ma,n),d(vs,s,n),e(s,ha,n),e(s,Zs,n),e(s,Ma,n),d(Bs,s,n),e(s,ja,n),e(s,Gs,n),e(s,da,n),d(Rs,s,n),e(s,ua,n),e(s,As,n),e(s,fa,n),d(Ws,s,n),e(s,ya,n),e(s,Es,n),e(s,ga,n),d(Xs,s,n),e(s,ba,n),d(ps,s,n),e(s,wa,n),d(rs,s,n),e(s,Ta,n),d(Vs,s,n),e(s,Ja,n),e(s,Fs,n),e(s,Ua,n),e(s,Hs,n),e(s,$a,n),d(zs,s,n),e(s,xa,n),e(s,Ns,n),e(s,_a,n),d(Qs,s,n),e(s,ka,n),e(s,Ys,n),e(s,Ca,n),d(os,s,n),e(s,Ia,n),e(s,Ss,n),va=!0},p(s,[n]){const Pa={};n&2&&(Pa.$$scope={dirty:n,ctx:s}),W.$set(Pa);const Ka={};n&2&&(Ka.$$scope={dirty:n,ctx:s}),ns.$set(Ka);const Oa={};n&2&&(Oa.$$scope={dirty:n,ctx:s}),ps.$set(Oa);const sl={};n&2&&(sl.$$scope={dirty:n,ctx:s}),rs.$set(sl);const al={};n&2&&(al.$$scope={dirty:n,ctx:s}),os.$set(al)},i(s){va||(u(g.$$.fragment,s),u(A.$$.fragment,s),u(I.$$.fragment,s),u(W.$$.fragment,s),u(H.$$.fragment,s),u(Q.$$.fragment,s),u(q.$$.fragment,s),u(D.$$.fragment,s),u(O.$$.fragment,s),u(J.$$.fragment,s),u(ys.$$.fragment,s),u(gs.$$.fragment,s),u(ws.$$.fragment,s),u(Js.$$.fragment,s),u(_s.$$.fragment,s),u(Cs.$$.fragment,s),u(ns.$$.fragment,s),u(vs.$$.fragment,s),u(Bs.$$.fragment,s),u(Rs.$$.fragment,s),u(Ws.$$.fragment,s),u(Xs.$$.fragment,s),u(ps.$$.fragment,s),u(rs.$$.fragment,s),u(Vs.$$.fragment,s),u(zs.$$.fragment,s),u(Qs.$$.fragment,s),u(os.$$.fragment,s),va=!0)},o(s){f(g.$$.fragment,s),f(A.$$.fragment,s),f(I.$$.fragment,s),f(W.$$.fragment,s),f(H.$$.fragment,s),f(Q.$$.fragment,s),f(q.$$.fragment,s),f(D.$$.fragment,s),f(O.$$.fragment,s),f(J.$$.fragment,s),f(ys.$$.fragment,s),f(gs.$$.fragment,s),f(ws.$$.fragment,s),f(Js.$$.fragment,s),f(_s.$$.fragment,s),f(Cs.$$.fragment,s),f(ns.$$.fragment,s),f(vs.$$.fragment,s),f(Bs.$$.fragment,s),f(Rs.$$.fragment,s),f(Ws.$$.fragment,s),f(Xs.$$.fragment,s),f(ps.$$.fragment,s),f(rs.$$.fragment,s),f(Vs.$$.fragment,s),f(zs.$$.fragment,s),f(Qs.$$.fragment,s),f(os.$$.fragment,s),va=!1},d(s){s&&(l(c),l(a),l(m),l(C),l(Z),l(B),l(x),l(G),l($),l(r),l(U),l(E),l(h),l(v),l(cs),l(z),l(N),l(ms),l(Y),l(L),l(ls),l(S),l(P),l(ts),l(K),l(ss),l(es),l(p),l(ds),l(us),l(Ds),l(fs),l(Ps),l(Ks),l(Os),l(bs),l(sa),l(aa),l(Ts),l(la),l(ta),l(Us),l(ea),l($s),l(na),l(xs),l(pa),l(ra),l(ks),l(oa),l(ia),l(Is),l(ca),l(ma),l(ha),l(Zs),l(Ma),l(ja),l(Gs),l(da),l(ua),l(As),l(fa),l(ya),l(Es),l(ga),l(ba),l(wa),l(Ta),l(Ja),l(Fs),l(Ua),l(Hs),l($a),l(xa),l(Ns),l(_a),l(ka),l(Ys),l(Ca),l(Ia),l(Ss)),l(t),y(g,s),y(A,s),y(I,s),y(W,s),y(H,s),y(Q,s),y(q,s),y(D,s),y(O,s),y(J,s),y(ys,s),y(gs,s),y(ws,s),y(Js,s),y(_s,s),y(Cs,s),y(ns,s),y(vs,s),y(Bs,s),y(Rs,s),y(Ws,s),y(Xs,s),y(ps,s),y(rs,s),y(Vs,s),y(zs,s),y(Qs,s),y(os,s)}}}const vl='{"title":"Token classification","local":"token-classification","sections":[{"title":"Load WNUT 17 dataset","local":"load-wnut-17-dataset","sections":[],"depth":2},{"title":"Preprocess","local":"preprocess","sections":[],"depth":2},{"title":"Evaluate","local":"evaluate","sections":[],"depth":2},{"title":"Train","local":"train","sections":[],"depth":2},{"title":"Inference","local":"inference","sections":[],"depth":2}],"depth":1}';function Zl(k){return ol(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Fl extends il{constructor(t){super(),cl(this,t,Zl,Il,rl,{})}}export{Fl as component};
