import{s as ko,o as $o,n as Ha}from"../chunks/scheduler.01eeda35.js";import{S as vo,i as xo,g as l,s as o,r as m,A as Co,h as s,f as a,c as n,j as k,u as p,x as d,k as _,y as t,a as i,v as c,d as g,t as f,w as u,m as yo,n as To}from"../chunks/index.6dd51b66.js";import{T as Ea}from"../chunks/Tip.de9bae2b.js";import{D as v}from"../chunks/Docstring.cb556860.js";import{H as W,E as Do}from"../chunks/index.58fe8f9d.js";function wo(R){let h,M=`For best performance, this data collator should be used with a dataset having items that are dictionaries or
BatchEncoding, with the <code>&quot;special_tokens_mask&quot;</code> key, as returned by a <a href="/docs/transformers/main/en/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> or a
<a href="/docs/transformers/main/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a> with the argument <code>return_special_tokens_mask=True</code>.`,b,T,E="<li><p>Default Behavior:</p> <ul><li><code>mask_replace_prob=0.8</code>, <code>random_replace_prob=0.1</code>.</li> <li>Expect 80% of masked tokens replaced with <code>[MASK]</code>, 10% replaced with random tokens, and 10% left unchanged.</li></ul></li> <li><p>All masked tokens replaced by <code>[MASK]</code>:</p> <ul><li><code>mask_replace_prob=1.0</code>, <code>random_replace_prob=0.0</code>.</li> <li>Expect all masked tokens to be replaced with <code>[MASK]</code>. No tokens are left unchanged or replaced with random tokens.</li></ul></li> <li><p>No <code>[MASK]</code> replacement, only random tokens:</p> <ul><li><code>mask_replace_prob=0.0</code>, <code>random_replace_prob=1.0</code>.</li> <li>Expect all masked tokens to be replaced with random tokens. No <code>[MASK]</code> replacements or unchanged tokens.</li></ul></li> <li><p>Balanced replacement:</p> <ul><li><code>mask_replace_prob=0.5</code>, <code>random_replace_prob=0.4</code>.</li> <li>Expect 50% of masked tokens replaced with <code>[MASK]</code>, 40% replaced with random tokens, and 10% left unchanged.</li></ul></li>",O,D,ut=`Note:
The sum of <code>mask_replace_prob</code> and <code>random_replace_prob</code> must not exceed 1. If their sum is less than 1, the
remaining proportion will consist of masked tokens left unchanged.`;return{c(){h=l("p"),h.innerHTML=M,b=yo(`
<Example Options and Expectations>
`),T=l("ol"),T.innerHTML=E,O=o(),D=l("p"),D.innerHTML=ut},l(x){h=s(x,"P",{"data-svelte-h":!0}),d(h)!=="svelte-1uxurri"&&(h.innerHTML=M),b=To(x,`
<Example Options and Expectations>
`),T=s(x,"OL",{"data-svelte-h":!0}),d(T)!=="svelte-14xwpjc"&&(T.innerHTML=E),O=n(x),D=s(x,"P",{"data-svelte-h":!0}),d(D)!=="svelte-10bg7dp"&&(D.innerHTML=ut)},m(x,F){i(x,h,F),i(x,b,F),i(x,T,F),i(x,O,F),i(x,D,F)},p:Ha,d(x){x&&(a(h),a(b),a(T),a(O),a(D))}}}function Po(R){let h,M=`This collator relies on details of the implementation of subword tokenization by <a href="/docs/transformers/main/en/model_doc/bert#transformers.BertTokenizer">BertTokenizer</a>, specifically
that subword tokens are prefixed with <em>##</em>. For tokenizers that do not adhere to this scheme, this collator will
produce an output that is roughly equivalent to <code>.DataCollatorForLanguageModeling</code>.`;return{c(){h=l("p"),h.innerHTML=M},l(b){h=s(b,"P",{"data-svelte-h":!0}),d(h)!=="svelte-kpcut1"&&(h.innerHTML=M)},m(b,T){i(b,h,T)},p:Ha,d(b){b&&a(h)}}}function Mo(R){let h,M=`Using <code>DataCollatorWithFlattening</code> will flatten the entire mini batch into single long sequence.
Make sure your attention computation is able to handle it!`;return{c(){h=l("p"),h.innerHTML=M},l(b){h=s(b,"P",{"data-svelte-h":!0}),d(h)!=="svelte-ai37qf"&&(h.innerHTML=M)},m(b,T){i(b,h,T)},p:Ha,d(b){b&&a(h)}}}function Fo(R){let h,M,b,T,E,O,D,ut=`Data collators are objects that will form a batch by using a list of dataset elements as input. These elements are of
the same type as the elements of <code>train_dataset</code> or <code>eval_dataset</code>.`,x,F,Na=`To be able to build batches, data collators may apply some processing (like padding). Some of them (like
<a href="/docs/transformers/main/en/main_classes/data_collator#transformers.DataCollatorForLanguageModeling">DataCollatorForLanguageModeling</a>) also apply some random data augmentation (like random masking)
on the formed batch.`,_t,ne,Va='Examples of use can be found in the <a href="../examples">example scripts</a> or <a href="../notebooks">example notebooks</a>.',bt,re,kt,L,le,Jt,He,Oa=`Very simple data collator that simply collates batches of dict-like objects and performs special handling for
potential keys named:`,Qt,Ne,ja="<li><code>label</code>: handles a single value (int or float) per object</li> <li><code>label_ids</code>: handles a list of values per object</li>",Yt,Ve,Ka=`Does not do any additional preprocessing: property names of the input object will be used as corresponding inputs
to the model. See glue and ner for example of how it’s useful.`,$t,se,vt,w,ie,Zt,Oe,Ba=`Very simple data collator that simply collates batches of dict-like objects and performs special handling for
potential keys named:`,ea,je,Ua="<li><code>label</code>: handles a single value (int or float) per object</li> <li><code>label_ids</code>: handles a list of values per object</li>",ta,Ke,Ra=`Does not do any additional preprocessing: property names of the input object will be used as corresponding inputs
to the model. See glue and ner for example of how it’s useful.`,aa,Be,Ga=`This is an object (like other data collators) rather than a pure function like default_data_collator. This can be
helpful if you need to set a return_tensors value at initialization.`,xt,de,Ct,j,me,oa,Ue,Xa="Data collator that will dynamically pad the inputs received.",yt,pe,Tt,K,ce,na,Re,Ja="Data collator that will dynamically pad the inputs received, as well as the labels.",Dt,ge,wt,B,fe,ra,Ge,Qa="Data collator that will dynamically pad the inputs received, as well as the labels.",Pt,ue,Mt,C,he,la,Xe,Ya=`Data collator used for language modeling. Inputs are dynamically padded to the maximum length of a batch if they
are not all of the same length.`,sa,G,ia,X,_e,da,Je,Za="Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original.",ma,J,be,pa,Qe,eo="Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original.",ca,Q,ke,ga,Ye,to="Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original.",Ft,$e,Lt,$,ve,fa,Ze,ao="Data collator used for language modeling that masks entire words.",ua,et,oo="<li>collates batches of tensors, honoring their tokenizer’s pad_token</li> <li>preprocesses batches for masked language modeling</li>",ha,Y,_a,Z,xe,ba,tt,no=`Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. Set
‘mask_labels’ means we use whole word mask (wwm), we directly mask idxs according to it’s ref.`,ka,ee,Ce,$a,at,ro=`Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. Set
‘mask_labels’ means we use whole word mask (wwm), we directly mask idxs according to it’s ref.`,va,te,ye,xa,ot,lo=`Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. Set
‘mask_labels’ means we use whole word mask (wwm), we directly mask idxs according to it’s ref.`,qt,Te,zt,y,De,Ca,nt,so="Data collator used for permutation language modeling.",ya,rt,io="<li>collates batches of tensors, honoring their tokenizer’s pad_token</li> <li>preprocesses batches for permutation language modeling with procedures specific to XLNet</li>",Ta,H,we,Da,lt,mo="The masked tokens to be predicted for a particular sequence are determined by the following algorithm:",wa,Pe,po=`<li>Start from the beginning of the sequence by setting <code>cur_len = 0</code> (number of tokens processed so far).</li> <li>Sample a <code>span_length</code> from the interval <code>[1, max_span_length]</code> (length of span of tokens to be masked)</li> <li>Reserve a context of length <code>context_length = span_length / plm_probability</code> to surround span to be
masked</li> <li>Sample a starting point <code>start_index</code> from the interval <code>[cur_len, cur_len + context_length - span_length]</code> and mask tokens <code>start_index:start_index + span_length</code></li> <li>Set <code>cur_len = cur_len + context_length</code>. If <code>cur_len &lt; max_len</code> (i.e. there are tokens remaining in the
sequence to be processed), repeat from Step 1.</li>`,Pa,N,Me,Ma,st,co="The masked tokens to be predicted for a particular sequence are determined by the following algorithm:",Fa,Fe,go=`<li>Start from the beginning of the sequence by setting <code>cur_len = 0</code> (number of tokens processed so far).</li> <li>Sample a <code>span_length</code> from the interval <code>[1, max_span_length]</code> (length of span of tokens to be masked)</li> <li>Reserve a context of length <code>context_length = span_length / plm_probability</code> to surround span to be
masked</li> <li>Sample a starting point <code>start_index</code> from the interval <code>[cur_len, cur_len + context_length - span_length]</code> and mask tokens <code>start_index:start_index + span_length</code></li> <li>Set <code>cur_len = cur_len + context_length</code>. If <code>cur_len &lt; max_len</code> (i.e. there are tokens remaining in the
sequence to be processed), repeat from Step 1.</li>`,La,V,Le,qa,it,fo="The masked tokens to be predicted for a particular sequence are determined by the following algorithm:",za,qe,uo=`<li>Start from the beginning of the sequence by setting <code>cur_len = 0</code> (number of tokens processed so far).</li> <li>Sample a <code>span_length</code> from the interval <code>[1, max_span_length]</code> (length of span of tokens to be masked)</li> <li>Reserve a context of length <code>context_length = span_length / plm_probability</code> to surround span to be
masked</li> <li>Sample a starting point <code>start_index</code> from the interval <code>[cur_len, cur_len + context_length - span_length]</code> and mask tokens <code>start_index:start_index + span_length</code></li> <li>Set <code>cur_len = cur_len + context_length</code>. If <code>cur_len &lt; max_len</code> (i.e. there are tokens remaining in the
sequence to be processed), repeat from Step 1.</li>`,St,ze,At,q,Se,Sa,dt,ho="Data collator used for padding free approach. Does the following:",Aa,mt,_o="<li>concatate the entire mini batch into single long sequence [1, total_tokens]</li> <li>uses <code>separator_id</code> to separate sequences within the concatenated <code>labels</code>, default value is -100</li> <li>no padding will be added, returns <code>input_ids</code>, <code>labels</code> and <code>position_ids</code></li>",Ia,ae,It,Ae,Wt,U,Ie,Wa,pt,bo=`Data collator that dynamically pads a batch of nested examples for multiple choice, so that all choices
of all examples have the same length.`,Et,We,Ht,ht,Nt;return E=new W({props:{title:"Data Collator",local:"data-collator",headingTag:"h1"}}),re=new W({props:{title:"Default data collator",local:"transformers.default_data_collator",headingTag:"h2"}}),le=new v({props:{name:"transformers.default_data_collator",anchor:"transformers.default_data_collator",parameters:[{name:"features",val:": typing.List[transformers.data.data_collator.InputDataClass]"},{name:"return_tensors",val:" = 'pt'"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L75"}}),se=new W({props:{title:"DefaultDataCollator",local:"transformers.DefaultDataCollator",headingTag:"h2"}}),ie=new v({props:{name:"class transformers.DefaultDataCollator",anchor:"transformers.DefaultDataCollator",parameters:[{name:"return_tensors",val:": str = 'pt'"}],parametersDescription:[{anchor:"transformers.DefaultDataCollator.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;pt&quot;</code>) &#x2014;
The type of Tensor to return. Allowable values are &#x201C;np&#x201D;, &#x201C;pt&#x201D; and &#x201C;tf&#x201D;.`,name:"return_tensors"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L100"}}),de=new W({props:{title:"DataCollatorWithPadding",local:"transformers.DataCollatorWithPadding",headingTag:"h2"}}),me=new v({props:{name:"class transformers.DataCollatorWithPadding",anchor:"transformers.DataCollatorWithPadding",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"padding",val:": typing.Union[bool, str, transformers.utils.generic.PaddingStrategy] = True"},{name:"max_length",val:": typing.Optional[int] = None"},{name:"pad_to_multiple_of",val:": typing.Optional[int] = None"},{name:"return_tensors",val:": str = 'pt'"}],parametersDescription:[{anchor:"transformers.DataCollatorWithPadding.tokenizer",description:`<strong>tokenizer</strong> (<a href="/docs/transformers/main/en/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> or <a href="/docs/transformers/main/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a>) &#x2014;
The tokenizer used for encoding the data.`,name:"tokenizer"},{anchor:"transformers.DataCollatorWithPadding.padding",description:`<strong>padding</strong> (<code>bool</code>, <code>str</code> or <a href="/docs/transformers/main/en/internal/file_utils#transformers.utils.PaddingStrategy">PaddingStrategy</a>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Select a strategy to pad the returned sequences (according to the model&#x2019;s padding side and padding index)
among:</p>
<ul>
<li><code>True</code> or <code>&apos;longest&apos;</code> (default): Pad to the longest sequence in the batch (or no padding if only a single
sequence is provided).</li>
<li><code>&apos;max_length&apos;</code>: Pad to a maximum length specified with the argument <code>max_length</code> or to the maximum
acceptable input length for the model if that argument is not provided.</li>
<li><code>False</code> or <code>&apos;do_not_pad&apos;</code>: No padding (i.e., can output a batch with sequences of different lengths).</li>
</ul>`,name:"padding"},{anchor:"transformers.DataCollatorWithPadding.max_length",description:`<strong>max_length</strong> (<code>int</code>, <em>optional</em>) &#x2014;
Maximum length of the returned list and optionally padding length (see above).`,name:"max_length"},{anchor:"transformers.DataCollatorWithPadding.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) &#x2014;
If set will pad the sequence to a multiple of the provided value.</p>
<p>This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability &gt;=
7.0 (Volta).`,name:"pad_to_multiple_of"},{anchor:"transformers.DataCollatorWithPadding.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;pt&quot;</code>) &#x2014;
The type of Tensor to return. Allowable values are &#x201C;np&#x201D;, &#x201C;pt&#x201D; and &#x201C;tf&#x201D;.`,name:"return_tensors"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L237"}}),pe=new W({props:{title:"DataCollatorForTokenClassification",local:"transformers.DataCollatorForTokenClassification",headingTag:"h2"}}),ce=new v({props:{name:"class transformers.DataCollatorForTokenClassification",anchor:"transformers.DataCollatorForTokenClassification",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"padding",val:": typing.Union[bool, str, transformers.utils.generic.PaddingStrategy] = True"},{name:"max_length",val:": typing.Optional[int] = None"},{name:"pad_to_multiple_of",val:": typing.Optional[int] = None"},{name:"label_pad_token_id",val:": int = -100"},{name:"return_tensors",val:": str = 'pt'"}],parametersDescription:[{anchor:"transformers.DataCollatorForTokenClassification.tokenizer",description:`<strong>tokenizer</strong> (<a href="/docs/transformers/main/en/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> or <a href="/docs/transformers/main/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a>) &#x2014;
The tokenizer used for encoding the data.`,name:"tokenizer"},{anchor:"transformers.DataCollatorForTokenClassification.padding",description:`<strong>padding</strong> (<code>bool</code>, <code>str</code> or <a href="/docs/transformers/main/en/internal/file_utils#transformers.utils.PaddingStrategy">PaddingStrategy</a>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Select a strategy to pad the returned sequences (according to the model&#x2019;s padding side and padding index)
among:</p>
<ul>
<li><code>True</code> or <code>&apos;longest&apos;</code> (default): Pad to the longest sequence in the batch (or no padding if only a single
sequence is provided).</li>
<li><code>&apos;max_length&apos;</code>: Pad to a maximum length specified with the argument <code>max_length</code> or to the maximum
acceptable input length for the model if that argument is not provided.</li>
<li><code>False</code> or <code>&apos;do_not_pad&apos;</code>: No padding (i.e., can output a batch with sequences of different lengths).</li>
</ul>`,name:"padding"},{anchor:"transformers.DataCollatorForTokenClassification.max_length",description:`<strong>max_length</strong> (<code>int</code>, <em>optional</em>) &#x2014;
Maximum length of the returned list and optionally padding length (see above).`,name:"max_length"},{anchor:"transformers.DataCollatorForTokenClassification.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) &#x2014;
If set will pad the sequence to a multiple of the provided value.</p>
<p>This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability &gt;=
7.0 (Volta).`,name:"pad_to_multiple_of"},{anchor:"transformers.DataCollatorForTokenClassification.label_pad_token_id",description:`<strong>label_pad_token_id</strong> (<code>int</code>, <em>optional</em>, defaults to -100) &#x2014;
The id to use when padding the labels (-100 will be automatically ignore by PyTorch loss functions).`,name:"label_pad_token_id"},{anchor:"transformers.DataCollatorForTokenClassification.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;pt&quot;</code>) &#x2014;
The type of Tensor to return. Allowable values are &#x201C;np&#x201D;, &#x201C;pt&#x201D; and &#x201C;tf&#x201D;.`,name:"return_tensors"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L289"}}),ge=new W({props:{title:"DataCollatorForSeq2Seq",local:"transformers.DataCollatorForSeq2Seq",headingTag:"h2"}}),fe=new v({props:{name:"class transformers.DataCollatorForSeq2Seq",anchor:"transformers.DataCollatorForSeq2Seq",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"model",val:": typing.Optional[typing.Any] = None"},{name:"padding",val:": typing.Union[bool, str, transformers.utils.generic.PaddingStrategy] = True"},{name:"max_length",val:": typing.Optional[int] = None"},{name:"pad_to_multiple_of",val:": typing.Optional[int] = None"},{name:"label_pad_token_id",val:": int = -100"},{name:"return_tensors",val:": str = 'pt'"}],parametersDescription:[{anchor:"transformers.DataCollatorForSeq2Seq.tokenizer",description:`<strong>tokenizer</strong> (<a href="/docs/transformers/main/en/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> or <a href="/docs/transformers/main/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a>) &#x2014;
The tokenizer used for encoding the data.`,name:"tokenizer"},{anchor:"transformers.DataCollatorForSeq2Seq.model",description:`<strong>model</strong> (<a href="/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel">PreTrainedModel</a>, <em>optional</em>) &#x2014;
The model that is being trained. If set and has the <em>prepare_decoder_input_ids_from_labels</em>, use it to
prepare the <em>decoder_input_ids</em></p>
<p>This is useful when using <em>label_smoothing</em> to avoid calculating loss twice.`,name:"model"},{anchor:"transformers.DataCollatorForSeq2Seq.padding",description:`<strong>padding</strong> (<code>bool</code>, <code>str</code> or <a href="/docs/transformers/main/en/internal/file_utils#transformers.utils.PaddingStrategy">PaddingStrategy</a>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Select a strategy to pad the returned sequences (according to the model&#x2019;s padding side and padding index)
among:</p>
<ul>
<li><code>True</code> or <code>&apos;longest&apos;</code> (default): Pad to the longest sequence in the batch (or no padding if only a single
sequence is provided).</li>
<li><code>&apos;max_length&apos;</code>: Pad to a maximum length specified with the argument <code>max_length</code> or to the maximum
acceptable input length for the model if that argument is not provided.</li>
<li><code>False</code> or <code>&apos;do_not_pad&apos;</code>: No padding (i.e., can output a batch with sequences of different lengths).</li>
</ul>`,name:"padding"},{anchor:"transformers.DataCollatorForSeq2Seq.max_length",description:`<strong>max_length</strong> (<code>int</code>, <em>optional</em>) &#x2014;
Maximum length of the returned list and optionally padding length (see above).`,name:"max_length"},{anchor:"transformers.DataCollatorForSeq2Seq.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) &#x2014;
If set will pad the sequence to a multiple of the provided value.</p>
<p>This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability &gt;=
7.0 (Volta).`,name:"pad_to_multiple_of"},{anchor:"transformers.DataCollatorForSeq2Seq.label_pad_token_id",description:`<strong>label_pad_token_id</strong> (<code>int</code>, <em>optional</em>, defaults to -100) &#x2014;
The id to use when padding the labels (-100 will be automatically ignored by PyTorch loss functions).`,name:"label_pad_token_id"},{anchor:"transformers.DataCollatorForSeq2Seq.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;pt&quot;</code>) &#x2014;
The type of Tensor to return. Allowable values are &#x201C;np&#x201D;, &#x201C;pt&#x201D; and &#x201C;tf&#x201D;.`,name:"return_tensors"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L627"}}),ue=new W({props:{title:"DataCollatorForLanguageModeling",local:"transformers.DataCollatorForLanguageModeling",headingTag:"h2"}}),he=new v({props:{name:"class transformers.DataCollatorForLanguageModeling",anchor:"transformers.DataCollatorForLanguageModeling",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"mlm",val:": bool = True"},{name:"mlm_probability",val:": float = 0.15"},{name:"mask_replace_prob",val:": float = 0.8"},{name:"random_replace_prob",val:": float = 0.1"},{name:"pad_to_multiple_of",val:": typing.Optional[int] = None"},{name:"tf_experimental_compile",val:": bool = False"},{name:"return_tensors",val:": str = 'pt'"},{name:"seed",val:": typing.Optional[int] = None"}],parametersDescription:[{anchor:"transformers.DataCollatorForLanguageModeling.tokenizer",description:`<strong>tokenizer</strong> (<a href="/docs/transformers/main/en/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> or <a href="/docs/transformers/main/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a>) &#x2014;
The tokenizer used for encoding the data.`,name:"tokenizer"},{anchor:"transformers.DataCollatorForLanguageModeling.mlm",description:`<strong>mlm</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether or not to use masked language modeling. If set to <code>False</code>, the labels are the same as the inputs
with the padding tokens ignored (by setting them to -100). Otherwise, the labels are -100 for non-masked
tokens and the value to predict for the masked token.`,name:"mlm"},{anchor:"transformers.DataCollatorForLanguageModeling.mlm_probability",description:`<strong>mlm_probability</strong> (<code>float</code>, <em>optional</em>, defaults to 0.15) &#x2014;
The probability with which to (randomly) mask tokens in the input, when <code>mlm</code> is set to <code>True</code>.`,name:"mlm_probability"},{anchor:"transformers.DataCollatorForLanguageModeling.mask_replace_prob",description:`<strong>mask_replace_prob</strong> (<code>float</code>, <em>optional</em>, defaults to 0.8) &#x2014;
The probability with which masked tokens are replaced by the tokenizer&#x2019;s mask token (e.g., <code>[MASK]</code>).
Defaults to 0.8, meaning 80% of the masked tokens will be replaced with <code>[MASK]</code>.
Only works when <code>mlm</code> is set to <code>True</code>.`,name:"mask_replace_prob"},{anchor:"transformers.DataCollatorForLanguageModeling.random_replace_prob",description:`<strong>random_replace_prob</strong> (<code>float</code>, <em>optional</em>, defaults to 0.1) &#x2014;
The probability with which masked tokens are replaced by random tokens from the tokenizer&#x2019;s vocabulary.
Defaults to 0.1, meaning 10% of the masked tokens will be replaced with random tokens. The remaining
masked tokens (1 - mask_replace_prob - random_replace_prob) are left unchanged.
Only works when <code>mlm</code> is set to <code>True</code>.`,name:"random_replace_prob"},{anchor:"transformers.DataCollatorForLanguageModeling.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) &#x2014;
If set, will pad the sequence to a multiple of the provided value.`,name:"pad_to_multiple_of"},{anchor:"transformers.DataCollatorForLanguageModeling.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>) &#x2014;
The type of Tensor to return. Allowable values are &#x201C;np&#x201D;, &#x201C;pt&#x201D; and &#x201C;tf&#x201D;.`,name:"return_tensors"},{anchor:"transformers.DataCollatorForLanguageModeling.seed",description:`<strong>seed</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The seed to use for the random number generator for masking. If not provided, the global RNG will be used.`,name:"seed"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L763"}}),G=new Ea({props:{$$slots:{default:[wo]},$$scope:{ctx:R}}}),_e=new v({props:{name:"numpy_mask_tokens",anchor:"transformers.DataCollatorForLanguageModeling.numpy_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"special_tokens_mask",val:": typing.Optional[typing.Any] = None"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L1113"}}),be=new v({props:{name:"tf_mask_tokens",anchor:"transformers.DataCollatorForLanguageModeling.tf_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"vocab_size",val:""},{name:"mask_token_id",val:""},{name:"special_tokens_mask",val:": typing.Optional[typing.Any] = None"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L911"}}),ke=new v({props:{name:"torch_mask_tokens",anchor:"transformers.DataCollatorForLanguageModeling.torch_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"special_tokens_mask",val:": typing.Optional[typing.Any] = None"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L1034"}}),$e=new W({props:{title:"DataCollatorForWholeWordMask",local:"transformers.DataCollatorForWholeWordMask",headingTag:"h2"}}),ve=new v({props:{name:"class transformers.DataCollatorForWholeWordMask",anchor:"transformers.DataCollatorForWholeWordMask",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"mlm",val:": bool = True"},{name:"mlm_probability",val:": float = 0.15"},{name:"mask_replace_prob",val:": float = 0.8"},{name:"random_replace_prob",val:": float = 0.1"},{name:"pad_to_multiple_of",val:": typing.Optional[int] = None"},{name:"tf_experimental_compile",val:": bool = False"},{name:"return_tensors",val:": str = 'pt'"},{name:"seed",val:": typing.Optional[int] = None"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L1179"}}),Y=new Ea({props:{$$slots:{default:[Po]},$$scope:{ctx:R}}}),xe=new v({props:{name:"numpy_mask_tokens",anchor:"transformers.DataCollatorForWholeWordMask.numpy_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"mask_labels",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L1476"}}),Ce=new v({props:{name:"tf_mask_tokens",anchor:"transformers.DataCollatorForWholeWordMask.tf_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"mask_labels",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L1416"}}),ye=new v({props:{name:"torch_mask_tokens",anchor:"transformers.DataCollatorForWholeWordMask.torch_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"mask_labels",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L1360"}}),Te=new W({props:{title:"DataCollatorForPermutationLanguageModeling",local:"transformers.DataCollatorForPermutationLanguageModeling",headingTag:"h2"}}),De=new v({props:{name:"class transformers.DataCollatorForPermutationLanguageModeling",anchor:"transformers.DataCollatorForPermutationLanguageModeling",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"plm_probability",val:": float = 0.16666666666666666"},{name:"max_span_length",val:": int = 5"},{name:"return_tensors",val:": str = 'pt'"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L1633"}}),we=new v({props:{name:"numpy_mask_tokens",anchor:"transformers.DataCollatorForPermutationLanguageModeling.numpy_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L1874"}}),Me=new v({props:{name:"tf_mask_tokens",anchor:"transformers.DataCollatorForPermutationLanguageModeling.tf_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L1767"}}),Le=new v({props:{name:"torch_mask_tokens",anchor:"transformers.DataCollatorForPermutationLanguageModeling.torch_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L1668"}}),ze=new W({props:{title:"DataCollatorWithFlattening",local:"transformers.DataCollatorWithFlattening",headingTag:"h2"}}),Se=new v({props:{name:"class transformers.DataCollatorWithFlattening",anchor:"transformers.DataCollatorWithFlattening",parameters:[{name:"*args",val:""},{name:"return_position_ids",val:" = True"},{name:"separator_id",val:" = -100"},{name:"**kwargs",val:""}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L1972"}}),ae=new Ea({props:{warning:!0,$$slots:{default:[Mo]},$$scope:{ctx:R}}}),Ae=new W({props:{title:"DataCollatorForMultipleChoice",local:"transformers.DataCollatorForMultipleChoice",headingTag:"h1"}}),Ie=new v({props:{name:"class transformers.DataCollatorForMultipleChoice",anchor:"transformers.DataCollatorForMultipleChoice",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"padding",val:": typing.Union[bool, str, transformers.utils.generic.PaddingStrategy] = True"},{name:"max_length",val:": typing.Optional[int] = None"},{name:"pad_to_multiple_of",val:": typing.Optional[int] = None"},{name:"return_tensors",val:": str = 'pt'"}],parametersDescription:[{anchor:"transformers.DataCollatorForMultipleChoice.tokenizer",description:`<strong>tokenizer</strong> (<a href="/docs/transformers/main/en/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> or <a href="/docs/transformers/main/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a>) &#x2014;
The tokenizer used for encoding the data.`,name:"tokenizer"},{anchor:"transformers.DataCollatorForMultipleChoice.padding",description:`<strong>padding</strong> (<code>bool</code>, <code>str</code> or <a href="/docs/transformers/main/en/internal/file_utils#transformers.utils.PaddingStrategy">PaddingStrategy</a>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Select a strategy to pad the returned sequences according to the model&#x2019;s padding side and padding index
among:</p>
<ul>
<li><code>True</code> or <code>&apos;longest&apos;</code>: Pad to the longest sequence in the batch (or no padding if only a single sequence
is provided).</li>
<li><code>&apos;max_length&apos;</code>: Pad to a maximum length specified with the argument <code>max_length</code> or to the maximum
acceptable input length for the model if that argument is not provided.</li>
<li><code>False</code> or <code>&apos;do_not_pad&apos;</code> (default): No padding (i.e., can output a batch with sequences of different
lengths).</li>
</ul>`,name:"padding"},{anchor:"transformers.DataCollatorForMultipleChoice.max_length",description:`<strong>max_length</strong> (<code>int</code>, <em>optional</em>) &#x2014;
Maximum length of the returned list and optionally padding length (see above).`,name:"max_length"},{anchor:"transformers.DataCollatorForMultipleChoice.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) &#x2014;
Pad the sequence to a multiple of the provided value.</p>
<p>This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability &gt;=
7.5 (Volta).`,name:"pad_to_multiple_of"},{anchor:"transformers.DataCollatorForMultipleChoice.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;pt&quot;</code>) &#x2014;
The type of Tensor to return. Allowable values are &#x201C;np&#x201D;, &#x201C;pt&#x201D; and &#x201C;tf&#x201D;.`,name:"return_tensors"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L536"}}),We=new Do({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/main_classes/data_collator.md"}}),{c(){h=l("meta"),M=o(),b=l("p"),T=o(),m(E.$$.fragment),O=o(),D=l("p"),D.innerHTML=ut,x=o(),F=l("p"),F.innerHTML=Na,_t=o(),ne=l("p"),ne.innerHTML=Va,bt=o(),m(re.$$.fragment),kt=o(),L=l("div"),m(le.$$.fragment),Jt=o(),He=l("p"),He.textContent=Oa,Qt=o(),Ne=l("ul"),Ne.innerHTML=ja,Yt=o(),Ve=l("p"),Ve.textContent=Ka,$t=o(),m(se.$$.fragment),vt=o(),w=l("div"),m(ie.$$.fragment),Zt=o(),Oe=l("p"),Oe.textContent=Ba,ea=o(),je=l("ul"),je.innerHTML=Ua,ta=o(),Ke=l("p"),Ke.textContent=Ra,aa=o(),Be=l("p"),Be.textContent=Ga,xt=o(),m(de.$$.fragment),Ct=o(),j=l("div"),m(me.$$.fragment),oa=o(),Ue=l("p"),Ue.textContent=Xa,yt=o(),m(pe.$$.fragment),Tt=o(),K=l("div"),m(ce.$$.fragment),na=o(),Re=l("p"),Re.textContent=Ja,Dt=o(),m(ge.$$.fragment),wt=o(),B=l("div"),m(fe.$$.fragment),ra=o(),Ge=l("p"),Ge.textContent=Qa,Pt=o(),m(ue.$$.fragment),Mt=o(),C=l("div"),m(he.$$.fragment),la=o(),Xe=l("p"),Xe.textContent=Ya,sa=o(),m(G.$$.fragment),ia=o(),X=l("div"),m(_e.$$.fragment),da=o(),Je=l("p"),Je.textContent=Za,ma=o(),J=l("div"),m(be.$$.fragment),pa=o(),Qe=l("p"),Qe.textContent=eo,ca=o(),Q=l("div"),m(ke.$$.fragment),ga=o(),Ye=l("p"),Ye.textContent=to,Ft=o(),m($e.$$.fragment),Lt=o(),$=l("div"),m(ve.$$.fragment),fa=o(),Ze=l("p"),Ze.textContent=ao,ua=o(),et=l("ul"),et.innerHTML=oo,ha=o(),m(Y.$$.fragment),_a=o(),Z=l("div"),m(xe.$$.fragment),ba=o(),tt=l("p"),tt.textContent=no,ka=o(),ee=l("div"),m(Ce.$$.fragment),$a=o(),at=l("p"),at.textContent=ro,va=o(),te=l("div"),m(ye.$$.fragment),xa=o(),ot=l("p"),ot.textContent=lo,qt=o(),m(Te.$$.fragment),zt=o(),y=l("div"),m(De.$$.fragment),Ca=o(),nt=l("p"),nt.textContent=so,ya=o(),rt=l("ul"),rt.innerHTML=io,Ta=o(),H=l("div"),m(we.$$.fragment),Da=o(),lt=l("p"),lt.textContent=mo,wa=o(),Pe=l("ol"),Pe.innerHTML=po,Pa=o(),N=l("div"),m(Me.$$.fragment),Ma=o(),st=l("p"),st.textContent=co,Fa=o(),Fe=l("ol"),Fe.innerHTML=go,La=o(),V=l("div"),m(Le.$$.fragment),qa=o(),it=l("p"),it.textContent=fo,za=o(),qe=l("ol"),qe.innerHTML=uo,St=o(),m(ze.$$.fragment),At=o(),q=l("div"),m(Se.$$.fragment),Sa=o(),dt=l("p"),dt.textContent=ho,Aa=o(),mt=l("ul"),mt.innerHTML=_o,Ia=o(),m(ae.$$.fragment),It=o(),m(Ae.$$.fragment),Wt=o(),U=l("div"),m(Ie.$$.fragment),Wa=o(),pt=l("p"),pt.textContent=bo,Et=o(),m(We.$$.fragment),Ht=o(),ht=l("p"),this.h()},l(e){const r=Co("svelte-u9bgzb",document.head);h=s(r,"META",{name:!0,content:!0}),r.forEach(a),M=n(e),b=s(e,"P",{}),k(b).forEach(a),T=n(e),p(E.$$.fragment,e),O=n(e),D=s(e,"P",{"data-svelte-h":!0}),d(D)!=="svelte-1kxfoc9"&&(D.innerHTML=ut),x=n(e),F=s(e,"P",{"data-svelte-h":!0}),d(F)!=="svelte-1e3gfra"&&(F.innerHTML=Na),_t=n(e),ne=s(e,"P",{"data-svelte-h":!0}),d(ne)!=="svelte-10kyi1a"&&(ne.innerHTML=Va),bt=n(e),p(re.$$.fragment,e),kt=n(e),L=s(e,"DIV",{class:!0});var I=k(L);p(le.$$.fragment,I),Jt=n(I),He=s(I,"P",{"data-svelte-h":!0}),d(He)!=="svelte-1hmsgsg"&&(He.textContent=Oa),Qt=n(I),Ne=s(I,"UL",{"data-svelte-h":!0}),d(Ne)!=="svelte-1pq8qks"&&(Ne.innerHTML=ja),Yt=n(I),Ve=s(I,"P",{"data-svelte-h":!0}),d(Ve)!=="svelte-1vi1gug"&&(Ve.textContent=Ka),I.forEach(a),$t=n(e),p(se.$$.fragment,e),vt=n(e),w=s(e,"DIV",{class:!0});var z=k(w);p(ie.$$.fragment,z),Zt=n(z),Oe=s(z,"P",{"data-svelte-h":!0}),d(Oe)!=="svelte-1hmsgsg"&&(Oe.textContent=Ba),ea=n(z),je=s(z,"UL",{"data-svelte-h":!0}),d(je)!=="svelte-1pq8qks"&&(je.innerHTML=Ua),ta=n(z),Ke=s(z,"P",{"data-svelte-h":!0}),d(Ke)!=="svelte-1vi1gug"&&(Ke.textContent=Ra),aa=n(z),Be=s(z,"P",{"data-svelte-h":!0}),d(Be)!=="svelte-vfvbwr"&&(Be.textContent=Ga),z.forEach(a),xt=n(e),p(de.$$.fragment,e),Ct=n(e),j=s(e,"DIV",{class:!0});var Ee=k(j);p(me.$$.fragment,Ee),oa=n(Ee),Ue=s(Ee,"P",{"data-svelte-h":!0}),d(Ue)!=="svelte-1iebpai"&&(Ue.textContent=Xa),Ee.forEach(a),yt=n(e),p(pe.$$.fragment,e),Tt=n(e),K=s(e,"DIV",{class:!0});var Vt=k(K);p(ce.$$.fragment,Vt),na=n(Vt),Re=s(Vt,"P",{"data-svelte-h":!0}),d(Re)!=="svelte-4uvw1w"&&(Re.textContent=Ja),Vt.forEach(a),Dt=n(e),p(ge.$$.fragment,e),wt=n(e),B=s(e,"DIV",{class:!0});var Ot=k(B);p(fe.$$.fragment,Ot),ra=n(Ot),Ge=s(Ot,"P",{"data-svelte-h":!0}),d(Ge)!=="svelte-4uvw1w"&&(Ge.textContent=Qa),Ot.forEach(a),Pt=n(e),p(ue.$$.fragment,e),Mt=n(e),C=s(e,"DIV",{class:!0});var S=k(C);p(he.$$.fragment,S),la=n(S),Xe=s(S,"P",{"data-svelte-h":!0}),d(Xe)!=="svelte-10km8jk"&&(Xe.textContent=Ya),sa=n(S),p(G.$$.fragment,S),ia=n(S),X=s(S,"DIV",{class:!0});var jt=k(X);p(_e.$$.fragment,jt),da=n(jt),Je=s(jt,"P",{"data-svelte-h":!0}),d(Je)!=="svelte-iv4xqf"&&(Je.textContent=Za),jt.forEach(a),ma=n(S),J=s(S,"DIV",{class:!0});var Kt=k(J);p(be.$$.fragment,Kt),pa=n(Kt),Qe=s(Kt,"P",{"data-svelte-h":!0}),d(Qe)!=="svelte-iv4xqf"&&(Qe.textContent=eo),Kt.forEach(a),ca=n(S),Q=s(S,"DIV",{class:!0});var Bt=k(Q);p(ke.$$.fragment,Bt),ga=n(Bt),Ye=s(Bt,"P",{"data-svelte-h":!0}),d(Ye)!=="svelte-iv4xqf"&&(Ye.textContent=to),Bt.forEach(a),S.forEach(a),Ft=n(e),p($e.$$.fragment,e),Lt=n(e),$=s(e,"DIV",{class:!0});var P=k($);p(ve.$$.fragment,P),fa=n(P),Ze=s(P,"P",{"data-svelte-h":!0}),d(Ze)!=="svelte-sjqk55"&&(Ze.textContent=ao),ua=n(P),et=s(P,"UL",{"data-svelte-h":!0}),d(et)!=="svelte-d2ozxj"&&(et.innerHTML=oo),ha=n(P),p(Y.$$.fragment,P),_a=n(P),Z=s(P,"DIV",{class:!0});var Ut=k(Z);p(xe.$$.fragment,Ut),ba=n(Ut),tt=s(Ut,"P",{"data-svelte-h":!0}),d(tt)!=="svelte-10msjh1"&&(tt.textContent=no),Ut.forEach(a),ka=n(P),ee=s(P,"DIV",{class:!0});var Rt=k(ee);p(Ce.$$.fragment,Rt),$a=n(Rt),at=s(Rt,"P",{"data-svelte-h":!0}),d(at)!=="svelte-10msjh1"&&(at.textContent=ro),Rt.forEach(a),va=n(P),te=s(P,"DIV",{class:!0});var Gt=k(te);p(ye.$$.fragment,Gt),xa=n(Gt),ot=s(Gt,"P",{"data-svelte-h":!0}),d(ot)!=="svelte-10msjh1"&&(ot.textContent=lo),Gt.forEach(a),P.forEach(a),qt=n(e),p(Te.$$.fragment,e),zt=n(e),y=s(e,"DIV",{class:!0});var A=k(y);p(De.$$.fragment,A),Ca=n(A),nt=s(A,"P",{"data-svelte-h":!0}),d(nt)!=="svelte-12inif1"&&(nt.textContent=so),ya=n(A),rt=s(A,"UL",{"data-svelte-h":!0}),d(rt)!=="svelte-1t4qh7k"&&(rt.innerHTML=io),Ta=n(A),H=s(A,"DIV",{class:!0});var ct=k(H);p(we.$$.fragment,ct),Da=n(ct),lt=s(ct,"P",{"data-svelte-h":!0}),d(lt)!=="svelte-1u70phx"&&(lt.textContent=mo),wa=n(ct),Pe=s(ct,"OL",{start:!0,"data-svelte-h":!0}),d(Pe)!=="svelte-1r6jrbi"&&(Pe.innerHTML=po),ct.forEach(a),Pa=n(A),N=s(A,"DIV",{class:!0});var gt=k(N);p(Me.$$.fragment,gt),Ma=n(gt),st=s(gt,"P",{"data-svelte-h":!0}),d(st)!=="svelte-1u70phx"&&(st.textContent=co),Fa=n(gt),Fe=s(gt,"OL",{start:!0,"data-svelte-h":!0}),d(Fe)!=="svelte-1r6jrbi"&&(Fe.innerHTML=go),gt.forEach(a),La=n(A),V=s(A,"DIV",{class:!0});var ft=k(V);p(Le.$$.fragment,ft),qa=n(ft),it=s(ft,"P",{"data-svelte-h":!0}),d(it)!=="svelte-1u70phx"&&(it.textContent=fo),za=n(ft),qe=s(ft,"OL",{start:!0,"data-svelte-h":!0}),d(qe)!=="svelte-1r6jrbi"&&(qe.innerHTML=uo),ft.forEach(a),A.forEach(a),St=n(e),p(ze.$$.fragment,e),At=n(e),q=s(e,"DIV",{class:!0});var oe=k(q);p(Se.$$.fragment,oe),Sa=n(oe),dt=s(oe,"P",{"data-svelte-h":!0}),d(dt)!=="svelte-4gb9ju"&&(dt.textContent=ho),Aa=n(oe),mt=s(oe,"UL",{"data-svelte-h":!0}),d(mt)!=="svelte-1w5rvhw"&&(mt.innerHTML=_o),Ia=n(oe),p(ae.$$.fragment,oe),oe.forEach(a),It=n(e),p(Ae.$$.fragment,e),Wt=n(e),U=s(e,"DIV",{class:!0});var Xt=k(U);p(Ie.$$.fragment,Xt),Wa=n(Xt),pt=s(Xt,"P",{"data-svelte-h":!0}),d(pt)!=="svelte-143gdbc"&&(pt.textContent=bo),Xt.forEach(a),Et=n(e),p(We.$$.fragment,e),Ht=n(e),ht=s(e,"P",{}),k(ht).forEach(a),this.h()},h(){_(h,"name","hf:doc:metadata"),_(h,"content",Lo),_(L,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(w,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(j,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(K,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(B,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(X,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(J,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(Q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(C,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(Z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(ee,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(te,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_($,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(Pe,"start","0"),_(H,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(Fe,"start","0"),_(N,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(qe,"start","0"),_(V,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(U,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,r){t(document.head,h),i(e,M,r),i(e,b,r),i(e,T,r),c(E,e,r),i(e,O,r),i(e,D,r),i(e,x,r),i(e,F,r),i(e,_t,r),i(e,ne,r),i(e,bt,r),c(re,e,r),i(e,kt,r),i(e,L,r),c(le,L,null),t(L,Jt),t(L,He),t(L,Qt),t(L,Ne),t(L,Yt),t(L,Ve),i(e,$t,r),c(se,e,r),i(e,vt,r),i(e,w,r),c(ie,w,null),t(w,Zt),t(w,Oe),t(w,ea),t(w,je),t(w,ta),t(w,Ke),t(w,aa),t(w,Be),i(e,xt,r),c(de,e,r),i(e,Ct,r),i(e,j,r),c(me,j,null),t(j,oa),t(j,Ue),i(e,yt,r),c(pe,e,r),i(e,Tt,r),i(e,K,r),c(ce,K,null),t(K,na),t(K,Re),i(e,Dt,r),c(ge,e,r),i(e,wt,r),i(e,B,r),c(fe,B,null),t(B,ra),t(B,Ge),i(e,Pt,r),c(ue,e,r),i(e,Mt,r),i(e,C,r),c(he,C,null),t(C,la),t(C,Xe),t(C,sa),c(G,C,null),t(C,ia),t(C,X),c(_e,X,null),t(X,da),t(X,Je),t(C,ma),t(C,J),c(be,J,null),t(J,pa),t(J,Qe),t(C,ca),t(C,Q),c(ke,Q,null),t(Q,ga),t(Q,Ye),i(e,Ft,r),c($e,e,r),i(e,Lt,r),i(e,$,r),c(ve,$,null),t($,fa),t($,Ze),t($,ua),t($,et),t($,ha),c(Y,$,null),t($,_a),t($,Z),c(xe,Z,null),t(Z,ba),t(Z,tt),t($,ka),t($,ee),c(Ce,ee,null),t(ee,$a),t(ee,at),t($,va),t($,te),c(ye,te,null),t(te,xa),t(te,ot),i(e,qt,r),c(Te,e,r),i(e,zt,r),i(e,y,r),c(De,y,null),t(y,Ca),t(y,nt),t(y,ya),t(y,rt),t(y,Ta),t(y,H),c(we,H,null),t(H,Da),t(H,lt),t(H,wa),t(H,Pe),t(y,Pa),t(y,N),c(Me,N,null),t(N,Ma),t(N,st),t(N,Fa),t(N,Fe),t(y,La),t(y,V),c(Le,V,null),t(V,qa),t(V,it),t(V,za),t(V,qe),i(e,St,r),c(ze,e,r),i(e,At,r),i(e,q,r),c(Se,q,null),t(q,Sa),t(q,dt),t(q,Aa),t(q,mt),t(q,Ia),c(ae,q,null),i(e,It,r),c(Ae,e,r),i(e,Wt,r),i(e,U,r),c(Ie,U,null),t(U,Wa),t(U,pt),i(e,Et,r),c(We,e,r),i(e,Ht,r),i(e,ht,r),Nt=!0},p(e,[r]){const I={};r&2&&(I.$$scope={dirty:r,ctx:e}),G.$set(I);const z={};r&2&&(z.$$scope={dirty:r,ctx:e}),Y.$set(z);const Ee={};r&2&&(Ee.$$scope={dirty:r,ctx:e}),ae.$set(Ee)},i(e){Nt||(g(E.$$.fragment,e),g(re.$$.fragment,e),g(le.$$.fragment,e),g(se.$$.fragment,e),g(ie.$$.fragment,e),g(de.$$.fragment,e),g(me.$$.fragment,e),g(pe.$$.fragment,e),g(ce.$$.fragment,e),g(ge.$$.fragment,e),g(fe.$$.fragment,e),g(ue.$$.fragment,e),g(he.$$.fragment,e),g(G.$$.fragment,e),g(_e.$$.fragment,e),g(be.$$.fragment,e),g(ke.$$.fragment,e),g($e.$$.fragment,e),g(ve.$$.fragment,e),g(Y.$$.fragment,e),g(xe.$$.fragment,e),g(Ce.$$.fragment,e),g(ye.$$.fragment,e),g(Te.$$.fragment,e),g(De.$$.fragment,e),g(we.$$.fragment,e),g(Me.$$.fragment,e),g(Le.$$.fragment,e),g(ze.$$.fragment,e),g(Se.$$.fragment,e),g(ae.$$.fragment,e),g(Ae.$$.fragment,e),g(Ie.$$.fragment,e),g(We.$$.fragment,e),Nt=!0)},o(e){f(E.$$.fragment,e),f(re.$$.fragment,e),f(le.$$.fragment,e),f(se.$$.fragment,e),f(ie.$$.fragment,e),f(de.$$.fragment,e),f(me.$$.fragment,e),f(pe.$$.fragment,e),f(ce.$$.fragment,e),f(ge.$$.fragment,e),f(fe.$$.fragment,e),f(ue.$$.fragment,e),f(he.$$.fragment,e),f(G.$$.fragment,e),f(_e.$$.fragment,e),f(be.$$.fragment,e),f(ke.$$.fragment,e),f($e.$$.fragment,e),f(ve.$$.fragment,e),f(Y.$$.fragment,e),f(xe.$$.fragment,e),f(Ce.$$.fragment,e),f(ye.$$.fragment,e),f(Te.$$.fragment,e),f(De.$$.fragment,e),f(we.$$.fragment,e),f(Me.$$.fragment,e),f(Le.$$.fragment,e),f(ze.$$.fragment,e),f(Se.$$.fragment,e),f(ae.$$.fragment,e),f(Ae.$$.fragment,e),f(Ie.$$.fragment,e),f(We.$$.fragment,e),Nt=!1},d(e){e&&(a(M),a(b),a(T),a(O),a(D),a(x),a(F),a(_t),a(ne),a(bt),a(kt),a(L),a($t),a(vt),a(w),a(xt),a(Ct),a(j),a(yt),a(Tt),a(K),a(Dt),a(wt),a(B),a(Pt),a(Mt),a(C),a(Ft),a(Lt),a($),a(qt),a(zt),a(y),a(St),a(At),a(q),a(It),a(Wt),a(U),a(Et),a(Ht),a(ht)),a(h),u(E,e),u(re,e),u(le),u(se,e),u(ie),u(de,e),u(me),u(pe,e),u(ce),u(ge,e),u(fe),u(ue,e),u(he),u(G),u(_e),u(be),u(ke),u($e,e),u(ve),u(Y),u(xe),u(Ce),u(ye),u(Te,e),u(De),u(we),u(Me),u(Le),u(ze,e),u(Se),u(ae),u(Ae,e),u(Ie),u(We,e)}}}const Lo='{"title":"Data Collator","local":"data-collator","sections":[{"title":"Default data collator","local":"transformers.default_data_collator","sections":[],"depth":2},{"title":"DefaultDataCollator","local":"transformers.DefaultDataCollator","sections":[],"depth":2},{"title":"DataCollatorWithPadding","local":"transformers.DataCollatorWithPadding","sections":[],"depth":2},{"title":"DataCollatorForTokenClassification","local":"transformers.DataCollatorForTokenClassification","sections":[],"depth":2},{"title":"DataCollatorForSeq2Seq","local":"transformers.DataCollatorForSeq2Seq","sections":[],"depth":2},{"title":"DataCollatorForLanguageModeling","local":"transformers.DataCollatorForLanguageModeling","sections":[],"depth":2},{"title":"DataCollatorForWholeWordMask","local":"transformers.DataCollatorForWholeWordMask","sections":[],"depth":2},{"title":"DataCollatorForPermutationLanguageModeling","local":"transformers.DataCollatorForPermutationLanguageModeling","sections":[],"depth":2},{"title":"DataCollatorWithFlattening","local":"transformers.DataCollatorWithFlattening","sections":[],"depth":2}],"depth":1}';function qo(R){return $o(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Eo extends vo{constructor(h){super(),xo(this,h,qo,Fo,ko,{})}}export{Eo as component};
