import{s as oo,o as no,n as wa}from"../chunks/scheduler.bdbef820.js";import{S as ro,i as lo,g as l,s as a,r as p,A as so,h as s,f as n,c as o,j as $,u as c,x as d,k as _,y as t,a as i,v as g,d as f,t as u,w as h,m as io,n as mo}from"../chunks/index.33f81d56.js";import{T as Ta}from"../chunks/Tip.34194030.js";import{D as x}from"../chunks/Docstring.b3df4217.js";import{H as V,E as po}from"../chunks/index.474b463a.js";function co(U){let m,F=`For best performance, this data collator should be used with a dataset having items that are dictionaries or
BatchEncoding, with the <code>&quot;special_tokens_mask&quot;</code> key, as returned by a <code>PreTrainedTokenizer</code> or a
<code>PreTrainedTokenizerFast</code> with the argument <code>return_special_tokens_mask=True</code>.`,b,D,I="<li><p>Default Behavior:</p> <ul><li><code>mask_replace_prob=0.8</code>, <code>random_replace_prob=0.1</code>.</li> <li>Expect 80% of masked tokens replaced with <code>[MASK]</code>, 10% replaced with random tokens, and 10% left unchanged.</li></ul></li> <li><p>All masked tokens replaced by <code>[MASK]</code>:</p> <ul><li><code>mask_replace_prob=1.0</code>, <code>random_replace_prob=0.0</code>.</li> <li>Expect all masked tokens to be replaced with <code>[MASK]</code>. No tokens are left unchanged or replaced with random tokens.</li></ul></li> <li><p>No <code>[MASK]</code> replacement, only random tokens:</p> <ul><li><code>mask_replace_prob=0.0</code>, <code>random_replace_prob=1.0</code>.</li> <li>Expect all masked tokens to be replaced with random tokens. No <code>[MASK]</code> replacements or unchanged tokens.</li></ul></li> <li><p>Balanced replacement:</p> <ul><li><code>mask_replace_prob=0.5</code>, <code>random_replace_prob=0.4</code>.</li> <li>Expect 50% of masked tokens replaced with <code>[MASK]</code>, 40% replaced with random tokens, and 10% left unchanged.</li></ul></li>",O,T,dt=`Note:
The sum of <code>mask_replace_prob</code> and <code>random_replace_prob</code> must not exceed 1. If their sum is less than 1, the
remaining proportion will consist of masked tokens left unchanged.`;return{c(){m=l("p"),m.innerHTML=F,b=io(`
<Example Options and Expectations>
`),D=l("ol"),D.innerHTML=I,O=a(),T=l("p"),T.innerHTML=dt},l(v){m=s(v,"P",{"data-svelte-h":!0}),d(m)!=="svelte-fxq6ow"&&(m.innerHTML=F),b=mo(v,`
<Example Options and Expectations>
`),D=s(v,"OL",{"data-svelte-h":!0}),d(D)!=="svelte-14xwpjc"&&(D.innerHTML=I),O=o(v),T=s(v,"P",{"data-svelte-h":!0}),d(T)!=="svelte-10bg7dp"&&(T.innerHTML=dt)},m(v,M){i(v,m,M),i(v,b,M),i(v,D,M),i(v,O,M),i(v,T,M)},p:wa,d(v){v&&(n(m),n(b),n(D),n(O),n(T))}}}function go(U){let m,F=`This collator relies on details of the implementation of subword tokenization by <a href="/docs/transformers/main/ko/model_doc/bert#transformers.BertTokenizer">BertTokenizer</a>, specifically
that subword tokens are prefixed with <em>##</em>. For tokenizers that do not adhere to this scheme, this collator will
produce an output that is roughly equivalent to <code>.DataCollatorForLanguageModeling</code>.`;return{c(){m=l("p"),m.innerHTML=F},l(b){m=s(b,"P",{"data-svelte-h":!0}),d(m)!=="svelte-bqxuku"&&(m.innerHTML=F)},m(b,D){i(b,m,D)},p:wa,d(b){b&&n(m)}}}function fo(U){let m,F=`Using <code>DataCollatorWithFlattening</code> will flatten the entire mini batch into single long sequence.
Make sure your attention computation is able to handle it!`;return{c(){m=l("p"),m.innerHTML=F},l(b){m=s(b,"P",{"data-svelte-h":!0}),d(m)!=="svelte-ai37qf"&&(m.innerHTML=F)},m(b,D){i(b,m,D)},p:wa,d(b){b&&n(m)}}}function uo(U){let m,F,b,D,I,O,T,dt='데이터 콜레이터는 데이터셋 요소들의 리스트를 입력으로 사용하여 배치를 형성하는 객체입니다. 이러한 요소들은 <code>train_dataset</code> 또는 <code>eval_dataset의</code> 요소들과 동일한 타입 입니다. 배치를 구성하기 위해, 데이터 콜레이터는 (패딩과 같은) 일부 처리를 적용할 수 있습니다. <a href="/docs/transformers/main/ko/main_classes/data_collator#transformers.DataCollatorForLanguageModeling">DataCollatorForLanguageModeling</a>과 같은 일부 콜레이터는 형성된 배치에 (무작위 마스킹과 같은) 일부 무작위 데이터 증강도 적용합니다. 사용 예시는 <a href="../examples">예제 스크립트</a>나 <a href="../notebooks">예제 노트북</a>에서 찾을 수 있습니다.',v,M,pt,L,oe,Ht,ze,Pa=`Very simple data collator that simply collates batches of dict-like objects and performs special handling for
potential keys named:`,Nt,We,Fa="<li><code>label</code>: handles a single value (int or float) per object</li> <li><code>label_ids</code>: handles a list of values per object</li>",Vt,Ae,Ma=`Does not do any additional preprocessing: property names of the input object will be used as corresponding inputs
to the model. See glue and ner for example of how it’s useful.`,ct,ne,gt,w,re,Ot,Ie,La=`Very simple data collator that simply collates batches of dict-like objects and performs special handling for
potential keys named:`,jt,Ee,qa="<li><code>label</code>: handles a single value (int or float) per object</li> <li><code>label_ids</code>: handles a list of values per object</li>",Kt,He,Sa=`Does not do any additional preprocessing: property names of the input object will be used as corresponding inputs
to the model. See glue and ner for example of how it’s useful.`,Bt,Ne,za=`This is an object (like other data collators) rather than a pure function like default_data_collator. This can be
helpful if you need to set a return_tensors value at initialization.`,ft,le,ut,j,se,Ut,Ve,Wa="Data collator that will dynamically pad the inputs received.",ht,ie,_t,K,de,Rt,Oe,Aa="Data collator that will dynamically pad the inputs received, as well as the labels.",bt,me,$t,B,pe,Gt,je,Ia="Data collator that will dynamically pad the inputs received, as well as the labels.",kt,ce,vt,C,ge,Xt,Ke,Ea=`Data collator used for language modeling. Inputs are dynamically padded to the maximum length of a batch if they
are not all of the same length.`,Jt,R,Qt,G,fe,Yt,Be,Ha="Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original.",Zt,X,ue,ea,Ue,Na="Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original.",ta,J,he,aa,Re,Va="Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original.",xt,_e,Ct,k,be,oa,Ge,Oa="Data collator used for language modeling that masks entire words.",na,Xe,ja="<li>collates batches of tensors, honoring their tokenizer’s pad_token</li> <li>preprocesses batches for masked language modeling</li>",ra,Q,la,Y,$e,sa,Je,Ka=`Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. Set
‘mask_labels’ means we use whole word mask (wwm), we directly mask idxs according to it’s ref.`,ia,Z,ke,da,Qe,Ba=`Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. Set
‘mask_labels’ means we use whole word mask (wwm), we directly mask idxs according to it’s ref.`,ma,ee,ve,pa,Ye,Ua=`Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original. Set
‘mask_labels’ means we use whole word mask (wwm), we directly mask idxs according to it’s ref.`,yt,xe,Dt,y,Ce,ca,Ze,Ra="Data collator used for permutation language modeling.",ga,et,Ga="<li>collates batches of tensors, honoring their tokenizer’s pad_token</li> <li>preprocesses batches for permutation language modeling with procedures specific to XLNet</li>",fa,E,ye,ua,tt,Xa="The masked tokens to be predicted for a particular sequence are determined by the following algorithm:",ha,De,Ja=`<li>Start from the beginning of the sequence by setting <code>cur_len = 0</code> (number of tokens processed so far).</li> <li>Sample a <code>span_length</code> from the interval <code>[1, max_span_length]</code> (length of span of tokens to be masked)</li> <li>Reserve a context of length <code>context_length = span_length / plm_probability</code> to surround span to be
masked</li> <li>Sample a starting point <code>start_index</code> from the interval <code>[cur_len, cur_len + context_length - span_length]</code> and mask tokens <code>start_index:start_index + span_length</code></li> <li>Set <code>cur_len = cur_len + context_length</code>. If <code>cur_len &lt; max_len</code> (i.e. there are tokens remaining in the
sequence to be processed), repeat from Step 1.</li>`,_a,H,Te,ba,at,Qa="The masked tokens to be predicted for a particular sequence are determined by the following algorithm:",$a,we,Ya=`<li>Start from the beginning of the sequence by setting <code>cur_len = 0</code> (number of tokens processed so far).</li> <li>Sample a <code>span_length</code> from the interval <code>[1, max_span_length]</code> (length of span of tokens to be masked)</li> <li>Reserve a context of length <code>context_length = span_length / plm_probability</code> to surround span to be
masked</li> <li>Sample a starting point <code>start_index</code> from the interval <code>[cur_len, cur_len + context_length - span_length]</code> and mask tokens <code>start_index:start_index + span_length</code></li> <li>Set <code>cur_len = cur_len + context_length</code>. If <code>cur_len &lt; max_len</code> (i.e. there are tokens remaining in the
sequence to be processed), repeat from Step 1.</li>`,ka,N,Pe,va,ot,Za="The masked tokens to be predicted for a particular sequence are determined by the following algorithm:",xa,Fe,eo=`<li>Start from the beginning of the sequence by setting <code>cur_len = 0</code> (number of tokens processed so far).</li> <li>Sample a <code>span_length</code> from the interval <code>[1, max_span_length]</code> (length of span of tokens to be masked)</li> <li>Reserve a context of length <code>context_length = span_length / plm_probability</code> to surround span to be
masked</li> <li>Sample a starting point <code>start_index</code> from the interval <code>[cur_len, cur_len + context_length - span_length]</code> and mask tokens <code>start_index:start_index + span_length</code></li> <li>Set <code>cur_len = cur_len + context_length</code>. If <code>cur_len &lt; max_len</code> (i.e. there are tokens remaining in the
sequence to be processed), repeat from Step 1.</li>`,Tt,Me,wt,q,Le,Ca,nt,to="Data collator used for padding free approach. Does the following:",ya,rt,ao="<li>concatate the entire mini batch into single long sequence [1, total_tokens]</li> <li>uses <code>separator_id</code> to separate sequences within the concatenated <code>labels</code>, default value is -100</li> <li>no padding will be added, returns <code>input_ids</code>, <code>labels</code> and <code>position_ids</code></li>",Da,te,Pt,qe,Ft,mt,Mt;return I=new V({props:{title:"데이터 콜레이터(Data Collator)",local:"data-collator",headingTag:"h1"}}),M=new V({props:{title:"기본 데이터 콜레이터",local:"transformers.default_data_collator ][ transformers.default_data_collator",headingTag:"h2"}}),oe=new x({props:{name:"transformers.default_data_collator",anchor:"transformers.default_data_collator",parameters:[{name:"features",val:": typing.List[transformers.data.data_collator.InputDataClass]"},{name:"return_tensors",val:" = 'pt'"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L75"}}),ne=new V({props:{title:"DefaultDataCollator",local:"transformers.DefaultDataCollator ][ transformers.DefaultDataCollator",headingTag:"h2"}}),re=new x({props:{name:"class transformers.DefaultDataCollator",anchor:"transformers.DefaultDataCollator",parameters:[{name:"return_tensors",val:": str = 'pt'"}],parametersDescription:[{anchor:"transformers.DefaultDataCollator.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;pt&quot;</code>) &#x2014;
The type of Tensor to return. Allowable values are &#x201C;np&#x201D;, &#x201C;pt&#x201D; and &#x201C;tf&#x201D;.`,name:"return_tensors"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L100"}}),le=new V({props:{title:"DataCollatorWithPadding",local:"transformers.DataCollatorWithPadding ][ transformers.DataCollatorWithPadding",headingTag:"h2"}}),se=new x({props:{name:"class transformers.DataCollatorWithPadding",anchor:"transformers.DataCollatorWithPadding",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"padding",val:": typing.Union[bool, str, transformers.utils.generic.PaddingStrategy] = True"},{name:"max_length",val:": typing.Optional[int] = None"},{name:"pad_to_multiple_of",val:": typing.Optional[int] = None"},{name:"return_tensors",val:": str = 'pt'"}],parametersDescription:[{anchor:"transformers.DataCollatorWithPadding.tokenizer",description:`<strong>tokenizer</strong> (<code>PreTrainedTokenizer</code> or <code>PreTrainedTokenizerFast</code>) &#x2014;
The tokenizer used for encoding the data.`,name:"tokenizer"},{anchor:"transformers.DataCollatorWithPadding.padding",description:`<strong>padding</strong> (<code>bool</code>, <code>str</code> or <a href="/docs/transformers/main/ko/internal/file_utils#transformers.utils.PaddingStrategy">PaddingStrategy</a>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Select a strategy to pad the returned sequences (according to the model&#x2019;s padding side and padding index)
among:</p>
<ul>
<li><code>True</code> or <code>&apos;longest&apos;</code> (default): Pad to the longest sequence in the batch (or no padding if only a single
sequence is provided).</li>
<li><code>&apos;max_length&apos;</code>: Pad to a maximum length specified with the argument <code>max_length</code> or to the maximum
acceptable input length for the model if that argument is not provided.</li>
<li><code>False</code> or <code>&apos;do_not_pad&apos;</code>: No padding (i.e., can output a batch with sequences of different lengths).</li>
</ul>`,name:"padding"},{anchor:"transformers.DataCollatorWithPadding.max_length",description:`<strong>max_length</strong> (<code>int</code>, <em>optional</em>) &#x2014;
Maximum length of the returned list and optionally padding length (see above).`,name:"max_length"},{anchor:"transformers.DataCollatorWithPadding.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) &#x2014;
If set will pad the sequence to a multiple of the provided value.</p>
<p>This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability &gt;=
7.0 (Volta).`,name:"pad_to_multiple_of"},{anchor:"transformers.DataCollatorWithPadding.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;pt&quot;</code>) &#x2014;
The type of Tensor to return. Allowable values are &#x201C;np&#x201D;, &#x201C;pt&#x201D; and &#x201C;tf&#x201D;.`,name:"return_tensors"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L237"}}),ie=new V({props:{title:"DataCollatorForTokenClassification",local:"transformers.DataCollatorForTokenClassification ][ transformers.DataCollatorForTokenClassification",headingTag:"h2"}}),de=new x({props:{name:"class transformers.DataCollatorForTokenClassification",anchor:"transformers.DataCollatorForTokenClassification",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"padding",val:": typing.Union[bool, str, transformers.utils.generic.PaddingStrategy] = True"},{name:"max_length",val:": typing.Optional[int] = None"},{name:"pad_to_multiple_of",val:": typing.Optional[int] = None"},{name:"label_pad_token_id",val:": int = -100"},{name:"return_tensors",val:": str = 'pt'"}],parametersDescription:[{anchor:"transformers.DataCollatorForTokenClassification.tokenizer",description:`<strong>tokenizer</strong> (<code>PreTrainedTokenizer</code> or <code>PreTrainedTokenizerFast</code>) &#x2014;
The tokenizer used for encoding the data.`,name:"tokenizer"},{anchor:"transformers.DataCollatorForTokenClassification.padding",description:`<strong>padding</strong> (<code>bool</code>, <code>str</code> or <a href="/docs/transformers/main/ko/internal/file_utils#transformers.utils.PaddingStrategy">PaddingStrategy</a>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Select a strategy to pad the returned sequences (according to the model&#x2019;s padding side and padding index)
among:</p>
<ul>
<li><code>True</code> or <code>&apos;longest&apos;</code> (default): Pad to the longest sequence in the batch (or no padding if only a single
sequence is provided).</li>
<li><code>&apos;max_length&apos;</code>: Pad to a maximum length specified with the argument <code>max_length</code> or to the maximum
acceptable input length for the model if that argument is not provided.</li>
<li><code>False</code> or <code>&apos;do_not_pad&apos;</code>: No padding (i.e., can output a batch with sequences of different lengths).</li>
</ul>`,name:"padding"},{anchor:"transformers.DataCollatorForTokenClassification.max_length",description:`<strong>max_length</strong> (<code>int</code>, <em>optional</em>) &#x2014;
Maximum length of the returned list and optionally padding length (see above).`,name:"max_length"},{anchor:"transformers.DataCollatorForTokenClassification.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) &#x2014;
If set will pad the sequence to a multiple of the provided value.</p>
<p>This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability &gt;=
7.0 (Volta).`,name:"pad_to_multiple_of"},{anchor:"transformers.DataCollatorForTokenClassification.label_pad_token_id",description:`<strong>label_pad_token_id</strong> (<code>int</code>, <em>optional</em>, defaults to -100) &#x2014;
The id to use when padding the labels (-100 will be automatically ignore by PyTorch loss functions).`,name:"label_pad_token_id"},{anchor:"transformers.DataCollatorForTokenClassification.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;pt&quot;</code>) &#x2014;
The type of Tensor to return. Allowable values are &#x201C;np&#x201D;, &#x201C;pt&#x201D; and &#x201C;tf&#x201D;.`,name:"return_tensors"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L289"}}),me=new V({props:{title:"DataCollatorForSeq2Seq",local:"transformers.DataCollatorForSeq2Seq ][ transformers.DataCollatorForSeq2Seq",headingTag:"h2"}}),pe=new x({props:{name:"class transformers.DataCollatorForSeq2Seq",anchor:"transformers.DataCollatorForSeq2Seq",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"model",val:": typing.Optional[typing.Any] = None"},{name:"padding",val:": typing.Union[bool, str, transformers.utils.generic.PaddingStrategy] = True"},{name:"max_length",val:": typing.Optional[int] = None"},{name:"pad_to_multiple_of",val:": typing.Optional[int] = None"},{name:"label_pad_token_id",val:": int = -100"},{name:"return_tensors",val:": str = 'pt'"}],parametersDescription:[{anchor:"transformers.DataCollatorForSeq2Seq.tokenizer",description:`<strong>tokenizer</strong> (<code>PreTrainedTokenizer</code> or <code>PreTrainedTokenizerFast</code>) &#x2014;
The tokenizer used for encoding the data.`,name:"tokenizer"},{anchor:"transformers.DataCollatorForSeq2Seq.model",description:`<strong>model</strong> (<a href="/docs/transformers/main/ko/main_classes/model#transformers.PreTrainedModel">PreTrainedModel</a>, <em>optional</em>) &#x2014;
The model that is being trained. If set and has the <em>prepare_decoder_input_ids_from_labels</em>, use it to
prepare the <em>decoder_input_ids</em></p>
<p>This is useful when using <em>label_smoothing</em> to avoid calculating loss twice.`,name:"model"},{anchor:"transformers.DataCollatorForSeq2Seq.padding",description:`<strong>padding</strong> (<code>bool</code>, <code>str</code> or <a href="/docs/transformers/main/ko/internal/file_utils#transformers.utils.PaddingStrategy">PaddingStrategy</a>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Select a strategy to pad the returned sequences (according to the model&#x2019;s padding side and padding index)
among:</p>
<ul>
<li><code>True</code> or <code>&apos;longest&apos;</code> (default): Pad to the longest sequence in the batch (or no padding if only a single
sequence is provided).</li>
<li><code>&apos;max_length&apos;</code>: Pad to a maximum length specified with the argument <code>max_length</code> or to the maximum
acceptable input length for the model if that argument is not provided.</li>
<li><code>False</code> or <code>&apos;do_not_pad&apos;</code>: No padding (i.e., can output a batch with sequences of different lengths).</li>
</ul>`,name:"padding"},{anchor:"transformers.DataCollatorForSeq2Seq.max_length",description:`<strong>max_length</strong> (<code>int</code>, <em>optional</em>) &#x2014;
Maximum length of the returned list and optionally padding length (see above).`,name:"max_length"},{anchor:"transformers.DataCollatorForSeq2Seq.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) &#x2014;
If set will pad the sequence to a multiple of the provided value.</p>
<p>This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability &gt;=
7.0 (Volta).`,name:"pad_to_multiple_of"},{anchor:"transformers.DataCollatorForSeq2Seq.label_pad_token_id",description:`<strong>label_pad_token_id</strong> (<code>int</code>, <em>optional</em>, defaults to -100) &#x2014;
The id to use when padding the labels (-100 will be automatically ignored by PyTorch loss functions).`,name:"label_pad_token_id"},{anchor:"transformers.DataCollatorForSeq2Seq.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;pt&quot;</code>) &#x2014;
The type of Tensor to return. Allowable values are &#x201C;np&#x201D;, &#x201C;pt&#x201D; and &#x201C;tf&#x201D;.`,name:"return_tensors"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L627"}}),ce=new V({props:{title:"DataCollatorForLanguageModeling",local:"transformers.DataCollatorForLanguageModeling ][ transformers.DataCollatorForLanguageModeling",headingTag:"h2"}}),ge=new x({props:{name:"class transformers.DataCollatorForLanguageModeling",anchor:"transformers.DataCollatorForLanguageModeling",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"mlm",val:": bool = True"},{name:"mlm_probability",val:": float = 0.15"},{name:"mask_replace_prob",val:": float = 0.8"},{name:"random_replace_prob",val:": float = 0.1"},{name:"pad_to_multiple_of",val:": typing.Optional[int] = None"},{name:"tf_experimental_compile",val:": bool = False"},{name:"return_tensors",val:": str = 'pt'"},{name:"seed",val:": typing.Optional[int] = None"}],parametersDescription:[{anchor:"transformers.DataCollatorForLanguageModeling.tokenizer",description:`<strong>tokenizer</strong> (<code>PreTrainedTokenizer</code> or <code>PreTrainedTokenizerFast</code>) &#x2014;
The tokenizer used for encoding the data.`,name:"tokenizer"},{anchor:"transformers.DataCollatorForLanguageModeling.mlm",description:`<strong>mlm</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether or not to use masked language modeling. If set to <code>False</code>, the labels are the same as the inputs
with the padding tokens ignored (by setting them to -100). Otherwise, the labels are -100 for non-masked
tokens and the value to predict for the masked token.`,name:"mlm"},{anchor:"transformers.DataCollatorForLanguageModeling.mlm_probability",description:`<strong>mlm_probability</strong> (<code>float</code>, <em>optional</em>, defaults to 0.15) &#x2014;
The probability with which to (randomly) mask tokens in the input, when <code>mlm</code> is set to <code>True</code>.`,name:"mlm_probability"},{anchor:"transformers.DataCollatorForLanguageModeling.mask_replace_prob",description:`<strong>mask_replace_prob</strong> (<code>float</code>, <em>optional</em>, defaults to 0.8) &#x2014;
The probability with which masked tokens are replaced by the tokenizer&#x2019;s mask token (e.g., <code>[MASK]</code>).
Defaults to 0.8, meaning 80% of the masked tokens will be replaced with <code>[MASK]</code>.
Only works when <code>mlm</code> is set to <code>True</code>.`,name:"mask_replace_prob"},{anchor:"transformers.DataCollatorForLanguageModeling.random_replace_prob",description:`<strong>random_replace_prob</strong> (<code>float</code>, <em>optional</em>, defaults to 0.1) &#x2014;
The probability with which masked tokens are replaced by random tokens from the tokenizer&#x2019;s vocabulary.
Defaults to 0.1, meaning 10% of the masked tokens will be replaced with random tokens. The remaining
masked tokens (1 - mask_replace_prob - random_replace_prob) are left unchanged.
Only works when <code>mlm</code> is set to <code>True</code>.`,name:"random_replace_prob"},{anchor:"transformers.DataCollatorForLanguageModeling.pad_to_multiple_of",description:`<strong>pad_to_multiple_of</strong> (<code>int</code>, <em>optional</em>) &#x2014;
If set, will pad the sequence to a multiple of the provided value.`,name:"pad_to_multiple_of"},{anchor:"transformers.DataCollatorForLanguageModeling.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code>) &#x2014;
The type of Tensor to return. Allowable values are &#x201C;np&#x201D;, &#x201C;pt&#x201D; and &#x201C;tf&#x201D;.`,name:"return_tensors"},{anchor:"transformers.DataCollatorForLanguageModeling.seed",description:`<strong>seed</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The seed to use for the random number generator for masking. If not provided, the global RNG will be used.`,name:"seed"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L763"}}),R=new Ta({props:{$$slots:{default:[co]},$$scope:{ctx:U}}}),fe=new x({props:{name:"numpy_mask_tokens",anchor:"transformers.DataCollatorForLanguageModeling.numpy_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"special_tokens_mask",val:": typing.Optional[typing.Any] = None"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L1113"}}),ue=new x({props:{name:"tf_mask_tokens",anchor:"transformers.DataCollatorForLanguageModeling.tf_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"vocab_size",val:""},{name:"mask_token_id",val:""},{name:"special_tokens_mask",val:": typing.Optional[typing.Any] = None"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L911"}}),he=new x({props:{name:"torch_mask_tokens",anchor:"transformers.DataCollatorForLanguageModeling.torch_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"special_tokens_mask",val:": typing.Optional[typing.Any] = None"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L1034"}}),_e=new V({props:{title:"DataCollatorForWholeWordMask",local:"transformers.DataCollatorForWholeWordMask ][ transformers.DataCollatorForWholeWordMask",headingTag:"h2"}}),be=new x({props:{name:"class transformers.DataCollatorForWholeWordMask",anchor:"transformers.DataCollatorForWholeWordMask",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"mlm",val:": bool = True"},{name:"mlm_probability",val:": float = 0.15"},{name:"mask_replace_prob",val:": float = 0.8"},{name:"random_replace_prob",val:": float = 0.1"},{name:"pad_to_multiple_of",val:": typing.Optional[int] = None"},{name:"tf_experimental_compile",val:": bool = False"},{name:"return_tensors",val:": str = 'pt'"},{name:"seed",val:": typing.Optional[int] = None"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L1179"}}),Q=new Ta({props:{$$slots:{default:[go]},$$scope:{ctx:U}}}),$e=new x({props:{name:"numpy_mask_tokens",anchor:"transformers.DataCollatorForWholeWordMask.numpy_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"mask_labels",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L1476"}}),ke=new x({props:{name:"tf_mask_tokens",anchor:"transformers.DataCollatorForWholeWordMask.tf_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"mask_labels",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L1416"}}),ve=new x({props:{name:"torch_mask_tokens",anchor:"transformers.DataCollatorForWholeWordMask.torch_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"},{name:"mask_labels",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L1360"}}),xe=new V({props:{title:"DataCollatorForPermutationLanguageModeling",local:"transformers.DataCollatorForPermutationLanguageModeling ][ transformers.DataCollatorForPermutationLanguageModeling",headingTag:"h2"}}),Ce=new x({props:{name:"class transformers.DataCollatorForPermutationLanguageModeling",anchor:"transformers.DataCollatorForPermutationLanguageModeling",parameters:[{name:"tokenizer",val:": PreTrainedTokenizerBase"},{name:"plm_probability",val:": float = 0.16666666666666666"},{name:"max_span_length",val:": int = 5"},{name:"return_tensors",val:": str = 'pt'"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L1633"}}),ye=new x({props:{name:"numpy_mask_tokens",anchor:"transformers.DataCollatorForPermutationLanguageModeling.numpy_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L1874"}}),Te=new x({props:{name:"tf_mask_tokens",anchor:"transformers.DataCollatorForPermutationLanguageModeling.tf_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L1767"}}),Pe=new x({props:{name:"torch_mask_tokens",anchor:"transformers.DataCollatorForPermutationLanguageModeling.torch_mask_tokens",parameters:[{name:"inputs",val:": typing.Any"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L1668"}}),Me=new V({props:{title:"DataCollatorWithFlatteningtransformers.DataCollatorWithFlattening",local:"transformers.DataCollatorWithFlattening",headingTag:"h2"}}),Le=new x({props:{name:"class transformers.DataCollatorWithFlattening",anchor:"transformers.DataCollatorWithFlattening",parameters:[{name:"*args",val:""},{name:"return_position_ids",val:" = True"},{name:"separator_id",val:" = -100"},{name:"**kwargs",val:""}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/data/data_collator.py#L1972"}}),te=new Ta({props:{warning:!0,$$slots:{default:[fo]},$$scope:{ctx:U}}}),qe=new po({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/ko/main_classes/data_collator.md"}}),{c(){m=l("meta"),F=a(),b=l("p"),D=a(),p(I.$$.fragment),O=a(),T=l("p"),T.innerHTML=dt,v=a(),p(M.$$.fragment),pt=a(),L=l("div"),p(oe.$$.fragment),Ht=a(),ze=l("p"),ze.textContent=Pa,Nt=a(),We=l("ul"),We.innerHTML=Fa,Vt=a(),Ae=l("p"),Ae.textContent=Ma,ct=a(),p(ne.$$.fragment),gt=a(),w=l("div"),p(re.$$.fragment),Ot=a(),Ie=l("p"),Ie.textContent=La,jt=a(),Ee=l("ul"),Ee.innerHTML=qa,Kt=a(),He=l("p"),He.textContent=Sa,Bt=a(),Ne=l("p"),Ne.textContent=za,ft=a(),p(le.$$.fragment),ut=a(),j=l("div"),p(se.$$.fragment),Ut=a(),Ve=l("p"),Ve.textContent=Wa,ht=a(),p(ie.$$.fragment),_t=a(),K=l("div"),p(de.$$.fragment),Rt=a(),Oe=l("p"),Oe.textContent=Aa,bt=a(),p(me.$$.fragment),$t=a(),B=l("div"),p(pe.$$.fragment),Gt=a(),je=l("p"),je.textContent=Ia,kt=a(),p(ce.$$.fragment),vt=a(),C=l("div"),p(ge.$$.fragment),Xt=a(),Ke=l("p"),Ke.textContent=Ea,Jt=a(),p(R.$$.fragment),Qt=a(),G=l("div"),p(fe.$$.fragment),Yt=a(),Be=l("p"),Be.textContent=Ha,Zt=a(),X=l("div"),p(ue.$$.fragment),ea=a(),Ue=l("p"),Ue.textContent=Na,ta=a(),J=l("div"),p(he.$$.fragment),aa=a(),Re=l("p"),Re.textContent=Va,xt=a(),p(_e.$$.fragment),Ct=a(),k=l("div"),p(be.$$.fragment),oa=a(),Ge=l("p"),Ge.textContent=Oa,na=a(),Xe=l("ul"),Xe.innerHTML=ja,ra=a(),p(Q.$$.fragment),la=a(),Y=l("div"),p($e.$$.fragment),sa=a(),Je=l("p"),Je.textContent=Ka,ia=a(),Z=l("div"),p(ke.$$.fragment),da=a(),Qe=l("p"),Qe.textContent=Ba,ma=a(),ee=l("div"),p(ve.$$.fragment),pa=a(),Ye=l("p"),Ye.textContent=Ua,yt=a(),p(xe.$$.fragment),Dt=a(),y=l("div"),p(Ce.$$.fragment),ca=a(),Ze=l("p"),Ze.textContent=Ra,ga=a(),et=l("ul"),et.innerHTML=Ga,fa=a(),E=l("div"),p(ye.$$.fragment),ua=a(),tt=l("p"),tt.textContent=Xa,ha=a(),De=l("ol"),De.innerHTML=Ja,_a=a(),H=l("div"),p(Te.$$.fragment),ba=a(),at=l("p"),at.textContent=Qa,$a=a(),we=l("ol"),we.innerHTML=Ya,ka=a(),N=l("div"),p(Pe.$$.fragment),va=a(),ot=l("p"),ot.textContent=Za,xa=a(),Fe=l("ol"),Fe.innerHTML=eo,Tt=a(),p(Me.$$.fragment),wt=a(),q=l("div"),p(Le.$$.fragment),Ca=a(),nt=l("p"),nt.textContent=to,ya=a(),rt=l("ul"),rt.innerHTML=ao,Da=a(),p(te.$$.fragment),Pt=a(),p(qe.$$.fragment),Ft=a(),mt=l("p"),this.h()},l(e){const r=so("svelte-u9bgzb",document.head);m=s(r,"META",{name:!0,content:!0}),r.forEach(n),F=o(e),b=s(e,"P",{}),$(b).forEach(n),D=o(e),c(I.$$.fragment,e),O=o(e),T=s(e,"P",{"data-svelte-h":!0}),d(T)!=="svelte-1tem19v"&&(T.innerHTML=dt),v=o(e),c(M.$$.fragment,e),pt=o(e),L=s(e,"DIV",{class:!0});var A=$(L);c(oe.$$.fragment,A),Ht=o(A),ze=s(A,"P",{"data-svelte-h":!0}),d(ze)!=="svelte-1hmsgsg"&&(ze.textContent=Pa),Nt=o(A),We=s(A,"UL",{"data-svelte-h":!0}),d(We)!=="svelte-1pq8qks"&&(We.innerHTML=Fa),Vt=o(A),Ae=s(A,"P",{"data-svelte-h":!0}),d(Ae)!=="svelte-1vi1gug"&&(Ae.textContent=Ma),A.forEach(n),ct=o(e),c(ne.$$.fragment,e),gt=o(e),w=s(e,"DIV",{class:!0});var S=$(w);c(re.$$.fragment,S),Ot=o(S),Ie=s(S,"P",{"data-svelte-h":!0}),d(Ie)!=="svelte-1hmsgsg"&&(Ie.textContent=La),jt=o(S),Ee=s(S,"UL",{"data-svelte-h":!0}),d(Ee)!=="svelte-1pq8qks"&&(Ee.innerHTML=qa),Kt=o(S),He=s(S,"P",{"data-svelte-h":!0}),d(He)!=="svelte-1vi1gug"&&(He.textContent=Sa),Bt=o(S),Ne=s(S,"P",{"data-svelte-h":!0}),d(Ne)!=="svelte-vfvbwr"&&(Ne.textContent=za),S.forEach(n),ft=o(e),c(le.$$.fragment,e),ut=o(e),j=s(e,"DIV",{class:!0});var Se=$(j);c(se.$$.fragment,Se),Ut=o(Se),Ve=s(Se,"P",{"data-svelte-h":!0}),d(Ve)!=="svelte-1iebpai"&&(Ve.textContent=Wa),Se.forEach(n),ht=o(e),c(ie.$$.fragment,e),_t=o(e),K=s(e,"DIV",{class:!0});var Lt=$(K);c(de.$$.fragment,Lt),Rt=o(Lt),Oe=s(Lt,"P",{"data-svelte-h":!0}),d(Oe)!=="svelte-4uvw1w"&&(Oe.textContent=Aa),Lt.forEach(n),bt=o(e),c(me.$$.fragment,e),$t=o(e),B=s(e,"DIV",{class:!0});var qt=$(B);c(pe.$$.fragment,qt),Gt=o(qt),je=s(qt,"P",{"data-svelte-h":!0}),d(je)!=="svelte-4uvw1w"&&(je.textContent=Ia),qt.forEach(n),kt=o(e),c(ce.$$.fragment,e),vt=o(e),C=s(e,"DIV",{class:!0});var z=$(C);c(ge.$$.fragment,z),Xt=o(z),Ke=s(z,"P",{"data-svelte-h":!0}),d(Ke)!=="svelte-10km8jk"&&(Ke.textContent=Ea),Jt=o(z),c(R.$$.fragment,z),Qt=o(z),G=s(z,"DIV",{class:!0});var St=$(G);c(fe.$$.fragment,St),Yt=o(St),Be=s(St,"P",{"data-svelte-h":!0}),d(Be)!=="svelte-iv4xqf"&&(Be.textContent=Ha),St.forEach(n),Zt=o(z),X=s(z,"DIV",{class:!0});var zt=$(X);c(ue.$$.fragment,zt),ea=o(zt),Ue=s(zt,"P",{"data-svelte-h":!0}),d(Ue)!=="svelte-iv4xqf"&&(Ue.textContent=Na),zt.forEach(n),ta=o(z),J=s(z,"DIV",{class:!0});var Wt=$(J);c(he.$$.fragment,Wt),aa=o(Wt),Re=s(Wt,"P",{"data-svelte-h":!0}),d(Re)!=="svelte-iv4xqf"&&(Re.textContent=Va),Wt.forEach(n),z.forEach(n),xt=o(e),c(_e.$$.fragment,e),Ct=o(e),k=s(e,"DIV",{class:!0});var P=$(k);c(be.$$.fragment,P),oa=o(P),Ge=s(P,"P",{"data-svelte-h":!0}),d(Ge)!=="svelte-sjqk55"&&(Ge.textContent=Oa),na=o(P),Xe=s(P,"UL",{"data-svelte-h":!0}),d(Xe)!=="svelte-d2ozxj"&&(Xe.innerHTML=ja),ra=o(P),c(Q.$$.fragment,P),la=o(P),Y=s(P,"DIV",{class:!0});var At=$(Y);c($e.$$.fragment,At),sa=o(At),Je=s(At,"P",{"data-svelte-h":!0}),d(Je)!=="svelte-10msjh1"&&(Je.textContent=Ka),At.forEach(n),ia=o(P),Z=s(P,"DIV",{class:!0});var It=$(Z);c(ke.$$.fragment,It),da=o(It),Qe=s(It,"P",{"data-svelte-h":!0}),d(Qe)!=="svelte-10msjh1"&&(Qe.textContent=Ba),It.forEach(n),ma=o(P),ee=s(P,"DIV",{class:!0});var Et=$(ee);c(ve.$$.fragment,Et),pa=o(Et),Ye=s(Et,"P",{"data-svelte-h":!0}),d(Ye)!=="svelte-10msjh1"&&(Ye.textContent=Ua),Et.forEach(n),P.forEach(n),yt=o(e),c(xe.$$.fragment,e),Dt=o(e),y=s(e,"DIV",{class:!0});var W=$(y);c(Ce.$$.fragment,W),ca=o(W),Ze=s(W,"P",{"data-svelte-h":!0}),d(Ze)!=="svelte-12inif1"&&(Ze.textContent=Ra),ga=o(W),et=s(W,"UL",{"data-svelte-h":!0}),d(et)!=="svelte-1t4qh7k"&&(et.innerHTML=Ga),fa=o(W),E=s(W,"DIV",{class:!0});var lt=$(E);c(ye.$$.fragment,lt),ua=o(lt),tt=s(lt,"P",{"data-svelte-h":!0}),d(tt)!=="svelte-1u70phx"&&(tt.textContent=Xa),ha=o(lt),De=s(lt,"OL",{start:!0,"data-svelte-h":!0}),d(De)!=="svelte-1r6jrbi"&&(De.innerHTML=Ja),lt.forEach(n),_a=o(W),H=s(W,"DIV",{class:!0});var st=$(H);c(Te.$$.fragment,st),ba=o(st),at=s(st,"P",{"data-svelte-h":!0}),d(at)!=="svelte-1u70phx"&&(at.textContent=Qa),$a=o(st),we=s(st,"OL",{start:!0,"data-svelte-h":!0}),d(we)!=="svelte-1r6jrbi"&&(we.innerHTML=Ya),st.forEach(n),ka=o(W),N=s(W,"DIV",{class:!0});var it=$(N);c(Pe.$$.fragment,it),va=o(it),ot=s(it,"P",{"data-svelte-h":!0}),d(ot)!=="svelte-1u70phx"&&(ot.textContent=Za),xa=o(it),Fe=s(it,"OL",{start:!0,"data-svelte-h":!0}),d(Fe)!=="svelte-1r6jrbi"&&(Fe.innerHTML=eo),it.forEach(n),W.forEach(n),Tt=o(e),c(Me.$$.fragment,e),wt=o(e),q=s(e,"DIV",{class:!0});var ae=$(q);c(Le.$$.fragment,ae),Ca=o(ae),nt=s(ae,"P",{"data-svelte-h":!0}),d(nt)!=="svelte-4gb9ju"&&(nt.textContent=to),ya=o(ae),rt=s(ae,"UL",{"data-svelte-h":!0}),d(rt)!=="svelte-1w5rvhw"&&(rt.innerHTML=ao),Da=o(ae),c(te.$$.fragment,ae),ae.forEach(n),Pt=o(e),c(qe.$$.fragment,e),Ft=o(e),mt=s(e,"P",{}),$(mt).forEach(n),this.h()},h(){_(m,"name","hf:doc:metadata"),_(m,"content",ho),_(L,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(w,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(j,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(K,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(B,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(G,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(X,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(J,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(C,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(Y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(Z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(ee,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(k,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(De,"start","0"),_(E,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(we,"start","0"),_(H,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(Fe,"start","0"),_(N,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),_(q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,r){t(document.head,m),i(e,F,r),i(e,b,r),i(e,D,r),g(I,e,r),i(e,O,r),i(e,T,r),i(e,v,r),g(M,e,r),i(e,pt,r),i(e,L,r),g(oe,L,null),t(L,Ht),t(L,ze),t(L,Nt),t(L,We),t(L,Vt),t(L,Ae),i(e,ct,r),g(ne,e,r),i(e,gt,r),i(e,w,r),g(re,w,null),t(w,Ot),t(w,Ie),t(w,jt),t(w,Ee),t(w,Kt),t(w,He),t(w,Bt),t(w,Ne),i(e,ft,r),g(le,e,r),i(e,ut,r),i(e,j,r),g(se,j,null),t(j,Ut),t(j,Ve),i(e,ht,r),g(ie,e,r),i(e,_t,r),i(e,K,r),g(de,K,null),t(K,Rt),t(K,Oe),i(e,bt,r),g(me,e,r),i(e,$t,r),i(e,B,r),g(pe,B,null),t(B,Gt),t(B,je),i(e,kt,r),g(ce,e,r),i(e,vt,r),i(e,C,r),g(ge,C,null),t(C,Xt),t(C,Ke),t(C,Jt),g(R,C,null),t(C,Qt),t(C,G),g(fe,G,null),t(G,Yt),t(G,Be),t(C,Zt),t(C,X),g(ue,X,null),t(X,ea),t(X,Ue),t(C,ta),t(C,J),g(he,J,null),t(J,aa),t(J,Re),i(e,xt,r),g(_e,e,r),i(e,Ct,r),i(e,k,r),g(be,k,null),t(k,oa),t(k,Ge),t(k,na),t(k,Xe),t(k,ra),g(Q,k,null),t(k,la),t(k,Y),g($e,Y,null),t(Y,sa),t(Y,Je),t(k,ia),t(k,Z),g(ke,Z,null),t(Z,da),t(Z,Qe),t(k,ma),t(k,ee),g(ve,ee,null),t(ee,pa),t(ee,Ye),i(e,yt,r),g(xe,e,r),i(e,Dt,r),i(e,y,r),g(Ce,y,null),t(y,ca),t(y,Ze),t(y,ga),t(y,et),t(y,fa),t(y,E),g(ye,E,null),t(E,ua),t(E,tt),t(E,ha),t(E,De),t(y,_a),t(y,H),g(Te,H,null),t(H,ba),t(H,at),t(H,$a),t(H,we),t(y,ka),t(y,N),g(Pe,N,null),t(N,va),t(N,ot),t(N,xa),t(N,Fe),i(e,Tt,r),g(Me,e,r),i(e,wt,r),i(e,q,r),g(Le,q,null),t(q,Ca),t(q,nt),t(q,ya),t(q,rt),t(q,Da),g(te,q,null),i(e,Pt,r),g(qe,e,r),i(e,Ft,r),i(e,mt,r),Mt=!0},p(e,[r]){const A={};r&2&&(A.$$scope={dirty:r,ctx:e}),R.$set(A);const S={};r&2&&(S.$$scope={dirty:r,ctx:e}),Q.$set(S);const Se={};r&2&&(Se.$$scope={dirty:r,ctx:e}),te.$set(Se)},i(e){Mt||(f(I.$$.fragment,e),f(M.$$.fragment,e),f(oe.$$.fragment,e),f(ne.$$.fragment,e),f(re.$$.fragment,e),f(le.$$.fragment,e),f(se.$$.fragment,e),f(ie.$$.fragment,e),f(de.$$.fragment,e),f(me.$$.fragment,e),f(pe.$$.fragment,e),f(ce.$$.fragment,e),f(ge.$$.fragment,e),f(R.$$.fragment,e),f(fe.$$.fragment,e),f(ue.$$.fragment,e),f(he.$$.fragment,e),f(_e.$$.fragment,e),f(be.$$.fragment,e),f(Q.$$.fragment,e),f($e.$$.fragment,e),f(ke.$$.fragment,e),f(ve.$$.fragment,e),f(xe.$$.fragment,e),f(Ce.$$.fragment,e),f(ye.$$.fragment,e),f(Te.$$.fragment,e),f(Pe.$$.fragment,e),f(Me.$$.fragment,e),f(Le.$$.fragment,e),f(te.$$.fragment,e),f(qe.$$.fragment,e),Mt=!0)},o(e){u(I.$$.fragment,e),u(M.$$.fragment,e),u(oe.$$.fragment,e),u(ne.$$.fragment,e),u(re.$$.fragment,e),u(le.$$.fragment,e),u(se.$$.fragment,e),u(ie.$$.fragment,e),u(de.$$.fragment,e),u(me.$$.fragment,e),u(pe.$$.fragment,e),u(ce.$$.fragment,e),u(ge.$$.fragment,e),u(R.$$.fragment,e),u(fe.$$.fragment,e),u(ue.$$.fragment,e),u(he.$$.fragment,e),u(_e.$$.fragment,e),u(be.$$.fragment,e),u(Q.$$.fragment,e),u($e.$$.fragment,e),u(ke.$$.fragment,e),u(ve.$$.fragment,e),u(xe.$$.fragment,e),u(Ce.$$.fragment,e),u(ye.$$.fragment,e),u(Te.$$.fragment,e),u(Pe.$$.fragment,e),u(Me.$$.fragment,e),u(Le.$$.fragment,e),u(te.$$.fragment,e),u(qe.$$.fragment,e),Mt=!1},d(e){e&&(n(F),n(b),n(D),n(O),n(T),n(v),n(pt),n(L),n(ct),n(gt),n(w),n(ft),n(ut),n(j),n(ht),n(_t),n(K),n(bt),n($t),n(B),n(kt),n(vt),n(C),n(xt),n(Ct),n(k),n(yt),n(Dt),n(y),n(Tt),n(wt),n(q),n(Pt),n(Ft),n(mt)),n(m),h(I,e),h(M,e),h(oe),h(ne,e),h(re),h(le,e),h(se),h(ie,e),h(de),h(me,e),h(pe),h(ce,e),h(ge),h(R),h(fe),h(ue),h(he),h(_e,e),h(be),h(Q),h($e),h(ke),h(ve),h(xe,e),h(Ce),h(ye),h(Te),h(Pe),h(Me,e),h(Le),h(te),h(qe,e)}}}const ho='{"title":"데이터 콜레이터(Data Collator)","local":"data-collator","sections":[{"title":"기본 데이터 콜레이터","local":"transformers.default_data_collator ][ transformers.default_data_collator","sections":[],"depth":2},{"title":"DefaultDataCollator","local":"transformers.DefaultDataCollator ][ transformers.DefaultDataCollator","sections":[],"depth":2},{"title":"DataCollatorWithPadding","local":"transformers.DataCollatorWithPadding ][ transformers.DataCollatorWithPadding","sections":[],"depth":2},{"title":"DataCollatorForTokenClassification","local":"transformers.DataCollatorForTokenClassification ][ transformers.DataCollatorForTokenClassification","sections":[],"depth":2},{"title":"DataCollatorForSeq2Seq","local":"transformers.DataCollatorForSeq2Seq ][ transformers.DataCollatorForSeq2Seq","sections":[],"depth":2},{"title":"DataCollatorForLanguageModeling","local":"transformers.DataCollatorForLanguageModeling ][ transformers.DataCollatorForLanguageModeling","sections":[],"depth":2},{"title":"DataCollatorForWholeWordMask","local":"transformers.DataCollatorForWholeWordMask ][ transformers.DataCollatorForWholeWordMask","sections":[],"depth":2},{"title":"DataCollatorForPermutationLanguageModeling","local":"transformers.DataCollatorForPermutationLanguageModeling ][ transformers.DataCollatorForPermutationLanguageModeling","sections":[],"depth":2},{"title":"DataCollatorWithFlatteningtransformers.DataCollatorWithFlattening","local":"transformers.DataCollatorWithFlattening","sections":[],"depth":2}],"depth":1}';function _o(U){return no(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Co extends ro{constructor(m){super(),lo(this,m,_o,uo,oo,{})}}export{Co as component};
