import{s as Ca,f as or,o as La,n as lr}from"../chunks/scheduler.25b97de1.js";import{S as Wa,i as ja,g as l,s as a,r as p,A as Da,h as m,f as t,c as n,j as z,u as d,x as w,k as b,y as s,a as o,v as h,d as u,t as g,w as f}from"../chunks/index.d9030fc9.js";import{D as M}from"../chunks/Docstring.ffac8efa.js";import{C as mr}from"../chunks/CodeBlock.e6cd0d95.js";import{E as ir}from"../chunks/ExampleCodeBlock.22dfe688.js";import{H as Z,E as Ua}from"../chunks/EditOnGithub.91d95064.js";function Ra(R){let c,A="Example:",x,_,v;return _=new mr({props:{code:"QWRhZmFjdG9yKG1vZGVsLnBhcmFtZXRlcnMoKSUyQyUyMHNjYWxlX3BhcmFtZXRlciUzREZhbHNlJTJDJTIwcmVsYXRpdmVfc3RlcCUzREZhbHNlJTJDJTIwd2FybXVwX2luaXQlM0RGYWxzZSUyQyUyMGxyJTNEMWUtMyk=",highlighted:'Adafactor(model.parameters(), scale_parameter=<span class="hljs-literal">False</span>, relative_step=<span class="hljs-literal">False</span>, warmup_init=<span class="hljs-literal">False</span>, lr=<span class="hljs-number">1e-3</span>)',wrap:!1}}),{c(){c=l("p"),c.textContent=A,x=a(),p(_.$$.fragment)},l(i){c=m(i,"P",{"data-svelte-h":!0}),w(c)!=="svelte-11lpom8"&&(c.textContent=A),x=n(i),d(_.$$.fragment,i)},m(i,T){o(i,c,T),o(i,x,T),h(_,i,T),v=!0},p:lr,i(i){v||(u(_.$$.fragment,i),v=!0)},o(i){g(_.$$.fragment,i),v=!1},d(i){i&&(t(c),t(x)),f(_,i)}}}function Ea(R){let c,A="Others reported the following combination to work well:",x,_,v;return _=new mr({props:{code:"QWRhZmFjdG9yKG1vZGVsLnBhcmFtZXRlcnMoKSUyQyUyMHNjYWxlX3BhcmFtZXRlciUzRFRydWUlMkMlMjByZWxhdGl2ZV9zdGVwJTNEVHJ1ZSUyQyUyMHdhcm11cF9pbml0JTNEVHJ1ZSUyQyUyMGxyJTNETm9uZSk=",highlighted:'Adafactor(model.parameters(), scale_parameter=<span class="hljs-literal">True</span>, relative_step=<span class="hljs-literal">True</span>, warmup_init=<span class="hljs-literal">True</span>, lr=<span class="hljs-literal">None</span>)',wrap:!1}}),{c(){c=l("p"),c.textContent=A,x=a(),p(_.$$.fragment)},l(i){c=m(i,"P",{"data-svelte-h":!0}),w(c)!=="svelte-mxeef1"&&(c.textContent=A),x=n(i),d(_.$$.fragment,i)},m(i,T){o(i,c,T),o(i,x,T),h(_,i,T),v=!0},p:lr,i(i){v||(u(_.$$.fragment,i),v=!0)},o(i){g(_.$$.fragment,i),v=!1},d(i){i&&(t(c),t(x)),f(_,i)}}}function ka(R){let c,A="scheduler as following:",x,_,v;return _=new mr({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5vcHRpbWl6YXRpb24lMjBpbXBvcnQlMjBBZGFmYWN0b3IlMkMlMjBBZGFmYWN0b3JTY2hlZHVsZSUwQSUwQW9wdGltaXplciUyMCUzRCUyMEFkYWZhY3Rvcihtb2RlbC5wYXJhbWV0ZXJzKCklMkMlMjBzY2FsZV9wYXJhbWV0ZXIlM0RUcnVlJTJDJTIwcmVsYXRpdmVfc3RlcCUzRFRydWUlMkMlMjB3YXJtdXBfaW5pdCUzRFRydWUlMkMlMjBsciUzRE5vbmUpJTBBbHJfc2NoZWR1bGVyJTIwJTNEJTIwQWRhZmFjdG9yU2NoZWR1bGUob3B0aW1pemVyKSUwQXRyYWluZXIlMjAlM0QlMjBUcmFpbmVyKC4uLiUyQyUyMG9wdGltaXplcnMlM0Qob3B0aW1pemVyJTJDJTIwbHJfc2NoZWR1bGVyKSk=",highlighted:`<span class="hljs-keyword">from</span> transformers.optimization <span class="hljs-keyword">import</span> Adafactor, AdafactorSchedule

optimizer = Adafactor(model.parameters(), scale_parameter=<span class="hljs-literal">True</span>, relative_step=<span class="hljs-literal">True</span>, warmup_init=<span class="hljs-literal">True</span>, lr=<span class="hljs-literal">None</span>)
lr_scheduler = AdafactorSchedule(optimizer)
trainer = Trainer(..., optimizers=(optimizer, lr_scheduler))`,wrap:!1}}),{c(){c=l("p"),c.textContent=A,x=a(),p(_.$$.fragment)},l(i){c=m(i,"P",{"data-svelte-h":!0}),w(c)!=="svelte-1eua222"&&(c.textContent=A),x=n(i),d(_.$$.fragment,i)},m(i,T){o(i,c,T),o(i,x,T),h(_,i,T),v=!0},p:lr,i(i){v||(u(_.$$.fragment,i),v=!0)},o(i){g(_.$$.fragment,i),v=!1},d(i){i&&(t(c),t(x)),f(_,i)}}}function Ia(R){let c,A="Usage:",x,_,v;return _=new mr({props:{code:"JTIzJTIwcmVwbGFjZSUyMEFkYW1XJTIwd2l0aCUyMEFkYWZhY3RvciUwQW9wdGltaXplciUyMCUzRCUyMEFkYWZhY3RvciglMEElMjAlMjAlMjAlMjBtb2RlbC5wYXJhbWV0ZXJzKCklMkMlMEElMjAlMjAlMjAlMjBsciUzRDFlLTMlMkMlMEElMjAlMjAlMjAlMjBlcHMlM0QoMWUtMzAlMkMlMjAxZS0zKSUyQyUwQSUyMCUyMCUyMCUyMGNsaXBfdGhyZXNob2xkJTNEMS4wJTJDJTBBJTIwJTIwJTIwJTIwZGVjYXlfcmF0ZSUzRC0wLjglMkMlMEElMjAlMjAlMjAlMjBiZXRhMSUzRE5vbmUlMkMlMEElMjAlMjAlMjAlMjB3ZWlnaHRfZGVjYXklM0QwLjAlMkMlMEElMjAlMjAlMjAlMjByZWxhdGl2ZV9zdGVwJTNERmFsc2UlMkMlMEElMjAlMjAlMjAlMjBzY2FsZV9wYXJhbWV0ZXIlM0RGYWxzZSUyQyUwQSUyMCUyMCUyMCUyMHdhcm11cF9pbml0JTNERmFsc2UlMkMlMEEp",highlighted:`<span class="hljs-comment"># replace AdamW with Adafactor</span>
optimizer = Adafactor(
    model.parameters(),
    lr=<span class="hljs-number">1e-3</span>,
    eps=(<span class="hljs-number">1e-30</span>, <span class="hljs-number">1e-3</span>),
    clip_threshold=<span class="hljs-number">1.0</span>,
    decay_rate=-<span class="hljs-number">0.8</span>,
    beta1=<span class="hljs-literal">None</span>,
    weight_decay=<span class="hljs-number">0.0</span>,
    relative_step=<span class="hljs-literal">False</span>,
    scale_parameter=<span class="hljs-literal">False</span>,
    warmup_init=<span class="hljs-literal">False</span>,
)`,wrap:!1}}),{c(){c=l("p"),c.textContent=A,x=a(),p(_.$$.fragment)},l(i){c=m(i,"P",{"data-svelte-h":!0}),w(c)!=="svelte-5wyjqd"&&(c.textContent=A),x=n(i),d(_.$$.fragment,i)},m(i,T){o(i,c,T),o(i,x,T),h(_,i,T),v=!0},p:lr,i(i){v||(u(_.$$.fragment,i),v=!0)},o(i){g(_.$$.fragment,i),v=!1},d(i){i&&(t(c),t(x)),f(_,i)}}}function Fa(R){let c,A,x,_,v,i,T,qr="The <code>.optimization</code> module provides:",_t,te,Br="<li>an optimizer with weight decay fixed that can be used to fine-tuned models, and</li> <li>several schedules in the form of schedule objects that inherit from <code>_LRSchedule</code>:</li> <li>a gradient accumulation class to accumulate the gradients of multiple batches</li>",wt,re,bt,L,ae,cr,Ie,Or=`Implements Adam algorithm with weight decay fix as introduced in <a href="https://arxiv.org/abs/1711.05101" rel="nofollow">Decoupled Weight Decay
Regularization</a>.`,pr,S,ne,dr,Fe,Xr="Performs a single optimization step.",yt,se,vt,y,oe,hr,Pe,Yr=`AdaFactor pytorch implementation can be used as a drop in replacement for Adam original fairseq code:
<a href="https://github.com/pytorch/fairseq/blob/master/fairseq/optim/adafactor.py" rel="nofollow">https://github.com/pytorch/fairseq/blob/master/fairseq/optim/adafactor.py</a>`,ur,Ve,Qr=`Paper: <em>Adafactor: Adaptive Learning Rates with Sublinear Memory Cost</em> <a href="https://arxiv.org/abs/1804.04235" rel="nofollow">https://arxiv.org/abs/1804.04235</a> Note that
this optimizer internally adjusts the learning rate depending on the <code>scale_parameter</code>, <code>relative_step</code> and
<code>warmup_init</code> options. To use a manual (external) learning rate schedule you should set <code>scale_parameter=False</code> and
<code>relative_step=False</code>.`,gr,Ne,Kr="This implementation handles low-precision (FP16, bfloat) values, but we have not thoroughly tested.",fr,Ge,ea='Recommended T5 finetuning settings (<a href="https://discuss.huggingface.co/t/t5-finetuning-tips/684/3" rel="nofollow">https://discuss.huggingface.co/t/t5-finetuning-tips/684/3</a>):',_r,He,ta='<li><p>Training without LR warmup or clip_threshold is not recommended.</p> <ul><li>use scheduled LR warm-up to fixed LR</li> <li>use clip_threshold=1.0 (<a href="https://arxiv.org/abs/1804.04235" rel="nofollow">https://arxiv.org/abs/1804.04235</a>)</li></ul></li> <li><p>Disable relative updates</p></li> <li><p>Use scale_parameter=False</p></li> <li><p>Additional optimizer operations like gradient clipping should not be used alongside Adafactor</p></li>',wr,q,br,B,yr,Je,ra='When using <code>lr=None</code> with <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a> you will most likely need to use <code>AdafactorSchedule</code>',vr,O,$r,X,xr,Y,ie,Tr,Ze,aa="Performs a single optimization step",$t,le,xt,C,me,zr,Se,na=`Adam enables L2 weight decay and clip_by_global_norm on gradients. Just adding the square of the weights to the
loss function is <em>not</em> the correct way of using L2 regularization/weight decay with Adam, since that will interact
with the m and v parameters in strange ways as shown in <a href="https://arxiv.org/abs/1711.05101" rel="nofollow">Decoupled Weight Decay
Regularization</a>.`,Mr,qe,sa=`Instead we want to decay the weights in a manner that doesn’t interact with the m/v parameters. This is equivalent
to adding the square of the weights to the loss with plain (non-momentum) SGD.`,Ar,Q,ce,Cr,Be,oa="Creates an optimizer from its config with WarmUp custom object.",Tt,E,pe,Lr,Oe,ia="Creates an optimizer with a learning rate schedule using a warmup phase followed by a linear decay.",zt,de,Mt,he,At,W,ue,Wr,Xe,la=`Scheduler names for the parameter <code>lr_scheduler_type</code> in <a href="/docs/transformers/main/en/main_classes/trainer#transformers.TrainingArguments">TrainingArguments</a>.
By default, it uses “linear”. Internally, this retrieves <code>get_linear_schedule_with_warmup</code> scheduler from <a href="/docs/transformers/main/en/main_classes/trainer#transformers.Trainer">Trainer</a>.
Scheduler types:`,jr,Ye,ma="<li>“linear” = get_linear_schedule_with_warmup</li> <li>“cosine” = get_cosine_schedule_with_warmup</li> <li>“cosine_with_restarts” = get_cosine_with_hard_restarts_schedule_with_warmup</li> <li>“polynomial” = get_polynomial_decay_schedule_with_warmup</li> <li>“constant” =  get_constant_schedule</li> <li>“constant_with_warmup” = get_constant_schedule_with_warmup</li> <li>“inverse_sqrt” = get_inverse_sqrt_schedule</li> <li>“reduce_lr_on_plateau” = get_reduce_on_plateau_schedule</li> <li>“cosine_with_min_lr” = get_cosine_with_min_lr_schedule_with_warmup</li> <li>“warmup_stable_decay” = get_wsd_schedule</li>",Ct,k,ge,Dr,Qe,ca="Unified API to get any scheduler from its name.",Lt,I,fe,Ur,Ke,pa="Create a schedule with a constant learning rate, using the learning rate set in optimizer.",Wt,F,_e,Rr,et,da=`Create a schedule with a constant learning rate preceded by a warmup period during which the learning rate
increases linearly between 0 and the initial lr set in the optimizer.`,jt,we,ha,Dt,P,be,Er,tt,ua=`Create a schedule with a learning rate that decreases following the values of the cosine function between the
initial lr set in the optimizer to 0, after a warmup period during which it increases linearly between 0 and the
initial lr set in the optimizer.`,Ut,ye,ga,Rt,V,ve,kr,rt,fa=`Create a schedule with a learning rate that decreases following the values of the cosine function between the
initial lr set in the optimizer to 0, with several hard restarts, after a warmup period during which it increases
linearly between 0 and the initial lr set in the optimizer.`,Et,$e,_a,kt,N,xe,Ir,at,wa=`Create a schedule with a learning rate that decreases linearly from the initial lr set in the optimizer to 0, after
a warmup period during which it increases linearly from 0 to the initial lr set in the optimizer.`,It,Te,ba,Ft,j,ze,Fr,nt,ya=`Create a schedule with a learning rate that decreases as a polynomial decay from the initial lr set in the
optimizer to end lr defined by <em>lr_end</em>, after a warmup period during which it increases linearly from 0 to the
initial lr set in the optimizer.`,Pr,st,va=`Note: <em>power</em> defaults to 1.0 as in the fairseq implementation, which in turn is based on the original BERT
implementation at
<a href="https://github.com/google-research/bert/blob/f39e881b169b9d53bea03d2d341b31707a6c052b/optimization.py#L37" rel="nofollow">https://github.com/google-research/bert/blob/f39e881b169b9d53bea03d2d341b31707a6c052b/optimization.py#L37</a>`,Pt,G,Me,Vr,ot,$a=`Create a schedule with an inverse square-root learning rate, from the initial lr set in the optimizer, after a
warmup period which increases lr linearly from 0 to the initial lr set in the optimizer.`,Vt,D,Ae,Nr,it,xa="Create a schedule with a learning rate that has three stages:",Gr,lt,Ta="<li>warmup: increase from min_lr_ratio times the initial learning rate to the initial learning rate following a warmup_type.</li> <li>stable: constant learning rate.</li> <li>decay: decrease from the initial learning rate to min_lr_ratio times the initial learning rate following a decay_type.</li>",Nt,Ce,Gt,H,Le,Hr,mt,za="Applies a warmup schedule on a given learning rate decay schedule.",Ht,We,Jt,je,Zt,U,De,Jr,ct,Ma=`Gradient accumulation utility. When used with a distribution strategy, the accumulator should be called in a
replica context. Gradients will be accumulated locally on each replica and without synchronization. Users should
then call <code>.gradients</code>, scale the gradients if required, and pass the result to <code>apply_gradients</code>.`,Zr,K,Ue,Sr,pt,Aa="Resets the accumulated gradients on the current replica.",St,Re,qt,ft,Bt;return v=new Z({props:{title:"Optimization",local:"optimization",headingTag:"h1"}}),re=new Z({props:{title:"AdamW (PyTorch)",local:"transformers.AdamW",headingTag:"h2"}}),ae=new M({props:{name:"class transformers.AdamW",anchor:"transformers.AdamW",parameters:[{name:"params",val:": typing.Iterable[torch.nn.parameter.Parameter]"},{name:"lr",val:": float = 0.001"},{name:"betas",val:": typing.Tuple[float, float] = (0.9, 0.999)"},{name:"eps",val:": float = 1e-06"},{name:"weight_decay",val:": float = 0.0"},{name:"correct_bias",val:": bool = True"},{name:"no_deprecation_warning",val:": bool = False"}],parametersDescription:[{anchor:"transformers.AdamW.params",description:`<strong>params</strong> (<code>Iterable[nn.parameter.Parameter]</code>) &#x2014;
Iterable of parameters to optimize or dictionaries defining parameter groups.`,name:"params"},{anchor:"transformers.AdamW.lr",description:`<strong>lr</strong> (<code>float</code>, <em>optional</em>, defaults to 0.001) &#x2014;
The learning rate to use.`,name:"lr"},{anchor:"transformers.AdamW.betas",description:`<strong>betas</strong> (<code>Tuple[float,float]</code>, <em>optional</em>, defaults to <code>(0.9, 0.999)</code>) &#x2014;
Adam&#x2019;s betas parameters (b1, b2).`,name:"betas"},{anchor:"transformers.AdamW.eps",description:`<strong>eps</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-06) &#x2014;
Adam&#x2019;s epsilon for numerical stability.`,name:"eps"},{anchor:"transformers.AdamW.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, <em>optional</em>, defaults to 0.0) &#x2014;
Decoupled weight decay to apply.`,name:"weight_decay"},{anchor:"transformers.AdamW.correct_bias",description:`<strong>correct_bias</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether or not to correct bias in Adam (for instance, in Bert TF repository they use <code>False</code>).`,name:"correct_bias"},{anchor:"transformers.AdamW.no_deprecation_warning",description:`<strong>no_deprecation_warning</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
A flag used to disable the deprecation warning (set to <code>True</code> to disable the warning).`,name:"no_deprecation_warning"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/optimization.py#L607"}}),ne=new M({props:{name:"step",anchor:"transformers.AdamW.step",parameters:[{name:"closure",val:": typing.Callable = None"}],parametersDescription:[{anchor:"transformers.AdamW.step.closure",description:"<strong>closure</strong> (<code>Callable</code>, <em>optional</em>) &#x2014; A closure that reevaluates the model and returns the loss.",name:"closure"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/optimization.py#L658"}}),se=new Z({props:{title:"AdaFactor (PyTorch)",local:"transformers.Adafactor",headingTag:"h2"}}),oe=new M({props:{name:"class transformers.Adafactor",anchor:"transformers.Adafactor",parameters:[{name:"params",val:""},{name:"lr",val:" = None"},{name:"eps",val:" = (1e-30, 0.001)"},{name:"clip_threshold",val:" = 1.0"},{name:"decay_rate",val:" = -0.8"},{name:"beta1",val:" = None"},{name:"weight_decay",val:" = 0.0"},{name:"scale_parameter",val:" = True"},{name:"relative_step",val:" = True"},{name:"warmup_init",val:" = False"}],parametersDescription:[{anchor:"transformers.Adafactor.params",description:`<strong>params</strong> (<code>Iterable[nn.parameter.Parameter]</code>) &#x2014;
Iterable of parameters to optimize or dictionaries defining parameter groups.`,name:"params"},{anchor:"transformers.Adafactor.lr",description:`<strong>lr</strong> (<code>float</code>, <em>optional</em>) &#x2014;
The external learning rate.`,name:"lr"},{anchor:"transformers.Adafactor.eps",description:`<strong>eps</strong> (<code>Tuple[float, float]</code>, <em>optional</em>, defaults to <code>(1e-30, 0.001)</code>) &#x2014;
Regularization constants for square gradient and parameter scale respectively`,name:"eps"},{anchor:"transformers.Adafactor.clip_threshold",description:`<strong>clip_threshold</strong> (<code>float</code>, <em>optional</em>, defaults to 1.0) &#x2014;
Threshold of root mean square of final gradient update`,name:"clip_threshold"},{anchor:"transformers.Adafactor.decay_rate",description:`<strong>decay_rate</strong> (<code>float</code>, <em>optional</em>, defaults to -0.8) &#x2014;
Coefficient used to compute running averages of square`,name:"decay_rate"},{anchor:"transformers.Adafactor.beta1",description:`<strong>beta1</strong> (<code>float</code>, <em>optional</em>) &#x2014;
Coefficient used for computing running averages of gradient`,name:"beta1"},{anchor:"transformers.Adafactor.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, <em>optional</em>, defaults to 0.0) &#x2014;
Weight decay (L2 penalty)`,name:"weight_decay"},{anchor:"transformers.Adafactor.scale_parameter",description:`<strong>scale_parameter</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
If True, learning rate is scaled by root mean square`,name:"scale_parameter"},{anchor:"transformers.Adafactor.relative_step",description:`<strong>relative_step</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
If True, time-dependent learning rate is computed instead of external learning rate`,name:"relative_step"},{anchor:"transformers.Adafactor.warmup_init",description:`<strong>warmup_init</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Time-dependent learning rate computation depends on whether warm-up initialization is being used`,name:"warmup_init"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/optimization.py#L721"}}),q=new ir({props:{anchor:"transformers.Adafactor.example",$$slots:{default:[Ra]},$$scope:{ctx:R}}}),B=new ir({props:{anchor:"transformers.Adafactor.example-2",$$slots:{default:[Ea]},$$scope:{ctx:R}}}),O=new ir({props:{anchor:"transformers.Adafactor.example-3",$$slots:{default:[ka]},$$scope:{ctx:R}}}),X=new ir({props:{anchor:"transformers.Adafactor.example-4",$$slots:{default:[Ia]},$$scope:{ctx:R}}}),ie=new M({props:{name:"step",anchor:"transformers.Adafactor.step",parameters:[{name:"closure",val:" = None"}],parametersDescription:[{anchor:"transformers.Adafactor.step.closure",description:`<strong>closure</strong> (callable, optional) &#x2014; A closure that reevaluates the model
and returns the loss.`,name:"closure"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/optimization.py#L867"}}),le=new Z({props:{title:"AdamWeightDecay (TensorFlow)",local:"transformers.AdamWeightDecay",headingTag:"h2"}}),me=new M({props:{name:"class transformers.AdamWeightDecay",anchor:"transformers.AdamWeightDecay",parameters:[{name:"learning_rate",val:": typing.Union[float, tf_keras.src.optimizers.schedules.learning_rate_schedule.LearningRateSchedule] = 0.001"},{name:"beta_1",val:": float = 0.9"},{name:"beta_2",val:": float = 0.999"},{name:"epsilon",val:": float = 1e-07"},{name:"amsgrad",val:": bool = False"},{name:"weight_decay_rate",val:": float = 0.0"},{name:"include_in_weight_decay",val:": typing.Optional[typing.List[str]] = None"},{name:"exclude_from_weight_decay",val:": typing.Optional[typing.List[str]] = None"},{name:"name",val:": str = 'AdamWeightDecay'"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.AdamWeightDecay.learning_rate",description:`<strong>learning_rate</strong> (<code>Union[float, LearningRateSchedule]</code>, <em>optional</em>, defaults to 0.001) &#x2014;
The learning rate to use or a schedule.`,name:"learning_rate"},{anchor:"transformers.AdamWeightDecay.beta_1",description:`<strong>beta_1</strong> (<code>float</code>, <em>optional</em>, defaults to 0.9) &#x2014;
The beta1 parameter in Adam, which is the exponential decay rate for the 1st momentum estimates.`,name:"beta_1"},{anchor:"transformers.AdamWeightDecay.beta_2",description:`<strong>beta_2</strong> (<code>float</code>, <em>optional</em>, defaults to 0.999) &#x2014;
The beta2 parameter in Adam, which is the exponential decay rate for the 2nd momentum estimates.`,name:"beta_2"},{anchor:"transformers.AdamWeightDecay.epsilon",description:`<strong>epsilon</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-07) &#x2014;
The epsilon parameter in Adam, which is a small constant for numerical stability.`,name:"epsilon"},{anchor:"transformers.AdamWeightDecay.amsgrad",description:`<strong>amsgrad</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to apply AMSGrad variant of this algorithm or not, see <a href="https://arxiv.org/abs/1904.09237" rel="nofollow">On the Convergence of Adam and
Beyond</a>.`,name:"amsgrad"},{anchor:"transformers.AdamWeightDecay.weight_decay_rate",description:`<strong>weight_decay_rate</strong> (<code>float</code>, <em>optional</em>, defaults to 0.0) &#x2014;
The weight decay to apply.`,name:"weight_decay_rate"},{anchor:"transformers.AdamWeightDecay.include_in_weight_decay",description:`<strong>include_in_weight_decay</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
List of the parameter names (or re patterns) to apply weight decay to. If none is passed, weight decay is
applied to all parameters by default (unless they are in <code>exclude_from_weight_decay</code>).`,name:"include_in_weight_decay"},{anchor:"transformers.AdamWeightDecay.exclude_from_weight_decay",description:`<strong>exclude_from_weight_decay</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
List of the parameter names (or re patterns) to exclude from applying weight decay to. If a
<code>include_in_weight_decay</code> is passed, the names in it will supersede this list.`,name:"exclude_from_weight_decay"},{anchor:"transformers.AdamWeightDecay.name",description:`<strong>name</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;AdamWeightDecay&quot;</code>) &#x2014;
Optional name for the operations created when applying gradients.`,name:"name"},{anchor:"transformers.AdamWeightDecay.kwargs",description:`<strong>kwargs</strong> (<code>Dict[str, Any]</code>, <em>optional</em>) &#x2014;
Keyword arguments. Allowed to be {<code>clipnorm</code>, <code>clipvalue</code>, <code>lr</code>, <code>decay</code>}. <code>clipnorm</code> is clip gradients by
norm; <code>clipvalue</code> is clip gradients by value, <code>decay</code> is included for backward compatibility to allow time
inverse decay of learning rate. <code>lr</code> is included for backward compatibility, recommended to use
<code>learning_rate</code> instead.`,name:"kwargs"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/optimization_tf.py#L180"}}),ce=new M({props:{name:"from_config",anchor:"transformers.AdamWeightDecay.from_config",parameters:[{name:"config",val:""}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/optimization_tf.py#L237"}}),pe=new M({props:{name:"transformers.create_optimizer",anchor:"transformers.create_optimizer",parameters:[{name:"init_lr",val:": float"},{name:"num_train_steps",val:": int"},{name:"num_warmup_steps",val:": int"},{name:"min_lr_ratio",val:": float = 0.0"},{name:"adam_beta1",val:": float = 0.9"},{name:"adam_beta2",val:": float = 0.999"},{name:"adam_epsilon",val:": float = 1e-08"},{name:"adam_clipnorm",val:": typing.Optional[float] = None"},{name:"adam_global_clipnorm",val:": typing.Optional[float] = None"},{name:"weight_decay_rate",val:": float = 0.0"},{name:"power",val:": float = 1.0"},{name:"include_in_weight_decay",val:": typing.Optional[typing.List[str]] = None"}],parametersDescription:[{anchor:"transformers.create_optimizer.init_lr",description:`<strong>init_lr</strong> (<code>float</code>) &#x2014;
The desired learning rate at the end of the warmup phase.`,name:"init_lr"},{anchor:"transformers.create_optimizer.num_train_steps",description:`<strong>num_train_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_train_steps"},{anchor:"transformers.create_optimizer.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of warmup steps.`,name:"num_warmup_steps"},{anchor:"transformers.create_optimizer.min_lr_ratio",description:`<strong>min_lr_ratio</strong> (<code>float</code>, <em>optional</em>, defaults to 0) &#x2014;
The final learning rate at the end of the linear decay will be <code>init_lr * min_lr_ratio</code>.`,name:"min_lr_ratio"},{anchor:"transformers.create_optimizer.adam_beta1",description:`<strong>adam_beta1</strong> (<code>float</code>, <em>optional</em>, defaults to 0.9) &#x2014;
The beta1 to use in Adam.`,name:"adam_beta1"},{anchor:"transformers.create_optimizer.adam_beta2",description:`<strong>adam_beta2</strong> (<code>float</code>, <em>optional</em>, defaults to 0.999) &#x2014;
The beta2 to use in Adam.`,name:"adam_beta2"},{anchor:"transformers.create_optimizer.adam_epsilon",description:`<strong>adam_epsilon</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-8) &#x2014;
The epsilon to use in Adam.`,name:"adam_epsilon"},{anchor:"transformers.create_optimizer.adam_clipnorm",description:`<strong>adam_clipnorm</strong> (<code>float</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
If not <code>None</code>, clip the gradient norm for each weight tensor to this value.`,name:"adam_clipnorm"},{anchor:"transformers.create_optimizer.adam_global_clipnorm",description:`<strong>adam_global_clipnorm</strong> (<code>float</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
If not <code>None</code>, clip gradient norm to this value. When using this argument, the norm is computed over all
weight tensors, as if they were concatenated into a single vector.`,name:"adam_global_clipnorm"},{anchor:"transformers.create_optimizer.weight_decay_rate",description:`<strong>weight_decay_rate</strong> (<code>float</code>, <em>optional</em>, defaults to 0) &#x2014;
The weight decay to use.`,name:"weight_decay_rate"},{anchor:"transformers.create_optimizer.power",description:`<strong>power</strong> (<code>float</code>, <em>optional</em>, defaults to 1.0) &#x2014;
The power to use for PolynomialDecay.`,name:"power"},{anchor:"transformers.create_optimizer.include_in_weight_decay",description:`<strong>include_in_weight_decay</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
List of the parameter names (or re patterns) to apply weight decay to. If none is passed, weight decay is
applied to all parameters except bias and layer norm parameters.`,name:"include_in_weight_decay"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/optimization_tf.py#L96"}}),de=new Z({props:{title:"Schedules",local:"schedules",headingTag:"h2"}}),he=new Z({props:{title:"Learning Rate Schedules (PyTorch)",local:"transformers.SchedulerType",headingTag:"h3"}}),ue=new M({props:{name:"class transformers.SchedulerType",anchor:"transformers.SchedulerType",parameters:[{name:"value",val:""},{name:"names",val:" = None"},{name:"module",val:" = None"},{name:"qualname",val:" = None"},{name:"type",val:" = None"},{name:"start",val:" = 1"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/trainer_utils.py#L418"}}),ge=new M({props:{name:"transformers.get_scheduler",anchor:"transformers.get_scheduler",parameters:[{name:"name",val:": typing.Union[str, transformers.trainer_utils.SchedulerType]"},{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": typing.Optional[int] = None"},{name:"num_training_steps",val:": typing.Optional[int] = None"},{name:"scheduler_specific_kwargs",val:": typing.Optional[dict] = None"}],parametersDescription:[{anchor:"transformers.get_scheduler.name",description:`<strong>name</strong> (<code>str</code> or <code>SchedulerType</code>) &#x2014;
The name of the scheduler to use.`,name:"name"},{anchor:"transformers.get_scheduler.optimizer",description:`<strong>optimizer</strong> (<code>torch.optim.Optimizer</code>) &#x2014;
The optimizer that will be used during training.`,name:"optimizer"},{anchor:"transformers.get_scheduler.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The number of warmup steps to do. This is not required by all schedulers (hence the argument being
optional), the function will raise an error if it&#x2019;s unset and the scheduler type requires it.`,name:"num_warmup_steps"},{anchor:"transformers.get_scheduler.num_training_steps",description:`<strong>num_training_steps</strong> (\`int&#x201C;, <em>optional</em>) &#x2014;
The number of training steps to do. This is not required by all schedulers (hence the argument being
optional), the function will raise an error if it&#x2019;s unset and the scheduler type requires it.`,name:"num_training_steps"},{anchor:"transformers.get_scheduler.scheduler_specific_kwargs",description:`<strong>scheduler_specific_kwargs</strong> (<code>dict</code>, <em>optional</em>) &#x2014;
Extra parameters for schedulers such as cosine with restarts. Mismatched scheduler types and scheduler
parameters will cause the scheduler function to raise a TypeError.`,name:"scheduler_specific_kwargs"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/optimization.py#L515"}}),fe=new M({props:{name:"transformers.get_constant_schedule",anchor:"transformers.get_constant_schedule",parameters:[{name:"optimizer",val:": Optimizer"},{name:"last_epoch",val:": int = -1"}],parametersDescription:[{anchor:"transformers.get_constant_schedule.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_constant_schedule.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/optimization.py#L40",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),_e=new M({props:{name:"transformers.get_constant_schedule_with_warmup",anchor:"transformers.get_constant_schedule_with_warmup",parameters:[{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": int"},{name:"last_epoch",val:": int = -1"}],parametersDescription:[{anchor:"transformers.get_constant_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_constant_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_constant_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/optimization.py#L81",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),be=new M({props:{name:"transformers.get_cosine_schedule_with_warmup",anchor:"transformers.get_cosine_schedule_with_warmup",parameters:[{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": int"},{name:"num_training_steps",val:": int"},{name:"num_cycles",val:": float = 0.5"},{name:"last_epoch",val:": int = -1"}],parametersDescription:[{anchor:"transformers.get_cosine_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_cosine_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_cosine_schedule_with_warmup.num_training_steps",description:`<strong>num_training_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_training_steps"},{anchor:"transformers.get_cosine_schedule_with_warmup.num_cycles",description:`<strong>num_cycles</strong> (<code>float</code>, <em>optional</em>, defaults to 0.5) &#x2014;
The number of waves in the cosine schedule (the defaults is to just decrease from the max value to 0
following a half-cosine).`,name:"num_cycles"},{anchor:"transformers.get_cosine_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/optimization.py#L144",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),ve=new M({props:{name:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup",anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup",parameters:[{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": int"},{name:"num_training_steps",val:": int"},{name:"num_cycles",val:": int = 1"},{name:"last_epoch",val:": int = -1"}],parametersDescription:[{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.num_training_steps",description:`<strong>num_training_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_training_steps"},{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.num_cycles",description:`<strong>num_cycles</strong> (<code>int</code>, <em>optional</em>, defaults to 1) &#x2014;
The number of hard restarts to use.`,name:"num_cycles"},{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/optimization.py#L189",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),xe=new M({props:{name:"transformers.get_linear_schedule_with_warmup",anchor:"transformers.get_linear_schedule_with_warmup",parameters:[{name:"optimizer",val:""},{name:"num_warmup_steps",val:""},{name:"num_training_steps",val:""},{name:"last_epoch",val:" = -1"}],parametersDescription:[{anchor:"transformers.get_linear_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_linear_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_linear_schedule_with_warmup.num_training_steps",description:`<strong>num_training_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_training_steps"},{anchor:"transformers.get_linear_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/optimization.py#L108",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),ze=new M({props:{name:"transformers.get_polynomial_decay_schedule_with_warmup",anchor:"transformers.get_polynomial_decay_schedule_with_warmup",parameters:[{name:"optimizer",val:""},{name:"num_warmup_steps",val:""},{name:"num_training_steps",val:""},{name:"lr_end",val:" = 1e-07"},{name:"power",val:" = 1.0"},{name:"last_epoch",val:" = -1"}],parametersDescription:[{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.num_training_steps",description:`<strong>num_training_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_training_steps"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.lr_end",description:`<strong>lr_end</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-7) &#x2014;
The end LR.`,name:"lr_end"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.power",description:`<strong>power</strong> (<code>float</code>, <em>optional</em>, defaults to 1.0) &#x2014;
Power factor.`,name:"power"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/optimization.py#L243",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),Me=new M({props:{name:"transformers.get_inverse_sqrt_schedule",anchor:"transformers.get_inverse_sqrt_schedule",parameters:[{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": int"},{name:"timescale",val:": int = None"},{name:"last_epoch",val:": int = -1"}],parametersDescription:[{anchor:"transformers.get_inverse_sqrt_schedule.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_inverse_sqrt_schedule.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_inverse_sqrt_schedule.timescale",description:`<strong>timescale</strong> (<code>int</code>, <em>optional</em>, defaults to <code>num_warmup_steps</code>) &#x2014;
Time scale.`,name:"timescale"},{anchor:"transformers.get_inverse_sqrt_schedule.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/optimization.py#L297",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),Ae=new M({props:{name:"transformers.get_wsd_schedule",anchor:"transformers.get_wsd_schedule",parameters:[{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": int"},{name:"num_decay_steps",val:": int"},{name:"num_training_steps",val:": typing.Optional[int] = None"},{name:"num_stable_steps",val:": typing.Optional[int] = None"},{name:"warmup_type",val:": str = 'linear'"},{name:"decay_type",val:": str = 'cosine'"},{name:"min_lr_ratio",val:": float = 0"},{name:"num_cycles",val:": float = 0.5"},{name:"last_epoch",val:": int = -1"}],parametersDescription:[{anchor:"transformers.get_wsd_schedule.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_wsd_schedule.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_wsd_schedule.num_decay_steps",description:`<strong>num_decay_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the decay phase.`,name:"num_decay_steps"},{anchor:"transformers.get_wsd_schedule.num_training_steps",description:`<strong>num_training_steps</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The total number of training steps. This is the sum of the warmup, stable and decay steps. If <code>num_stable_steps</code> is not provided, the stable phase will be <code>num_training_steps - num_warmup_steps - num_decay_steps</code>.`,name:"num_training_steps"},{anchor:"transformers.get_wsd_schedule.num_stable_steps",description:`<strong>num_stable_steps</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The number of steps for the stable phase. Please ensure that <code>num_warmup_steps + num_stable_steps + num_decay_steps</code> equals <code>num_training_steps</code>, otherwise the other steps will default to the minimum learning rate.`,name:"num_stable_steps"},{anchor:"transformers.get_wsd_schedule.warmup_type",description:`<strong>warmup_type</strong> (<code>str</code>, <em>optional</em>, defaults to &#x201C;linear&#x201D;) &#x2014;
The type of warmup to use. Can be &#x2018;linear&#x2019;, &#x2018;cosine&#x2019; or &#x2018;1-sqrt&#x2019;.`,name:"warmup_type"},{anchor:"transformers.get_wsd_schedule.decay_type",description:`<strong>decay_type</strong> (<code>str</code>, <em>optional</em>, defaults to &#x201C;cosine&#x201D;) &#x2014;
The type of decay to use. Can be &#x2018;linear&#x2019;, &#x2018;cosine&#x2019; or &#x2018;1-sqrt&#x2019;.`,name:"decay_type"},{anchor:"transformers.get_wsd_schedule.min_lr_ratio",description:`<strong>min_lr_ratio</strong> (<code>float</code>, <em>optional</em>, defaults to 0) &#x2014;
The minimum learning rate as a ratio of the initial learning rate.`,name:"min_lr_ratio"},{anchor:"transformers.get_wsd_schedule.num_cycles",description:`<strong>num_cycles</strong> (<code>float</code>, <em>optional</em>, defaults to 0.5) &#x2014;
The number of waves in the cosine schedule (the defaults is to just decrease from the max value to 0
following a half-cosine).`,name:"num_cycles"},{anchor:"transformers.get_wsd_schedule.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/optimization.py#L428",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),Ce=new Z({props:{title:"Warmup (TensorFlow)",local:"transformers.WarmUp",headingTag:"h3"}}),Le=new M({props:{name:"class transformers.WarmUp",anchor:"transformers.WarmUp",parameters:[{name:"initial_learning_rate",val:": float"},{name:"decay_schedule_fn",val:": typing.Callable"},{name:"warmup_steps",val:": int"},{name:"power",val:": float = 1.0"},{name:"name",val:": str = None"}],parametersDescription:[{anchor:"transformers.WarmUp.initial_learning_rate",description:`<strong>initial_learning_rate</strong> (<code>float</code>) &#x2014;
The initial learning rate for the schedule after the warmup (so this will be the learning rate at the end
of the warmup).`,name:"initial_learning_rate"},{anchor:"transformers.WarmUp.decay_schedule_fn",description:`<strong>decay_schedule_fn</strong> (<code>Callable</code>) &#x2014;
The schedule function to apply after the warmup for the rest of training.`,name:"decay_schedule_fn"},{anchor:"transformers.WarmUp.warmup_steps",description:`<strong>warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup part of training.`,name:"warmup_steps"},{anchor:"transformers.WarmUp.power",description:`<strong>power</strong> (<code>float</code>, <em>optional</em>, defaults to 1.0) &#x2014;
The power to use for the polynomial warmup (defaults is a linear warmup).`,name:"power"},{anchor:"transformers.WarmUp.name",description:`<strong>name</strong> (<code>str</code>, <em>optional</em>) &#x2014;
Optional name prefix for the returned tensors during the schedule.`,name:"name"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/optimization_tf.py#L38"}}),We=new Z({props:{title:"Gradient Strategies",local:"gradient-strategies",headingTag:"h2"}}),je=new Z({props:{title:"GradientAccumulator (TensorFlow)",local:"transformers.GradientAccumulator",headingTag:"h3"}}),De=new M({props:{name:"class transformers.GradientAccumulator",anchor:"transformers.GradientAccumulator",parameters:[],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/optimization_tf.py#L310"}}),Ue=new M({props:{name:"reset",anchor:"transformers.GradientAccumulator.reset",parameters:[],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/optimization_tf.py#L372"}}),Re=new Ua({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/main_classes/optimizer_schedules.md"}}),{c(){c=l("meta"),A=a(),x=l("p"),_=a(),p(v.$$.fragment),i=a(),T=l("p"),T.innerHTML=qr,_t=a(),te=l("ul"),te.innerHTML=Br,wt=a(),p(re.$$.fragment),bt=a(),L=l("div"),p(ae.$$.fragment),cr=a(),Ie=l("p"),Ie.innerHTML=Or,pr=a(),S=l("div"),p(ne.$$.fragment),dr=a(),Fe=l("p"),Fe.textContent=Xr,yt=a(),p(se.$$.fragment),vt=a(),y=l("div"),p(oe.$$.fragment),hr=a(),Pe=l("p"),Pe.innerHTML=Yr,ur=a(),Ve=l("p"),Ve.innerHTML=Qr,gr=a(),Ne=l("p"),Ne.textContent=Kr,fr=a(),Ge=l("p"),Ge.innerHTML=ea,_r=a(),He=l("ul"),He.innerHTML=ta,wr=a(),p(q.$$.fragment),br=a(),p(B.$$.fragment),yr=a(),Je=l("p"),Je.innerHTML=ra,vr=a(),p(O.$$.fragment),$r=a(),p(X.$$.fragment),xr=a(),Y=l("div"),p(ie.$$.fragment),Tr=a(),Ze=l("p"),Ze.textContent=aa,$t=a(),p(le.$$.fragment),xt=a(),C=l("div"),p(me.$$.fragment),zr=a(),Se=l("p"),Se.innerHTML=na,Mr=a(),qe=l("p"),qe.textContent=sa,Ar=a(),Q=l("div"),p(ce.$$.fragment),Cr=a(),Be=l("p"),Be.textContent=oa,Tt=a(),E=l("div"),p(pe.$$.fragment),Lr=a(),Oe=l("p"),Oe.textContent=ia,zt=a(),p(de.$$.fragment),Mt=a(),p(he.$$.fragment),At=a(),W=l("div"),p(ue.$$.fragment),Wr=a(),Xe=l("p"),Xe.innerHTML=la,jr=a(),Ye=l("ul"),Ye.innerHTML=ma,Ct=a(),k=l("div"),p(ge.$$.fragment),Dr=a(),Qe=l("p"),Qe.textContent=ca,Lt=a(),I=l("div"),p(fe.$$.fragment),Ur=a(),Ke=l("p"),Ke.textContent=pa,Wt=a(),F=l("div"),p(_e.$$.fragment),Rr=a(),et=l("p"),et.textContent=da,jt=a(),we=l("img"),Dt=a(),P=l("div"),p(be.$$.fragment),Er=a(),tt=l("p"),tt.textContent=ua,Ut=a(),ye=l("img"),Rt=a(),V=l("div"),p(ve.$$.fragment),kr=a(),rt=l("p"),rt.textContent=fa,Et=a(),$e=l("img"),kt=a(),N=l("div"),p(xe.$$.fragment),Ir=a(),at=l("p"),at.textContent=wa,It=a(),Te=l("img"),Ft=a(),j=l("div"),p(ze.$$.fragment),Fr=a(),nt=l("p"),nt.innerHTML=ya,Pr=a(),st=l("p"),st.innerHTML=va,Pt=a(),G=l("div"),p(Me.$$.fragment),Vr=a(),ot=l("p"),ot.textContent=$a,Vt=a(),D=l("div"),p(Ae.$$.fragment),Nr=a(),it=l("p"),it.textContent=xa,Gr=a(),lt=l("ol"),lt.innerHTML=Ta,Nt=a(),p(Ce.$$.fragment),Gt=a(),H=l("div"),p(Le.$$.fragment),Hr=a(),mt=l("p"),mt.textContent=za,Ht=a(),p(We.$$.fragment),Jt=a(),p(je.$$.fragment),Zt=a(),U=l("div"),p(De.$$.fragment),Jr=a(),ct=l("p"),ct.innerHTML=Ma,Zr=a(),K=l("div"),p(Ue.$$.fragment),Sr=a(),pt=l("p"),pt.textContent=Aa,St=a(),p(Re.$$.fragment),qt=a(),ft=l("p"),this.h()},l(e){const r=Da("svelte-u9bgzb",document.head);c=m(r,"META",{name:!0,content:!0}),r.forEach(t),A=n(e),x=m(e,"P",{}),z(x).forEach(t),_=n(e),d(v.$$.fragment,e),i=n(e),T=m(e,"P",{"data-svelte-h":!0}),w(T)!=="svelte-312sfx"&&(T.innerHTML=qr),_t=n(e),te=m(e,"UL",{"data-svelte-h":!0}),w(te)!=="svelte-ps1e52"&&(te.innerHTML=Br),wt=n(e),d(re.$$.fragment,e),bt=n(e),L=m(e,"DIV",{class:!0});var J=z(L);d(ae.$$.fragment,J),cr=n(J),Ie=m(J,"P",{"data-svelte-h":!0}),w(Ie)!=="svelte-b86q21"&&(Ie.innerHTML=Or),pr=n(J),S=m(J,"DIV",{class:!0});var Ee=z(S);d(ne.$$.fragment,Ee),dr=n(Ee),Fe=m(Ee,"P",{"data-svelte-h":!0}),w(Fe)!=="svelte-1gr03w4"&&(Fe.textContent=Xr),Ee.forEach(t),J.forEach(t),yt=n(e),d(se.$$.fragment,e),vt=n(e),y=m(e,"DIV",{class:!0});var $=z(y);d(oe.$$.fragment,$),hr=n($),Pe=m($,"P",{"data-svelte-h":!0}),w(Pe)!=="svelte-114ll6r"&&(Pe.innerHTML=Yr),ur=n($),Ve=m($,"P",{"data-svelte-h":!0}),w(Ve)!=="svelte-9bvrpy"&&(Ve.innerHTML=Qr),gr=n($),Ne=m($,"P",{"data-svelte-h":!0}),w(Ne)!=="svelte-1dxk71o"&&(Ne.textContent=Kr),fr=n($),Ge=m($,"P",{"data-svelte-h":!0}),w(Ge)!=="svelte-kc6644"&&(Ge.innerHTML=ea),_r=n($),He=m($,"UL",{"data-svelte-h":!0}),w(He)!=="svelte-p81jc"&&(He.innerHTML=ta),wr=n($),d(q.$$.fragment,$),br=n($),d(B.$$.fragment,$),yr=n($),Je=m($,"P",{"data-svelte-h":!0}),w(Je)!=="svelte-1qo676b"&&(Je.innerHTML=ra),vr=n($),d(O.$$.fragment,$),$r=n($),d(X.$$.fragment,$),xr=n($),Y=m($,"DIV",{class:!0});var ke=z(Y);d(ie.$$.fragment,ke),Tr=n(ke),Ze=m(ke,"P",{"data-svelte-h":!0}),w(Ze)!=="svelte-6lkj7q"&&(Ze.textContent=aa),ke.forEach(t),$.forEach(t),$t=n(e),d(le.$$.fragment,e),xt=n(e),C=m(e,"DIV",{class:!0});var ee=z(C);d(me.$$.fragment,ee),zr=n(ee),Se=m(ee,"P",{"data-svelte-h":!0}),w(Se)!=="svelte-towids"&&(Se.innerHTML=na),Mr=n(ee),qe=m(ee,"P",{"data-svelte-h":!0}),w(qe)!=="svelte-id029h"&&(qe.textContent=sa),Ar=n(ee),Q=m(ee,"DIV",{class:!0});var Ot=z(Q);d(ce.$$.fragment,Ot),Cr=n(Ot),Be=m(Ot,"P",{"data-svelte-h":!0}),w(Be)!=="svelte-3iict9"&&(Be.textContent=oa),Ot.forEach(t),ee.forEach(t),Tt=n(e),E=m(e,"DIV",{class:!0});var Xt=z(E);d(pe.$$.fragment,Xt),Lr=n(Xt),Oe=m(Xt,"P",{"data-svelte-h":!0}),w(Oe)!=="svelte-16qojcu"&&(Oe.textContent=ia),Xt.forEach(t),zt=n(e),d(de.$$.fragment,e),Mt=n(e),d(he.$$.fragment,e),At=n(e),W=m(e,"DIV",{class:!0});var dt=z(W);d(ue.$$.fragment,dt),Wr=n(dt),Xe=m(dt,"P",{"data-svelte-h":!0}),w(Xe)!=="svelte-xzcfpn"&&(Xe.innerHTML=la),jr=n(dt),Ye=m(dt,"UL",{"data-svelte-h":!0}),w(Ye)!=="svelte-1i9daf3"&&(Ye.innerHTML=ma),dt.forEach(t),Ct=n(e),k=m(e,"DIV",{class:!0});var Yt=z(k);d(ge.$$.fragment,Yt),Dr=n(Yt),Qe=m(Yt,"P",{"data-svelte-h":!0}),w(Qe)!=="svelte-1oh310h"&&(Qe.textContent=ca),Yt.forEach(t),Lt=n(e),I=m(e,"DIV",{class:!0});var Qt=z(I);d(fe.$$.fragment,Qt),Ur=n(Qt),Ke=m(Qt,"P",{"data-svelte-h":!0}),w(Ke)!=="svelte-cadr4a"&&(Ke.textContent=pa),Qt.forEach(t),Wt=n(e),F=m(e,"DIV",{class:!0});var Kt=z(F);d(_e.$$.fragment,Kt),Rr=n(Kt),et=m(Kt,"P",{"data-svelte-h":!0}),w(et)!=="svelte-1lj8vcj"&&(et.textContent=da),Kt.forEach(t),jt=n(e),we=m(e,"IMG",{alt:!0,src:!0}),Dt=n(e),P=m(e,"DIV",{class:!0});var er=z(P);d(be.$$.fragment,er),Er=n(er),tt=m(er,"P",{"data-svelte-h":!0}),w(tt)!=="svelte-1lhkhz0"&&(tt.textContent=ua),er.forEach(t),Ut=n(e),ye=m(e,"IMG",{alt:!0,src:!0}),Rt=n(e),V=m(e,"DIV",{class:!0});var tr=z(V);d(ve.$$.fragment,tr),kr=n(tr),rt=m(tr,"P",{"data-svelte-h":!0}),w(rt)!=="svelte-1pclsvt"&&(rt.textContent=fa),tr.forEach(t),Et=n(e),$e=m(e,"IMG",{alt:!0,src:!0}),kt=n(e),N=m(e,"DIV",{class:!0});var rr=z(N);d(xe.$$.fragment,rr),Ir=n(rr),at=m(rr,"P",{"data-svelte-h":!0}),w(at)!=="svelte-17q4ov5"&&(at.textContent=wa),rr.forEach(t),It=n(e),Te=m(e,"IMG",{alt:!0,src:!0}),Ft=n(e),j=m(e,"DIV",{class:!0});var ht=z(j);d(ze.$$.fragment,ht),Fr=n(ht),nt=m(ht,"P",{"data-svelte-h":!0}),w(nt)!=="svelte-1y55gjq"&&(nt.innerHTML=ya),Pr=n(ht),st=m(ht,"P",{"data-svelte-h":!0}),w(st)!=="svelte-1tn0l2l"&&(st.innerHTML=va),ht.forEach(t),Pt=n(e),G=m(e,"DIV",{class:!0});var ar=z(G);d(Me.$$.fragment,ar),Vr=n(ar),ot=m(ar,"P",{"data-svelte-h":!0}),w(ot)!=="svelte-yp2yn6"&&(ot.textContent=$a),ar.forEach(t),Vt=n(e),D=m(e,"DIV",{class:!0});var ut=z(D);d(Ae.$$.fragment,ut),Nr=n(ut),it=m(ut,"P",{"data-svelte-h":!0}),w(it)!=="svelte-1g5u0qv"&&(it.textContent=xa),Gr=n(ut),lt=m(ut,"OL",{"data-svelte-h":!0}),w(lt)!=="svelte-128o9uy"&&(lt.innerHTML=Ta),ut.forEach(t),Nt=n(e),d(Ce.$$.fragment,e),Gt=n(e),H=m(e,"DIV",{class:!0});var nr=z(H);d(Le.$$.fragment,nr),Hr=n(nr),mt=m(nr,"P",{"data-svelte-h":!0}),w(mt)!=="svelte-1mfn66a"&&(mt.textContent=za),nr.forEach(t),Ht=n(e),d(We.$$.fragment,e),Jt=n(e),d(je.$$.fragment,e),Zt=n(e),U=m(e,"DIV",{class:!0});var gt=z(U);d(De.$$.fragment,gt),Jr=n(gt),ct=m(gt,"P",{"data-svelte-h":!0}),w(ct)!=="svelte-8tkiw"&&(ct.innerHTML=Ma),Zr=n(gt),K=m(gt,"DIV",{class:!0});var sr=z(K);d(Ue.$$.fragment,sr),Sr=n(sr),pt=m(sr,"P",{"data-svelte-h":!0}),w(pt)!=="svelte-v0y289"&&(pt.textContent=Aa),sr.forEach(t),gt.forEach(t),St=n(e),d(Re.$$.fragment,e),qt=n(e),ft=m(e,"P",{}),z(ft).forEach(t),this.h()},h(){b(c,"name","hf:doc:metadata"),b(c,"content",Pa),b(S,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),b(L,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),b(Y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),b(y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),b(Q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),b(C,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),b(E,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),b(W,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),b(k,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),b(I,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),b(F,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),b(we,"alt",""),or(we.src,ha="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/warmup_constant_schedule.png")||b(we,"src",ha),b(P,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),b(ye,"alt",""),or(ye.src,ga="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/warmup_cosine_schedule.png")||b(ye,"src",ga),b(V,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),b($e,"alt",""),or($e.src,_a="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/warmup_cosine_hard_restarts_schedule.png")||b($e,"src",_a),b(N,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),b(Te,"alt",""),or(Te.src,ba="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/warmup_linear_schedule.png")||b(Te,"src",ba),b(j,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),b(G,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),b(D,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),b(H,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),b(K,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),b(U,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,r){s(document.head,c),o(e,A,r),o(e,x,r),o(e,_,r),h(v,e,r),o(e,i,r),o(e,T,r),o(e,_t,r),o(e,te,r),o(e,wt,r),h(re,e,r),o(e,bt,r),o(e,L,r),h(ae,L,null),s(L,cr),s(L,Ie),s(L,pr),s(L,S),h(ne,S,null),s(S,dr),s(S,Fe),o(e,yt,r),h(se,e,r),o(e,vt,r),o(e,y,r),h(oe,y,null),s(y,hr),s(y,Pe),s(y,ur),s(y,Ve),s(y,gr),s(y,Ne),s(y,fr),s(y,Ge),s(y,_r),s(y,He),s(y,wr),h(q,y,null),s(y,br),h(B,y,null),s(y,yr),s(y,Je),s(y,vr),h(O,y,null),s(y,$r),h(X,y,null),s(y,xr),s(y,Y),h(ie,Y,null),s(Y,Tr),s(Y,Ze),o(e,$t,r),h(le,e,r),o(e,xt,r),o(e,C,r),h(me,C,null),s(C,zr),s(C,Se),s(C,Mr),s(C,qe),s(C,Ar),s(C,Q),h(ce,Q,null),s(Q,Cr),s(Q,Be),o(e,Tt,r),o(e,E,r),h(pe,E,null),s(E,Lr),s(E,Oe),o(e,zt,r),h(de,e,r),o(e,Mt,r),h(he,e,r),o(e,At,r),o(e,W,r),h(ue,W,null),s(W,Wr),s(W,Xe),s(W,jr),s(W,Ye),o(e,Ct,r),o(e,k,r),h(ge,k,null),s(k,Dr),s(k,Qe),o(e,Lt,r),o(e,I,r),h(fe,I,null),s(I,Ur),s(I,Ke),o(e,Wt,r),o(e,F,r),h(_e,F,null),s(F,Rr),s(F,et),o(e,jt,r),o(e,we,r),o(e,Dt,r),o(e,P,r),h(be,P,null),s(P,Er),s(P,tt),o(e,Ut,r),o(e,ye,r),o(e,Rt,r),o(e,V,r),h(ve,V,null),s(V,kr),s(V,rt),o(e,Et,r),o(e,$e,r),o(e,kt,r),o(e,N,r),h(xe,N,null),s(N,Ir),s(N,at),o(e,It,r),o(e,Te,r),o(e,Ft,r),o(e,j,r),h(ze,j,null),s(j,Fr),s(j,nt),s(j,Pr),s(j,st),o(e,Pt,r),o(e,G,r),h(Me,G,null),s(G,Vr),s(G,ot),o(e,Vt,r),o(e,D,r),h(Ae,D,null),s(D,Nr),s(D,it),s(D,Gr),s(D,lt),o(e,Nt,r),h(Ce,e,r),o(e,Gt,r),o(e,H,r),h(Le,H,null),s(H,Hr),s(H,mt),o(e,Ht,r),h(We,e,r),o(e,Jt,r),h(je,e,r),o(e,Zt,r),o(e,U,r),h(De,U,null),s(U,Jr),s(U,ct),s(U,Zr),s(U,K),h(Ue,K,null),s(K,Sr),s(K,pt),o(e,St,r),h(Re,e,r),o(e,qt,r),o(e,ft,r),Bt=!0},p(e,[r]){const J={};r&2&&(J.$$scope={dirty:r,ctx:e}),q.$set(J);const Ee={};r&2&&(Ee.$$scope={dirty:r,ctx:e}),B.$set(Ee);const $={};r&2&&($.$$scope={dirty:r,ctx:e}),O.$set($);const ke={};r&2&&(ke.$$scope={dirty:r,ctx:e}),X.$set(ke)},i(e){Bt||(u(v.$$.fragment,e),u(re.$$.fragment,e),u(ae.$$.fragment,e),u(ne.$$.fragment,e),u(se.$$.fragment,e),u(oe.$$.fragment,e),u(q.$$.fragment,e),u(B.$$.fragment,e),u(O.$$.fragment,e),u(X.$$.fragment,e),u(ie.$$.fragment,e),u(le.$$.fragment,e),u(me.$$.fragment,e),u(ce.$$.fragment,e),u(pe.$$.fragment,e),u(de.$$.fragment,e),u(he.$$.fragment,e),u(ue.$$.fragment,e),u(ge.$$.fragment,e),u(fe.$$.fragment,e),u(_e.$$.fragment,e),u(be.$$.fragment,e),u(ve.$$.fragment,e),u(xe.$$.fragment,e),u(ze.$$.fragment,e),u(Me.$$.fragment,e),u(Ae.$$.fragment,e),u(Ce.$$.fragment,e),u(Le.$$.fragment,e),u(We.$$.fragment,e),u(je.$$.fragment,e),u(De.$$.fragment,e),u(Ue.$$.fragment,e),u(Re.$$.fragment,e),Bt=!0)},o(e){g(v.$$.fragment,e),g(re.$$.fragment,e),g(ae.$$.fragment,e),g(ne.$$.fragment,e),g(se.$$.fragment,e),g(oe.$$.fragment,e),g(q.$$.fragment,e),g(B.$$.fragment,e),g(O.$$.fragment,e),g(X.$$.fragment,e),g(ie.$$.fragment,e),g(le.$$.fragment,e),g(me.$$.fragment,e),g(ce.$$.fragment,e),g(pe.$$.fragment,e),g(de.$$.fragment,e),g(he.$$.fragment,e),g(ue.$$.fragment,e),g(ge.$$.fragment,e),g(fe.$$.fragment,e),g(_e.$$.fragment,e),g(be.$$.fragment,e),g(ve.$$.fragment,e),g(xe.$$.fragment,e),g(ze.$$.fragment,e),g(Me.$$.fragment,e),g(Ae.$$.fragment,e),g(Ce.$$.fragment,e),g(Le.$$.fragment,e),g(We.$$.fragment,e),g(je.$$.fragment,e),g(De.$$.fragment,e),g(Ue.$$.fragment,e),g(Re.$$.fragment,e),Bt=!1},d(e){e&&(t(A),t(x),t(_),t(i),t(T),t(_t),t(te),t(wt),t(bt),t(L),t(yt),t(vt),t(y),t($t),t(xt),t(C),t(Tt),t(E),t(zt),t(Mt),t(At),t(W),t(Ct),t(k),t(Lt),t(I),t(Wt),t(F),t(jt),t(we),t(Dt),t(P),t(Ut),t(ye),t(Rt),t(V),t(Et),t($e),t(kt),t(N),t(It),t(Te),t(Ft),t(j),t(Pt),t(G),t(Vt),t(D),t(Nt),t(Gt),t(H),t(Ht),t(Jt),t(Zt),t(U),t(St),t(qt),t(ft)),t(c),f(v,e),f(re,e),f(ae),f(ne),f(se,e),f(oe),f(q),f(B),f(O),f(X),f(ie),f(le,e),f(me),f(ce),f(pe),f(de,e),f(he,e),f(ue),f(ge),f(fe),f(_e),f(be),f(ve),f(xe),f(ze),f(Me),f(Ae),f(Ce,e),f(Le),f(We,e),f(je,e),f(De),f(Ue),f(Re,e)}}}const Pa='{"title":"Optimization","local":"optimization","sections":[{"title":"AdamW (PyTorch)","local":"transformers.AdamW","sections":[],"depth":2},{"title":"AdaFactor (PyTorch)","local":"transformers.Adafactor","sections":[],"depth":2},{"title":"AdamWeightDecay (TensorFlow)","local":"transformers.AdamWeightDecay","sections":[],"depth":2},{"title":"Schedules","local":"schedules","sections":[{"title":"Learning Rate Schedules (PyTorch)","local":"transformers.SchedulerType","sections":[],"depth":3},{"title":"Warmup (TensorFlow)","local":"transformers.WarmUp","sections":[],"depth":3}],"depth":2},{"title":"Gradient Strategies","local":"gradient-strategies","sections":[{"title":"GradientAccumulator (TensorFlow)","local":"transformers.GradientAccumulator","sections":[],"depth":3}],"depth":2}],"depth":1}';function Va(R){return La(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class qa extends Wa{constructor(c){super(),ja(this,c,Va,Fa,Ca,{})}}export{qa as component};
