import{s as fo,o as ho,n as qe}from"../chunks/scheduler.25b97de1.js";import{S as go,i as _o,g as d,s,r as m,A as Mo,h as l,f as n,c as a,j as q,u as p,x as b,k as P,y as r,a as c,v as u,d as f,t as h,w as g}from"../chunks/index.d9030fc9.js";import{T as Xe}from"../chunks/Tip.baa67368.js";import{D as oe}from"../chunks/Docstring.e257edda.js";import{C as bo}from"../chunks/CodeBlock.e6cd0d95.js";import{E as vo}from"../chunks/ExampleCodeBlock.20db4b6e.js";import{H as ge,E as $o}from"../chunks/EditOnGithub.91d95064.js";function To(J){let t,_;return t=new bo({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEpldE1vZU1vZGVsJTJDJTIwSmV0TW9lQ29uZmlnJTBBJTBBJTIzJTIwSW5pdGlhbGl6aW5nJTIwYSUyMEpldE1vZSUyMDRCJTIwc3R5bGUlMjBjb25maWd1cmF0aW9uJTBBY29uZmlndXJhdGlvbiUyMCUzRCUyMEpldE1vZUNvbmZpZygpJTBBJTBBJTIzJTIwSW5pdGlhbGl6aW5nJTIwYSUyMG1vZGVsJTIwZnJvbSUyMHRoZSUyMEpldE1vZSUyMDRCJTIwc3R5bGUlMjBjb25maWd1cmF0aW9uJTBBbW9kZWwlMjAlM0QlMjBKZXRNb2VNb2RlbChjb25maWd1cmF0aW9uKSUwQSUwQSUyMyUyMEFjY2Vzc2luZyUyMHRoZSUyMG1vZGVsJTIwY29uZmlndXJhdGlvbiUwQWNvbmZpZ3VyYXRpb24lMjAlM0QlMjBtb2RlbC5jb25maWc=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> JetMoeModel, JetMoeConfig

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a JetMoe 4B style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = JetMoeConfig()

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a model from the JetMoe 4B style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = JetMoeModel(configuration)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Accessing the model configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = model.config`,wrap:!1}}),{c(){m(t.$$.fragment)},l(i){p(t.$$.fragment,i)},m(i,w){u(t,i,w),_=!0},p:qe,i(i){_||(f(t.$$.fragment,i),_=!0)},o(i){h(t.$$.fragment,i),_=!1},d(i){g(t,i)}}}function wo(J){let t,_=`Although the recipe for forward pass needs to be defined within this function, one should call the <code>Module</code>
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`;return{c(){t=d("p"),t.innerHTML=_},l(i){t=l(i,"P",{"data-svelte-h":!0}),b(t)!=="svelte-fincs2"&&(t.innerHTML=_)},m(i,w){c(i,t,w)},p:qe,d(i){i&&n(t)}}}function yo(J){let t,_=`Although the recipe for forward pass needs to be defined within this function, one should call the <code>Module</code>
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`;return{c(){t=d("p"),t.innerHTML=_},l(i){t=l(i,"P",{"data-svelte-h":!0}),b(t)!=="svelte-fincs2"&&(t.innerHTML=_)},m(i,w){c(i,t,w)},p:qe,d(i){i&&n(t)}}}function ko(J){let t,_=`Although the recipe for forward pass needs to be defined within this function, one should call the <code>Module</code>
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`;return{c(){t=d("p"),t.innerHTML=_},l(i){t=l(i,"P",{"data-svelte-h":!0}),b(t)!=="svelte-fincs2"&&(t.innerHTML=_)},m(i,w){c(i,t,w)},p:qe,d(i){i&&n(t)}}}function Jo(J){let t,_,i,w,W,Me,H,be,N,Ke=`<strong>JetMoe-8B</strong> is an 8B Mixture-of-Experts (MoE) language model developed by <a href="https://scholar.google.com.hk/citations?user=qff5rRYAAAAJ" rel="nofollow">Yikang Shen</a> and <a href="https://myshell.ai/" rel="nofollow">MyShell</a>.
JetMoe project aims to provide a LLaMA2-level performance and efficient language model with a limited budget.
To achieve this goal, JetMoe uses a sparsely activated architecture inspired by the <a href="https://arxiv.org/abs/2306.04640" rel="nofollow">ModuleFormer</a>.
Each JetMoe block consists of two MoE layers: Mixture of Attention Heads and Mixture of MLP Experts.
Given the input tokens, it activates a subset of its experts to process them.
This sparse activation schema enables JetMoe to achieve much better training throughput than similar size dense models.
The training throughput of JetMoe-8B is around 100B tokens per day on a cluster of 96 H100 GPUs with a straightforward 3-way pipeline parallelism strategy.`,ve,E,eo='This model was contributed by <a href="https://huggingface.co/YikangS" rel="nofollow">Yikang Shen</a>.',$e,D,Te,v,B,Pe,te,oo=`This is the configuration class to store the configuration of a <a href="/docs/transformers/main/en/model_doc/jetmoe#transformers.JetMoeModel">JetMoeModel</a>. It is used to instantiate a
JetMoe model according to the specified arguments, defining the model architecture. Instantiating a configuration
with the defaults will yield a configuration of the JetMoe-4B.`,Se,ne,to='<a href="https://huggingface.co/jetmoe/jetmoe-8b" rel="nofollow">jetmoe/jetmoe-8b</a>',je,re,no=`Configuration objects inherit from <a href="/docs/transformers/main/en/main_classes/configuration#transformers.PretrainedConfig">PretrainedConfig</a> and can be used to control the model outputs. Read the
documentation from <a href="/docs/transformers/main/en/main_classes/configuration#transformers.PretrainedConfig">PretrainedConfig</a> for more information.`,Ie,S,we,U,ye,$,A,Oe,se,ro=`The bare JetMoe Model outputting raw hidden-states without any specific head on top.
This model is a PyTorch <a href="https://pytorch.org/docs/stable/nn.html#torch.nn.Module" rel="nofollow">torch.nn.Module</a> sub-class. Use
it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
behavior.`,We,ae,so="Transformer decoder consisting of <em>config.num_hidden_layers</em> layers. Each layer is a <code>JetMoeBlock</code>",He,x,Z,Ne,ie,ao='The <a href="/docs/transformers/main/en/model_doc/jetmoe#transformers.JetMoeModel">JetMoeModel</a> forward method, overrides the <code>__call__</code> special method.',Ee,j,ke,G,Je,F,R,De,C,V,Be,de,io='The <a href="/docs/transformers/main/en/model_doc/jetmoe#transformers.JetMoeForCausalLM">JetMoeForCausalLM</a> forward method, overrides the <code>__call__</code> special method.',Ue,I,xe,Y,Ce,M,Q,Ae,le,lo="The JetMoe Model transformer with a sequence classification head on top (linear layer).",Ze,ce,co=`<a href="/docs/transformers/main/en/model_doc/jetmoe#transformers.JetMoeForSequenceClassification">JetMoeForSequenceClassification</a> uses the last token in order to do the classification, as other causal models
(e.g. GPT-2) do.`,Ge,me,mo=`Since it does classification on the last token, it requires to know the position of the last token. If a
<code>pad_token_id</code> is defined in the configuration, it finds the last token that is not a padding token in each row. If
no <code>pad_token_id</code> is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
padding tokens when <code>inputs_embeds</code> are passed instead of <code>input_ids</code>, it does the same (take the last value in
each row of the batch).`,Re,pe,po=`This model is a PyTorch <a href="https://pytorch.org/docs/stable/nn.html#torch.nn.Module" rel="nofollow">torch.nn.Module</a> sub-class. Use
it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
behavior.`,Ve,L,X,Ye,ue,uo='The <a href="/docs/transformers/main/en/model_doc/jetmoe#transformers.JetMoeForSequenceClassification">JetMoeForSequenceClassification</a> forward method, overrides the <code>__call__</code> special method.',Qe,O,Le,K,Fe,_e,ze;return W=new ge({props:{title:"JetMoe",local:"jetmoe",headingTag:"h1"}}),H=new ge({props:{title:"Overview",local:"overview",headingTag:"h2"}}),D=new ge({props:{title:"JetMoeConfig",local:"transformers.JetMoeConfig",headingTag:"h2"}}),B=new oe({props:{name:"class transformers.JetMoeConfig",anchor:"transformers.JetMoeConfig",parameters:[{name:"vocab_size",val:" = 32000"},{name:"hidden_size",val:" = 2048"},{name:"num_hidden_layers",val:" = 12"},{name:"num_key_value_heads",val:" = 16"},{name:"kv_channels",val:" = 128"},{name:"intermediate_size",val:" = 5632"},{name:"max_position_embeddings",val:" = 4096"},{name:"activation_function",val:" = 'silu'"},{name:"num_local_experts",val:" = 8"},{name:"num_experts_per_tok",val:" = 2"},{name:"output_router_logits",val:" = False"},{name:"aux_loss_coef",val:" = 0.01"},{name:"use_cache",val:" = True"},{name:"bos_token_id",val:" = 1"},{name:"eos_token_id",val:" = 2"},{name:"tie_word_embeddings",val:" = True"},{name:"rope_theta",val:" = 10000.0"},{name:"rms_norm_eps",val:" = 1e-06"},{name:"initializer_range",val:" = 0.01"},{name:"attention_dropout",val:" = 0.0"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.JetMoeConfig.vocab_size",description:`<strong>vocab_size</strong> (<code>int</code>, <em>optional</em>, defaults to 32000) &#x2014;
Vocabulary size of the JetMoe model. Defines the number of different tokens that can be represented by the
<code>inputs_ids</code> passed when calling <a href="/docs/transformers/main/en/model_doc/jetmoe#transformers.JetMoeModel">JetMoeModel</a>`,name:"vocab_size"},{anchor:"transformers.JetMoeConfig.hidden_size",description:`<strong>hidden_size</strong> (<code>int</code>, <em>optional</em>, defaults to 2048) &#x2014;
Dimension of the hidden representations.`,name:"hidden_size"},{anchor:"transformers.JetMoeConfig.num_hidden_layers",description:`<strong>num_hidden_layers</strong> (<code>int</code>, <em>optional</em>, defaults to 12) &#x2014;
Number of hidden layers in the Transformer encoder.`,name:"num_hidden_layers"},{anchor:"transformers.JetMoeConfig.num_key_value_heads",description:`<strong>num_key_value_heads</strong> (<code>int</code>, <em>optional</em>, defaults to 16) &#x2014;
Number of attention heads for each key and value in the Transformer encoder.`,name:"num_key_value_heads"},{anchor:"transformers.JetMoeConfig.kv_channels",description:`<strong>kv_channels</strong> (<code>int</code>, <em>optional</em>, defaults to 128) &#x2014;
Defines the number of channels for the key and value tensors.`,name:"kv_channels"},{anchor:"transformers.JetMoeConfig.intermediate_size",description:`<strong>intermediate_size</strong> (<code>int</code>, <em>optional</em>, defaults to 5632) &#x2014;
Dimension of the MLP representations.`,name:"intermediate_size"},{anchor:"transformers.JetMoeConfig.max_position_embeddings",description:`<strong>max_position_embeddings</strong> (<code>int</code>, <em>optional</em>, defaults to 4096) &#x2014;
The maximum sequence length that this model might ever be used with. JetMoe&#x2019;s attention allows sequence of
up to 4096 tokens.`,name:"max_position_embeddings"},{anchor:"transformers.JetMoeConfig.activation_function",description:`<strong>activation_function</strong> (<code>string</code>, <em>optional</em>, defaults to <code>&quot;silu&quot;</code>) &#x2014;
Defines the activation function for MLP experts.`,name:"activation_function"},{anchor:"transformers.JetMoeConfig.num_local_experts",description:`<strong>num_local_experts</strong> (<code>int</code>, <em>optional</em>, defaults to 8) &#x2014;
Defines the number of experts in the MoE and MoA.`,name:"num_local_experts"},{anchor:"transformers.JetMoeConfig.num_experts_per_tok",description:"<strong>num_experts_per_tok</strong> (`int, <em>optional</em>, defaults to 2) &#x2014;\nThe number of experts to route per-token and for MoE and MoA.",name:"num_experts_per_tok"},{anchor:"transformers.JetMoeConfig.output_router_logits",description:`<strong>output_router_logits</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not the router logits should be returned by the model. Enabeling this will also
allow the model to output the auxiliary loss.`,name:"output_router_logits"},{anchor:"transformers.JetMoeConfig.aux_loss_coef",description:`<strong>aux_loss_coef</strong> (<code>float</code>, <em>optional</em>, defaults to 0.01) &#x2014;
The coefficient for the auxiliary loss.`,name:"aux_loss_coef"},{anchor:"transformers.JetMoeConfig.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether or not the model should return the last key/values attentions (not used by all models). Only
relevant if <code>config.is_decoder=True</code>.`,name:"use_cache"},{anchor:"transformers.JetMoeConfig.bos_token_id",description:`<strong>bos_token_id</strong> (<code>int</code>, <em>optional</em>, defaults to 1) &#x2014;
The id of the &#x201C;beginning-of-sequence&#x201D; token.`,name:"bos_token_id"},{anchor:"transformers.JetMoeConfig.eos_token_id",description:`<strong>eos_token_id</strong> (<code>int</code>, <em>optional</em>, defaults to 2) &#x2014;
The id of the &#x201C;end-of-sequence&#x201D; token.`,name:"eos_token_id"},{anchor:"transformers.JetMoeConfig.tie_word_embeddings",description:`<strong>tie_word_embeddings</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether the model&#x2019;s input and output word embeddings should be tied.`,name:"tie_word_embeddings"},{anchor:"transformers.JetMoeConfig.rope_theta",description:`<strong>rope_theta</strong> (<code>float</code>, <em>optional</em>, defaults to 10000.0) &#x2014;
The base period of the RoPE embeddings.`,name:"rope_theta"},{anchor:"transformers.JetMoeConfig.rms_norm_eps",description:`<strong>rms_norm_eps</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-06) &#x2014;
The epsilon used by the rms normalization layers.`,name:"rms_norm_eps"},{anchor:"transformers.JetMoeConfig.initializer_range",description:`<strong>initializer_range</strong> (<code>float</code>, <em>optional</em>, defaults to 0.01) &#x2014;
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.`,name:"initializer_range"},{anchor:"transformers.JetMoeConfig.attention_dropout",description:`<strong>attention_dropout</strong> (<code>float</code>, <em>optional</em>, defaults to 0.0) &#x2014;
The dropout ratio for the attention probabilities.`,name:"attention_dropout"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/jetmoe/configuration_jetmoe.py#L24"}}),S=new vo({props:{anchor:"transformers.JetMoeConfig.example",$$slots:{default:[To]},$$scope:{ctx:J}}}),U=new ge({props:{title:"JetMoeModel",local:"transformers.JetMoeModel",headingTag:"h2"}}),A=new oe({props:{name:"class transformers.JetMoeModel",anchor:"transformers.JetMoeModel",parameters:[{name:"config",val:": JetMoeConfig"}],parametersDescription:[{anchor:"transformers.JetMoeModel.config",description:`<strong>config</strong> (<a href="/docs/transformers/main/en/model_doc/jetmoe#transformers.JetMoeConfig">JetMoeConfig</a>) &#x2014; Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
configuration. Check out the <a href="/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.
config &#x2014;
JetMoeConfig`,name:"config"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/jetmoe/modeling_jetmoe.py#L961"}}),Z=new oe({props:{name:"forward",anchor:"transformers.JetMoeModel.forward",parameters:[{name:"input_ids",val:": LongTensor = None"},{name:"attention_mask",val:": Optional = None"},{name:"position_ids",val:": Optional = None"},{name:"past_key_values",val:": Union = None"},{name:"inputs_embeds",val:": Optional = None"},{name:"use_cache",val:": Optional = None"},{name:"output_attentions",val:": Optional = None"},{name:"output_hidden_states",val:": Optional = None"},{name:"output_router_logits",val:": Optional = None"},{name:"return_dict",val:": Optional = None"},{name:"cache_position",val:": Optional = None"}],parametersDescription:[{anchor:"transformers.JetMoeModel.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.LongTensor</code> of shape <code>({0})</code>) &#x2014;
Indices of input sequence tokens in the vocabulary.</p>
<p>Indices can be obtained using <code>AutoProcenizer</code>. See <a href="/docs/transformers/main/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/main/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p>
<p><a href="../glossary#input-ids">What are input IDs?</a>`,name:"input_ids"},{anchor:"transformers.JetMoeModel.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.FloatTensor</code> of shape <code>({0})</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>not masked</strong>,</li>
<li>0 for tokens that are <strong>masked</strong>.</li>
</ul>
<p><a href="../glossary#attention-mask">What are attention masks?</a>`,name:"attention_mask"},{anchor:"transformers.JetMoeModel.forward.position_ids",description:`<strong>position_ids</strong> (<code>torch.LongTensor</code> of shape <code>({0})</code>, <em>optional</em>) &#x2014;
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range <code>[0, config.n_positions - 1]</code>.</p>
<p><a href="../glossary#position-ids">What are position IDs?</a>`,name:"position_ids"},{anchor:"transformers.JetMoeModel.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.FloatTensor</code> of shape <code>({0}, hidden_dim)</code>, <em>optional</em>) &#x2014;
Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This
is useful if you want more control over how to convert <em>input_ids</em> indices into associated vectors than the
model&#x2019;s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.JetMoeModel.forward.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.JetMoeModel.forward.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.JetMoeModel.forward.output_router_logits",description:`<strong>output_router_logits</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the logits of all the routers. They are useful for computing the router loss, and
should not be returned during inference.`,name:"output_router_logits"},{anchor:"transformers.JetMoeModel.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return a <a href="/docs/transformers/main/en/main_classes/output#transformers.utils.ModelOutput">ModelOutput</a> instead of a plain tuple.`,name:"return_dict"},{anchor:"transformers.JetMoeModel.forward.cache_position",description:`<strong>cache_position</strong> (<code>torch.LongTensor</code> of shape <code>(sequence_length)</code>, <em>optional</em>) &#x2014;
Indices depicting the position of the input sequence tokens in the sequence. Contrarily to <code>position_ids</code>,
this tensor is not affected by padding. It is used to update the cache in the correct position and to infer
the complete sequence length.`,name:"cache_position"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/jetmoe/modeling_jetmoe.py#L996"}}),j=new Xe({props:{$$slots:{default:[wo]},$$scope:{ctx:J}}}),G=new ge({props:{title:"JetMoeForCausalLM",local:"transformers.JetMoeForCausalLM",headingTag:"h2"}}),R=new oe({props:{name:"class transformers.JetMoeForCausalLM",anchor:"transformers.JetMoeForCausalLM",parameters:[{name:"config",val:""}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/jetmoe/modeling_jetmoe.py#L1202"}}),V=new oe({props:{name:"forward",anchor:"transformers.JetMoeForCausalLM.forward",parameters:[{name:"input_ids",val:": LongTensor = None"},{name:"attention_mask",val:": Optional = None"},{name:"position_ids",val:": Optional = None"},{name:"past_key_values",val:": Optional = None"},{name:"inputs_embeds",val:": Optional = None"},{name:"labels",val:": Optional = None"},{name:"use_cache",val:": Optional = None"},{name:"output_attentions",val:": Optional = None"},{name:"output_hidden_states",val:": Optional = None"},{name:"output_router_logits",val:": Optional = None"},{name:"return_dict",val:": Optional = None"},{name:"cache_position",val:": Optional = None"}],parametersDescription:[{anchor:"transformers.JetMoeForCausalLM.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.LongTensor</code> of shape <code>({0})</code>) &#x2014;
Indices of input sequence tokens in the vocabulary.</p>
<p>Indices can be obtained using <code>AutoProcenizer</code>. See <a href="/docs/transformers/main/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/main/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p>
<p><a href="../glossary#input-ids">What are input IDs?</a>`,name:"input_ids"},{anchor:"transformers.JetMoeForCausalLM.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.FloatTensor</code> of shape <code>({0})</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>not masked</strong>,</li>
<li>0 for tokens that are <strong>masked</strong>.</li>
</ul>
<p><a href="../glossary#attention-mask">What are attention masks?</a>`,name:"attention_mask"},{anchor:"transformers.JetMoeForCausalLM.forward.position_ids",description:`<strong>position_ids</strong> (<code>torch.LongTensor</code> of shape <code>({0})</code>, <em>optional</em>) &#x2014;
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range <code>[0, config.n_positions - 1]</code>.</p>
<p><a href="../glossary#position-ids">What are position IDs?</a>`,name:"position_ids"},{anchor:"transformers.JetMoeForCausalLM.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.FloatTensor</code> of shape <code>({0}, hidden_dim)</code>, <em>optional</em>) &#x2014;
Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This
is useful if you want more control over how to convert <em>input_ids</em> indices into associated vectors than the
model&#x2019;s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.JetMoeForCausalLM.forward.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.JetMoeForCausalLM.forward.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.JetMoeForCausalLM.forward.output_router_logits",description:`<strong>output_router_logits</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the logits of all the routers. They are useful for computing the router loss, and
should not be returned during inference.`,name:"output_router_logits"},{anchor:"transformers.JetMoeForCausalLM.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return a <a href="/docs/transformers/main/en/main_classes/output#transformers.utils.ModelOutput">ModelOutput</a> instead of a plain tuple.`,name:"return_dict"},{anchor:"transformers.JetMoeForCausalLM.forward.cache_position",description:`<strong>cache_position</strong> (<code>torch.LongTensor</code> of shape <code>(sequence_length)</code>, <em>optional</em>) &#x2014;
Indices depicting the position of the input sequence tokens in the sequence. Contrarily to <code>position_ids</code>,
this tensor is not affected by padding. It is used to update the cache in the correct position and to infer
the complete sequence length.</p>
<p>Args &#x2014;
labels (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>):
Labels for computing the masked language modeling loss. Indices should either be in <code>[0, ..., config.vocab_size]</code> or -100 (see <code>input_ids</code> docstring). Tokens with indices set to <code>-100</code> are ignored
(masked), the loss is only computed for the tokens with labels in <code>[0, ..., config.vocab_size]</code>.`,name:"cache_position"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/jetmoe/modeling_jetmoe.py#L1240",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A <code>transformers.modeling_outputs.MoeCausalLMOutputWithPast</code> or a tuple of
<code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various
elements depending on the configuration (<a
  href="/docs/transformers/main/en/model_doc/jetmoe#transformers.JetMoeConfig"
>JetMoeConfig</a>) and inputs.</p>
<ul>
<li>
<p><strong>loss</strong> (<code>torch.FloatTensor</code> of shape <code>(1,)</code>, <em>optional</em>, returned when <code>labels</code> is provided) — Language modeling loss (for next-token prediction).</p>
</li>
<li>
<p><strong>logits</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, config.vocab_size)</code>) — Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).</p>
</li>
<li>
<p><strong>aux_loss</strong> (<code>torch.FloatTensor</code>, <em>optional</em>, returned when <code>labels</code> is provided) — aux_loss for the sparse modules.</p>
</li>
<li>
<p><strong>router_logits</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_router_probs=True</code> and <code>config.add_router_probs=True</code> is passed or when <code>config.output_router_probs=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, sequence_length, num_experts)</code>.</p>
<p>Raw router logtis (post-softmax) that are computed by MoE routers, these terms are used to compute the auxiliary
loss for Mixture of Experts models.</p>
</li>
<li>
<p><strong>past_key_values</strong> (<code>tuple(tuple(torch.FloatTensor))</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) — Tuple of <code>tuple(torch.FloatTensor)</code> of length <code>config.n_layers</code>, with each tuple having 2 tensors of shape
<code>(batch_size, num_heads, sequence_length, embed_size_per_head)</code>)</p>
<p>Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
<code>past_key_values</code> input) to speed up sequential decoding.</p>
</li>
<li>
<p><strong>hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings, if the model has an embedding layer, +
one for the output of each layer) of shape <code>(batch_size, sequence_length, hidden_size)</code>.</p>
<p>Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.</p>
</li>
<li>
<p><strong>attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p>
<p>Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.</p>
</li>
</ul>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>transformers.modeling_outputs.MoeCausalLMOutputWithPast</code> or <code>tuple(torch.FloatTensor)</code></p>
`}}),I=new Xe({props:{$$slots:{default:[yo]},$$scope:{ctx:J}}}),Y=new ge({props:{title:"JetMoeForSequenceClassification",local:"transformers.JetMoeForSequenceClassification",headingTag:"h2"}}),Q=new oe({props:{name:"class transformers.JetMoeForSequenceClassification",anchor:"transformers.JetMoeForSequenceClassification",parameters:[{name:"config",val:""}],parametersDescription:[{anchor:"transformers.JetMoeForSequenceClassification.config",description:`<strong>config</strong> (<a href="/docs/transformers/main/en/model_doc/jetmoe#transformers.JetMoeConfig">JetMoeConfig</a>) &#x2014; Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
configuration. Check out the <a href="/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/jetmoe/modeling_jetmoe.py#L1379"}}),X=new oe({props:{name:"forward",anchor:"transformers.JetMoeForSequenceClassification.forward",parameters:[{name:"input_ids",val:": Optional = None"},{name:"attention_mask",val:": Optional = None"},{name:"position_ids",val:": Optional = None"},{name:"past_key_values",val:": Union = None"},{name:"inputs_embeds",val:": Optional = None"},{name:"labels",val:": Optional = None"},{name:"use_cache",val:": Optional = None"},{name:"output_attentions",val:": Optional = None"},{name:"output_hidden_states",val:": Optional = None"},{name:"return_dict",val:": Optional = None"}],parametersDescription:[{anchor:"transformers.JetMoeForSequenceClassification.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.LongTensor</code> of shape <code>({0})</code>) &#x2014;
Indices of input sequence tokens in the vocabulary.</p>
<p>Indices can be obtained using <code>AutoProcenizer</code>. See <a href="/docs/transformers/main/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/main/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p>
<p><a href="../glossary#input-ids">What are input IDs?</a>`,name:"input_ids"},{anchor:"transformers.JetMoeForSequenceClassification.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.FloatTensor</code> of shape <code>({0})</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>not masked</strong>,</li>
<li>0 for tokens that are <strong>masked</strong>.</li>
</ul>
<p><a href="../glossary#attention-mask">What are attention masks?</a>`,name:"attention_mask"},{anchor:"transformers.JetMoeForSequenceClassification.forward.position_ids",description:`<strong>position_ids</strong> (<code>torch.LongTensor</code> of shape <code>({0})</code>, <em>optional</em>) &#x2014;
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range <code>[0, config.n_positions - 1]</code>.</p>
<p><a href="../glossary#position-ids">What are position IDs?</a>`,name:"position_ids"},{anchor:"transformers.JetMoeForSequenceClassification.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.FloatTensor</code> of shape <code>({0}, hidden_dim)</code>, <em>optional</em>) &#x2014;
Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This
is useful if you want more control over how to convert <em>input_ids</em> indices into associated vectors than the
model&#x2019;s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.JetMoeForSequenceClassification.forward.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.JetMoeForSequenceClassification.forward.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.JetMoeForSequenceClassification.forward.output_router_logits",description:`<strong>output_router_logits</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the logits of all the routers. They are useful for computing the router loss, and
should not be returned during inference.`,name:"output_router_logits"},{anchor:"transformers.JetMoeForSequenceClassification.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return a <a href="/docs/transformers/main/en/main_classes/output#transformers.utils.ModelOutput">ModelOutput</a> instead of a plain tuple.`,name:"return_dict"},{anchor:"transformers.JetMoeForSequenceClassification.forward.cache_position",description:`<strong>cache_position</strong> (<code>torch.LongTensor</code> of shape <code>(sequence_length)</code>, <em>optional</em>) &#x2014;
Indices depicting the position of the input sequence tokens in the sequence. Contrarily to <code>position_ids</code>,
this tensor is not affected by padding. It is used to update the cache in the correct position and to infer
the complete sequence length.`,name:"cache_position"},{anchor:"transformers.JetMoeForSequenceClassification.forward.labels",description:`<strong>labels</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size,)</code>, <em>optional</em>) &#x2014;
Labels for computing the sequence classification/regression loss. Indices should be in <code>[0, ..., config.num_labels - 1]</code>. If <code>config.num_labels == 1</code> a regression loss is computed (Mean-Square loss), If
<code>config.num_labels &gt; 1</code> a classification loss is computed (Cross-Entropy).`,name:"labels"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/jetmoe/modeling_jetmoe.py#L1411"}}),O=new Xe({props:{$$slots:{default:[ko]},$$scope:{ctx:J}}}),K=new $o({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/model_doc/jetmoe.md"}}),{c(){t=d("meta"),_=s(),i=d("p"),w=s(),m(W.$$.fragment),Me=s(),m(H.$$.fragment),be=s(),N=d("p"),N.innerHTML=Ke,ve=s(),E=d("p"),E.innerHTML=eo,$e=s(),m(D.$$.fragment),Te=s(),v=d("div"),m(B.$$.fragment),Pe=s(),te=d("p"),te.innerHTML=oo,Se=s(),ne=d("p"),ne.innerHTML=to,je=s(),re=d("p"),re.innerHTML=no,Ie=s(),m(S.$$.fragment),we=s(),m(U.$$.fragment),ye=s(),$=d("div"),m(A.$$.fragment),Oe=s(),se=d("p"),se.innerHTML=ro,We=s(),ae=d("p"),ae.innerHTML=so,He=s(),x=d("div"),m(Z.$$.fragment),Ne=s(),ie=d("p"),ie.innerHTML=ao,Ee=s(),m(j.$$.fragment),ke=s(),m(G.$$.fragment),Je=s(),F=d("div"),m(R.$$.fragment),De=s(),C=d("div"),m(V.$$.fragment),Be=s(),de=d("p"),de.innerHTML=io,Ue=s(),m(I.$$.fragment),xe=s(),m(Y.$$.fragment),Ce=s(),M=d("div"),m(Q.$$.fragment),Ae=s(),le=d("p"),le.textContent=lo,Ze=s(),ce=d("p"),ce.innerHTML=co,Ge=s(),me=d("p"),me.innerHTML=mo,Re=s(),pe=d("p"),pe.innerHTML=po,Ve=s(),L=d("div"),m(X.$$.fragment),Ye=s(),ue=d("p"),ue.innerHTML=uo,Qe=s(),m(O.$$.fragment),Le=s(),m(K.$$.fragment),Fe=s(),_e=d("p"),this.h()},l(e){const o=Mo("svelte-u9bgzb",document.head);t=l(o,"META",{name:!0,content:!0}),o.forEach(n),_=a(e),i=l(e,"P",{}),q(i).forEach(n),w=a(e),p(W.$$.fragment,e),Me=a(e),p(H.$$.fragment,e),be=a(e),N=l(e,"P",{"data-svelte-h":!0}),b(N)!=="svelte-o95nje"&&(N.innerHTML=Ke),ve=a(e),E=l(e,"P",{"data-svelte-h":!0}),b(E)!=="svelte-8foy02"&&(E.innerHTML=eo),$e=a(e),p(D.$$.fragment,e),Te=a(e),v=l(e,"DIV",{class:!0});var T=q(v);p(B.$$.fragment,T),Pe=a(T),te=l(T,"P",{"data-svelte-h":!0}),b(te)!=="svelte-14idbpz"&&(te.innerHTML=oo),Se=a(T),ne=l(T,"P",{"data-svelte-h":!0}),b(ne)!=="svelte-aqfqqx"&&(ne.innerHTML=to),je=a(T),re=l(T,"P",{"data-svelte-h":!0}),b(re)!=="svelte-o55m63"&&(re.innerHTML=no),Ie=a(T),p(S.$$.fragment,T),T.forEach(n),we=a(e),p(U.$$.fragment,e),ye=a(e),$=l(e,"DIV",{class:!0});var k=q($);p(A.$$.fragment,k),Oe=a(k),se=l(k,"P",{"data-svelte-h":!0}),b(se)!=="svelte-18tbkqx"&&(se.innerHTML=ro),We=a(k),ae=l(k,"P",{"data-svelte-h":!0}),b(ae)!=="svelte-1yw4f7n"&&(ae.innerHTML=so),He=a(k),x=l(k,"DIV",{class:!0});var z=q(x);p(Z.$$.fragment,z),Ne=a(z),ie=l(z,"P",{"data-svelte-h":!0}),b(ie)!=="svelte-nvbk3h"&&(ie.innerHTML=ao),Ee=a(z),p(j.$$.fragment,z),z.forEach(n),k.forEach(n),ke=a(e),p(G.$$.fragment,e),Je=a(e),F=l(e,"DIV",{class:!0});var ee=q(F);p(R.$$.fragment,ee),De=a(ee),C=l(ee,"DIV",{class:!0});var fe=q(C);p(V.$$.fragment,fe),Be=a(fe),de=l(fe,"P",{"data-svelte-h":!0}),b(de)!=="svelte-jct3wh"&&(de.innerHTML=io),Ue=a(fe),p(I.$$.fragment,fe),fe.forEach(n),ee.forEach(n),xe=a(e),p(Y.$$.fragment,e),Ce=a(e),M=l(e,"DIV",{class:!0});var y=q(M);p(Q.$$.fragment,y),Ae=a(y),le=l(y,"P",{"data-svelte-h":!0}),b(le)!=="svelte-1pmdz94"&&(le.textContent=lo),Ze=a(y),ce=l(y,"P",{"data-svelte-h":!0}),b(ce)!=="svelte-qp2lg8"&&(ce.innerHTML=co),Ge=a(y),me=l(y,"P",{"data-svelte-h":!0}),b(me)!=="svelte-10ugs3m"&&(me.innerHTML=mo),Re=a(y),pe=l(y,"P",{"data-svelte-h":!0}),b(pe)!=="svelte-68lg8f"&&(pe.innerHTML=po),Ve=a(y),L=l(y,"DIV",{class:!0});var he=q(L);p(X.$$.fragment,he),Ye=a(he),ue=l(he,"P",{"data-svelte-h":!0}),b(ue)!=="svelte-l80est"&&(ue.innerHTML=uo),Qe=a(he),p(O.$$.fragment,he),he.forEach(n),y.forEach(n),Le=a(e),p(K.$$.fragment,e),Fe=a(e),_e=l(e,"P",{}),q(_e).forEach(n),this.h()},h(){P(t,"name","hf:doc:metadata"),P(t,"content",xo),P(v,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),P(x,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),P($,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),P(C,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),P(F,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),P(L,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),P(M,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,o){r(document.head,t),c(e,_,o),c(e,i,o),c(e,w,o),u(W,e,o),c(e,Me,o),u(H,e,o),c(e,be,o),c(e,N,o),c(e,ve,o),c(e,E,o),c(e,$e,o),u(D,e,o),c(e,Te,o),c(e,v,o),u(B,v,null),r(v,Pe),r(v,te),r(v,Se),r(v,ne),r(v,je),r(v,re),r(v,Ie),u(S,v,null),c(e,we,o),u(U,e,o),c(e,ye,o),c(e,$,o),u(A,$,null),r($,Oe),r($,se),r($,We),r($,ae),r($,He),r($,x),u(Z,x,null),r(x,Ne),r(x,ie),r(x,Ee),u(j,x,null),c(e,ke,o),u(G,e,o),c(e,Je,o),c(e,F,o),u(R,F,null),r(F,De),r(F,C),u(V,C,null),r(C,Be),r(C,de),r(C,Ue),u(I,C,null),c(e,xe,o),u(Y,e,o),c(e,Ce,o),c(e,M,o),u(Q,M,null),r(M,Ae),r(M,le),r(M,Ze),r(M,ce),r(M,Ge),r(M,me),r(M,Re),r(M,pe),r(M,Ve),r(M,L),u(X,L,null),r(L,Ye),r(L,ue),r(L,Qe),u(O,L,null),c(e,Le,o),u(K,e,o),c(e,Fe,o),c(e,_e,o),ze=!0},p(e,[o]){const T={};o&2&&(T.$$scope={dirty:o,ctx:e}),S.$set(T);const k={};o&2&&(k.$$scope={dirty:o,ctx:e}),j.$set(k);const z={};o&2&&(z.$$scope={dirty:o,ctx:e}),I.$set(z);const ee={};o&2&&(ee.$$scope={dirty:o,ctx:e}),O.$set(ee)},i(e){ze||(f(W.$$.fragment,e),f(H.$$.fragment,e),f(D.$$.fragment,e),f(B.$$.fragment,e),f(S.$$.fragment,e),f(U.$$.fragment,e),f(A.$$.fragment,e),f(Z.$$.fragment,e),f(j.$$.fragment,e),f(G.$$.fragment,e),f(R.$$.fragment,e),f(V.$$.fragment,e),f(I.$$.fragment,e),f(Y.$$.fragment,e),f(Q.$$.fragment,e),f(X.$$.fragment,e),f(O.$$.fragment,e),f(K.$$.fragment,e),ze=!0)},o(e){h(W.$$.fragment,e),h(H.$$.fragment,e),h(D.$$.fragment,e),h(B.$$.fragment,e),h(S.$$.fragment,e),h(U.$$.fragment,e),h(A.$$.fragment,e),h(Z.$$.fragment,e),h(j.$$.fragment,e),h(G.$$.fragment,e),h(R.$$.fragment,e),h(V.$$.fragment,e),h(I.$$.fragment,e),h(Y.$$.fragment,e),h(Q.$$.fragment,e),h(X.$$.fragment,e),h(O.$$.fragment,e),h(K.$$.fragment,e),ze=!1},d(e){e&&(n(_),n(i),n(w),n(Me),n(be),n(N),n(ve),n(E),n($e),n(Te),n(v),n(we),n(ye),n($),n(ke),n(Je),n(F),n(xe),n(Ce),n(M),n(Le),n(Fe),n(_e)),n(t),g(W,e),g(H,e),g(D,e),g(B),g(S),g(U,e),g(A),g(Z),g(j),g(G,e),g(R),g(V),g(I),g(Y,e),g(Q),g(X),g(O),g(K,e)}}}const xo='{"title":"JetMoe","local":"jetmoe","sections":[{"title":"Overview","local":"overview","sections":[],"depth":2},{"title":"JetMoeConfig","local":"transformers.JetMoeConfig","sections":[],"depth":2},{"title":"JetMoeModel","local":"transformers.JetMoeModel","sections":[],"depth":2},{"title":"JetMoeForCausalLM","local":"transformers.JetMoeForCausalLM","sections":[],"depth":2},{"title":"JetMoeForSequenceClassification","local":"transformers.JetMoeForSequenceClassification","sections":[],"depth":2}],"depth":1}';function Co(J){return ho(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Io extends go{constructor(t){super(),_o(this,t,Co,Jo,fo,{})}}export{Io as component};
