import{s as _e,o as be,n as ke}from"../chunks/scheduler.9bc65507.js";import{S as $e,i as ve,g,s as r,r as h,A as ye,h as f,f as n,c as i,j as ae,u as w,x as v,k as oe,y as z,a,v as k,d as T,t as _,w as b}from"../chunks/index.707bf1b6.js";import{T as Me}from"../chunks/Tip.c2ecdbf4.js";import{D as we}from"../chunks/Docstring.17db21ae.js";import{C as Te}from"../chunks/CodeBlock.54a9f38d.js";import{E as ze}from"../chunks/ExampleCodeBlock.4f515aa9.js";import{H as N}from"../chunks/Heading.342b1fa6.js";function xe(Z){let o,$=`The implementation uses the <code>GPT2Model</code> coupled with our <code>GPTSw3Tokenizer</code>. Refer to <a href="gpt2">GPT2Model documentation</a>
for API reference and examples.`,p,l,c="Note that sentencepiece is required to use our tokenizer and can be installed with <code>pip install transformers[sentencepiece]</code> or <code>pip install sentencepiece</code>";return{c(){o=g("p"),o.innerHTML=$,p=r(),l=g("p"),l.innerHTML=c},l(s){o=f(s,"P",{"data-svelte-h":!0}),v(o)!=="svelte-13njngq"&&(o.innerHTML=$),p=i(s),l=f(s,"P",{"data-svelte-h":!0}),v(l)!=="svelte-1oicrkv"&&(l.innerHTML=c)},m(s,m){a(s,o,m),a(s,p,m),a(s,l,m)},p:ke,d(s){s&&(n(o),n(p),n(l))}}}function Se(Z){let o,$="Example usage:",p,l,c;return l=new Te({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEdQVFN3M1Rva2VuaXplciUwQSUwQXRva2VuaXplciUyMCUzRCUyMEdQVFN3M1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyQUktU3dlZGVuJTJGZ3B0LXN3My0xMjZtJTIyKSUwQXRva2VuaXplciglMjJTdmVuc2thJTIwJUMzJUE0ciUyMGt1bCElMjIpJTVCJTIyaW5wdXRfaWRzJTIyJTVE",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> GPTSw3Tokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = GPTSw3Tokenizer.from_pretrained(<span class="hljs-string">&quot;AI-Sweden/gpt-sw3-126m&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer(<span class="hljs-string">&quot;Svenska är kul!&quot;</span>)[<span class="hljs-string">&quot;input_ids&quot;</span>]
[<span class="hljs-number">1814</span>, <span class="hljs-number">377</span>, <span class="hljs-number">3617</span>, <span class="hljs-number">63504</span>]`,wrap:!1}}),{c(){o=g("p"),o.textContent=$,p=r(),h(l.$$.fragment)},l(s){o=f(s,"P",{"data-svelte-h":!0}),v(o)!=="svelte-1ni337v"&&(o.textContent=$),p=i(s),w(l.$$.fragment,s)},m(s,m){a(s,o,m),a(s,p,m),k(l,s,m),c=!0},p:ke,i(s){c||(T(l.$$.fragment,s),c=!0)},o(s){_(l.$$.fragment,s),c=!1},d(s){s&&(n(o),n(p)),b(l,s)}}}function Pe(Z){let o,$,p,l,c,s,m,q,x,ce=`The GPT-Sw3 model was first proposed in
<a href="http://www.lrec-conf.org/proceedings/lrec2022/pdf/2022.lrec-1.376.pdf" rel="nofollow">Lessons Learned from GPT-SW3: Building the First Large-Scale Generative Language Model for Swedish</a>
by Ariel Ekgren, Amaru Cuba Gyllensten, Evangelia Gogoulou, Alice Heiman, Severine Verlinden, Joey Öhman,
Fredrik Carlsson, Magnus Sahlgren.`,F,S,me="Since that first paper the authors have extended their work and trained new models on their new 1.2TB corpora named The Nordic Pile.",X,P,de=`GPT-Sw3 is a collection of large decoder-only pretrained transformer language models that were developed by AI Sweden
in collaboration with RISE and the WASP WARA for Media and Language. GPT-Sw3 has been trained on a dataset containing
320B tokens in Swedish, Norwegian, Danish, Icelandic, English, and programming code. The model was pretrained using a
causal language modeling (CLM) objective utilizing the NeMo Megatron GPT implementation.`,A,G,ge='This model was contributed by <a href="https://huggingface.co/AI-Sweden" rel="nofollow">AI Sweden</a>.',Q,U,D,C,Y,j,K,J,fe='<li><a href="../tasks/sequence_classification">Text classification task guide</a></li> <li><a href="../tasks/token_classification">Token classification task guide</a></li> <li><a href="../tasks/language_modeling">Causal language modeling task guide</a></li>',O,y,ee,R,te,d,I,re,E,ue='Construct an GPTSw3 tokenizer. Based on <a href="https://github.com/google/sentencepiece" rel="nofollow">SentencePiece</a>.',ie,W,he=`This tokenizer inherits from <a href="/docs/transformers/main/en/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> which contains most of the main methods. Users should refer to
this superclass for more information regarding those methods.`,le,M,pe,L,V,ne,B,se;return c=new N({props:{title:"GPT-Sw3",local:"gpt-sw3",headingTag:"h1"}}),m=new N({props:{title:"Overview",local:"overview",headingTag:"h2"}}),U=new N({props:{title:"Usage example",local:"usage-example",headingTag:"h2"}}),C=new Te({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMkFJLVN3ZWRlbiUyRmdwdC1zdzMtMzU2bSUyMiklMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJBSS1Td2VkZW4lMkZncHQtc3czLTM1Nm0lMjIpJTBBJTBBaW5wdXRfaWRzJTIwJTNEJTIwdG9rZW5pemVyKCUyMlRyJUMzJUE0ZCUyMCVDMyVBNHIlMjBmaW5hJTIwZiVDMyVCNnIlMjBhdHQlMjIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSU1QiUyMmlucHV0X2lkcyUyMiU1RCUwQSUwQWdlbmVyYXRlZF90b2tlbl9pZHMlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZShpbnB1dHMlM0RpbnB1dF9pZHMlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDEwJTJDJTIwZG9fc2FtcGxlJTNEVHJ1ZSklNUIwJTVEJTBBJTBBcHJpbnQodG9rZW5pemVyLmRlY29kZShnZW5lcmF0ZWRfdG9rZW5faWRzKSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;AI-Sweden/gpt-sw3-356m&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;AI-Sweden/gpt-sw3-356m&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>input_ids = tokenizer(<span class="hljs-string">&quot;Träd är fina för att&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)[<span class="hljs-string">&quot;input_ids&quot;</span>]

<span class="hljs-meta">&gt;&gt;&gt; </span>generated_token_ids = model.generate(inputs=input_ids, max_new_tokens=<span class="hljs-number">10</span>, do_sample=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(tokenizer.decode(generated_token_ids))
Träd är fina för att de är färgstarka. Men ibland är det fint`,wrap:!1}}),j=new N({props:{title:"Resources",local:"resources",headingTag:"h2"}}),y=new Me({props:{$$slots:{default:[xe]},$$scope:{ctx:Z}}}),R=new N({props:{title:"GPTSw3Tokenizer",local:"transformers.GPTSw3Tokenizer",headingTag:"h2"}}),I=new we({props:{name:"class transformers.GPTSw3Tokenizer",anchor:"transformers.GPTSw3Tokenizer",parameters:[{name:"vocab_file",val:""},{name:"do_lower_case",val:" = False"},{name:"remove_space",val:" = False"},{name:"keep_accents",val:" = False"},{name:"pad_token",val:" = None"},{name:"unk_token",val:" = None"},{name:"eos_token",val:" = None"},{name:"bos_token",val:" = None"},{name:"sp_model_kwargs",val:": typing.Union[typing.Dict[str, typing.Any], NoneType] = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.GPTSw3Tokenizer.vocab_file",description:`<strong>vocab_file</strong> (<code>str</code>) &#x2014;
<a href="https://github.com/google/sentencepiece" rel="nofollow">SentencePiece</a> file (generally has a <em>.spm</em> extension) that
contains the vocabulary necessary to instantiate a tokenizer.`,name:"vocab_file"},{anchor:"transformers.GPTSw3Tokenizer.do_lower_case",description:`<strong>do_lower_case</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to lowercase the input when tokenizing.`,name:"do_lower_case"},{anchor:"transformers.GPTSw3Tokenizer.remove_space",description:`<strong>remove_space</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to strip the text when tokenizing (removing excess spaces before and after the string).`,name:"remove_space"},{anchor:"transformers.GPTSw3Tokenizer.keep_accents",description:`<strong>keep_accents</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to keep accents when tokenizing.`,name:"keep_accents"},{anchor:"transformers.GPTSw3Tokenizer.pad_token",description:`<strong>pad_token</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The token used for padding, for example when batching sequences of different lengths. If not provided, will
default to &#x2019;<pad>&#x2019; or &#x2019;<unk>&#x2019; depending on model size.</unk></pad>`,name:"pad_token"},{anchor:"transformers.GPTSw3Tokenizer.unk_token",description:`<strong>unk_token</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
token instead. If not provided, will default to &#x2019;<unk>&#x2018;.</unk>`,name:"unk_token"},{anchor:"transformers.GPTSw3Tokenizer.eos_token",description:`<strong>eos_token</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The end of sequence token seen during pretraining. If not provided, will default to &#x2019;&lt;|endoftext|&gt;&#x2019;`,name:"eos_token"},{anchor:"transformers.GPTSw3Tokenizer.bos_token",description:`<strong>bos_token</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The beginning of sequence token that can be used for downstream task, was not seen during pretraining. If
not provided, will default to &#x2019;<s>&#x2019; or &#x2019;&lt;|endoftext|&gt;&#x2019;, depending on model size.</s>`,name:"bos_token"},{anchor:"transformers.GPTSw3Tokenizer.sp_model_kwargs",description:`<strong>sp_model_kwargs</strong> (<code>dict</code>, <em>optional</em>) &#x2014;
Will be passed to the <code>SentencePieceProcessor.__init__()</code> method. The <a href="https://github.com/google/sentencepiece/tree/master/python" rel="nofollow">Python wrapper for
SentencePiece</a> can be used, among other things,
to set:</p>
<ul>
<li>
<p><code>enable_sampling</code>: Enable subword regularization.</p>
</li>
<li>
<p><code>nbest_size</code>: Sampling parameters for unigram. Invalid for BPE-Dropout.</p>
<ul>
<li><code>nbest_size = {0,1}</code>: No sampling is performed.</li>
<li><code>nbest_size &gt; 1</code>: samples from the nbest_size results.</li>
<li><code>nbest_size &lt; 0</code>: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)
using forward-filtering-and-backward-sampling algorithm.</li>
</ul>
</li>
<li>
<p><code>alpha</code>: Smoothing parameter for unigram sampling, and dropout probability of merge operations for
BPE-dropout.</p>
</li>
</ul>`,name:"sp_model_kwargs"},{anchor:"transformers.GPTSw3Tokenizer.sp_model",description:`<strong>sp_model</strong> (<code>SentencePieceProcessor</code>) &#x2014;
The <em>SentencePiece</em> processor that is used for every conversion (string, tokens and IDs).`,name:"sp_model"},{anchor:"transformers.GPTSw3Tokenizer.whitespaces",description:`<strong>whitespaces</strong> (<code>set</code>) &#x2014;
The whitespaces that are replaced in the whitespace normalization in preprocessing.`,name:"whitespaces"},{anchor:"transformers.GPTSw3Tokenizer.non_printing_characters_re",description:`<strong>non_printing_characters_re</strong> (<code>Pattern</code>) &#x2014;
The compiled regular expression to remove non-printing characters in preprocessing.`,name:"non_printing_characters_re"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/gpt_sw3/tokenization_gpt_sw3.py#L41"}}),M=new ze({props:{anchor:"transformers.GPTSw3Tokenizer.example",$$slots:{default:[Se]},$$scope:{ctx:Z}}}),V=new we({props:{name:"save_vocabulary",anchor:"transformers.GPTSw3Tokenizer.save_vocabulary",parameters:[{name:"save_directory",val:": str"},{name:"filename_prefix",val:": typing.Optional[str] = None"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/gpt_sw3/tokenization_gpt_sw3.py#L254"}}),{c(){o=g("meta"),$=r(),p=g("p"),l=r(),h(c.$$.fragment),s=r(),h(m.$$.fragment),q=r(),x=g("p"),x.innerHTML=ce,F=r(),S=g("p"),S.textContent=me,X=r(),P=g("p"),P.textContent=de,A=r(),G=g("p"),G.innerHTML=ge,Q=r(),h(U.$$.fragment),D=r(),h(C.$$.fragment),Y=r(),h(j.$$.fragment),K=r(),J=g("ul"),J.innerHTML=fe,O=r(),h(y.$$.fragment),ee=r(),h(R.$$.fragment),te=r(),d=g("div"),h(I.$$.fragment),re=r(),E=g("p"),E.innerHTML=ue,ie=r(),W=g("p"),W.innerHTML=he,le=r(),h(M.$$.fragment),pe=r(),L=g("div"),h(V.$$.fragment),ne=r(),B=g("p"),this.h()},l(e){const t=ye("svelte-u9bgzb",document.head);o=f(t,"META",{name:!0,content:!0}),t.forEach(n),$=i(e),p=f(e,"P",{}),ae(p).forEach(n),l=i(e),w(c.$$.fragment,e),s=i(e),w(m.$$.fragment,e),q=i(e),x=f(e,"P",{"data-svelte-h":!0}),v(x)!=="svelte-1qucz45"&&(x.innerHTML=ce),F=i(e),S=f(e,"P",{"data-svelte-h":!0}),v(S)!=="svelte-1uddber"&&(S.textContent=me),X=i(e),P=f(e,"P",{"data-svelte-h":!0}),v(P)!=="svelte-19zkpwq"&&(P.textContent=de),A=i(e),G=f(e,"P",{"data-svelte-h":!0}),v(G)!=="svelte-1u6kw5m"&&(G.innerHTML=ge),Q=i(e),w(U.$$.fragment,e),D=i(e),w(C.$$.fragment,e),Y=i(e),w(j.$$.fragment,e),K=i(e),J=f(e,"UL",{"data-svelte-h":!0}),v(J)!=="svelte-v6nfyh"&&(J.innerHTML=fe),O=i(e),w(y.$$.fragment,e),ee=i(e),w(R.$$.fragment,e),te=i(e),d=f(e,"DIV",{class:!0});var u=ae(d);w(I.$$.fragment,u),re=i(u),E=f(u,"P",{"data-svelte-h":!0}),v(E)!=="svelte-19f98fg"&&(E.innerHTML=ue),ie=i(u),W=f(u,"P",{"data-svelte-h":!0}),v(W)!=="svelte-1c3t5ty"&&(W.innerHTML=he),le=i(u),w(M.$$.fragment,u),pe=i(u),L=f(u,"DIV",{class:!0});var H=ae(L);w(V.$$.fragment,H),H.forEach(n),u.forEach(n),ne=i(e),B=f(e,"P",{}),ae(B).forEach(n),this.h()},h(){oe(o,"name","hf:doc:metadata"),oe(o,"content",Ge),oe(L,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),oe(d,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,t){z(document.head,o),a(e,$,t),a(e,p,t),a(e,l,t),k(c,e,t),a(e,s,t),k(m,e,t),a(e,q,t),a(e,x,t),a(e,F,t),a(e,S,t),a(e,X,t),a(e,P,t),a(e,A,t),a(e,G,t),a(e,Q,t),k(U,e,t),a(e,D,t),k(C,e,t),a(e,Y,t),k(j,e,t),a(e,K,t),a(e,J,t),a(e,O,t),k(y,e,t),a(e,ee,t),k(R,e,t),a(e,te,t),a(e,d,t),k(I,d,null),z(d,re),z(d,E),z(d,ie),z(d,W),z(d,le),k(M,d,null),z(d,pe),z(d,L),k(V,L,null),a(e,ne,t),a(e,B,t),se=!0},p(e,[t]){const u={};t&2&&(u.$$scope={dirty:t,ctx:e}),y.$set(u);const H={};t&2&&(H.$$scope={dirty:t,ctx:e}),M.$set(H)},i(e){se||(T(c.$$.fragment,e),T(m.$$.fragment,e),T(U.$$.fragment,e),T(C.$$.fragment,e),T(j.$$.fragment,e),T(y.$$.fragment,e),T(R.$$.fragment,e),T(I.$$.fragment,e),T(M.$$.fragment,e),T(V.$$.fragment,e),se=!0)},o(e){_(c.$$.fragment,e),_(m.$$.fragment,e),_(U.$$.fragment,e),_(C.$$.fragment,e),_(j.$$.fragment,e),_(y.$$.fragment,e),_(R.$$.fragment,e),_(I.$$.fragment,e),_(M.$$.fragment,e),_(V.$$.fragment,e),se=!1},d(e){e&&(n($),n(p),n(l),n(s),n(q),n(x),n(F),n(S),n(X),n(P),n(A),n(G),n(Q),n(D),n(Y),n(K),n(J),n(O),n(ee),n(te),n(d),n(ne),n(B)),n(o),b(c,e),b(m,e),b(U,e),b(C,e),b(j,e),b(y,e),b(R,e),b(I),b(M),b(V)}}}const Ge='{"title":"GPT-Sw3","local":"gpt-sw3","sections":[{"title":"Overview","local":"overview","sections":[],"depth":2},{"title":"Usage example","local":"usage-example","sections":[],"depth":2},{"title":"Resources","local":"resources","sections":[],"depth":2},{"title":"GPTSw3Tokenizer","local":"transformers.GPTSw3Tokenizer","sections":[],"depth":2}],"depth":1}';function Ue(Z){return be(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ee extends $e{constructor(o){super(),ve(this,o,Ue,Pe,_e,{})}}export{Ee as component};
