import{S as qn,i as Pn,s as Ln,e as r,k as c,w as b,t as l,M as Dn,c as i,d as t,m as d,a as p,x as g,h as n,b as h,N as P,G as s,g as a,y as k,q as T,o as M,B as $,v as On,L as Kn}from"../../chunks/vendor-hf-doc-builder.js";import{T as Ns}from"../../chunks/Tip-hf-doc-builder.js";import{I as xa}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as E}from"../../chunks/CodeBlock-hf-doc-builder.js";function ep(C){let u,w;return{c(){u=r("p"),w=l(`Even though the fine-tuned models start out using the same set of weights from the shared pre-trained model, the
final versions are all quite different in the end. You can\u2019t take a fine-tuned ASR model and swap out the pre-nets and
post-net to get a working TTS model, for example. SpeechT5 is flexible, but not that flexible ;)`)},l(m){u=i(m,"P",{});var f=p(u);w=n(f,`Even though the fine-tuned models start out using the same set of weights from the shared pre-trained model, the
final versions are all quite different in the end. You can\u2019t take a fine-tuned ASR model and swap out the pre-nets and
post-net to get a working TTS model, for example. SpeechT5 is flexible, but not that flexible ;)`),f.forEach(t)},m(m,f){a(m,u,f),s(u,w)},d(m){m&&t(u)}}}function tp(C){let u,w,m,f,J,_,S,I,B,x,U,j,N,v;return{c(){u=r("p"),w=l(`Speaker embeddings is a method of representing a speaker\u2019s identity in a compact way, as a vector of
fixed size, regardless of the length of the utterance. These embeddings capture essential information about a speaker\u2019s
voice, accent, intonation, and other unique characteristics that distinguish one speaker from another. Such embeddings can
be used for speaker verification, speaker diarization, speaker identification, and more.
The most common techniques for generating speaker embeddings include:`),m=c(),f=r("ul"),J=r("li"),_=l("I-Vectors (identity vectors): I-Vectors are based on a Gaussian mixture model (GMM). They represent speakers as low-dimensional fixed-length vectors derived from the statistics of a speaker-specific GMM, and are obtained in unsupervised manner."),S=c(),I=r("li"),B=l("X-Vectors: X-Vectors are derived using deep neural networks (DNNs) and capture frame-level speaker information by incorporating temporal context."),x=c(),U=r("p"),j=r("a"),N=l("X-Vectors"),v=l(` are a state-of-the-art method that shows superior performance
on evaluation datasets compared to I-Vectors. The deep neural network is used to obtain X-Vectors: it trains to discriminate
between speakers, and maps variable-length utterances to fixed-dimensional embeddings. You can also load an X-Vector speaker embedding that has been computed ahead of time, which will encapsulate the speaking characteristics of a particular speaker.`),this.h()},l(y){u=i(y,"P",{});var G=p(u);w=n(G,`Speaker embeddings is a method of representing a speaker\u2019s identity in a compact way, as a vector of
fixed size, regardless of the length of the utterance. These embeddings capture essential information about a speaker\u2019s
voice, accent, intonation, and other unique characteristics that distinguish one speaker from another. Such embeddings can
be used for speaker verification, speaker diarization, speaker identification, and more.
The most common techniques for generating speaker embeddings include:`),G.forEach(t),m=d(y),f=i(y,"UL",{});var Z=p(f);J=i(Z,"LI",{});var R=p(J);_=n(R,"I-Vectors (identity vectors): I-Vectors are based on a Gaussian mixture model (GMM). They represent speakers as low-dimensional fixed-length vectors derived from the statistics of a speaker-specific GMM, and are obtained in unsupervised manner."),R.forEach(t),S=d(Z),I=i(Z,"LI",{});var z=p(I);B=n(z,"X-Vectors: X-Vectors are derived using deep neural networks (DNNs) and capture frame-level speaker information by incorporating temporal context."),z.forEach(t),Z.forEach(t),x=d(y),U=i(y,"P",{});var Te=p(U);j=i(Te,"A",{href:!0,rel:!0});var Me=p(j);N=n(Me,"X-Vectors"),Me.forEach(t),v=n(Te,` are a state-of-the-art method that shows superior performance
on evaluation datasets compared to I-Vectors. The deep neural network is used to obtain X-Vectors: it trains to discriminate
between speakers, and maps variable-length utterances to fixed-dimensional embeddings. You can also load an X-Vector speaker embedding that has been computed ahead of time, which will encapsulate the speaking characteristics of a particular speaker.`),Te.forEach(t),this.h()},h(){h(j,"href","https://www.danielpovey.com/files/2018_icassp_xvectors.pdf"),h(j,"rel","nofollow")},m(y,G){a(y,u,G),s(u,w),a(y,m,G),a(y,f,G),s(f,J),s(J,_),s(f,S),s(f,I),s(I,B),a(y,x,G),a(y,U,G),s(U,j),s(j,N),s(U,v)},d(y){y&&t(u),y&&t(m),y&&t(f),y&&t(x),y&&t(U)}}}function sp(C){let u,w,m,f,J,_,S,I,B,x,U,j,N;return{c(){u=r("p"),w=r("a"),m=l("HiFi-GAN"),f=l(` is a state-of-the-art generative adversarial network (GAN) designed
for high-fidelity speech synthesis. It is capable of generating high-quality and realistic audio waveforms from spectrogram inputs.`),J=c(),_=r("p"),S=l(`On a high level, HiFi-GAN consists of one generator and two discriminators. The generator is a fully convolutional
neural network that takes a mel-spectrogram as input and learns to produce raw audio waveforms. The discriminators\u2019
role is to distinguish between real and generated audio. The two discriminators focus on different aspects of the audio.`),I=c(),B=r("p"),x=l("HiFi-GAN is trained on a large dataset of high-quality audio recordings. It uses a so-called "),U=r("em"),j=l("adversarial training"),N=l(`,
where the generator and discriminator networks compete against each other. Initially, the generator produces low-quality
audio, and the discriminator can easily differentiate it from real audio. As training progresses, the generator improves
its output, aiming to fool the discriminator. The discriminator, in turn, becomes more accurate in distinguishing real
and generated audio. This adversarial feedback loop helps both networks improve over time. Ultimately, HiFi-GAN learns to
generate high-fidelity audio that closely resembles the characteristics of the training data.`),this.h()},l(v){u=i(v,"P",{});var y=p(u);w=i(y,"A",{href:!0,rel:!0});var G=p(w);m=n(G,"HiFi-GAN"),G.forEach(t),f=n(y,` is a state-of-the-art generative adversarial network (GAN) designed
for high-fidelity speech synthesis. It is capable of generating high-quality and realistic audio waveforms from spectrogram inputs.`),y.forEach(t),J=d(v),_=i(v,"P",{});var Z=p(_);S=n(Z,`On a high level, HiFi-GAN consists of one generator and two discriminators. The generator is a fully convolutional
neural network that takes a mel-spectrogram as input and learns to produce raw audio waveforms. The discriminators\u2019
role is to distinguish between real and generated audio. The two discriminators focus on different aspects of the audio.`),Z.forEach(t),I=d(v),B=i(v,"P",{});var R=p(B);x=n(R,"HiFi-GAN is trained on a large dataset of high-quality audio recordings. It uses a so-called "),U=i(R,"EM",{});var z=p(U);j=n(z,"adversarial training"),z.forEach(t),N=n(R,`,
where the generator and discriminator networks compete against each other. Initially, the generator produces low-quality
audio, and the discriminator can easily differentiate it from real audio. As training progresses, the generator improves
its output, aiming to fool the discriminator. The discriminator, in turn, becomes more accurate in distinguishing real
and generated audio. This adversarial feedback loop helps both networks improve over time. Ultimately, HiFi-GAN learns to
generate high-fidelity audio that closely resembles the characteristics of the training data.`),R.forEach(t),this.h()},h(){h(w,"href","https://arxiv.org/pdf/2010.05646v2.pdf"),h(w,"rel","nofollow")},m(v,y){a(v,u,y),s(u,w),s(w,m),s(u,f),a(v,J,y),a(v,_,y),s(_,S),a(v,I,y),a(v,B,y),s(B,x),s(B,U),s(U,j),s(B,N)},d(v){v&&t(u),v&&t(J),v&&t(_),v&&t(I),v&&t(B)}}}function op(C){let u,w,m,f,J;return{c(){u=r("p"),w=l("Bark, like other \u{1F917} Transformers models, can be optimized in just a few lines of code regarding speed and memory impact. To find out how, click on "),m=r("a"),f=l("this colab demonstration notebook"),J=l("."),this.h()},l(_){u=i(_,"P",{});var S=p(u);w=n(S,"Bark, like other \u{1F917} Transformers models, can be optimized in just a few lines of code regarding speed and memory impact. To find out how, click on "),m=i(S,"A",{href:!0,rel:!0});var I=p(m);f=n(I,"this colab demonstration notebook"),I.forEach(t),J=n(S,"."),S.forEach(t),this.h()},h(){h(m,"href","https://colab.research.google.com/github/ylacombe/notebooks/blob/main/Benchmark_Bark_HuggingFace.ipynb"),h(m,"rel","nofollow")},m(_,S){a(_,u,S),s(u,w),s(u,m),s(m,f),s(u,J)},d(_){_&&t(u)}}}function ap(C){let u,w,m;return w=new E({props:{code:"cGlwJTIwaW5zdGFsbCUyMGdpdCUyQmh0dHBzJTNBJTJGJTJGZ2l0aHViLmNvbSUyRmhvbGxhbmNlJTJGdHJhbnNmb3JtZXJzLmdpdCU0MDY5MDBlOGJhNjUzMjE2MmE4NjEzZDIyNzBlYzIyODZjM2Y1OGY1N2I=",highlighted:"pip install git+https://github.com/hollance/transformers.git@6900e8ba6532162a8613d2270ec2286c3f58f57b"}}),{c(){u=l(`MMS model has been added to \u{1F917} Transformers very recently, so the API may still slightly change. At the time of this Unit's 
release, the MMS for TTS is scheduled to be merged in Transformers \`main\` in a couple of days. 
Once it's merged, you will have to install the library from source. Meanwhile, if you're eager to try the following inference 
example, you can install the model from the PR's branch:

	`),b(w.$$.fragment)},l(f){u=n(f,`MMS model has been added to \u{1F917} Transformers very recently, so the API may still slightly change. At the time of this Unit's 
release, the MMS for TTS is scheduled to be merged in Transformers \`main\` in a couple of days. 
Once it's merged, you will have to install the library from source. Meanwhile, if you're eager to try the following inference 
example, you can install the model from the PR's branch:

	`),g(w.$$.fragment,f)},m(f,J){a(f,u,J),k(w,f,J),m=!0},p:Kn,i(f){m||(T(w.$$.fragment,f),m=!0)},o(f){M(w.$$.fragment,f),m=!1},d(f){f&&t(u),$(w,f)}}}function rp(C){let u,w,m,f,J,_,S,I,B,x,U,j,N,v,y,G,Z,R,z,Te,Me,$e,_e,Ca,Na,Ws,it,Wa,Rs,X,Ra,ns,za,Xa,ps,Fa,Aa,zs,lt,Ha,Xs,Je,nt,pl,Fs,pt,Ya,As,ct,Qa,Hs,dt,qa,Ys,O,Qs,ht,Pa,qs,F,cs,La,Da,ds,Oa,Ka,hs,er,Ps,ut,tr,Ls,Ee,ft,cl,Ds,K,sr,mt,or,ar,Os,yt,rr,Ks,wt,ir,eo,Ue,to,vt,lr,so,Ie,oo,bt,nr,ao,ee,ro,A,pr,Se,cr,dr,Ge,hr,ur,io,Be,lo,gt,fr,no,kt,mr,po,je,co,Tt,yr,ho,Mt,wr,uo,te,fo,$t,vr,mo,Ze,yo,_t,br,wo,Ve,vo,Jt,gr,bo,xe,go,Et,kr,ko,Ut,Tr,To,H,dl,Mo,L,se,us,Ce,Mr,fs,$r,$o,oe,_r,Ne,Jr,Er,_o,ae,Ur,We,ms,Ir,Sr,Jo,re,Gr,ys,Br,jr,Eo,It,Zr,Uo,W,St,ws,Vr,xr,Cr,ie,vs,Nr,Wr,bs,Rr,zr,Xr,Gt,gs,Fr,Ar,Hr,Re,Yr,ks,Qr,qr,Io,Bt,Pr,So,jt,Lr,Go,Zt,Dr,Bo,Vt,Or,jo,le,Ts,Kr,ei,Ms,ti,Zo,ze,Vo,ne,si,Xe,oi,ai,xo,Fe,Co,pe,xt,hl,ri,No,ce,ii,Ae,li,ni,Wo,He,Ro,de,Ct,ul,pi,zo,V,ci,$s,di,hi,_s,ui,fi,Js,mi,yi,Es,wi,vi,Xo,Ye,Fo,he,Nt,fl,bi,Ao,Wt,gi,Ho,Qe,Yo,ue,Rt,ml,ki,Qo,zt,Ti,qo,Xt,Mi,Po,qe,Lo,Ft,$i,Do,At,_i,Oo,Pe,Ko,fe,Ht,yl,Ji,ea,Yt,Ei,ta,Le,sa,me,Qt,wl,Ui,oa,qt,Ii,aa,De,ra,ye,Pt,vl,Si,ia,we,la,D,ve,Us,Oe,Gi,Is,Bi,na,Lt,ji,pa,be,Zi,Ke,Vi,xi,ca,Dt,Ci,da,ge,ha,Ot,Ni,ua,et,fa,Y,Wi,Ss,Ri,zi,Gs,Xi,Fi,ma,Kt,Ai,ya,tt,wa,es,Hi,va,st,ba,ts,Yi,ga,ot,ka,Q,Qi,Bs,qi,Pi,at,Li,Di,Ta,ss,Oi,Ma;return _=new xa({}),Z=new xa({}),O=new Ns({props:{$$slots:{default:[ep]},$$scope:{ctx:C}}}),Ue=new E({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFNwZWVjaFQ1UHJvY2Vzc29yJTJDJTIwU3BlZWNoVDVGb3JUZXh0VG9TcGVlY2glMEElMEFwcm9jZXNzb3IlMjAlM0QlMjBTcGVlY2hUNVByb2Nlc3Nvci5mcm9tX3ByZXRyYWluZWQoJTIybWljcm9zb2Z0JTJGc3BlZWNodDVfdHRzJTIyKSUwQW1vZGVsJTIwJTNEJTIwU3BlZWNoVDVGb3JUZXh0VG9TcGVlY2guZnJvbV9wcmV0cmFpbmVkKCUyMm1pY3Jvc29mdCUyRnNwZWVjaHQ1X3R0cyUyMik=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> SpeechT5Processor, SpeechT5ForTextToSpeech

processor = SpeechT5Processor.from_pretrained(<span class="hljs-string">&quot;microsoft/speecht5_tts&quot;</span>)
model = SpeechT5ForTextToSpeech.from_pretrained(<span class="hljs-string">&quot;microsoft/speecht5_tts&quot;</span>)`}}),Ie=new E({props:{code:"aW5wdXRzJTIwJTNEJTIwcHJvY2Vzc29yKHRleHQlM0QlMjJEb24ndCUyMGNvdW50JTIwdGhlJTIwZGF5cyUyQyUyMG1ha2UlMjB0aGUlMjBkYXlzJTIwY291bnQuJTIyJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMik=",highlighted:'inputs = processor(text=<span class="hljs-string">&quot;Don&#x27;t count the days, make the days count.&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)'}}),ee=new Ns({props:{$$slots:{default:[tp]},$$scope:{ctx:C}}}),Be=new E({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBZW1iZWRkaW5nc19kYXRhc2V0JTIwJTNEJTIwbG9hZF9kYXRhc2V0KCUyMk1hdHRoaWpzJTJGY211LWFyY3RpYy14dmVjdG9ycyUyMiUyQyUyMHNwbGl0JTNEJTIydmFsaWRhdGlvbiUyMiklMEElMEFpbXBvcnQlMjB0b3JjaCUwQSUwQXNwZWFrZXJfZW1iZWRkaW5ncyUyMCUzRCUyMHRvcmNoLnRlbnNvcihlbWJlZGRpbmdzX2RhdGFzZXQlNUI3MzA2JTVEJTVCJTIyeHZlY3RvciUyMiU1RCkudW5zcXVlZXplKDAp",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

embeddings_dataset = load_dataset(<span class="hljs-string">&quot;Matthijs/cmu-arctic-xvectors&quot;</span>, split=<span class="hljs-string">&quot;validation&quot;</span>)

<span class="hljs-keyword">import</span> torch

speaker_embeddings = torch.tensor(embeddings_dataset[<span class="hljs-number">7306</span>][<span class="hljs-string">&quot;xvector&quot;</span>]).unsqueeze(<span class="hljs-number">0</span>)`}}),je=new E({props:{code:"c3BlY3Ryb2dyYW0lMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZV9zcGVlY2goaW5wdXRzJTVCJTIyaW5wdXRfaWRzJTIyJTVEJTJDJTIwc3BlYWtlcl9lbWJlZGRpbmdzKQ==",highlighted:'spectrogram = model.generate_speech(inputs[<span class="hljs-string">&quot;input_ids&quot;</span>], speaker_embeddings)'}}),te=new Ns({props:{$$slots:{default:[sp]},$$scope:{ctx:C}}}),Ze=new E({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFNwZWVjaFQ1SGlmaUdhbiUwQSUwQXZvY29kZXIlMjAlM0QlMjBTcGVlY2hUNUhpZmlHYW4uZnJvbV9wcmV0cmFpbmVkKCUyMm1pY3Jvc29mdCUyRnNwZWVjaHQ1X2hpZmlnYW4lMjIp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> SpeechT5HifiGan

vocoder = SpeechT5HifiGan.from_pretrained(<span class="hljs-string">&quot;microsoft/speecht5_hifigan&quot;</span>)`}}),Ve=new E({props:{code:"c3BlZWNoJTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGVfc3BlZWNoKGlucHV0cyU1QiUyMmlucHV0X2lkcyUyMiU1RCUyQyUyMHNwZWFrZXJfZW1iZWRkaW5ncyUyQyUyMHZvY29kZXIlM0R2b2NvZGVyKQ==",highlighted:'speech = model.generate_speech(inputs[<span class="hljs-string">&quot;input_ids&quot;</span>], speaker_embeddings, vocoder=vocoder)'}}),xe=new E({props:{code:"ZnJvbSUyMElQeXRob24uZGlzcGxheSUyMGltcG9ydCUyMEF1ZGlvJTBBJTBBQXVkaW8oc3BlZWNoJTJDJTIwcmF0ZSUzRDE2MDAwKQ==",highlighted:`<span class="hljs-keyword">from</span> IPython.display <span class="hljs-keyword">import</span> Audio

Audio(speech, rate=<span class="hljs-number">16000</span>)`}}),Ce=new xa({}),ze=new E({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJhcmtNb2RlbCUyQyUyMEJhcmtQcm9jZXNzb3IlMEElMEFtb2RlbCUyMCUzRCUyMEJhcmtNb2RlbC5mcm9tX3ByZXRyYWluZWQoJTIyc3VubyUyRmJhcmstc21hbGwlMjIpJTBBcHJvY2Vzc29yJTIwJTNEJTIwQmFya1Byb2Nlc3Nvci5mcm9tX3ByZXRyYWluZWQoJTIyc3VubyUyRmJhcmstc21hbGwlMjIp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BarkModel, BarkProcessor

model = BarkModel.from_pretrained(<span class="hljs-string">&quot;suno/bark-small&quot;</span>)
processor = BarkProcessor.from_pretrained(<span class="hljs-string">&quot;suno/bark-small&quot;</span>)`}}),Fe=new E({props:{code:"JTIzJTIwYWRkJTIwYSUyMHNwZWFrZXIlMjBlbWJlZGRpbmclMEFpbnB1dHMlMjAlM0QlMjBwcm9jZXNzb3IoJTIyVGhpcyUyMGlzJTIwYSUyMHRlc3QhJTIyJTJDJTIwdm9pY2VfcHJlc2V0JTNEJTIydjIlMkZlbl9zcGVha2VyXzMlMjIpJTBBJTBBc3BlZWNoX291dHB1dCUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqaW5wdXRzKS5jcHUoKS5udW1weSgp",highlighted:`<span class="hljs-comment"># add a speaker embedding</span>
inputs = processor(<span class="hljs-string">&quot;This is a test!&quot;</span>, voice_preset=<span class="hljs-string">&quot;v2/en_speaker_3&quot;</span>)

speech_output = model.generate(**inputs).cpu().numpy()`}}),He=new E({props:{code:"JTIzJTIwdHJ5JTIwaXQlMjBpbiUyMEZyZW5jaCUyQyUyMGxldCdzJTIwYWxzbyUyMGFkZCUyMGElMjBGcmVuY2glMjBzcGVha2VyJTIwZW1iZWRkaW5nJTBBaW5wdXRzJTIwJTNEJTIwcHJvY2Vzc29yKCUyMkMnZXN0JTIwdW4lMjB0ZXN0ISUyMiUyQyUyMHZvaWNlX3ByZXNldCUzRCUyMnYyJTJGZnJfc3BlYWtlcl8xJTIyKSUwQSUwQXNwZWVjaF9vdXRwdXQlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSgqKmlucHV0cykuY3B1KCkubnVtcHkoKQ==",highlighted:`<span class="hljs-comment"># try it in French, let&#x27;s also add a French speaker embedding</span>
inputs = processor(<span class="hljs-string">&quot;C&#x27;est un test!&quot;</span>, voice_preset=<span class="hljs-string">&quot;v2/fr_speaker_1&quot;</span>)

speech_output = model.generate(**inputs).cpu().numpy()`}}),Ye=new E({props:{code:"aW5wdXRzJTIwJTNEJTIwcHJvY2Vzc29yKCUwQSUyMCUyMCUyMCUyMCUyMiU1QmNsZWFycyUyMHRocm9hdCU1RCUyMFRoaXMlMjBpcyUyMGElMjB0ZXN0JTIwLi4uJTIwYW5kJTIwSSUyMGp1c3QlMjB0b29rJTIwYSUyMGxvbmclMjBwYXVzZS4lMjIlMkMlMEElMjAlMjAlMjAlMjB2b2ljZV9wcmVzZXQlM0QlMjJ2MiUyRmZyX3NwZWFrZXJfMSUyMiUyQyUwQSklMEElMEFzcGVlY2hfb3V0cHV0JTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMpLmNwdSgpLm51bXB5KCk=",highlighted:`inputs = processor(
    <span class="hljs-string">&quot;[clears throat] This is a test ... and I just took a long pause.&quot;</span>,
    voice_preset=<span class="hljs-string">&quot;v2/fr_speaker_1&quot;</span>,
)

speech_output = model.generate(**inputs).cpu().numpy()`}}),Qe=new E({props:{code:"aW5wdXRzJTIwJTNEJTIwcHJvY2Vzc29yKCUwQSUyMCUyMCUyMCUyMCUyMiVFMiU5OSVBQSUyMEluJTIwdGhlJTIwbWlnaHR5JTIwanVuZ2xlJTJDJTIwSSdtJTIwdHJ5aW5nJTIwdG8lMjBnZW5lcmF0ZSUyMGJhcmtzLiUyMiUyQyUwQSklMEElMEFzcGVlY2hfb3V0cHV0JTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMpLmNwdSgpLm51bXB5KCk=",highlighted:`inputs = processor(
    <span class="hljs-string">&quot;\u266A In the mighty jungle, I&#x27;m trying to generate barks.&quot;</span>,
)

speech_output = model.generate(**inputs).cpu().numpy()`}}),qe=new E({props:{code:"aW5wdXRfbGlzdCUyMCUzRCUyMCU1QiUwQSUyMCUyMCUyMCUyMCUyMiU1QmNsZWFycyUyMHRocm9hdCU1RCUyMEhlbGxvJTIwdWglMjAuLi4lMkMlMjBteSUyMGRvZyUyMGlzJTIwY3V0ZSUyMCU1QmxhdWdodGVyJTVEJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIyTGV0J3MlMjB0cnklMjBnZW5lcmF0aW5nJTIwc3BlZWNoJTJDJTIwd2l0aCUyMEJhcmslMkMlMjBhJTIwdGV4dC10by1zcGVlY2glMjBtb2RlbCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMiVFMiU5OSVBQSUyMEluJTIwdGhlJTIwanVuZ2xlJTJDJTIwdGhlJTIwbWlnaHR5JTIwanVuZ2xlJTJDJTIwdGhlJTIwbGlvbiUyMGJhcmtzJTIwdG9uaWdodCUyMCVFMiU5OSVBQSUyMiUyQyUwQSU1RCUwQSUwQSUyMyUyMGFsc28lMjBhZGQlMjBhJTIwc3BlYWtlciUyMGVtYmVkZGluZyUwQWlucHV0cyUyMCUzRCUyMHByb2Nlc3NvcihpbnB1dF9saXN0JTJDJTIwdm9pY2VfcHJlc2V0JTNEJTIydjIlMkZlbl9zcGVha2VyXzMlMjIpJTBBJTBBc3BlZWNoX291dHB1dCUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqaW5wdXRzKS5jcHUoKS5udW1weSgp",highlighted:`input_list = [
    <span class="hljs-string">&quot;[clears throat] Hello uh ..., my dog is cute [laughter]&quot;</span>,
    <span class="hljs-string">&quot;Let&#x27;s try generating speech, with Bark, a text-to-speech model&quot;</span>,
    <span class="hljs-string">&quot;\u266A In the jungle, the mighty jungle, the lion barks tonight \u266A&quot;</span>,
]

<span class="hljs-comment"># also add a speaker embedding</span>
inputs = processor(input_list, voice_preset=<span class="hljs-string">&quot;v2/en_speaker_3&quot;</span>)

speech_output = model.generate(**inputs).cpu().numpy()`}}),Pe=new E({props:{code:"ZnJvbSUyMElQeXRob24uZGlzcGxheSUyMGltcG9ydCUyMEF1ZGlvJTBBJTBBc2FtcGxpbmdfcmF0ZSUyMCUzRCUyMG1vZGVsLmdlbmVyYXRpb25fY29uZmlnLnNhbXBsZV9yYXRlJTBBQXVkaW8oc3BlZWNoX291dHB1dCU1QjAlNUQlMkMlMjByYXRlJTNEc2FtcGxpbmdfcmF0ZSk=",highlighted:`<span class="hljs-keyword">from</span> IPython.display <span class="hljs-keyword">import</span> Audio

sampling_rate = model.generation_config.sample_rate
Audio(speech_output[<span class="hljs-number">0</span>], rate=sampling_rate)`}}),Le=new E({props:{code:"QXVkaW8oc3BlZWNoX291dHB1dCU1QjElNUQlMkMlMjByYXRlJTNEc2FtcGxpbmdfcmF0ZSk=",highlighted:'Audio(speech_output[<span class="hljs-number">1</span>], rate=sampling_rate)'}}),De=new E({props:{code:"QXVkaW8oc3BlZWNoX291dHB1dCU1QjIlNUQlMkMlMjByYXRlJTNEc2FtcGxpbmdfcmF0ZSk=",highlighted:'Audio(speech_output[<span class="hljs-number">2</span>], rate=sampling_rate)'}}),we=new Ns({props:{$$slots:{default:[op]},$$scope:{ctx:C}}}),Oe=new xa({}),ge=new Ns({props:{warning:!0,$$slots:{default:[ap]},$$scope:{ctx:C}}}),et=new E({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFZpdHNNb2RlbCUyQyUyMFZpdHNUb2tlbml6ZXIlMEElMEFtb2RlbCUyMCUzRCUyMFZpdHNNb2RlbC5mcm9tX3ByZXRyYWluZWQoJTIyTWF0dGhpanMlMkZtbXMtdHRzLWRldSUyMiklMEF0b2tlbml6ZXIlMjAlM0QlMjBWaXRzVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJNYXR0aGlqcyUyRm1tcy10dHMtZGV1JTIyKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> VitsModel, VitsTokenizer

model = VitsModel.from_pretrained(<span class="hljs-string">&quot;Matthijs/mms-tts-deu&quot;</span>)
tokenizer = VitsTokenizer.from_pretrained(<span class="hljs-string">&quot;Matthijs/mms-tts-deu&quot;</span>)`}}),tt=new E({props:{code:"dGV4dF9leGFtcGxlJTIwJTNEJTIwKCUwQSUyMCUyMCUyMCUyMCUyMkljaCUyMGJpbiUyMFNjaG5hcHBpJTIwZGFzJTIwa2xlaW5lJTIwS3Jva29kaWwlMkMlMjBrb21tJTIwYXVzJTIwJUMzJTg0Z3lwdGVuJTIwZGFzJTIwbGllZ3QlMjBkaXJla3QlMjBhbSUyME5pbC4lMjIlMEEp",highlighted:`text_example = (
    <span class="hljs-string">&quot;Ich bin Schnappi das kleine Krokodil, komm aus \xC4gypten das liegt direkt am Nil.&quot;</span>
)`}}),st=new E({props:{code:"aW1wb3J0JTIwdG9yY2glMEElMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIodGV4dF9leGFtcGxlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEFpbnB1dF9pZHMlMjAlM0QlMjBpbnB1dHMlNUIlMjJpbnB1dF9pZHMlMjIlNUQlMEElMEElMEF3aXRoJTIwdG9yY2gubm9fZ3JhZCgpJTNBJTBBJTIwJTIwJTIwJTIwb3V0cHV0cyUyMCUzRCUyMG1vZGVsKGlucHV0X2lkcyklMEElMEFzcGVlY2glMjAlM0QlMjBvdXRwdXRzLmF1ZGlvJTVCMCU1RA==",highlighted:`<span class="hljs-keyword">import</span> torch

inputs = tokenizer(text_example, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
input_ids = inputs[<span class="hljs-string">&quot;input_ids&quot;</span>]


<span class="hljs-keyword">with</span> torch.no_grad():
    outputs = model(input_ids)

speech = outputs.audio[<span class="hljs-number">0</span>]`}}),ot=new E({props:{code:"ZnJvbSUyMElQeXRob24uZGlzcGxheSUyMGltcG9ydCUyMEF1ZGlvJTBBJTBBQXVkaW8oc3BlZWNoJTJDJTIwcmF0ZSUzRDE2MDAwKQ==",highlighted:`<span class="hljs-keyword">from</span> IPython.display <span class="hljs-keyword">import</span> Audio

Audio(speech, rate=<span class="hljs-number">16000</span>)`}}),{c(){u=r("meta"),w=c(),m=r("h1"),f=r("a"),J=r("span"),b(_.$$.fragment),S=c(),I=r("span"),B=l("Pre-trained models for text-to-speech"),x=c(),U=r("p"),j=l(`Compared to ASR (automatic speech recognition) and audio classification tasks, there are significantly fewer pre-trained
model checkpoints available. On the \u{1F917} Hub, you\u2019ll find close to 300 suitable checkpoints. Among
these pre-trained models we\u2019ll focus on two architectures that are readily available for you in the \u{1F917} Transformers library -
SpeechT5 and Massive Multilingual Speech (MMS). In this section, we\u2019ll explore how to use these pre-trained models in the
Transformers library for TTS.`),N=c(),v=r("h2"),y=r("a"),G=r("span"),b(Z.$$.fragment),R=c(),z=r("span"),Te=l("SpeechT5"),Me=c(),$e=r("p"),_e=r("a"),Ca=l("SpeechT5"),Na=l(` is a model published by Junyi Ao et al. from Microsoft that is capable of
handling a range of speech tasks. While in this unit, we focus on the text-to-speech aspect,
this model can be tailored to speech-to-text tasks (automatic speech recognition or speaker identification),
as well as speech-to-speech (e.g. speech enhancement or converting between different voices). This is due to how the model
is designed and pre-trained.`),Ws=c(),it=r("p"),Wa=l(`At the heart of SpeechT5 is a regular Transformer encoder-decoder model. Just like any other Transformer, the encoder-decoder
network models a sequence-to-sequence transformation using hidden representations. This Transformer backbone is the same
for all tasks SpeechT5 supports.`),Rs=c(),X=r("p"),Ra=l("This Transformer is complemented with six modal-specific (speech/text) "),ns=r("em"),za=l("pre-nets"),Xa=l(" and "),ps=r("em"),Fa=l("post-nets"),Aa=l(`. The input speech or text
(depending on the task) is preprocessed through a corresponding pre-net to obtain the hidden representations that Transformer
can use. The Transformer\u2019s output is then passed to a post-net that will use it to generate the output in the target modality.`),zs=c(),lt=r("p"),Ha=l("This is what the architecture looks like (image from the original paper):"),Xs=c(),Je=r("div"),nt=r("img"),Fs=c(),pt=r("p"),Ya=l(`SpeechT5 is first pre-trained using large-scale unlabeled speech and text data, to acquire a unified representation
of different modalities. During the pre-training phase all pre-nets and post-nets are used simultaneously.`),As=c(),ct=r("p"),Qa=l(`After pre-training, the entire encoder-decoder backbone is fine-tuned for each individual task. At this step, only the
pre-nets and post-nets relevant to the specific task are employed. For example, to use SpeechT5 for text-to-speech, you\u2019d
need the text encoder pre-net for the text inputs and the speech decoder pre- and post-nets for the speech outputs.`),Hs=c(),dt=r("p"),qa=l(`This approach allows to obtain several models fine-tuned for different speech tasks that all benefit from the initial
pre-training on unlabeled data.`),Ys=c(),b(O.$$.fragment),Qs=c(),ht=r("p"),Pa=l("Let\u2019s see what are the pre- and post-nets that SpeechT5 uses for the TTS task specifically:"),qs=c(),F=r("ul"),cs=r("li"),La=l("Text encoder pre-net: A text embedding layer that maps text tokens to the hidden representations that the encoder expects. This is similar to what happens in an NLP model such as BERT."),Da=c(),ds=r("li"),Oa=l("Speech decoder pre-net: This takes a log mel spectrogram as input and uses a sequence of linear layers to compress the spectrogram into hidden representations."),Ka=c(),hs=r("li"),er=l("Speech decoder post-net: This predicts a residual to add to the output spectrogram and is used to refine the results."),Ps=c(),ut=r("p"),tr=l("When combined, this is what SpeechT5 architecture for text-to-speech looks like:"),Ls=c(),Ee=r("div"),ft=r("img"),Ds=c(),K=r("p"),sr=l(`As you can see, the output is a log mel spectrogram and not a final waveform. If you recall, we briefly touched on
this topic in `),mt=r("a"),or=l("Unit 3"),ar=l(`. It is common for models that generate audio to produce
a log mel spectrogram, which needs to be converted to a waveform with an additional neural network known as a vocoder.`),Os=c(),yt=r("p"),rr=l("Let\u2019s see how you could do that."),Ks=c(),wt=r("p"),ir=l(`First, let\u2019s load the fine-tuned TTS SpeechT5 model from the \u{1F917} Hub, along with the processor object used for tokenization
and feature extraction:`),eo=c(),b(Ue.$$.fragment),to=c(),vt=r("p"),lr=l("Next, tokenize the input text."),so=c(),b(Ie.$$.fragment),oo=c(),bt=r("p"),nr=l(`The SpeechT5 TTS model is not limited to creating speech for a single speaker. Instead, it uses so-called speaker embeddings
that capture a particular speaker\u2019s voice characteristics.`),ao=c(),b(ee.$$.fragment),ro=c(),A=r("p"),pr=l(`Let\u2019s load such a speaker embedding from a dataset on the Hub. The embeddings
were obtained from the `),Se=r("a"),cr=l("CMU ARCTIC dataset"),dr=l(` using
`),Ge=r("a"),hr=l("this script"),ur=l(`, but
any X-Vector embedding should work.`),io=c(),b(Be.$$.fragment),lo=c(),gt=r("p"),fr=l("The speaker embedding is a tensor of shape (1, 512). This particular speaker embedding describes a female voice."),no=c(),kt=r("p"),mr=l("At this point we already have enough inputs to generate a log mel spectrogram as an output, you can do it like this:"),po=c(),b(je.$$.fragment),co=c(),Tt=r("p"),yr=l(`This outputs a tensor of shape (140, 80) containing a log mel spectrogram. The first dimension is the sequence length, and
it may vary between runs as the speech decoder pre-net always applies dropout to the input sequence. This adds a bit of
random variability to the generated speech.`),ho=c(),Mt=r("p"),wr=l(`However, if we are looking to generate speech waveform, we need to specify a vocoder to use for the spectrogram to waveform conversion.
In theory, you can use any vocoder that works on 80-bin mel spectrograms. Conveniently, \u{1F917} Transformers offers a vocoder
based on HiFi-GAN. Its weights were kindly provided by the original authors of SpeechT5.`),uo=c(),b(te.$$.fragment),fo=c(),$t=r("p"),vr=l("Loading the vocoder is as easy as any other \u{1F917} Transformers model."),mo=c(),b(Ze.$$.fragment),yo=c(),_t=r("p"),br=l("Now all you need to do is pass it as an argument when generating speech, and the outputs will be automatically converted to the speech waveform."),wo=c(),b(Ve.$$.fragment),vo=c(),Jt=r("p"),gr=l("Let\u2019s listen to the result. The sample rate used by SpeechT5 is always 16 kHz."),bo=c(),b(xe.$$.fragment),go=c(),Et=r("p"),kr=l("Neat!"),ko=c(),Ut=r("p"),Tr=l(`Feel free to play with the SpeechT5 text-to-speech demo, explore other voices, experiment with inputs. Note that this
pre-trained checkpoint only supports English language:`),To=c(),H=r("iframe"),Mo=c(),L=r("h2"),se=r("a"),us=r("span"),b(Ce.$$.fragment),Mr=c(),fs=r("span"),$r=l("Bark"),$o=c(),oe=r("p"),_r=l("Bark is a transformer-based text-to-speech model proposed by Suno AI in "),Ne=r("a"),Jr=l("suno-ai/bark"),Er=l("."),_o=c(),ae=r("p"),Ur=l("Unlike SpeechT5, Bark generates raw speech waveforms directly, eliminating the need for a separate vocoder during inference \u2013 it\u2019s already integrated. This efficiency is achieved through the utilization of "),We=r("a"),ms=r("code"),Ir=l("Encodec"),Sr=l(", which serves as both a codec and a compression tool."),Jo=c(),re=r("p"),Gr=l("With "),ys=r("code"),Br=l("Encodec"),jr=l(", you can compress audio into a lightweight format to reduce memory usage and subsequently decompress it to restore the original audio. This compression process is facilitated by 8 codebooks, each consisting of integer vectors. Think of these codebooks as representations or embeddings of the audio in integer form. It\u2019s important to note that each successive codebook improves the quality of the audio reconstruction from the previous codebooks. As codebooks are integer vectors, they can be learned by transformer models, which are very efficient in this task. This is what Bark was specifically trained to do."),Eo=c(),It=r("p"),Zr=l("To be more specific, Bark is made of 4 main models:"),Uo=c(),W=r("ul"),St=r("li"),ws=r("code"),Vr=l("BarkSemanticModel"),xr=l(" (also referred to as the \u2018text\u2019 model): a causal auto-regressive transformer model that takes as input tokenized text, and predicts semantic text tokens that capture the meaning of the text."),Cr=c(),ie=r("li"),vs=r("code"),Nr=l("BarkCoarseModel"),Wr=l(" (also referred to as the \u2018coarse acoustics\u2019 model): a causal autoregressive transformer, that takes as input the results of the "),bs=r("code"),Rr=l("BarkSemanticModel"),zr=l(" model. It aims at predicting the first two audio codebooks necessary for EnCodec."),Xr=c(),Gt=r("li"),gs=r("code"),Fr=l("BarkFineModel"),Ar=l(" (the \u2018fine acoustics\u2019 model), this time a non-causal autoencoder transformer, which iteratively predicts the last codebooks based on the sum of the previous codebooks embeddings."),Hr=c(),Re=r("li"),Yr=l("having predicted all the codebook channels from the "),ks=r("code"),Qr=l("EncodecModel"),qr=l(", Bark uses it to decode the output audio array."),Io=c(),Bt=r("p"),Pr=l("It should be noted that each of the first three modules can support conditional speaker embeddings to condition the output sound according to specific predefined voice."),So=c(),jt=r("p"),Lr=l("Bark is an highly-controllable text-to-speech model, meaning you can use with various settings, as we are going to see."),Go=c(),Zt=r("p"),Dr=l("Before everything, load the model and its processor."),Bo=c(),Vt=r("p"),Or=l("The processor role here is two-sides:"),jo=c(),le=r("ol"),Ts=r("li"),Kr=l("It is used to tokenize the input text, i.e. to cut it into small pieces that the model can understand."),ei=c(),Ms=r("li"),ti=l("It stores speaker embeddings, i.e voice presets that can condition the generation."),Zo=c(),b(ze.$$.fragment),Vo=c(),ne=r("p"),si=l("Bark is very versatile and can generate audio conditioned by "),Xe=r("a"),oi=l("a speaker embeddings library"),ai=l(" which can be loaded via the processor."),xo=c(),b(Fe.$$.fragment),Co=c(),pe=r("audio"),xt=r("source"),ri=l(` 
Your browser does not support the audio element.`),No=c(),ce=r("p"),ii=l("It can also generate ready-to-use multilingual speeches, such as French and Chinese. You can find a list of supported languages "),Ae=r("a"),li=l("here"),ni=l(". Unlike MMS, discussed below, it is not necessary to specify the language used, but simply adapt the input text to the corresponding language."),Wo=c(),b(He.$$.fragment),Ro=c(),de=r("audio"),Ct=r("source"),pi=l(` 
Your browser does not support the audio element.`),zo=c(),V=r("p"),ci=l("The model can also generate "),$s=r("strong"),di=l("non-verbal communications"),hi=l(" such as laughing, sighing and crying. You just have to modify the input text with corresponding cues such as "),_s=r("code"),ui=l("[clears throat]"),fi=l(", "),Js=r("code"),mi=l("[laughter]"),yi=l(", or "),Es=r("code"),wi=l("..."),vi=l("."),Xo=c(),b(Ye.$$.fragment),Fo=c(),he=r("audio"),Nt=r("source"),bi=l(` 
Your browser does not support the audio element.`),Ao=c(),Wt=r("p"),gi=l("Bark can even generate music. You can help by adding \u266A musical notes \u266A around your words."),Ho=c(),b(Qe.$$.fragment),Yo=c(),ue=r("audio"),Rt=r("source"),ki=l(` 
Your browser does not support the audio element.`),Qo=c(),zt=r("p"),Ti=l(`In addition to all these features, Bark supports batch processing, which means you can process several text entries at the same time, at the expense of more intensive computation.
On some hardware, such as GPUs, batching enables faster overall generation, which means it can be faster to generate samples all at once than to generate them one by one.`),qo=c(),Xt=r("p"),Mi=l("Let\u2019s try generating a few examples:"),Po=c(),b(qe.$$.fragment),Lo=c(),Ft=r("p"),$i=l("Let\u2019s listen to the outputs one by one."),Do=c(),At=r("p"),_i=l("First one:"),Oo=c(),b(Pe.$$.fragment),Ko=c(),fe=r("audio"),Ht=r("source"),Ji=l(` 
Your browser does not support the audio element.`),ea=c(),Yt=r("p"),Ei=l("Second one:"),ta=c(),b(Le.$$.fragment),sa=c(),me=r("audio"),Qt=r("source"),Ui=l(` 
Your browser does not support the audio element.`),oa=c(),qt=r("p"),Ii=l("Third one:"),aa=c(),b(De.$$.fragment),ra=c(),ye=r("audio"),Pt=r("source"),Si=l(` 
Your browser does not support the audio element.`),ia=c(),b(we.$$.fragment),la=c(),D=r("h2"),ve=r("a"),Us=r("span"),b(Oe.$$.fragment),Gi=c(),Is=r("span"),Bi=l("Massive Multilingual Speech (MMS)"),na=c(),Lt=r("p"),ji=l(`What if you are looking for a pre-trained model in a language other than English? Massive Multilingual Speech (MMS) is
another model that covers an array of speech tasks, however, it supports a large number of languages. For instance, it can
synthesize speech in over 1,100 languages.`),pa=c(),be=r("p"),Zi=l("MMS for text-to-speech is based on "),Ke=r("a"),Vi=l("VITS Kim et al., 2021"),xi=l(`, which is one of the
state-of-the-art TTS approaches.`),ca=c(),Dt=r("p"),Ci=l(`VITS is a speech generation network that converts text into raw speech waveforms. It works like a conditional variational
auto-encoder, estimating audio features from the input text. First, acoustic features, represented as spectrograms, are
generated. The waveform is then decoded using transposed convolutional layers adapted from HiFi-GAN.
During inference, the text encodings are upsampled and transformed into waveforms using the flow module and HiFi-GAN decoder.
Like Bark, there\u2019s no need for a vocoder, as waveforms are generated directly.`),da=c(),b(ge.$$.fragment),ha=c(),Ot=r("p"),Ni=l(`Let\u2019s give MMS a go, and see how we can synthesize speech in a language other than English, e.g. German.
First, we\u2019ll load the model checkpoint and the tokenizer for the correct language:`),ua=c(),b(et.$$.fragment),fa=c(),Y=r("p"),Wi=l("You may notice that to load the MMS model you need to use "),Ss=r("code"),Ri=l("VitsModel"),zi=l(" and "),Gs=r("code"),Xi=l("VitsTokenizer"),Fi=l(`. This is because MMS for text-to-speech
is based on the VITS model as mentioned earlier.`),ma=c(),Kt=r("p"),Ai=l("Let\u2019s pick an example text in German, like these first two lines from a children\u2019s song:"),ya=c(),b(tt.$$.fragment),wa=c(),es=r("p"),Hi=l("To generate a waveform output, preprocess the text with the tokenizer, and pass it to the model:"),va=c(),b(st.$$.fragment),ba=c(),ts=r("p"),Yi=l("Let\u2019s listen to it:"),ga=c(),b(ot.$$.fragment),ka=c(),Q=r("p"),Qi=l("Wunderbar! If you\u2019d like to try MMS with another language, find other suitable "),Bs=r("code"),qi=l("vits"),Pi=l(" checkpoints "),at=r("a"),Li=l("on \u{1F917} Hub"),Di=l("."),Ta=c(),ss=r("p"),Oi=l("Now let\u2019s see how you can fine-tune a TTS model yourself!"),this.h()},l(e){const o=Dn('[data-svelte="svelte-1phssyn"]',document.head);u=i(o,"META",{name:!0,content:!0}),o.forEach(t),w=d(e),m=i(e,"H1",{class:!0});var rt=p(m);f=i(rt,"A",{id:!0,class:!0,href:!0});var js=p(f);J=i(js,"SPAN",{});var Zs=p(J);g(_.$$.fragment,Zs),Zs.forEach(t),js.forEach(t),S=d(rt),I=i(rt,"SPAN",{});var Vs=p(I);B=n(Vs,"Pre-trained models for text-to-speech"),Vs.forEach(t),rt.forEach(t),x=d(e),U=i(e,"P",{});var xs=p(U);j=n(xs,`Compared to ASR (automatic speech recognition) and audio classification tasks, there are significantly fewer pre-trained
model checkpoints available. On the \u{1F917} Hub, you\u2019ll find close to 300 suitable checkpoints. Among
these pre-trained models we\u2019ll focus on two architectures that are readily available for you in the \u{1F917} Transformers library -
SpeechT5 and Massive Multilingual Speech (MMS). In this section, we\u2019ll explore how to use these pre-trained models in the
Transformers library for TTS.`),xs.forEach(t),N=d(e),v=i(e,"H2",{class:!0});var $a=p(v);y=i($a,"A",{id:!0,class:!0,href:!0});var bl=p(y);G=i(bl,"SPAN",{});var gl=p(G);g(Z.$$.fragment,gl),gl.forEach(t),bl.forEach(t),R=d($a),z=i($a,"SPAN",{});var kl=p(z);Te=n(kl,"SpeechT5"),kl.forEach(t),$a.forEach(t),Me=d(e),$e=i(e,"P",{});var Ki=p($e);_e=i(Ki,"A",{href:!0,rel:!0});var Tl=p(_e);Ca=n(Tl,"SpeechT5"),Tl.forEach(t),Na=n(Ki,` is a model published by Junyi Ao et al. from Microsoft that is capable of
handling a range of speech tasks. While in this unit, we focus on the text-to-speech aspect,
this model can be tailored to speech-to-text tasks (automatic speech recognition or speaker identification),
as well as speech-to-speech (e.g. speech enhancement or converting between different voices). This is due to how the model
is designed and pre-trained.`),Ki.forEach(t),Ws=d(e),it=i(e,"P",{});var Ml=p(it);Wa=n(Ml,`At the heart of SpeechT5 is a regular Transformer encoder-decoder model. Just like any other Transformer, the encoder-decoder
network models a sequence-to-sequence transformation using hidden representations. This Transformer backbone is the same
for all tasks SpeechT5 supports.`),Ml.forEach(t),Rs=d(e),X=i(e,"P",{});var os=p(X);Ra=n(os,"This Transformer is complemented with six modal-specific (speech/text) "),ns=i(os,"EM",{});var $l=p(ns);za=n($l,"pre-nets"),$l.forEach(t),Xa=n(os," and "),ps=i(os,"EM",{});var _l=p(ps);Fa=n(_l,"post-nets"),_l.forEach(t),Aa=n(os,`. The input speech or text
(depending on the task) is preprocessed through a corresponding pre-net to obtain the hidden representations that Transformer
can use. The Transformer\u2019s output is then passed to a post-net that will use it to generate the output in the target modality.`),os.forEach(t),zs=d(e),lt=i(e,"P",{});var Jl=p(lt);Ha=n(Jl,"This is what the architecture looks like (image from the original paper):"),Jl.forEach(t),Xs=d(e),Je=i(e,"DIV",{class:!0});var El=p(Je);nt=i(El,"IMG",{src:!0,alt:!0}),El.forEach(t),Fs=d(e),pt=i(e,"P",{});var Ul=p(pt);Ya=n(Ul,`SpeechT5 is first pre-trained using large-scale unlabeled speech and text data, to acquire a unified representation
of different modalities. During the pre-training phase all pre-nets and post-nets are used simultaneously.`),Ul.forEach(t),As=d(e),ct=i(e,"P",{});var Il=p(ct);Qa=n(Il,`After pre-training, the entire encoder-decoder backbone is fine-tuned for each individual task. At this step, only the
pre-nets and post-nets relevant to the specific task are employed. For example, to use SpeechT5 for text-to-speech, you\u2019d
need the text encoder pre-net for the text inputs and the speech decoder pre- and post-nets for the speech outputs.`),Il.forEach(t),Hs=d(e),dt=i(e,"P",{});var Sl=p(dt);qa=n(Sl,`This approach allows to obtain several models fine-tuned for different speech tasks that all benefit from the initial
pre-training on unlabeled data.`),Sl.forEach(t),Ys=d(e),g(O.$$.fragment,e),Qs=d(e),ht=i(e,"P",{});var Gl=p(ht);Pa=n(Gl,"Let\u2019s see what are the pre- and post-nets that SpeechT5 uses for the TTS task specifically:"),Gl.forEach(t),qs=d(e),F=i(e,"UL",{});var as=p(F);cs=i(as,"LI",{});var Bl=p(cs);La=n(Bl,"Text encoder pre-net: A text embedding layer that maps text tokens to the hidden representations that the encoder expects. This is similar to what happens in an NLP model such as BERT."),Bl.forEach(t),Da=d(as),ds=i(as,"LI",{});var jl=p(ds);Oa=n(jl,"Speech decoder pre-net: This takes a log mel spectrogram as input and uses a sequence of linear layers to compress the spectrogram into hidden representations."),jl.forEach(t),Ka=d(as),hs=i(as,"LI",{});var Zl=p(hs);er=n(Zl,"Speech decoder post-net: This predicts a residual to add to the output spectrogram and is used to refine the results."),Zl.forEach(t),as.forEach(t),Ps=d(e),ut=i(e,"P",{});var Vl=p(ut);tr=n(Vl,"When combined, this is what SpeechT5 architecture for text-to-speech looks like:"),Vl.forEach(t),Ls=d(e),Ee=i(e,"DIV",{class:!0});var xl=p(Ee);ft=i(xl,"IMG",{src:!0,alt:!0}),xl.forEach(t),Ds=d(e),K=i(e,"P",{});var _a=p(K);sr=n(_a,`As you can see, the output is a log mel spectrogram and not a final waveform. If you recall, we briefly touched on
this topic in `),mt=i(_a,"A",{href:!0});var Cl=p(mt);or=n(Cl,"Unit 3"),Cl.forEach(t),ar=n(_a,`. It is common for models that generate audio to produce
a log mel spectrogram, which needs to be converted to a waveform with an additional neural network known as a vocoder.`),_a.forEach(t),Os=d(e),yt=i(e,"P",{});var Nl=p(yt);rr=n(Nl,"Let\u2019s see how you could do that."),Nl.forEach(t),Ks=d(e),wt=i(e,"P",{});var Wl=p(wt);ir=n(Wl,`First, let\u2019s load the fine-tuned TTS SpeechT5 model from the \u{1F917} Hub, along with the processor object used for tokenization
and feature extraction:`),Wl.forEach(t),eo=d(e),g(Ue.$$.fragment,e),to=d(e),vt=i(e,"P",{});var Rl=p(vt);lr=n(Rl,"Next, tokenize the input text."),Rl.forEach(t),so=d(e),g(Ie.$$.fragment,e),oo=d(e),bt=i(e,"P",{});var zl=p(bt);nr=n(zl,`The SpeechT5 TTS model is not limited to creating speech for a single speaker. Instead, it uses so-called speaker embeddings
that capture a particular speaker\u2019s voice characteristics.`),zl.forEach(t),ao=d(e),g(ee.$$.fragment,e),ro=d(e),A=i(e,"P",{});var rs=p(A);pr=n(rs,`Let\u2019s load such a speaker embedding from a dataset on the Hub. The embeddings
were obtained from the `),Se=i(rs,"A",{href:!0,rel:!0});var Xl=p(Se);cr=n(Xl,"CMU ARCTIC dataset"),Xl.forEach(t),dr=n(rs,` using
`),Ge=i(rs,"A",{href:!0,rel:!0});var Fl=p(Ge);hr=n(Fl,"this script"),Fl.forEach(t),ur=n(rs,`, but
any X-Vector embedding should work.`),rs.forEach(t),io=d(e),g(Be.$$.fragment,e),lo=d(e),gt=i(e,"P",{});var Al=p(gt);fr=n(Al,"The speaker embedding is a tensor of shape (1, 512). This particular speaker embedding describes a female voice."),Al.forEach(t),no=d(e),kt=i(e,"P",{});var Hl=p(kt);mr=n(Hl,"At this point we already have enough inputs to generate a log mel spectrogram as an output, you can do it like this:"),Hl.forEach(t),po=d(e),g(je.$$.fragment,e),co=d(e),Tt=i(e,"P",{});var Yl=p(Tt);yr=n(Yl,`This outputs a tensor of shape (140, 80) containing a log mel spectrogram. The first dimension is the sequence length, and
it may vary between runs as the speech decoder pre-net always applies dropout to the input sequence. This adds a bit of
random variability to the generated speech.`),Yl.forEach(t),ho=d(e),Mt=i(e,"P",{});var Ql=p(Mt);wr=n(Ql,`However, if we are looking to generate speech waveform, we need to specify a vocoder to use for the spectrogram to waveform conversion.
In theory, you can use any vocoder that works on 80-bin mel spectrograms. Conveniently, \u{1F917} Transformers offers a vocoder
based on HiFi-GAN. Its weights were kindly provided by the original authors of SpeechT5.`),Ql.forEach(t),uo=d(e),g(te.$$.fragment,e),fo=d(e),$t=i(e,"P",{});var ql=p($t);vr=n(ql,"Loading the vocoder is as easy as any other \u{1F917} Transformers model."),ql.forEach(t),mo=d(e),g(Ze.$$.fragment,e),yo=d(e),_t=i(e,"P",{});var Pl=p(_t);br=n(Pl,"Now all you need to do is pass it as an argument when generating speech, and the outputs will be automatically converted to the speech waveform."),Pl.forEach(t),wo=d(e),g(Ve.$$.fragment,e),vo=d(e),Jt=i(e,"P",{});var Ll=p(Jt);gr=n(Ll,"Let\u2019s listen to the result. The sample rate used by SpeechT5 is always 16 kHz."),Ll.forEach(t),bo=d(e),g(xe.$$.fragment,e),go=d(e),Et=i(e,"P",{});var Dl=p(Et);kr=n(Dl,"Neat!"),Dl.forEach(t),ko=d(e),Ut=i(e,"P",{});var Ol=p(Ut);Tr=n(Ol,`Feel free to play with the SpeechT5 text-to-speech demo, explore other voices, experiment with inputs. Note that this
pre-trained checkpoint only supports English language:`),Ol.forEach(t),To=d(e),H=i(e,"IFRAME",{src:!0,frameborder:!0,width:!0,height:!0});var Qn=p(H);Qn.forEach(t),Mo=d(e),L=i(e,"H2",{class:!0});var Ja=p(L);se=i(Ja,"A",{id:!0,class:!0,href:!0});var Kl=p(se);us=i(Kl,"SPAN",{});var en=p(us);g(Ce.$$.fragment,en),en.forEach(t),Kl.forEach(t),Mr=d(Ja),fs=i(Ja,"SPAN",{});var tn=p(fs);$r=n(tn,"Bark"),tn.forEach(t),Ja.forEach(t),$o=d(e),oe=i(e,"P",{});var Ea=p(oe);_r=n(Ea,"Bark is a transformer-based text-to-speech model proposed by Suno AI in "),Ne=i(Ea,"A",{href:!0,rel:!0});var sn=p(Ne);Jr=n(sn,"suno-ai/bark"),sn.forEach(t),Er=n(Ea,"."),Ea.forEach(t),_o=d(e),ae=i(e,"P",{});var Ua=p(ae);Ur=n(Ua,"Unlike SpeechT5, Bark generates raw speech waveforms directly, eliminating the need for a separate vocoder during inference \u2013 it\u2019s already integrated. This efficiency is achieved through the utilization of "),We=i(Ua,"A",{href:!0,rel:!0});var on=p(We);ms=i(on,"CODE",{});var an=p(ms);Ir=n(an,"Encodec"),an.forEach(t),on.forEach(t),Sr=n(Ua,", which serves as both a codec and a compression tool."),Ua.forEach(t),Jo=d(e),re=i(e,"P",{});var Ia=p(re);Gr=n(Ia,"With "),ys=i(Ia,"CODE",{});var rn=p(ys);Br=n(rn,"Encodec"),rn.forEach(t),jr=n(Ia,", you can compress audio into a lightweight format to reduce memory usage and subsequently decompress it to restore the original audio. This compression process is facilitated by 8 codebooks, each consisting of integer vectors. Think of these codebooks as representations or embeddings of the audio in integer form. It\u2019s important to note that each successive codebook improves the quality of the audio reconstruction from the previous codebooks. As codebooks are integer vectors, they can be learned by transformer models, which are very efficient in this task. This is what Bark was specifically trained to do."),Ia.forEach(t),Eo=d(e),It=i(e,"P",{});var ln=p(It);Zr=n(ln,"To be more specific, Bark is made of 4 main models:"),ln.forEach(t),Uo=d(e),W=i(e,"UL",{});var ke=p(W);St=i(ke,"LI",{});var el=p(St);ws=i(el,"CODE",{});var nn=p(ws);Vr=n(nn,"BarkSemanticModel"),nn.forEach(t),xr=n(el," (also referred to as the \u2018text\u2019 model): a causal auto-regressive transformer model that takes as input tokenized text, and predicts semantic text tokens that capture the meaning of the text."),el.forEach(t),Cr=d(ke),ie=i(ke,"LI",{});var Cs=p(ie);vs=i(Cs,"CODE",{});var pn=p(vs);Nr=n(pn,"BarkCoarseModel"),pn.forEach(t),Wr=n(Cs," (also referred to as the \u2018coarse acoustics\u2019 model): a causal autoregressive transformer, that takes as input the results of the "),bs=i(Cs,"CODE",{});var cn=p(bs);Rr=n(cn,"BarkSemanticModel"),cn.forEach(t),zr=n(Cs," model. It aims at predicting the first two audio codebooks necessary for EnCodec."),Cs.forEach(t),Xr=d(ke),Gt=i(ke,"LI",{});var tl=p(Gt);gs=i(tl,"CODE",{});var dn=p(gs);Fr=n(dn,"BarkFineModel"),dn.forEach(t),Ar=n(tl," (the \u2018fine acoustics\u2019 model), this time a non-causal autoencoder transformer, which iteratively predicts the last codebooks based on the sum of the previous codebooks embeddings."),tl.forEach(t),Hr=d(ke),Re=i(ke,"LI",{});var Sa=p(Re);Yr=n(Sa,"having predicted all the codebook channels from the "),ks=i(Sa,"CODE",{});var hn=p(ks);Qr=n(hn,"EncodecModel"),hn.forEach(t),qr=n(Sa,", Bark uses it to decode the output audio array."),Sa.forEach(t),ke.forEach(t),Io=d(e),Bt=i(e,"P",{});var un=p(Bt);Pr=n(un,"It should be noted that each of the first three modules can support conditional speaker embeddings to condition the output sound according to specific predefined voice."),un.forEach(t),So=d(e),jt=i(e,"P",{});var fn=p(jt);Lr=n(fn,"Bark is an highly-controllable text-to-speech model, meaning you can use with various settings, as we are going to see."),fn.forEach(t),Go=d(e),Zt=i(e,"P",{});var mn=p(Zt);Dr=n(mn,"Before everything, load the model and its processor."),mn.forEach(t),Bo=d(e),Vt=i(e,"P",{});var yn=p(Vt);Or=n(yn,"The processor role here is two-sides:"),yn.forEach(t),jo=d(e),le=i(e,"OL",{});var Ga=p(le);Ts=i(Ga,"LI",{});var wn=p(Ts);Kr=n(wn,"It is used to tokenize the input text, i.e. to cut it into small pieces that the model can understand."),wn.forEach(t),ei=d(Ga),Ms=i(Ga,"LI",{});var vn=p(Ms);ti=n(vn,"It stores speaker embeddings, i.e voice presets that can condition the generation."),vn.forEach(t),Ga.forEach(t),Zo=d(e),g(ze.$$.fragment,e),Vo=d(e),ne=i(e,"P",{});var Ba=p(ne);si=n(Ba,"Bark is very versatile and can generate audio conditioned by "),Xe=i(Ba,"A",{href:!0,rel:!0});var bn=p(Xe);oi=n(bn,"a speaker embeddings library"),bn.forEach(t),ai=n(Ba," which can be loaded via the processor."),Ba.forEach(t),xo=d(e),g(Fe.$$.fragment,e),Co=d(e),pe=i(e,"AUDIO",{});var sl=p(pe);xt=i(sl,"SOURCE",{src:!0,type:!0}),ri=n(sl,` 
Your browser does not support the audio element.`),sl.forEach(t),No=d(e),ce=i(e,"P",{});var ja=p(ce);ii=n(ja,"It can also generate ready-to-use multilingual speeches, such as French and Chinese. You can find a list of supported languages "),Ae=i(ja,"A",{href:!0,rel:!0});var gn=p(Ae);li=n(gn,"here"),gn.forEach(t),ni=n(ja,". Unlike MMS, discussed below, it is not necessary to specify the language used, but simply adapt the input text to the corresponding language."),ja.forEach(t),Wo=d(e),g(He.$$.fragment,e),Ro=d(e),de=i(e,"AUDIO",{});var ol=p(de);Ct=i(ol,"SOURCE",{src:!0,type:!0}),pi=n(ol,` 
Your browser does not support the audio element.`),ol.forEach(t),zo=d(e),V=i(e,"P",{});var q=p(V);ci=n(q,"The model can also generate "),$s=i(q,"STRONG",{});var kn=p($s);di=n(kn,"non-verbal communications"),kn.forEach(t),hi=n(q," such as laughing, sighing and crying. You just have to modify the input text with corresponding cues such as "),_s=i(q,"CODE",{});var Tn=p(_s);ui=n(Tn,"[clears throat]"),Tn.forEach(t),fi=n(q,", "),Js=i(q,"CODE",{});var Mn=p(Js);mi=n(Mn,"[laughter]"),Mn.forEach(t),yi=n(q,", or "),Es=i(q,"CODE",{});var $n=p(Es);wi=n($n,"..."),$n.forEach(t),vi=n(q,"."),q.forEach(t),Xo=d(e),g(Ye.$$.fragment,e),Fo=d(e),he=i(e,"AUDIO",{});var al=p(he);Nt=i(al,"SOURCE",{src:!0,type:!0}),bi=n(al,` 
Your browser does not support the audio element.`),al.forEach(t),Ao=d(e),Wt=i(e,"P",{});var _n=p(Wt);gi=n(_n,"Bark can even generate music. You can help by adding \u266A musical notes \u266A around your words."),_n.forEach(t),Ho=d(e),g(Qe.$$.fragment,e),Yo=d(e),ue=i(e,"AUDIO",{});var rl=p(ue);Rt=i(rl,"SOURCE",{src:!0,type:!0}),ki=n(rl,` 
Your browser does not support the audio element.`),rl.forEach(t),Qo=d(e),zt=i(e,"P",{});var Jn=p(zt);Ti=n(Jn,`In addition to all these features, Bark supports batch processing, which means you can process several text entries at the same time, at the expense of more intensive computation.
On some hardware, such as GPUs, batching enables faster overall generation, which means it can be faster to generate samples all at once than to generate them one by one.`),Jn.forEach(t),qo=d(e),Xt=i(e,"P",{});var En=p(Xt);Mi=n(En,"Let\u2019s try generating a few examples:"),En.forEach(t),Po=d(e),g(qe.$$.fragment,e),Lo=d(e),Ft=i(e,"P",{});var Un=p(Ft);$i=n(Un,"Let\u2019s listen to the outputs one by one."),Un.forEach(t),Do=d(e),At=i(e,"P",{});var In=p(At);_i=n(In,"First one:"),In.forEach(t),Oo=d(e),g(Pe.$$.fragment,e),Ko=d(e),fe=i(e,"AUDIO",{});var il=p(fe);Ht=i(il,"SOURCE",{src:!0,type:!0}),Ji=n(il,` 
Your browser does not support the audio element.`),il.forEach(t),ea=d(e),Yt=i(e,"P",{});var Sn=p(Yt);Ei=n(Sn,"Second one:"),Sn.forEach(t),ta=d(e),g(Le.$$.fragment,e),sa=d(e),me=i(e,"AUDIO",{});var ll=p(me);Qt=i(ll,"SOURCE",{src:!0,type:!0}),Ui=n(ll,` 
Your browser does not support the audio element.`),ll.forEach(t),oa=d(e),qt=i(e,"P",{});var Gn=p(qt);Ii=n(Gn,"Third one:"),Gn.forEach(t),aa=d(e),g(De.$$.fragment,e),ra=d(e),ye=i(e,"AUDIO",{});var nl=p(ye);Pt=i(nl,"SOURCE",{src:!0,type:!0}),Si=n(nl,` 
Your browser does not support the audio element.`),nl.forEach(t),ia=d(e),g(we.$$.fragment,e),la=d(e),D=i(e,"H2",{class:!0});var Za=p(D);ve=i(Za,"A",{id:!0,class:!0,href:!0});var Bn=p(ve);Us=i(Bn,"SPAN",{});var jn=p(Us);g(Oe.$$.fragment,jn),jn.forEach(t),Bn.forEach(t),Gi=d(Za),Is=i(Za,"SPAN",{});var Zn=p(Is);Bi=n(Zn,"Massive Multilingual Speech (MMS)"),Zn.forEach(t),Za.forEach(t),na=d(e),Lt=i(e,"P",{});var Vn=p(Lt);ji=n(Vn,`What if you are looking for a pre-trained model in a language other than English? Massive Multilingual Speech (MMS) is
another model that covers an array of speech tasks, however, it supports a large number of languages. For instance, it can
synthesize speech in over 1,100 languages.`),Vn.forEach(t),pa=d(e),be=i(e,"P",{});var Va=p(be);Zi=n(Va,"MMS for text-to-speech is based on "),Ke=i(Va,"A",{href:!0,rel:!0});var xn=p(Ke);Vi=n(xn,"VITS Kim et al., 2021"),xn.forEach(t),xi=n(Va,`, which is one of the
state-of-the-art TTS approaches.`),Va.forEach(t),ca=d(e),Dt=i(e,"P",{});var Cn=p(Dt);Ci=n(Cn,`VITS is a speech generation network that converts text into raw speech waveforms. It works like a conditional variational
auto-encoder, estimating audio features from the input text. First, acoustic features, represented as spectrograms, are
generated. The waveform is then decoded using transposed convolutional layers adapted from HiFi-GAN.
During inference, the text encodings are upsampled and transformed into waveforms using the flow module and HiFi-GAN decoder.
Like Bark, there\u2019s no need for a vocoder, as waveforms are generated directly.`),Cn.forEach(t),da=d(e),g(ge.$$.fragment,e),ha=d(e),Ot=i(e,"P",{});var Nn=p(Ot);Ni=n(Nn,`Let\u2019s give MMS a go, and see how we can synthesize speech in a language other than English, e.g. German.
First, we\u2019ll load the model checkpoint and the tokenizer for the correct language:`),Nn.forEach(t),ua=d(e),g(et.$$.fragment,e),fa=d(e),Y=i(e,"P",{});var is=p(Y);Wi=n(is,"You may notice that to load the MMS model you need to use "),Ss=i(is,"CODE",{});var Wn=p(Ss);Ri=n(Wn,"VitsModel"),Wn.forEach(t),zi=n(is," and "),Gs=i(is,"CODE",{});var Rn=p(Gs);Xi=n(Rn,"VitsTokenizer"),Rn.forEach(t),Fi=n(is,`. This is because MMS for text-to-speech
is based on the VITS model as mentioned earlier.`),is.forEach(t),ma=d(e),Kt=i(e,"P",{});var zn=p(Kt);Ai=n(zn,"Let\u2019s pick an example text in German, like these first two lines from a children\u2019s song:"),zn.forEach(t),ya=d(e),g(tt.$$.fragment,e),wa=d(e),es=i(e,"P",{});var Xn=p(es);Hi=n(Xn,"To generate a waveform output, preprocess the text with the tokenizer, and pass it to the model:"),Xn.forEach(t),va=d(e),g(st.$$.fragment,e),ba=d(e),ts=i(e,"P",{});var Fn=p(ts);Yi=n(Fn,"Let\u2019s listen to it:"),Fn.forEach(t),ga=d(e),g(ot.$$.fragment,e),ka=d(e),Q=i(e,"P",{});var ls=p(Q);Qi=n(ls,"Wunderbar! If you\u2019d like to try MMS with another language, find other suitable "),Bs=i(ls,"CODE",{});var An=p(Bs);qi=n(An,"vits"),An.forEach(t),Pi=n(ls," checkpoints "),at=i(ls,"A",{href:!0,rel:!0});var Hn=p(at);Li=n(Hn,"on \u{1F917} Hub"),Hn.forEach(t),Di=n(ls,"."),ls.forEach(t),Ta=d(e),ss=i(e,"P",{});var Yn=p(ss);Oi=n(Yn,"Now let\u2019s see how you can fine-tune a TTS model yourself!"),Yn.forEach(t),this.h()},h(){h(u,"name","hf:doc:metadata"),h(u,"content",JSON.stringify(ip)),h(f,"id","pretrained-models-for-texttospeech"),h(f,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(f,"href","#pretrained-models-for-texttospeech"),h(m,"class","relative group"),h(y,"id","speecht5"),h(y,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(y,"href","#speecht5"),h(v,"class","relative group"),h(_e,"href","https://arxiv.org/abs/2110.07205"),h(_e,"rel","nofollow"),P(nt.src,pl="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/blog/speecht5/architecture.jpg")||h(nt,"src",pl),h(nt,"alt","SpeechT5 architecture from the original paper"),h(Je,"class","flex justify-center"),P(ft.src,cl="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/blog/speecht5/tts.jpg")||h(ft,"src",cl),h(ft,"alt","SpeechT5 architecture for TTS"),h(Ee,"class","flex justify-center"),h(mt,"href","../chapter3/introduction#spectrogram-output"),h(Se,"href","http://www.festvox.org/cmu_arctic/"),h(Se,"rel","nofollow"),h(Ge,"href","https://huggingface.co/mechanicalsea/speecht5-vc/blob/main/manifest/utils/prep_cmu_arctic_spkemb.py"),h(Ge,"rel","nofollow"),P(H.src,dl="https://matthijs-speecht5-tts-demo.hf.space")||h(H,"src",dl),h(H,"frameborder","0"),h(H,"width","850"),h(H,"height","450"),h(se,"id","bark"),h(se,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(se,"href","#bark"),h(L,"class","relative group"),h(Ne,"href","https://github.com/suno-ai/bark"),h(Ne,"rel","nofollow"),h(We,"href","https://huggingface.co/docs/transformers/main/en/model_doc/encodec"),h(We,"rel","nofollow"),h(Xe,"href","https://suno-ai.notion.site/8b8e8749ed514b0cbf3f699013548683?v=bc67cff786b04b50b3ceb756fd05f68c"),h(Xe,"rel","nofollow"),P(xt.src,hl="https://huggingface.co/datasets/ylacombe/hf-course-audio-files/resolve/main/first_sample.wav")||h(xt,"src",hl),h(xt,"type","audio/wav"),pe.controls=!0,h(Ae,"href","https://huggingface.co/suno/bark"),h(Ae,"rel","nofollow"),P(Ct.src,ul="https://huggingface.co/datasets/ylacombe/hf-course-audio-files/resolve/main/second_sample.wav")||h(Ct,"src",ul),h(Ct,"type","audio/wav"),de.controls=!0,P(Nt.src,fl="https://huggingface.co/datasets/ylacombe/hf-course-audio-files/resolve/main/third_sample.wav")||h(Nt,"src",fl),h(Nt,"type","audio/wav"),he.controls=!0,P(Rt.src,ml="https://huggingface.co/datasets/ylacombe/hf-course-audio-files/resolve/main/fourth_sample.wav")||h(Rt,"src",ml),h(Rt,"type","audio/wav"),ue.controls=!0,P(Ht.src,yl="https://huggingface.co/datasets/ylacombe/hf-course-audio-files/resolve/main/batch_1.wav")||h(Ht,"src",yl),h(Ht,"type","audio/wav"),fe.controls=!0,P(Qt.src,wl="https://huggingface.co/datasets/ylacombe/hf-course-audio-files/resolve/main/batch_2.wav")||h(Qt,"src",wl),h(Qt,"type","audio/wav"),me.controls=!0,P(Pt.src,vl="https://huggingface.co/datasets/ylacombe/hf-course-audio-files/resolve/main/batch_3.wav")||h(Pt,"src",vl),h(Pt,"type","audio/wav"),ye.controls=!0,h(ve,"id","massive-multilingual-speech-mms"),h(ve,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(ve,"href","#massive-multilingual-speech-mms"),h(D,"class","relative group"),h(Ke,"href","https://arxiv.org/pdf/2106.06103.pdf"),h(Ke,"rel","nofollow"),h(at,"href","https://huggingface.co/models?filter=vits"),h(at,"rel","nofollow")},m(e,o){s(document.head,u),a(e,w,o),a(e,m,o),s(m,f),s(f,J),k(_,J,null),s(m,S),s(m,I),s(I,B),a(e,x,o),a(e,U,o),s(U,j),a(e,N,o),a(e,v,o),s(v,y),s(y,G),k(Z,G,null),s(v,R),s(v,z),s(z,Te),a(e,Me,o),a(e,$e,o),s($e,_e),s(_e,Ca),s($e,Na),a(e,Ws,o),a(e,it,o),s(it,Wa),a(e,Rs,o),a(e,X,o),s(X,Ra),s(X,ns),s(ns,za),s(X,Xa),s(X,ps),s(ps,Fa),s(X,Aa),a(e,zs,o),a(e,lt,o),s(lt,Ha),a(e,Xs,o),a(e,Je,o),s(Je,nt),a(e,Fs,o),a(e,pt,o),s(pt,Ya),a(e,As,o),a(e,ct,o),s(ct,Qa),a(e,Hs,o),a(e,dt,o),s(dt,qa),a(e,Ys,o),k(O,e,o),a(e,Qs,o),a(e,ht,o),s(ht,Pa),a(e,qs,o),a(e,F,o),s(F,cs),s(cs,La),s(F,Da),s(F,ds),s(ds,Oa),s(F,Ka),s(F,hs),s(hs,er),a(e,Ps,o),a(e,ut,o),s(ut,tr),a(e,Ls,o),a(e,Ee,o),s(Ee,ft),a(e,Ds,o),a(e,K,o),s(K,sr),s(K,mt),s(mt,or),s(K,ar),a(e,Os,o),a(e,yt,o),s(yt,rr),a(e,Ks,o),a(e,wt,o),s(wt,ir),a(e,eo,o),k(Ue,e,o),a(e,to,o),a(e,vt,o),s(vt,lr),a(e,so,o),k(Ie,e,o),a(e,oo,o),a(e,bt,o),s(bt,nr),a(e,ao,o),k(ee,e,o),a(e,ro,o),a(e,A,o),s(A,pr),s(A,Se),s(Se,cr),s(A,dr),s(A,Ge),s(Ge,hr),s(A,ur),a(e,io,o),k(Be,e,o),a(e,lo,o),a(e,gt,o),s(gt,fr),a(e,no,o),a(e,kt,o),s(kt,mr),a(e,po,o),k(je,e,o),a(e,co,o),a(e,Tt,o),s(Tt,yr),a(e,ho,o),a(e,Mt,o),s(Mt,wr),a(e,uo,o),k(te,e,o),a(e,fo,o),a(e,$t,o),s($t,vr),a(e,mo,o),k(Ze,e,o),a(e,yo,o),a(e,_t,o),s(_t,br),a(e,wo,o),k(Ve,e,o),a(e,vo,o),a(e,Jt,o),s(Jt,gr),a(e,bo,o),k(xe,e,o),a(e,go,o),a(e,Et,o),s(Et,kr),a(e,ko,o),a(e,Ut,o),s(Ut,Tr),a(e,To,o),a(e,H,o),a(e,Mo,o),a(e,L,o),s(L,se),s(se,us),k(Ce,us,null),s(L,Mr),s(L,fs),s(fs,$r),a(e,$o,o),a(e,oe,o),s(oe,_r),s(oe,Ne),s(Ne,Jr),s(oe,Er),a(e,_o,o),a(e,ae,o),s(ae,Ur),s(ae,We),s(We,ms),s(ms,Ir),s(ae,Sr),a(e,Jo,o),a(e,re,o),s(re,Gr),s(re,ys),s(ys,Br),s(re,jr),a(e,Eo,o),a(e,It,o),s(It,Zr),a(e,Uo,o),a(e,W,o),s(W,St),s(St,ws),s(ws,Vr),s(St,xr),s(W,Cr),s(W,ie),s(ie,vs),s(vs,Nr),s(ie,Wr),s(ie,bs),s(bs,Rr),s(ie,zr),s(W,Xr),s(W,Gt),s(Gt,gs),s(gs,Fr),s(Gt,Ar),s(W,Hr),s(W,Re),s(Re,Yr),s(Re,ks),s(ks,Qr),s(Re,qr),a(e,Io,o),a(e,Bt,o),s(Bt,Pr),a(e,So,o),a(e,jt,o),s(jt,Lr),a(e,Go,o),a(e,Zt,o),s(Zt,Dr),a(e,Bo,o),a(e,Vt,o),s(Vt,Or),a(e,jo,o),a(e,le,o),s(le,Ts),s(Ts,Kr),s(le,ei),s(le,Ms),s(Ms,ti),a(e,Zo,o),k(ze,e,o),a(e,Vo,o),a(e,ne,o),s(ne,si),s(ne,Xe),s(Xe,oi),s(ne,ai),a(e,xo,o),k(Fe,e,o),a(e,Co,o),a(e,pe,o),s(pe,xt),s(pe,ri),a(e,No,o),a(e,ce,o),s(ce,ii),s(ce,Ae),s(Ae,li),s(ce,ni),a(e,Wo,o),k(He,e,o),a(e,Ro,o),a(e,de,o),s(de,Ct),s(de,pi),a(e,zo,o),a(e,V,o),s(V,ci),s(V,$s),s($s,di),s(V,hi),s(V,_s),s(_s,ui),s(V,fi),s(V,Js),s(Js,mi),s(V,yi),s(V,Es),s(Es,wi),s(V,vi),a(e,Xo,o),k(Ye,e,o),a(e,Fo,o),a(e,he,o),s(he,Nt),s(he,bi),a(e,Ao,o),a(e,Wt,o),s(Wt,gi),a(e,Ho,o),k(Qe,e,o),a(e,Yo,o),a(e,ue,o),s(ue,Rt),s(ue,ki),a(e,Qo,o),a(e,zt,o),s(zt,Ti),a(e,qo,o),a(e,Xt,o),s(Xt,Mi),a(e,Po,o),k(qe,e,o),a(e,Lo,o),a(e,Ft,o),s(Ft,$i),a(e,Do,o),a(e,At,o),s(At,_i),a(e,Oo,o),k(Pe,e,o),a(e,Ko,o),a(e,fe,o),s(fe,Ht),s(fe,Ji),a(e,ea,o),a(e,Yt,o),s(Yt,Ei),a(e,ta,o),k(Le,e,o),a(e,sa,o),a(e,me,o),s(me,Qt),s(me,Ui),a(e,oa,o),a(e,qt,o),s(qt,Ii),a(e,aa,o),k(De,e,o),a(e,ra,o),a(e,ye,o),s(ye,Pt),s(ye,Si),a(e,ia,o),k(we,e,o),a(e,la,o),a(e,D,o),s(D,ve),s(ve,Us),k(Oe,Us,null),s(D,Gi),s(D,Is),s(Is,Bi),a(e,na,o),a(e,Lt,o),s(Lt,ji),a(e,pa,o),a(e,be,o),s(be,Zi),s(be,Ke),s(Ke,Vi),s(be,xi),a(e,ca,o),a(e,Dt,o),s(Dt,Ci),a(e,da,o),k(ge,e,o),a(e,ha,o),a(e,Ot,o),s(Ot,Ni),a(e,ua,o),k(et,e,o),a(e,fa,o),a(e,Y,o),s(Y,Wi),s(Y,Ss),s(Ss,Ri),s(Y,zi),s(Y,Gs),s(Gs,Xi),s(Y,Fi),a(e,ma,o),a(e,Kt,o),s(Kt,Ai),a(e,ya,o),k(tt,e,o),a(e,wa,o),a(e,es,o),s(es,Hi),a(e,va,o),k(st,e,o),a(e,ba,o),a(e,ts,o),s(ts,Yi),a(e,ga,o),k(ot,e,o),a(e,ka,o),a(e,Q,o),s(Q,Qi),s(Q,Bs),s(Bs,qi),s(Q,Pi),s(Q,at),s(at,Li),s(Q,Di),a(e,Ta,o),a(e,ss,o),s(ss,Oi),Ma=!0},p(e,[o]){const rt={};o&2&&(rt.$$scope={dirty:o,ctx:e}),O.$set(rt);const js={};o&2&&(js.$$scope={dirty:o,ctx:e}),ee.$set(js);const Zs={};o&2&&(Zs.$$scope={dirty:o,ctx:e}),te.$set(Zs);const Vs={};o&2&&(Vs.$$scope={dirty:o,ctx:e}),we.$set(Vs);const xs={};o&2&&(xs.$$scope={dirty:o,ctx:e}),ge.$set(xs)},i(e){Ma||(T(_.$$.fragment,e),T(Z.$$.fragment,e),T(O.$$.fragment,e),T(Ue.$$.fragment,e),T(Ie.$$.fragment,e),T(ee.$$.fragment,e),T(Be.$$.fragment,e),T(je.$$.fragment,e),T(te.$$.fragment,e),T(Ze.$$.fragment,e),T(Ve.$$.fragment,e),T(xe.$$.fragment,e),T(Ce.$$.fragment,e),T(ze.$$.fragment,e),T(Fe.$$.fragment,e),T(He.$$.fragment,e),T(Ye.$$.fragment,e),T(Qe.$$.fragment,e),T(qe.$$.fragment,e),T(Pe.$$.fragment,e),T(Le.$$.fragment,e),T(De.$$.fragment,e),T(we.$$.fragment,e),T(Oe.$$.fragment,e),T(ge.$$.fragment,e),T(et.$$.fragment,e),T(tt.$$.fragment,e),T(st.$$.fragment,e),T(ot.$$.fragment,e),Ma=!0)},o(e){M(_.$$.fragment,e),M(Z.$$.fragment,e),M(O.$$.fragment,e),M(Ue.$$.fragment,e),M(Ie.$$.fragment,e),M(ee.$$.fragment,e),M(Be.$$.fragment,e),M(je.$$.fragment,e),M(te.$$.fragment,e),M(Ze.$$.fragment,e),M(Ve.$$.fragment,e),M(xe.$$.fragment,e),M(Ce.$$.fragment,e),M(ze.$$.fragment,e),M(Fe.$$.fragment,e),M(He.$$.fragment,e),M(Ye.$$.fragment,e),M(Qe.$$.fragment,e),M(qe.$$.fragment,e),M(Pe.$$.fragment,e),M(Le.$$.fragment,e),M(De.$$.fragment,e),M(we.$$.fragment,e),M(Oe.$$.fragment,e),M(ge.$$.fragment,e),M(et.$$.fragment,e),M(tt.$$.fragment,e),M(st.$$.fragment,e),M(ot.$$.fragment,e),Ma=!1},d(e){t(u),e&&t(w),e&&t(m),$(_),e&&t(x),e&&t(U),e&&t(N),e&&t(v),$(Z),e&&t(Me),e&&t($e),e&&t(Ws),e&&t(it),e&&t(Rs),e&&t(X),e&&t(zs),e&&t(lt),e&&t(Xs),e&&t(Je),e&&t(Fs),e&&t(pt),e&&t(As),e&&t(ct),e&&t(Hs),e&&t(dt),e&&t(Ys),$(O,e),e&&t(Qs),e&&t(ht),e&&t(qs),e&&t(F),e&&t(Ps),e&&t(ut),e&&t(Ls),e&&t(Ee),e&&t(Ds),e&&t(K),e&&t(Os),e&&t(yt),e&&t(Ks),e&&t(wt),e&&t(eo),$(Ue,e),e&&t(to),e&&t(vt),e&&t(so),$(Ie,e),e&&t(oo),e&&t(bt),e&&t(ao),$(ee,e),e&&t(ro),e&&t(A),e&&t(io),$(Be,e),e&&t(lo),e&&t(gt),e&&t(no),e&&t(kt),e&&t(po),$(je,e),e&&t(co),e&&t(Tt),e&&t(ho),e&&t(Mt),e&&t(uo),$(te,e),e&&t(fo),e&&t($t),e&&t(mo),$(Ze,e),e&&t(yo),e&&t(_t),e&&t(wo),$(Ve,e),e&&t(vo),e&&t(Jt),e&&t(bo),$(xe,e),e&&t(go),e&&t(Et),e&&t(ko),e&&t(Ut),e&&t(To),e&&t(H),e&&t(Mo),e&&t(L),$(Ce),e&&t($o),e&&t(oe),e&&t(_o),e&&t(ae),e&&t(Jo),e&&t(re),e&&t(Eo),e&&t(It),e&&t(Uo),e&&t(W),e&&t(Io),e&&t(Bt),e&&t(So),e&&t(jt),e&&t(Go),e&&t(Zt),e&&t(Bo),e&&t(Vt),e&&t(jo),e&&t(le),e&&t(Zo),$(ze,e),e&&t(Vo),e&&t(ne),e&&t(xo),$(Fe,e),e&&t(Co),e&&t(pe),e&&t(No),e&&t(ce),e&&t(Wo),$(He,e),e&&t(Ro),e&&t(de),e&&t(zo),e&&t(V),e&&t(Xo),$(Ye,e),e&&t(Fo),e&&t(he),e&&t(Ao),e&&t(Wt),e&&t(Ho),$(Qe,e),e&&t(Yo),e&&t(ue),e&&t(Qo),e&&t(zt),e&&t(qo),e&&t(Xt),e&&t(Po),$(qe,e),e&&t(Lo),e&&t(Ft),e&&t(Do),e&&t(At),e&&t(Oo),$(Pe,e),e&&t(Ko),e&&t(fe),e&&t(ea),e&&t(Yt),e&&t(ta),$(Le,e),e&&t(sa),e&&t(me),e&&t(oa),e&&t(qt),e&&t(aa),$(De,e),e&&t(ra),e&&t(ye),e&&t(ia),$(we,e),e&&t(la),e&&t(D),$(Oe),e&&t(na),e&&t(Lt),e&&t(pa),e&&t(be),e&&t(ca),e&&t(Dt),e&&t(da),$(ge,e),e&&t(ha),e&&t(Ot),e&&t(ua),$(et,e),e&&t(fa),e&&t(Y),e&&t(ma),e&&t(Kt),e&&t(ya),$(tt,e),e&&t(wa),e&&t(es),e&&t(va),$(st,e),e&&t(ba),e&&t(ts),e&&t(ga),$(ot,e),e&&t(ka),e&&t(Q),e&&t(Ta),e&&t(ss)}}}const ip={local:"pretrained-models-for-texttospeech",sections:[{local:"speecht5",title:"SpeechT5 "},{local:"bark",title:"Bark"},{local:"massive-multilingual-speech-mms",title:"Massive Multilingual Speech (MMS)"}],title:"Pre-trained models for text-to-speech"};function lp(C){return On(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class hp extends qn{constructor(u){super();Pn(this,u,lp,rp,Ln,{})}}export{hp as default,ip as metadata};
