import{S as gu,i as bu,s as Mu,e as n,k as c,w as d,t as o,M as vu,c as i,d as t,m as p,a as h,x as f,h as l,b as u,N as Rt,G as s,g as r,y as m,q as w,o as y,B as g,v as ku}from"../../chunks/vendor-hf-doc-builder.js";import{T as Qh}from"../../chunks/Tip-hf-doc-builder.js";import{I as N}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as T}from"../../chunks/CodeBlock-hf-doc-builder.js";function Tu(se){let b;return{c(){b=o(`The following section requires the use of a microphone to record a voice input. Since Google Colab machines do not
    have microphone compatibility, it is recommended to run this section locally, either on your CPU, or on a GPU if you
    have local access. The checkpoint sizes have been selected as those small enough to run adequately fast on CPU, so
    you will still get good performance without a GPU.`)},l(M){b=l(M,`The following section requires the use of a microphone to record a voice input. Since Google Colab machines do not
    have microphone compatibility, it is recommended to run this section locally, either on your CPU, or on a GPU if you
    have local access. The checkpoint sizes have been selected as those small enough to run adequately fast on CPU, so
    you will still get good performance without a GPU.`)},m(M,U){r(M,b,U)},d(M){M&&t(b)}}}function Ju(se){let b,M,U,E,I,S,ae;return{c(){b=o("If you're using a GPU, you can increase the checkpoint size to use the "),M=n("a"),U=o("Whisper Small English"),E=o(` 
    checkpoint, which will return better transcription accuracy and still be within the required latency threshold. Simply swap the 
    model id to: `),I=n("code"),S=o('"openai/whisper-small.en"'),ae=o("."),this.h()},l(v){b=l(v,"If you're using a GPU, you can increase the checkpoint size to use the "),M=i(v,"A",{href:!0});var $=h(M);U=l($,"Whisper Small English"),$.forEach(t),E=l(v,` 
    checkpoint, which will return better transcription accuracy and still be within the required latency threshold. Simply swap the 
    model id to: `),I=i(v,"CODE",{});var xe=h(I);S=l(xe,'"openai/whisper-small.en"'),xe.forEach(t),ae=l(v,"."),this.h()},h(){u(M,"href","https://huggingface.co/openai/whisper-small.en")},m(v,$){r(v,b,$),r(v,M,$),s(M,U),r(v,E,$),r(v,I,$),s(I,S),r(v,ae,$)},d(v){v&&t(b),v&&t(M),v&&t(E),v&&t(I),v&&t(ae)}}}function Uu(se){let b;return{c(){b=o(`Note that the first time calling this will trigger the model weights to be downloaded, which might take 
    some time depending on your Hub download speed.`)},l(M){b=l(M,`Note that the first time calling this will trigger the model weights to be downloaded, which might take 
    some time depending on your Hub download speed.`)},m(M,U){r(M,b,U)},d(M){M&&t(b)}}}function _u(se){let b,M,U,E,I,S,ae,v,$,xe,oe,cn,Js,pn,un,Qa,Xt,dn,Ra,He,Us,Rh,Xa,R,le,_s,Ve,fn,js,mn,Fa,Ft,wn,za,zt,yn,Pa,X,ne,Es,qe,gn,Is,bn,Da,Pt,Mn,Oa,Dt,vn,Ka,F,ie,$s,Ye,kn,Ws,Tn,eo,re,Jn,Cs,Un,_n,to,Ot,jn,so,z,he,As,Le,En,Zs,In,ao,Kt,$n,oo,es,Wn,lo,ce,no,P,pe,Ss,Qe,Cn,Ns,An,io,k,Zn,ts,Sn,Nn,Re,Bn,Gn,Bs,xn,Hn,Gs,Vn,qn,xs,Yn,Ln,Hs,Qn,Rn,Vs,Xn,Fn,Xe,zn,Pn,ro,ss,Dn,ho,Fe,On,ze,Kn,co,as,ei,po,Pe,qs,Xh,uo,os,ti,fo,De,Ys,Fh,mo,ue,si,Ls,ai,oi,wo,Oe,Qs,zh,yo,de,li,Ke,ni,ii,go,fe,ri,Rs,hi,ci,bo,et,Mo,me,pi,Xs,ui,di,vo,tt,ko,_,fi,Fs,mi,wi,zs,yi,gi,Ps,bi,Mi,Ds,vi,ki,To,st,Jo,at,Uo,we,Ti,Os,Ji,Ui,_o,ye,_i,ot,Ks,ji,Ei,jo,j,Ii,ea,$i,Wi,ta,Ci,Ai,sa,Zi,Si,aa,Ni,Bi,Eo,W,Gi,oa,xi,Hi,la,Vi,qi,na,Yi,Li,Io,B,Qi,ia,Ri,Xi,lt,ra,Fi,zi,$o,nt,Wo,C,Pi,ha,Di,Oi,ca,Ki,er,pa,tr,sr,Co,it,Ao,rt,Zo,ge,ar,ua,or,lr,So,D,be,da,ht,nr,fa,ir,No,G,rr,ct,hr,cr,pt,pr,ur,Bo,ut,Go,Me,xo,x,dr,ma,fr,mr,wa,wr,yr,Ho,A,gr,ya,br,Mr,ga,vr,kr,dt,Tr,Jr,Vo,ft,qo,ls,Ur,Yo,mt,Lo,wt,Qo,Z,_r,ba,jr,Er,Ma,Ir,$r,va,Wr,Cr,Ro,O,ve,ka,yt,Ar,Ta,Zr,Xo,ke,Sr,gt,Nr,Br,Fo,Te,Gr,bt,xr,Hr,zo,Mt,Ja,Ph,Po,H,Vr,vt,qr,Yr,kt,Lr,Qr,Do,Tt,Ua,Dh,Oo,Je,Rr,_a,Xr,Fr,Ko,Jt,el,ns,zr,tl,Ut,sl,_t,al,is,Pr,ol,K,Ue,ja,jt,Dr,Ea,Or,ll,_e,Kr,Et,eh,th,nl,It,il,rs,sh,rl,$t,hl,V,ah,Ia,oh,lh,hs,nh,ih,cl,Wt,pl,cs,rh,ul,Ct,dl,ps,hh,fl,ee,je,$a,At,ch,Wa,ph,ml,Ee,uh,Ca,dh,fh,wl,Zt,yl,us,mh,gl,q,Aa,Za,wh,yh,Sa,Na,gh,bh,Ba,Ga,Mh,bl,ds,vh,Ml,te,Ie,xa,St,kh,Ha,Th,vl,fs,Jh,kl,$e,Uh,Nt,_h,jh,Tl,Y,Eh,Bt,Ih,$h,Gt,Wh,Ch,Jl,xt,Ul,We,Ah,Va,Zh,Sh,_l,Ht,jl,Vt,qa,Oh,El,Ce,Il,Ae,Nh,qt,Bh,Gh,$l,ms,xh,Wl,Yt,Cl,ws,Hh,Al,Ze,Vh,Lt,qh,Yh,Zl;return S=new N({}),Ve=new N({}),qe=new N({}),Ye=new N({}),Le=new N({}),ce=new Qh({props:{$$slots:{default:[Tu]},$$scope:{ctx:se}}}),Qe=new N({}),et=new T({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMHBpcGVsaW5lJTBBaW1wb3J0JTIwdG9yY2glMEElMEFkZXZpY2UlMjAlM0QlMjAlMjJjdWRhJTNBMCUyMiUyMGlmJTIwdG9yY2guY3VkYS5pc19hdmFpbGFibGUoKSUyMGVsc2UlMjAlMjJjcHUlMjIlMEElMEFjbGFzc2lmaWVyJTIwJTNEJTIwcGlwZWxpbmUoJTBBJTIwJTIwJTIwJTIwJTIyYXVkaW8tY2xhc3NpZmljYXRpb24lMjIlMkMlMjBtb2RlbCUzRCUyMk1JVCUyRmFzdC1maW5ldHVuZWQtc3BlZWNoLWNvbW1hbmRzLXYyJTIyJTJDJTIwZGV2aWNlJTNEZGV2aWNlJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline
<span class="hljs-keyword">import</span> torch

device = <span class="hljs-string">&quot;cuda:0&quot;</span> <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> <span class="hljs-string">&quot;cpu&quot;</span>

classifier = pipeline(
    <span class="hljs-string">&quot;audio-classification&quot;</span>, model=<span class="hljs-string">&quot;MIT/ast-finetuned-speech-commands-v2&quot;</span>, device=device
)`}}),tt=new T({props:{code:"Y2xhc3NpZmllci5tb2RlbC5jb25maWcuaWQybGFiZWw=",highlighted:"classifier.model.config.id2label"}}),st=new T({props:{code:"Y2xhc3NpZmllci5tb2RlbC5jb25maWcuaWQybGFiZWwlNUIyNyU1RA==",highlighted:'classifier.model.config.id2label[<span class="hljs-number">27</span>]'}}),at=new T({props:{code:"J21hcnZpbic=",highlighted:'<span class="hljs-symbol">&#x27;marvin</span>&#x27;'}}),nt=new T({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5waXBlbGluZXMuYXVkaW9fdXRpbHMlMjBpbXBvcnQlMjBmZm1wZWdfbWljcm9waG9uZV9saXZlJTBBJTBBJTBBZGVmJTIwbGF1bmNoX2ZuKCUwQSUyMCUyMCUyMCUyMHdha2Vfd29yZCUzRCUyMm1hcnZpbiUyMiUyQyUwQSUyMCUyMCUyMCUyMHByb2JfdGhyZXNob2xkJTNEMC41JTJDJTBBJTIwJTIwJTIwJTIwY2h1bmtfbGVuZ3RoX3MlM0QyLjAlMkMlMEElMjAlMjAlMjAlMjBzdHJlYW1fY2h1bmtfcyUzRDAuMjUlMkMlMEElMjAlMjAlMjAlMjBkZWJ1ZyUzREZhbHNlJTJDJTBBKSUzQSUwQSUyMCUyMCUyMCUyMGlmJTIwd2FrZV93b3JkJTIwbm90JTIwaW4lMjBjbGFzc2lmaWVyLm1vZGVsLmNvbmZpZy5sYWJlbDJpZC5rZXlzKCklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByYWlzZSUyMFZhbHVlRXJyb3IoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZiUyMldha2UlMjB3b3JkJTIwJTdCd2FrZV93b3JkJTdEJTIwbm90JTIwaW4lMjBzZXQlMjBvZiUyMHZhbGlkJTIwY2xhc3MlMjBsYWJlbHMlMkMlMjBwaWNrJTIwYSUyMHdha2UlMjB3b3JkJTIwaW4lMjB0aGUlMjBzZXQlMjAlN0JjbGFzc2lmaWVyLm1vZGVsLmNvbmZpZy5sYWJlbDJpZC5rZXlzKCklN0QuJTIyJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKSUwQSUwQSUyMCUyMCUyMCUyMHNhbXBsaW5nX3JhdGUlMjAlM0QlMjBjbGFzc2lmaWVyLmZlYXR1cmVfZXh0cmFjdG9yLnNhbXBsaW5nX3JhdGUlMEElMEElMjAlMjAlMjAlMjBtaWMlMjAlM0QlMjBmZm1wZWdfbWljcm9waG9uZV9saXZlKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHNhbXBsaW5nX3JhdGUlM0RzYW1wbGluZ19yYXRlJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY2h1bmtfbGVuZ3RoX3MlM0RjaHVua19sZW5ndGhfcyUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHN0cmVhbV9jaHVua19zJTNEc3RyZWFtX2NodW5rX3MlMkMlMEElMjAlMjAlMjAlMjApJTBBJTBBJTIwJTIwJTIwJTIwcHJpbnQoJTIyTGlzdGVuaW5nJTIwZm9yJTIwd2FrZSUyMHdvcmQuLi4lMjIpJTBBJTIwJTIwJTIwJTIwZm9yJTIwcHJlZGljdGlvbiUyMGluJTIwY2xhc3NpZmllcihtaWMpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcHJlZGljdGlvbiUyMCUzRCUyMHByZWRpY3Rpb24lNUIwJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWYlMjBkZWJ1ZyUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHByaW50KHByZWRpY3Rpb24pJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWYlMjBwcmVkaWN0aW9uJTVCJTIybGFiZWwlMjIlNUQlMjAlM0QlM0QlMjB3YWtlX3dvcmQlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZiUyMHByZWRpY3Rpb24lNUIlMjJzY29yZSUyMiU1RCUyMCUzRSUyMHByb2JfdGhyZXNob2xkJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV0dXJuJTIwVHJ1ZQ==",highlighted:`<span class="hljs-keyword">from</span> transformers.pipelines.audio_utils <span class="hljs-keyword">import</span> ffmpeg_microphone_live


<span class="hljs-keyword">def</span> <span class="hljs-title function_">launch_fn</span>(<span class="hljs-params">
    wake_word=<span class="hljs-string">&quot;marvin&quot;</span>,
    prob_threshold=<span class="hljs-number">0.5</span>,
    chunk_length_s=<span class="hljs-number">2.0</span>,
    stream_chunk_s=<span class="hljs-number">0.25</span>,
    debug=<span class="hljs-literal">False</span>,
</span>):
    <span class="hljs-keyword">if</span> wake_word <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> classifier.model.config.label2id.keys():
        <span class="hljs-keyword">raise</span> ValueError(
            <span class="hljs-string">f&quot;Wake word <span class="hljs-subst">{wake_word}</span> not in set of valid class labels, pick a wake word in the set <span class="hljs-subst">{classifier.model.config.label2id.keys()}</span>.&quot;</span>
        )

    sampling_rate = classifier.feature_extractor.sampling_rate

    mic = ffmpeg_microphone_live(
        sampling_rate=sampling_rate,
        chunk_length_s=chunk_length_s,
        stream_chunk_s=stream_chunk_s,
    )

    <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Listening for wake word...&quot;</span>)
    <span class="hljs-keyword">for</span> prediction <span class="hljs-keyword">in</span> classifier(mic):
        prediction = prediction[<span class="hljs-number">0</span>]
        <span class="hljs-keyword">if</span> debug:
            <span class="hljs-built_in">print</span>(prediction)
        <span class="hljs-keyword">if</span> prediction[<span class="hljs-string">&quot;label&quot;</span>] == wake_word:
            <span class="hljs-keyword">if</span> prediction[<span class="hljs-string">&quot;score&quot;</span>] &gt; prob_threshold:
                <span class="hljs-keyword">return</span> <span class="hljs-literal">True</span>`}}),it=new T({props:{code:"bGF1bmNoX2ZuKGRlYnVnJTNEVHJ1ZSk=",highlighted:'launch_fn(debug=<span class="hljs-literal">True</span>)'}}),rt=new T({props:{code:"TGlzdGVuaW5nJTIwZm9yJTIwd2FrZSUyMHdvcmQuLi4lMEElN0Inc2NvcmUnJTNBJTIwMC4wNTUzMjYxMDc4ODk0MTM4MzQlMkMlMjAnbGFiZWwnJTNBJTIwJ29uZSclN0QlMEElN0Inc2NvcmUnJTNBJTIwMC4wNTk5OTg1NjgxNDc0MjA4OCUyQyUyMCdsYWJlbCclM0ElMjAnb2ZmJyU3RCUwQSU3QidzY29yZSclM0ElMjAwLjEyODI3NDg0MzA5NjczMzElMkMlMjAnbGFiZWwnJTNBJTIwJ2ZpdmUnJTdEJTBBJTdCJ3Njb3JlJyUzQSUyMDAuMDczMTAxMTAzMzA1ODE2NjUlMkMlMjAnbGFiZWwnJTNBJTIwJ2ZvbGxvdyclN0QlMEElN0Inc2NvcmUnJTNBJTIwMC4wNjYzNDgwOTgyMTg0NDEwMSUyQyUyMCdsYWJlbCclM0ElMjAnZm9sbG93JyU3RCUwQSU3QidzY29yZSclM0ElMjAwLjA1OTkyNjQyNDE3NTUwMDg3JTJDJTIwJ2xhYmVsJyUzQSUyMCd0cmVlJyU3RCUwQSU3QidzY29yZSclM0ElMjAwLjA1OTkyNjQyNDE3NTUwMDg3JTJDJTIwJ2xhYmVsJyUzQSUyMCd0cmVlJyU3RCUwQSU3QidzY29yZSclM0ElMjAwLjk5OTkxMzIxNTYzNzIwNyUyQyUyMCdsYWJlbCclM0ElMjAnbWFydmluJyU3RA==",highlighted:`Listening for wake word...
{&#x27;score&#x27;: 0.055326107889413834, &#x27;label&#x27;: &#x27;one&#x27;}
{&#x27;score&#x27;: 0.05999856814742088, &#x27;label&#x27;: &#x27;off&#x27;}
{&#x27;score&#x27;: 0.1282748430967331, &#x27;label&#x27;: &#x27;five&#x27;}
{&#x27;score&#x27;: 0.07310110330581665, &#x27;label&#x27;: &#x27;follow&#x27;}
{&#x27;score&#x27;: 0.06634809821844101, &#x27;label&#x27;: &#x27;follow&#x27;}
{&#x27;score&#x27;: 0.05992642417550087, &#x27;label&#x27;: &#x27;tree&#x27;}
{&#x27;score&#x27;: 0.05992642417550087, &#x27;label&#x27;: &#x27;tree&#x27;}
{&#x27;score&#x27;: 0.999913215637207, &#x27;label&#x27;: &#x27;marvin&#x27;}`}}),ht=new N({}),ut=new T({props:{code:"dHJhbnNjcmliZXIlMjAlM0QlMjBwaXBlbGluZSglMEElMjAlMjAlMjAlMjAlMjJhdXRvbWF0aWMtc3BlZWNoLXJlY29nbml0aW9uJTIyJTJDJTIwbW9kZWwlM0QlMjJvcGVuYWklMkZ3aGlzcGVyLWJhc2UuZW4lMjIlMkMlMjBkZXZpY2UlM0RkZXZpY2UlMEEp",highlighted:`transcriber = pipeline(
    <span class="hljs-string">&quot;automatic-speech-recognition&quot;</span>, model=<span class="hljs-string">&quot;openai/whisper-base.en&quot;</span>, device=device
)`}}),Me=new Qh({props:{$$slots:{default:[Ju]},$$scope:{ctx:se}}}),ft=new T({props:{code:"aW1wb3J0JTIwc3lzJTBBJTBBJTBBZGVmJTIwdHJhbnNjcmliZShjaHVua19sZW5ndGhfcyUzRDUuMCUyQyUyMHN0cmVhbV9jaHVua19zJTNEMS4wKSUzQSUwQSUyMCUyMCUyMCUyMHNhbXBsaW5nX3JhdGUlMjAlM0QlMjB0cmFuc2NyaWJlci5mZWF0dXJlX2V4dHJhY3Rvci5zYW1wbGluZ19yYXRlJTBBJTBBJTIwJTIwJTIwJTIwbWljJTIwJTNEJTIwZmZtcGVnX21pY3JvcGhvbmVfbGl2ZSglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzYW1wbGluZ19yYXRlJTNEc2FtcGxpbmdfcmF0ZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGNodW5rX2xlbmd0aF9zJTNEY2h1bmtfbGVuZ3RoX3MlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzdHJlYW1fY2h1bmtfcyUzRHN0cmVhbV9jaHVua19zJTJDJTBBJTIwJTIwJTIwJTIwKSUwQSUwQSUyMCUyMCUyMCUyMHByaW50KCUyMlN0YXJ0JTIwc3BlYWtpbmcuLi4lMjIpJTBBJTIwJTIwJTIwJTIwZm9yJTIwaXRlbSUyMGluJTIwdHJhbnNjcmliZXIobWljJTJDJTIwZ2VuZXJhdGVfa3dhcmdzJTNEJTdCJTIybWF4X25ld190b2tlbnMlMjIlM0ElMjAxMjglN0QpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc3lzLnN0ZG91dC53cml0ZSglMjIlNUMwMzMlNUJLJTIyKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHByaW50KGl0ZW0lNUIlMjJ0ZXh0JTIyJTVEJTJDJTIwZW5kJTNEJTIyJTVDciUyMiklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZiUyMG5vdCUyMGl0ZW0lNUIlMjJwYXJ0aWFsJTIyJTVEJTVCMCU1RCUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGJyZWFrJTBBJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwaXRlbSU1QiUyMnRleHQlMjIlNUQ=",highlighted:`<span class="hljs-keyword">import</span> sys


<span class="hljs-keyword">def</span> <span class="hljs-title function_">transcribe</span>(<span class="hljs-params">chunk_length_s=<span class="hljs-number">5.0</span>, stream_chunk_s=<span class="hljs-number">1.0</span></span>):
    sampling_rate = transcriber.feature_extractor.sampling_rate

    mic = ffmpeg_microphone_live(
        sampling_rate=sampling_rate,
        chunk_length_s=chunk_length_s,
        stream_chunk_s=stream_chunk_s,
    )

    <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Start speaking...&quot;</span>)
    <span class="hljs-keyword">for</span> item <span class="hljs-keyword">in</span> transcriber(mic, generate_kwargs={<span class="hljs-string">&quot;max_new_tokens&quot;</span>: <span class="hljs-number">128</span>}):
        sys.stdout.write(<span class="hljs-string">&quot;\\033[K&quot;</span>)
        <span class="hljs-built_in">print</span>(item[<span class="hljs-string">&quot;text&quot;</span>], end=<span class="hljs-string">&quot;\\r&quot;</span>)
        <span class="hljs-keyword">if</span> <span class="hljs-keyword">not</span> item[<span class="hljs-string">&quot;partial&quot;</span>][<span class="hljs-number">0</span>]:
            <span class="hljs-keyword">break</span>

    <span class="hljs-keyword">return</span> item[<span class="hljs-string">&quot;text&quot;</span>]`}}),mt=new T({props:{code:"dHJhbnNjcmliZSgp",highlighted:"transcribe()"}}),wt=new T({props:{code:"U3RhcnQlMjBzcGVha2luZy4uLiUwQSUyMEhleSUyQyUyMHRoaXMlMjBpcyUyMGElMjB0ZXN0JTIwd2l0aCUyMHRoZSUyMHdoaXNwZXIlMjBtb2RlbC4=",highlighted:`Start speaking...
 Hey, this is a test with the whisper model.`}}),yt=new N({}),Jt=new T({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMEhmRm9sZGVyJTBBaW1wb3J0JTIwcmVxdWVzdHMlMEElMEElMEFkZWYlMjBxdWVyeSh0ZXh0JTJDJTIwbW9kZWxfaWQlM0QlMjJ0aWl1YWUlMkZmYWxjb24tN2ItaW5zdHJ1Y3QlMjIpJTNBJTBBJTIwJTIwJTIwJTIwYXBpX3VybCUyMCUzRCUyMGYlMjJodHRwcyUzQSUyRiUyRmFwaS1pbmZlcmVuY2UuaHVnZ2luZ2ZhY2UuY28lMkZtb2RlbHMlMkYlN0Jtb2RlbF9pZCU3RCUyMiUwQSUyMCUyMCUyMCUyMGhlYWRlcnMlMjAlM0QlMjAlN0IlMjJBdXRob3JpemF0aW9uJTIyJTNBJTIwZiUyMkJlYXJlciUyMCU3QkhmRm9sZGVyKCkuZ2V0X3Rva2VuKCklN0QlMjIlN0QlMEElMjAlMjAlMjAlMjBwYXlsb2FkJTIwJTNEJTIwJTdCJTIyaW5wdXRzJTIyJTNBJTIwdGV4dCU3RCUwQSUwQSUyMCUyMCUyMCUyMHByaW50KGYlMjJRdWVyeWluZy4uLiUzQSUyMCU3QnRleHQlN0QlMjIpJTBBJTIwJTIwJTIwJTIwcmVzcG9uc2UlMjAlM0QlMjByZXF1ZXN0cy5wb3N0KGFwaV91cmwlMkMlMjBoZWFkZXJzJTNEaGVhZGVycyUyQyUyMGpzb24lM0RwYXlsb2FkKSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMHJlc3BvbnNlLmpzb24oKSU1QjAlNUQlNUIlMjJnZW5lcmF0ZWRfdGV4dCUyMiU1RCU1Qmxlbih0ZXh0KSUyMCUyQiUyMDElMjAlM0ElNUQ=",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> HfFolder
<span class="hljs-keyword">import</span> requests


<span class="hljs-keyword">def</span> <span class="hljs-title function_">query</span>(<span class="hljs-params">text, model_id=<span class="hljs-string">&quot;tiiuae/falcon-7b-instruct&quot;</span></span>):
    api_url = <span class="hljs-string">f&quot;https://api-inference.huggingface.co/models/<span class="hljs-subst">{model_id}</span>&quot;</span>
    headers = {<span class="hljs-string">&quot;Authorization&quot;</span>: <span class="hljs-string">f&quot;Bearer <span class="hljs-subst">{HfFolder().get_token()}</span>&quot;</span>}
    payload = {<span class="hljs-string">&quot;inputs&quot;</span>: text}

    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Querying...: <span class="hljs-subst">{text}</span>&quot;</span>)
    response = requests.post(api_url, headers=headers, json=payload)
    <span class="hljs-keyword">return</span> response.json()[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;generated_text&quot;</span>][<span class="hljs-built_in">len</span>(text) + <span class="hljs-number">1</span> :]`}}),Ut=new T({props:{code:"cXVlcnkoJTIyV2hhdCUyMGRvZXMlMjBIdWdnaW5nJTIwRmFjZSUyMGRvJTNGJTIyKQ==",highlighted:'query(<span class="hljs-string">&quot;What does Hugging Face do?&quot;</span>)'}}),_t=new T({props:{code:"J0h1Z2dpbmclMjBGYWNlJTIwaXMlMjBhJTIwY29tcGFueSUyMHRoYXQlMjBwcm92aWRlcyUyMG5hdHVyYWwlMjBsYW5ndWFnZSUyMHByb2Nlc3NpbmclMjBhbmQlMjBtYWNoaW5lJTIwbGVhcm5pbmclMjB0b29scyUyMGZvciUyMGRldmVsb3BlcnMuJTIwVGhleSc=",highlighted:'<span class="hljs-comment">&#x27;Hugging Face is a company that provides natural language processing and machine learning tools for developers. They&#x27;</span>'}}),jt=new N({}),It=new T({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFNwZWVjaFQ1UHJvY2Vzc29yJTJDJTIwU3BlZWNoVDVGb3JUZXh0VG9TcGVlY2glMkMlMjBTcGVlY2hUNUhpZmlHYW4lMEElMEFwcm9jZXNzb3IlMjAlM0QlMjBTcGVlY2hUNVByb2Nlc3Nvci5mcm9tX3ByZXRyYWluZWQoJTIybWljcm9zb2Z0JTJGc3BlZWNodDVfdHRzJTIyKSUwQSUwQW1vZGVsJTIwJTNEJTIwU3BlZWNoVDVGb3JUZXh0VG9TcGVlY2guZnJvbV9wcmV0cmFpbmVkKCUyMm1pY3Jvc29mdCUyRnNwZWVjaHQ1X3R0cyUyMikudG8oZGV2aWNlKSUwQXZvY29kZXIlMjAlM0QlMjBTcGVlY2hUNUhpZmlHYW4uZnJvbV9wcmV0cmFpbmVkKCUyMm1pY3Jvc29mdCUyRnNwZWVjaHQ1X2hpZmlnYW4lMjIpLnRvKGRldmljZSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> SpeechT5Processor, SpeechT5ForTextToSpeech, SpeechT5HifiGan

processor = SpeechT5Processor.from_pretrained(<span class="hljs-string">&quot;microsoft/speecht5_tts&quot;</span>)

model = SpeechT5ForTextToSpeech.from_pretrained(<span class="hljs-string">&quot;microsoft/speecht5_tts&quot;</span>).to(device)
vocoder = SpeechT5HifiGan.from_pretrained(<span class="hljs-string">&quot;microsoft/speecht5_hifigan&quot;</span>).to(device)`}}),$t=new T({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBZW1iZWRkaW5nc19kYXRhc2V0JTIwJTNEJTIwbG9hZF9kYXRhc2V0KCUyMk1hdHRoaWpzJTJGY211LWFyY3RpYy14dmVjdG9ycyUyMiUyQyUyMHNwbGl0JTNEJTIydmFsaWRhdGlvbiUyMiklMEFzcGVha2VyX2VtYmVkZGluZ3MlMjAlM0QlMjB0b3JjaC50ZW5zb3IoZW1iZWRkaW5nc19kYXRhc2V0JTVCNzMwNiU1RCU1QiUyMnh2ZWN0b3IlMjIlNUQpLnVuc3F1ZWV6ZSgwKQ==",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

embeddings_dataset = load_dataset(<span class="hljs-string">&quot;Matthijs/cmu-arctic-xvectors&quot;</span>, split=<span class="hljs-string">&quot;validation&quot;</span>)
speaker_embeddings = torch.tensor(embeddings_dataset[<span class="hljs-number">7306</span>][<span class="hljs-string">&quot;xvector&quot;</span>]).unsqueeze(<span class="hljs-number">0</span>)`}}),Wt=new T({props:{code:"ZGVmJTIwc3ludGhlc2lzZSh0ZXh0KSUzQSUwQSUyMCUyMCUyMCUyMGlucHV0cyUyMCUzRCUyMHByb2Nlc3Nvcih0ZXh0JTNEdGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTBBJTIwJTIwJTIwJTIwc3BlZWNoJTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGVfc3BlZWNoKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlucHV0cyU1QiUyMmlucHV0X2lkcyUyMiU1RC50byhkZXZpY2UpJTJDJTIwc3BlYWtlcl9lbWJlZGRpbmdzLnRvKGRldmljZSklMkMlMjB2b2NvZGVyJTNEdm9jb2RlciUwQSUyMCUyMCUyMCUyMCklMEElMjAlMjAlMjAlMjByZXR1cm4lMjBzcGVlY2guY3B1KCk=",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">synthesise</span>(<span class="hljs-params">text</span>):
    inputs = processor(text=text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
    speech = model.generate_speech(
        inputs[<span class="hljs-string">&quot;input_ids&quot;</span>].to(device), speaker_embeddings.to(device), vocoder=vocoder
    )
    <span class="hljs-keyword">return</span> speech.cpu()`}}),Ct=new T({props:{code:"ZnJvbSUyMElQeXRob24uZGlzcGxheSUyMGltcG9ydCUyMEF1ZGlvJTBBJTBBYXVkaW8lMjAlM0QlMjBzeW50aGVzaXNlKCUwQSUyMCUyMCUyMCUyMCUyMkh1Z2dpbmclMjBGYWNlJTIwaXMlMjBhJTIwY29tcGFueSUyMHRoYXQlMjBwcm92aWRlcyUyMG5hdHVyYWwlMjBsYW5ndWFnZSUyMHByb2Nlc3NpbmclMjBhbmQlMjBtYWNoaW5lJTIwbGVhcm5pbmclMjB0b29scyUyMGZvciUyMGRldmVsb3BlcnMuJTIyJTBBKSUwQSUwQUF1ZGlvKGF1ZGlvJTJDJTIwcmF0ZSUzRDE2MDAwKQ==",highlighted:`<span class="hljs-keyword">from</span> IPython.display <span class="hljs-keyword">import</span> Audio

audio = synthesise(
    <span class="hljs-string">&quot;Hugging Face is a company that provides natural language processing and machine learning tools for developers.&quot;</span>
)

Audio(audio, rate=<span class="hljs-number">16000</span>)`}}),At=new N({}),Zt=new T({props:{code:"bGF1bmNoX2ZuKCklMEF0cmFuc2NyaXB0aW9uJTIwJTNEJTIwdHJhbnNjcmliZSgpJTBBcmVzcG9uc2UlMjAlM0QlMjBxdWVyeSh0cmFuc2NyaXB0aW9uKSUwQWF1ZGlvJTIwJTNEJTIwc3ludGhlc2lzZShyZXNwb25zZSklMEElMEFBdWRpbyhhdWRpbyUyQyUyMHJhdGUlM0QxNjAwMCUyQyUyMGF1dG9wbGF5JTNEVHJ1ZSk=",highlighted:`launch_fn()
transcription = transcribe()
response = query(transcription)
audio = synthesise(response)

Audio(audio, rate=<span class="hljs-number">16000</span>, autoplay=<span class="hljs-literal">True</span>)`}}),St=new N({}),xt=new T({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEhmQWdlbnQlMEElMEFhZ2VudCUyMCUzRCUyMEhmQWdlbnQoJTBBJTIwJTIwJTIwJTIwdXJsX2VuZHBvaW50JTNEJTIyaHR0cHMlM0ElMkYlMkZhcGktaW5mZXJlbmNlLmh1Z2dpbmdmYWNlLmNvJTJGbW9kZWxzJTJGYmlnY29kZSUyRnN0YXJjb2RlciUyMiUwQSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> HfAgent

agent = HfAgent(
    url_endpoint=<span class="hljs-string">&quot;https://api-inference.huggingface.co/models/bigcode/starcoder&quot;</span>
)`}}),Ht=new T({props:{code:"YWdlbnQucnVuKCUyMkdlbmVyYXRlJTIwYW4lMjBpbWFnZSUyMG9mJTIwYSUyMGNhdCUyMik=",highlighted:'agent.run(<span class="hljs-string">&quot;Generate an image of a cat&quot;</span>)'}}),Ce=new Qh({props:{$$slots:{default:[Uu]},$$scope:{ctx:se}}}),Yt=new T({props:{code:"bGF1bmNoX2ZuKCklMEF0cmFuc2NyaXB0aW9uJTIwJTNEJTIwdHJhbnNjcmliZSgpJTBBYWdlbnQucnVuKHRyYW5zY3JpcHRpb24p",highlighted:`launch_fn()
transcription = transcribe()
agent.run(transcription)`}}),{c(){b=n("meta"),M=c(),U=n("h1"),E=n("a"),I=n("span"),d(S.$$.fragment),ae=c(),v=n("span"),$=o("Creating a voice assistant"),xe=c(),oe=n("p"),cn=o(`In this section, we\u2019ll piece together three models that we\u2019ve already had hands-on experience with to build an end-to-end
voice assistant called `),Js=n("strong"),pn=o("Marvin"),un=o(` \u{1F916}. Like Amazon\u2019s Alexa or Apple\u2019s Siri, Marvin is a virtual voice assistant who
responds to a particular \u2018wake word\u2019, then listens out for a spoken query, and finally responds with a spoken answer.`),Qa=c(),Xt=n("p"),dn=o("We can break down the voice assistant pipeline into four stages, each of which requires a standalone model:"),Ra=c(),He=n("div"),Us=n("img"),Xa=c(),R=n("h3"),le=n("a"),_s=n("span"),d(Ve.$$.fragment),fn=c(),js=n("span"),mn=o("1. Wake word detection"),Fa=c(),Ft=n("p"),wn=o(`Voice assistants are constantly listening to the audio inputs coming through your device\u2019s microphone, however they only
boot into action when a particular \u2018wake word\u2019 or \u2018trigger word\u2019 is spoken.`),za=c(),zt=n("p"),yn=o(`The wake word detection task is handled by a small on-device audio classification model, which is much smaller and lighter
than the speech recognition model, often only several millions of parameters compared to several hundred millions for
speech recognition. Thus, it can be run continuously on your device without draining your battery. Only when the wake
word is detected is the larger speech recognition model launched, and afterwards it is shut down again.`),Pa=c(),X=n("h3"),ne=n("a"),Es=n("span"),d(qe.$$.fragment),gn=c(),Is=n("span"),bn=o("2. Speech transcription"),Da=c(),Pt=n("p"),Mn=o(`The next stage in the pipeline is transcribing the spoken query to text. In practice, transferring audio files from your
local device to the Cloud is slow due to the large nature of audio files, so it\u2019s more efficient to transcribe them
directly using an automatic speech recognition (ASR) model on-device rather than using a model in the Cloud. The on-device
model might be smaller and thus less accurate than one hosted in the Cloud, but the faster inference speed makes it
worthwhile since we can run speech recognition in near real-time, our spoken audio utterance being transcribed as we say it.`),Oa=c(),Dt=n("p"),vn=o("We\u2019re very familiar with the speech recognition process now, so this should be a piece of cake!"),Ka=c(),F=n("h3"),ie=n("a"),$s=n("span"),d(Ye.$$.fragment),kn=c(),Ws=n("span"),Tn=o("3. Language model query"),eo=c(),re=n("p"),Jn=o(`Now that we know what the user asked, we need to generate a response! The best candidate models for this task are
`),Cs=n("em"),Un=o("large language models (LLMs)"),_n=o(`, since they are effectively able to understand the semantics of the text query and
generate a suitable response.`),to=c(),Ot=n("p"),jn=o(`Since our text query is small (just a few text tokens), and language models large (many billions of parameters), the most
efficient way of running LLM inference is to send our text query from our device to an LLM running in the Cloud,
generate a text response, and return the response back to the device.`),so=c(),z=n("h3"),he=n("a"),As=n("span"),d(Le.$$.fragment),En=c(),Zs=n("span"),In=o("4. Synthesise speech"),ao=c(),Kt=n("p"),$n=o(`Finally, we\u2019ll use a text-to-speech (TTS) model to synthesise the text response as spoken speech. This is done
on-device, but you could feasibly run a TTS model in the Cloud, generating the audio output and transferring it back to
the device.`),oo=c(),es=n("p"),Wn=o("Again, we\u2019ve done this several times now, so the process will be very familiar!"),lo=c(),d(ce.$$.fragment),no=c(),P=n("h2"),pe=n("a"),Ss=n("span"),d(Qe.$$.fragment),Cn=c(),Ns=n("span"),An=o("Wake word detection"),io=c(),k=n("p"),Zn=o(`The first stage in the voice assistant pipeline is detecting whether the wake word was spoken, and we need to find ourselves
an appropriate pre-trained model for this task! You\u2019ll remember from the section on `),ts=n("a"),Sn=o("pre-trained models for audio classification"),Nn=o(`
that `),Re=n("a"),Bn=o("Speech Commands"),Gn=o(` is a dataset of spoken words designed to
evaluate audio classification models on 15+ simple command words like `),Bs=n("code"),xn=o('"up"'),Hn=o(", "),Gs=n("code"),Vn=o('"down"'),qn=o(", "),xs=n("code"),Yn=o('"yes"'),Ln=o(" and "),Hs=n("code"),Qn=o('"no"'),Rn=o(`, as well as a
`),Vs=n("code"),Xn=o('"silence"'),Fn=o(` label to classify no speech. Take a minute to listen through the samples on the datasets viewer on
the Hub and re-acquaint yourself with the Speech Commands dataset: `),Xe=n("a"),zn=o("datasets viewer"),Pn=o("."),ro=c(),ss=n("p"),Dn=o(`We can take an audio classification model pre-trained on the Speech Commands dataset and pick one of these simple command
words to be our chosen wake word. Out of the 15+ possible command words, if the model predicts our chosen wake word with the
highest probability, we can be fairly certain that the wake word has been said.`),ho=c(),Fe=n("p"),On=o("Let\u2019s head to the Hugging Face Hub and click on the \u201CModels\u201D tab: "),ze=n("a"),Kn=o("https://huggingface.co/models"),co=c(),as=n("p"),ei=o("This is going to bring up all the models on the Hugging Face Hub, sorted by downloads in the past 30 days:"),po=c(),Pe=n("div"),qs=n("img"),uo=c(),os=n("p"),ti=o(`You\u2019ll notice on the left-hand side that we have a selection of tabs that we can select to filter models by task, library,
dataset, etc. Scroll down and select the task \u201CAudio Classification\u201D from the list of audio tasks:`),fo=c(),De=n("div"),Ys=n("img"),mo=c(),ue=n("p"),si=o(`We\u2019re now presented with the sub-set of 500+ audio classification models on the Hub. To further refine this selection, we
can filter models by dataset. Click on the tab \u201CDatasets\u201D, and in the search box type \u201Cspeech_commands\u201D. As you begin typing,
you\u2019ll see the selection for `),Ls=n("code"),ai=o("speech_commands"),oi=o(` appear underneath the search tab. You can click this button to filter all
audio classification models to those fine-tuned on the Speech Commands dataset:`),wo=c(),Oe=n("div"),Qs=n("img"),yo=c(),de=n("p"),li=o(`Great! We see that we have six pre-trained models available to us for this specific dataset and task (although there may
be new models added if you\u2019re reading at a later date!). You\u2019ll recognise the first of these models as the `),Ke=n("a"),ni=o("Audio Spectrogram Transformer checkpoint"),ii=o(`
that we used in Unit 4 example. We\u2019ll use this checkpoint again for our wake word detection task.`),go=c(),fe=n("p"),ri=o("Let\u2019s go ahead and load the checkpoint using the "),Rs=n("code"),hi=o("pipeline"),ci=o(" class:"),bo=c(),d(et.$$.fragment),Mo=c(),me=n("p"),pi=o("We can check what labels the model was trained on by checking the "),Xs=n("code"),ui=o("id2label"),di=o(" attribute in the model config:"),vo=c(),d(tt.$$.fragment),ko=c(),_=n("p"),fi=o(`Alright! We see that the model was trained on 35 class labels, including some simple command words that we described above,
as well as some particular objects like `),Fs=n("code"),mi=o('"bed"'),wi=o(", "),zs=n("code"),yi=o('"house"'),gi=o(" and "),Ps=n("code"),bi=o('"cat"'),Mi=o(`. We see that there is one name in these class labels:
id 27 corresponds to the label `),Ds=n("strong"),vi=o("\u201Cmarvin\u201D"),ki=o(":"),To=c(),d(st.$$.fragment),Jo=c(),d(at.$$.fragment),Uo=c(),we=n("p"),Ti=o(`Perfect! We can use this name as our wake word for our voice assistant, similar to how \u201CAlexa\u201D is used for Amazon\u2019s Alexa,
or \u201CHey Siri\u201D is used for Apple\u2019s Siri. Of all the possible labels, if the model predicts `),Os=n("code"),Ji=o('"marvin"'),Ui=o(` with the highest class
probability, we can be fairly sure that our chosen wake word has been said.`),_o=c(),ye=n("p"),_i=o(`Now we need to define a function that is constantly listening to our device\u2019s microphone input, and continuously
passes the audio to the classification model for inference. To do this, we\u2019ll use a handy helper function that comes
with \u{1F917} Transformers called `),ot=n("a"),Ks=n("code"),ji=o("ffmpeg_microphone_live"),Ei=o("."),jo=c(),j=n("p"),Ii=o("This function forwards small chunks of audio of specified length "),ea=n("code"),$i=o("chunk_length_s"),Wi=o(` to the model to be classified. To ensure that
we get smooth boundaries across chunks of audio, we run a sliding window across our audio with stride `),ta=n("code"),Ci=o("chunk_length_s / 6"),Ai=o(`.
So that we don\u2019t have to wait for the entire first chunk to be recorded before we start inferring, we also define a minimal
temporary audio input length `),sa=n("code"),Zi=o("stream_chunk_s"),Si=o(" that is forwarded to the model before "),aa=n("code"),Ni=o("chunk_length_s"),Bi=o(" time is reached."),Eo=c(),W=n("p"),Gi=o("The function "),oa=n("code"),xi=o("ffmpeg_microphone_live"),Hi=o(" returns a "),la=n("em"),Vi=o("generator"),qi=o(` object, yielding a sequence of audio chunks that can each
be passed to the classification model to make a prediction. We can pass this generator directly to the `),na=n("code"),Yi=o("pipeline"),Li=o(`,
which in turn returns a sequence of output predictions, one for each chunk of audio input. We can inspect the class
label probabilities for each audio chunk, and stop our wake word detection loop when we detect that the wake word
has been spoken.`),Io=c(),B=n("p"),Qi=o(`We\u2019ll use a very simple criteria for classifying whether our wake word was spoken: if the class label with the highest
probability was our wake word, and this probability exceeds a threshold `),ia=n("code"),Ri=o("prob_threshold"),Xi=o(`, we declare that the wake word
as having been spoken. Using a probability threshold to gate our classifier this way ensures that the wake word is not
erroneously predicted if the audio input is noise, which is typically when the model is very uncertain and all the class
label probabilities low. You might want to tune this probability threshold, or explore more sophisticated means for
the wake word decision through an `),lt=n("a"),ra=n("em"),Fi=o("entropy"),zi=o(" (or uncertainty) based metric."),$o=c(),d(nt.$$.fragment),Wo=c(),C=n("p"),Pi=o("Let\u2019s give this function a try to see how it works! We\u2019ll set the flag "),ha=n("code"),Di=o("debug=True"),Oi=o(` to print out the prediction for each
chunk of audio. Let the model run for a few seconds to see the kinds of predictions that it makes when there is no speech
input, then clearly say the wake word `),ca=n("code"),Ki=o('"marvin"'),er=o(" and watch the class label prediction for "),pa=n("code"),tr=o('"marvin"'),sr=o(" spike to near 1:"),Co=c(),d(it.$$.fragment),Ao=c(),d(rt.$$.fragment),Zo=c(),ge=n("p"),ar=o(`Awesome! As we expect, the model generates garbage predictions for the first few seconds. There is no speech input, so the
model makes close to random predictions, but with very low probability. As soon as we say the wake word, the model predicts
`),ua=n("code"),or=o('"marvin"'),lr=o(` with probability close to 1 and terminates the loop, signalling that the wake word has been detected and that the
ASR system should be activated!`),So=c(),D=n("h2"),be=n("a"),da=n("span"),d(ht.$$.fragment),nr=c(),fa=n("span"),ir=o("Speech transcription"),No=c(),G=n("p"),rr=o("Once again, we\u2019ll use the Whisper model for our speech transcription system. Specifically, we\u2019ll load the "),ct=n("a"),hr=o("Whisper Base English"),cr=o(`
checkpoint, since it\u2019s small enough to give good inference speed with reasonable transcription accuracy. We\u2019ll use a trick
to get near real-time transcription by being clever with how we forward our audio inputs to the model. As before, feel
free to use any speech recognition checkpoint on `),pt=n("a"),pr=o("the Hub"),ur=o(`,
including Wav2Vec2, MMS ASR or other Whisper checkpoints:`),Bo=c(),d(ut.$$.fragment),Go=c(),d(Me.$$.fragment),xo=c(),x=n("p"),dr=o("We can now define a function to record our microphone input and transcribe the corresponding text. With the "),ma=n("code"),fr=o("ffmpeg_microphone_live"),mr=o(`
helper function, we can control how \u2018real-time\u2019 our speech recognition model is. Using a smaller `),wa=n("code"),wr=o("stream_chunk_s"),yr=o(` lends
itself to more real-time speech recognition, since we divide our input audio into smaller chunks and transcribe them on
the fly. However, this comes at the expense of poorer accuracy, since there\u2019s less context for the model to infer from.`),Ho=c(),A=n("p"),gr=o("As we\u2019re transcribing the speech, we also need to have an idea of when the user "),ya=n("strong"),br=o("stops"),Mr=o(` speaking, so that we can terminate
the recording. For simplicity, we\u2019ll terminate our microphone recording after the first `),ga=n("code"),vr=o("chunk_length_s"),kr=o(` (which is set to
5 seconds by default), but you can experiment with using a `),dt=n("a"),Tr=o("voice activity detection (VAD)"),Jr=o(`
model to predict when the user has stopped speaking.`),Vo=c(),d(ft.$$.fragment),qo=c(),ls=n("p"),Ur=o(`Let\u2019s give this a go and see how we get on! Once the microphone is live, start speaking and watch your transcription
appear in semi real-time:`),Yo=c(),d(mt.$$.fragment),Lo=c(),d(wt.$$.fragment),Qo=c(),Z=n("p"),_r=o("Nice! You can adjust the maximum audio length "),ba=n("code"),jr=o("chunk_length_s"),Er=o(` based on how fast or slow you speak (increase it if you
felt like you didn\u2019t have enough time to speak, decrease it if you were left waiting at the end), and the
`),Ma=n("code"),Ir=o("stream_chunk_s"),$r=o(" for the real-time factor. Just pass these as arguments to the "),va=n("code"),Wr=o("transcribe"),Cr=o(" function."),Ro=c(),O=n("h2"),ve=n("a"),ka=n("span"),d(yt.$$.fragment),Ar=c(),Ta=n("span"),Zr=o("Language model query"),Xo=c(),ke=n("p"),Sr=o(`Now that we have our spoken query transcribed, we want to generate a meaningful response. To do this, we\u2019ll use an LLM
hosted on the Cloud. Specifically, we\u2019ll pick an LLM on the Hugging Face Hub and use the `),gt=n("a"),Nr=o("Inference API"),Br=o(`
to easily query the model.`),Fo=c(),Te=n("p"),Gr=o("First, let\u2019s head over to the Hugging Face Hub. To find our LLM, we\u2019ll use the "),bt=n("a"),xr=o("\u{1F917} Open LLM Leaderboard"),Hr=o(`,
a Space that ranks LLM models by performance over four generation tasks. We\u2019ll search by \u201Cinstruct\u201D to filter out models
that have been instruction fine-tuned, since these should work better for our querying task:`),zo=c(),Mt=n("div"),Ja=n("img"),Po=c(),H=n("p"),Vr=o("We\u2019ll use the "),vt=n("a"),qr=o("tiiuae/falcon-7b-instruct"),Yr=o(" checkpoint by "),kt=n("a"),Lr=o("TII"),Qr=o(`,
a 7B parameter decoder-only LM fine-tuned on a mixture of chat and instruction datasets. You can use any LLM on the Hugging
Face Hub that has the \u201CHosted inference API\u201D enabled, just look out for the widget on the right-side of the model card:`),Do=c(),Tt=n("div"),Ua=n("img"),Oo=c(),Je=n("p"),Rr=o(`The Inference API allows us to send a HTTP request from our local machine to the LLM hosted on the Hub, and returns the
response as a `),_a=n("code"),Xr=o("json"),Fr=o(` file. All we need to provide is our Hugging Face Hub token (which we retrieve directly from our Hugging Face
Hub folder) and the model id of the LLM we wish to query:`),Ko=c(),d(Jt.$$.fragment),el=c(),ns=n("p"),zr=o("Let\u2019s give it a try with a test input!"),tl=c(),d(Ut.$$.fragment),sl=c(),d(_t.$$.fragment),al=c(),is=n("p"),Pr=o(`You\u2019ll notice just how fast inference is using the Inference API - we only have to send a small number of text tokens
from our local machine to the hosted model, so the communication cost is very low. The LLM is hosted on GPU accelerators,
so inference runs very quickly. Finally, the generated response is transferred back from the model to our local machine,
again with low communication overhead.`),ol=c(),K=n("h2"),Ue=n("a"),ja=n("span"),d(jt.$$.fragment),Dr=c(),Ea=n("span"),Or=o("Synthesise speech"),ll=c(),_e=n("p"),Kr=o("And now we\u2019re ready to get the final spoken output! Once again, we\u2019ll use the Microsoft "),Et=n("a"),eh=o("SpeechT5 TTS"),th=o(`
model for English TTS, but you can use any TTS model of your choice. Let\u2019s go ahead and load the processor and model:`),nl=c(),d(It.$$.fragment),il=c(),rs=n("p"),sh=o("And also the speaker embeddings:"),rl=c(),d($t.$$.fragment),hl=c(),V=n("p"),ah=o("We\u2019ll re-use the "),Ia=n("code"),oh=o("synthesise"),lh=o(" function that we defined in the previous chapter on "),hs=n("a"),nh=o("Speech-to-speech translation"),ih=o(":"),cl=c(),d(Wt.$$.fragment),pl=c(),cs=n("p"),rh=o("Let\u2019s quickly verify this works as expected:"),ul=c(),d(Ct.$$.fragment),dl=c(),ps=n("p"),hh=o("Nice job \u{1F44D}"),fl=c(),ee=n("h2"),je=n("a"),$a=n("span"),d(At.$$.fragment),ch=c(),Wa=n("span"),ph=o("Marvin \u{1F916}"),ml=c(),Ee=n("p"),uh=o(`Now that we\u2019ve defined a function for each of the four stages of the voice assistant pipeline, all that\u2019s left to do is
piece them together to get our end-to-end voice assistant. We\u2019ll simply concatenate the four stages, starting with
wake word detection (`),Ca=n("code"),dh=o("launch_fn"),fh=o("), speech transcription, querying the LLM, and finally speech synthesis."),wl=c(),d(Zt.$$.fragment),yl=c(),us=n("p"),mh=o("Try it out with a few prompts! Here are some examples to get you started:"),gl=c(),q=n("ul"),Aa=n("li"),Za=n("em"),wh=o("What is the hottest country in the world?"),yh=c(),Sa=n("li"),Na=n("em"),gh=o("How do Transformer models work?"),bh=c(),Ba=n("li"),Ga=n("em"),Mh=o("Do you know Spanish?"),bl=c(),ds=n("p"),vh=o(`And with that, we have our end-to-end voice assistant complete, made using the \u{1F917} audio tools you\u2019ve learnt throughout
this course, with a sprinkling of LLM magic at the end. There are several extensions that we could make to improve the
voice assistant. Firstly, the audio classification model classifies 35 different labels. We could use a smaller, more
lightweight binary classification model that only predicts whether the wake word was spoken or not. Secondly, we pre-load
all the models ahead and keep them running on our device. If we wanted to save power, we would only load each model at
the time it was required, and subsequently un-load them afterwards. Thirdly, we\u2019re missing a voice activity detection model
in our transcription function, transcribing for a fixed amount of time, which in some cases is too long, and in others too
short.`),Ml=c(),te=n("h2"),Ie=n("a"),xa=n("span"),d(St.$$.fragment),kh=c(),Ha=n("span"),Th=o("Generalise to anything \u{1FA84}"),vl=c(),fs=n("p"),Jh=o(`So far, we\u2019ve seen how we can generate speech outputs with our voice assistant Marvin. To finish, we\u2019ll demonstrate how
we can generalise these speech outputs to text, audio and image.`),kl=c(),$e=n("p"),Uh=o("We\u2019ll use "),Nt=n("a"),_h=o("Transformers Agents"),jh=o(` to build our assistant.
Transformers Agents provides a natural language API on top of the \u{1F917} Transformers and Diffusers libraries, interpreting
a natural language input using an LLM with carefully crafted prompts, and using a set of curated tools to provide
multimodal outputs.`),Tl=c(),Y=n("p"),Eh=o("Let\u2019s go ahead and instantiate an agent. There are "),Bt=n("a"),Ih=o("three LLMs available"),$h=o(`
for Transformers Agents, two of which are open-source and free on the Hugging Face Hub. The third is a model from OpenAI
that requires an OpenAI API key. We\u2019ll use the free `),Gt=n("a"),Wh=o("Bigcode Starcoder"),Ch=o(` model
in this example, but you can also try either of the other LLMs available:`),Jl=c(),d(xt.$$.fragment),Ul=c(),We=n("p"),Ah=o("To use the agent, we simply have to call "),Va=n("code"),Zh=o("agent.run"),Sh=o(` with our text prompt. As an example, we\u2019ll get it to generate an
image of a cat \u{1F408} (that hopefully looks a bit better than this emoji):`),_l=c(),d(Ht.$$.fragment),jl=c(),Vt=n("div"),qa=n("img"),El=c(),d(Ce.$$.fragment),Il=c(),Ae=n("p"),Nh=o("Easy as that! The Agent interpreted our prompt, and used "),qt=n("a"),Bh=o("Stable Diffusion"),Gh=o(`
under the hood to generate the image, without us having to worry about loading the model, writing the function or executing
the code.`),$l=c(),ms=n("p"),xh=o(`We can now replace our LLM query function and text synthesis step with our Transformers Agent in our voice assistant,
since the Agent is going to take care of both of these steps for us:`),Wl=c(),d(Yt.$$.fragment),Cl=c(),ws=n("p"),Hh=o(`Try speaking the same prompt \u201CGenerate an image of a cat\u201D and see how the system gets on. If you ask the Agent a simple
question / answer query, the Agent will respond with a text answer. You can encourage it to generate multimodal outputs
by asking it to return an image or speech. For example, you can ask it to: \u201CGenerate an image of a cat, caption it, and
speak the caption\u201D.`),Al=c(),Ze=n("p"),Vh=o(`While the Agent is more flexible than our first iteration Marvin \u{1F916} assistant, generalising the voice assistant task in this way
may lead to inferior performance on standard voice assistant queries. To recover performance, you can try using a
more performant LLM checkpoint, such as the one from OpenAI, or define a set of `),Lt=n("a"),qh=o("custom tools"),Yh=o(`
that are specific to the voice assistant task.`),this.h()},l(e){const a=vu('[data-svelte="svelte-1phssyn"]',document.head);b=i(a,"META",{name:!0,content:!0}),a.forEach(t),M=p(e),U=i(e,"H1",{class:!0});var Qt=h(U);E=i(Qt,"A",{id:!0,class:!0,href:!0});var Ya=h(E);I=i(Ya,"SPAN",{});var La=h(I);f(S.$$.fragment,La),La.forEach(t),Ya.forEach(t),ae=p(Qt),v=i(Qt,"SPAN",{});var Kh=h(v);$=l(Kh,"Creating a voice assistant"),Kh.forEach(t),Qt.forEach(t),xe=p(e),oe=i(e,"P",{});var Sl=h(oe);cn=l(Sl,`In this section, we\u2019ll piece together three models that we\u2019ve already had hands-on experience with to build an end-to-end
voice assistant called `),Js=i(Sl,"STRONG",{});var ec=h(Js);pn=l(ec,"Marvin"),ec.forEach(t),un=l(Sl,` \u{1F916}. Like Amazon\u2019s Alexa or Apple\u2019s Siri, Marvin is a virtual voice assistant who
responds to a particular \u2018wake word\u2019, then listens out for a spoken query, and finally responds with a spoken answer.`),Sl.forEach(t),Qa=p(e),Xt=i(e,"P",{});var tc=h(Xt);dn=l(tc,"We can break down the voice assistant pipeline into four stages, each of which requires a standalone model:"),tc.forEach(t),Ra=p(e),He=i(e,"DIV",{class:!0});var sc=h(He);Us=i(sc,"IMG",{src:!0}),sc.forEach(t),Xa=p(e),R=i(e,"H3",{class:!0});var Nl=h(R);le=i(Nl,"A",{id:!0,class:!0,href:!0});var ac=h(le);_s=i(ac,"SPAN",{});var oc=h(_s);f(Ve.$$.fragment,oc),oc.forEach(t),ac.forEach(t),fn=p(Nl),js=i(Nl,"SPAN",{});var lc=h(js);mn=l(lc,"1. Wake word detection"),lc.forEach(t),Nl.forEach(t),Fa=p(e),Ft=i(e,"P",{});var nc=h(Ft);wn=l(nc,`Voice assistants are constantly listening to the audio inputs coming through your device\u2019s microphone, however they only
boot into action when a particular \u2018wake word\u2019 or \u2018trigger word\u2019 is spoken.`),nc.forEach(t),za=p(e),zt=i(e,"P",{});var ic=h(zt);yn=l(ic,`The wake word detection task is handled by a small on-device audio classification model, which is much smaller and lighter
than the speech recognition model, often only several millions of parameters compared to several hundred millions for
speech recognition. Thus, it can be run continuously on your device without draining your battery. Only when the wake
word is detected is the larger speech recognition model launched, and afterwards it is shut down again.`),ic.forEach(t),Pa=p(e),X=i(e,"H3",{class:!0});var Bl=h(X);ne=i(Bl,"A",{id:!0,class:!0,href:!0});var rc=h(ne);Es=i(rc,"SPAN",{});var hc=h(Es);f(qe.$$.fragment,hc),hc.forEach(t),rc.forEach(t),gn=p(Bl),Is=i(Bl,"SPAN",{});var cc=h(Is);bn=l(cc,"2. Speech transcription"),cc.forEach(t),Bl.forEach(t),Da=p(e),Pt=i(e,"P",{});var pc=h(Pt);Mn=l(pc,`The next stage in the pipeline is transcribing the spoken query to text. In practice, transferring audio files from your
local device to the Cloud is slow due to the large nature of audio files, so it\u2019s more efficient to transcribe them
directly using an automatic speech recognition (ASR) model on-device rather than using a model in the Cloud. The on-device
model might be smaller and thus less accurate than one hosted in the Cloud, but the faster inference speed makes it
worthwhile since we can run speech recognition in near real-time, our spoken audio utterance being transcribed as we say it.`),pc.forEach(t),Oa=p(e),Dt=i(e,"P",{});var uc=h(Dt);vn=l(uc,"We\u2019re very familiar with the speech recognition process now, so this should be a piece of cake!"),uc.forEach(t),Ka=p(e),F=i(e,"H3",{class:!0});var Gl=h(F);ie=i(Gl,"A",{id:!0,class:!0,href:!0});var dc=h(ie);$s=i(dc,"SPAN",{});var fc=h($s);f(Ye.$$.fragment,fc),fc.forEach(t),dc.forEach(t),kn=p(Gl),Ws=i(Gl,"SPAN",{});var mc=h(Ws);Tn=l(mc,"3. Language model query"),mc.forEach(t),Gl.forEach(t),eo=p(e),re=i(e,"P",{});var xl=h(re);Jn=l(xl,`Now that we know what the user asked, we need to generate a response! The best candidate models for this task are
`),Cs=i(xl,"EM",{});var wc=h(Cs);Un=l(wc,"large language models (LLMs)"),wc.forEach(t),_n=l(xl,`, since they are effectively able to understand the semantics of the text query and
generate a suitable response.`),xl.forEach(t),to=p(e),Ot=i(e,"P",{});var yc=h(Ot);jn=l(yc,`Since our text query is small (just a few text tokens), and language models large (many billions of parameters), the most
efficient way of running LLM inference is to send our text query from our device to an LLM running in the Cloud,
generate a text response, and return the response back to the device.`),yc.forEach(t),so=p(e),z=i(e,"H3",{class:!0});var Hl=h(z);he=i(Hl,"A",{id:!0,class:!0,href:!0});var gc=h(he);As=i(gc,"SPAN",{});var bc=h(As);f(Le.$$.fragment,bc),bc.forEach(t),gc.forEach(t),En=p(Hl),Zs=i(Hl,"SPAN",{});var Mc=h(Zs);In=l(Mc,"4. Synthesise speech"),Mc.forEach(t),Hl.forEach(t),ao=p(e),Kt=i(e,"P",{});var vc=h(Kt);$n=l(vc,`Finally, we\u2019ll use a text-to-speech (TTS) model to synthesise the text response as spoken speech. This is done
on-device, but you could feasibly run a TTS model in the Cloud, generating the audio output and transferring it back to
the device.`),vc.forEach(t),oo=p(e),es=i(e,"P",{});var kc=h(es);Wn=l(kc,"Again, we\u2019ve done this several times now, so the process will be very familiar!"),kc.forEach(t),lo=p(e),f(ce.$$.fragment,e),no=p(e),P=i(e,"H2",{class:!0});var Vl=h(P);pe=i(Vl,"A",{id:!0,class:!0,href:!0});var Tc=h(pe);Ss=i(Tc,"SPAN",{});var Jc=h(Ss);f(Qe.$$.fragment,Jc),Jc.forEach(t),Tc.forEach(t),Cn=p(Vl),Ns=i(Vl,"SPAN",{});var Uc=h(Ns);An=l(Uc,"Wake word detection"),Uc.forEach(t),Vl.forEach(t),io=p(e),k=i(e,"P",{});var J=h(k);Zn=l(J,`The first stage in the voice assistant pipeline is detecting whether the wake word was spoken, and we need to find ourselves
an appropriate pre-trained model for this task! You\u2019ll remember from the section on `),ts=i(J,"A",{href:!0});var _c=h(ts);Sn=l(_c,"pre-trained models for audio classification"),_c.forEach(t),Nn=l(J,`
that `),Re=i(J,"A",{href:!0,rel:!0});var jc=h(Re);Bn=l(jc,"Speech Commands"),jc.forEach(t),Gn=l(J,` is a dataset of spoken words designed to
evaluate audio classification models on 15+ simple command words like `),Bs=i(J,"CODE",{});var Ec=h(Bs);xn=l(Ec,'"up"'),Ec.forEach(t),Hn=l(J,", "),Gs=i(J,"CODE",{});var Ic=h(Gs);Vn=l(Ic,'"down"'),Ic.forEach(t),qn=l(J,", "),xs=i(J,"CODE",{});var $c=h(xs);Yn=l($c,'"yes"'),$c.forEach(t),Ln=l(J," and "),Hs=i(J,"CODE",{});var Wc=h(Hs);Qn=l(Wc,'"no"'),Wc.forEach(t),Rn=l(J,`, as well as a
`),Vs=i(J,"CODE",{});var Cc=h(Vs);Xn=l(Cc,'"silence"'),Cc.forEach(t),Fn=l(J,` label to classify no speech. Take a minute to listen through the samples on the datasets viewer on
the Hub and re-acquaint yourself with the Speech Commands dataset: `),Xe=i(J,"A",{href:!0,rel:!0});var Ac=h(Xe);zn=l(Ac,"datasets viewer"),Ac.forEach(t),Pn=l(J,"."),J.forEach(t),ro=p(e),ss=i(e,"P",{});var Zc=h(ss);Dn=l(Zc,`We can take an audio classification model pre-trained on the Speech Commands dataset and pick one of these simple command
words to be our chosen wake word. Out of the 15+ possible command words, if the model predicts our chosen wake word with the
highest probability, we can be fairly certain that the wake word has been said.`),Zc.forEach(t),ho=p(e),Fe=i(e,"P",{});var Lh=h(Fe);On=l(Lh,"Let\u2019s head to the Hugging Face Hub and click on the \u201CModels\u201D tab: "),ze=i(Lh,"A",{href:!0,rel:!0});var Sc=h(ze);Kn=l(Sc,"https://huggingface.co/models"),Sc.forEach(t),Lh.forEach(t),co=p(e),as=i(e,"P",{});var Nc=h(as);ei=l(Nc,"This is going to bring up all the models on the Hugging Face Hub, sorted by downloads in the past 30 days:"),Nc.forEach(t),po=p(e),Pe=i(e,"DIV",{class:!0});var Bc=h(Pe);qs=i(Bc,"IMG",{src:!0}),Bc.forEach(t),uo=p(e),os=i(e,"P",{});var Gc=h(os);ti=l(Gc,`You\u2019ll notice on the left-hand side that we have a selection of tabs that we can select to filter models by task, library,
dataset, etc. Scroll down and select the task \u201CAudio Classification\u201D from the list of audio tasks:`),Gc.forEach(t),fo=p(e),De=i(e,"DIV",{class:!0});var xc=h(De);Ys=i(xc,"IMG",{src:!0}),xc.forEach(t),mo=p(e),ue=i(e,"P",{});var ql=h(ue);si=l(ql,`We\u2019re now presented with the sub-set of 500+ audio classification models on the Hub. To further refine this selection, we
can filter models by dataset. Click on the tab \u201CDatasets\u201D, and in the search box type \u201Cspeech_commands\u201D. As you begin typing,
you\u2019ll see the selection for `),Ls=i(ql,"CODE",{});var Hc=h(Ls);ai=l(Hc,"speech_commands"),Hc.forEach(t),oi=l(ql,` appear underneath the search tab. You can click this button to filter all
audio classification models to those fine-tuned on the Speech Commands dataset:`),ql.forEach(t),wo=p(e),Oe=i(e,"DIV",{class:!0});var Vc=h(Oe);Qs=i(Vc,"IMG",{src:!0}),Vc.forEach(t),yo=p(e),de=i(e,"P",{});var Yl=h(de);li=l(Yl,`Great! We see that we have six pre-trained models available to us for this specific dataset and task (although there may
be new models added if you\u2019re reading at a later date!). You\u2019ll recognise the first of these models as the `),Ke=i(Yl,"A",{href:!0,rel:!0});var qc=h(Ke);ni=l(qc,"Audio Spectrogram Transformer checkpoint"),qc.forEach(t),ii=l(Yl,`
that we used in Unit 4 example. We\u2019ll use this checkpoint again for our wake word detection task.`),Yl.forEach(t),go=p(e),fe=i(e,"P",{});var Ll=h(fe);ri=l(Ll,"Let\u2019s go ahead and load the checkpoint using the "),Rs=i(Ll,"CODE",{});var Yc=h(Rs);hi=l(Yc,"pipeline"),Yc.forEach(t),ci=l(Ll," class:"),Ll.forEach(t),bo=p(e),f(et.$$.fragment,e),Mo=p(e),me=i(e,"P",{});var Ql=h(me);pi=l(Ql,"We can check what labels the model was trained on by checking the "),Xs=i(Ql,"CODE",{});var Lc=h(Xs);ui=l(Lc,"id2label"),Lc.forEach(t),di=l(Ql," attribute in the model config:"),Ql.forEach(t),vo=p(e),f(tt.$$.fragment,e),ko=p(e),_=i(e,"P",{});var L=h(_);fi=l(L,`Alright! We see that the model was trained on 35 class labels, including some simple command words that we described above,
as well as some particular objects like `),Fs=i(L,"CODE",{});var Qc=h(Fs);mi=l(Qc,'"bed"'),Qc.forEach(t),wi=l(L,", "),zs=i(L,"CODE",{});var Rc=h(zs);yi=l(Rc,'"house"'),Rc.forEach(t),gi=l(L," and "),Ps=i(L,"CODE",{});var Xc=h(Ps);bi=l(Xc,'"cat"'),Xc.forEach(t),Mi=l(L,`. We see that there is one name in these class labels:
id 27 corresponds to the label `),Ds=i(L,"STRONG",{});var Fc=h(Ds);vi=l(Fc,"\u201Cmarvin\u201D"),Fc.forEach(t),ki=l(L,":"),L.forEach(t),To=p(e),f(st.$$.fragment,e),Jo=p(e),f(at.$$.fragment,e),Uo=p(e),we=i(e,"P",{});var Rl=h(we);Ti=l(Rl,`Perfect! We can use this name as our wake word for our voice assistant, similar to how \u201CAlexa\u201D is used for Amazon\u2019s Alexa,
or \u201CHey Siri\u201D is used for Apple\u2019s Siri. Of all the possible labels, if the model predicts `),Os=i(Rl,"CODE",{});var zc=h(Os);Ji=l(zc,'"marvin"'),zc.forEach(t),Ui=l(Rl,` with the highest class
probability, we can be fairly sure that our chosen wake word has been said.`),Rl.forEach(t),_o=p(e),ye=i(e,"P",{});var Xl=h(ye);_i=l(Xl,`Now we need to define a function that is constantly listening to our device\u2019s microphone input, and continuously
passes the audio to the classification model for inference. To do this, we\u2019ll use a handy helper function that comes
with \u{1F917} Transformers called `),ot=i(Xl,"A",{href:!0,rel:!0});var Pc=h(ot);Ks=i(Pc,"CODE",{});var Dc=h(Ks);ji=l(Dc,"ffmpeg_microphone_live"),Dc.forEach(t),Pc.forEach(t),Ei=l(Xl,"."),Xl.forEach(t),jo=p(e),j=i(e,"P",{});var Q=h(j);Ii=l(Q,"This function forwards small chunks of audio of specified length "),ea=i(Q,"CODE",{});var Oc=h(ea);$i=l(Oc,"chunk_length_s"),Oc.forEach(t),Wi=l(Q,` to the model to be classified. To ensure that
we get smooth boundaries across chunks of audio, we run a sliding window across our audio with stride `),ta=i(Q,"CODE",{});var Kc=h(ta);Ci=l(Kc,"chunk_length_s / 6"),Kc.forEach(t),Ai=l(Q,`.
So that we don\u2019t have to wait for the entire first chunk to be recorded before we start inferring, we also define a minimal
temporary audio input length `),sa=i(Q,"CODE",{});var ep=h(sa);Zi=l(ep,"stream_chunk_s"),ep.forEach(t),Si=l(Q," that is forwarded to the model before "),aa=i(Q,"CODE",{});var tp=h(aa);Ni=l(tp,"chunk_length_s"),tp.forEach(t),Bi=l(Q," time is reached."),Q.forEach(t),Eo=p(e),W=i(e,"P",{});var Se=h(W);Gi=l(Se,"The function "),oa=i(Se,"CODE",{});var sp=h(oa);xi=l(sp,"ffmpeg_microphone_live"),sp.forEach(t),Hi=l(Se," returns a "),la=i(Se,"EM",{});var ap=h(la);Vi=l(ap,"generator"),ap.forEach(t),qi=l(Se,` object, yielding a sequence of audio chunks that can each
be passed to the classification model to make a prediction. We can pass this generator directly to the `),na=i(Se,"CODE",{});var op=h(na);Yi=l(op,"pipeline"),op.forEach(t),Li=l(Se,`,
which in turn returns a sequence of output predictions, one for each chunk of audio input. We can inspect the class
label probabilities for each audio chunk, and stop our wake word detection loop when we detect that the wake word
has been spoken.`),Se.forEach(t),Io=p(e),B=i(e,"P",{});var ys=h(B);Qi=l(ys,`We\u2019ll use a very simple criteria for classifying whether our wake word was spoken: if the class label with the highest
probability was our wake word, and this probability exceeds a threshold `),ia=i(ys,"CODE",{});var lp=h(ia);Ri=l(lp,"prob_threshold"),lp.forEach(t),Xi=l(ys,`, we declare that the wake word
as having been spoken. Using a probability threshold to gate our classifier this way ensures that the wake word is not
erroneously predicted if the audio input is noise, which is typically when the model is very uncertain and all the class
label probabilities low. You might want to tune this probability threshold, or explore more sophisticated means for
the wake word decision through an `),lt=i(ys,"A",{href:!0,rel:!0});var np=h(lt);ra=i(np,"EM",{});var ip=h(ra);Fi=l(ip,"entropy"),ip.forEach(t),np.forEach(t),zi=l(ys," (or uncertainty) based metric."),ys.forEach(t),$o=p(e),f(nt.$$.fragment,e),Wo=p(e),C=i(e,"P",{});var Ne=h(C);Pi=l(Ne,"Let\u2019s give this function a try to see how it works! We\u2019ll set the flag "),ha=i(Ne,"CODE",{});var rp=h(ha);Di=l(rp,"debug=True"),rp.forEach(t),Oi=l(Ne,` to print out the prediction for each
chunk of audio. Let the model run for a few seconds to see the kinds of predictions that it makes when there is no speech
input, then clearly say the wake word `),ca=i(Ne,"CODE",{});var hp=h(ca);Ki=l(hp,'"marvin"'),hp.forEach(t),er=l(Ne," and watch the class label prediction for "),pa=i(Ne,"CODE",{});var cp=h(pa);tr=l(cp,'"marvin"'),cp.forEach(t),sr=l(Ne," spike to near 1:"),Ne.forEach(t),Co=p(e),f(it.$$.fragment,e),Ao=p(e),f(rt.$$.fragment,e),Zo=p(e),ge=i(e,"P",{});var Fl=h(ge);ar=l(Fl,`Awesome! As we expect, the model generates garbage predictions for the first few seconds. There is no speech input, so the
model makes close to random predictions, but with very low probability. As soon as we say the wake word, the model predicts
`),ua=i(Fl,"CODE",{});var pp=h(ua);or=l(pp,'"marvin"'),pp.forEach(t),lr=l(Fl,` with probability close to 1 and terminates the loop, signalling that the wake word has been detected and that the
ASR system should be activated!`),Fl.forEach(t),So=p(e),D=i(e,"H2",{class:!0});var zl=h(D);be=i(zl,"A",{id:!0,class:!0,href:!0});var up=h(be);da=i(up,"SPAN",{});var dp=h(da);f(ht.$$.fragment,dp),dp.forEach(t),up.forEach(t),nr=p(zl),fa=i(zl,"SPAN",{});var fp=h(fa);ir=l(fp,"Speech transcription"),fp.forEach(t),zl.forEach(t),No=p(e),G=i(e,"P",{});var gs=h(G);rr=l(gs,"Once again, we\u2019ll use the Whisper model for our speech transcription system. Specifically, we\u2019ll load the "),ct=i(gs,"A",{href:!0,rel:!0});var mp=h(ct);hr=l(mp,"Whisper Base English"),mp.forEach(t),cr=l(gs,`
checkpoint, since it\u2019s small enough to give good inference speed with reasonable transcription accuracy. We\u2019ll use a trick
to get near real-time transcription by being clever with how we forward our audio inputs to the model. As before, feel
free to use any speech recognition checkpoint on `),pt=i(gs,"A",{href:!0,rel:!0});var wp=h(pt);pr=l(wp,"the Hub"),wp.forEach(t),ur=l(gs,`,
including Wav2Vec2, MMS ASR or other Whisper checkpoints:`),gs.forEach(t),Bo=p(e),f(ut.$$.fragment,e),Go=p(e),f(Me.$$.fragment,e),xo=p(e),x=i(e,"P",{});var bs=h(x);dr=l(bs,"We can now define a function to record our microphone input and transcribe the corresponding text. With the "),ma=i(bs,"CODE",{});var yp=h(ma);fr=l(yp,"ffmpeg_microphone_live"),yp.forEach(t),mr=l(bs,`
helper function, we can control how \u2018real-time\u2019 our speech recognition model is. Using a smaller `),wa=i(bs,"CODE",{});var gp=h(wa);wr=l(gp,"stream_chunk_s"),gp.forEach(t),yr=l(bs,` lends
itself to more real-time speech recognition, since we divide our input audio into smaller chunks and transcribe them on
the fly. However, this comes at the expense of poorer accuracy, since there\u2019s less context for the model to infer from.`),bs.forEach(t),Ho=p(e),A=i(e,"P",{});var Be=h(A);gr=l(Be,"As we\u2019re transcribing the speech, we also need to have an idea of when the user "),ya=i(Be,"STRONG",{});var bp=h(ya);br=l(bp,"stops"),bp.forEach(t),Mr=l(Be,` speaking, so that we can terminate
the recording. For simplicity, we\u2019ll terminate our microphone recording after the first `),ga=i(Be,"CODE",{});var Mp=h(ga);vr=l(Mp,"chunk_length_s"),Mp.forEach(t),kr=l(Be,` (which is set to
5 seconds by default), but you can experiment with using a `),dt=i(Be,"A",{href:!0,rel:!0});var vp=h(dt);Tr=l(vp,"voice activity detection (VAD)"),vp.forEach(t),Jr=l(Be,`
model to predict when the user has stopped speaking.`),Be.forEach(t),Vo=p(e),f(ft.$$.fragment,e),qo=p(e),ls=i(e,"P",{});var kp=h(ls);Ur=l(kp,`Let\u2019s give this a go and see how we get on! Once the microphone is live, start speaking and watch your transcription
appear in semi real-time:`),kp.forEach(t),Yo=p(e),f(mt.$$.fragment,e),Lo=p(e),f(wt.$$.fragment,e),Qo=p(e),Z=i(e,"P",{});var Ge=h(Z);_r=l(Ge,"Nice! You can adjust the maximum audio length "),ba=i(Ge,"CODE",{});var Tp=h(ba);jr=l(Tp,"chunk_length_s"),Tp.forEach(t),Er=l(Ge,` based on how fast or slow you speak (increase it if you
felt like you didn\u2019t have enough time to speak, decrease it if you were left waiting at the end), and the
`),Ma=i(Ge,"CODE",{});var Jp=h(Ma);Ir=l(Jp,"stream_chunk_s"),Jp.forEach(t),$r=l(Ge," for the real-time factor. Just pass these as arguments to the "),va=i(Ge,"CODE",{});var Up=h(va);Wr=l(Up,"transcribe"),Up.forEach(t),Cr=l(Ge," function."),Ge.forEach(t),Ro=p(e),O=i(e,"H2",{class:!0});var Pl=h(O);ve=i(Pl,"A",{id:!0,class:!0,href:!0});var _p=h(ve);ka=i(_p,"SPAN",{});var jp=h(ka);f(yt.$$.fragment,jp),jp.forEach(t),_p.forEach(t),Ar=p(Pl),Ta=i(Pl,"SPAN",{});var Ep=h(Ta);Zr=l(Ep,"Language model query"),Ep.forEach(t),Pl.forEach(t),Xo=p(e),ke=i(e,"P",{});var Dl=h(ke);Sr=l(Dl,`Now that we have our spoken query transcribed, we want to generate a meaningful response. To do this, we\u2019ll use an LLM
hosted on the Cloud. Specifically, we\u2019ll pick an LLM on the Hugging Face Hub and use the `),gt=i(Dl,"A",{href:!0,rel:!0});var Ip=h(gt);Nr=l(Ip,"Inference API"),Ip.forEach(t),Br=l(Dl,`
to easily query the model.`),Dl.forEach(t),Fo=p(e),Te=i(e,"P",{});var Ol=h(Te);Gr=l(Ol,"First, let\u2019s head over to the Hugging Face Hub. To find our LLM, we\u2019ll use the "),bt=i(Ol,"A",{href:!0,rel:!0});var $p=h(bt);xr=l($p,"\u{1F917} Open LLM Leaderboard"),$p.forEach(t),Hr=l(Ol,`,
a Space that ranks LLM models by performance over four generation tasks. We\u2019ll search by \u201Cinstruct\u201D to filter out models
that have been instruction fine-tuned, since these should work better for our querying task:`),Ol.forEach(t),zo=p(e),Mt=i(e,"DIV",{class:!0});var Wp=h(Mt);Ja=i(Wp,"IMG",{src:!0}),Wp.forEach(t),Po=p(e),H=i(e,"P",{});var Ms=h(H);Vr=l(Ms,"We\u2019ll use the "),vt=i(Ms,"A",{href:!0,rel:!0});var Cp=h(vt);qr=l(Cp,"tiiuae/falcon-7b-instruct"),Cp.forEach(t),Yr=l(Ms," checkpoint by "),kt=i(Ms,"A",{href:!0,rel:!0});var Ap=h(kt);Lr=l(Ap,"TII"),Ap.forEach(t),Qr=l(Ms,`,
a 7B parameter decoder-only LM fine-tuned on a mixture of chat and instruction datasets. You can use any LLM on the Hugging
Face Hub that has the \u201CHosted inference API\u201D enabled, just look out for the widget on the right-side of the model card:`),Ms.forEach(t),Do=p(e),Tt=i(e,"DIV",{class:!0});var Zp=h(Tt);Ua=i(Zp,"IMG",{src:!0}),Zp.forEach(t),Oo=p(e),Je=i(e,"P",{});var Kl=h(Je);Rr=l(Kl,`The Inference API allows us to send a HTTP request from our local machine to the LLM hosted on the Hub, and returns the
response as a `),_a=i(Kl,"CODE",{});var Sp=h(_a);Xr=l(Sp,"json"),Sp.forEach(t),Fr=l(Kl,` file. All we need to provide is our Hugging Face Hub token (which we retrieve directly from our Hugging Face
Hub folder) and the model id of the LLM we wish to query:`),Kl.forEach(t),Ko=p(e),f(Jt.$$.fragment,e),el=p(e),ns=i(e,"P",{});var Np=h(ns);zr=l(Np,"Let\u2019s give it a try with a test input!"),Np.forEach(t),tl=p(e),f(Ut.$$.fragment,e),sl=p(e),f(_t.$$.fragment,e),al=p(e),is=i(e,"P",{});var Bp=h(is);Pr=l(Bp,`You\u2019ll notice just how fast inference is using the Inference API - we only have to send a small number of text tokens
from our local machine to the hosted model, so the communication cost is very low. The LLM is hosted on GPU accelerators,
so inference runs very quickly. Finally, the generated response is transferred back from the model to our local machine,
again with low communication overhead.`),Bp.forEach(t),ol=p(e),K=i(e,"H2",{class:!0});var en=h(K);Ue=i(en,"A",{id:!0,class:!0,href:!0});var Gp=h(Ue);ja=i(Gp,"SPAN",{});var xp=h(ja);f(jt.$$.fragment,xp),xp.forEach(t),Gp.forEach(t),Dr=p(en),Ea=i(en,"SPAN",{});var Hp=h(Ea);Or=l(Hp,"Synthesise speech"),Hp.forEach(t),en.forEach(t),ll=p(e),_e=i(e,"P",{});var tn=h(_e);Kr=l(tn,"And now we\u2019re ready to get the final spoken output! Once again, we\u2019ll use the Microsoft "),Et=i(tn,"A",{href:!0,rel:!0});var Vp=h(Et);eh=l(Vp,"SpeechT5 TTS"),Vp.forEach(t),th=l(tn,`
model for English TTS, but you can use any TTS model of your choice. Let\u2019s go ahead and load the processor and model:`),tn.forEach(t),nl=p(e),f(It.$$.fragment,e),il=p(e),rs=i(e,"P",{});var qp=h(rs);sh=l(qp,"And also the speaker embeddings:"),qp.forEach(t),rl=p(e),f($t.$$.fragment,e),hl=p(e),V=i(e,"P",{});var vs=h(V);ah=l(vs,"We\u2019ll re-use the "),Ia=i(vs,"CODE",{});var Yp=h(Ia);oh=l(Yp,"synthesise"),Yp.forEach(t),lh=l(vs," function that we defined in the previous chapter on "),hs=i(vs,"A",{href:!0});var Lp=h(hs);nh=l(Lp,"Speech-to-speech translation"),Lp.forEach(t),ih=l(vs,":"),vs.forEach(t),cl=p(e),f(Wt.$$.fragment,e),pl=p(e),cs=i(e,"P",{});var Qp=h(cs);rh=l(Qp,"Let\u2019s quickly verify this works as expected:"),Qp.forEach(t),ul=p(e),f(Ct.$$.fragment,e),dl=p(e),ps=i(e,"P",{});var Rp=h(ps);hh=l(Rp,"Nice job \u{1F44D}"),Rp.forEach(t),fl=p(e),ee=i(e,"H2",{class:!0});var sn=h(ee);je=i(sn,"A",{id:!0,class:!0,href:!0});var Xp=h(je);$a=i(Xp,"SPAN",{});var Fp=h($a);f(At.$$.fragment,Fp),Fp.forEach(t),Xp.forEach(t),ch=p(sn),Wa=i(sn,"SPAN",{});var zp=h(Wa);ph=l(zp,"Marvin \u{1F916}"),zp.forEach(t),sn.forEach(t),ml=p(e),Ee=i(e,"P",{});var an=h(Ee);uh=l(an,`Now that we\u2019ve defined a function for each of the four stages of the voice assistant pipeline, all that\u2019s left to do is
piece them together to get our end-to-end voice assistant. We\u2019ll simply concatenate the four stages, starting with
wake word detection (`),Ca=i(an,"CODE",{});var Pp=h(Ca);dh=l(Pp,"launch_fn"),Pp.forEach(t),fh=l(an,"), speech transcription, querying the LLM, and finally speech synthesis."),an.forEach(t),wl=p(e),f(Zt.$$.fragment,e),yl=p(e),us=i(e,"P",{});var Dp=h(us);mh=l(Dp,"Try it out with a few prompts! Here are some examples to get you started:"),Dp.forEach(t),gl=p(e),q=i(e,"UL",{});var ks=h(q);Aa=i(ks,"LI",{});var Op=h(Aa);Za=i(Op,"EM",{});var Kp=h(Za);wh=l(Kp,"What is the hottest country in the world?"),Kp.forEach(t),Op.forEach(t),yh=p(ks),Sa=i(ks,"LI",{});var eu=h(Sa);Na=i(eu,"EM",{});var tu=h(Na);gh=l(tu,"How do Transformer models work?"),tu.forEach(t),eu.forEach(t),bh=p(ks),Ba=i(ks,"LI",{});var su=h(Ba);Ga=i(su,"EM",{});var au=h(Ga);Mh=l(au,"Do you know Spanish?"),au.forEach(t),su.forEach(t),ks.forEach(t),bl=p(e),ds=i(e,"P",{});var ou=h(ds);vh=l(ou,`And with that, we have our end-to-end voice assistant complete, made using the \u{1F917} audio tools you\u2019ve learnt throughout
this course, with a sprinkling of LLM magic at the end. There are several extensions that we could make to improve the
voice assistant. Firstly, the audio classification model classifies 35 different labels. We could use a smaller, more
lightweight binary classification model that only predicts whether the wake word was spoken or not. Secondly, we pre-load
all the models ahead and keep them running on our device. If we wanted to save power, we would only load each model at
the time it was required, and subsequently un-load them afterwards. Thirdly, we\u2019re missing a voice activity detection model
in our transcription function, transcribing for a fixed amount of time, which in some cases is too long, and in others too
short.`),ou.forEach(t),Ml=p(e),te=i(e,"H2",{class:!0});var on=h(te);Ie=i(on,"A",{id:!0,class:!0,href:!0});var lu=h(Ie);xa=i(lu,"SPAN",{});var nu=h(xa);f(St.$$.fragment,nu),nu.forEach(t),lu.forEach(t),kh=p(on),Ha=i(on,"SPAN",{});var iu=h(Ha);Th=l(iu,"Generalise to anything \u{1FA84}"),iu.forEach(t),on.forEach(t),vl=p(e),fs=i(e,"P",{});var ru=h(fs);Jh=l(ru,`So far, we\u2019ve seen how we can generate speech outputs with our voice assistant Marvin. To finish, we\u2019ll demonstrate how
we can generalise these speech outputs to text, audio and image.`),ru.forEach(t),kl=p(e),$e=i(e,"P",{});var ln=h($e);Uh=l(ln,"We\u2019ll use "),Nt=i(ln,"A",{href:!0,rel:!0});var hu=h(Nt);_h=l(hu,"Transformers Agents"),hu.forEach(t),jh=l(ln,` to build our assistant.
Transformers Agents provides a natural language API on top of the \u{1F917} Transformers and Diffusers libraries, interpreting
a natural language input using an LLM with carefully crafted prompts, and using a set of curated tools to provide
multimodal outputs.`),ln.forEach(t),Tl=p(e),Y=i(e,"P",{});var Ts=h(Y);Eh=l(Ts,"Let\u2019s go ahead and instantiate an agent. There are "),Bt=i(Ts,"A",{href:!0,rel:!0});var cu=h(Bt);Ih=l(cu,"three LLMs available"),cu.forEach(t),$h=l(Ts,`
for Transformers Agents, two of which are open-source and free on the Hugging Face Hub. The third is a model from OpenAI
that requires an OpenAI API key. We\u2019ll use the free `),Gt=i(Ts,"A",{href:!0,rel:!0});var pu=h(Gt);Wh=l(pu,"Bigcode Starcoder"),pu.forEach(t),Ch=l(Ts,` model
in this example, but you can also try either of the other LLMs available:`),Ts.forEach(t),Jl=p(e),f(xt.$$.fragment,e),Ul=p(e),We=i(e,"P",{});var nn=h(We);Ah=l(nn,"To use the agent, we simply have to call "),Va=i(nn,"CODE",{});var uu=h(Va);Zh=l(uu,"agent.run"),uu.forEach(t),Sh=l(nn,` with our text prompt. As an example, we\u2019ll get it to generate an
image of a cat \u{1F408} (that hopefully looks a bit better than this emoji):`),nn.forEach(t),_l=p(e),f(Ht.$$.fragment,e),jl=p(e),Vt=i(e,"DIV",{class:!0});var du=h(Vt);qa=i(du,"IMG",{src:!0}),du.forEach(t),El=p(e),f(Ce.$$.fragment,e),Il=p(e),Ae=i(e,"P",{});var rn=h(Ae);Nh=l(rn,"Easy as that! The Agent interpreted our prompt, and used "),qt=i(rn,"A",{href:!0,rel:!0});var fu=h(qt);Bh=l(fu,"Stable Diffusion"),fu.forEach(t),Gh=l(rn,`
under the hood to generate the image, without us having to worry about loading the model, writing the function or executing
the code.`),rn.forEach(t),$l=p(e),ms=i(e,"P",{});var mu=h(ms);xh=l(mu,`We can now replace our LLM query function and text synthesis step with our Transformers Agent in our voice assistant,
since the Agent is going to take care of both of these steps for us:`),mu.forEach(t),Wl=p(e),f(Yt.$$.fragment,e),Cl=p(e),ws=i(e,"P",{});var wu=h(ws);Hh=l(wu,`Try speaking the same prompt \u201CGenerate an image of a cat\u201D and see how the system gets on. If you ask the Agent a simple
question / answer query, the Agent will respond with a text answer. You can encourage it to generate multimodal outputs
by asking it to return an image or speech. For example, you can ask it to: \u201CGenerate an image of a cat, caption it, and
speak the caption\u201D.`),wu.forEach(t),Al=p(e),Ze=i(e,"P",{});var hn=h(Ze);Vh=l(hn,`While the Agent is more flexible than our first iteration Marvin \u{1F916} assistant, generalising the voice assistant task in this way
may lead to inferior performance on standard voice assistant queries. To recover performance, you can try using a
more performant LLM checkpoint, such as the one from OpenAI, or define a set of `),Lt=i(hn,"A",{href:!0,rel:!0});var yu=h(Lt);qh=l(yu,"custom tools"),yu.forEach(t),Yh=l(hn,`
that are specific to the voice assistant task.`),hn.forEach(t),this.h()},h(){u(b,"name","hf:doc:metadata"),u(b,"content",JSON.stringify(ju)),u(E,"id","creating-a-voice-assistant"),u(E,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(E,"href","#creating-a-voice-assistant"),u(U,"class","relative group"),Rt(Us.src,Rh="https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/voice_assistant.png")||u(Us,"src",Rh),u(He,"class","flex justify-center"),u(le,"id","1-wake-word-detection"),u(le,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(le,"href","#1-wake-word-detection"),u(R,"class","relative group"),u(ne,"id","2-speech-transcription"),u(ne,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ne,"href","#2-speech-transcription"),u(X,"class","relative group"),u(ie,"id","3-language-model-query"),u(ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ie,"href","#3-language-model-query"),u(F,"class","relative group"),u(he,"id","4-synthesise-speech"),u(he,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(he,"href","#4-synthesise-speech"),u(z,"class","relative group"),u(pe,"id","wake-word-detection"),u(pe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(pe,"href","#wake-word-detection"),u(P,"class","relative group"),u(ts,"href","../chapter4/classification_models"),u(Re,"href","https://huggingface.co/datasets/speech_commands"),u(Re,"rel","nofollow"),u(Xe,"href","https://huggingface.co/datasets/speech_commands/viewer/v0.01/train"),u(Xe,"rel","nofollow"),u(ze,"href","https://huggingface.co/models"),u(ze,"rel","nofollow"),Rt(qs.src,Xh="https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/all_models.png")||u(qs,"src",Xh),u(Pe,"class","flex justify-center"),Rt(Ys.src,Fh="https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/by_audio_classification.png")||u(Ys,"src",Fh),u(De,"class","flex justify-center"),Rt(Qs.src,zh="https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/by_speech_commands.png")||u(Qs,"src",zh),u(Oe,"class","flex justify-center"),u(Ke,"href","https://huggingface.co/MIT/ast-finetuned-speech-commands-v2"),u(Ke,"rel","nofollow"),u(ot,"href","https://github.com/huggingface/transformers/blob/fb78769b9c053876ed7ae152ee995b0439a4462a/src/transformers/pipelines/audio_utils.py#L98"),u(ot,"rel","nofollow"),u(lt,"href","https://en.wikipedia.org/wiki/Entropy_(information_theory)"),u(lt,"rel","nofollow"),u(be,"id","speech-transcription"),u(be,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(be,"href","#speech-transcription"),u(D,"class","relative group"),u(ct,"href","https://huggingface.co/openai/whisper-base.en"),u(ct,"rel","nofollow"),u(pt,"href","https://huggingface.co/models?pipeline_tag=automatic-speech-recognition&library=transformers&sort=trending"),u(pt,"rel","nofollow"),u(dt,"href","https://huggingface.co/models?pipeline_tag=voice-activity-detection&sort=trending"),u(dt,"rel","nofollow"),u(ve,"id","language-model-query"),u(ve,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ve,"href","#language-model-query"),u(O,"class","relative group"),u(gt,"href","https://huggingface.co/inference-api"),u(gt,"rel","nofollow"),u(bt,"href","https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard"),u(bt,"rel","nofollow"),Rt(Ja.src,Ph="https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/llm_leaderboard.png")||u(Ja,"src",Ph),u(Mt,"class","flex justify-center"),u(vt,"href","https://huggingface.co/tiiuae/falcon-7b-instruct"),u(vt,"rel","nofollow"),u(kt,"href","https://www.tii.ae/"),u(kt,"rel","nofollow"),Rt(Ua.src,Dh="https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/inference_api.png")||u(Ua,"src",Dh),u(Tt,"class","flex justify-center"),u(Ue,"id","synthesise-speech"),u(Ue,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Ue,"href","#synthesise-speech"),u(K,"class","relative group"),u(Et,"href","https://huggingface.co/microsoft/speecht5_tts"),u(Et,"rel","nofollow"),u(hs,"href","speech-to-speech"),u(je,"id","marvin"),u(je,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(je,"href","#marvin"),u(ee,"class","relative group"),u(Ie,"id","generalise-to-anything"),u(Ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Ie,"href","#generalise-to-anything"),u(te,"class","relative group"),u(Nt,"href","https://huggingface.co/docs/transformers/transformers_agents"),u(Nt,"rel","nofollow"),u(Bt,"href","https://huggingface.co/docs/transformers/transformers_agents#quickstart"),u(Bt,"rel","nofollow"),u(Gt,"href","https://huggingface.co/bigcode/starcoder"),u(Gt,"rel","nofollow"),Rt(qa.src,Oh="https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/generated_cat.png")||u(qa,"src",Oh),u(Vt,"class","flex justify-center"),u(qt,"href","https://huggingface.co/docs/diffusers/using-diffusers/conditional_image_generation"),u(qt,"rel","nofollow"),u(Lt,"href","https://huggingface.co/docs/transformers/transformers_agents#custom-tools"),u(Lt,"rel","nofollow")},m(e,a){s(document.head,b),r(e,M,a),r(e,U,a),s(U,E),s(E,I),m(S,I,null),s(U,ae),s(U,v),s(v,$),r(e,xe,a),r(e,oe,a),s(oe,cn),s(oe,Js),s(Js,pn),s(oe,un),r(e,Qa,a),r(e,Xt,a),s(Xt,dn),r(e,Ra,a),r(e,He,a),s(He,Us),r(e,Xa,a),r(e,R,a),s(R,le),s(le,_s),m(Ve,_s,null),s(R,fn),s(R,js),s(js,mn),r(e,Fa,a),r(e,Ft,a),s(Ft,wn),r(e,za,a),r(e,zt,a),s(zt,yn),r(e,Pa,a),r(e,X,a),s(X,ne),s(ne,Es),m(qe,Es,null),s(X,gn),s(X,Is),s(Is,bn),r(e,Da,a),r(e,Pt,a),s(Pt,Mn),r(e,Oa,a),r(e,Dt,a),s(Dt,vn),r(e,Ka,a),r(e,F,a),s(F,ie),s(ie,$s),m(Ye,$s,null),s(F,kn),s(F,Ws),s(Ws,Tn),r(e,eo,a),r(e,re,a),s(re,Jn),s(re,Cs),s(Cs,Un),s(re,_n),r(e,to,a),r(e,Ot,a),s(Ot,jn),r(e,so,a),r(e,z,a),s(z,he),s(he,As),m(Le,As,null),s(z,En),s(z,Zs),s(Zs,In),r(e,ao,a),r(e,Kt,a),s(Kt,$n),r(e,oo,a),r(e,es,a),s(es,Wn),r(e,lo,a),m(ce,e,a),r(e,no,a),r(e,P,a),s(P,pe),s(pe,Ss),m(Qe,Ss,null),s(P,Cn),s(P,Ns),s(Ns,An),r(e,io,a),r(e,k,a),s(k,Zn),s(k,ts),s(ts,Sn),s(k,Nn),s(k,Re),s(Re,Bn),s(k,Gn),s(k,Bs),s(Bs,xn),s(k,Hn),s(k,Gs),s(Gs,Vn),s(k,qn),s(k,xs),s(xs,Yn),s(k,Ln),s(k,Hs),s(Hs,Qn),s(k,Rn),s(k,Vs),s(Vs,Xn),s(k,Fn),s(k,Xe),s(Xe,zn),s(k,Pn),r(e,ro,a),r(e,ss,a),s(ss,Dn),r(e,ho,a),r(e,Fe,a),s(Fe,On),s(Fe,ze),s(ze,Kn),r(e,co,a),r(e,as,a),s(as,ei),r(e,po,a),r(e,Pe,a),s(Pe,qs),r(e,uo,a),r(e,os,a),s(os,ti),r(e,fo,a),r(e,De,a),s(De,Ys),r(e,mo,a),r(e,ue,a),s(ue,si),s(ue,Ls),s(Ls,ai),s(ue,oi),r(e,wo,a),r(e,Oe,a),s(Oe,Qs),r(e,yo,a),r(e,de,a),s(de,li),s(de,Ke),s(Ke,ni),s(de,ii),r(e,go,a),r(e,fe,a),s(fe,ri),s(fe,Rs),s(Rs,hi),s(fe,ci),r(e,bo,a),m(et,e,a),r(e,Mo,a),r(e,me,a),s(me,pi),s(me,Xs),s(Xs,ui),s(me,di),r(e,vo,a),m(tt,e,a),r(e,ko,a),r(e,_,a),s(_,fi),s(_,Fs),s(Fs,mi),s(_,wi),s(_,zs),s(zs,yi),s(_,gi),s(_,Ps),s(Ps,bi),s(_,Mi),s(_,Ds),s(Ds,vi),s(_,ki),r(e,To,a),m(st,e,a),r(e,Jo,a),m(at,e,a),r(e,Uo,a),r(e,we,a),s(we,Ti),s(we,Os),s(Os,Ji),s(we,Ui),r(e,_o,a),r(e,ye,a),s(ye,_i),s(ye,ot),s(ot,Ks),s(Ks,ji),s(ye,Ei),r(e,jo,a),r(e,j,a),s(j,Ii),s(j,ea),s(ea,$i),s(j,Wi),s(j,ta),s(ta,Ci),s(j,Ai),s(j,sa),s(sa,Zi),s(j,Si),s(j,aa),s(aa,Ni),s(j,Bi),r(e,Eo,a),r(e,W,a),s(W,Gi),s(W,oa),s(oa,xi),s(W,Hi),s(W,la),s(la,Vi),s(W,qi),s(W,na),s(na,Yi),s(W,Li),r(e,Io,a),r(e,B,a),s(B,Qi),s(B,ia),s(ia,Ri),s(B,Xi),s(B,lt),s(lt,ra),s(ra,Fi),s(B,zi),r(e,$o,a),m(nt,e,a),r(e,Wo,a),r(e,C,a),s(C,Pi),s(C,ha),s(ha,Di),s(C,Oi),s(C,ca),s(ca,Ki),s(C,er),s(C,pa),s(pa,tr),s(C,sr),r(e,Co,a),m(it,e,a),r(e,Ao,a),m(rt,e,a),r(e,Zo,a),r(e,ge,a),s(ge,ar),s(ge,ua),s(ua,or),s(ge,lr),r(e,So,a),r(e,D,a),s(D,be),s(be,da),m(ht,da,null),s(D,nr),s(D,fa),s(fa,ir),r(e,No,a),r(e,G,a),s(G,rr),s(G,ct),s(ct,hr),s(G,cr),s(G,pt),s(pt,pr),s(G,ur),r(e,Bo,a),m(ut,e,a),r(e,Go,a),m(Me,e,a),r(e,xo,a),r(e,x,a),s(x,dr),s(x,ma),s(ma,fr),s(x,mr),s(x,wa),s(wa,wr),s(x,yr),r(e,Ho,a),r(e,A,a),s(A,gr),s(A,ya),s(ya,br),s(A,Mr),s(A,ga),s(ga,vr),s(A,kr),s(A,dt),s(dt,Tr),s(A,Jr),r(e,Vo,a),m(ft,e,a),r(e,qo,a),r(e,ls,a),s(ls,Ur),r(e,Yo,a),m(mt,e,a),r(e,Lo,a),m(wt,e,a),r(e,Qo,a),r(e,Z,a),s(Z,_r),s(Z,ba),s(ba,jr),s(Z,Er),s(Z,Ma),s(Ma,Ir),s(Z,$r),s(Z,va),s(va,Wr),s(Z,Cr),r(e,Ro,a),r(e,O,a),s(O,ve),s(ve,ka),m(yt,ka,null),s(O,Ar),s(O,Ta),s(Ta,Zr),r(e,Xo,a),r(e,ke,a),s(ke,Sr),s(ke,gt),s(gt,Nr),s(ke,Br),r(e,Fo,a),r(e,Te,a),s(Te,Gr),s(Te,bt),s(bt,xr),s(Te,Hr),r(e,zo,a),r(e,Mt,a),s(Mt,Ja),r(e,Po,a),r(e,H,a),s(H,Vr),s(H,vt),s(vt,qr),s(H,Yr),s(H,kt),s(kt,Lr),s(H,Qr),r(e,Do,a),r(e,Tt,a),s(Tt,Ua),r(e,Oo,a),r(e,Je,a),s(Je,Rr),s(Je,_a),s(_a,Xr),s(Je,Fr),r(e,Ko,a),m(Jt,e,a),r(e,el,a),r(e,ns,a),s(ns,zr),r(e,tl,a),m(Ut,e,a),r(e,sl,a),m(_t,e,a),r(e,al,a),r(e,is,a),s(is,Pr),r(e,ol,a),r(e,K,a),s(K,Ue),s(Ue,ja),m(jt,ja,null),s(K,Dr),s(K,Ea),s(Ea,Or),r(e,ll,a),r(e,_e,a),s(_e,Kr),s(_e,Et),s(Et,eh),s(_e,th),r(e,nl,a),m(It,e,a),r(e,il,a),r(e,rs,a),s(rs,sh),r(e,rl,a),m($t,e,a),r(e,hl,a),r(e,V,a),s(V,ah),s(V,Ia),s(Ia,oh),s(V,lh),s(V,hs),s(hs,nh),s(V,ih),r(e,cl,a),m(Wt,e,a),r(e,pl,a),r(e,cs,a),s(cs,rh),r(e,ul,a),m(Ct,e,a),r(e,dl,a),r(e,ps,a),s(ps,hh),r(e,fl,a),r(e,ee,a),s(ee,je),s(je,$a),m(At,$a,null),s(ee,ch),s(ee,Wa),s(Wa,ph),r(e,ml,a),r(e,Ee,a),s(Ee,uh),s(Ee,Ca),s(Ca,dh),s(Ee,fh),r(e,wl,a),m(Zt,e,a),r(e,yl,a),r(e,us,a),s(us,mh),r(e,gl,a),r(e,q,a),s(q,Aa),s(Aa,Za),s(Za,wh),s(q,yh),s(q,Sa),s(Sa,Na),s(Na,gh),s(q,bh),s(q,Ba),s(Ba,Ga),s(Ga,Mh),r(e,bl,a),r(e,ds,a),s(ds,vh),r(e,Ml,a),r(e,te,a),s(te,Ie),s(Ie,xa),m(St,xa,null),s(te,kh),s(te,Ha),s(Ha,Th),r(e,vl,a),r(e,fs,a),s(fs,Jh),r(e,kl,a),r(e,$e,a),s($e,Uh),s($e,Nt),s(Nt,_h),s($e,jh),r(e,Tl,a),r(e,Y,a),s(Y,Eh),s(Y,Bt),s(Bt,Ih),s(Y,$h),s(Y,Gt),s(Gt,Wh),s(Y,Ch),r(e,Jl,a),m(xt,e,a),r(e,Ul,a),r(e,We,a),s(We,Ah),s(We,Va),s(Va,Zh),s(We,Sh),r(e,_l,a),m(Ht,e,a),r(e,jl,a),r(e,Vt,a),s(Vt,qa),r(e,El,a),m(Ce,e,a),r(e,Il,a),r(e,Ae,a),s(Ae,Nh),s(Ae,qt),s(qt,Bh),s(Ae,Gh),r(e,$l,a),r(e,ms,a),s(ms,xh),r(e,Wl,a),m(Yt,e,a),r(e,Cl,a),r(e,ws,a),s(ws,Hh),r(e,Al,a),r(e,Ze,a),s(Ze,Vh),s(Ze,Lt),s(Lt,qh),s(Ze,Yh),Zl=!0},p(e,[a]){const Qt={};a&2&&(Qt.$$scope={dirty:a,ctx:e}),ce.$set(Qt);const Ya={};a&2&&(Ya.$$scope={dirty:a,ctx:e}),Me.$set(Ya);const La={};a&2&&(La.$$scope={dirty:a,ctx:e}),Ce.$set(La)},i(e){Zl||(w(S.$$.fragment,e),w(Ve.$$.fragment,e),w(qe.$$.fragment,e),w(Ye.$$.fragment,e),w(Le.$$.fragment,e),w(ce.$$.fragment,e),w(Qe.$$.fragment,e),w(et.$$.fragment,e),w(tt.$$.fragment,e),w(st.$$.fragment,e),w(at.$$.fragment,e),w(nt.$$.fragment,e),w(it.$$.fragment,e),w(rt.$$.fragment,e),w(ht.$$.fragment,e),w(ut.$$.fragment,e),w(Me.$$.fragment,e),w(ft.$$.fragment,e),w(mt.$$.fragment,e),w(wt.$$.fragment,e),w(yt.$$.fragment,e),w(Jt.$$.fragment,e),w(Ut.$$.fragment,e),w(_t.$$.fragment,e),w(jt.$$.fragment,e),w(It.$$.fragment,e),w($t.$$.fragment,e),w(Wt.$$.fragment,e),w(Ct.$$.fragment,e),w(At.$$.fragment,e),w(Zt.$$.fragment,e),w(St.$$.fragment,e),w(xt.$$.fragment,e),w(Ht.$$.fragment,e),w(Ce.$$.fragment,e),w(Yt.$$.fragment,e),Zl=!0)},o(e){y(S.$$.fragment,e),y(Ve.$$.fragment,e),y(qe.$$.fragment,e),y(Ye.$$.fragment,e),y(Le.$$.fragment,e),y(ce.$$.fragment,e),y(Qe.$$.fragment,e),y(et.$$.fragment,e),y(tt.$$.fragment,e),y(st.$$.fragment,e),y(at.$$.fragment,e),y(nt.$$.fragment,e),y(it.$$.fragment,e),y(rt.$$.fragment,e),y(ht.$$.fragment,e),y(ut.$$.fragment,e),y(Me.$$.fragment,e),y(ft.$$.fragment,e),y(mt.$$.fragment,e),y(wt.$$.fragment,e),y(yt.$$.fragment,e),y(Jt.$$.fragment,e),y(Ut.$$.fragment,e),y(_t.$$.fragment,e),y(jt.$$.fragment,e),y(It.$$.fragment,e),y($t.$$.fragment,e),y(Wt.$$.fragment,e),y(Ct.$$.fragment,e),y(At.$$.fragment,e),y(Zt.$$.fragment,e),y(St.$$.fragment,e),y(xt.$$.fragment,e),y(Ht.$$.fragment,e),y(Ce.$$.fragment,e),y(Yt.$$.fragment,e),Zl=!1},d(e){t(b),e&&t(M),e&&t(U),g(S),e&&t(xe),e&&t(oe),e&&t(Qa),e&&t(Xt),e&&t(Ra),e&&t(He),e&&t(Xa),e&&t(R),g(Ve),e&&t(Fa),e&&t(Ft),e&&t(za),e&&t(zt),e&&t(Pa),e&&t(X),g(qe),e&&t(Da),e&&t(Pt),e&&t(Oa),e&&t(Dt),e&&t(Ka),e&&t(F),g(Ye),e&&t(eo),e&&t(re),e&&t(to),e&&t(Ot),e&&t(so),e&&t(z),g(Le),e&&t(ao),e&&t(Kt),e&&t(oo),e&&t(es),e&&t(lo),g(ce,e),e&&t(no),e&&t(P),g(Qe),e&&t(io),e&&t(k),e&&t(ro),e&&t(ss),e&&t(ho),e&&t(Fe),e&&t(co),e&&t(as),e&&t(po),e&&t(Pe),e&&t(uo),e&&t(os),e&&t(fo),e&&t(De),e&&t(mo),e&&t(ue),e&&t(wo),e&&t(Oe),e&&t(yo),e&&t(de),e&&t(go),e&&t(fe),e&&t(bo),g(et,e),e&&t(Mo),e&&t(me),e&&t(vo),g(tt,e),e&&t(ko),e&&t(_),e&&t(To),g(st,e),e&&t(Jo),g(at,e),e&&t(Uo),e&&t(we),e&&t(_o),e&&t(ye),e&&t(jo),e&&t(j),e&&t(Eo),e&&t(W),e&&t(Io),e&&t(B),e&&t($o),g(nt,e),e&&t(Wo),e&&t(C),e&&t(Co),g(it,e),e&&t(Ao),g(rt,e),e&&t(Zo),e&&t(ge),e&&t(So),e&&t(D),g(ht),e&&t(No),e&&t(G),e&&t(Bo),g(ut,e),e&&t(Go),g(Me,e),e&&t(xo),e&&t(x),e&&t(Ho),e&&t(A),e&&t(Vo),g(ft,e),e&&t(qo),e&&t(ls),e&&t(Yo),g(mt,e),e&&t(Lo),g(wt,e),e&&t(Qo),e&&t(Z),e&&t(Ro),e&&t(O),g(yt),e&&t(Xo),e&&t(ke),e&&t(Fo),e&&t(Te),e&&t(zo),e&&t(Mt),e&&t(Po),e&&t(H),e&&t(Do),e&&t(Tt),e&&t(Oo),e&&t(Je),e&&t(Ko),g(Jt,e),e&&t(el),e&&t(ns),e&&t(tl),g(Ut,e),e&&t(sl),g(_t,e),e&&t(al),e&&t(is),e&&t(ol),e&&t(K),g(jt),e&&t(ll),e&&t(_e),e&&t(nl),g(It,e),e&&t(il),e&&t(rs),e&&t(rl),g($t,e),e&&t(hl),e&&t(V),e&&t(cl),g(Wt,e),e&&t(pl),e&&t(cs),e&&t(ul),g(Ct,e),e&&t(dl),e&&t(ps),e&&t(fl),e&&t(ee),g(At),e&&t(ml),e&&t(Ee),e&&t(wl),g(Zt,e),e&&t(yl),e&&t(us),e&&t(gl),e&&t(q),e&&t(bl),e&&t(ds),e&&t(Ml),e&&t(te),g(St),e&&t(vl),e&&t(fs),e&&t(kl),e&&t($e),e&&t(Tl),e&&t(Y),e&&t(Jl),g(xt,e),e&&t(Ul),e&&t(We),e&&t(_l),g(Ht,e),e&&t(jl),e&&t(Vt),e&&t(El),g(Ce,e),e&&t(Il),e&&t(Ae),e&&t($l),e&&t(ms),e&&t(Wl),g(Yt,e),e&&t(Cl),e&&t(ws),e&&t(Al),e&&t(Ze)}}}const ju={local:"creating-a-voice-assistant",sections:[{local:null,sections:[{local:"1-wake-word-detection",title:"1. Wake word detection"},{local:"2-speech-transcription",title:"2. Speech transcription"},{local:"3-language-model-query",title:"3. Language model query"},{local:"4-synthesise-speech",title:"4. Synthesise speech"}],title:null},{local:"wake-word-detection",title:"Wake word detection"},{local:"speech-transcription",title:"Speech transcription"},{local:"language-model-query",title:"Language model query"},{local:"synthesise-speech",title:"Synthesise speech"},{local:"marvin",title:"Marvin \u{1F916}"},{local:"generalise-to-anything",title:"Generalise to anything \u{1FA84}"}],title:"Creating a voice assistant"};function Eu(se){return ku(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Au extends gu{constructor(b){super();bu(this,b,Eu,_u,Mu,{})}}export{Au as default,ju as metadata};
