import{S as Wh,i as Gh,s as Ch,e as a,k as c,w as d,t as n,M as kh,c as i,d as t,m,a as o,x as M,h as r,b as f,N as ne,G as l,g as p,y as w,q as y,o as J,B as T,v as Xh}from"../../chunks/vendor-hf-doc-builder.js";import{T as en}from"../../chunks/Tip-hf-doc-builder.js";import{I as nl}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as U}from"../../chunks/CodeBlock-hf-doc-builder.js";function Vh(A){let h,v,u,j,_;return{c(){h=a("p"),v=n(`It is useful to look at some inference samples while a model is training to measure the
training progress. In our `),u=a("a"),j=n("training scripts"),_=n(`, we support this utility with additional support for
logging to TensorBoard and Weights & Biases.`),this.h()},l(I){h=i(I,"P",{});var g=o(h);v=r(g,`It is useful to look at some inference samples while a model is training to measure the
training progress. In our `),u=i(g,"A",{href:!0,rel:!0});var Z=o(u);j=r(Z,"training scripts"),Z.forEach(t),_=r(g,`, we support this utility with additional support for
logging to TensorBoard and Weights & Biases.`),g.forEach(t),this.h()},h(){f(u,"href","https://github.com/huggingface/diffusers/tree/main/examples/"),f(u,"rel","nofollow")},m(I,g){p(I,h,g),l(h,v),l(h,u),l(u,j),l(h,_)},d(I){I&&t(h)}}}function Nh(A){let h,v,u,j,_;return{c(){h=a("p"),v=n(`By construction, there are some limitations in this score. The captions in the training dataset
were crawled from the web and extracted from `),u=a("code"),j=n("alt"),_=n(` and similar tags associated an image on the internet.
They are not necessarily representative of what a human being would use to describe an image. Hence we
had to \u201Cengineer\u201D some prompts here.`)},l(I){h=i(I,"P",{});var g=o(h);v=r(g,`By construction, there are some limitations in this score. The captions in the training dataset
were crawled from the web and extracted from `),u=i(g,"CODE",{});var Z=o(u);j=r(Z,"alt"),Z.forEach(t),_=r(g,` and similar tags associated an image on the internet.
They are not necessarily representative of what a human being would use to describe an image. Hence we
had to \u201Cengineer\u201D some prompts here.`),g.forEach(t)},m(I,g){p(I,h,g),l(h,v),l(h,u),l(u,j),l(h,_)},d(I){I&&t(h)}}}function Rh(A){let h,v;return{c(){h=a("p"),v=n("Both CLIP score and CLIP direction similarity rely on the CLIP model, which can make the evaluations biased.")},l(u){h=i(u,"P",{});var j=o(h);v=r(j,"Both CLIP score and CLIP direction similarity rely on the CLIP model, which can make the evaluations biased."),j.forEach(t)},m(u,j){p(u,h,j),l(h,v)},d(u){u&&t(h)}}}function Fh(A){let h,v,u,j,_,I,g,Z,ze,re,E,S,He,R,k,pe,W,De;return{c(){h=a("p"),v=n("FID results tend to be fragile as they depend on a lot of factors:"),u=c(),j=a("ul"),_=a("li"),I=n("The specific Inception model used during computation."),g=c(),Z=a("li"),ze=n("The implementation accuracy of the computation."),re=c(),E=a("li"),S=n("The image format (not the same if we start from PNGs vs JPGs)."),He=c(),R=a("p"),k=n(`Keeping that in mind, FID is often most useful when comparing similar runs, but it is
hard to reproduce paper results unless the authors carefully disclose the FID
measurement code.`),pe=c(),W=a("p"),De=n("These points apply to other related metrics too, such as KID and IS.")},l(b){h=i(b,"P",{});var B=o(h);v=r(B,"FID results tend to be fragile as they depend on a lot of factors:"),B.forEach(t),u=m(b),j=i(b,"UL",{});var C=o(j);_=i(C,"LI",{});var rl=o(_);I=r(rl,"The specific Inception model used during computation."),rl.forEach(t),g=m(C),Z=i(C,"LI",{});var qe=o(Z);ze=r(qe,"The implementation accuracy of the computation."),qe.forEach(t),re=m(C),E=i(C,"LI",{});var F=o(E);S=r(F,"The image format (not the same if we start from PNGs vs JPGs)."),F.forEach(t),C.forEach(t),He=m(b),R=i(b,"P",{});var pl=o(R);k=r(pl,`Keeping that in mind, FID is often most useful when comparing similar runs, but it is
hard to reproduce paper results unless the authors carefully disclose the FID
measurement code.`),pl.forEach(t),pe=m(b),W=i(b,"P",{});var ce=o(W);De=r(ce,"These points apply to other related metrics too, such as KID and IS."),ce.forEach(t)},m(b,B){p(b,h,B),l(h,v),p(b,u,B),p(b,j,B),l(j,_),l(_,I),l(j,g),l(j,Z),l(Z,ze),l(j,re),l(j,E),l(E,S),p(b,He,B),p(b,R,B),l(R,k),p(b,pe,B),p(b,W,B),l(W,De)},d(b){b&&t(h),b&&t(u),b&&t(j),b&&t(He),b&&t(R),b&&t(pe),b&&t(W)}}}function Ah(A){let h,v,u,j,_,I,g,Z,ze,re,E,S,He,R,k,pe,W,De,b,B,C,rl,qe,F,pl,ce,tn,ln,Ua,me,sn,Le,an,on,ga,O,fe,us,Ke,nn,ds,rn,va,cl,pn,_a,x,Oe,cn,et,Ms,mn,fn,hn,ee,un,tt,ws,dn,Mn,lt,ys,wn,yn,Jn,st,Tn,at,Js,jn,bn,Za,te,he,Ts,it,In,js,Un,Ba,$,gn,ot,vn,_n,nt,Zn,Bn,Ea,ue,En,rt,Wn,Gn,Wa,ml,bs,Cn,Ga,fl,hl,qc,Ca,ul,kn,ka,P,Is,Xn,Vn,Us,Nn,Rn,gs,Fn,Xa,dl,An,Va,de,Sn,vs,xn,$n,Na,Me,Ml,pt,Pn,Yn,Qn,wl,ct,zn,Hn,Ra,we,Dn,_s,qn,Ln,Fa,ye,Kn,mt,On,er,Aa,ft,Sa,Je,tr,ht,lr,sr,xa,ut,$a,yl,Jl,Lc,Pa,Y,ar,Zs,ir,or,dt,nr,rr,Ya,Tl,jl,Kc,Qa,bl,pr,za,Te,Ha,le,je,Bs,Mt,cr,Es,mr,Da,Il,fr,qa,Q,Ws,hr,ur,Gs,dr,Mr,Cs,wr,La,se,be,ks,wt,yr,Xs,Jr,Ka,yt,Jt,Tr,jr,Oa,Ie,br,Ul,Ir,Ur,ei,Tt,ti,gl,gr,li,jt,si,vl,vr,ai,bt,ii,_l,_r,oi,z,Zr,Zl,Br,Er,It,Wr,Gr,ni,Ut,ri,Ue,Cr,gt,kr,Xr,pi,vt,ci,Bl,Vr,mi,_t,fi,ge,Nr,Zt,Rr,Fr,hi,ve,ui,ae,_e,Vs,Bt,Ar,Ns,Sr,di,Ze,xr,El,$r,Pr,Mi,Wl,Yr,wi,Gl,Cl,Oc,yi,X,Qr,Et,zr,Hr,Wt,Dr,qr,Rs,Lr,Kr,Ji,Be,Fs,Or,ep,As,tp,Ti,kl,lp,ji,Xl,Vl,em,bi,Nl,sp,Ii,Gt,Ui,Ct,gi,Rl,ap,vi,H,Ee,Ss,ip,op,xs,np,rp,pp,Fl,$s,cp,mp,fp,We,Ps,hp,up,Ys,dp,Mp,_i,Al,wp,Zi,kt,Bi,Xt,Ei,Sl,yp,Wi,Vt,Gi,xl,$l,tm,Ci,Pl,Jp,ki,Ge,Tp,Yl,jp,bp,Xi,Nt,Vi,Ql,Ip,Ni,Rt,Ri,zl,Up,Fi,Ft,Ai,D,gp,Qs,vp,_p,At,Zp,Bp,Si,Ce,Ep,zs,Wp,Gp,xi,St,$i,ke,Cp,Hs,kp,Xp,Pi,xt,Yi,Hl,Vp,Qi,V,Np,Ds,Rp,Fp,qs,Ap,Sp,Ls,xp,$p,zi,Xe,Pp,Ks,Yp,Qp,Hi,Ve,zp,$t,Os,Hp,Dp,Di,Ne,qi,ie,ea,ta,qp,Lp,Pt,Kp,Op,Li,Dl,la,Yt,ec,Qt,tc,lc,Ki,oe,Re,sa,zt,sc,aa,ac,Oi,G,ic,Ht,oc,nc,Dt,rc,pc,qt,ia,cc,mc,Lt,fc,hc,eo,Fe,uc,Kt,dc,Mc,to,ql,oa,wc,lo,Ll,yc,so,Kl,Jc,ao,Ot,io,el,oo,Ol,Tc,no,q,es,lm,jc,bc,na,Ic,ro,ts,Uc,po,tl,co,Ae,gc,ll,ra,vc,_c,mo,sl,fo,Se,Zc,al,pa,Bc,Ec,ho,il,uo,ls,Wc,Mo,N,ca,Gc,Cc,ma,kc,Xc,fa,Vc,Nc,ha,Rc,wo,ss,Fc,yo,xe,Jo,$e,Ac,ua,Sc,xc,To,L,as,sm,$c,Pc,da,Yc,jo;return I=new nl({}),Ke=new nl({}),it=new nl({}),ft=new U({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBJTIzJTIwcHJvbXB0cyUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJuYXRlcmF3JTJGcGFydGktcHJvbXB0cyUyMiUyQyUyMHNwbGl0JTNEJTIydHJhaW4lMjIpJTBBJTIzJTIwcHJvbXB0cyUyMCUzRCUyMHByb21wdHMuc2h1ZmZsZSgpJTBBJTIzJTIwc2FtcGxlX3Byb21wdHMlMjAlM0QlMjAlNUJwcm9tcHRzJTVCaSU1RCU1QiUyMlByb21wdCUyMiU1RCUyMGZvciUyMGklMjBpbiUyMHJhbmdlKDUpJTVEJTBBJTBBJTIzJTIwRml4aW5nJTIwdGhlc2UlMjBzYW1wbGUlMjBwcm9tcHRzJTIwaW4lMjB0aGUlMjBpbnRlcmVzdCUyMG9mJTIwcmVwcm9kdWNpYmlsaXR5LiUwQXNhbXBsZV9wcm9tcHRzJTIwJTNEJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIyYSUyMGNvcmdpJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIyYSUyMGhvdCUyMGFpciUyMGJhbGxvb24lMjB3aXRoJTIwYSUyMHlpbi15YW5nJTIwc3ltYm9sJTJDJTIwd2l0aCUyMHRoZSUyMG1vb24lMjB2aXNpYmxlJTIwaW4lMjB0aGUlMjBkYXl0aW1lJTIwc2t5JTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIyYSUyMGNhciUyMHdpdGglMjBubyUyMHdpbmRvd3MlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJhJTIwY3ViZSUyMG1hZGUlMjBvZiUyMHBvcmN1cGluZSUyMiUyQyUwQSUyMCUyMCUyMCUyMCdUaGUlMjBzYXlpbmclMjAlMjJCRSUyMEVYQ0VMTEVOVCUyMFRPJTIwRUFDSCUyME9USEVSJTIyJTIwd3JpdHRlbiUyMG9uJTIwYSUyMHJlZCUyMGJyaWNrJTIwd2FsbCUyMHdpdGglMjBhJTIwZ3JhZmZpdGklMjBpbWFnZSUyMG9mJTIwYSUyMGdyZWVuJTIwYWxpZW4lMjB3ZWFyaW5nJTIwYSUyMHR1eGVkby4lMjBBJTIweWVsbG93JTIwZmlyZSUyMGh5ZHJhbnQlMjBpcyUyMG9uJTIwYSUyMHNpZGV3YWxrJTIwaW4lMjB0aGUlMjBmb3JlZ3JvdW5kLiclMkMlMEElNUQ=",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-comment"># prompts = load_dataset(&quot;nateraw/parti-prompts&quot;, split=&quot;train&quot;)</span>
<span class="hljs-comment"># prompts = prompts.shuffle()</span>
<span class="hljs-comment"># sample_prompts = [prompts[i][&quot;Prompt&quot;] for i in range(5)]</span>

<span class="hljs-comment"># Fixing these sample prompts in the interest of reproducibility.</span>
sample_prompts = [
    <span class="hljs-string">&quot;a corgi&quot;</span>,
    <span class="hljs-string">&quot;a hot air balloon with a yin-yang symbol, with the moon visible in the daytime sky&quot;</span>,
    <span class="hljs-string">&quot;a car with no windows&quot;</span>,
    <span class="hljs-string">&quot;a cube made of porcupine&quot;</span>,
    <span class="hljs-string">&#x27;The saying &quot;BE EXCELLENT TO EACH OTHER&quot; written on a red brick wall with a graffiti image of a green alien wearing a tuxedo. A yellow fire hydrant is on a sidewalk in the foreground.&#x27;</span>,
]`}}),ut=new U({props:{code:"aW1wb3J0JTIwdG9yY2glMEElMEFzZWVkJTIwJTNEJTIwMCUwQWdlbmVyYXRvciUyMCUzRCUyMHRvcmNoLm1hbnVhbF9zZWVkKHNlZWQpJTBBJTBBaW1hZ2VzJTIwJTNEJTIwc2RfcGlwZWxpbmUoc2FtcGxlX3Byb21wdHMlMkMlMjBudW1faW1hZ2VzX3Blcl9wcm9tcHQlM0QxJTJDJTIwZ2VuZXJhdG9yJTNEZ2VuZXJhdG9yJTJDJTIwb3V0cHV0X3R5cGUlM0QlMjJudW1weSUyMikuaW1hZ2Vz",highlighted:`<span class="hljs-keyword">import</span> torch

seed = <span class="hljs-number">0</span>
generator = torch.manual_seed(seed)

images = sd_pipeline(sample_prompts, num_images_per_prompt=<span class="hljs-number">1</span>, generator=generator, output_type=<span class="hljs-string">&quot;numpy&quot;</span>).images`}}),Te=new en({props:{$$slots:{default:[Vh]},$$scope:{ctx:A}}}),Mt=new nl({}),wt=new nl({}),Tt=new U({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMFN0YWJsZURpZmZ1c2lvblBpcGVsaW5lJTBBaW1wb3J0JTIwdG9yY2glMEElMEFtb2RlbF9ja3B0JTIwJTNEJTIwJTIyQ29tcFZpcyUyRnN0YWJsZS1kaWZmdXNpb24tdjEtNCUyMiUwQXNkX3BpcGVsaW5lJTIwJTNEJTIwU3RhYmxlRGlmZnVzaW9uUGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2NrcHQlMkMlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYpLnRvKCUyMmN1ZGElMjIp",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline
<span class="hljs-keyword">import</span> torch

model_ckpt = <span class="hljs-string">&quot;CompVis/stable-diffusion-v1-4&quot;</span>
sd_pipeline = StableDiffusionPipeline.from_pretrained(model_ckpt, torch_dtype=torch.float16).to(<span class="hljs-string">&quot;cuda&quot;</span>)`}}),jt=new U({props:{code:"cHJvbXB0cyUyMCUzRCUyMCU1QiUwQSUyMCUyMCUyMCUyMCUyMmElMjBwaG90byUyMG9mJTIwYW4lMjBhc3Ryb25hdXQlMjByaWRpbmclMjBhJTIwaG9yc2UlMjBvbiUyMG1hcnMlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJBJTIwaGlnaCUyMHRlY2glMjBzb2xhcnB1bmslMjB1dG9waWElMjBpbiUyMHRoZSUyMEFtYXpvbiUyMHJhaW5mb3Jlc3QlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJBJTIwcGlrYWNodSUyMGZpbmUlMjBkaW5pbmclMjB3aXRoJTIwYSUyMHZpZXclMjB0byUyMHRoZSUyMEVpZmZlbCUyMFRvd2VyJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIyQSUyMG1lY2hhJTIwcm9ib3QlMjBpbiUyMGElMjBmYXZlbGElMjBpbiUyMGV4cHJlc3Npb25pc3QlMjBzdHlsZSUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMmFuJTIwaW5zZWN0JTIwcm9ib3QlMjBwcmVwYXJpbmclMjBhJTIwZGVsaWNpb3VzJTIwbWVhbCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMkElMjBzbWFsbCUyMGNhYmluJTIwb24lMjB0b3AlMjBvZiUyMGElMjBzbm93eSUyMG1vdW50YWluJTIwaW4lMjB0aGUlMjBzdHlsZSUyMG9mJTIwRGlzbmV5JTJDJTIwYXJ0c3RhdGlvbiUyMiUyQyUwQSU1RCUwQSUwQWltYWdlcyUyMCUzRCUyMHNkX3BpcGVsaW5lKHByb21wdHMlMkMlMjBudW1faW1hZ2VzX3Blcl9wcm9tcHQlM0QxJTJDJTIwb3V0cHV0X3R5cGUlM0QlMjJudW1weSUyMikuaW1hZ2VzJTBBJTBBcHJpbnQoaW1hZ2VzLnNoYXBlKSUwQSUyMyUyMCg2JTJDJTIwNTEyJTJDJTIwNTEyJTJDJTIwMyk=",highlighted:`prompts = [
    <span class="hljs-string">&quot;a photo of an astronaut riding a horse on mars&quot;</span>,
    <span class="hljs-string">&quot;A high tech solarpunk utopia in the Amazon rainforest&quot;</span>,
    <span class="hljs-string">&quot;A pikachu fine dining with a view to the Eiffel Tower&quot;</span>,
    <span class="hljs-string">&quot;A mecha robot in a favela in expressionist style&quot;</span>,
    <span class="hljs-string">&quot;an insect robot preparing a delicious meal&quot;</span>,
    <span class="hljs-string">&quot;A small cabin on top of a snowy mountain in the style of Disney, artstation&quot;</span>,
]

images = sd_pipeline(prompts, num_images_per_prompt=<span class="hljs-number">1</span>, output_type=<span class="hljs-string">&quot;numpy&quot;</span>).images

<span class="hljs-built_in">print</span>(images.shape)
<span class="hljs-comment"># (6, 512, 512, 3)</span>`}}),bt=new U({props:{code:"ZnJvbSUyMHRvcmNobWV0cmljcy5mdW5jdGlvbmFsLm11bHRpbW9kYWwlMjBpbXBvcnQlMjBjbGlwX3Njb3JlJTBBZnJvbSUyMGZ1bmN0b29scyUyMGltcG9ydCUyMHBhcnRpYWwlMEElMEFjbGlwX3Njb3JlX2ZuJTIwJTNEJTIwcGFydGlhbChjbGlwX3Njb3JlJTJDJTIwbW9kZWxfbmFtZV9vcl9wYXRoJTNEJTIyb3BlbmFpJTJGY2xpcC12aXQtYmFzZS1wYXRjaDE2JTIyKSUwQSUwQSUwQWRlZiUyMGNhbGN1bGF0ZV9jbGlwX3Njb3JlKGltYWdlcyUyQyUyMHByb21wdHMpJTNBJTBBJTIwJTIwJTIwJTIwaW1hZ2VzX2ludCUyMCUzRCUyMChpbWFnZXMlMjAqJTIwMjU1KS5hc3R5cGUoJTIydWludDglMjIpJTBBJTIwJTIwJTIwJTIwY2xpcF9zY29yZSUyMCUzRCUyMGNsaXBfc2NvcmVfZm4odG9yY2guZnJvbV9udW1weShpbWFnZXNfaW50KS5wZXJtdXRlKDAlMkMlMjAzJTJDJTIwMSUyQyUyMDIpJTJDJTIwcHJvbXB0cykuZGV0YWNoKCklMEElMjAlMjAlMjAlMjByZXR1cm4lMjByb3VuZChmbG9hdChjbGlwX3Njb3JlKSUyQyUyMDQpJTBBJTBBJTBBc2RfY2xpcF9zY29yZSUyMCUzRCUyMGNhbGN1bGF0ZV9jbGlwX3Njb3JlKGltYWdlcyUyQyUyMHByb21wdHMpJTBBcHJpbnQoZiUyMkNMSVAlMjBzY29yZSUzQSUyMCU3QnNkX2NsaXBfc2NvcmUlN0QlMjIpJTBBJTIzJTIwQ0xJUCUyMHNjb3JlJTNBJTIwMzUuNzAzOA==",highlighted:`<span class="hljs-keyword">from</span> torchmetrics.functional.multimodal <span class="hljs-keyword">import</span> clip_score
<span class="hljs-keyword">from</span> functools <span class="hljs-keyword">import</span> partial

clip_score_fn = partial(clip_score, model_name_or_path=<span class="hljs-string">&quot;openai/clip-vit-base-patch16&quot;</span>)


<span class="hljs-keyword">def</span> <span class="hljs-title function_">calculate_clip_score</span>(<span class="hljs-params">images, prompts</span>):
    images_int = (images * <span class="hljs-number">255</span>).astype(<span class="hljs-string">&quot;uint8&quot;</span>)
    clip_score = clip_score_fn(torch.from_numpy(images_int).permute(<span class="hljs-number">0</span>, <span class="hljs-number">3</span>, <span class="hljs-number">1</span>, <span class="hljs-number">2</span>), prompts).detach()
    <span class="hljs-keyword">return</span> <span class="hljs-built_in">round</span>(<span class="hljs-built_in">float</span>(clip_score), <span class="hljs-number">4</span>)


sd_clip_score = calculate_clip_score(images, prompts)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;CLIP score: <span class="hljs-subst">{sd_clip_score}</span>&quot;</span>)
<span class="hljs-comment"># CLIP score: 35.7038</span>`}}),Ut=new U({props:{code:"c2VlZCUyMCUzRCUyMDAlMEFnZW5lcmF0b3IlMjAlM0QlMjB0b3JjaC5tYW51YWxfc2VlZChzZWVkKSUwQSUwQWltYWdlcyUyMCUzRCUyMHNkX3BpcGVsaW5lKHByb21wdHMlMkMlMjBudW1faW1hZ2VzX3Blcl9wcm9tcHQlM0QxJTJDJTIwZ2VuZXJhdG9yJTNEZ2VuZXJhdG9yJTJDJTIwb3V0cHV0X3R5cGUlM0QlMjJudW1weSUyMikuaW1hZ2Vz",highlighted:`seed = <span class="hljs-number">0</span>
generator = torch.manual_seed(seed)

images = sd_pipeline(prompts, num_images_per_prompt=<span class="hljs-number">1</span>, generator=generator, output_type=<span class="hljs-string">&quot;numpy&quot;</span>).images`}}),vt=new U({props:{code:"bW9kZWxfY2twdF8xXzUlMjAlM0QlMjAlMjJydW53YXltbCUyRnN0YWJsZS1kaWZmdXNpb24tdjEtNSUyMiUwQXNkX3BpcGVsaW5lXzFfNSUyMCUzRCUyMFN0YWJsZURpZmZ1c2lvblBpcGVsaW5lLmZyb21fcHJldHJhaW5lZChtb2RlbF9ja3B0XzFfNSUyQyUyMHRvcmNoX2R0eXBlJTNEd2VpZ2h0X2R0eXBlKS50byhkZXZpY2UpJTBBJTBBaW1hZ2VzXzFfNSUyMCUzRCUyMHNkX3BpcGVsaW5lXzFfNShwcm9tcHRzJTJDJTIwbnVtX2ltYWdlc19wZXJfcHJvbXB0JTNEMSUyQyUyMGdlbmVyYXRvciUzRGdlbmVyYXRvciUyQyUyMG91dHB1dF90eXBlJTNEJTIybnVtcHklMjIpLmltYWdlcw==",highlighted:`model_ckpt_1_5 = <span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>
sd_pipeline_1_5 = StableDiffusionPipeline.from_pretrained(model_ckpt_1_5, torch_dtype=weight_dtype).to(device)

images_1_5 = sd_pipeline_1_5(prompts, num_images_per_prompt=<span class="hljs-number">1</span>, generator=generator, output_type=<span class="hljs-string">&quot;numpy&quot;</span>).images`}}),_t=new U({props:{code:"c2RfY2xpcF9zY29yZV8xXzQlMjAlM0QlMjBjYWxjdWxhdGVfY2xpcF9zY29yZShpbWFnZXMlMkMlMjBwcm9tcHRzKSUwQXByaW50KGYlMjJDTElQJTIwU2NvcmUlMjB3aXRoJTIwdi0xLTQlM0ElMjAlN0JzZF9jbGlwX3Njb3JlXzFfNCU3RCUyMiklMEElMjMlMjBDTElQJTIwU2NvcmUlMjB3aXRoJTIwdi0xLTQlM0ElMjAzNC45MTAyJTBBJTBBc2RfY2xpcF9zY29yZV8xXzUlMjAlM0QlMjBjYWxjdWxhdGVfY2xpcF9zY29yZShpbWFnZXNfMV81JTJDJTIwcHJvbXB0cyklMEFwcmludChmJTIyQ0xJUCUyMFNjb3JlJTIwd2l0aCUyMHYtMS01JTNBJTIwJTdCc2RfY2xpcF9zY29yZV8xXzUlN0QlMjIpJTBBJTIzJTIwQ0xJUCUyMFNjb3JlJTIwd2l0aCUyMHYtMS01JTNBJTIwMzYuMjEzNw==",highlighted:`sd_clip_score_1_4 = calculate_clip_score(images, prompts)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;CLIP Score with v-1-4: <span class="hljs-subst">{sd_clip_score_1_4}</span>&quot;</span>)
<span class="hljs-comment"># CLIP Score with v-1-4: 34.9102</span>

sd_clip_score_1_5 = calculate_clip_score(images_1_5, prompts)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;CLIP Score with v-1-5: <span class="hljs-subst">{sd_clip_score_1_5}</span>&quot;</span>)
<span class="hljs-comment"># CLIP Score with v-1-5: 36.2137</span>`}}),ve=new en({props:{warning:!0,$$slots:{default:[Nh]},$$scope:{ctx:A}}}),Bt=new nl({}),Gt=new U({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBZGF0YXNldCUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJzYXlha3BhdWwlMkZpbnN0cnVjdHBpeDJwaXgtZGVtbyUyMiUyQyUyMHNwbGl0JTNEJTIydHJhaW4lMjIpJTBBZGF0YXNldC5mZWF0dXJlcw==",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

dataset = load_dataset(<span class="hljs-string">&quot;sayakpaul/instructpix2pix-demo&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
dataset.features`}}),Ct=new U({props:{code:"JTdCJ2lucHV0JyUzQSUyMFZhbHVlKGR0eXBlJTNEJ3N0cmluZyclMkMlMjBpZCUzRE5vbmUpJTJDJTBBJTIwJ2VkaXQnJTNBJTIwVmFsdWUoZHR5cGUlM0Qnc3RyaW5nJyUyQyUyMGlkJTNETm9uZSklMkMlMEElMjAnb3V0cHV0JyUzQSUyMFZhbHVlKGR0eXBlJTNEJ3N0cmluZyclMkMlMjBpZCUzRE5vbmUpJTJDJTBBJTIwJ2ltYWdlJyUzQSUyMEltYWdlKGRlY29kZSUzRFRydWUlMkMlMjBpZCUzRE5vbmUpJTdE",highlighted:`{<span class="hljs-string">&#x27;input&#x27;</span>: Value(dtype=<span class="hljs-string">&#x27;string&#x27;</span>, <span class="hljs-built_in">id</span>=None),
 <span class="hljs-string">&#x27;edit&#x27;</span>: Value(dtype=<span class="hljs-string">&#x27;string&#x27;</span>, <span class="hljs-built_in">id</span>=None),
 <span class="hljs-string">&#x27;output&#x27;</span>: Value(dtype=<span class="hljs-string">&#x27;string&#x27;</span>, <span class="hljs-built_in">id</span>=None),
 <span class="hljs-string">&#x27;image&#x27;</span>: Image(decode=True, <span class="hljs-built_in">id</span>=None)}`}}),kt=new U({props:{code:"aWR4JTIwJTNEJTIwMCUwQXByaW50KGYlMjJPcmlnaW5hbCUyMGNhcHRpb24lM0ElMjAlN0JkYXRhc2V0JTVCaWR4JTVEJTVCJ2lucHV0JyU1RCU3RCUyMiklMEFwcmludChmJTIyRWRpdCUyMGluc3RydWN0aW9uJTNBJTIwJTdCZGF0YXNldCU1QmlkeCU1RCU1QidlZGl0JyU1RCU3RCUyMiklMEFwcmludChmJTIyTW9kaWZpZWQlMjBjYXB0aW9uJTNBJTIwJTdCZGF0YXNldCU1QmlkeCU1RCU1QidvdXRwdXQnJTVEJTdEJTIyKQ==",highlighted:`idx = <span class="hljs-number">0</span>
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Original caption: <span class="hljs-subst">{dataset[idx][<span class="hljs-string">&#x27;input&#x27;</span>]}</span>&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Edit instruction: <span class="hljs-subst">{dataset[idx][<span class="hljs-string">&#x27;edit&#x27;</span>]}</span>&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Modified caption: <span class="hljs-subst">{dataset[idx][<span class="hljs-string">&#x27;output&#x27;</span>]}</span>&quot;</span>)`}}),Xt=new U({props:{code:"T3JpZ2luYWwlMjBjYXB0aW9uJTNBJTIwMi4lMjBGQVJPRSUyMElTTEFORFMlM0ElMjBBbiUyMGFyY2hpcGVsYWdvJTIwb2YlMjAxOCUyMG1vdW50YWlub3VzJTIwaXNsZXMlMjBpbiUyMHRoZSUyME5vcnRoJTIwQXRsYW50aWMlMjBPY2VhbiUyMGJldHdlZW4lMjBOb3J3YXklMjBhbmQlMjBJY2VsYW5kJTJDJTIwdGhlJTIwRmFyb2UlMjBJc2xhbmRzJTIwaGFzJTIwJ2V2ZXJ5dGhpbmclMjB5b3UlMjBjb3VsZCUyMGhvcGUlMjBmb3InJTJDJTIwYWNjb3JkaW5nJTIwdG8lMjBCaWclMjA3JTIwVHJhdmVsLiUyMEl0JTIwYm9hc3RzJTIwJ2NyeXN0YWwlMjBjbGVhciUyMHdhdGVyZmFsbHMlMkMlMjByb2NreSUyMGNsaWZmcyUyMHRoYXQlMjBzZWVtJTIwdG8lMjBqdXQlMjBvdXQlMjBvZiUyMG5vd2hlcmUlMjBhbmQlMjB2ZWx2ZXR5JTIwZ3JlZW4lMjBoaWxscyclMEFFZGl0JTIwaW5zdHJ1Y3Rpb24lM0ElMjBtYWtlJTIwdGhlJTIwaXNsZXMlMjBhbGwlMjB3aGl0ZSUyMG1hcmJsZSUwQU1vZGlmaWVkJTIwY2FwdGlvbiUzQSUyMDIuJTIwV0hJVEUlMjBNQVJCTEUlMjBJU0xBTkRTJTNBJTIwQW4lMjBhcmNoaXBlbGFnbyUyMG9mJTIwMTglMjBtb3VudGFpbm91cyUyMHdoaXRlJTIwbWFyYmxlJTIwaXNsZXMlMjBpbiUyMHRoZSUyME5vcnRoJTIwQXRsYW50aWMlMjBPY2VhbiUyMGJldHdlZW4lMjBOb3J3YXklMjBhbmQlMjBJY2VsYW5kJTJDJTIwdGhlJTIwV2hpdGUlMjBNYXJibGUlMjBJc2xhbmRzJTIwaGFzJTIwJ2V2ZXJ5dGhpbmclMjB5b3UlMjBjb3VsZCUyMGhvcGUlMjBmb3InJTJDJTIwYWNjb3JkaW5nJTIwdG8lMjBCaWclMjA3JTIwVHJhdmVsLiUyMEl0JTIwYm9hc3RzJTIwJ2NyeXN0YWwlMjBjbGVhciUyMHdhdGVyZmFsbHMlMkMlMjByb2NreSUyMGNsaWZmcyUyMHRoYXQlMjBzZWVtJTIwdG8lMjBqdXQlMjBvdXQlMjBvZiUyMG5vd2hlcmUlMjBhbmQlMjB2ZWx2ZXR5JTIwZ3JlZW4lMjBoaWxscyc=",highlighted:`Original caption: 2. FAROE ISLANDS: An archipelago of 18 mountainous isles <span class="hljs-keyword">in</span> the North Atlantic Ocean between Norway and Iceland, the Faroe Islands has <span class="hljs-string">&#x27;everything you could hope for&#x27;</span>, according to Big 7 Travel. It boasts <span class="hljs-string">&#x27;crystal clear waterfalls, rocky cliffs that seem to jut out of nowhere and velvety green hills&#x27;</span>
Edit instruction: make the isles all white marble
Modified caption: 2. WHITE MARBLE ISLANDS: An archipelago of 18 mountainous white marble isles <span class="hljs-keyword">in</span> the North Atlantic Ocean between Norway and Iceland, the White Marble Islands has <span class="hljs-string">&#x27;everything you could hope for&#x27;</span>, according to Big 7 Travel. It boasts <span class="hljs-string">&#x27;crystal clear waterfalls, rocky cliffs that seem to jut out of nowhere and velvety green hills&#x27;</span>`}}),Vt=new U({props:{code:"ZGF0YXNldCU1QmlkeCU1RCU1QiUyMmltYWdlJTIyJTVE",highlighted:'dataset[idx][<span class="hljs-string">&quot;image&quot;</span>]'}}),Nt=new U({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMFN0YWJsZURpZmZ1c2lvbkluc3RydWN0UGl4MlBpeFBpcGVsaW5lJTBBJTBBaW5zdHJ1Y3RfcGl4MnBpeF9waXBlbGluZSUyMCUzRCUyMFN0YWJsZURpZmZ1c2lvbkluc3RydWN0UGl4MlBpeFBpcGVsaW5lLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJ0aW1icm9va3MlMkZpbnN0cnVjdC1waXgycGl4JTIyJTJDJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTBBKS50byhkZXZpY2Up",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionInstructPix2PixPipeline

instruct_pix2pix_pipeline = StableDiffusionInstructPix2PixPipeline.from_pretrained(
    <span class="hljs-string">&quot;timbrooks/instruct-pix2pix&quot;</span>, torch_dtype=torch.float16
).to(device)`}}),Rt=new U({props:{code:"aW1wb3J0JTIwbnVtcHklMjBhcyUyMG5wJTBBJTBBJTBBZGVmJTIwZWRpdF9pbWFnZShpbnB1dF9pbWFnZSUyQyUyMGluc3RydWN0aW9uKSUzQSUwQSUyMCUyMCUyMCUyMGltYWdlJTIwJTNEJTIwaW5zdHJ1Y3RfcGl4MnBpeF9waXBlbGluZSglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpbnN0cnVjdGlvbiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGltYWdlJTNEaW5wdXRfaW1hZ2UlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvdXRwdXRfdHlwZSUzRCUyMm51bXB5JTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZ2VuZXJhdG9yJTNEZ2VuZXJhdG9yJTJDJTBBJTIwJTIwJTIwJTIwKS5pbWFnZXMlNUIwJTVEJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwaW1hZ2UlMEElMEElMEFpbnB1dF9pbWFnZXMlMjAlM0QlMjAlNUIlNUQlMEFvcmlnaW5hbF9jYXB0aW9ucyUyMCUzRCUyMCU1QiU1RCUwQW1vZGlmaWVkX2NhcHRpb25zJTIwJTNEJTIwJTVCJTVEJTBBZWRpdGVkX2ltYWdlcyUyMCUzRCUyMCU1QiU1RCUwQSUwQWZvciUyMGlkeCUyMGluJTIwcmFuZ2UobGVuKGRhdGFzZXQpKSUzQSUwQSUyMCUyMCUyMCUyMGlucHV0X2ltYWdlJTIwJTNEJTIwZGF0YXNldCU1QmlkeCU1RCU1QiUyMmltYWdlJTIyJTVEJTBBJTIwJTIwJTIwJTIwZWRpdF9pbnN0cnVjdGlvbiUyMCUzRCUyMGRhdGFzZXQlNUJpZHglNUQlNUIlMjJlZGl0JTIyJTVEJTBBJTIwJTIwJTIwJTIwZWRpdGVkX2ltYWdlJTIwJTNEJTIwZWRpdF9pbWFnZShpbnB1dF9pbWFnZSUyQyUyMGVkaXRfaW5zdHJ1Y3Rpb24pJTBBJTBBJTIwJTIwJTIwJTIwaW5wdXRfaW1hZ2VzLmFwcGVuZChucC5hcnJheShpbnB1dF9pbWFnZSkpJTBBJTIwJTIwJTIwJTIwb3JpZ2luYWxfY2FwdGlvbnMuYXBwZW5kKGRhdGFzZXQlNUJpZHglNUQlNUIlMjJpbnB1dCUyMiU1RCklMEElMjAlMjAlMjAlMjBtb2RpZmllZF9jYXB0aW9ucy5hcHBlbmQoZGF0YXNldCU1QmlkeCU1RCU1QiUyMm91dHB1dCUyMiU1RCklMEElMjAlMjAlMjAlMjBlZGl0ZWRfaW1hZ2VzLmFwcGVuZChlZGl0ZWRfaW1hZ2Up",highlighted:`<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np


<span class="hljs-keyword">def</span> <span class="hljs-title function_">edit_image</span>(<span class="hljs-params">input_image, instruction</span>):
    image = instruct_pix2pix_pipeline(
        instruction,
        image=input_image,
        output_type=<span class="hljs-string">&quot;numpy&quot;</span>,
        generator=generator,
    ).images[<span class="hljs-number">0</span>]
    <span class="hljs-keyword">return</span> image


input_images = []
original_captions = []
modified_captions = []
edited_images = []

<span class="hljs-keyword">for</span> idx <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(dataset)):
    input_image = dataset[idx][<span class="hljs-string">&quot;image&quot;</span>]
    edit_instruction = dataset[idx][<span class="hljs-string">&quot;edit&quot;</span>]
    edited_image = edit_image(input_image, edit_instruction)

    input_images.append(np.array(input_image))
    original_captions.append(dataset[idx][<span class="hljs-string">&quot;input&quot;</span>])
    modified_captions.append(dataset[idx][<span class="hljs-string">&quot;output&quot;</span>])
    edited_images.append(edited_image)`}}),Ft=new U({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMCglMEElMjAlMjAlMjAlMjBDTElQVG9rZW5pemVyJTJDJTBBJTIwJTIwJTIwJTIwQ0xJUFRleHRNb2RlbFdpdGhQcm9qZWN0aW9uJTJDJTBBJTIwJTIwJTIwJTIwQ0xJUFZpc2lvbk1vZGVsV2l0aFByb2plY3Rpb24lMkMlMEElMjAlMjAlMjAlMjBDTElQSW1hZ2VQcm9jZXNzb3IlMkMlMEEpJTBBJTBBY2xpcF9pZCUyMCUzRCUyMCUyMm9wZW5haSUyRmNsaXAtdml0LWxhcmdlLXBhdGNoMTQlMjIlMEF0b2tlbml6ZXIlMjAlM0QlMjBDTElQVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChjbGlwX2lkKSUwQXRleHRfZW5jb2RlciUyMCUzRCUyMENMSVBUZXh0TW9kZWxXaXRoUHJvamVjdGlvbi5mcm9tX3ByZXRyYWluZWQoY2xpcF9pZCkudG8oZGV2aWNlKSUwQWltYWdlX3Byb2Nlc3NvciUyMCUzRCUyMENMSVBJbWFnZVByb2Nlc3Nvci5mcm9tX3ByZXRyYWluZWQoY2xpcF9pZCklMEFpbWFnZV9lbmNvZGVyJTIwJTNEJTIwQ0xJUFZpc2lvbk1vZGVsV2l0aFByb2plY3Rpb24uZnJvbV9wcmV0cmFpbmVkKGNsaXBfaWQpLnRvKGRldmljZSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> (
    CLIPTokenizer,
    CLIPTextModelWithProjection,
    CLIPVisionModelWithProjection,
    CLIPImageProcessor,
)

clip_id = <span class="hljs-string">&quot;openai/clip-vit-large-patch14&quot;</span>
tokenizer = CLIPTokenizer.from_pretrained(clip_id)
text_encoder = CLIPTextModelWithProjection.from_pretrained(clip_id).to(device)
image_processor = CLIPImageProcessor.from_pretrained(clip_id)
image_encoder = CLIPVisionModelWithProjection.from_pretrained(clip_id).to(device)`}}),St=new U({props:{code:" ",highlighted:`<span class="hljs-keyword">import</span> torch.nn <span class="hljs-keyword">as</span> nn
<span class="hljs-keyword">import</span> torch.nn.functional <span class="hljs-keyword">as</span> F


<span class="hljs-keyword">class</span> <span class="hljs-title class_">DirectionalSimilarity</span>(nn.Module):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, tokenizer, text_encoder, image_processor, image_encoder</span>):
        <span class="hljs-built_in">super</span>().__init__()
        self.tokenizer = tokenizer
        self.text_encoder = text_encoder
        self.image_processor = image_processor
        self.image_encoder = image_encoder

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">preprocess_image</span>(<span class="hljs-params">self, image</span>):
        image = self.image_processor(image, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)[<span class="hljs-string">&quot;pixel_values&quot;</span>]
        <span class="hljs-keyword">return</span> {<span class="hljs-string">&quot;pixel_values&quot;</span>: image.to(device)}

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize_text</span>(<span class="hljs-params">self, text</span>):
        inputs = self.tokenizer(
            text,
            max_length=self.tokenizer.model_max_length,
            padding=<span class="hljs-string">&quot;max_length&quot;</span>,
            truncation=<span class="hljs-literal">True</span>,
            return_tensors=<span class="hljs-string">&quot;pt&quot;</span>,
        )
        <span class="hljs-keyword">return</span> {<span class="hljs-string">&quot;input_ids&quot;</span>: inputs.input_ids.to(device)}

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">encode_image</span>(<span class="hljs-params">self, image</span>):
        preprocessed_image = self.preprocess_image(image)
        image_features = self.image_encoder(**preprocessed_image).image_embeds
        image_features = image_features / image_features.norm(dim=<span class="hljs-number">1</span>, keepdim=<span class="hljs-literal">True</span>)
        <span class="hljs-keyword">return</span> image_features

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">encode_text</span>(<span class="hljs-params">self, text</span>):
        tokenized_text = self.tokenize_text(text)
        text_features = self.text_encoder(**tokenized_text).text_embeds
        text_features = text_features / text_features.norm(dim=<span class="hljs-number">1</span>, keepdim=<span class="hljs-literal">True</span>)
        <span class="hljs-keyword">return</span> text_features

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_directional_similarity</span>(<span class="hljs-params">self, img_feat_one, img_feat_two, text_feat_one, text_feat_two</span>):
        sim_direction = F.cosine_similarity(img_feat_two - img_feat_one, text_feat_two - text_feat_one)
        <span class="hljs-keyword">return</span> sim_direction

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, image_one, image_two, caption_one, caption_two</span>):
        img_feat_one = self.encode_image(image_one)
        img_feat_two = self.encode_image(image_two)
        text_feat_one = self.encode_text(caption_one)
        text_feat_two = self.encode_text(caption_two)
        directional_similarity = self.compute_directional_similarity(
            img_feat_one, img_feat_two, text_feat_one, text_feat_two
        )
        <span class="hljs-keyword">return</span> directional_similarity`}}),xt=new U({props:{code:"ZGlyX3NpbWlsYXJpdHklMjAlM0QlMjBEaXJlY3Rpb25hbFNpbWlsYXJpdHkodG9rZW5pemVyJTJDJTIwdGV4dF9lbmNvZGVyJTJDJTIwaW1hZ2VfcHJvY2Vzc29yJTJDJTIwaW1hZ2VfZW5jb2RlciklMEFzY29yZXMlMjAlM0QlMjAlNUIlNUQlMEElMEFmb3IlMjBpJTIwaW4lMjByYW5nZShsZW4oaW5wdXRfaW1hZ2VzKSklM0ElMEElMjAlMjAlMjAlMjBvcmlnaW5hbF9pbWFnZSUyMCUzRCUyMGlucHV0X2ltYWdlcyU1QmklNUQlMEElMjAlMjAlMjAlMjBvcmlnaW5hbF9jYXB0aW9uJTIwJTNEJTIwb3JpZ2luYWxfY2FwdGlvbnMlNUJpJTVEJTBBJTIwJTIwJTIwJTIwZWRpdGVkX2ltYWdlJTIwJTNEJTIwZWRpdGVkX2ltYWdlcyU1QmklNUQlMEElMjAlMjAlMjAlMjBtb2RpZmllZF9jYXB0aW9uJTIwJTNEJTIwbW9kaWZpZWRfY2FwdGlvbnMlNUJpJTVEJTBBJTBBJTIwJTIwJTIwJTIwc2ltaWxhcml0eV9zY29yZSUyMCUzRCUyMGRpcl9zaW1pbGFyaXR5KG9yaWdpbmFsX2ltYWdlJTJDJTIwZWRpdGVkX2ltYWdlJTJDJTIwb3JpZ2luYWxfY2FwdGlvbiUyQyUyMG1vZGlmaWVkX2NhcHRpb24pJTBBJTIwJTIwJTIwJTIwc2NvcmVzLmFwcGVuZChmbG9hdChzaW1pbGFyaXR5X3Njb3JlLmRldGFjaCgpLmNwdSgpKSklMEElMEFwcmludChmJTIyQ0xJUCUyMGRpcmVjdGlvbmFsJTIwc2ltaWxhcml0eSUzQSUyMCU3Qm5wLm1lYW4oc2NvcmVzKSU3RCUyMiklMEElMjMlMjBDTElQJTIwZGlyZWN0aW9uYWwlMjBzaW1pbGFyaXR5JTNBJTIwMC4wNzk3OTc2NTUzNDQwMDk0",highlighted:`dir_similarity = DirectionalSimilarity(tokenizer, text_encoder, image_processor, image_encoder)
scores = []

<span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(input_images)):
    original_image = input_images[i]
    original_caption = original_captions[i]
    edited_image = edited_images[i]
    modified_caption = modified_captions[i]

    similarity_score = dir_similarity(original_image, edited_image, original_caption, modified_caption)
    scores.append(<span class="hljs-built_in">float</span>(similarity_score.detach().cpu()))

<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;CLIP directional similarity: <span class="hljs-subst">{np.mean(scores)}</span>&quot;</span>)
<span class="hljs-comment"># CLIP directional similarity: 0.0797976553440094</span>`}}),Ne=new en({props:{$$slots:{default:[Rh]},$$scope:{ctx:A}}}),zt=new nl({}),Ot=new U({props:{code:"ZnJvbSUyMHppcGZpbGUlMjBpbXBvcnQlMjBaaXBGaWxlJTBBaW1wb3J0JTIwcmVxdWVzdHMlMEElMEElMEFkZWYlMjBkb3dubG9hZCh1cmwlMkMlMjBsb2NhbF9maWxlcGF0aCklM0ElMEElMjAlMjAlMjAlMjByJTIwJTNEJTIwcmVxdWVzdHMuZ2V0KHVybCklMEElMjAlMjAlMjAlMjB3aXRoJTIwb3Blbihsb2NhbF9maWxlcGF0aCUyQyUyMCUyMndiJTIyKSUyMGFzJTIwZiUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGYud3JpdGUoci5jb250ZW50KSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMGxvY2FsX2ZpbGVwYXRoJTBBJTBBJTBBZHVtbXlfZGF0YXNldF91cmwlMjAlM0QlMjAlMjJodHRwcyUzQSUyRiUyRmhmLmNvJTJGZGF0YXNldHMlMkZzYXlha3BhdWwlMkZzYW1wbGUtZGF0YXNldHMlMkZyZXNvbHZlJTJGbWFpbiUyRnNhbXBsZS1pbWFnZW5ldC1pbWFnZXMuemlwJTIyJTBBbG9jYWxfZmlsZXBhdGglMjAlM0QlMjBkb3dubG9hZChkdW1teV9kYXRhc2V0X3VybCUyQyUyMGR1bW15X2RhdGFzZXRfdXJsLnNwbGl0KCUyMiUyRiUyMiklNUItMSU1RCklMEElMEF3aXRoJTIwWmlwRmlsZShsb2NhbF9maWxlcGF0aCUyQyUyMCUyMnIlMjIpJTIwYXMlMjB6aXBwZXIlM0ElMEElMjAlMjAlMjAlMjB6aXBwZXIuZXh0cmFjdGFsbCglMjIuJTIyKQ==",highlighted:`<span class="hljs-keyword">from</span> zipfile <span class="hljs-keyword">import</span> ZipFile
<span class="hljs-keyword">import</span> requests


<span class="hljs-keyword">def</span> <span class="hljs-title function_">download</span>(<span class="hljs-params">url, local_filepath</span>):
    r = requests.get(url)
    <span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(local_filepath, <span class="hljs-string">&quot;wb&quot;</span>) <span class="hljs-keyword">as</span> f:
        f.write(r.content)
    <span class="hljs-keyword">return</span> local_filepath


dummy_dataset_url = <span class="hljs-string">&quot;https://hf.co/datasets/sayakpaul/sample-datasets/resolve/main/sample-imagenet-images.zip&quot;</span>
local_filepath = download(dummy_dataset_url, dummy_dataset_url.split(<span class="hljs-string">&quot;/&quot;</span>)[-<span class="hljs-number">1</span>])

<span class="hljs-keyword">with</span> ZipFile(local_filepath, <span class="hljs-string">&quot;r&quot;</span>) <span class="hljs-keyword">as</span> zipper:
    zipper.extractall(<span class="hljs-string">&quot;.&quot;</span>)`}}),el=new U({props:{code:"ZnJvbSUyMFBJTCUyMGltcG9ydCUyMEltYWdlJTBBaW1wb3J0JTIwb3MlMEElMEFkYXRhc2V0X3BhdGglMjAlM0QlMjAlMjJzYW1wbGUtaW1hZ2VuZXQtaW1hZ2VzJTIyJTBBaW1hZ2VfcGF0aHMlMjAlM0QlMjBzb3J0ZWQoJTVCb3MucGF0aC5qb2luKGRhdGFzZXRfcGF0aCUyQyUyMHgpJTIwZm9yJTIweCUyMGluJTIwb3MubGlzdGRpcihkYXRhc2V0X3BhdGgpJTVEKSUwQSUwQXJlYWxfaW1hZ2VzJTIwJTNEJTIwJTVCbnAuYXJyYXkoSW1hZ2Uub3BlbihwYXRoKS5jb252ZXJ0KCUyMlJHQiUyMikpJTIwZm9yJTIwcGF0aCUyMGluJTIwaW1hZ2VfcGF0aHMlNUQ=",highlighted:`<span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image
<span class="hljs-keyword">import</span> os

dataset_path = <span class="hljs-string">&quot;sample-imagenet-images&quot;</span>
image_paths = <span class="hljs-built_in">sorted</span>([os.path.join(dataset_path, x) <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> os.listdir(dataset_path)])

real_images = [np.array(Image.<span class="hljs-built_in">open</span>(path).convert(<span class="hljs-string">&quot;RGB&quot;</span>)) <span class="hljs-keyword">for</span> path <span class="hljs-keyword">in</span> image_paths]`}}),tl=new U({props:{code:"ZnJvbSUyMHRvcmNodmlzaW9uLnRyYW5zZm9ybXMlMjBpbXBvcnQlMjBmdW5jdGlvbmFsJTIwYXMlMjBGJTBBJTBBJTBBZGVmJTIwcHJlcHJvY2Vzc19pbWFnZShpbWFnZSklM0ElMEElMjAlMjAlMjAlMjBpbWFnZSUyMCUzRCUyMHRvcmNoLnRlbnNvcihpbWFnZSkudW5zcXVlZXplKDApJTBBJTIwJTIwJTIwJTIwaW1hZ2UlMjAlM0QlMjBpbWFnZS5wZXJtdXRlKDAlMkMlMjAzJTJDJTIwMSUyQyUyMDIpJTIwJTJGJTIwMjU1LjAlMEElMjAlMjAlMjAlMjByZXR1cm4lMjBGLmNlbnRlcl9jcm9wKGltYWdlJTJDJTIwKDI1NiUyQyUyMDI1NikpJTBBJTBBJTBBcmVhbF9pbWFnZXMlMjAlM0QlMjB0b3JjaC5jYXQoJTVCcHJlcHJvY2Vzc19pbWFnZShpbWFnZSklMjBmb3IlMjBpbWFnZSUyMGluJTIwcmVhbF9pbWFnZXMlNUQpJTBBcHJpbnQocmVhbF9pbWFnZXMuc2hhcGUpJTBBJTIzJTIwdG9yY2guU2l6ZSglNUIxMCUyQyUyMDMlMkMlMjAyNTYlMkMlMjAyNTYlNUQp",highlighted:`<span class="hljs-keyword">from</span> torchvision.transforms <span class="hljs-keyword">import</span> functional <span class="hljs-keyword">as</span> F


<span class="hljs-keyword">def</span> <span class="hljs-title function_">preprocess_image</span>(<span class="hljs-params">image</span>):
    image = torch.tensor(image).unsqueeze(<span class="hljs-number">0</span>)
    image = image.permute(<span class="hljs-number">0</span>, <span class="hljs-number">3</span>, <span class="hljs-number">1</span>, <span class="hljs-number">2</span>) / <span class="hljs-number">255.0</span>
    <span class="hljs-keyword">return</span> F.center_crop(image, (<span class="hljs-number">256</span>, <span class="hljs-number">256</span>))


real_images = torch.cat([preprocess_image(image) <span class="hljs-keyword">for</span> image <span class="hljs-keyword">in</span> real_images])
<span class="hljs-built_in">print</span>(real_images.shape)
<span class="hljs-comment"># torch.Size([10, 3, 256, 256])</span>`}}),sl=new U({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMERpVFBpcGVsaW5lJTJDJTIwRFBNU29sdmVyTXVsdGlzdGVwU2NoZWR1bGVyJTBBJTBBZGl0X3BpcGVsaW5lJTIwJTNEJTIwRGlUUGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKCUyMmZhY2Vib29rJTJGRGlULVhMLTItMjU2JTIyJTJDJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2KSUwQWRpdF9waXBlbGluZS5zY2hlZHVsZXIlMjAlM0QlMjBEUE1Tb2x2ZXJNdWx0aXN0ZXBTY2hlZHVsZXIuZnJvbV9jb25maWcoZGl0X3BpcGVsaW5lLnNjaGVkdWxlci5jb25maWcpJTBBZGl0X3BpcGVsaW5lJTIwJTNEJTIwZGl0X3BpcGVsaW5lLnRvKCUyMmN1ZGElMjIpJTBBJTBBd29yZHMlMjAlM0QlMjAlNUIlMEElMjAlMjAlMjAlMjAlMjJjYXNzZXR0ZSUyMHBsYXllciUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMmNoYWluc2F3JTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIyY2hhaW5zYXclMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJjaHVyY2glMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJnYXMlMjBwdW1wJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIyZ2FzJTIwcHVtcCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMmdhcyUyMHB1bXAlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJwYXJhY2h1dGUlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJwYXJhY2h1dGUlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJ0ZW5jaCUyMiUyQyUwQSU1RCUwQSUwQWNsYXNzX2lkcyUyMCUzRCUyMGRpdF9waXBlbGluZS5nZXRfbGFiZWxfaWRzKHdvcmRzKSUwQW91dHB1dCUyMCUzRCUyMGRpdF9waXBlbGluZShjbGFzc19sYWJlbHMlM0RjbGFzc19pZHMlMkMlMjBnZW5lcmF0b3IlM0RnZW5lcmF0b3IlMkMlMjBvdXRwdXRfdHlwZSUzRCUyMm51bXB5JTIyKSUwQSUwQWZha2VfaW1hZ2VzJTIwJTNEJTIwb3V0cHV0LmltYWdlcyUwQWZha2VfaW1hZ2VzJTIwJTNEJTIwdG9yY2gudGVuc29yKGZha2VfaW1hZ2VzKSUwQWZha2VfaW1hZ2VzJTIwJTNEJTIwZmFrZV9pbWFnZXMucGVybXV0ZSgwJTJDJTIwMyUyQyUyMDElMkMlMjAyKSUwQXByaW50KGZha2VfaW1hZ2VzLnNoYXBlKSUwQSUyMyUyMHRvcmNoLlNpemUoJTVCMTAlMkMlMjAzJTJDJTIwMjU2JTJDJTIwMjU2JTVEKQ==",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> DiTPipeline, DPMSolverMultistepScheduler

dit_pipeline = DiTPipeline.from_pretrained(<span class="hljs-string">&quot;facebook/DiT-XL-2-256&quot;</span>, torch_dtype=torch.float16)
dit_pipeline.scheduler = DPMSolverMultistepScheduler.from_config(dit_pipeline.scheduler.config)
dit_pipeline = dit_pipeline.to(<span class="hljs-string">&quot;cuda&quot;</span>)

words = [
    <span class="hljs-string">&quot;cassette player&quot;</span>,
    <span class="hljs-string">&quot;chainsaw&quot;</span>,
    <span class="hljs-string">&quot;chainsaw&quot;</span>,
    <span class="hljs-string">&quot;church&quot;</span>,
    <span class="hljs-string">&quot;gas pump&quot;</span>,
    <span class="hljs-string">&quot;gas pump&quot;</span>,
    <span class="hljs-string">&quot;gas pump&quot;</span>,
    <span class="hljs-string">&quot;parachute&quot;</span>,
    <span class="hljs-string">&quot;parachute&quot;</span>,
    <span class="hljs-string">&quot;tench&quot;</span>,
]

class_ids = dit_pipeline.get_label_ids(words)
output = dit_pipeline(class_labels=class_ids, generator=generator, output_type=<span class="hljs-string">&quot;numpy&quot;</span>)

fake_images = output.images
fake_images = torch.tensor(fake_images)
fake_images = fake_images.permute(<span class="hljs-number">0</span>, <span class="hljs-number">3</span>, <span class="hljs-number">1</span>, <span class="hljs-number">2</span>)
<span class="hljs-built_in">print</span>(fake_images.shape)
<span class="hljs-comment"># torch.Size([10, 3, 256, 256])</span>`}}),il=new U({props:{code:"ZnJvbSUyMHRvcmNobWV0cmljcy5pbWFnZS5maWQlMjBpbXBvcnQlMjBGcmVjaGV0SW5jZXB0aW9uRGlzdGFuY2UlMEElMEFmaWQlMjAlM0QlMjBGcmVjaGV0SW5jZXB0aW9uRGlzdGFuY2Uobm9ybWFsaXplJTNEVHJ1ZSklMEFmaWQudXBkYXRlKHJlYWxfaW1hZ2VzJTJDJTIwcmVhbCUzRFRydWUpJTBBZmlkLnVwZGF0ZShmYWtlX2ltYWdlcyUyQyUyMHJlYWwlM0RGYWxzZSklMEElMEFwcmludChmJTIyRklEJTNBJTIwJTdCZmxvYXQoZmlkLmNvbXB1dGUoKSklN0QlMjIpJTBBJTIzJTIwRklEJTNBJTIwMTc3LjcxNDcyMTY3OTY4NzU=",highlighted:`<span class="hljs-keyword">from</span> torchmetrics.image.fid <span class="hljs-keyword">import</span> FrechetInceptionDistance

fid = FrechetInceptionDistance(normalize=<span class="hljs-literal">True</span>)
fid.update(real_images, real=<span class="hljs-literal">True</span>)
fid.update(fake_images, real=<span class="hljs-literal">False</span>)

<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;FID: <span class="hljs-subst">{<span class="hljs-built_in">float</span>(fid.compute())}</span>&quot;</span>)
<span class="hljs-comment"># FID: 177.7147216796875</span>`}}),xe=new en({props:{warning:!0,$$slots:{default:[Fh]},$$scope:{ctx:A}}}),{c(){h=a("meta"),v=c(),u=a("h1"),j=a("a"),_=a("span"),d(I.$$.fragment),g=c(),Z=a("span"),ze=n("Evaluating Diffusion Models"),re=c(),E=a("a"),S=a("img"),R=c(),k=a("p"),pe=n("Evaluation of generative models like "),W=a("a"),De=n("Stable Diffusion"),b=n(" is subjective in nature. But as practitioners and researchers, we often have to make careful choices amongst many different possibilities. So, when working with different generative models (like GANs, Diffusion, etc.), how do we choose one over the other?"),B=c(),C=a("p"),rl=n(`Qualitative evaluation of such models can be error-prone and might incorrectly influence a decision.
However, quantitative metrics don\u2019t necessarily correspond to image quality. So, usually, a combination
of both qualitative and quantitative evaluations provides a stronger signal when choosing one model
over the other.`),qe=c(),F=a("p"),pl=n("In this document, we provide a non-exhaustive overview of qualitative and quantitative methods to evaluate Diffusion models. For quantitative methods, we specifically focus on how to implement them alongside "),ce=a("code"),tn=n("diffusers"),ln=n("."),Ua=c(),me=a("p"),sn=n("The methods shown in this document can also be used to evaluate different "),Le=a("a"),an=n("noise schedulers"),on=n(" keeping the underlying generation model fixed."),ga=c(),O=a("h2"),fe=a("a"),us=a("span"),d(Ke.$$.fragment),nn=c(),ds=a("span"),rn=n("Scenarios"),va=c(),cl=a("p"),pn=n("We cover Diffusion models with the following pipelines:"),_a=c(),x=a("ul"),Oe=a("li"),cn=n("Text-guided image generation (such as the "),et=a("a"),Ms=a("code"),mn=n("StableDiffusionPipeline"),fn=n(")."),hn=c(),ee=a("li"),un=n("Text-guided image generation, additionally conditioned on an input image (such as the "),tt=a("a"),ws=a("code"),dn=n("StableDiffusionImg2ImgPipeline"),Mn=n(", and "),lt=a("a"),ys=a("code"),wn=n("StableDiffusionInstructPix2PixPipeline"),yn=n(")."),Jn=c(),st=a("li"),Tn=n("Class-conditioned image generation models (such as the "),at=a("a"),Js=a("code"),jn=n("DiTPipeline"),bn=n(")."),Za=c(),te=a("h2"),he=a("a"),Ts=a("span"),d(it.$$.fragment),In=c(),js=a("span"),Un=n("Qualitative Evaluation"),Ba=c(),$=a("p"),gn=n(`Qualitative evaluation typically involves human assessment of generated images. Quality is measured across aspects such as compositionality, image-text alignment, and spatial relations. Common prompts provide a degree of uniformity for subjective metrics.
DrawBench and PartiPrompts are prompt datasets used for qualitative benchmarking. DrawBench and PartiPrompts were introduced by `),ot=a("a"),vn=n("Imagen"),_n=n(" and "),nt=a("a"),Zn=n("Parti"),Bn=n(" respectively."),Ea=c(),ue=a("p"),En=n("From the "),rt=a("a"),Wn=n("official Parti website"),Gn=n(":"),Wa=c(),ml=a("blockquote"),bs=a("p"),Cn=n("PartiPrompts (P2) is a rich set of over 1600 prompts in English that we release as part of this work. P2 can be used to measure model capabilities across various categories and challenge aspects."),Ga=c(),fl=a("p"),hl=a("img"),Ca=c(),ul=a("p"),kn=n("PartiPrompts has the following columns:"),ka=c(),P=a("ul"),Is=a("li"),Xn=n("Prompt"),Vn=c(),Us=a("li"),Nn=n("Category of the prompt (such as \u201CAbstract\u201D, \u201CWorld Knowledge\u201D, etc.)"),Rn=c(),gs=a("li"),Fn=n("Challenge reflecting the difficulty (such as \u201CBasic\u201D, \u201CComplex\u201D, \u201CWriting & Symbols\u201D, etc.)"),Xa=c(),dl=a("p"),An=n("These benchmarks allow for side-by-side human evaluation of different image generation models."),Va=c(),de=a("p"),Sn=n("For this, the \u{1F9E8} Diffusers team has built "),vs=a("strong"),xn=n("Open Parti Prompts"),$n=n(", which is a community-driven qualitative benchmark based on Parti Prompts to compare state-of-the-art open-source diffusion models:"),Na=c(),Me=a("ul"),Ml=a("li"),pt=a("a"),Pn=n("Open Parti Prompts Game"),Yn=n(": For 10 parti prompts, 4 generated images are shown and the user selects the image that suits the prompt best."),Qn=c(),wl=a("li"),ct=a("a"),zn=n("Open Parti Prompts Leaderboard"),Hn=n(": The leaderboard comparing the currently best open-sourced diffusion models to each other."),Ra=c(),we=a("p"),Dn=n("To manually compare images, let\u2019s see how we can use "),_s=a("code"),qn=n("diffusers"),Ln=n(" on a couple of PartiPrompts."),Fa=c(),ye=a("p"),Kn=n("Below we show some prompts sampled across different challenges: Basic, Complex, Linguistic Structures, Imagination, and Writing & Symbols. Here we are using PartiPrompts as a "),mt=a("a"),On=n("dataset"),er=n("."),Aa=c(),d(ft.$$.fragment),Sa=c(),Je=a("p"),tr=n("Now we can use these prompts to generate some images using Stable Diffusion ("),ht=a("a"),lr=n("v1-4 checkpoint"),sr=n("):"),xa=c(),d(ut.$$.fragment),$a=c(),yl=a("p"),Jl=a("img"),Pa=c(),Y=a("p"),ar=n("We can also set "),Zs=a("code"),ir=n("num_images_per_prompt"),or=n(" accordingly to compare different images for the same prompt. Running the same pipeline but with a different checkpoint ("),dt=a("a"),nr=n("v1-5"),rr=n("), yields:"),Ya=c(),Tl=a("p"),jl=a("img"),Qa=c(),bl=a("p"),pr=n(`Once several images are generated from all the prompts using multiple models (under evaluation), these results are presented to human evaluators for scoring. For
more details on the DrawBench and PartiPrompts benchmarks, refer to their respective papers.`),za=c(),d(Te.$$.fragment),Ha=c(),le=a("h2"),je=a("a"),Bs=a("span"),d(Mt.$$.fragment),cr=c(),Es=a("span"),mr=n("Quantitative Evaluation"),Da=c(),Il=a("p"),fr=n("In this section, we will walk you through how to evaluate three different diffusion pipelines using:"),qa=c(),Q=a("ul"),Ws=a("li"),hr=n("CLIP score"),ur=c(),Gs=a("li"),dr=n("CLIP directional similarity"),Mr=c(),Cs=a("li"),wr=n("FID"),La=c(),se=a("h3"),be=a("a"),ks=a("span"),d(wt.$$.fragment),yr=c(),Xs=a("span"),Jr=n("Text-guided image generation"),Ka=c(),yt=a("p"),Jt=a("a"),Tr=n("CLIP score"),jr=n(" measures the compatibility of image-caption pairs. Higher CLIP scores imply higher compatibility \u{1F53C}. The CLIP score is a quantitative measurement of the qualitative concept \u201Ccompatibility\u201D. Image-caption pair compatibility can also be thought of as the semantic similarity between the image and the caption. CLIP score was found to have high correlation with human judgement."),Oa=c(),Ie=a("p"),br=n("Let\u2019s first load a "),Ul=a("a"),Ir=n("StableDiffusionPipeline"),Ur=n(":"),ei=c(),d(Tt.$$.fragment),ti=c(),gl=a("p"),gr=n("Generate some images with multiple prompts:"),li=c(),d(jt.$$.fragment),si=c(),vl=a("p"),vr=n("And then, we calculate the CLIP score."),ai=c(),d(bt.$$.fragment),ii=c(),_l=a("p"),_r=n("In the above example, we generated one image per prompt. If we generated multiple images per prompt, we would have to take the average score from the generated images per prompt."),oi=c(),z=a("p"),Zr=n("Now, if we wanted to compare two checkpoints compatible with the "),Zl=a("a"),Br=n("StableDiffusionPipeline"),Er=n(` we should pass a generator while calling the pipeline. First, we generate images with a
fixed seed with the `),It=a("a"),Wr=n("v1-4 Stable Diffusion checkpoint"),Gr=n(":"),ni=c(),d(Ut.$$.fragment),ri=c(),Ue=a("p"),Cr=n("Then we load the "),gt=a("a"),kr=n("v1-5 checkpoint"),Xr=n(" to generate images:"),pi=c(),d(vt.$$.fragment),ci=c(),Bl=a("p"),Vr=n("And finally, we compare their CLIP scores:"),mi=c(),d(_t.$$.fragment),fi=c(),ge=a("p"),Nr=n("It seems like the "),Zt=a("a"),Rr=n("v1-5"),Fr=n(" checkpoint performs better than its predecessor. Note, however, that the number of prompts we used to compute the CLIP scores is quite low. For a more practical evaluation, this number should be way higher, and the prompts should be diverse."),hi=c(),d(ve.$$.fragment),ui=c(),ae=a("h3"),_e=a("a"),Vs=a("span"),d(Bt.$$.fragment),Ar=c(),Ns=a("span"),Sr=n("Image-conditioned text-to-image generation"),di=c(),Ze=a("p"),xr=n("In this case, we condition the generation pipeline with an input image as well as a text prompt. Let\u2019s take the "),El=a("a"),$r=n("StableDiffusionInstructPix2PixPipeline"),Pr=n(", as an example. It takes an edit instruction as an input prompt and an input image to be edited."),Mi=c(),Wl=a("p"),Yr=n("Here is one example:"),wi=c(),Gl=a("p"),Cl=a("img"),yi=c(),X=a("p"),Qr=n("One strategy to evaluate such a model is to measure the consistency of the change between the two images (in "),Et=a("a"),zr=n("CLIP"),Hr=n(" space) with the change between the two image captions (as shown in "),Wt=a("a"),Dr=n("CLIP-Guided Domain Adaptation of Image Generators"),qr=n("). This is referred to as the \u201D"),Rs=a("strong"),Lr=n("CLIP directional similarity"),Kr=n("\u201C."),Ji=c(),Be=a("ul"),Fs=a("li"),Or=n("Caption 1 corresponds to the input image (image 1) that is to be edited."),ep=c(),As=a("li"),tp=n("Caption 2 corresponds to the edited image (image 2). It should reflect the edit instruction."),Ti=c(),kl=a("p"),lp=n("Following is a pictorial overview:"),ji=c(),Xl=a("p"),Vl=a("img"),bi=c(),Nl=a("p"),sp=n("We have prepared a mini dataset to implement this metric. Let\u2019s first load the dataset."),Ii=c(),d(Gt.$$.fragment),Ui=c(),d(Ct.$$.fragment),gi=c(),Rl=a("p"),ap=n("Here we have:"),vi=c(),H=a("ul"),Ee=a("li"),Ss=a("code"),ip=n("input"),op=n(" is a caption corresponding to the "),xs=a("code"),np=n("image"),rp=n("."),pp=c(),Fl=a("li"),$s=a("code"),cp=n("edit"),mp=n(" denotes the edit instruction."),fp=c(),We=a("li"),Ps=a("code"),hp=n("output"),up=n(" denotes the modified caption reflecting the "),Ys=a("code"),dp=n("edit"),Mp=n(" instruction."),_i=c(),Al=a("p"),wp=n("Let\u2019s take a look at a sample."),Zi=c(),d(kt.$$.fragment),Bi=c(),d(Xt.$$.fragment),Ei=c(),Sl=a("p"),yp=n("And here is the image:"),Wi=c(),d(Vt.$$.fragment),Gi=c(),xl=a("p"),$l=a("img"),Ci=c(),Pl=a("p"),Jp=n("We will first edit the images of our dataset with the edit instruction and compute the directional similarity."),ki=c(),Ge=a("p"),Tp=n("Let\u2019s first load the "),Yl=a("a"),jp=n("StableDiffusionInstructPix2PixPipeline"),bp=n(":"),Xi=c(),d(Nt.$$.fragment),Vi=c(),Ql=a("p"),Ip=n("Now, we perform the edits:"),Ni=c(),d(Rt.$$.fragment),Ri=c(),zl=a("p"),Up=n("To measure the directional similarity, we first load CLIP\u2019s image and text encoders:"),Fi=c(),d(Ft.$$.fragment),Ai=c(),D=a("p"),gp=n("Notice that we are using a particular CLIP checkpoint, i.e.,\xA0"),Qs=a("code"),vp=n("openai/clip-vit-large-patch14"),_p=n(". This is because the Stable Diffusion pre-training was performed with this CLIP variant. For more details, refer to the\xA0"),At=a("a"),Zp=n("documentation"),Bp=n("."),Si=c(),Ce=a("p"),Ep=n("Next, we prepare a PyTorch\xA0"),zs=a("code"),Wp=n("nn.Module"),Gp=n("\xA0to compute directional similarity:"),xi=c(),d(St.$$.fragment),$i=c(),ke=a("p"),Cp=n("Let\u2019s put\xA0"),Hs=a("code"),kp=n("DirectionalSimilarity"),Xp=n("\xA0to use now."),Pi=c(),d(xt.$$.fragment),Yi=c(),Hl=a("p"),Vp=n("Like the CLIP Score, the higher the CLIP directional similarity, the better it is."),Qi=c(),V=a("p"),Np=n("It should be noted that the\xA0"),Ds=a("code"),Rp=n("StableDiffusionInstructPix2PixPipeline"),Fp=n("\xA0exposes two arguments, namely,\xA0"),qs=a("code"),Ap=n("image_guidance_scale"),Sp=n("\xA0and\xA0"),Ls=a("code"),xp=n("guidance_scale"),$p=n("\xA0that let you control the quality of the final edited image. We encourage you to experiment with these two arguments and see the impact of that on the directional similarity."),zi=c(),Xe=a("p"),Pp=n("We can extend the idea of this metric to measure how similar the original image and edited version are. To do that, we can just do\xA0"),Ks=a("code"),Yp=n("F.cosine_similarity(img_feat_two, img_feat_one)"),Qp=n(". For these kinds of edits, we would still want the primary semantics of the images to be preserved as much as possible, i.e., a high similarity score."),Hi=c(),Ve=a("p"),zp=n("We can use these metrics for similar pipelines such as the "),$t=a("a"),Os=a("code"),Hp=n("StableDiffusionPix2PixZeroPipeline"),Dp=n("."),Di=c(),d(Ne.$$.fragment),qi=c(),ie=a("p"),ea=a("strong"),ta=a("em"),qp=n("Extending metrics like IS, FID (discussed later), or KID can be difficult"),Lp=n(" when the model under evaluation was pre-trained on a large image-captioning dataset (such as the "),Pt=a("a"),Kp=n("LAION-5B dataset"),Op=n("). This is because underlying these metrics is an InceptionNet (pre-trained on the ImageNet-1k dataset) used for extracting intermediate image features. The pre-training dataset of Stable Diffusion may have limited overlap with the pre-training dataset of InceptionNet, so it is not a good candidate here for feature extraction."),Li=c(),Dl=a("p"),la=a("strong"),Yt=a("em"),ec=n("Using the above metrics helps evaluate models that are class-conditioned. For example, "),Qt=a("a"),tc=n("DiT"),lc=n(". It was pre-trained being conditioned on the ImageNet-1k classes."),Ki=c(),oe=a("h3"),Re=a("a"),sa=a("span"),d(zt.$$.fragment),sc=c(),aa=a("span"),ac=n("Class-conditioned image generation"),Oi=c(),G=a("p"),ic=n("Class-conditioned generative models are usually pre-trained on a class-labeled dataset such as "),Ht=a("a"),oc=n("ImageNet-1k"),nc=n(". Popular metrics for evaluating these models include Fr\xE9chet Inception Distance (FID), Kernel Inception Distance (KID), and Inception Score (IS). In this document, we focus on FID ("),Dt=a("a"),rc=n("Heusel et al."),pc=n("). We show how to compute it with the "),qt=a("a"),ia=a("code"),cc=n("DiTPipeline"),mc=n(", which uses the "),Lt=a("a"),fc=n("DiT model"),hc=n(" under the hood."),eo=c(),Fe=a("p"),uc=n("FID aims to measure how similar are two datasets of images. As per "),Kt=a("a"),dc=n("this resource"),Mc=n(":"),to=c(),ql=a("blockquote"),oa=a("p"),wc=n("Fr\xE9chet Inception Distance is a measure of similarity between two datasets of images. It was shown to correlate well with the human judgment of visual quality and is most often used to evaluate the quality of samples of Generative Adversarial Networks. FID is calculated by computing the Fr\xE9chet distance between two Gaussians fitted to feature representations of the Inception network."),lo=c(),Ll=a("p"),yc=n("These two datasets are essentially the dataset of real images and the dataset of fake images (generated images in our case). FID is usually calculated with two large datasets. However, for this document, we will work with two mini datasets."),so=c(),Kl=a("p"),Jc=n("Let\u2019s first download a few images from the ImageNet-1k training set:"),ao=c(),d(Ot.$$.fragment),io=c(),d(el.$$.fragment),oo=c(),Ol=a("p"),Tc=n("These are 10 images from the following Imagenet-1k classes: \u201Ccassette_player\u201D, \u201Cchain_saw\u201D (x2), \u201Cchurch\u201D, \u201Cgas_pump\u201D (x3), \u201Cparachute\u201D (x2), and \u201Ctench\u201D."),no=c(),q=a("p"),es=a("img"),jc=a("br"),bc=c(),na=a("em"),Ic=n("Real images."),ro=c(),ts=a("p"),Uc=n("Now that the images are loaded, let\u2019s apply some lightweight pre-processing on them to use them for FID calculation."),po=c(),d(tl.$$.fragment),co=c(),Ae=a("p"),gc=n("We now load the\xA0"),ll=a("a"),ra=a("code"),vc=n("DiTPipeline"),_c=n(" to generate images conditioned on the above-mentioned classes."),mo=c(),d(sl.$$.fragment),fo=c(),Se=a("p"),Zc=n("Now, we can compute the FID using\xA0"),al=a("a"),pa=a("code"),Bc=n("torchmetrics"),Ec=n("."),ho=c(),d(il.$$.fragment),uo=c(),ls=a("p"),Wc=n("The lower the FID, the better it is. Several things can influence FID here:"),Mo=c(),N=a("ul"),ca=a("li"),Gc=n("Number of images (both real and fake)"),Cc=c(),ma=a("li"),kc=n("Randomness induced in the diffusion process"),Xc=c(),fa=a("li"),Vc=n("Number of inference steps in the diffusion process"),Nc=c(),ha=a("li"),Rc=n("The scheduler being used in the diffusion process"),wo=c(),ss=a("p"),Fc=n("For the last two points, it is, therefore, a good practice to run the evaluation across different seeds and inference steps, and then report an average result."),yo=c(),d(xe.$$.fragment),Jo=c(),$e=a("p"),Ac=n("As a final step, let\u2019s visually inspect the\xA0"),ua=a("code"),Sc=n("fake_images"),xc=n("."),To=c(),L=a("p"),as=a("img"),$c=a("br"),Pc=c(),da=a("em"),Yc=n("Fake images."),this.h()},l(e){const s=kh('[data-svelte="svelte-1phssyn"]',document.head);h=i(s,"META",{name:!0,content:!0}),s.forEach(t),v=m(e),u=i(e,"H1",{class:!0});var ol=o(u);j=i(ol,"A",{id:!0,class:!0,href:!0});var Ma=o(j);_=i(Ma,"SPAN",{});var wa=o(_);M(I.$$.fragment,wa),wa.forEach(t),Ma.forEach(t),g=m(ol),Z=i(ol,"SPAN",{});var ya=o(Z);ze=r(ya,"Evaluating Diffusion Models"),ya.forEach(t),ol.forEach(t),re=m(e),E=i(e,"A",{target:!0,href:!0});var am=o(E);S=i(am,"IMG",{src:!0,alt:!0}),am.forEach(t),R=m(e),k=i(e,"P",{});var bo=o(k);pe=r(bo,"Evaluation of generative models like "),W=i(bo,"A",{href:!0,rel:!0});var im=o(W);De=r(im,"Stable Diffusion"),im.forEach(t),b=r(bo," is subjective in nature. But as practitioners and researchers, we often have to make careful choices amongst many different possibilities. So, when working with different generative models (like GANs, Diffusion, etc.), how do we choose one over the other?"),bo.forEach(t),B=m(e),C=i(e,"P",{});var om=o(C);rl=r(om,`Qualitative evaluation of such models can be error-prone and might incorrectly influence a decision.
However, quantitative metrics don\u2019t necessarily correspond to image quality. So, usually, a combination
of both qualitative and quantitative evaluations provides a stronger signal when choosing one model
over the other.`),om.forEach(t),qe=m(e),F=i(e,"P",{});var Io=o(F);pl=r(Io,"In this document, we provide a non-exhaustive overview of qualitative and quantitative methods to evaluate Diffusion models. For quantitative methods, we specifically focus on how to implement them alongside "),ce=i(Io,"CODE",{});var nm=o(ce);tn=r(nm,"diffusers"),nm.forEach(t),ln=r(Io,"."),Io.forEach(t),Ua=m(e),me=i(e,"P",{});var Uo=o(me);sn=r(Uo,"The methods shown in this document can also be used to evaluate different "),Le=i(Uo,"A",{href:!0,rel:!0});var rm=o(Le);an=r(rm,"noise schedulers"),rm.forEach(t),on=r(Uo," keeping the underlying generation model fixed."),Uo.forEach(t),ga=m(e),O=i(e,"H2",{class:!0});var go=o(O);fe=i(go,"A",{id:!0,class:!0,href:!0});var pm=o(fe);us=i(pm,"SPAN",{});var cm=o(us);M(Ke.$$.fragment,cm),cm.forEach(t),pm.forEach(t),nn=m(go),ds=i(go,"SPAN",{});var mm=o(ds);rn=r(mm,"Scenarios"),mm.forEach(t),go.forEach(t),va=m(e),cl=i(e,"P",{});var fm=o(cl);pn=r(fm,"We cover Diffusion models with the following pipelines:"),fm.forEach(t),_a=m(e),x=i(e,"UL",{});var is=o(x);Oe=i(is,"LI",{});var vo=o(Oe);cn=r(vo,"Text-guided image generation (such as the "),et=i(vo,"A",{href:!0,rel:!0});var hm=o(et);Ms=i(hm,"CODE",{});var um=o(Ms);mn=r(um,"StableDiffusionPipeline"),um.forEach(t),hm.forEach(t),fn=r(vo,")."),vo.forEach(t),hn=m(is),ee=i(is,"LI",{});var os=o(ee);un=r(os,"Text-guided image generation, additionally conditioned on an input image (such as the "),tt=i(os,"A",{href:!0,rel:!0});var dm=o(tt);ws=i(dm,"CODE",{});var Mm=o(ws);dn=r(Mm,"StableDiffusionImg2ImgPipeline"),Mm.forEach(t),dm.forEach(t),Mn=r(os,", and "),lt=i(os,"A",{href:!0,rel:!0});var wm=o(lt);ys=i(wm,"CODE",{});var ym=o(ys);wn=r(ym,"StableDiffusionInstructPix2PixPipeline"),ym.forEach(t),wm.forEach(t),yn=r(os,")."),os.forEach(t),Jn=m(is),st=i(is,"LI",{});var _o=o(st);Tn=r(_o,"Class-conditioned image generation models (such as the "),at=i(_o,"A",{href:!0,rel:!0});var Jm=o(at);Js=i(Jm,"CODE",{});var Tm=o(Js);jn=r(Tm,"DiTPipeline"),Tm.forEach(t),Jm.forEach(t),bn=r(_o,")."),_o.forEach(t),is.forEach(t),Za=m(e),te=i(e,"H2",{class:!0});var Zo=o(te);he=i(Zo,"A",{id:!0,class:!0,href:!0});var jm=o(he);Ts=i(jm,"SPAN",{});var bm=o(Ts);M(it.$$.fragment,bm),bm.forEach(t),jm.forEach(t),In=m(Zo),js=i(Zo,"SPAN",{});var Im=o(js);Un=r(Im,"Qualitative Evaluation"),Im.forEach(t),Zo.forEach(t),Ba=m(e),$=i(e,"P",{});var ns=o($);gn=r(ns,`Qualitative evaluation typically involves human assessment of generated images. Quality is measured across aspects such as compositionality, image-text alignment, and spatial relations. Common prompts provide a degree of uniformity for subjective metrics.
DrawBench and PartiPrompts are prompt datasets used for qualitative benchmarking. DrawBench and PartiPrompts were introduced by `),ot=i(ns,"A",{href:!0,rel:!0});var Um=o(ot);vn=r(Um,"Imagen"),Um.forEach(t),_n=r(ns," and "),nt=i(ns,"A",{href:!0,rel:!0});var gm=o(nt);Zn=r(gm,"Parti"),gm.forEach(t),Bn=r(ns," respectively."),ns.forEach(t),Ea=m(e),ue=i(e,"P",{});var Bo=o(ue);En=r(Bo,"From the "),rt=i(Bo,"A",{href:!0,rel:!0});var vm=o(rt);Wn=r(vm,"official Parti website"),vm.forEach(t),Gn=r(Bo,":"),Bo.forEach(t),Wa=m(e),ml=i(e,"BLOCKQUOTE",{});var _m=o(ml);bs=i(_m,"P",{});var Zm=o(bs);Cn=r(Zm,"PartiPrompts (P2) is a rich set of over 1600 prompts in English that we release as part of this work. P2 can be used to measure model capabilities across various categories and challenge aspects."),Zm.forEach(t),_m.forEach(t),Ga=m(e),fl=i(e,"P",{});var Bm=o(fl);hl=i(Bm,"IMG",{src:!0,alt:!0}),Bm.forEach(t),Ca=m(e),ul=i(e,"P",{});var Em=o(ul);kn=r(Em,"PartiPrompts has the following columns:"),Em.forEach(t),ka=m(e),P=i(e,"UL",{});var rs=o(P);Is=i(rs,"LI",{});var Wm=o(Is);Xn=r(Wm,"Prompt"),Wm.forEach(t),Vn=m(rs),Us=i(rs,"LI",{});var Gm=o(Us);Nn=r(Gm,"Category of the prompt (such as \u201CAbstract\u201D, \u201CWorld Knowledge\u201D, etc.)"),Gm.forEach(t),Rn=m(rs),gs=i(rs,"LI",{});var Cm=o(gs);Fn=r(Cm,"Challenge reflecting the difficulty (such as \u201CBasic\u201D, \u201CComplex\u201D, \u201CWriting & Symbols\u201D, etc.)"),Cm.forEach(t),rs.forEach(t),Xa=m(e),dl=i(e,"P",{});var km=o(dl);An=r(km,"These benchmarks allow for side-by-side human evaluation of different image generation models."),km.forEach(t),Va=m(e),de=i(e,"P",{});var Eo=o(de);Sn=r(Eo,"For this, the \u{1F9E8} Diffusers team has built "),vs=i(Eo,"STRONG",{});var Xm=o(vs);xn=r(Xm,"Open Parti Prompts"),Xm.forEach(t),$n=r(Eo,", which is a community-driven qualitative benchmark based on Parti Prompts to compare state-of-the-art open-source diffusion models:"),Eo.forEach(t),Na=m(e),Me=i(e,"UL",{});var Wo=o(Me);Ml=i(Wo,"LI",{});var Qc=o(Ml);pt=i(Qc,"A",{href:!0,rel:!0});var Vm=o(pt);Pn=r(Vm,"Open Parti Prompts Game"),Vm.forEach(t),Yn=r(Qc,": For 10 parti prompts, 4 generated images are shown and the user selects the image that suits the prompt best."),Qc.forEach(t),Qn=m(Wo),wl=i(Wo,"LI",{});var zc=o(wl);ct=i(zc,"A",{href:!0,rel:!0});var Nm=o(ct);zn=r(Nm,"Open Parti Prompts Leaderboard"),Nm.forEach(t),Hn=r(zc,": The leaderboard comparing the currently best open-sourced diffusion models to each other."),zc.forEach(t),Wo.forEach(t),Ra=m(e),we=i(e,"P",{});var Go=o(we);Dn=r(Go,"To manually compare images, let\u2019s see how we can use "),_s=i(Go,"CODE",{});var Rm=o(_s);qn=r(Rm,"diffusers"),Rm.forEach(t),Ln=r(Go," on a couple of PartiPrompts."),Go.forEach(t),Fa=m(e),ye=i(e,"P",{});var Co=o(ye);Kn=r(Co,"Below we show some prompts sampled across different challenges: Basic, Complex, Linguistic Structures, Imagination, and Writing & Symbols. Here we are using PartiPrompts as a "),mt=i(Co,"A",{href:!0,rel:!0});var Fm=o(mt);On=r(Fm,"dataset"),Fm.forEach(t),er=r(Co,"."),Co.forEach(t),Aa=m(e),M(ft.$$.fragment,e),Sa=m(e),Je=i(e,"P",{});var ko=o(Je);tr=r(ko,"Now we can use these prompts to generate some images using Stable Diffusion ("),ht=i(ko,"A",{href:!0,rel:!0});var Am=o(ht);lr=r(Am,"v1-4 checkpoint"),Am.forEach(t),sr=r(ko,"):"),ko.forEach(t),xa=m(e),M(ut.$$.fragment,e),$a=m(e),yl=i(e,"P",{});var Sm=o(yl);Jl=i(Sm,"IMG",{src:!0,alt:!0}),Sm.forEach(t),Pa=m(e),Y=i(e,"P",{});var ps=o(Y);ar=r(ps,"We can also set "),Zs=i(ps,"CODE",{});var xm=o(Zs);ir=r(xm,"num_images_per_prompt"),xm.forEach(t),or=r(ps," accordingly to compare different images for the same prompt. Running the same pipeline but with a different checkpoint ("),dt=i(ps,"A",{href:!0,rel:!0});var $m=o(dt);nr=r($m,"v1-5"),$m.forEach(t),rr=r(ps,"), yields:"),ps.forEach(t),Ya=m(e),Tl=i(e,"P",{});var Pm=o(Tl);jl=i(Pm,"IMG",{src:!0,alt:!0}),Pm.forEach(t),Qa=m(e),bl=i(e,"P",{});var Ym=o(bl);pr=r(Ym,`Once several images are generated from all the prompts using multiple models (under evaluation), these results are presented to human evaluators for scoring. For
more details on the DrawBench and PartiPrompts benchmarks, refer to their respective papers.`),Ym.forEach(t),za=m(e),M(Te.$$.fragment,e),Ha=m(e),le=i(e,"H2",{class:!0});var Xo=o(le);je=i(Xo,"A",{id:!0,class:!0,href:!0});var Qm=o(je);Bs=i(Qm,"SPAN",{});var zm=o(Bs);M(Mt.$$.fragment,zm),zm.forEach(t),Qm.forEach(t),cr=m(Xo),Es=i(Xo,"SPAN",{});var Hm=o(Es);mr=r(Hm,"Quantitative Evaluation"),Hm.forEach(t),Xo.forEach(t),Da=m(e),Il=i(e,"P",{});var Dm=o(Il);fr=r(Dm,"In this section, we will walk you through how to evaluate three different diffusion pipelines using:"),Dm.forEach(t),qa=m(e),Q=i(e,"UL",{});var cs=o(Q);Ws=i(cs,"LI",{});var qm=o(Ws);hr=r(qm,"CLIP score"),qm.forEach(t),ur=m(cs),Gs=i(cs,"LI",{});var Lm=o(Gs);dr=r(Lm,"CLIP directional similarity"),Lm.forEach(t),Mr=m(cs),Cs=i(cs,"LI",{});var Km=o(Cs);wr=r(Km,"FID"),Km.forEach(t),cs.forEach(t),La=m(e),se=i(e,"H3",{class:!0});var Vo=o(se);be=i(Vo,"A",{id:!0,class:!0,href:!0});var Om=o(be);ks=i(Om,"SPAN",{});var ef=o(ks);M(wt.$$.fragment,ef),ef.forEach(t),Om.forEach(t),yr=m(Vo),Xs=i(Vo,"SPAN",{});var tf=o(Xs);Jr=r(tf,"Text-guided image generation"),tf.forEach(t),Vo.forEach(t),Ka=m(e),yt=i(e,"P",{});var Hc=o(yt);Jt=i(Hc,"A",{href:!0,rel:!0});var lf=o(Jt);Tr=r(lf,"CLIP score"),lf.forEach(t),jr=r(Hc," measures the compatibility of image-caption pairs. Higher CLIP scores imply higher compatibility \u{1F53C}. The CLIP score is a quantitative measurement of the qualitative concept \u201Ccompatibility\u201D. Image-caption pair compatibility can also be thought of as the semantic similarity between the image and the caption. CLIP score was found to have high correlation with human judgement."),Hc.forEach(t),Oa=m(e),Ie=i(e,"P",{});var No=o(Ie);br=r(No,"Let\u2019s first load a "),Ul=i(No,"A",{href:!0});var sf=o(Ul);Ir=r(sf,"StableDiffusionPipeline"),sf.forEach(t),Ur=r(No,":"),No.forEach(t),ei=m(e),M(Tt.$$.fragment,e),ti=m(e),gl=i(e,"P",{});var af=o(gl);gr=r(af,"Generate some images with multiple prompts:"),af.forEach(t),li=m(e),M(jt.$$.fragment,e),si=m(e),vl=i(e,"P",{});var of=o(vl);vr=r(of,"And then, we calculate the CLIP score."),of.forEach(t),ai=m(e),M(bt.$$.fragment,e),ii=m(e),_l=i(e,"P",{});var nf=o(_l);_r=r(nf,"In the above example, we generated one image per prompt. If we generated multiple images per prompt, we would have to take the average score from the generated images per prompt."),nf.forEach(t),oi=m(e),z=i(e,"P",{});var ms=o(z);Zr=r(ms,"Now, if we wanted to compare two checkpoints compatible with the "),Zl=i(ms,"A",{href:!0});var rf=o(Zl);Br=r(rf,"StableDiffusionPipeline"),rf.forEach(t),Er=r(ms,` we should pass a generator while calling the pipeline. First, we generate images with a
fixed seed with the `),It=i(ms,"A",{href:!0,rel:!0});var pf=o(It);Wr=r(pf,"v1-4 Stable Diffusion checkpoint"),pf.forEach(t),Gr=r(ms,":"),ms.forEach(t),ni=m(e),M(Ut.$$.fragment,e),ri=m(e),Ue=i(e,"P",{});var Ro=o(Ue);Cr=r(Ro,"Then we load the "),gt=i(Ro,"A",{href:!0,rel:!0});var cf=o(gt);kr=r(cf,"v1-5 checkpoint"),cf.forEach(t),Xr=r(Ro," to generate images:"),Ro.forEach(t),pi=m(e),M(vt.$$.fragment,e),ci=m(e),Bl=i(e,"P",{});var mf=o(Bl);Vr=r(mf,"And finally, we compare their CLIP scores:"),mf.forEach(t),mi=m(e),M(_t.$$.fragment,e),fi=m(e),ge=i(e,"P",{});var Fo=o(ge);Nr=r(Fo,"It seems like the "),Zt=i(Fo,"A",{href:!0,rel:!0});var ff=o(Zt);Rr=r(ff,"v1-5"),ff.forEach(t),Fr=r(Fo," checkpoint performs better than its predecessor. Note, however, that the number of prompts we used to compute the CLIP scores is quite low. For a more practical evaluation, this number should be way higher, and the prompts should be diverse."),Fo.forEach(t),hi=m(e),M(ve.$$.fragment,e),ui=m(e),ae=i(e,"H3",{class:!0});var Ao=o(ae);_e=i(Ao,"A",{id:!0,class:!0,href:!0});var hf=o(_e);Vs=i(hf,"SPAN",{});var uf=o(Vs);M(Bt.$$.fragment,uf),uf.forEach(t),hf.forEach(t),Ar=m(Ao),Ns=i(Ao,"SPAN",{});var df=o(Ns);Sr=r(df,"Image-conditioned text-to-image generation"),df.forEach(t),Ao.forEach(t),di=m(e),Ze=i(e,"P",{});var So=o(Ze);xr=r(So,"In this case, we condition the generation pipeline with an input image as well as a text prompt. Let\u2019s take the "),El=i(So,"A",{href:!0});var Mf=o(El);$r=r(Mf,"StableDiffusionInstructPix2PixPipeline"),Mf.forEach(t),Pr=r(So,", as an example. It takes an edit instruction as an input prompt and an input image to be edited."),So.forEach(t),Mi=m(e),Wl=i(e,"P",{});var wf=o(Wl);Yr=r(wf,"Here is one example:"),wf.forEach(t),wi=m(e),Gl=i(e,"P",{});var yf=o(Gl);Cl=i(yf,"IMG",{src:!0,alt:!0}),yf.forEach(t),yi=m(e),X=i(e,"P",{});var Pe=o(X);Qr=r(Pe,"One strategy to evaluate such a model is to measure the consistency of the change between the two images (in "),Et=i(Pe,"A",{href:!0,rel:!0});var Jf=o(Et);zr=r(Jf,"CLIP"),Jf.forEach(t),Hr=r(Pe," space) with the change between the two image captions (as shown in "),Wt=i(Pe,"A",{href:!0,rel:!0});var Tf=o(Wt);Dr=r(Tf,"CLIP-Guided Domain Adaptation of Image Generators"),Tf.forEach(t),qr=r(Pe,"). This is referred to as the \u201D"),Rs=i(Pe,"STRONG",{});var jf=o(Rs);Lr=r(jf,"CLIP directional similarity"),jf.forEach(t),Kr=r(Pe,"\u201C."),Pe.forEach(t),Ji=m(e),Be=i(e,"UL",{});var xo=o(Be);Fs=i(xo,"LI",{});var bf=o(Fs);Or=r(bf,"Caption 1 corresponds to the input image (image 1) that is to be edited."),bf.forEach(t),ep=m(xo),As=i(xo,"LI",{});var If=o(As);tp=r(If,"Caption 2 corresponds to the edited image (image 2). It should reflect the edit instruction."),If.forEach(t),xo.forEach(t),Ti=m(e),kl=i(e,"P",{});var Uf=o(kl);lp=r(Uf,"Following is a pictorial overview:"),Uf.forEach(t),ji=m(e),Xl=i(e,"P",{});var gf=o(Xl);Vl=i(gf,"IMG",{src:!0,alt:!0}),gf.forEach(t),bi=m(e),Nl=i(e,"P",{});var vf=o(Nl);sp=r(vf,"We have prepared a mini dataset to implement this metric. Let\u2019s first load the dataset."),vf.forEach(t),Ii=m(e),M(Gt.$$.fragment,e),Ui=m(e),M(Ct.$$.fragment,e),gi=m(e),Rl=i(e,"P",{});var _f=o(Rl);ap=r(_f,"Here we have:"),_f.forEach(t),vi=m(e),H=i(e,"UL",{});var fs=o(H);Ee=i(fs,"LI",{});var Ja=o(Ee);Ss=i(Ja,"CODE",{});var Zf=o(Ss);ip=r(Zf,"input"),Zf.forEach(t),op=r(Ja," is a caption corresponding to the "),xs=i(Ja,"CODE",{});var Bf=o(xs);np=r(Bf,"image"),Bf.forEach(t),rp=r(Ja,"."),Ja.forEach(t),pp=m(fs),Fl=i(fs,"LI",{});var Dc=o(Fl);$s=i(Dc,"CODE",{});var Ef=o($s);cp=r(Ef,"edit"),Ef.forEach(t),mp=r(Dc," denotes the edit instruction."),Dc.forEach(t),fp=m(fs),We=i(fs,"LI",{});var Ta=o(We);Ps=i(Ta,"CODE",{});var Wf=o(Ps);hp=r(Wf,"output"),Wf.forEach(t),up=r(Ta," denotes the modified caption reflecting the "),Ys=i(Ta,"CODE",{});var Gf=o(Ys);dp=r(Gf,"edit"),Gf.forEach(t),Mp=r(Ta," instruction."),Ta.forEach(t),fs.forEach(t),_i=m(e),Al=i(e,"P",{});var Cf=o(Al);wp=r(Cf,"Let\u2019s take a look at a sample."),Cf.forEach(t),Zi=m(e),M(kt.$$.fragment,e),Bi=m(e),M(Xt.$$.fragment,e),Ei=m(e),Sl=i(e,"P",{});var kf=o(Sl);yp=r(kf,"And here is the image:"),kf.forEach(t),Wi=m(e),M(Vt.$$.fragment,e),Gi=m(e),xl=i(e,"P",{});var Xf=o(xl);$l=i(Xf,"IMG",{src:!0,alt:!0}),Xf.forEach(t),Ci=m(e),Pl=i(e,"P",{});var Vf=o(Pl);Jp=r(Vf,"We will first edit the images of our dataset with the edit instruction and compute the directional similarity."),Vf.forEach(t),ki=m(e),Ge=i(e,"P",{});var $o=o(Ge);Tp=r($o,"Let\u2019s first load the "),Yl=i($o,"A",{href:!0});var Nf=o(Yl);jp=r(Nf,"StableDiffusionInstructPix2PixPipeline"),Nf.forEach(t),bp=r($o,":"),$o.forEach(t),Xi=m(e),M(Nt.$$.fragment,e),Vi=m(e),Ql=i(e,"P",{});var Rf=o(Ql);Ip=r(Rf,"Now, we perform the edits:"),Rf.forEach(t),Ni=m(e),M(Rt.$$.fragment,e),Ri=m(e),zl=i(e,"P",{});var Ff=o(zl);Up=r(Ff,"To measure the directional similarity, we first load CLIP\u2019s image and text encoders:"),Ff.forEach(t),Fi=m(e),M(Ft.$$.fragment,e),Ai=m(e),D=i(e,"P",{});var hs=o(D);gp=r(hs,"Notice that we are using a particular CLIP checkpoint, i.e.,\xA0"),Qs=i(hs,"CODE",{});var Af=o(Qs);vp=r(Af,"openai/clip-vit-large-patch14"),Af.forEach(t),_p=r(hs,". This is because the Stable Diffusion pre-training was performed with this CLIP variant. For more details, refer to the\xA0"),At=i(hs,"A",{href:!0,rel:!0});var Sf=o(At);Zp=r(Sf,"documentation"),Sf.forEach(t),Bp=r(hs,"."),hs.forEach(t),Si=m(e),Ce=i(e,"P",{});var Po=o(Ce);Ep=r(Po,"Next, we prepare a PyTorch\xA0"),zs=i(Po,"CODE",{});var xf=o(zs);Wp=r(xf,"nn.Module"),xf.forEach(t),Gp=r(Po,"\xA0to compute directional similarity:"),Po.forEach(t),xi=m(e),M(St.$$.fragment,e),$i=m(e),ke=i(e,"P",{});var Yo=o(ke);Cp=r(Yo,"Let\u2019s put\xA0"),Hs=i(Yo,"CODE",{});var $f=o(Hs);kp=r($f,"DirectionalSimilarity"),$f.forEach(t),Xp=r(Yo,"\xA0to use now."),Yo.forEach(t),Pi=m(e),M(xt.$$.fragment,e),Yi=m(e),Hl=i(e,"P",{});var Pf=o(Hl);Vp=r(Pf,"Like the CLIP Score, the higher the CLIP directional similarity, the better it is."),Pf.forEach(t),Qi=m(e),V=i(e,"P",{});var Ye=o(V);Np=r(Ye,"It should be noted that the\xA0"),Ds=i(Ye,"CODE",{});var Yf=o(Ds);Rp=r(Yf,"StableDiffusionInstructPix2PixPipeline"),Yf.forEach(t),Fp=r(Ye,"\xA0exposes two arguments, namely,\xA0"),qs=i(Ye,"CODE",{});var Qf=o(qs);Ap=r(Qf,"image_guidance_scale"),Qf.forEach(t),Sp=r(Ye,"\xA0and\xA0"),Ls=i(Ye,"CODE",{});var zf=o(Ls);xp=r(zf,"guidance_scale"),zf.forEach(t),$p=r(Ye,"\xA0that let you control the quality of the final edited image. We encourage you to experiment with these two arguments and see the impact of that on the directional similarity."),Ye.forEach(t),zi=m(e),Xe=i(e,"P",{});var Qo=o(Xe);Pp=r(Qo,"We can extend the idea of this metric to measure how similar the original image and edited version are. To do that, we can just do\xA0"),Ks=i(Qo,"CODE",{});var Hf=o(Ks);Yp=r(Hf,"F.cosine_similarity(img_feat_two, img_feat_one)"),Hf.forEach(t),Qp=r(Qo,". For these kinds of edits, we would still want the primary semantics of the images to be preserved as much as possible, i.e., a high similarity score."),Qo.forEach(t),Hi=m(e),Ve=i(e,"P",{});var zo=o(Ve);zp=r(zo,"We can use these metrics for similar pipelines such as the "),$t=i(zo,"A",{href:!0,rel:!0});var Df=o($t);Os=i(Df,"CODE",{});var qf=o(Os);Hp=r(qf,"StableDiffusionPix2PixZeroPipeline"),qf.forEach(t),Df.forEach(t),Dp=r(zo,"."),zo.forEach(t),Di=m(e),M(Ne.$$.fragment,e),qi=m(e),ie=i(e,"P",{});var ja=o(ie);ea=i(ja,"STRONG",{});var Lf=o(ea);ta=i(Lf,"EM",{});var Kf=o(ta);qp=r(Kf,"Extending metrics like IS, FID (discussed later), or KID can be difficult"),Kf.forEach(t),Lf.forEach(t),Lp=r(ja," when the model under evaluation was pre-trained on a large image-captioning dataset (such as the "),Pt=i(ja,"A",{href:!0,rel:!0});var Of=o(Pt);Kp=r(Of,"LAION-5B dataset"),Of.forEach(t),Op=r(ja,"). This is because underlying these metrics is an InceptionNet (pre-trained on the ImageNet-1k dataset) used for extracting intermediate image features. The pre-training dataset of Stable Diffusion may have limited overlap with the pre-training dataset of InceptionNet, so it is not a good candidate here for feature extraction."),ja.forEach(t),Li=m(e),Dl=i(e,"P",{});var eh=o(Dl);la=i(eh,"STRONG",{});var th=o(la);Yt=i(th,"EM",{});var Ho=o(Yt);ec=r(Ho,"Using the above metrics helps evaluate models that are class-conditioned. For example, "),Qt=i(Ho,"A",{href:!0,rel:!0});var lh=o(Qt);tc=r(lh,"DiT"),lh.forEach(t),lc=r(Ho,". It was pre-trained being conditioned on the ImageNet-1k classes."),Ho.forEach(t),th.forEach(t),eh.forEach(t),Ki=m(e),oe=i(e,"H3",{class:!0});var Do=o(oe);Re=i(Do,"A",{id:!0,class:!0,href:!0});var sh=o(Re);sa=i(sh,"SPAN",{});var ah=o(sa);M(zt.$$.fragment,ah),ah.forEach(t),sh.forEach(t),sc=m(Do),aa=i(Do,"SPAN",{});var ih=o(aa);ac=r(ih,"Class-conditioned image generation"),ih.forEach(t),Do.forEach(t),Oi=m(e),G=i(e,"P",{});var K=o(G);ic=r(K,"Class-conditioned generative models are usually pre-trained on a class-labeled dataset such as "),Ht=i(K,"A",{href:!0,rel:!0});var oh=o(Ht);oc=r(oh,"ImageNet-1k"),oh.forEach(t),nc=r(K,". Popular metrics for evaluating these models include Fr\xE9chet Inception Distance (FID), Kernel Inception Distance (KID), and Inception Score (IS). In this document, we focus on FID ("),Dt=i(K,"A",{href:!0,rel:!0});var nh=o(Dt);rc=r(nh,"Heusel et al."),nh.forEach(t),pc=r(K,"). We show how to compute it with the "),qt=i(K,"A",{href:!0,rel:!0});var rh=o(qt);ia=i(rh,"CODE",{});var ph=o(ia);cc=r(ph,"DiTPipeline"),ph.forEach(t),rh.forEach(t),mc=r(K,", which uses the "),Lt=i(K,"A",{href:!0,rel:!0});var ch=o(Lt);fc=r(ch,"DiT model"),ch.forEach(t),hc=r(K," under the hood."),K.forEach(t),eo=m(e),Fe=i(e,"P",{});var qo=o(Fe);uc=r(qo,"FID aims to measure how similar are two datasets of images. As per "),Kt=i(qo,"A",{href:!0,rel:!0});var mh=o(Kt);dc=r(mh,"this resource"),mh.forEach(t),Mc=r(qo,":"),qo.forEach(t),to=m(e),ql=i(e,"BLOCKQUOTE",{});var fh=o(ql);oa=i(fh,"P",{});var hh=o(oa);wc=r(hh,"Fr\xE9chet Inception Distance is a measure of similarity between two datasets of images. It was shown to correlate well with the human judgment of visual quality and is most often used to evaluate the quality of samples of Generative Adversarial Networks. FID is calculated by computing the Fr\xE9chet distance between two Gaussians fitted to feature representations of the Inception network."),hh.forEach(t),fh.forEach(t),lo=m(e),Ll=i(e,"P",{});var uh=o(Ll);yc=r(uh,"These two datasets are essentially the dataset of real images and the dataset of fake images (generated images in our case). FID is usually calculated with two large datasets. However, for this document, we will work with two mini datasets."),uh.forEach(t),so=m(e),Kl=i(e,"P",{});var dh=o(Kl);Jc=r(dh,"Let\u2019s first download a few images from the ImageNet-1k training set:"),dh.forEach(t),ao=m(e),M(Ot.$$.fragment,e),io=m(e),M(el.$$.fragment,e),oo=m(e),Ol=i(e,"P",{});var Mh=o(Ol);Tc=r(Mh,"These are 10 images from the following Imagenet-1k classes: \u201Ccassette_player\u201D, \u201Cchain_saw\u201D (x2), \u201Cchurch\u201D, \u201Cgas_pump\u201D (x3), \u201Cparachute\u201D (x2), and \u201Ctench\u201D."),Mh.forEach(t),no=m(e),q=i(e,"P",{align:!0});var ba=o(q);es=i(ba,"IMG",{src:!0,alt:!0}),jc=i(ba,"BR",{}),bc=m(ba),na=i(ba,"EM",{});var wh=o(na);Ic=r(wh,"Real images."),wh.forEach(t),ba.forEach(t),ro=m(e),ts=i(e,"P",{});var yh=o(ts);Uc=r(yh,"Now that the images are loaded, let\u2019s apply some lightweight pre-processing on them to use them for FID calculation."),yh.forEach(t),po=m(e),M(tl.$$.fragment,e),co=m(e),Ae=i(e,"P",{});var Lo=o(Ae);gc=r(Lo,"We now load the\xA0"),ll=i(Lo,"A",{href:!0,rel:!0});var Jh=o(ll);ra=i(Jh,"CODE",{});var Th=o(ra);vc=r(Th,"DiTPipeline"),Th.forEach(t),Jh.forEach(t),_c=r(Lo," to generate images conditioned on the above-mentioned classes."),Lo.forEach(t),mo=m(e),M(sl.$$.fragment,e),fo=m(e),Se=i(e,"P",{});var Ko=o(Se);Zc=r(Ko,"Now, we can compute the FID using\xA0"),al=i(Ko,"A",{href:!0,rel:!0});var jh=o(al);pa=i(jh,"CODE",{});var bh=o(pa);Bc=r(bh,"torchmetrics"),bh.forEach(t),jh.forEach(t),Ec=r(Ko,"."),Ko.forEach(t),ho=m(e),M(il.$$.fragment,e),uo=m(e),ls=i(e,"P",{});var Ih=o(ls);Wc=r(Ih,"The lower the FID, the better it is. Several things can influence FID here:"),Ih.forEach(t),Mo=m(e),N=i(e,"UL",{});var Qe=o(N);ca=i(Qe,"LI",{});var Uh=o(ca);Gc=r(Uh,"Number of images (both real and fake)"),Uh.forEach(t),Cc=m(Qe),ma=i(Qe,"LI",{});var gh=o(ma);kc=r(gh,"Randomness induced in the diffusion process"),gh.forEach(t),Xc=m(Qe),fa=i(Qe,"LI",{});var vh=o(fa);Vc=r(vh,"Number of inference steps in the diffusion process"),vh.forEach(t),Nc=m(Qe),ha=i(Qe,"LI",{});var _h=o(ha);Rc=r(_h,"The scheduler being used in the diffusion process"),_h.forEach(t),Qe.forEach(t),wo=m(e),ss=i(e,"P",{});var Zh=o(ss);Fc=r(Zh,"For the last two points, it is, therefore, a good practice to run the evaluation across different seeds and inference steps, and then report an average result."),Zh.forEach(t),yo=m(e),M(xe.$$.fragment,e),Jo=m(e),$e=i(e,"P",{});var Oo=o($e);Ac=r(Oo,"As a final step, let\u2019s visually inspect the\xA0"),ua=i(Oo,"CODE",{});var Bh=o(ua);Sc=r(Bh,"fake_images"),Bh.forEach(t),xc=r(Oo,"."),Oo.forEach(t),To=m(e),L=i(e,"P",{align:!0});var Ia=o(L);as=i(Ia,"IMG",{src:!0,alt:!0}),$c=i(Ia,"BR",{}),Pc=m(Ia),da=i(Ia,"EM",{});var Eh=o(da);Yc=r(Eh,"Fake images."),Eh.forEach(t),Ia.forEach(t),this.h()},h(){f(h,"name","hf:doc:metadata"),f(h,"content",JSON.stringify(Sh)),f(j,"id","evaluating-diffusion-models"),f(j,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(j,"href","#evaluating-diffusion-models"),f(u,"class","relative group"),ne(S.src,He="https://colab.research.google.com/assets/colab-badge.svg")||f(S,"src",He),f(S,"alt","Open In Colab"),f(E,"target","_blank"),f(E,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/main/diffusers/evaluation.ipynb"),f(W,"href","https://huggingface.co/docs/diffusers/stable_diffusion"),f(W,"rel","nofollow"),f(Le,"href","https://huggingface.co/docs/diffusers/main/en/api/schedulers/overview"),f(Le,"rel","nofollow"),f(fe,"id","scenarios"),f(fe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(fe,"href","#scenarios"),f(O,"class","relative group"),f(et,"href","https://huggingface.co/docs/diffusers/main/en/api/pipelines/stable_diffusion/text2img"),f(et,"rel","nofollow"),f(tt,"href","https://huggingface.co/docs/diffusers/main/en/api/pipelines/stable_diffusion/img2img"),f(tt,"rel","nofollow"),f(lt,"href","https://huggingface.co/docs/diffusers/main/en/api/pipelines/stable_diffusion/pix2pix"),f(lt,"rel","nofollow"),f(at,"href","https://huggingface.co/docs/diffusers/main/en/api/pipelines/dit"),f(at,"rel","nofollow"),f(he,"id","qualitative-evaluation"),f(he,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(he,"href","#qualitative-evaluation"),f(te,"class","relative group"),f(ot,"href","https://imagen.research.google/"),f(ot,"rel","nofollow"),f(nt,"href","https://parti.research.google/"),f(nt,"rel","nofollow"),f(rt,"href","https://parti.research.google/"),f(rt,"rel","nofollow"),ne(hl.src,qc="https://huggingface.co/datasets/diffusers/docs-images/resolve/main/evaluation_diffusion_models/parti-prompts.png")||f(hl,"src",qc),f(hl,"alt","parti-prompts"),f(pt,"href","https://huggingface.co/spaces/OpenGenAI/open-parti-prompts"),f(pt,"rel","nofollow"),f(ct,"href","https://huggingface.co/spaces/OpenGenAI/parti-prompts-leaderboard"),f(ct,"rel","nofollow"),f(mt,"href","https://huggingface.co/datasets/nateraw/parti-prompts"),f(mt,"rel","nofollow"),f(ht,"href","https://huggingface.co/CompVis/stable-diffusion-v1-4"),f(ht,"rel","nofollow"),ne(Jl.src,Lc="https://huggingface.co/datasets/diffusers/docs-images/resolve/main/evaluation_diffusion_models/parti-prompts-14.png")||f(Jl,"src",Lc),f(Jl,"alt","parti-prompts-14"),f(dt,"href","https://huggingface.co/runwayml/stable-diffusion-v1-5"),f(dt,"rel","nofollow"),ne(jl.src,Kc="https://huggingface.co/datasets/diffusers/docs-images/resolve/main/evaluation_diffusion_models/parti-prompts-15.png")||f(jl,"src",Kc),f(jl,"alt","parti-prompts-15"),f(je,"id","quantitative-evaluation"),f(je,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(je,"href","#quantitative-evaluation"),f(le,"class","relative group"),f(be,"id","textguided-image-generation"),f(be,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(be,"href","#textguided-image-generation"),f(se,"class","relative group"),f(Jt,"href","https://arxiv.org/abs/2104.08718"),f(Jt,"rel","nofollow"),f(Ul,"href","/docs/diffusers/v0.17.1/en/api/pipelines/stable_diffusion/text2img#diffusers.StableDiffusionPipeline"),f(Zl,"href","/docs/diffusers/v0.17.1/en/api/pipelines/stable_diffusion/text2img#diffusers.StableDiffusionPipeline"),f(It,"href","https://huggingface.co/CompVis/stable-diffusion-v1-4"),f(It,"rel","nofollow"),f(gt,"href","https://huggingface.co/runwayml/stable-diffusion-v1-5"),f(gt,"rel","nofollow"),f(Zt,"href","https://huggingface.co/runwayml/stable-diffusion-v1-5"),f(Zt,"rel","nofollow"),f(_e,"id","imageconditioned-texttoimage-generation"),f(_e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(_e,"href","#imageconditioned-texttoimage-generation"),f(ae,"class","relative group"),f(El,"href","/docs/diffusers/v0.17.1/en/api/pipelines/pix2pix#diffusers.StableDiffusionInstructPix2PixPipeline"),ne(Cl.src,Oc="https://huggingface.co/datasets/diffusers/docs-images/resolve/main/evaluation_diffusion_models/edit-instruction.png")||f(Cl,"src",Oc),f(Cl,"alt","edit-instruction"),f(Et,"href","https://huggingface.co/docs/transformers/model_doc/clip"),f(Et,"rel","nofollow"),f(Wt,"href","https://arxiv.org/abs/2108.00946"),f(Wt,"rel","nofollow"),ne(Vl.src,em="https://huggingface.co/datasets/diffusers/docs-images/resolve/main/evaluation_diffusion_models/edit-consistency.png")||f(Vl,"src",em),f(Vl,"alt","edit-consistency"),ne($l.src,tm="https://huggingface.co/datasets/diffusers/docs-images/resolve/main/evaluation_diffusion_models/edit-dataset.png")||f($l,"src",tm),f($l,"alt","edit-dataset"),f(Yl,"href","/docs/diffusers/v0.17.1/en/api/pipelines/pix2pix#diffusers.StableDiffusionInstructPix2PixPipeline"),f(At,"href","https://huggingface.co/docs/diffusers/main/en/api/pipelines/stable_diffusion/pix2pix#diffusers.StableDiffusionInstructPix2PixPipeline.text_encoder"),f(At,"rel","nofollow"),f($t,"href","https://huggingface.co/docs/diffusers/main/en/api/pipelines/stable_diffusion/pix2pix_zero#diffusers.StableDiffusionPix2PixZeroPipeline"),f($t,"rel","nofollow"),f(Pt,"href","https://laion.ai/blog/laion-5b/"),f(Pt,"rel","nofollow"),f(Qt,"href","https://huggingface.co/docs/diffusers/main/en/api/pipelines/stable_diffusion/overview"),f(Qt,"rel","nofollow"),f(Re,"id","classconditioned-image-generation"),f(Re,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Re,"href","#classconditioned-image-generation"),f(oe,"class","relative group"),f(Ht,"href","https://huggingface.co/datasets/imagenet-1k"),f(Ht,"rel","nofollow"),f(Dt,"href","https://arxiv.org/abs/1706.08500"),f(Dt,"rel","nofollow"),f(qt,"href","https://huggingface.co/docs/diffusers/api/pipelines/dit"),f(qt,"rel","nofollow"),f(Lt,"href","https://arxiv.org/abs/2212.09748"),f(Lt,"rel","nofollow"),f(Kt,"href","https://mmgeneration.readthedocs.io/en/latest/quick_run.html#fid"),f(Kt,"rel","nofollow"),ne(es.src,lm="https://huggingface.co/datasets/diffusers/docs-images/resolve/main/evaluation_diffusion_models/real-images.png")||f(es,"src",lm),f(es,"alt","real-images"),f(q,"align","center"),f(ll,"href","https://huggingface.co/docs/diffusers/api/pipelines/dit"),f(ll,"rel","nofollow"),f(al,"href","https://torchmetrics.readthedocs.io/"),f(al,"rel","nofollow"),ne(as.src,sm="https://huggingface.co/datasets/diffusers/docs-images/resolve/main/evaluation_diffusion_models/fake-images.png")||f(as,"src",sm),f(as,"alt","fake-images"),f(L,"align","center")},m(e,s){l(document.head,h),p(e,v,s),p(e,u,s),l(u,j),l(j,_),w(I,_,null),l(u,g),l(u,Z),l(Z,ze),p(e,re,s),p(e,E,s),l(E,S),p(e,R,s),p(e,k,s),l(k,pe),l(k,W),l(W,De),l(k,b),p(e,B,s),p(e,C,s),l(C,rl),p(e,qe,s),p(e,F,s),l(F,pl),l(F,ce),l(ce,tn),l(F,ln),p(e,Ua,s),p(e,me,s),l(me,sn),l(me,Le),l(Le,an),l(me,on),p(e,ga,s),p(e,O,s),l(O,fe),l(fe,us),w(Ke,us,null),l(O,nn),l(O,ds),l(ds,rn),p(e,va,s),p(e,cl,s),l(cl,pn),p(e,_a,s),p(e,x,s),l(x,Oe),l(Oe,cn),l(Oe,et),l(et,Ms),l(Ms,mn),l(Oe,fn),l(x,hn),l(x,ee),l(ee,un),l(ee,tt),l(tt,ws),l(ws,dn),l(ee,Mn),l(ee,lt),l(lt,ys),l(ys,wn),l(ee,yn),l(x,Jn),l(x,st),l(st,Tn),l(st,at),l(at,Js),l(Js,jn),l(st,bn),p(e,Za,s),p(e,te,s),l(te,he),l(he,Ts),w(it,Ts,null),l(te,In),l(te,js),l(js,Un),p(e,Ba,s),p(e,$,s),l($,gn),l($,ot),l(ot,vn),l($,_n),l($,nt),l(nt,Zn),l($,Bn),p(e,Ea,s),p(e,ue,s),l(ue,En),l(ue,rt),l(rt,Wn),l(ue,Gn),p(e,Wa,s),p(e,ml,s),l(ml,bs),l(bs,Cn),p(e,Ga,s),p(e,fl,s),l(fl,hl),p(e,Ca,s),p(e,ul,s),l(ul,kn),p(e,ka,s),p(e,P,s),l(P,Is),l(Is,Xn),l(P,Vn),l(P,Us),l(Us,Nn),l(P,Rn),l(P,gs),l(gs,Fn),p(e,Xa,s),p(e,dl,s),l(dl,An),p(e,Va,s),p(e,de,s),l(de,Sn),l(de,vs),l(vs,xn),l(de,$n),p(e,Na,s),p(e,Me,s),l(Me,Ml),l(Ml,pt),l(pt,Pn),l(Ml,Yn),l(Me,Qn),l(Me,wl),l(wl,ct),l(ct,zn),l(wl,Hn),p(e,Ra,s),p(e,we,s),l(we,Dn),l(we,_s),l(_s,qn),l(we,Ln),p(e,Fa,s),p(e,ye,s),l(ye,Kn),l(ye,mt),l(mt,On),l(ye,er),p(e,Aa,s),w(ft,e,s),p(e,Sa,s),p(e,Je,s),l(Je,tr),l(Je,ht),l(ht,lr),l(Je,sr),p(e,xa,s),w(ut,e,s),p(e,$a,s),p(e,yl,s),l(yl,Jl),p(e,Pa,s),p(e,Y,s),l(Y,ar),l(Y,Zs),l(Zs,ir),l(Y,or),l(Y,dt),l(dt,nr),l(Y,rr),p(e,Ya,s),p(e,Tl,s),l(Tl,jl),p(e,Qa,s),p(e,bl,s),l(bl,pr),p(e,za,s),w(Te,e,s),p(e,Ha,s),p(e,le,s),l(le,je),l(je,Bs),w(Mt,Bs,null),l(le,cr),l(le,Es),l(Es,mr),p(e,Da,s),p(e,Il,s),l(Il,fr),p(e,qa,s),p(e,Q,s),l(Q,Ws),l(Ws,hr),l(Q,ur),l(Q,Gs),l(Gs,dr),l(Q,Mr),l(Q,Cs),l(Cs,wr),p(e,La,s),p(e,se,s),l(se,be),l(be,ks),w(wt,ks,null),l(se,yr),l(se,Xs),l(Xs,Jr),p(e,Ka,s),p(e,yt,s),l(yt,Jt),l(Jt,Tr),l(yt,jr),p(e,Oa,s),p(e,Ie,s),l(Ie,br),l(Ie,Ul),l(Ul,Ir),l(Ie,Ur),p(e,ei,s),w(Tt,e,s),p(e,ti,s),p(e,gl,s),l(gl,gr),p(e,li,s),w(jt,e,s),p(e,si,s),p(e,vl,s),l(vl,vr),p(e,ai,s),w(bt,e,s),p(e,ii,s),p(e,_l,s),l(_l,_r),p(e,oi,s),p(e,z,s),l(z,Zr),l(z,Zl),l(Zl,Br),l(z,Er),l(z,It),l(It,Wr),l(z,Gr),p(e,ni,s),w(Ut,e,s),p(e,ri,s),p(e,Ue,s),l(Ue,Cr),l(Ue,gt),l(gt,kr),l(Ue,Xr),p(e,pi,s),w(vt,e,s),p(e,ci,s),p(e,Bl,s),l(Bl,Vr),p(e,mi,s),w(_t,e,s),p(e,fi,s),p(e,ge,s),l(ge,Nr),l(ge,Zt),l(Zt,Rr),l(ge,Fr),p(e,hi,s),w(ve,e,s),p(e,ui,s),p(e,ae,s),l(ae,_e),l(_e,Vs),w(Bt,Vs,null),l(ae,Ar),l(ae,Ns),l(Ns,Sr),p(e,di,s),p(e,Ze,s),l(Ze,xr),l(Ze,El),l(El,$r),l(Ze,Pr),p(e,Mi,s),p(e,Wl,s),l(Wl,Yr),p(e,wi,s),p(e,Gl,s),l(Gl,Cl),p(e,yi,s),p(e,X,s),l(X,Qr),l(X,Et),l(Et,zr),l(X,Hr),l(X,Wt),l(Wt,Dr),l(X,qr),l(X,Rs),l(Rs,Lr),l(X,Kr),p(e,Ji,s),p(e,Be,s),l(Be,Fs),l(Fs,Or),l(Be,ep),l(Be,As),l(As,tp),p(e,Ti,s),p(e,kl,s),l(kl,lp),p(e,ji,s),p(e,Xl,s),l(Xl,Vl),p(e,bi,s),p(e,Nl,s),l(Nl,sp),p(e,Ii,s),w(Gt,e,s),p(e,Ui,s),w(Ct,e,s),p(e,gi,s),p(e,Rl,s),l(Rl,ap),p(e,vi,s),p(e,H,s),l(H,Ee),l(Ee,Ss),l(Ss,ip),l(Ee,op),l(Ee,xs),l(xs,np),l(Ee,rp),l(H,pp),l(H,Fl),l(Fl,$s),l($s,cp),l(Fl,mp),l(H,fp),l(H,We),l(We,Ps),l(Ps,hp),l(We,up),l(We,Ys),l(Ys,dp),l(We,Mp),p(e,_i,s),p(e,Al,s),l(Al,wp),p(e,Zi,s),w(kt,e,s),p(e,Bi,s),w(Xt,e,s),p(e,Ei,s),p(e,Sl,s),l(Sl,yp),p(e,Wi,s),w(Vt,e,s),p(e,Gi,s),p(e,xl,s),l(xl,$l),p(e,Ci,s),p(e,Pl,s),l(Pl,Jp),p(e,ki,s),p(e,Ge,s),l(Ge,Tp),l(Ge,Yl),l(Yl,jp),l(Ge,bp),p(e,Xi,s),w(Nt,e,s),p(e,Vi,s),p(e,Ql,s),l(Ql,Ip),p(e,Ni,s),w(Rt,e,s),p(e,Ri,s),p(e,zl,s),l(zl,Up),p(e,Fi,s),w(Ft,e,s),p(e,Ai,s),p(e,D,s),l(D,gp),l(D,Qs),l(Qs,vp),l(D,_p),l(D,At),l(At,Zp),l(D,Bp),p(e,Si,s),p(e,Ce,s),l(Ce,Ep),l(Ce,zs),l(zs,Wp),l(Ce,Gp),p(e,xi,s),w(St,e,s),p(e,$i,s),p(e,ke,s),l(ke,Cp),l(ke,Hs),l(Hs,kp),l(ke,Xp),p(e,Pi,s),w(xt,e,s),p(e,Yi,s),p(e,Hl,s),l(Hl,Vp),p(e,Qi,s),p(e,V,s),l(V,Np),l(V,Ds),l(Ds,Rp),l(V,Fp),l(V,qs),l(qs,Ap),l(V,Sp),l(V,Ls),l(Ls,xp),l(V,$p),p(e,zi,s),p(e,Xe,s),l(Xe,Pp),l(Xe,Ks),l(Ks,Yp),l(Xe,Qp),p(e,Hi,s),p(e,Ve,s),l(Ve,zp),l(Ve,$t),l($t,Os),l(Os,Hp),l(Ve,Dp),p(e,Di,s),w(Ne,e,s),p(e,qi,s),p(e,ie,s),l(ie,ea),l(ea,ta),l(ta,qp),l(ie,Lp),l(ie,Pt),l(Pt,Kp),l(ie,Op),p(e,Li,s),p(e,Dl,s),l(Dl,la),l(la,Yt),l(Yt,ec),l(Yt,Qt),l(Qt,tc),l(Yt,lc),p(e,Ki,s),p(e,oe,s),l(oe,Re),l(Re,sa),w(zt,sa,null),l(oe,sc),l(oe,aa),l(aa,ac),p(e,Oi,s),p(e,G,s),l(G,ic),l(G,Ht),l(Ht,oc),l(G,nc),l(G,Dt),l(Dt,rc),l(G,pc),l(G,qt),l(qt,ia),l(ia,cc),l(G,mc),l(G,Lt),l(Lt,fc),l(G,hc),p(e,eo,s),p(e,Fe,s),l(Fe,uc),l(Fe,Kt),l(Kt,dc),l(Fe,Mc),p(e,to,s),p(e,ql,s),l(ql,oa),l(oa,wc),p(e,lo,s),p(e,Ll,s),l(Ll,yc),p(e,so,s),p(e,Kl,s),l(Kl,Jc),p(e,ao,s),w(Ot,e,s),p(e,io,s),w(el,e,s),p(e,oo,s),p(e,Ol,s),l(Ol,Tc),p(e,no,s),p(e,q,s),l(q,es),l(q,jc),l(q,bc),l(q,na),l(na,Ic),p(e,ro,s),p(e,ts,s),l(ts,Uc),p(e,po,s),w(tl,e,s),p(e,co,s),p(e,Ae,s),l(Ae,gc),l(Ae,ll),l(ll,ra),l(ra,vc),l(Ae,_c),p(e,mo,s),w(sl,e,s),p(e,fo,s),p(e,Se,s),l(Se,Zc),l(Se,al),l(al,pa),l(pa,Bc),l(Se,Ec),p(e,ho,s),w(il,e,s),p(e,uo,s),p(e,ls,s),l(ls,Wc),p(e,Mo,s),p(e,N,s),l(N,ca),l(ca,Gc),l(N,Cc),l(N,ma),l(ma,kc),l(N,Xc),l(N,fa),l(fa,Vc),l(N,Nc),l(N,ha),l(ha,Rc),p(e,wo,s),p(e,ss,s),l(ss,Fc),p(e,yo,s),w(xe,e,s),p(e,Jo,s),p(e,$e,s),l($e,Ac),l($e,ua),l(ua,Sc),l($e,xc),p(e,To,s),p(e,L,s),l(L,as),l(L,$c),l(L,Pc),l(L,da),l(da,Yc),jo=!0},p(e,[s]){const ol={};s&2&&(ol.$$scope={dirty:s,ctx:e}),Te.$set(ol);const Ma={};s&2&&(Ma.$$scope={dirty:s,ctx:e}),ve.$set(Ma);const wa={};s&2&&(wa.$$scope={dirty:s,ctx:e}),Ne.$set(wa);const ya={};s&2&&(ya.$$scope={dirty:s,ctx:e}),xe.$set(ya)},i(e){jo||(y(I.$$.fragment,e),y(Ke.$$.fragment,e),y(it.$$.fragment,e),y(ft.$$.fragment,e),y(ut.$$.fragment,e),y(Te.$$.fragment,e),y(Mt.$$.fragment,e),y(wt.$$.fragment,e),y(Tt.$$.fragment,e),y(jt.$$.fragment,e),y(bt.$$.fragment,e),y(Ut.$$.fragment,e),y(vt.$$.fragment,e),y(_t.$$.fragment,e),y(ve.$$.fragment,e),y(Bt.$$.fragment,e),y(Gt.$$.fragment,e),y(Ct.$$.fragment,e),y(kt.$$.fragment,e),y(Xt.$$.fragment,e),y(Vt.$$.fragment,e),y(Nt.$$.fragment,e),y(Rt.$$.fragment,e),y(Ft.$$.fragment,e),y(St.$$.fragment,e),y(xt.$$.fragment,e),y(Ne.$$.fragment,e),y(zt.$$.fragment,e),y(Ot.$$.fragment,e),y(el.$$.fragment,e),y(tl.$$.fragment,e),y(sl.$$.fragment,e),y(il.$$.fragment,e),y(xe.$$.fragment,e),jo=!0)},o(e){J(I.$$.fragment,e),J(Ke.$$.fragment,e),J(it.$$.fragment,e),J(ft.$$.fragment,e),J(ut.$$.fragment,e),J(Te.$$.fragment,e),J(Mt.$$.fragment,e),J(wt.$$.fragment,e),J(Tt.$$.fragment,e),J(jt.$$.fragment,e),J(bt.$$.fragment,e),J(Ut.$$.fragment,e),J(vt.$$.fragment,e),J(_t.$$.fragment,e),J(ve.$$.fragment,e),J(Bt.$$.fragment,e),J(Gt.$$.fragment,e),J(Ct.$$.fragment,e),J(kt.$$.fragment,e),J(Xt.$$.fragment,e),J(Vt.$$.fragment,e),J(Nt.$$.fragment,e),J(Rt.$$.fragment,e),J(Ft.$$.fragment,e),J(St.$$.fragment,e),J(xt.$$.fragment,e),J(Ne.$$.fragment,e),J(zt.$$.fragment,e),J(Ot.$$.fragment,e),J(el.$$.fragment,e),J(tl.$$.fragment,e),J(sl.$$.fragment,e),J(il.$$.fragment,e),J(xe.$$.fragment,e),jo=!1},d(e){t(h),e&&t(v),e&&t(u),T(I),e&&t(re),e&&t(E),e&&t(R),e&&t(k),e&&t(B),e&&t(C),e&&t(qe),e&&t(F),e&&t(Ua),e&&t(me),e&&t(ga),e&&t(O),T(Ke),e&&t(va),e&&t(cl),e&&t(_a),e&&t(x),e&&t(Za),e&&t(te),T(it),e&&t(Ba),e&&t($),e&&t(Ea),e&&t(ue),e&&t(Wa),e&&t(ml),e&&t(Ga),e&&t(fl),e&&t(Ca),e&&t(ul),e&&t(ka),e&&t(P),e&&t(Xa),e&&t(dl),e&&t(Va),e&&t(de),e&&t(Na),e&&t(Me),e&&t(Ra),e&&t(we),e&&t(Fa),e&&t(ye),e&&t(Aa),T(ft,e),e&&t(Sa),e&&t(Je),e&&t(xa),T(ut,e),e&&t($a),e&&t(yl),e&&t(Pa),e&&t(Y),e&&t(Ya),e&&t(Tl),e&&t(Qa),e&&t(bl),e&&t(za),T(Te,e),e&&t(Ha),e&&t(le),T(Mt),e&&t(Da),e&&t(Il),e&&t(qa),e&&t(Q),e&&t(La),e&&t(se),T(wt),e&&t(Ka),e&&t(yt),e&&t(Oa),e&&t(Ie),e&&t(ei),T(Tt,e),e&&t(ti),e&&t(gl),e&&t(li),T(jt,e),e&&t(si),e&&t(vl),e&&t(ai),T(bt,e),e&&t(ii),e&&t(_l),e&&t(oi),e&&t(z),e&&t(ni),T(Ut,e),e&&t(ri),e&&t(Ue),e&&t(pi),T(vt,e),e&&t(ci),e&&t(Bl),e&&t(mi),T(_t,e),e&&t(fi),e&&t(ge),e&&t(hi),T(ve,e),e&&t(ui),e&&t(ae),T(Bt),e&&t(di),e&&t(Ze),e&&t(Mi),e&&t(Wl),e&&t(wi),e&&t(Gl),e&&t(yi),e&&t(X),e&&t(Ji),e&&t(Be),e&&t(Ti),e&&t(kl),e&&t(ji),e&&t(Xl),e&&t(bi),e&&t(Nl),e&&t(Ii),T(Gt,e),e&&t(Ui),T(Ct,e),e&&t(gi),e&&t(Rl),e&&t(vi),e&&t(H),e&&t(_i),e&&t(Al),e&&t(Zi),T(kt,e),e&&t(Bi),T(Xt,e),e&&t(Ei),e&&t(Sl),e&&t(Wi),T(Vt,e),e&&t(Gi),e&&t(xl),e&&t(Ci),e&&t(Pl),e&&t(ki),e&&t(Ge),e&&t(Xi),T(Nt,e),e&&t(Vi),e&&t(Ql),e&&t(Ni),T(Rt,e),e&&t(Ri),e&&t(zl),e&&t(Fi),T(Ft,e),e&&t(Ai),e&&t(D),e&&t(Si),e&&t(Ce),e&&t(xi),T(St,e),e&&t($i),e&&t(ke),e&&t(Pi),T(xt,e),e&&t(Yi),e&&t(Hl),e&&t(Qi),e&&t(V),e&&t(zi),e&&t(Xe),e&&t(Hi),e&&t(Ve),e&&t(Di),T(Ne,e),e&&t(qi),e&&t(ie),e&&t(Li),e&&t(Dl),e&&t(Ki),e&&t(oe),T(zt),e&&t(Oi),e&&t(G),e&&t(eo),e&&t(Fe),e&&t(to),e&&t(ql),e&&t(lo),e&&t(Ll),e&&t(so),e&&t(Kl),e&&t(ao),T(Ot,e),e&&t(io),T(el,e),e&&t(oo),e&&t(Ol),e&&t(no),e&&t(q),e&&t(ro),e&&t(ts),e&&t(po),T(tl,e),e&&t(co),e&&t(Ae),e&&t(mo),T(sl,e),e&&t(fo),e&&t(Se),e&&t(ho),T(il,e),e&&t(uo),e&&t(ls),e&&t(Mo),e&&t(N),e&&t(wo),e&&t(ss),e&&t(yo),T(xe,e),e&&t(Jo),e&&t($e),e&&t(To),e&&t(L)}}}const Sh={local:"evaluating-diffusion-models",sections:[{local:"scenarios",title:"Scenarios"},{local:"qualitative-evaluation",title:"Qualitative Evaluation"},{local:"quantitative-evaluation",sections:[{local:"textguided-image-generation",title:"Text-guided image generation"},{local:"imageconditioned-texttoimage-generation",title:"Image-conditioned text-to-image generation"},{local:"classconditioned-image-generation",title:"Class-conditioned image generation"}],title:"Quantitative Evaluation"}],title:"Evaluating Diffusion Models"};function xh(A){return Xh(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class zh extends Wh{constructor(h){super();Gh(this,h,xh,Ah,Ch,{})}}export{zh as default,Sh as metadata};
