import{s as Ee,n as Ge,o as Be}from"../chunks/scheduler.7bc62968.js";import{S as ke,i as Ne,g as l,s as a,r as p,A as Qe,h as o,f as i,c as s,j as ze,u as m,x as r,k as He,y as We,a as n,v as c,d,t as f,w as g}from"../chunks/index.2f8492b0.js";import{C as Ve}from"../chunks/CodeBlock.4e96d6f7.js";import{H as h,E as Ze}from"../chunks/EditOnGithub.2a9ce03a.js";function Re(we){let u,R,V,q,y,S,w,Y,b,be="CLIP is a neural network adept at grasping visual concepts through natural language supervision. It operates by concurrently training a text encoder and an image encoder, focusing on a pretraining task that involves matching captions with corresponding images. This architecture allows CLIP to adapt to a variety of visual classification benchmarks seamlessly. It does so by simply receiving the names of the visual categories to be recognized, demonstrating “zero-shot” learning capabilities akin to those observed in GPT-2 and GPT-3 models.",X,$,F,v,$e="Given a batch of image-text pairs, CLIP computes the dense cosine similarity matrix between all possible (image, text) candidates within this batch. The core idea is to maximize the similarity between the correct pairs (shown in blue in the figure below) and minimize the similarity for incorrect pairs (shown in grey in the image). To do it, they optimize a symmetric cross-entropy loss over these similarity scores.",A,M,ve='<img src="https://images.openai.com/blob/fbc4f633-9ad4-4dc2-bd94-0b6f1feee22f/overview-a.svg" alt="CLIP contrastive pre-training"/> <em>Image taken from OpenAI</em>',D,C,Me="Explaining in simple terms, we want to make the similarity between the image and its corresponding caption as high as we can, while the similarity between the image and the other captions should be small. We apply this logic to the caption too, so we want to maximize the similarity of the caption with its corresponding image, and minimize between all other images.",O,T,K,x,Ce="CLIP’s design features independent encoders for images and text, allowing flexibility in their choice. Users can switch the standard image encoder, like a Vision Transformer, for alternatives like ResNet, or opt for different text encoders, enhancing adaptability and experimentation. Of course, if you switch one of the encoders, you will need to train your model again, as your embedding distribution will be different.",ee,U,te,I,Te="CLIP, can be leveraged for a variety of applications. Here are some notable use cases:",ie,J,xe="<li>Zero-shot image classification;</li> <li>Similarity search;</li> <li>Diffusion models conditioning.</li>",ne,L,ae,P,Ue="For practical applications, one typically uses an image, and pre-defined classes as input. The provided Python example demonstrates how to use the transformers library for running CLIP. In this example, we want to zero-shot classify the image below between <code>dog</code> or <code>cat</code>.",se,_,Ie='<img src="http://images.cocodataset.org/val2017/000000039769.jpg" alt="A photo of cats"/>',le,j,oe,z,Je="After executing this code, we got the following probabilities:",re,H,Le="<li>“a photo of a cat”: 99.49%</li> <li>“a photo of a dog”: 0.51%</li>",pe,E,me,G,Pe=`Despite CLIP’s proficiency in zero-shot classification, it is unlikely to outperform a specialized, fine-tuned model. Moreover, its generalization capabilities are somewhat limited, particularly in scenarios involving data or examples not encountered during training.
The paper also shows that CLIP’s effectiveness and biases are impacted by the choice of categories, as demonstrated in tests using the Fairface dataset. Notable disparities were found in gender and racial classifications, with gender accuracy over 96% and racial accuracy around 93%.`,ce,B,de,k,_e="In conclusion, the CLIP model from OpenAI has revolutionized the multimodal field. What sets CLIP apart is its proficiency in zero-shot learning, allowing it to classify images into categories it wasn’t explicitly trained on. This remarkable ability to generalize comes from its innovative training method, where it learns to match images with text captions.",fe,N,ge,Q,je='<li><a href="https://arxiv.org/abs/2103.00020" rel="nofollow">CLIP paper</a></li> <li><a href="https://lilianweng.github.io/posts/2021-05-31-contrastive/#clip" rel="nofollow">CLIP by Lilian Weng</a></li>',ue,W,he,Z,ye;return y=new h({props:{title:"Contrastive Language-Image Pre-training (CLIP)",local:"contrastive-language-image-pre-training-clip",headingTag:"h1"}}),w=new h({props:{title:"Introduction",local:"introduction",headingTag:"h2"}}),$=new h({props:{title:"Contrastive pre-training",local:"contrastive-pre-training",headingTag:"h2"}}),T=new h({props:{title:"Text Encoder and Image Encoder",local:"text-encoder-and-image-encoder",headingTag:"h2"}}),U=new h({props:{title:"Use cases",local:"use-cases",headingTag:"h2"}}),L=new h({props:{title:"Usage",local:"usage",headingTag:"h2"}}),j=new Ve({props:{code:"ZnJvbSUyMFBJTCUyMGltcG9ydCUyMEltYWdlJTBBaW1wb3J0JTIwcmVxdWVzdHMlMEElMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQ0xJUFByb2Nlc3NvciUyQyUyMENMSVBNb2RlbCUwQSUwQW1vZGVsJTIwJTNEJTIwQ0xJUE1vZGVsLmZyb21fcHJldHJhaW5lZCglMjJvcGVuYWklMkZjbGlwLXZpdC1iYXNlLXBhdGNoMzIlMjIpJTBBcHJvY2Vzc29yJTIwJTNEJTIwQ0xJUFByb2Nlc3Nvci5mcm9tX3ByZXRyYWluZWQoJTIyb3BlbmFpJTJGY2xpcC12aXQtYmFzZS1wYXRjaDMyJTIyKSUwQSUwQXVybCUyMCUzRCUyMCUyMmh0dHAlM0ElMkYlMkZpbWFnZXMuY29jb2RhdGFzZXQub3JnJTJGdmFsMjAxNyUyRjAwMDAwMDAzOTc2OS5qcGclMjIlMEFpbWFnZSUyMCUzRCUyMEltYWdlLm9wZW4ocmVxdWVzdHMuZ2V0KHVybCUyQyUyMHN0cmVhbSUzRFRydWUpLnJhdyklMEElMEFpbnB1dHMlMjAlM0QlMjBwcm9jZXNzb3IoJTBBJTIwJTIwJTIwJTIwdGV4dCUzRCU1QiUyMmElMjBwaG90byUyMG9mJTIwYSUyMGNhdCUyMiUyQyUyMCUyMmElMjBwaG90byUyMG9mJTIwYSUyMGRvZyUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMGltYWdlcyUzRGltYWdlJTJDJTBBJTIwJTIwJTIwJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiUyQyUwQSUyMCUyMCUyMCUyMHBhZGRpbmclM0RUcnVlJTJDJTBBKSUwQSUwQW91dHB1dHMlMjAlM0QlMjBtb2RlbCgqKmlucHV0cyklMEFsb2dpdHNfcGVyX2ltYWdlJTIwJTNEJTIwb3V0cHV0cy5sb2dpdHNfcGVyX2ltYWdlJTBBcHJvYnMlMjAlM0QlMjBsb2dpdHNfcGVyX2ltYWdlLnNvZnRtYXgoZGltJTNEMSk=",highlighted:`<span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image
<span class="hljs-keyword">import</span> requests

<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> CLIPProcessor, CLIPModel

model = CLIPModel.from_pretrained(<span class="hljs-string">&quot;openai/clip-vit-base-patch32&quot;</span>)
processor = CLIPProcessor.from_pretrained(<span class="hljs-string">&quot;openai/clip-vit-base-patch32&quot;</span>)

url = <span class="hljs-string">&quot;http://images.cocodataset.org/val2017/000000039769.jpg&quot;</span>
image = Image.<span class="hljs-built_in">open</span>(requests.get(url, stream=<span class="hljs-literal">True</span>).raw)

inputs = processor(
    text=[<span class="hljs-string">&quot;a photo of a cat&quot;</span>, <span class="hljs-string">&quot;a photo of a dog&quot;</span>],
    images=image,
    return_tensors=<span class="hljs-string">&quot;pt&quot;</span>,
    padding=<span class="hljs-literal">True</span>,
)

outputs = model(**inputs)
logits_per_image = outputs.logits_per_image
probs = logits_per_image.softmax(dim=<span class="hljs-number">1</span>)`,wrap:!1}}),E=new h({props:{title:"Limitations",local:"limitations",headingTag:"h2"}}),B=new h({props:{title:"Conclusion",local:"conclusion",headingTag:"h2"}}),N=new h({props:{title:"References",local:"references",headingTag:"h2"}}),W=new Ze({props:{source:"https://github.com/huggingface/computer-vision-course/blob/main/chapters/en/unit4/multimodal-models/clip-and-relatives/clip.mdx"}}),{c(){u=l("meta"),R=a(),V=l("p"),q=a(),p(y.$$.fragment),S=a(),p(w.$$.fragment),Y=a(),b=l("p"),b.textContent=be,X=a(),p($.$$.fragment),F=a(),v=l("p"),v.textContent=$e,A=a(),M=l("p"),M.innerHTML=ve,D=a(),C=l("p"),C.textContent=Me,O=a(),p(T.$$.fragment),K=a(),x=l("p"),x.textContent=Ce,ee=a(),p(U.$$.fragment),te=a(),I=l("p"),I.textContent=Te,ie=a(),J=l("ul"),J.innerHTML=xe,ne=a(),p(L.$$.fragment),ae=a(),P=l("p"),P.innerHTML=Ue,se=a(),_=l("p"),_.innerHTML=Ie,le=a(),p(j.$$.fragment),oe=a(),z=l("p"),z.textContent=Je,re=a(),H=l("ul"),H.innerHTML=Le,pe=a(),p(E.$$.fragment),me=a(),G=l("p"),G.textContent=Pe,ce=a(),p(B.$$.fragment),de=a(),k=l("p"),k.textContent=_e,fe=a(),p(N.$$.fragment),ge=a(),Q=l("ul"),Q.innerHTML=je,ue=a(),p(W.$$.fragment),he=a(),Z=l("p"),this.h()},l(e){const t=Qe("svelte-u9bgzb",document.head);u=o(t,"META",{name:!0,content:!0}),t.forEach(i),R=s(e),V=o(e,"P",{}),ze(V).forEach(i),q=s(e),m(y.$$.fragment,e),S=s(e),m(w.$$.fragment,e),Y=s(e),b=o(e,"P",{"data-svelte-h":!0}),r(b)!=="svelte-ehzu06"&&(b.textContent=be),X=s(e),m($.$$.fragment,e),F=s(e),v=o(e,"P",{"data-svelte-h":!0}),r(v)!=="svelte-1k56xzf"&&(v.textContent=$e),A=s(e),M=o(e,"P",{"data-svelte-h":!0}),r(M)!=="svelte-1f2xlz1"&&(M.innerHTML=ve),D=s(e),C=o(e,"P",{"data-svelte-h":!0}),r(C)!=="svelte-hxbljq"&&(C.textContent=Me),O=s(e),m(T.$$.fragment,e),K=s(e),x=o(e,"P",{"data-svelte-h":!0}),r(x)!=="svelte-1nn5dxn"&&(x.textContent=Ce),ee=s(e),m(U.$$.fragment,e),te=s(e),I=o(e,"P",{"data-svelte-h":!0}),r(I)!=="svelte-qs9t2n"&&(I.textContent=Te),ie=s(e),J=o(e,"UL",{"data-svelte-h":!0}),r(J)!=="svelte-d3etr2"&&(J.innerHTML=xe),ne=s(e),m(L.$$.fragment,e),ae=s(e),P=o(e,"P",{"data-svelte-h":!0}),r(P)!=="svelte-147dtgw"&&(P.innerHTML=Ue),se=s(e),_=o(e,"P",{"data-svelte-h":!0}),r(_)!=="svelte-xhmj4j"&&(_.innerHTML=Ie),le=s(e),m(j.$$.fragment,e),oe=s(e),z=o(e,"P",{"data-svelte-h":!0}),r(z)!=="svelte-16udf98"&&(z.textContent=Je),re=s(e),H=o(e,"UL",{"data-svelte-h":!0}),r(H)!=="svelte-cuwovx"&&(H.innerHTML=Le),pe=s(e),m(E.$$.fragment,e),me=s(e),G=o(e,"P",{"data-svelte-h":!0}),r(G)!=="svelte-1z1bz9"&&(G.textContent=Pe),ce=s(e),m(B.$$.fragment,e),de=s(e),k=o(e,"P",{"data-svelte-h":!0}),r(k)!=="svelte-1taci8q"&&(k.textContent=_e),fe=s(e),m(N.$$.fragment,e),ge=s(e),Q=o(e,"UL",{"data-svelte-h":!0}),r(Q)!=="svelte-18pv8si"&&(Q.innerHTML=je),ue=s(e),m(W.$$.fragment,e),he=s(e),Z=o(e,"P",{}),ze(Z).forEach(i),this.h()},h(){He(u,"name","hf:doc:metadata"),He(u,"content",qe)},m(e,t){We(document.head,u),n(e,R,t),n(e,V,t),n(e,q,t),c(y,e,t),n(e,S,t),c(w,e,t),n(e,Y,t),n(e,b,t),n(e,X,t),c($,e,t),n(e,F,t),n(e,v,t),n(e,A,t),n(e,M,t),n(e,D,t),n(e,C,t),n(e,O,t),c(T,e,t),n(e,K,t),n(e,x,t),n(e,ee,t),c(U,e,t),n(e,te,t),n(e,I,t),n(e,ie,t),n(e,J,t),n(e,ne,t),c(L,e,t),n(e,ae,t),n(e,P,t),n(e,se,t),n(e,_,t),n(e,le,t),c(j,e,t),n(e,oe,t),n(e,z,t),n(e,re,t),n(e,H,t),n(e,pe,t),c(E,e,t),n(e,me,t),n(e,G,t),n(e,ce,t),c(B,e,t),n(e,de,t),n(e,k,t),n(e,fe,t),c(N,e,t),n(e,ge,t),n(e,Q,t),n(e,ue,t),c(W,e,t),n(e,he,t),n(e,Z,t),ye=!0},p:Ge,i(e){ye||(d(y.$$.fragment,e),d(w.$$.fragment,e),d($.$$.fragment,e),d(T.$$.fragment,e),d(U.$$.fragment,e),d(L.$$.fragment,e),d(j.$$.fragment,e),d(E.$$.fragment,e),d(B.$$.fragment,e),d(N.$$.fragment,e),d(W.$$.fragment,e),ye=!0)},o(e){f(y.$$.fragment,e),f(w.$$.fragment,e),f($.$$.fragment,e),f(T.$$.fragment,e),f(U.$$.fragment,e),f(L.$$.fragment,e),f(j.$$.fragment,e),f(E.$$.fragment,e),f(B.$$.fragment,e),f(N.$$.fragment,e),f(W.$$.fragment,e),ye=!1},d(e){e&&(i(R),i(V),i(q),i(S),i(Y),i(b),i(X),i(F),i(v),i(A),i(M),i(D),i(C),i(O),i(K),i(x),i(ee),i(te),i(I),i(ie),i(J),i(ne),i(ae),i(P),i(se),i(_),i(le),i(oe),i(z),i(re),i(H),i(pe),i(me),i(G),i(ce),i(de),i(k),i(fe),i(ge),i(Q),i(ue),i(he),i(Z)),i(u),g(y,e),g(w,e),g($,e),g(T,e),g(U,e),g(L,e),g(j,e),g(E,e),g(B,e),g(N,e),g(W,e)}}}const qe='{"title":"Contrastive Language-Image Pre-training (CLIP)","local":"contrastive-language-image-pre-training-clip","sections":[{"title":"Introduction","local":"introduction","sections":[],"depth":2},{"title":"Contrastive pre-training","local":"contrastive-pre-training","sections":[],"depth":2},{"title":"Text Encoder and Image Encoder","local":"text-encoder-and-image-encoder","sections":[],"depth":2},{"title":"Use cases","local":"use-cases","sections":[],"depth":2},{"title":"Usage","local":"usage","sections":[],"depth":2},{"title":"Limitations","local":"limitations","sections":[],"depth":2},{"title":"Conclusion","local":"conclusion","sections":[],"depth":2},{"title":"References","local":"references","sections":[],"depth":2}],"depth":1}';function Se(we){return Be(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class De extends ke{constructor(u){super(),Ne(this,u,Se,Re,Ee,{})}}export{De as component};
