import{s as Ds,f as Os,o as Ks,n as st}from"../chunks/scheduler.25b97de1.js";import{S as eo,i as to,g as r,s as n,r as p,A as so,h as i,f as s,c as a,j as B,u as h,x as m,k as J,y as c,a as o,v as g,d as f,t as u,w as _}from"../chunks/index.d9030fc9.js";import{T as Ls}from"../chunks/Tip.baa67368.js";import{D as q}from"../chunks/Docstring.e257edda.js";import{C as A}from"../chunks/CodeBlock.e6cd0d95.js";import{E as fs}from"../chunks/ExampleCodeBlock.20db4b6e.js";import{H as F,E as oo}from"../chunks/EditOnGithub.91d95064.js";function no(C){let l,y="Example:",M,T,b;return T=new A({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMElkZWZpY3MyTW9kZWwlMkMlMjBJZGVmaWNzMkNvbmZpZyUwQSUyMyUyMEluaXRpYWxpemluZyUyMGNvbmZpZ3VyYXRpb24lMEFjb25maWd1cmF0aW9uJTIwJTNEJTIwSWRlZmljczJDb25maWcoKSUwQSUyMyUyMEluaXRpYWxpemluZyUyMGElMjBtb2RlbCUyMGZyb20lMjB0aGUlMjBjb25maWd1cmF0aW9uJTBBbW9kZWwlMjAlM0QlMjBJZGVmaWNzMk1vZGVsKGNvbmZpZ3VyYXRpb24pJTBBJTIzJTIwQWNjZXNzaW5nJTIwdGhlJTIwbW9kZWwlMjBjb25maWd1cmF0aW9uJTBBY29uZmlndXJhdGlvbiUyMCUzRCUyMG1vZGVsLmNvbmZpZw==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Idefics2Model, Idefics2Config
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = Idefics2Config()
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a model from the configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = Idefics2Model(configuration)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Accessing the model configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>configuration = model.config`,wrap:!1}}),{c(){l=r("p"),l.textContent=y,M=n(),p(T.$$.fragment)},l(d){l=i(d,"P",{"data-svelte-h":!0}),m(l)!=="svelte-11lpom8"&&(l.textContent=y),M=a(d),h(T.$$.fragment,d)},m(d,I){o(d,l,I),o(d,M,I),g(T,d,I),b=!0},p:st,i(d){b||(f(T.$$.fragment,d),b=!0)},o(d){u(T.$$.fragment,d),b=!1},d(d){d&&(s(l),s(M)),_(T,d)}}}function ao(C){let l,y=`Although the recipe for forward pass needs to be defined within this function, one should call the <code>Module</code>
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`;return{c(){l=r("p"),l.innerHTML=y},l(M){l=i(M,"P",{"data-svelte-h":!0}),m(l)!=="svelte-fincs2"&&(l.innerHTML=y)},m(M,T){o(M,l,T)},p:st,d(M){M&&s(l)}}}function ro(C){let l,y=`Although the recipe for forward pass needs to be defined within this function, one should call the <code>Module</code>
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`;return{c(){l=r("p"),l.innerHTML=y},l(M){l=i(M,"P",{"data-svelte-h":!0}),m(l)!=="svelte-fincs2"&&(l.innerHTML=y)},m(M,T){o(M,l,T)},p:st,d(M){M&&s(l)}}}function io(C){let l,y="Example:",M,T,b;return T=new A({props:{code:"aW1wb3J0JTIwcmVxdWVzdHMlMEFpbXBvcnQlMjB0b3JjaCUwQWZyb20lMjBQSUwlMjBpbXBvcnQlMjBJbWFnZSUwQWZyb20lMjBpbyUyMGltcG9ydCUyMEJ5dGVzSU8lMEElMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Byb2Nlc3NvciUyQyUyMEF1dG9Nb2RlbEZvclZpc2lvbjJTZXElMEFmcm9tJTIwdHJhbnNmb3JtZXJzLmltYWdlX3V0aWxzJTIwaW1wb3J0JTIwbG9hZF9pbWFnZSUwQSUwQSUyMyUyME5vdGUlMjB0aGF0JTIwcGFzc2luZyUyMHRoZSUyMGltYWdlJTIwdXJscyUyMChpbnN0ZWFkJTIwb2YlMjB0aGUlMjBhY3R1YWwlMjBwaWwlMjBpbWFnZXMpJTIwdG8lMjB0aGUlMjBwcm9jZXNzb3IlMjBpcyUyMGFsc28lMjBwb3NzaWJsZSUwQWltYWdlMSUyMCUzRCUyMGxvYWRfaW1hZ2UoJTIyaHR0cHMlM0ElMkYlMkZjZG4uYnJpdGFubmljYS5jb20lMkY2MSUyRjkzMDYxLTA1MC05OTE0N0RDRSUyRlN0YXR1ZS1vZi1MaWJlcnR5LUlzbGFuZC1OZXctWW9yay1CYXkuanBnJTIyKSUwQWltYWdlMiUyMCUzRCUyMGxvYWRfaW1hZ2UoJTIyaHR0cHMlM0ElMkYlMkZjZG4uYnJpdGFubmljYS5jb20lMkY1OSUyRjk0NDU5LTA1MC1EQkE0MjQ2NyUyRlNreWxpbmUtQ2hpY2Fnby5qcGclMjIpJTBBaW1hZ2UzJTIwJTNEJTIwbG9hZF9pbWFnZSglMjJodHRwcyUzQSUyRiUyRmNkbi5icml0YW5uaWNhLmNvbSUyRjY4JTJGMTcwODY4LTA1MC04RERFODI2MyUyRkdvbGRlbi1HYXRlLUJyaWRnZS1TYW4tRnJhbmNpc2NvLmpwZyUyMiklMEElMEFwcm9jZXNzb3IlMjAlM0QlMjBBdXRvUHJvY2Vzc29yLmZyb21fcHJldHJhaW5lZCglMjJIdWdnaW5nRmFjZU00JTJGaWRlZmljczItOGItYmFzZSUyMiklMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvclZpc2lvbjJTZXEuZnJvbV9wcmV0cmFpbmVkKCUyMkh1Z2dpbmdGYWNlTTQlMkZpZGVmaWNzMi04Yi1iYXNlJTIyJTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIpJTBBJTBBQkFEX1dPUkRTX0lEUyUyMCUzRCUyMHByb2Nlc3Nvci50b2tlbml6ZXIoJTVCJTIyJTNDaW1hZ2UlM0UlMjIlMkMlMjAlMjIlM0NmYWtlX3Rva2VuX2Fyb3VuZF9pbWFnZSUzRSUyMiU1RCUyQyUyMGFkZF9zcGVjaWFsX3Rva2VucyUzREZhbHNlKS5pbnB1dF9pZHMlMEFFT1NfV09SRFNfSURTJTIwJTNEJTIwJTVCcHJvY2Vzc29yLnRva2VuaXplci5lb3NfdG9rZW5faWQlNUQlMEElMEElMjMlMjBDcmVhdGUlMjBpbnB1dHMlMEFwcm9tcHRzJTIwJTNEJTIwJTVCJTBBJTIwJTIwJTIyJTNDaW1hZ2UlM0VJbiUyMHRoaXMlMjBpbWFnZSUyQyUyMHdlJTIwY2FuJTIwc2VlJTIwdGhlJTIwY2l0eSUyMG9mJTIwTmV3JTIwWW9yayUyQyUyMGFuZCUyMG1vcmUlMjBzcGVjaWZpY2FsbHklMjB0aGUlMjBTdGF0dWUlMjBvZiUyMExpYmVydHkuJTNDaW1hZ2UlM0VJbiUyMHRoaXMlMjBpbWFnZSUyQyUyMiUyQyUwQSUyMCUyMCUyMkluJTIwd2hpY2glMjBjaXR5JTIwaXMlMjB0aGF0JTIwYnJpZGdlJTIwbG9jYXRlZCUzRiUzQ2ltYWdlJTNFJTIyJTJDJTBBJTVEJTBBaW1hZ2VzJTIwJTNEJTIwJTVCJTVCaW1hZ2UxJTJDJTIwaW1hZ2UyJTVEJTJDJTIwJTVCaW1hZ2UzJTVEJTVEJTBBaW5wdXRzJTIwJTNEJTIwcHJvY2Vzc29yKHRleHQlM0Rwcm9tcHRzJTJDJTIwaW1hZ2VzJTNEaW1hZ2VzJTJDJTIwcGFkZGluZyUzRFRydWUlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS50byglMjJjdWRhJTIyKSUwQSUwQSUyMyUyMEdlbmVyYXRlJTBBZ2VuZXJhdGVkX2lkcyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqaW5wdXRzJTJDJTIwYmFkX3dvcmRzX2lkcyUzREJBRF9XT1JEU19JRFMlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDIwKSUwQWdlbmVyYXRlZF90ZXh0cyUyMCUzRCUyMHByb2Nlc3Nvci5iYXRjaF9kZWNvZGUoZ2VuZXJhdGVkX2lkcyUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSUwQSUwQXByaW50KGdlbmVyYXRlZF90ZXh0cyk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> requests
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> io <span class="hljs-keyword">import</span> BytesIO

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor, AutoModelForVision2Seq
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers.image_utils <span class="hljs-keyword">import</span> load_image

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Note that passing the image urls (instead of the actual pil images) to the processor is also possible</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>image1 = load_image(<span class="hljs-string">&quot;https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>image2 = load_image(<span class="hljs-string">&quot;https://cdn.britannica.com/59/94459-050-DBA42467/Skyline-Chicago.jpg&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>image3 = load_image(<span class="hljs-string">&quot;https://cdn.britannica.com/68/170868-050-8DDE8263/Golden-Gate-Bridge-San-Francisco.jpg&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>processor = AutoProcessor.from_pretrained(<span class="hljs-string">&quot;HuggingFaceM4/idefics2-8b-base&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForVision2Seq.from_pretrained(<span class="hljs-string">&quot;HuggingFaceM4/idefics2-8b-base&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>BAD_WORDS_IDS = processor.tokenizer([<span class="hljs-string">&quot;&lt;image&gt;&quot;</span>, <span class="hljs-string">&quot;&lt;fake_token_around_image&gt;&quot;</span>], add_special_tokens=<span class="hljs-literal">False</span>).input_ids
<span class="hljs-meta">&gt;&gt;&gt; </span>EOS_WORDS_IDS = [processor.tokenizer.eos_token_id]

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Create inputs</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>prompts = [
<span class="hljs-meta">... </span>  <span class="hljs-string">&quot;&lt;image&gt;In this image, we can see the city of New York, and more specifically the Statue of Liberty.&lt;image&gt;In this image,&quot;</span>,
<span class="hljs-meta">... </span>  <span class="hljs-string">&quot;In which city is that bridge located?&lt;image&gt;&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>images = [[image1, image2], [image3]]
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = processor(text=prompts, images=images, padding=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Generate</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>generated_ids = model.generate(**inputs, bad_words_ids=BAD_WORDS_IDS, max_new_tokens=<span class="hljs-number">20</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>generated_texts = processor.batch_decode(generated_ids, skip_special_tokens=<span class="hljs-literal">True</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(generated_texts)
[<span class="hljs-string">&#x27;In this image, we can see the city of New York, and more specifically the Statue of Liberty. In this image, we can see the city of New York, and more specifically the Statue of Liberty.\\n\\n&#x27;</span>, <span class="hljs-string">&#x27;In which city is that bridge located?\\n\\nThe bridge is located in the city of Pittsburgh, Pennsylvania.\\n\\n\\nThe bridge is&#x27;</span>]`,wrap:!1}}),{c(){l=r("p"),l.textContent=y,M=n(),p(T.$$.fragment)},l(d){l=i(d,"P",{"data-svelte-h":!0}),m(l)!=="svelte-11lpom8"&&(l.textContent=y),M=a(d),h(T.$$.fragment,d)},m(d,I){o(d,l,I),o(d,M,I),g(T,d,I),b=!0},p:st,i(d){b||(f(T.$$.fragment,d),b=!0)},o(d){u(T.$$.fragment,d),b=!1},d(d){d&&(s(l),s(M)),_(T,d)}}}function lo(C){let l,y="Example:",M,T,b;return T=new A({props:{code:"aW1wb3J0JTIwcmVxdWVzdHMlMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwSWRlZmljczJQcm9jZXNzb3IlMEFmcm9tJTIwdHJhbnNmb3JtZXJzLmltYWdlX3V0aWxzJTIwaW1wb3J0JTIwbG9hZF9pbWFnZSUwQSUwQXByb2Nlc3NvciUyMCUzRCUyMElkZWZpY3MyUHJvY2Vzc29yLmZyb21fcHJldHJhaW5lZCglMjJIdWdnaW5nRmFjZU00JTJGaWRlZmljczItOGIlMjIlMkMlMjBpbWFnZV9zZXFfbGVuJTNEMiklMEFwcm9jZXNzb3IuaW1hZ2VfcHJvY2Vzc29yLmRvX2ltYWdlX3NwbGl0dGluZyUyMCUzRCUyMEZhbHNlJTIwJTIwJTIzJTIwRm9yY2UlMjBhcyUyMEZhbHNlJTIwdG8lMjBzaW1wbGlmeSUyMHRoZSUyMGV4YW1wbGUlMEElMEF1cmwxJTIwJTNEJTIwJTIyaHR0cHMlM0ElMkYlMkZjZG4uYnJpdGFubmljYS5jb20lMkY2MSUyRjkzMDYxLTA1MC05OTE0N0RDRSUyRlN0YXR1ZS1vZi1MaWJlcnR5LUlzbGFuZC1OZXctWW9yay1CYXkuanBnJTIyJTBBdXJsMiUyMCUzRCUyMCUyMmh0dHBzJTNBJTJGJTJGY2RuLmJyaXRhbm5pY2EuY29tJTJGNTklMkY5NDQ1OS0wNTAtREJBNDI0NjclMkZTa3lsaW5lLUNoaWNhZ28uanBnJTIyJTBBJTBBaW1hZ2UxJTJDJTIwaW1hZ2UyJTIwJTNEJTIwbG9hZF9pbWFnZSh1cmwxKSUyQyUyMGxvYWRfaW1hZ2UodXJsMiklMEFpbWFnZXMlMjAlM0QlMjAlNUIlNUJpbWFnZTElNUQlMkMlMjAlNUJpbWFnZTIlNUQlNUQlMEElMEF0ZXh0JTIwJTNEJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIyJTNDaW1hZ2UlM0VJbiUyMHRoaXMlMjBpbWFnZSUyQyUyMHdlJTIwc2VlJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIyYmxhJTIwYmxhJTIwYmxhJTNDaW1hZ2UlM0UlMjIlMkMlMEElNUQlMEFvdXRwdXRzJTIwJTNEJTIwcHJvY2Vzc29yKHRleHQlM0R0ZXh0JTJDJTIwaW1hZ2VzJTNEaW1hZ2VzJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiUyQyUyMHBhZGRpbmclM0RUcnVlKSUwQWlucHV0X2lkcyUyMCUzRCUyMG91dHB1dHMuaW5wdXRfaWRzJTBBaW5wdXRfdG9rZW5zJTIwJTNEJTIwcHJvY2Vzc29yLnRva2VuaXplci5iYXRjaF9kZWNvZGUoaW5wdXRfaWRzKSUwQXByaW50KGlucHV0X3Rva2Vucyk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> requests
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Idefics2Processor
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers.image_utils <span class="hljs-keyword">import</span> load_image

<span class="hljs-meta">&gt;&gt;&gt; </span>processor = Idefics2Processor.from_pretrained(<span class="hljs-string">&quot;HuggingFaceM4/idefics2-8b&quot;</span>, image_seq_len=<span class="hljs-number">2</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>processor.image_processor.do_image_splitting = <span class="hljs-literal">False</span>  <span class="hljs-comment"># Force as False to simplify the example</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>url1 = <span class="hljs-string">&quot;https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>url2 = <span class="hljs-string">&quot;https://cdn.britannica.com/59/94459-050-DBA42467/Skyline-Chicago.jpg&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>image1, image2 = load_image(url1), load_image(url2)
<span class="hljs-meta">&gt;&gt;&gt; </span>images = [[image1], [image2]]

<span class="hljs-meta">&gt;&gt;&gt; </span>text = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;&lt;image&gt;In this image, we see&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;bla bla bla&lt;image&gt;&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = processor(text=text, images=images, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>, padding=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>input_ids = outputs.input_ids
<span class="hljs-meta">&gt;&gt;&gt; </span>input_tokens = processor.tokenizer.batch_decode(input_ids)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(input_tokens)
[<span class="hljs-string">&#x27;&lt;s&gt;&lt;fake_token_around_image&gt;&lt;image&gt;&lt;image&gt;&lt;fake_token_around_image&gt; In this image, we see&#x27;</span>, <span class="hljs-string">&#x27;&lt;s&gt; bla bla bla&lt;fake_token_around_image&gt;&lt;image&gt;&lt;image&gt;&lt;fake_token_around_image&gt;&#x27;</span>]`,wrap:!1}}),{c(){l=r("p"),l.textContent=y,M=n(),p(T.$$.fragment)},l(d){l=i(d,"P",{"data-svelte-h":!0}),m(l)!=="svelte-11lpom8"&&(l.textContent=y),M=a(d),h(T.$$.fragment,d)},m(d,I){o(d,l,I),o(d,M,I),g(T,d,I),b=!0},p:st,i(d){b||(f(T.$$.fragment,d),b=!0)},o(d){u(T.$$.fragment,d),b=!1},d(d){d&&(s(l),s(M)),_(T,d)}}}function co(C){let l,y,M,T,b,d,I,ot,L,us='The Idefics2 model was proposed in <a href="https://arxiv.org/abs/2405.02246" rel="nofollow">What matters when building vision-language models?</a> by Léo Tronchon, Hugo Laurencon, Victor Sanh. The accompanying blog post can be found <a href="https://huggingface.co/blog/idefics2" rel="nofollow">here</a>.',nt,D,_s=`Idefics2 is an open multimodal model that accepts arbitrary sequences of image and text inputs and produces text
outputs. The model can answer questions about images, describe visual content, create stories grounded on multiple
images, or simply behave as a pure language model without visual inputs. It improves upon IDEFICS-1, notably on
document understanding, OCR, or visual reasoning. Idefics2 is lightweight (8 billion parameters) and treats
images in their native aspect ratio and resolution, which allows for varying inference efficiency.`,at,O,Ms="The abstract from the paper is the following:",rt,K,Ts="<em>The growing interest in vision-language models (VLMs) has been driven by improvements in large language models and vision transformers. Despite the abundance of literature on this subject, we observe that critical decisions regarding the design of VLMs are often not justified. We argue that these unsupported decisions impede progress in the field by making it difficult to identify which choices improve model performance. To address this issue, we conduct extensive experiments around pre-trained models, architecture choice, data, and training methods. Our consolidation of findings includes the development of Idefics2, an efficient foundational VLM of 8 billion parameters. Idefics2 achieves state-of-the-art performance within its size category across various multimodal benchmarks, and is often on par with models four times its size. We release the model (base, instructed, and chat) along with the datasets created for its training.</em>",it,V,bs,lt,ee,ys='Idefics2 architecture. Taken from the <a href="https://arxiv.org/abs/2405.02246">original paper.</a>',dt,te,Is=`This model was contributed by <a href="https://huggingface.co/amyeroberts" rel="nofollow">amyeroberts</a>.
The original code can be found <a href="https://huggingface.co/HuggingFaceM4/idefics2" rel="nofollow">here</a>.`,ct,se,mt,oe,ws="<li>Each sample can contain multiple images, and the number of images can vary between samples. The processor will pad the inputs to the maximum number of images in a batch for input to the model.</li> <li>The processor has a <code>do_image_splitting</code> option. If <code>True</code>, each input image will be split into 4 sub-images, and concatenated with the original to form 5 images. This is useful for increasing model performance. Make sure <code>processor.image_processor.do_image_splitting</code> is set to <code>False</code> if the model was not trained with this option.</li> <li><code>text</code> passed to the processor should have the <code>&lt;image&gt;</code> tokens where the images should be inserted. And <code>&lt;end_of_utterance&gt;</code> at the end of each utterance if the text is a chat message.</li> <li>The processor has its own <code>apply_chat_template</code> method to convert chat messages to text that can then be passed as <code>text</code> to the processor.</li>",pt,ne,Js="Example of how to use the processor on chat messages:",ht,ae,gt,re,vs="<li>During training, it’s important to determine which tokens the model should not learn. For Idefics2, this typically comes down to the image and padding tokens. This means that one can create the labels as follows:</li>",ft,ie,ut,le,js="Do note that when training Idefics2 on multi-turn conversations between a user and an assistant, one typically also sets all the tokens corresponding to the user messages to -100.",_t,de,Mt,ce,Us='The code snippets above showcase inference without any optimization tricks. However, one can drastically speed up the model by leveraging <a href="../perf_train_gpu_one.md#flash-attention-2">Flash Attention</a>, which is a faster implementation of the attention mechanism used inside the model.',Tt,me,ks="First, make sure to install the latest version of Flash Attention 2 to include the sliding window attention feature.",bt,pe,yt,he,zs='Make also sure that you have a hardware that is compatible with Flash-Attention 2. Read more about it in the official documentation of the <a href="https://github.com/Dao-AILab/flash-attention" rel="nofollow">flash attention repository</a>. Make also sure to load your model in half-precision (e.g. <code>torch.float16</code>)',It,ge,Cs="To load and run a model using Flash Attention-2, simply change the code snippet above with the following change:",wt,fe,Jt,ue,vt,_e,Zs='As the Idefics2 model has 8 billion parameters, that would require about 16GB of GPU RAM in half precision (float16), since each parameter is stored in 2 bytes. However, one can shrink down the size of the model using <a href="../quantization.md">quantization</a>. If the model is quantized to 4 bits (or half a byte per parameter), that requires only about 3.5GB of RAM.',jt,Me,xs='Quantizing a model is as simple as passing a <code>quantization_config</code> to the model. One can change the code snippet above with the changes below. We’ll leverage the BitsAndyBytes quantization (but refer to <a href="../quantization.md">this page</a> for other quantization methods):',Ut,Te,kt,be,zt,ye,Ws="A list of official Hugging Face and community (indicated by 🌎) resources to help you get started with Idefics2. If you’re interested in submitting a resource to be included here, please feel free to open a Pull Request and we’ll review it! The resource should ideally demonstrate something new instead of duplicating an existing resource.",Ct,Ie,$s='<li>A notebook on how to fine-tune Idefics2 on a custom dataset using the <a href="../main_classes/trainer.md">Trainer</a> can be found <a href="https://colab.research.google.com/drive/1NtcTgRbSBKN7pYD3Vdx1j9m8pt3fhFDB?usp=sharing" rel="nofollow">here</a>. It supports both full fine-tuning as well as (quantized) LoRa.</li> <li>A script regarding how to fine-tune Idefics2 using the TRL library can be found <a href="https://gist.github.com/edbeeching/228652fc6c2b29a1641be5a5778223cb" rel="nofollow">here</a>.</li> <li>Demo notebook regarding fine-tuning Idefics2 for JSON extraction use cases can be found <a href="https://github.com/NielsRogge/Transformers-Tutorials/tree/master/Idefics2" rel="nofollow">here</a>. 🌎</li>',Zt,we,xt,v,Je,Ht,Fe,Ns=`This is the configuration class to store the configuration of a <a href="/docs/transformers/main/en/model_doc/idefics2#transformers.Idefics2Model">Idefics2Model</a>. It is used to instantiate a
Idefics2 model according to the specified arguments, defining the model architecture. Instantiating a
configuration with the defaults will yield a similar configuration to that of the model of the Idefics2
<a href="https://huggingface.co/HuggingFaceM4/idefics2-8b" rel="nofollow">HuggingFaceM4/idefics2-8b</a> architecture.`,St,Ge,Rs=`Configuration objects inherit from <a href="/docs/transformers/main/en/main_classes/configuration#transformers.PretrainedConfig">PretrainedConfig</a> and can be used to control the model outputs. Read the
documentation from <a href="/docs/transformers/main/en/main_classes/configuration#transformers.PretrainedConfig">PretrainedConfig</a> for more information.`,Yt,E,Wt,ve,$t,j,je,Qt,qe,Bs=`Idefics2 model consisting of a SIGLIP vision encoder and Mistral language decoder
This model inherits from <a href="/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel">PreTrainedModel</a>. Check the superclass documentation for the generic methods the
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
etc.)`,At,Ve,Fs=`This model is also a PyTorch <a href="https://pytorch.org/docs/stable/nn.html#torch.nn.Module" rel="nofollow">torch.nn.Module</a> subclass.
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
and behavior.`,Lt,w,Ue,Dt,Ee,Gs='The <a href="/docs/transformers/main/en/model_doc/idefics2#transformers.Idefics2Model">Idefics2Model</a> forward method, overrides the <code>__call__</code> special method.',Ot,X,Kt,Xe,qs=`Inputs fed to the model can have an arbitrary number of images. To account for this, pixel_values fed to
the model have image padding -> (batch_size, max_num_images, 3, max_heights, max_widths) where
max_num_images is the maximum number of images among the batch_size samples in the batch.`,es,Pe,Vs=`Padding images are not needed beyond padding the pixel_values at the entrance of the model.
For efficiency, we only pass through the vision_model’s forward the real images by
discarding the padding images i.e. pixel_values of size (image_batch_size, 3, height, width) where
image_batch_size would be 7 when num_images_per_sample=[1, 3, 1, 2] and max_num_images would be 3.`,Nt,ke,Rt,U,ze,ts,He,Es=`The Idefics2 Model with a language modeling head. It is made up a SigLIP vision encoder, with a language modeling head on top.
This model inherits from <a href="/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel">PreTrainedModel</a>. Check the superclass documentation for the generic methods the
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
etc.)`,ss,Se,Xs=`This model is also a PyTorch <a href="https://pytorch.org/docs/stable/nn.html#torch.nn.Module" rel="nofollow">torch.nn.Module</a> subclass.
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
and behavior.`,os,Z,Ce,ns,Ye,Ps='The <a href="/docs/transformers/main/en/model_doc/idefics2#transformers.Idefics2ForConditionalGeneration">Idefics2ForConditionalGeneration</a> forward method, overrides the <code>__call__</code> special method.',as,P,rs,H,Bt,Ze,Ft,x,xe,is,Qe,Hs="Constructs a Idefics image processor.",ls,S,We,ds,Ae,Ss="Preprocess a batch of images.",Gt,$e,qt,k,Ne,cs,Le,Ys="Constructs a IDEFICS2 processor which wraps a LLama tokenizer and IDEFICS2 image processor into a single processor.",ms,De,Qs=`<a href="/docs/transformers/main/en/model_doc/idefics#transformers.IdeficsProcessor">IdeficsProcessor</a> offers all the functionalities of <a href="/docs/transformers/main/en/model_doc/idefics2#transformers.Idefics2ImageProcessor">Idefics2ImageProcessor</a> and <a href="/docs/transformers/main/en/model_doc/llama#transformers.LlamaTokenizerFast">LlamaTokenizerFast</a>. See
the docstring of <a href="/docs/transformers/main/en/model_doc/idefics#transformers.IdeficsProcessor.__call__"><strong>call</strong>()</a> and <code>decode()</code> for more information.`,ps,G,Re,hs,Oe,As="Processes the input prompts and returns a BatchEncoding.",gs,Y,Vt,Be,Et,tt,Xt;return b=new F({props:{title:"Idefics2",local:"idefics2",headingTag:"h1"}}),I=new F({props:{title:"Overview",local:"overview",headingTag:"h2"}}),se=new F({props:{title:"Usage tips",local:"usage-tips",headingTag:"h2"}}),ae=new A({props:{code:"aW1wb3J0JTIwcmVxdWVzdHMlMEFmcm9tJTIwUElMJTIwaW1wb3J0JTIwSW1hZ2UlMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwSWRlZmljczJQcm9jZXNzb3IlMkMlMjBJZGVmaWNzMkZvckNvbmRpdGlvbmFsR2VuZXJhdGlvbiUwQWltcG9ydCUyMHRvcmNoJTBBJTBBZGV2aWNlJTIwJTNEJTIwJTIyY3VkYSUyMiUyMGlmJTIwdG9yY2guY3VkYS5pc19hdmFpbGFibGUoKSUyMGVsc2UlMjAlMjJjcHUlMjIlMEElMEF1cmxfMSUyMCUzRCUyMCUyMmh0dHAlM0ElMkYlMkZpbWFnZXMuY29jb2RhdGFzZXQub3JnJTJGdmFsMjAxNyUyRjAwMDAwMDAzOTc2OS5qcGclMjIlMEF1cmxfMiUyMCUzRCUyMCUyMmh0dHAlM0ElMkYlMkZpbWFnZXMuY29jb2RhdGFzZXQub3JnJTJGdmFsMjAxNyUyRjAwMDAwMDIxOTU3OC5qcGclMjIlMEElMEFpbWFnZV8xJTIwJTNEJTIwSW1hZ2Uub3BlbihyZXF1ZXN0cy5nZXQodXJsXzElMkMlMjBzdHJlYW0lM0RUcnVlKS5yYXcpJTBBaW1hZ2VfMiUyMCUzRCUyMEltYWdlLm9wZW4ocmVxdWVzdHMuZ2V0KHVybF8yJTJDJTIwc3RyZWFtJTNEVHJ1ZSkucmF3KSUwQWltYWdlcyUyMCUzRCUyMCU1QmltYWdlXzElMkMlMjBpbWFnZV8yJTVEJTBBJTBBbWVzc2FnZXMlMjAlM0QlMjAlNUIlN0IlMEElMjAlMjAlMjAlMjAlMjJyb2xlJTIyJTNBJTIwJTIydXNlciUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMmNvbnRlbnQlMjIlM0ElMjAlNUIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0IlMjJ0eXBlJTIyJTNBJTIwJTIydGV4dCUyMiUyQyUyMCUyMnRleHQlMjIlM0ElMjAlMjJXaGF0JUUyJTgwJTk5cyUyMHRoZSUyMGRpZmZlcmVuY2UlMjBiZXR3ZWVuJTIwdGhlc2UlMjB0d28lMjBpbWFnZXMlM0YlMjIlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0IlMjJ0eXBlJTIyJTNBJTIwJTIyaW1hZ2UlMjIlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0IlMjJ0eXBlJTIyJTNBJTIwJTIyaW1hZ2UlMjIlN0QlMkMlMEElMjAlMjAlMjAlMjAlNUQlMkMlMEElN0QlNUQlMEElMEFwcm9jZXNzb3IlMjAlM0QlMjBJZGVmaWNzMlByb2Nlc3Nvci5mcm9tX3ByZXRyYWluZWQoJTIySHVnZ2luZ0ZhY2VNNCUyRmlkZWZpY3MyLThiJTIyKSUwQW1vZGVsJTIwJTNEJTIwSWRlZmljczJGb3JDb25kaXRpb25hbEdlbmVyYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUyMkh1Z2dpbmdGYWNlTTQlMkZpZGVmaWNzMi04YiUyMiklMEFtb2RlbC50byhkZXZpY2UpJTBBJTBBJTIzJTIwYXQlMjBpbmZlcmVuY2UlMjB0aW1lJTJDJTIwb25lJTIwbmVlZHMlMjB0byUyMHBhc3MlMjAlNjBhZGRfZ2VuZXJhdGlvbl9wcm9tcHQlM0RUcnVlJTYwJTIwaW4lMjBvcmRlciUyMHRvJTIwbWFrZSUyMHN1cmUlMjB0aGUlMjBtb2RlbCUyMGNvbXBsZXRlcyUyMHRoZSUyMHByb21wdCUwQXRleHQlMjAlM0QlMjBwcm9jZXNzb3IuYXBwbHlfY2hhdF90ZW1wbGF0ZShtZXNzYWdlcyUyQyUyMGFkZF9nZW5lcmF0aW9uX3Byb21wdCUzRFRydWUpJTBBcHJpbnQodGV4dCklMEElMjMlMjAnVXNlciUzQSUyMFdoYXQlRTIlODAlOTlzJTIwdGhlJTIwZGlmZmVyZW5jZSUyMGJldHdlZW4lMjB0aGVzZSUyMHR3byUyMGltYWdlcyUzRiUzQ2ltYWdlJTNFJTNDaW1hZ2UlM0UlM0NlbmRfb2ZfdXR0ZXJhbmNlJTNFJTVDbkFzc2lzdGFudCUzQSclMEElMEFpbnB1dHMlMjAlM0QlMjBwcm9jZXNzb3IoaW1hZ2VzJTNEaW1hZ2VzJTJDJTIwdGV4dCUzRHRleHQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS50byhkZXZpY2UpJTBBJTBBZ2VuZXJhdGVkX3RleHQlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSgqKmlucHV0cyUyQyUyMG1heF9uZXdfdG9rZW5zJTNENTAwKSUwQWdlbmVyYXRlZF90ZXh0JTIwJTNEJTIwcHJvY2Vzc29yLmJhdGNoX2RlY29kZShnZW5lcmF0ZWRfdGV4dCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSU1QjAlNUQlMEFwcmludCglMjJHZW5lcmF0ZWQlMjB0ZXh0JTNBJTIyJTJDJTIwZ2VuZXJhdGVkX3RleHQp",highlighted:`<span class="hljs-keyword">import</span> requests
<span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Idefics2Processor, Idefics2ForConditionalGeneration
<span class="hljs-keyword">import</span> torch

device = <span class="hljs-string">&quot;cuda&quot;</span> <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> <span class="hljs-string">&quot;cpu&quot;</span>

url_1 = <span class="hljs-string">&quot;http://images.cocodataset.org/val2017/000000039769.jpg&quot;</span>
url_2 = <span class="hljs-string">&quot;http://images.cocodataset.org/val2017/000000219578.jpg&quot;</span>

image_1 = Image.<span class="hljs-built_in">open</span>(requests.get(url_1, stream=<span class="hljs-literal">True</span>).raw)
image_2 = Image.<span class="hljs-built_in">open</span>(requests.get(url_2, stream=<span class="hljs-literal">True</span>).raw)
images = [image_1, image_2]

messages = [{
    <span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>,
    <span class="hljs-string">&quot;content&quot;</span>: [
        {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;text&quot;</span>: <span class="hljs-string">&quot;What’s the difference between these two images?&quot;</span>},
        {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;image&quot;</span>},
        {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;image&quot;</span>},
    ],
}]

processor = Idefics2Processor.from_pretrained(<span class="hljs-string">&quot;HuggingFaceM4/idefics2-8b&quot;</span>)
model = Idefics2ForConditionalGeneration.from_pretrained(<span class="hljs-string">&quot;HuggingFaceM4/idefics2-8b&quot;</span>)
model.to(device)

<span class="hljs-comment"># at inference time, one needs to pass \`add_generation_prompt=True\` in order to make sure the model completes the prompt</span>
text = processor.apply_chat_template(messages, add_generation_prompt=<span class="hljs-literal">True</span>)
<span class="hljs-built_in">print</span>(text)
<span class="hljs-comment"># &#x27;User: What’s the difference between these two images?&lt;image&gt;&lt;image&gt;&lt;end_of_utterance&gt;\\nAssistant:&#x27;</span>

inputs = processor(images=images, text=text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(device)

generated_text = model.generate(**inputs, max_new_tokens=<span class="hljs-number">500</span>)
generated_text = processor.batch_decode(generated_text, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Generated text:&quot;</span>, generated_text)`,wrap:!1}}),ie=new A({props:{code:"aW1wb3J0JTIwcmVxdWVzdHMlMEFmcm9tJTIwUElMJTIwaW1wb3J0JTIwSW1hZ2UlMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwSWRlZmljczJQcm9jZXNzb3IlMkMlMjBJZGVmaWNzMkZvckNvbmRpdGlvbmFsR2VuZXJhdGlvbiUwQWltcG9ydCUyMHRvcmNoJTBBJTBBdXJsXzElMjAlM0QlMjAlMjJodHRwJTNBJTJGJTJGaW1hZ2VzLmNvY29kYXRhc2V0Lm9yZyUyRnZhbDIwMTclMkYwMDAwMDAwMzk3NjkuanBnJTIyJTBBdXJsXzIlMjAlM0QlMjAlMjJodHRwJTNBJTJGJTJGaW1hZ2VzLmNvY29kYXRhc2V0Lm9yZyUyRnZhbDIwMTclMkYwMDAwMDAyMTk1NzguanBnJTIyJTBBJTBBaW1hZ2VfMSUyMCUzRCUyMEltYWdlLm9wZW4ocmVxdWVzdHMuZ2V0KHVybF8xJTJDJTIwc3RyZWFtJTNEVHJ1ZSkucmF3KSUwQWltYWdlXzIlMjAlM0QlMjBJbWFnZS5vcGVuKHJlcXVlc3RzLmdldCh1cmxfMiUyQyUyMHN0cmVhbSUzRFRydWUpLnJhdyklMEFpbWFnZXMlMjAlM0QlMjAlNUJpbWFnZV8xJTJDJTIwaW1hZ2VfMiU1RCUwQSUwQW1lc3NhZ2VzJTIwJTNEJTIwJTVCJTdCJTBBJTIwJTIwJTIwJTIwJTIycm9sZSUyMiUzQSUyMCUyMnVzZXIlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJjb250ZW50JTIyJTNBJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdCJTIydHlwZSUyMiUzQSUyMCUyMnRleHQlMjIlMkMlMjAlMjJ0ZXh0JTIyJTNBJTIwJTIyV2hhdCVFMiU4MCU5OXMlMjB0aGUlMjBkaWZmZXJlbmNlJTIwYmV0d2VlbiUyMHRoZXNlJTIwdHdvJTIwaW1hZ2VzJTNGJTIyJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdCJTIydHlwZSUyMiUzQSUyMCUyMmltYWdlJTIyJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdCJTIydHlwZSUyMiUzQSUyMCUyMmltYWdlJTIyJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTVEJTJDJTBBJTdEJTJDJTBBJTdCJTBBJTIwJTIwJTIwJTIwJTIycm9sZSUyMiUzQSUyMCUyMmFzc2lzdGFudCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMmNvbnRlbnQlMjIlM0ElMjAlNUIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0IlMjJ0eXBlJTIyJTNBJTIwJTIydGV4dCUyMiUyQyUyMCUyMnRleHQlMjIlM0ElMjAlMjJUaGUlMjBkaWZmZXJlbmNlJTIwaXMlMjB0aGF0JTIwb25lJTIwaW1hZ2UlMjBpcyUyMGFib3V0JTIwZG9ncyUyMGFuZCUyMHRoZSUyMG90aGVyJTIwb25lJTIwYWJvdXQlMjBjYXRzLiUyMiU3RCUyQyUwQSUyMCUyMCUyMCUyMCU1RCUyQyUwQSU3RCU1RCUwQSUwQWRldmljZSUyMCUzRCUyMCUyMmN1ZGElMjIlMjBpZiUyMHRvcmNoLmN1ZGEuaXNfYXZhaWxhYmxlKCklMjBlbHNlJTIwJTIyY3B1JTIyJTBBJTBBcHJvY2Vzc29yJTIwJTNEJTIwSWRlZmljczJQcm9jZXNzb3IuZnJvbV9wcmV0cmFpbmVkKCUyMkh1Z2dpbmdGYWNlTTQlMkZpZGVmaWNzMi04YiUyMiklMEFtb2RlbCUyMCUzRCUyMElkZWZpY3MyRm9yQ29uZGl0aW9uYWxHZW5lcmF0aW9uLmZyb21fcHJldHJhaW5lZCglMjJIdWdnaW5nRmFjZU00JTJGaWRlZmljczItOGIlMjIpJTBBbW9kZWwudG8oZGV2aWNlKSUwQSUwQXRleHQlMjAlM0QlMjBwcm9jZXNzb3IuYXBwbHlfY2hhdF90ZW1wbGF0ZShtZXNzYWdlcyUyQyUyMGFkZF9nZW5lcmF0aW9uX3Byb21wdCUzREZhbHNlKSUwQWlucHV0cyUyMCUzRCUyMHByb2Nlc3NvcihpbWFnZXMlM0RpbWFnZXMlMkMlMjB0ZXh0JTNEdGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnRvKGRldmljZSklMEElMEFsYWJlbHMlMjAlM0QlMjBpbnB1dHMuaW5wdXRfaWRzLmNsb25lKCklMEFsYWJlbHMlNUJsYWJlbHMlMjAlM0QlM0QlMjBwcm9jZXNzb3IudG9rZW5pemVyLnBhZF90b2tlbl9pZCU1RCUyMCUzRCUyMC0xMDAlMEFsYWJlbHMlNUJsYWJlbHMlMjAlM0QlM0QlMjBtb2RlbC5jb25maWcuaW1hZ2VfdG9rZW5faWQlNUQlMjAlM0QlMjAtMTAwJTBBJTBBaW5wdXRzJTVCJTIybGFiZWxzJTIyJTVEJTIwJTNEJTIwbGFiZWxzJTBBJTBBb3V0cHV0cyUyMCUzRCUyMG1vZGVsKCoqaW5wdXRzKSUwQWxvc3MlMjAlM0QlMjBvdXRwdXRzLmxvc3MlMEFsb3NzLmJhY2t3YXJkKCk=",highlighted:`<span class="hljs-keyword">import</span> requests
<span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Idefics2Processor, Idefics2ForConditionalGeneration
<span class="hljs-keyword">import</span> torch

url_1 = <span class="hljs-string">&quot;http://images.cocodataset.org/val2017/000000039769.jpg&quot;</span>
url_2 = <span class="hljs-string">&quot;http://images.cocodataset.org/val2017/000000219578.jpg&quot;</span>

image_1 = Image.<span class="hljs-built_in">open</span>(requests.get(url_1, stream=<span class="hljs-literal">True</span>).raw)
image_2 = Image.<span class="hljs-built_in">open</span>(requests.get(url_2, stream=<span class="hljs-literal">True</span>).raw)
images = [image_1, image_2]

messages = [{
    <span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>,
    <span class="hljs-string">&quot;content&quot;</span>: [
        {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;text&quot;</span>: <span class="hljs-string">&quot;What’s the difference between these two images?&quot;</span>},
        {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;image&quot;</span>},
        {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;image&quot;</span>},
    ],
},
{
    <span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;assistant&quot;</span>,
    <span class="hljs-string">&quot;content&quot;</span>: [
        {<span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;text&quot;</span>: <span class="hljs-string">&quot;The difference is that one image is about dogs and the other one about cats.&quot;</span>},
    ],
}]

device = <span class="hljs-string">&quot;cuda&quot;</span> <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> <span class="hljs-string">&quot;cpu&quot;</span>

processor = Idefics2Processor.from_pretrained(<span class="hljs-string">&quot;HuggingFaceM4/idefics2-8b&quot;</span>)
model = Idefics2ForConditionalGeneration.from_pretrained(<span class="hljs-string">&quot;HuggingFaceM4/idefics2-8b&quot;</span>)
model.to(device)

text = processor.apply_chat_template(messages, add_generation_prompt=<span class="hljs-literal">False</span>)
inputs = processor(images=images, text=text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(device)

labels = inputs.input_ids.clone()
labels[labels == processor.tokenizer.pad_token_id] = -<span class="hljs-number">100</span>
labels[labels == model.config.image_token_id] = -<span class="hljs-number">100</span>

inputs[<span class="hljs-string">&quot;labels&quot;</span>] = labels

outputs = model(**inputs)
loss = outputs.loss
loss.backward()`,wrap:!1}}),de=new F({props:{title:"Model optimizations: Flash Attention",local:"model-optimizations-flash-attention",headingTag:"h2"}}),pe=new A({props:{code:"cGlwJTIwaW5zdGFsbCUyMC1VJTIwZmxhc2gtYXR0biUyMC0tbm8tYnVpbGQtaXNvbGF0aW9u",highlighted:"pip install -U flash-attn --no-build-isolation",wrap:!1}}),fe=new A({props:{code:"bW9kZWwlMjAlM0QlMjBJZGVmaWNzMkZvckNvbmRpdGlvbmFsR2VuZXJhdGlvbi5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIySHVnZ2luZ0ZhY2VNNCUyRmlkZWZpY3MyLThiJTIyJTJDJTBBJTJCJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTIwJTIwJTIwJTIwJTBBJTJCJTIwJTIwJTIwJTIwYXR0bl9pbXBsZW1lbnRhdGlvbiUzRCUyMmZsYXNoX2F0dGVudGlvbl8yJTIyJTJDJTBBKS50byhkZXZpY2Up",highlighted:`model = Idefics2ForConditionalGeneration.from_pretrained(
    &quot;HuggingFaceM4/idefics2-8b&quot;,
<span class="hljs-addition">+    torch_dtype=torch.float16,    </span>
<span class="hljs-addition">+    attn_implementation=&quot;flash_attention_2&quot;,</span>
).to(device)`,wrap:!1}}),ue=new F({props:{title:"Shrinking down Idefics2 using quantization",local:"shrinking-down-idefics2-using-quantization",headingTag:"h2"}}),Te=new A({props:{code:"JTJCJTIwZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQSUyQiUyMHF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTJCJTIwJTIwJTIwJTIwbG9hZF9pbl80Yml0JTNEVHJ1ZSUyQyUwQSUyQiUyMCUyMCUyMCUyMGJuYl80Yml0X3F1YW50X3R5cGUlM0QlMjJuZjQlMjIlMkMlMEElMkIlMjAlMjAlMjAlMjBibmJfNGJpdF91c2VfZG91YmxlX3F1YW50JTNEVHJ1ZSUyQyUwQSUyQiUyMCUyMCUyMCUyMGJuYl80Yml0X2NvbXB1dGVfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTBBJTJCJTIwKSUwQW1vZGVsJTIwJTNEJTIwSWRlZmljczJGb3JDb25kaXRpb25hbEdlbmVyYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMkh1Z2dpbmdGYWNlTTQlMkZpZGVmaWNzMi04YiUyMiUyQyUwQSUyQiUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNiUyQyUyMCUyMCUyMCUyMCUwQSUyQiUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudGl6YXRpb25fY29uZmlnJTJDJTBBKS50byhkZXZpY2Up",highlighted:`<span class="hljs-addition">+ from transformers import BitsAndBytesConfig</span>

<span class="hljs-addition">+ quantization_config = BitsAndBytesConfig(</span>
<span class="hljs-addition">+    load_in_4bit=True,</span>
<span class="hljs-addition">+    bnb_4bit_quant_type=&quot;nf4&quot;,</span>
<span class="hljs-addition">+    bnb_4bit_use_double_quant=True,</span>
<span class="hljs-addition">+    bnb_4bit_compute_dtype=torch.float16</span>
<span class="hljs-addition">+ )</span>
model = Idefics2ForConditionalGeneration.from_pretrained(
    &quot;HuggingFaceM4/idefics2-8b&quot;,
<span class="hljs-addition">+    torch_dtype=torch.float16,    </span>
<span class="hljs-addition">+    quantization_config=quantization_config,</span>
).to(device)`,wrap:!1}}),be=new F({props:{title:"Resources",local:"resources",headingTag:"h2"}}),we=new F({props:{title:"Idefics2Config",local:"transformers.Idefics2Config",headingTag:"h2"}}),Je=new q({props:{name:"class transformers.Idefics2Config",anchor:"transformers.Idefics2Config",parameters:[{name:"use_cache",val:" = True"},{name:"image_token_id",val:" = 32001"},{name:"tie_word_embeddings",val:" = False"},{name:"vision_config",val:" = None"},{name:"perceiver_config",val:" = None"},{name:"text_config",val:" = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.Idefics2Config.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether or not the model should cache the key/value pairs of the attention mechanism.`,name:"use_cache"},{anchor:"transformers.Idefics2Config.image_token_id",description:`<strong>image_token_id</strong> (<code>int</code>, <em>optional</em>, defaults to 32001) &#x2014;
The id of the &#x201C;image&#x201D; token.`,name:"image_token_id"},{anchor:"transformers.Idefics2Config.tie_word_embeddings",description:`<strong>tie_word_embeddings</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to tie the word embeddings with the token embeddings.`,name:"tie_word_embeddings"},{anchor:"transformers.Idefics2Config.vision_config",description:`<strong>vision_config</strong> (<code>IdeficsVisionConfig</code> or <code>dict</code>, <em>optional</em>) &#x2014;
Custom vision config or dict`,name:"vision_config"},{anchor:"transformers.Idefics2Config.perceiver_config",description:`<strong>perceiver_config</strong> (<code>IdeficsPerceiverConfig</code> or <code>dict</code>, <em>optional</em>) &#x2014;
Custom perceiver config or dict`,name:"perceiver_config"},{anchor:"transformers.Idefics2Config.text_config",description:`<strong>text_config</strong> (<code>MistralConfig</code> or <code>dict</code>, <em>optional</em>) &#x2014;
Custom text config or dict for the text model`,name:"text_config"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/idefics2/configuration_idefics2.py#L179"}}),E=new fs({props:{anchor:"transformers.Idefics2Config.example",$$slots:{default:[no]},$$scope:{ctx:C}}}),ve=new F({props:{title:"Idefics2Model",local:"transformers.Idefics2Model",headingTag:"h2"}}),je=new q({props:{name:"class transformers.Idefics2Model",anchor:"transformers.Idefics2Model",parameters:[{name:"config",val:": Idefics2Config"}],parametersDescription:[{anchor:"transformers.Idefics2Model.config",description:`<strong>config</strong> (<a href="/docs/transformers/main/en/model_doc/idefics2#transformers.Idefics2Config">Idefics2Config</a> or <code>Idefics2VisionConfig</code>) &#x2014;
Model configuration class with all the parameters of the model. Initializing with a config file does not
load the weights associated with the model, only the configuration. Check out the
<a href="/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/idefics2/modeling_idefics2.py#L1211"}}),Ue=new q({props:{name:"forward",anchor:"transformers.Idefics2Model.forward",parameters:[{name:"input_ids",val:": LongTensor = None"},{name:"attention_mask",val:": Optional = None"},{name:"position_ids",val:": Optional = None"},{name:"past_key_values",val:": Optional = None"},{name:"inputs_embeds",val:": Optional = None"},{name:"pixel_values",val:": Optional = None"},{name:"pixel_attention_mask",val:": Optional = None"},{name:"image_hidden_states",val:": Optional = None"},{name:"use_cache",val:": Optional = None"},{name:"output_attentions",val:": Optional = None"},{name:"output_hidden_states",val:": Optional = None"},{name:"return_dict",val:": Optional = None"}],parametersDescription:[{anchor:"transformers.Idefics2Model.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>) &#x2014;
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.</p>
<p>Indices can be obtained using <a href="/docs/transformers/main/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See <a href="/docs/transformers/main/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/main/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p>
<p><a href="../glossary#input-ids">What are input IDs?</a>`,name:"input_ids"},{anchor:"transformers.Idefics2Model.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>not masked</strong>,</li>
<li>0 for tokens that are <strong>masked</strong>.</li>
</ul>
<p><a href="../glossary#attention-mask">What are attention masks?</a></p>
<p>Indices can be obtained using <a href="/docs/transformers/main/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See <a href="/docs/transformers/main/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/main/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p>
<p>If <code>past_key_values</code> is used, optionally only the last <code>decoder_input_ids</code> have to be input (see
<code>past_key_values</code>).</p>
<p>If you want to change padding behavior, you should read <code>modeling_opt._prepare_decoder_attention_mask</code>
and modify to your needs. See diagram 1 in <a href="https://arxiv.org/abs/1910.13461" rel="nofollow">the paper</a> for more
information on the default strategy.</p>
<ul>
<li>1 indicates the head is <strong>not masked</strong>,</li>
<li>0 indicates the head is <strong>masked</strong>.</li>
</ul>`,name:"attention_mask"},{anchor:"transformers.Idefics2Model.forward.position_ids",description:`<strong>position_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range <code>[0, config.n_positions - 1]</code>. <a href="../glossary#position-ids">What are position IDs?</a>`,name:"position_ids"},{anchor:"transformers.Idefics2Model.forward.past_key_values",description:`<strong>past_key_values</strong> (<code>tuple(tuple(torch.FloatTensor))</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) &#x2014;
Tuple of <code>tuple(torch.FloatTensor)</code> of length <code>config.n_layers</code>, with each tuple having 2 tensors of shape
<code>(batch_size, num_heads, sequence_length, embed_size_per_head)</code>) and 2 additional tensors of shape
<code>(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)</code>.</p>
<p>Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used (see <code>past_key_values</code> input) to speed up sequential decoding.</p>
<p>If <code>past_key_values</code> are used, the user can optionally input only the last <code>decoder_input_ids</code> (those that
don&#x2019;t have their past key value states given to this model) of shape <code>(batch_size, 1)</code> instead of all
<code>decoder_input_ids</code> of shape <code>(batch_size, sequence_length)</code>.`,name:"past_key_values"},{anchor:"transformers.Idefics2Model.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) &#x2014;
Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This
is useful if you want more control over how to convert <code>input_ids</code> indices into associated vectors than the
model&#x2019;s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.Idefics2Model.forward.pixel_values",description:'<strong>pixel_values</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_channels, image_size, image_size)) -- The tensors corresponding to the input images. Pixel values can be obtained using [AutoImageProcessor](/docs/transformers/main/en/model_doc/auto#transformers.AutoImageProcessor). See [CLIPImageProcessor.__call__()](/docs/transformers/main/en/model_doc/vit#transformers.ViTFeatureExtractor.__call__) for details ([]</code>LlavaProcessor`] uses\n<a href="/docs/transformers/main/en/model_doc/clip#transformers.CLIPImageProcessor">CLIPImageProcessor</a> for processing images).',name:"pixel_values"},{anchor:"transformers.Idefics2Model.forward.pixel_attention_mask",description:`<strong>pixel_attention_mask</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, image_size, image_size)</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding pixel indices.`,name:"pixel_attention_mask"},{anchor:"transformers.Idefics2Model.forward.image_hidden_states",description:`<strong>image_hidden_states</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_channels, image_size, image_size)</code>) &#x2014;
The hidden states of the image encoder after modality projection and perceiver resampling.`,name:"image_hidden_states"},{anchor:"transformers.Idefics2Model.forward.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
If set to <code>True</code>, <code>past_key_values</code> key value states are returned and can be used to speed up decoding (see
<code>past_key_values</code>).`,name:"use_cache"},{anchor:"transformers.Idefics2Model.forward.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.Idefics2Model.forward.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.Idefics2Model.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return a <a href="/docs/transformers/main/en/main_classes/output#transformers.utils.ModelOutput">ModelOutput</a> instead of a plain tuple.`,name:"return_dict"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/idefics2/modeling_idefics2.py#L1293"}}),X=new Ls({props:{$$slots:{default:[ao]},$$scope:{ctx:C}}}),ke=new F({props:{title:"Idefics2ForConditionalGeneration",local:"transformers.Idefics2ForConditionalGeneration",headingTag:"h2"}}),ze=new q({props:{name:"class transformers.Idefics2ForConditionalGeneration",anchor:"transformers.Idefics2ForConditionalGeneration",parameters:[{name:"config",val:""}],parametersDescription:[{anchor:"transformers.Idefics2ForConditionalGeneration.config",description:`<strong>config</strong> (<a href="/docs/transformers/main/en/model_doc/idefics2#transformers.Idefics2Config">Idefics2Config</a> or <code>Idefics2VisionConfig</code>) &#x2014;
Model configuration class with all the parameters of the model. Initializing with a config file does not
load the weights associated with the model, only the configuration. Check out the
<a href="/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/idefics2/modeling_idefics2.py#L1436"}}),Ce=new q({props:{name:"forward",anchor:"transformers.Idefics2ForConditionalGeneration.forward",parameters:[{name:"input_ids",val:": LongTensor = None"},{name:"attention_mask",val:": Optional = None"},{name:"position_ids",val:": Optional = None"},{name:"past_key_values",val:": Optional = None"},{name:"inputs_embeds",val:": Optional = None"},{name:"pixel_values",val:": Optional = None"},{name:"pixel_attention_mask",val:": Optional = None"},{name:"image_hidden_states",val:": Optional = None"},{name:"labels",val:": Optional = None"},{name:"use_cache",val:": Optional = None"},{name:"output_attentions",val:": Optional = None"},{name:"output_hidden_states",val:": Optional = None"},{name:"return_dict",val:": Optional = None"}],parametersDescription:[{anchor:"transformers.Idefics2ForConditionalGeneration.forward.input_ids",description:`<strong>input_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>) &#x2014;
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.</p>
<p>Indices can be obtained using <a href="/docs/transformers/main/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See <a href="/docs/transformers/main/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/main/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p>
<p><a href="../glossary#input-ids">What are input IDs?</a>`,name:"input_ids"},{anchor:"transformers.Idefics2ForConditionalGeneration.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>not masked</strong>,</li>
<li>0 for tokens that are <strong>masked</strong>.</li>
</ul>
<p><a href="../glossary#attention-mask">What are attention masks?</a></p>
<p>Indices can be obtained using <a href="/docs/transformers/main/en/model_doc/auto#transformers.AutoTokenizer">AutoTokenizer</a>. See <a href="/docs/transformers/main/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/main/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p>
<p>If <code>past_key_values</code> is used, optionally only the last <code>decoder_input_ids</code> have to be input (see
<code>past_key_values</code>).</p>
<p>If you want to change padding behavior, you should read <code>modeling_opt._prepare_decoder_attention_mask</code>
and modify to your needs. See diagram 1 in <a href="https://arxiv.org/abs/1910.13461" rel="nofollow">the paper</a> for more
information on the default strategy.</p>
<ul>
<li>1 indicates the head is <strong>not masked</strong>,</li>
<li>0 indicates the head is <strong>masked</strong>.</li>
</ul>`,name:"attention_mask"},{anchor:"transformers.Idefics2ForConditionalGeneration.forward.position_ids",description:`<strong>position_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range <code>[0, config.n_positions - 1]</code>. <a href="../glossary#position-ids">What are position IDs?</a>`,name:"position_ids"},{anchor:"transformers.Idefics2ForConditionalGeneration.forward.past_key_values",description:`<strong>past_key_values</strong> (<code>tuple(tuple(torch.FloatTensor))</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) &#x2014;
Tuple of <code>tuple(torch.FloatTensor)</code> of length <code>config.n_layers</code>, with each tuple having 2 tensors of shape
<code>(batch_size, num_heads, sequence_length, embed_size_per_head)</code>) and 2 additional tensors of shape
<code>(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)</code>.</p>
<p>Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used (see <code>past_key_values</code> input) to speed up sequential decoding.</p>
<p>If <code>past_key_values</code> are used, the user can optionally input only the last <code>decoder_input_ids</code> (those that
don&#x2019;t have their past key value states given to this model) of shape <code>(batch_size, 1)</code> instead of all
<code>decoder_input_ids</code> of shape <code>(batch_size, sequence_length)</code>.`,name:"past_key_values"},{anchor:"transformers.Idefics2ForConditionalGeneration.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) &#x2014;
Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This
is useful if you want more control over how to convert <code>input_ids</code> indices into associated vectors than the
model&#x2019;s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.Idefics2ForConditionalGeneration.forward.pixel_values",description:'<strong>pixel_values</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_channels, image_size, image_size)) -- The tensors corresponding to the input images. Pixel values can be obtained using [AutoImageProcessor](/docs/transformers/main/en/model_doc/auto#transformers.AutoImageProcessor). See [CLIPImageProcessor.__call__()](/docs/transformers/main/en/model_doc/vit#transformers.ViTFeatureExtractor.__call__) for details ([]</code>LlavaProcessor`] uses\n<a href="/docs/transformers/main/en/model_doc/clip#transformers.CLIPImageProcessor">CLIPImageProcessor</a> for processing images).',name:"pixel_values"},{anchor:"transformers.Idefics2ForConditionalGeneration.forward.pixel_attention_mask",description:`<strong>pixel_attention_mask</strong> (<code>torch.Tensor</code> of shape <code>(batch_size, image_size, image_size)</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding pixel indices.`,name:"pixel_attention_mask"},{anchor:"transformers.Idefics2ForConditionalGeneration.forward.image_hidden_states",description:`<strong>image_hidden_states</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_channels, image_size, image_size)</code>) &#x2014;
The hidden states of the image encoder after modality projection and perceiver resampling.`,name:"image_hidden_states"},{anchor:"transformers.Idefics2ForConditionalGeneration.forward.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
If set to <code>True</code>, <code>past_key_values</code> key value states are returned and can be used to speed up decoding (see
<code>past_key_values</code>).`,name:"use_cache"},{anchor:"transformers.Idefics2ForConditionalGeneration.forward.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.Idefics2ForConditionalGeneration.forward.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.Idefics2ForConditionalGeneration.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return a <a href="/docs/transformers/main/en/main_classes/output#transformers.utils.ModelOutput">ModelOutput</a> instead of a plain tuple.</p>
<p>Args &#x2014;
labels (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>):
Labels for computing the masked language modeling loss. Indices should either be in <code>[0, ..., config.vocab_size]</code> or <code>model.image_token_id</code> (where <code>model</code> is your instance of <code>Idefics2ForConditionalGeneration</code>).
Tokens with indices set to <code>model.image_token_id</code> are ignored (masked), the loss is only
computed for the tokens with labels in <code>[0, ..., config.vocab_size]</code>.`,name:"return_dict"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/idefics2/modeling_idefics2.py#L1506",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A <code>transformers.models.idefics2.modeling_idefics2.Idefics2CausalLMOutputWithPast</code> or a tuple of
<code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various
elements depending on the configuration (<a
  href="/docs/transformers/main/en/model_doc/idefics2#transformers.Idefics2Config"
>Idefics2Config</a>) and inputs.</p>
<ul>
<li><strong>loss</strong> (<code>torch.FloatTensor</code> of shape <code>(1,)</code>, <em>optional</em>, returned when <code>labels</code> is provided) — Language modeling loss (for next-token prediction).</li>
<li><strong>logits</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, config.vocab_size)</code>) — Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).</li>
<li><strong>past_key_values</strong> (<code>tuple(tuple(torch.FloatTensor))</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) — Tuple of <code>tuple(torch.FloatTensor)</code> of length <code>config.n_layers</code>, with each tuple having 2 tensors of shape
<code>(batch_size, num_heads, sequence_length, embed_size_per_head)</code>)
Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
<code>past_key_values</code> input) to speed up sequential decoding.</li>
<li><strong>hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings, if the model has an embedding layer, +
one for the output of each layer) of shape <code>(batch_size, sequence_length, hidden_size)</code>.
Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.</li>
<li><strong>attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) — Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.</li>
<li><strong>image_hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>) — Tuple of <code>torch.FloatTensor</code> (one for the output of the image embeddings, <code>(batch_size, num_images, sequence_length, hidden_size)</code>.
image_hidden_states of the model produced by the vision encoder, and optionally by the perceiver</li>
</ul>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>transformers.models.idefics2.modeling_idefics2.Idefics2CausalLMOutputWithPast</code> or <code>tuple(torch.FloatTensor)</code></p>
`}}),P=new Ls({props:{$$slots:{default:[ro]},$$scope:{ctx:C}}}),H=new fs({props:{anchor:"transformers.Idefics2ForConditionalGeneration.forward.example",$$slots:{default:[io]},$$scope:{ctx:C}}}),Ze=new F({props:{title:"Idefics2ImageProcessor",local:"transformers.Idefics2ImageProcessor",headingTag:"h2"}}),xe=new q({props:{name:"class transformers.Idefics2ImageProcessor",anchor:"transformers.Idefics2ImageProcessor",parameters:[{name:"do_convert_rgb",val:": bool = True"},{name:"do_resize",val:": bool = True"},{name:"size",val:": Dict = None"},{name:"resample",val:": Resampling = <Resampling.BILINEAR: 2>"},{name:"do_rescale",val:": bool = True"},{name:"rescale_factor",val:": float = 0.00392156862745098"},{name:"do_normalize",val:": bool = True"},{name:"image_mean",val:": Union = None"},{name:"image_std",val:": Union = None"},{name:"do_pad",val:": bool = True"},{name:"do_image_splitting",val:": bool = False"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.Idefics2ImageProcessor.do_convert_rgb",description:`<strong>do_convert_rgb</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to convert the image to RGB. This is useful if the input image is of a different format e.g. RGBA.
Only has an effect if the input image is in the PIL format.`,name:"do_convert_rgb"},{anchor:"transformers.Idefics2ImageProcessor.do_resize",description:`<strong>do_resize</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to resize the image. The longest edge of the image is resized to  be &lt;= <code>size[&quot;longest_edge&quot;]</code>, with the
shortest edge resized to keep the input aspect ratio, with a minimum size of <code>size[&quot;shortest_edge&quot;]</code>.`,name:"do_resize"},{anchor:"transformers.Idefics2ImageProcessor.size",description:`<strong>size</strong> (<code>Dict</code>, <em>optional</em>) &#x2014;
Controls the size of the output image. This is a dictionary containing the keys &#x201C;shortest_edge&#x201D; and &#x201C;longest_edge&#x201D;.`,name:"size"},{anchor:"transformers.Idefics2ImageProcessor.resample",description:`<strong>resample</strong> (<code>Resampling</code>, <em>optional</em>, defaults to <code>Resampling.BILINEAR</code>) &#x2014;
Resampling filter to use when resizing the image.`,name:"resample"},{anchor:"transformers.Idefics2ImageProcessor.do_rescale",description:`<strong>do_rescale</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to rescale the image. If set to <code>True</code>, the image is rescaled to have pixel values between 0 and 1.`,name:"do_rescale"},{anchor:"transformers.Idefics2ImageProcessor.rescale_factor",description:`<strong>rescale_factor</strong> (<code>float</code>, <em>optional</em>, defaults to <code>1/255</code>) &#x2014;
Rescale factor to rescale the image by if <code>do_rescale</code> is set to <code>True</code>.`,name:"rescale_factor"},{anchor:"transformers.Idefics2ImageProcessor.do_normalize",description:`<strong>do_normalize</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to normalize the image. If set to <code>True</code>, the image is normalized to have a mean of <code>image_mean</code> and
a standard deviation of <code>image_std</code>.`,name:"do_normalize"},{anchor:"transformers.Idefics2ImageProcessor.image_mean",description:`<strong>image_mean</strong> (<code>float</code> or <code>List[float]</code>, <em>optional</em>, defaults to <code>IDEFICS_STANDARD_MEAN</code>) &#x2014;
Mean to use if normalizing the image. This is a float or list of floats the length of the number of
channels in the image. Can be overridden by the <code>image_mean</code> parameter in the <code>preprocess</code> method. Can be
overridden by the <code>image_mean</code> parameter in the <code>preprocess</code> method.`,name:"image_mean"},{anchor:"transformers.Idefics2ImageProcessor.image_std",description:`<strong>image_std</strong> (<code>float</code> or <code>List[float]</code>, <em>optional</em>, defaults to <code>IDEFICS_STANDARD_STD</code>) &#x2014;
Standard deviation to use if normalizing the image. This is a float or list of floats the length of the
number of channels in the image. Can be overridden by the <code>image_std</code> parameter in the <code>preprocess</code> method.
Can be overridden by the <code>image_std</code> parameter in the <code>preprocess</code> method.`,name:"image_std"},{anchor:"transformers.Idefics2ImageProcessor.do_pad",description:`<strong>do_pad</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether or not to pad the images to the largest height and width in the batch and number of images per
sample in the batch, such that the returned tensor is of shape (batch_size, max_num_images, num_channels, max_height, max_width).`,name:"do_pad"},{anchor:"transformers.Idefics2ImageProcessor.do_image_splitting",description:`<strong>do_image_splitting</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to split the image into a sequence 4 equal sub-images concatenated with the original image. That
strategy was first introduced in <a href="https://arxiv.org/abs/2311.06607" rel="nofollow">https://arxiv.org/abs/2311.06607</a>.`,name:"do_image_splitting"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/idefics2/image_processing_idefics2.py#L181"}}),We=new q({props:{name:"preprocess",anchor:"transformers.Idefics2ImageProcessor.preprocess",parameters:[{name:"images",val:": Union"},{name:"do_convert_rgb",val:": Optional = None"},{name:"do_resize",val:": Optional = None"},{name:"size",val:": Optional = None"},{name:"resample",val:": Resampling = None"},{name:"do_rescale",val:": Optional = None"},{name:"rescale_factor",val:": Optional = None"},{name:"do_normalize",val:": Optional = None"},{name:"image_mean",val:": Union = None"},{name:"image_std",val:": Union = None"},{name:"do_pad",val:": Optional = None"},{name:"do_image_splitting",val:": Optional = None"},{name:"return_tensors",val:": Union = None"},{name:"input_data_format",val:": Optional = None"},{name:"data_format",val:": Optional = <ChannelDimension.FIRST: 'channels_first'>"}],parametersDescription:[{anchor:"transformers.Idefics2ImageProcessor.preprocess.images",description:`<strong>images</strong> (<code>ImageInput</code>) &#x2014;
A list of images to preprocess.`,name:"images"},{anchor:"transformers.Idefics2ImageProcessor.preprocess.do_convert_rgb",description:`<strong>do_convert_rgb</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>self.do_convert_rgb</code>) &#x2014;
Whether to convert the image to RGB.`,name:"do_convert_rgb"},{anchor:"transformers.Idefics2ImageProcessor.preprocess.do_resize",description:`<strong>do_resize</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>self.do_resize</code>) &#x2014;
Whether to resize the image.`,name:"do_resize"},{anchor:"transformers.Idefics2ImageProcessor.preprocess.size",description:`<strong>size</strong> (<code>Dict[str, int]</code>, <em>optional</em>, defaults to <code>self.size</code>) &#x2014;
Size of the image after resizing. Shortest edge of the image is resized to size[&#x201C;shortest_edge&#x201D;], with
the longest edge resized to keep the input aspect ratio.`,name:"size"},{anchor:"transformers.Idefics2ImageProcessor.preprocess.resample",description:`<strong>resample</strong> (<code>int</code>, <em>optional</em>, defaults to <code>self.resample</code>) &#x2014;
Resampling filter to use if resizing the image. This can be one of the enum <code>PILImageResampling</code>. Only
has an effect if <code>do_resize</code> is set to <code>True</code>.`,name:"resample"},{anchor:"transformers.Idefics2ImageProcessor.preprocess.do_rescale",description:`<strong>do_rescale</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>self.do_rescale</code>) &#x2014;
Whether to rescale the image.`,name:"do_rescale"},{anchor:"transformers.Idefics2ImageProcessor.preprocess.rescale_factor",description:`<strong>rescale_factor</strong> (<code>float</code>, <em>optional</em>, defaults to <code>self.rescale_factor</code>) &#x2014;
Rescale factor to rescale the image by if <code>do_rescale</code> is set to <code>True</code>.`,name:"rescale_factor"},{anchor:"transformers.Idefics2ImageProcessor.preprocess.do_normalize",description:`<strong>do_normalize</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>self.do_normalize</code>) &#x2014;
Whether to normalize the image.`,name:"do_normalize"},{anchor:"transformers.Idefics2ImageProcessor.preprocess.image_mean",description:`<strong>image_mean</strong> (<code>float</code> or <code>List[float]</code>, <em>optional</em>, defaults to <code>self.image_mean</code>) &#x2014;
Image mean to use for normalization. Only has an effect if <code>do_normalize</code> is set to <code>True</code>.`,name:"image_mean"},{anchor:"transformers.Idefics2ImageProcessor.preprocess.image_std",description:`<strong>image_std</strong> (<code>float</code> or <code>List[float]</code>, <em>optional</em>, defaults to <code>self.image_std</code>) &#x2014;
Image standard deviation to use for normalization. Only has an effect if <code>do_normalize</code> is set to
<code>True</code>.`,name:"image_std"},{anchor:"transformers.Idefics2ImageProcessor.preprocess.do_pad",description:`<strong>do_pad</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>self.do_pad</code>) &#x2014;
Whether or not to pad the images to the largest height and width in the batch.`,name:"do_pad"},{anchor:"transformers.Idefics2ImageProcessor.preprocess.do_image_splitting",description:`<strong>do_image_splitting</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>self.do_image_splitting</code>) &#x2014;
Whether to split the image into a sequence 4 equal sub-images concatenated with the original image. That
strategy was first introduced in <a href="https://arxiv.org/abs/2311.06607" rel="nofollow">https://arxiv.org/abs/2311.06607</a>.`,name:"do_image_splitting"},{anchor:"transformers.Idefics2ImageProcessor.preprocess.return_tensors",description:`<strong>return_tensors</strong> (<code>str</code> or <code>TensorType</code>, <em>optional</em>) &#x2014;
The type of tensors to return. Can be one of:<ul>
<li>Unset: Return a list of <code>np.ndarray</code>.</li>
<li><code>TensorType.TENSORFLOW</code> or <code>&apos;tf&apos;</code>: Return a batch of type <code>tf.Tensor</code>.</li>
<li><code>TensorType.PYTORCH</code> or <code>&apos;pt&apos;</code>: Return a batch of type <code>torch.Tensor</code>.</li>
<li><code>TensorType.NUMPY</code> or <code>&apos;np&apos;</code>: Return a batch of type <code>np.ndarray</code>.</li>
<li><code>TensorType.JAX</code> or <code>&apos;jax&apos;</code>: Return a batch of type <code>jax.numpy.ndarray</code>.</li>
</ul>`,name:"return_tensors"},{anchor:"transformers.Idefics2ImageProcessor.preprocess.data_format",description:`<strong>data_format</strong> (<code>ChannelDimension</code> or <code>str</code>, <em>optional</em>, defaults to <code>ChannelDimension.FIRST</code>) &#x2014;
The channel dimension format for the output image. Can be one of:<ul>
<li><code>&quot;channels_first&quot;</code> or <code>ChannelDimension.FIRST</code>: image in (num_channels, height, width) format.</li>
<li><code>&quot;channels_last&quot;</code> or <code>ChannelDimension.LAST</code>: image in (height, width, num_channels) format.</li>
<li>Unset: Use the channel dimension format of the input image.</li>
</ul>`,name:"data_format"},{anchor:"transformers.Idefics2ImageProcessor.preprocess.input_data_format",description:`<strong>input_data_format</strong> (<code>ChannelDimension</code> or <code>str</code>, <em>optional</em>) &#x2014;
The channel dimension format for the input image. If unset, the channel dimension format is inferred
from the input image. Can be one of:<ul>
<li><code>&quot;channels_first&quot;</code> or <code>ChannelDimension.FIRST</code>: image in (num_channels, height, width) format.</li>
<li><code>&quot;channels_last&quot;</code> or <code>ChannelDimension.LAST</code>: image in (height, width, num_channels) format.</li>
<li><code>&quot;none&quot;</code> or <code>ChannelDimension.NONE</code>: image in (height, width) format.</li>
</ul>`,name:"input_data_format"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/idefics2/image_processing_idefics2.py#L425"}}),$e=new F({props:{title:"Idefics2Processor",local:"transformers.Idefics2Processor",headingTag:"h2"}}),Ne=new q({props:{name:"class transformers.Idefics2Processor",anchor:"transformers.Idefics2Processor",parameters:[{name:"image_processor",val:""},{name:"tokenizer",val:" = None"},{name:"image_seq_len",val:": int = 64"},{name:"chat_template",val:": str = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.Idefics2Processor.image_processor",description:`<strong>image_processor</strong> (<code>Idefics2ImageProcessor</code>) &#x2014;
An instance of <a href="/docs/transformers/main/en/model_doc/idefics2#transformers.Idefics2ImageProcessor">Idefics2ImageProcessor</a>. The image processor is a required input.`,name:"image_processor"},{anchor:"transformers.Idefics2Processor.tokenizer",description:`<strong>tokenizer</strong> (<code>PreTrainedTokenizerBase</code>, <em>optional</em>) &#x2014;
An instance of <a href="/docs/transformers/main/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase">PreTrainedTokenizerBase</a>. This should correspond with the model&#x2019;s text model. The tokenizer is a required input.`,name:"tokenizer"},{anchor:"transformers.Idefics2Processor.image_seq_len",description:`<strong>image_seq_len</strong> (<code>int</code>, <em>optional</em>, defaults to 64) &#x2014;
The length of the image sequence i.e. the number of <image> tokens per image in the input.
This parameter is used to build the string from the input prompt and image tokens and should match the
config.perceiver_config.resampler_n_latents value for the model used.</image>`,name:"image_seq_len"},{anchor:"transformers.Idefics2Processor.chat_template",description:`<strong>chat_template</strong> (<code>str</code>, <em>optional</em>) &#x2014; A Jinja template which will be used to convert lists of messages
in a chat into a tokenizable string.`,name:"chat_template"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/idefics2/processing_idefics2.py#L43"}}),Re=new q({props:{name:"__call__",anchor:"transformers.Idefics2Processor.__call__",parameters:[{name:"text",val:": Union = None"},{name:"images",val:": Union = None"},{name:"image_seq_len",val:": Optional = None"},{name:"padding",val:": Union = False"},{name:"truncation",val:": Union = None"},{name:"max_length",val:": Optional = None"},{name:"is_split_into_words",val:": bool = False"},{name:"add_special_tokens",val:": bool = True"},{name:"return_tensors",val:": Union = None"}],parametersDescription:[{anchor:"transformers.Idefics2Processor.__call__.text",description:`<strong>text</strong> (<code>Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]]</code>, <em>optional</em>) &#x2014;
The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
(pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
<code>is_split_into_words=True</code> (to lift the ambiguity with a batch of sequences).</p>
<p>Wherever an image token, <code>&lt;image&gt;</code> is encountered it is expanded to
<code>&lt;fake_token_around_image&gt;</code> + <code>&lt;image&gt;</code> <em> <code>image_seq_len</code> </em> <fake_token_around_image>\`.</fake_token_around_image>`,name:"text"},{anchor:"transformers.Idefics2Processor.__call__.images",description:`<strong>images</strong> (<code>PIL.Image.Image</code>, <code>np.ndarray</code>, <code>torch.Tensor</code>, <code>List[PIL.Image.Image]</code>, <code>List[np.ndarray]</code>, <code>List[torch.Tensor]</code>, <em>optional</em>) &#x2014;
The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch
tensor. If is of type <code>List[ImageInput]</code>, it&#x2019;s assumed that this is for a single prompt i.e. of batch size 1.`,name:"images"},{anchor:"transformers.Idefics2Processor.__call__.image_seq_len",description:`<strong>image_seq_len</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The length of the image sequence. If not provided, the default value is used.`,name:"image_seq_len"},{anchor:"transformers.Idefics2Processor.__call__.padding",description:`<strong>padding</strong> (<code>Union[bool, str, PaddingStrategy]</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Padding strategy applied to the input ids. See <a href="/docs/transformers/main/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.pad">PreTrainedTokenizerFast.pad()</a> for more information.`,name:"padding"},{anchor:"transformers.Idefics2Processor.__call__.truncation",description:`<strong>truncation</strong> (<code>Union[bool, str, TruncationStrategy]</code>, <em>optional</em>) &#x2014;
Truncation strategy applied to the input ids. See <code>PreTrainedTokenizerFast.truncate</code> for more information.`,name:"truncation"},{anchor:"transformers.Idefics2Processor.__call__.max_length",description:`<strong>max_length</strong> (<code>int</code>, <em>optional</em>) &#x2014;
Maximum length of the returned list and optionally padding/truncation length. See
<a href="/docs/transformers/main/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizerFast.<strong>call</strong>()</a> for more information.`,name:"max_length"},{anchor:"transformers.Idefics2Processor.__call__.is_split_into_words",description:`<strong>is_split_into_words</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether the input text is split into words or not. If set to <code>True</code>, the tokenizer will skip the
tokenization process and assume the input is already tokenized.`,name:"is_split_into_words"},{anchor:"transformers.Idefics2Processor.__call__.add_special_tokens",description:`<strong>add_special_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to add special tokens or not. See <a href="/docs/transformers/main/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizerFast.<strong>call</strong>()</a> for more information.`,name:"add_special_tokens"},{anchor:"transformers.Idefics2Processor.__call__.return_tensors",description:`<strong>return_tensors</strong> (<code>Union[str, TensorType]</code>, <em>optional</em>) &#x2014;
If set, will return tensors of a particular framework. See <a href="/docs/transformers/main/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizerFast.<strong>call</strong>()</a> for more
information.`,name:"return_tensors"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/idefics2/processing_idefics2.py#L98"}}),Y=new fs({props:{anchor:"transformers.Idefics2Processor.__call__.example",$$slots:{default:[lo]},$$scope:{ctx:C}}}),Be=new oo({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/model_doc/idefics2.md"}}),{c(){l=r("meta"),y=n(),M=r("p"),T=n(),p(b.$$.fragment),d=n(),p(I.$$.fragment),ot=n(),L=r("p"),L.innerHTML=us,nt=n(),D=r("p"),D.textContent=_s,at=n(),O=r("p"),O.textContent=Ms,rt=n(),K=r("p"),K.innerHTML=Ts,it=n(),V=r("img"),lt=n(),ee=r("small"),ee.innerHTML=ys,dt=n(),te=r("p"),te.innerHTML=Is,ct=n(),p(se.$$.fragment),mt=n(),oe=r("ul"),oe.innerHTML=ws,pt=n(),ne=r("p"),ne.textContent=Js,ht=n(),p(ae.$$.fragment),gt=n(),re=r("ul"),re.innerHTML=vs,ft=n(),p(ie.$$.fragment),ut=n(),le=r("p"),le.textContent=js,_t=n(),p(de.$$.fragment),Mt=n(),ce=r("p"),ce.innerHTML=Us,Tt=n(),me=r("p"),me.textContent=ks,bt=n(),p(pe.$$.fragment),yt=n(),he=r("p"),he.innerHTML=zs,It=n(),ge=r("p"),ge.textContent=Cs,wt=n(),p(fe.$$.fragment),Jt=n(),p(ue.$$.fragment),vt=n(),_e=r("p"),_e.innerHTML=Zs,jt=n(),Me=r("p"),Me.innerHTML=xs,Ut=n(),p(Te.$$.fragment),kt=n(),p(be.$$.fragment),zt=n(),ye=r("p"),ye.textContent=Ws,Ct=n(),Ie=r("ul"),Ie.innerHTML=$s,Zt=n(),p(we.$$.fragment),xt=n(),v=r("div"),p(Je.$$.fragment),Ht=n(),Fe=r("p"),Fe.innerHTML=Ns,St=n(),Ge=r("p"),Ge.innerHTML=Rs,Yt=n(),p(E.$$.fragment),Wt=n(),p(ve.$$.fragment),$t=n(),j=r("div"),p(je.$$.fragment),Qt=n(),qe=r("p"),qe.innerHTML=Bs,At=n(),Ve=r("p"),Ve.innerHTML=Fs,Lt=n(),w=r("div"),p(Ue.$$.fragment),Dt=n(),Ee=r("p"),Ee.innerHTML=Gs,Ot=n(),p(X.$$.fragment),Kt=n(),Xe=r("p"),Xe.textContent=qs,es=n(),Pe=r("p"),Pe.textContent=Vs,Nt=n(),p(ke.$$.fragment),Rt=n(),U=r("div"),p(ze.$$.fragment),ts=n(),He=r("p"),He.innerHTML=Es,ss=n(),Se=r("p"),Se.innerHTML=Xs,os=n(),Z=r("div"),p(Ce.$$.fragment),ns=n(),Ye=r("p"),Ye.innerHTML=Ps,as=n(),p(P.$$.fragment),rs=n(),p(H.$$.fragment),Bt=n(),p(Ze.$$.fragment),Ft=n(),x=r("div"),p(xe.$$.fragment),is=n(),Qe=r("p"),Qe.textContent=Hs,ls=n(),S=r("div"),p(We.$$.fragment),ds=n(),Ae=r("p"),Ae.textContent=Ss,Gt=n(),p($e.$$.fragment),qt=n(),k=r("div"),p(Ne.$$.fragment),cs=n(),Le=r("p"),Le.textContent=Ys,ms=n(),De=r("p"),De.innerHTML=Qs,ps=n(),G=r("div"),p(Re.$$.fragment),hs=n(),Oe=r("p"),Oe.textContent=As,gs=n(),p(Y.$$.fragment),Vt=n(),p(Be.$$.fragment),Et=n(),tt=r("p"),this.h()},l(e){const t=so("svelte-u9bgzb",document.head);l=i(t,"META",{name:!0,content:!0}),t.forEach(s),y=a(e),M=i(e,"P",{}),B(M).forEach(s),T=a(e),h(b.$$.fragment,e),d=a(e),h(I.$$.fragment,e),ot=a(e),L=i(e,"P",{"data-svelte-h":!0}),m(L)!=="svelte-2yybt7"&&(L.innerHTML=us),nt=a(e),D=i(e,"P",{"data-svelte-h":!0}),m(D)!=="svelte-5h9xv8"&&(D.textContent=_s),at=a(e),O=i(e,"P",{"data-svelte-h":!0}),m(O)!=="svelte-vfdo9a"&&(O.textContent=Ms),rt=a(e),K=i(e,"P",{"data-svelte-h":!0}),m(K)!=="svelte-1d8hoqd"&&(K.innerHTML=Ts),it=a(e),V=i(e,"IMG",{src:!0,alt:!0,width:!0}),lt=a(e),ee=i(e,"SMALL",{"data-svelte-h":!0}),m(ee)!=="svelte-1ry3ulw"&&(ee.innerHTML=ys),dt=a(e),te=i(e,"P",{"data-svelte-h":!0}),m(te)!=="svelte-yuhhqz"&&(te.innerHTML=Is),ct=a(e),h(se.$$.fragment,e),mt=a(e),oe=i(e,"UL",{"data-svelte-h":!0}),m(oe)!=="svelte-ejgnpl"&&(oe.innerHTML=ws),pt=a(e),ne=i(e,"P",{"data-svelte-h":!0}),m(ne)!=="svelte-hckg3t"&&(ne.textContent=Js),ht=a(e),h(ae.$$.fragment,e),gt=a(e),re=i(e,"UL",{"data-svelte-h":!0}),m(re)!=="svelte-18iwa30"&&(re.innerHTML=vs),ft=a(e),h(ie.$$.fragment,e),ut=a(e),le=i(e,"P",{"data-svelte-h":!0}),m(le)!=="svelte-7z8spz"&&(le.textContent=js),_t=a(e),h(de.$$.fragment,e),Mt=a(e),ce=i(e,"P",{"data-svelte-h":!0}),m(ce)!=="svelte-1sb6n2y"&&(ce.innerHTML=Us),Tt=a(e),me=i(e,"P",{"data-svelte-h":!0}),m(me)!=="svelte-o3pzzu"&&(me.textContent=ks),bt=a(e),h(pe.$$.fragment,e),yt=a(e),he=i(e,"P",{"data-svelte-h":!0}),m(he)!=="svelte-1ehkek3"&&(he.innerHTML=zs),It=a(e),ge=i(e,"P",{"data-svelte-h":!0}),m(ge)!=="svelte-1pidlee"&&(ge.textContent=Cs),wt=a(e),h(fe.$$.fragment,e),Jt=a(e),h(ue.$$.fragment,e),vt=a(e),_e=i(e,"P",{"data-svelte-h":!0}),m(_e)!=="svelte-1ai4ii"&&(_e.innerHTML=Zs),jt=a(e),Me=i(e,"P",{"data-svelte-h":!0}),m(Me)!=="svelte-mp60zk"&&(Me.innerHTML=xs),Ut=a(e),h(Te.$$.fragment,e),kt=a(e),h(be.$$.fragment,e),zt=a(e),ye=i(e,"P",{"data-svelte-h":!0}),m(ye)!=="svelte-ig9dca"&&(ye.textContent=Ws),Ct=a(e),Ie=i(e,"UL",{"data-svelte-h":!0}),m(Ie)!=="svelte-1sz5atq"&&(Ie.innerHTML=$s),Zt=a(e),h(we.$$.fragment,e),xt=a(e),v=i(e,"DIV",{class:!0});var W=B(v);h(Je.$$.fragment,W),Ht=a(W),Fe=i(W,"P",{"data-svelte-h":!0}),m(Fe)!=="svelte-1ngts0f"&&(Fe.innerHTML=Ns),St=a(W),Ge=i(W,"P",{"data-svelte-h":!0}),m(Ge)!=="svelte-o55m63"&&(Ge.innerHTML=Rs),Yt=a(W),h(E.$$.fragment,W),W.forEach(s),Wt=a(e),h(ve.$$.fragment,e),$t=a(e),j=i(e,"DIV",{class:!0});var $=B(j);h(je.$$.fragment,$),Qt=a($),qe=i($,"P",{"data-svelte-h":!0}),m(qe)!=="svelte-1ybhnwc"&&(qe.innerHTML=Bs),At=a($),Ve=i($,"P",{"data-svelte-h":!0}),m(Ve)!=="svelte-hswkmf"&&(Ve.innerHTML=Fs),Lt=a($),w=i($,"DIV",{class:!0});var z=B(w);h(Ue.$$.fragment,z),Dt=a(z),Ee=i(z,"P",{"data-svelte-h":!0}),m(Ee)!=="svelte-1ljsomm"&&(Ee.innerHTML=Gs),Ot=a(z),h(X.$$.fragment,z),Kt=a(z),Xe=i(z,"P",{"data-svelte-h":!0}),m(Xe)!=="svelte-felb7u"&&(Xe.textContent=qs),es=a(z),Pe=i(z,"P",{"data-svelte-h":!0}),m(Pe)!=="svelte-58ee4k"&&(Pe.textContent=Vs),z.forEach(s),$.forEach(s),Nt=a(e),h(ke.$$.fragment,e),Rt=a(e),U=i(e,"DIV",{class:!0});var N=B(U);h(ze.$$.fragment,N),ts=a(N),He=i(N,"P",{"data-svelte-h":!0}),m(He)!=="svelte-hgurjl"&&(He.innerHTML=Es),ss=a(N),Se=i(N,"P",{"data-svelte-h":!0}),m(Se)!=="svelte-hswkmf"&&(Se.innerHTML=Xs),os=a(N),Z=i(N,"DIV",{class:!0});var R=B(Z);h(Ce.$$.fragment,R),ns=a(R),Ye=i(R,"P",{"data-svelte-h":!0}),m(Ye)!=="svelte-sq5c5e"&&(Ye.innerHTML=Ps),as=a(R),h(P.$$.fragment,R),rs=a(R),h(H.$$.fragment,R),R.forEach(s),N.forEach(s),Bt=a(e),h(Ze.$$.fragment,e),Ft=a(e),x=i(e,"DIV",{class:!0});var Ke=B(x);h(xe.$$.fragment,Ke),is=a(Ke),Qe=i(Ke,"P",{"data-svelte-h":!0}),m(Qe)!=="svelte-1m1ucrf"&&(Qe.textContent=Hs),ls=a(Ke),S=i(Ke,"DIV",{class:!0});var Pt=B(S);h(We.$$.fragment,Pt),ds=a(Pt),Ae=i(Pt,"P",{"data-svelte-h":!0}),m(Ae)!=="svelte-1xds4wy"&&(Ae.textContent=Ss),Pt.forEach(s),Ke.forEach(s),Gt=a(e),h($e.$$.fragment,e),qt=a(e),k=i(e,"DIV",{class:!0});var Q=B(k);h(Ne.$$.fragment,Q),cs=a(Q),Le=i(Q,"P",{"data-svelte-h":!0}),m(Le)!=="svelte-1jys543"&&(Le.textContent=Ys),ms=a(Q),De=i(Q,"P",{"data-svelte-h":!0}),m(De)!=="svelte-1efqgl8"&&(De.innerHTML=Qs),ps=a(Q),G=i(Q,"DIV",{class:!0});var et=B(G);h(Re.$$.fragment,et),hs=a(et),Oe=i(et,"P",{"data-svelte-h":!0}),m(Oe)!=="svelte-421t2f"&&(Oe.textContent=As),gs=a(et),h(Y.$$.fragment,et),et.forEach(s),Q.forEach(s),Vt=a(e),h(Be.$$.fragment,e),Et=a(e),tt=i(e,"P",{}),B(tt).forEach(s),this.h()},h(){J(l,"name","hf:doc:metadata"),J(l,"content",mo),Os(V.src,bs="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/model_doc/idefics2_architecture.png")||J(V,"src",bs),J(V,"alt","drawing"),J(V,"width","600"),J(v,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),J(w,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),J(j,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),J(Z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),J(U,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),J(S,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),J(x,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),J(G,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),J(k,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,t){c(document.head,l),o(e,y,t),o(e,M,t),o(e,T,t),g(b,e,t),o(e,d,t),g(I,e,t),o(e,ot,t),o(e,L,t),o(e,nt,t),o(e,D,t),o(e,at,t),o(e,O,t),o(e,rt,t),o(e,K,t),o(e,it,t),o(e,V,t),o(e,lt,t),o(e,ee,t),o(e,dt,t),o(e,te,t),o(e,ct,t),g(se,e,t),o(e,mt,t),o(e,oe,t),o(e,pt,t),o(e,ne,t),o(e,ht,t),g(ae,e,t),o(e,gt,t),o(e,re,t),o(e,ft,t),g(ie,e,t),o(e,ut,t),o(e,le,t),o(e,_t,t),g(de,e,t),o(e,Mt,t),o(e,ce,t),o(e,Tt,t),o(e,me,t),o(e,bt,t),g(pe,e,t),o(e,yt,t),o(e,he,t),o(e,It,t),o(e,ge,t),o(e,wt,t),g(fe,e,t),o(e,Jt,t),g(ue,e,t),o(e,vt,t),o(e,_e,t),o(e,jt,t),o(e,Me,t),o(e,Ut,t),g(Te,e,t),o(e,kt,t),g(be,e,t),o(e,zt,t),o(e,ye,t),o(e,Ct,t),o(e,Ie,t),o(e,Zt,t),g(we,e,t),o(e,xt,t),o(e,v,t),g(Je,v,null),c(v,Ht),c(v,Fe),c(v,St),c(v,Ge),c(v,Yt),g(E,v,null),o(e,Wt,t),g(ve,e,t),o(e,$t,t),o(e,j,t),g(je,j,null),c(j,Qt),c(j,qe),c(j,At),c(j,Ve),c(j,Lt),c(j,w),g(Ue,w,null),c(w,Dt),c(w,Ee),c(w,Ot),g(X,w,null),c(w,Kt),c(w,Xe),c(w,es),c(w,Pe),o(e,Nt,t),g(ke,e,t),o(e,Rt,t),o(e,U,t),g(ze,U,null),c(U,ts),c(U,He),c(U,ss),c(U,Se),c(U,os),c(U,Z),g(Ce,Z,null),c(Z,ns),c(Z,Ye),c(Z,as),g(P,Z,null),c(Z,rs),g(H,Z,null),o(e,Bt,t),g(Ze,e,t),o(e,Ft,t),o(e,x,t),g(xe,x,null),c(x,is),c(x,Qe),c(x,ls),c(x,S),g(We,S,null),c(S,ds),c(S,Ae),o(e,Gt,t),g($e,e,t),o(e,qt,t),o(e,k,t),g(Ne,k,null),c(k,cs),c(k,Le),c(k,ms),c(k,De),c(k,ps),c(k,G),g(Re,G,null),c(G,hs),c(G,Oe),c(G,gs),g(Y,G,null),o(e,Vt,t),g(Be,e,t),o(e,Et,t),o(e,tt,t),Xt=!0},p(e,[t]){const W={};t&2&&(W.$$scope={dirty:t,ctx:e}),E.$set(W);const $={};t&2&&($.$$scope={dirty:t,ctx:e}),X.$set($);const z={};t&2&&(z.$$scope={dirty:t,ctx:e}),P.$set(z);const N={};t&2&&(N.$$scope={dirty:t,ctx:e}),H.$set(N);const R={};t&2&&(R.$$scope={dirty:t,ctx:e}),Y.$set(R)},i(e){Xt||(f(b.$$.fragment,e),f(I.$$.fragment,e),f(se.$$.fragment,e),f(ae.$$.fragment,e),f(ie.$$.fragment,e),f(de.$$.fragment,e),f(pe.$$.fragment,e),f(fe.$$.fragment,e),f(ue.$$.fragment,e),f(Te.$$.fragment,e),f(be.$$.fragment,e),f(we.$$.fragment,e),f(Je.$$.fragment,e),f(E.$$.fragment,e),f(ve.$$.fragment,e),f(je.$$.fragment,e),f(Ue.$$.fragment,e),f(X.$$.fragment,e),f(ke.$$.fragment,e),f(ze.$$.fragment,e),f(Ce.$$.fragment,e),f(P.$$.fragment,e),f(H.$$.fragment,e),f(Ze.$$.fragment,e),f(xe.$$.fragment,e),f(We.$$.fragment,e),f($e.$$.fragment,e),f(Ne.$$.fragment,e),f(Re.$$.fragment,e),f(Y.$$.fragment,e),f(Be.$$.fragment,e),Xt=!0)},o(e){u(b.$$.fragment,e),u(I.$$.fragment,e),u(se.$$.fragment,e),u(ae.$$.fragment,e),u(ie.$$.fragment,e),u(de.$$.fragment,e),u(pe.$$.fragment,e),u(fe.$$.fragment,e),u(ue.$$.fragment,e),u(Te.$$.fragment,e),u(be.$$.fragment,e),u(we.$$.fragment,e),u(Je.$$.fragment,e),u(E.$$.fragment,e),u(ve.$$.fragment,e),u(je.$$.fragment,e),u(Ue.$$.fragment,e),u(X.$$.fragment,e),u(ke.$$.fragment,e),u(ze.$$.fragment,e),u(Ce.$$.fragment,e),u(P.$$.fragment,e),u(H.$$.fragment,e),u(Ze.$$.fragment,e),u(xe.$$.fragment,e),u(We.$$.fragment,e),u($e.$$.fragment,e),u(Ne.$$.fragment,e),u(Re.$$.fragment,e),u(Y.$$.fragment,e),u(Be.$$.fragment,e),Xt=!1},d(e){e&&(s(y),s(M),s(T),s(d),s(ot),s(L),s(nt),s(D),s(at),s(O),s(rt),s(K),s(it),s(V),s(lt),s(ee),s(dt),s(te),s(ct),s(mt),s(oe),s(pt),s(ne),s(ht),s(gt),s(re),s(ft),s(ut),s(le),s(_t),s(Mt),s(ce),s(Tt),s(me),s(bt),s(yt),s(he),s(It),s(ge),s(wt),s(Jt),s(vt),s(_e),s(jt),s(Me),s(Ut),s(kt),s(zt),s(ye),s(Ct),s(Ie),s(Zt),s(xt),s(v),s(Wt),s($t),s(j),s(Nt),s(Rt),s(U),s(Bt),s(Ft),s(x),s(Gt),s(qt),s(k),s(Vt),s(Et),s(tt)),s(l),_(b,e),_(I,e),_(se,e),_(ae,e),_(ie,e),_(de,e),_(pe,e),_(fe,e),_(ue,e),_(Te,e),_(be,e),_(we,e),_(Je),_(E),_(ve,e),_(je),_(Ue),_(X),_(ke,e),_(ze),_(Ce),_(P),_(H),_(Ze,e),_(xe),_(We),_($e,e),_(Ne),_(Re),_(Y),_(Be,e)}}}const mo='{"title":"Idefics2","local":"idefics2","sections":[{"title":"Overview","local":"overview","sections":[],"depth":2},{"title":"Usage tips","local":"usage-tips","sections":[],"depth":2},{"title":"Model optimizations: Flash Attention","local":"model-optimizations-flash-attention","sections":[],"depth":2},{"title":"Shrinking down Idefics2 using quantization","local":"shrinking-down-idefics2-using-quantization","sections":[],"depth":2},{"title":"Resources","local":"resources","sections":[],"depth":2},{"title":"Idefics2Config","local":"transformers.Idefics2Config","sections":[],"depth":2},{"title":"Idefics2Model","local":"transformers.Idefics2Model","sections":[],"depth":2},{"title":"Idefics2ForConditionalGeneration","local":"transformers.Idefics2ForConditionalGeneration","sections":[],"depth":2},{"title":"Idefics2ImageProcessor","local":"transformers.Idefics2ImageProcessor","sections":[],"depth":2},{"title":"Idefics2Processor","local":"transformers.Idefics2Processor","sections":[],"depth":2}],"depth":1}';function po(C){return Ks(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class bo extends eo{constructor(l){super(),to(this,l,po,co,Ds,{})}}export{bo as component};
