import{s as me,o as ce,n as ue}from"../chunks/scheduler.25b97de1.js";import{S as de,i as he,g as r,s as l,r as c,A as fe,h as p,f as n,c as i,j as oe,u,x as M,k as re,y as ge,a as s,v as d,d as h,t as f,w as g}from"../chunks/index.d9030fc9.js";import{T as Me}from"../chunks/Tip.baa67368.js";import{C as pe}from"../chunks/CodeBlock.e6cd0d95.js";import{H as K,E as be}from"../chunks/EditOnGithub.91d95064.js";function we(C){let a,b='DePlot is a model trained using <code>Pix2Struct</code> architecture. For API reference, see <a href="pix2struct"><code>Pix2Struct</code> documentation</a>.';return{c(){a=r("p"),a.innerHTML=b},l(o){a=p(o,"P",{"data-svelte-h":!0}),M(a)!=="svelte-zgoioq"&&(a.innerHTML=b)},m(o,W){s(o,a,W)},p:ue,d(o){o&&n(a)}}}function ye(C){let a,b,o,W,w,G,y,X,T,O='DePlot was proposed in the paper <a href="https://arxiv.org/abs/2212.10505" rel="nofollow">DePlot: One-shot visual language reasoning by plot-to-table translation</a> from Fangyu Liu, Julian Martin Eisenschlos, Francesco Piccinno, Syrine Krichene, Chenxi Pang, Kenton Lee, Mandar Joshi, Wenhu Chen, Nigel Collier, Yasemin Altun.',V,$,ee="The abstract of the paper states the following:",Y,J,te="<em>Visual language such as charts and plots is ubiquitous in the human world. Comprehending plots and charts requires strong reasoning skills. Prior state-of-the-art (SOTA) models require at least tens of thousands of training examples and their reasoning capabilities are still much limited, especially on complex human-written queries. This paper presents the first one-shot solution to visual language reasoning. We decompose the challenge of visual language reasoning into two steps: (1) plot-to-text translation, and (2) reasoning over the translated text. The key in this method is a modality conversion module, named as DePlot, which translates the image of a plot or chart to a linearized table. The output of DePlot can then be directly used to prompt a pretrained large language model (LLM), exploiting the few-shot reasoning capabilities of LLMs. To obtain DePlot, we standardize the plot-to-table task by establishing unified task formats and metrics, and train DePlot end-to-end on this task. DePlot can then be used off-the-shelf together with LLMs in a plug-and-play fashion. Compared with a SOTA model finetuned on more than &gt;28k data points, DePlot+LLM with just one-shot prompting achieves a 24.0% improvement over finetuned SOTA on human-written queries from the task of chart QA.</em>",H,v,ne=`DePlot is a model that is trained using <code>Pix2Struct</code> architecture. You can find more information about <code>Pix2Struct</code> in the <a href="https://huggingface.co/docs/transformers/main/en/model_doc/pix2struct" rel="nofollow">Pix2Struct documentation</a>.
DePlot is a Visual Question Answering subset of <code>Pix2Struct</code> architecture. It renders the input question on the image and predicts the answer.`,I,j,z,_,se="Currently one checkpoint is available for DePlot:",F,k,ae="<li><code>google/deplot</code>: DePlot fine-tuned on ChartQA dataset</li>",E,x,D,R,L,U,le='To fine-tune DePlot, refer to the pix2struct <a href="https://github.com/huggingface/notebooks/blob/main/examples/image_captioning_pix2struct.ipynb" rel="nofollow">fine-tuning notebook</a>. For <code>Pix2Struct</code> models, we have found out that fine-tuning the model with Adafactor and cosine learning rate scheduler leads to faster convergence:',S,Z,N,m,A,P,Q,B,q;return w=new K({props:{title:"DePlot",local:"deplot",headingTag:"h1"}}),y=new K({props:{title:"Overview",local:"overview",headingTag:"h2"}}),j=new K({props:{title:"Usage example",local:"usage-example",headingTag:"h2"}}),x=new pe({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Qcm9jZXNzb3IlMkMlMjBQaXgyU3RydWN0Rm9yQ29uZGl0aW9uYWxHZW5lcmF0aW9uJTBBaW1wb3J0JTIwcmVxdWVzdHMlMEFmcm9tJTIwUElMJTIwaW1wb3J0JTIwSW1hZ2UlMEElMEFtb2RlbCUyMCUzRCUyMFBpeDJTdHJ1Y3RGb3JDb25kaXRpb25hbEdlbmVyYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZSUyRmRlcGxvdCUyMiklMEFwcm9jZXNzb3IlMjAlM0QlMjBBdXRvUHJvY2Vzc29yLmZyb21fcHJldHJhaW5lZCglMjJnb29nbGUlMkZkZXBsb3QlMjIpJTBBdXJsJTIwJTNEJTIwJTIyaHR0cHMlM0ElMkYlMkZyYXcuZ2l0aHVidXNlcmNvbnRlbnQuY29tJTJGdmlzLW5scCUyRkNoYXJ0UUElMkZtYWluJTJGQ2hhcnRRQSUyNTIwRGF0YXNldCUyRnZhbCUyRnBuZyUyRjUwOTAucG5nJTIyJTBBaW1hZ2UlMjAlM0QlMjBJbWFnZS5vcGVuKHJlcXVlc3RzLmdldCh1cmwlMkMlMjBzdHJlYW0lM0RUcnVlKS5yYXcpJTBBJTBBaW5wdXRzJTIwJTNEJTIwcHJvY2Vzc29yKGltYWdlcyUzRGltYWdlJTJDJTIwdGV4dCUzRCUyMkdlbmVyYXRlJTIwdW5kZXJseWluZyUyMGRhdGElMjB0YWJsZSUyMG9mJTIwdGhlJTIwZmlndXJlJTIwYmVsb3clM0ElMjIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSUwQXByZWRpY3Rpb25zJTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDUxMiklMEFwcmludChwcm9jZXNzb3IuZGVjb2RlKHByZWRpY3Rpb25zJTVCMCU1RCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor, Pix2StructForConditionalGeneration
<span class="hljs-keyword">import</span> requests
<span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image

model = Pix2StructForConditionalGeneration.from_pretrained(<span class="hljs-string">&quot;google/deplot&quot;</span>)
processor = AutoProcessor.from_pretrained(<span class="hljs-string">&quot;google/deplot&quot;</span>)
url = <span class="hljs-string">&quot;https://raw.githubusercontent.com/vis-nlp/ChartQA/main/ChartQA%20Dataset/val/png/5090.png&quot;</span>
image = Image.<span class="hljs-built_in">open</span>(requests.get(url, stream=<span class="hljs-literal">True</span>).raw)

inputs = processor(images=image, text=<span class="hljs-string">&quot;Generate underlying data table of the figure below:&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
predictions = model.generate(**inputs, max_new_tokens=<span class="hljs-number">512</span>)
<span class="hljs-built_in">print</span>(processor.decode(predictions[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>))`,wrap:!1}}),R=new K({props:{title:"Fine-tuning",local:"fine-tuning",headingTag:"h2"}}),Z=new pe({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5vcHRpbWl6YXRpb24lMjBpbXBvcnQlMjBBZGFmYWN0b3IlMkMlMjBnZXRfY29zaW5lX3NjaGVkdWxlX3dpdGhfd2FybXVwJTBBJTBBb3B0aW1pemVyJTIwJTNEJTIwQWRhZmFjdG9yKHNlbGYucGFyYW1ldGVycygpJTJDJTIwc2NhbGVfcGFyYW1ldGVyJTNERmFsc2UlMkMlMjByZWxhdGl2ZV9zdGVwJTNERmFsc2UlMkMlMjBsciUzRDAuMDElMkMlMjB3ZWlnaHRfZGVjYXklM0QxZS0wNSklMEFzY2hlZHVsZXIlMjAlM0QlMjBnZXRfY29zaW5lX3NjaGVkdWxlX3dpdGhfd2FybXVwKG9wdGltaXplciUyQyUyMG51bV93YXJtdXBfc3RlcHMlM0QxMDAwJTJDJTIwbnVtX3RyYWluaW5nX3N0ZXBzJTNENDAwMDAp",highlighted:`<span class="hljs-keyword">from</span> transformers.optimization <span class="hljs-keyword">import</span> Adafactor, get_cosine_schedule_with_warmup

optimizer = Adafactor(self.parameters(), scale_parameter=<span class="hljs-literal">False</span>, relative_step=<span class="hljs-literal">False</span>, lr=<span class="hljs-number">0.01</span>, weight_decay=<span class="hljs-number">1e-05</span>)
scheduler = get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=<span class="hljs-number">1000</span>, num_training_steps=<span class="hljs-number">40000</span>)`,wrap:!1}}),m=new Me({props:{$$slots:{default:[we]},$$scope:{ctx:C}}}),P=new be({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/model_doc/deplot.md"}}),{c(){a=r("meta"),b=l(),o=r("p"),W=l(),c(w.$$.fragment),G=l(),c(y.$$.fragment),X=l(),T=r("p"),T.innerHTML=O,V=l(),$=r("p"),$.textContent=ee,Y=l(),J=r("p"),J.innerHTML=te,H=l(),v=r("p"),v.innerHTML=ne,I=l(),c(j.$$.fragment),z=l(),_=r("p"),_.textContent=se,F=l(),k=r("ul"),k.innerHTML=ae,E=l(),c(x.$$.fragment),D=l(),c(R.$$.fragment),L=l(),U=r("p"),U.innerHTML=le,S=l(),c(Z.$$.fragment),N=l(),c(m.$$.fragment),A=l(),c(P.$$.fragment),Q=l(),B=r("p"),this.h()},l(e){const t=fe("svelte-u9bgzb",document.head);a=p(t,"META",{name:!0,content:!0}),t.forEach(n),b=i(e),o=p(e,"P",{}),oe(o).forEach(n),W=i(e),u(w.$$.fragment,e),G=i(e),u(y.$$.fragment,e),X=i(e),T=p(e,"P",{"data-svelte-h":!0}),M(T)!=="svelte-19mspuy"&&(T.innerHTML=O),V=i(e),$=p(e,"P",{"data-svelte-h":!0}),M($)!=="svelte-1pvthah"&&($.textContent=ee),Y=i(e),J=p(e,"P",{"data-svelte-h":!0}),M(J)!=="svelte-tgrbdp"&&(J.innerHTML=te),H=i(e),v=p(e,"P",{"data-svelte-h":!0}),M(v)!=="svelte-1ihm3oy"&&(v.innerHTML=ne),I=i(e),u(j.$$.fragment,e),z=i(e),_=p(e,"P",{"data-svelte-h":!0}),M(_)!=="svelte-1010knq"&&(_.textContent=se),F=i(e),k=p(e,"UL",{"data-svelte-h":!0}),M(k)!=="svelte-kycb4s"&&(k.innerHTML=ae),E=i(e),u(x.$$.fragment,e),D=i(e),u(R.$$.fragment,e),L=i(e),U=p(e,"P",{"data-svelte-h":!0}),M(U)!=="svelte-11vjzlg"&&(U.innerHTML=le),S=i(e),u(Z.$$.fragment,e),N=i(e),u(m.$$.fragment,e),A=i(e),u(P.$$.fragment,e),Q=i(e),B=p(e,"P",{}),oe(B).forEach(n),this.h()},h(){re(a,"name","hf:doc:metadata"),re(a,"content",Te)},m(e,t){ge(document.head,a),s(e,b,t),s(e,o,t),s(e,W,t),d(w,e,t),s(e,G,t),d(y,e,t),s(e,X,t),s(e,T,t),s(e,V,t),s(e,$,t),s(e,Y,t),s(e,J,t),s(e,H,t),s(e,v,t),s(e,I,t),d(j,e,t),s(e,z,t),s(e,_,t),s(e,F,t),s(e,k,t),s(e,E,t),d(x,e,t),s(e,D,t),d(R,e,t),s(e,L,t),s(e,U,t),s(e,S,t),d(Z,e,t),s(e,N,t),d(m,e,t),s(e,A,t),d(P,e,t),s(e,Q,t),s(e,B,t),q=!0},p(e,[t]){const ie={};t&2&&(ie.$$scope={dirty:t,ctx:e}),m.$set(ie)},i(e){q||(h(w.$$.fragment,e),h(y.$$.fragment,e),h(j.$$.fragment,e),h(x.$$.fragment,e),h(R.$$.fragment,e),h(Z.$$.fragment,e),h(m.$$.fragment,e),h(P.$$.fragment,e),q=!0)},o(e){f(w.$$.fragment,e),f(y.$$.fragment,e),f(j.$$.fragment,e),f(x.$$.fragment,e),f(R.$$.fragment,e),f(Z.$$.fragment,e),f(m.$$.fragment,e),f(P.$$.fragment,e),q=!1},d(e){e&&(n(b),n(o),n(W),n(G),n(X),n(T),n(V),n($),n(Y),n(J),n(H),n(v),n(I),n(z),n(_),n(F),n(k),n(E),n(D),n(L),n(U),n(S),n(N),n(A),n(Q),n(B)),n(a),g(w,e),g(y,e),g(j,e),g(x,e),g(R,e),g(Z,e),g(m,e),g(P,e)}}}const Te='{"title":"DePlot","local":"deplot","sections":[{"title":"Overview","local":"overview","sections":[],"depth":2},{"title":"Usage example","local":"usage-example","sections":[],"depth":2},{"title":"Fine-tuning","local":"fine-tuning","sections":[],"depth":2}],"depth":1}';function $e(C){return ce(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class xe extends de{constructor(a){super(),he(this,a,$e,ye,me,{})}}export{xe as component};
