import{s as mt,n as ut,o as ct}from"../chunks/scheduler.bdbef820.js";import{S as dt,i as ft,g as i,s as n,r as v,A as wt,h as p,f as s,c as l,j as ot,u as I,x as r,k as rt,y as bt,a,v as _,d as k,t as x,w as z}from"../chunks/index.c0aea24a.js";import{C as ht}from"../chunks/CodeBlock.6ccca92e.js";import{H as S,E as gt}from"../chunks/EditOnGithub.725ee0c1.js";function yt(K){let o,G,U,N,h,F,m,O='Combining the utility of <a href="/docs/datasets/v2.21.0/en/package_reference/main_classes#datasets.Dataset.map">Dataset.map()</a> with batch mode is very powerful. It allows you to speed up processing, and freely control the size of the generated dataset.',R,u,Y,c,tt='The primary objective of batch mapping is to speed up processing. Often times, it is faster to work with batches of data instead of single examples. Naturally, batch mapping lends itself to tokenization. For example, the 🤗 <a href="https://huggingface.co/docs/tokenizers/python/latest/" rel="nofollow">Tokenizers</a> library works faster with batches because it parallelizes the tokenization of all the examples in a batch.',E,d,H,f,et='The ability to control the size of the generated dataset can be leveraged for many interesting use-cases. In the How-to <a href="#map">map</a> section, there are examples of using batch mapping to:',L,w,st="<li>Split long sentences into shorter chunks.</li> <li>Augment a dataset with additional tokens.</li>",Z,b,at="It is helpful to understand how this works, so you can come up with your own ways to use batch mapping. At this point, you may be wondering how you can control the size of the generated dataset. The answer is: <strong>the mapped function does not have to return an output batch of the same size</strong>.",A,g,nt="In other words, your mapped function input can be a batch of size <code>N</code> and return a batch of size <code>M</code>. The output <code>M</code> can be greater than or less than <code>N</code>. This means you can concatenate your examples, divide it up, and even add more examples!",Q,y,lt="However, remember that all values in the output dictionary must contain the <strong>same number of elements</strong> as the other fields in the output dictionary. Otherwise, it is not possible to define the number of examples in the output returned by the mapped function. The number can vary between successive batches processed by the mapped function. For a single batch though, all values of the output dictionary should have the same length (i.e., the number of elements).",X,T,it=`For example, from a dataset of 1 column and 3 rows, if you use <code>map</code> to return a new column with twice as many rows, then you will have an error.
In this case, you end up with one column with 3 rows, and one column with 6 rows. As you can see, the table will not be valid:`,q,M,W,j,pt="To make it valid, you have to drop one of the columns:",D,J,P,$,V,C,B;return h=new S({props:{title:"Batch mapping",local:"batch-mapping",headingTag:"h1"}}),u=new S({props:{title:"Need for speed",local:"need-for-speed",headingTag:"h2"}}),d=new S({props:{title:"Input size != output size",local:"input-size--output-size",headingTag:"h2"}}),M=new ht({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwRGF0YXNldCUwQWRhdGFzZXQlMjAlM0QlMjBEYXRhc2V0LmZyb21fZGljdCglN0IlMjJhJTIyJTNBJTIwJTVCMCUyQyUyMDElMkMlMjAyJTVEJTdEKSUwQWRhdGFzZXQubWFwKGxhbWJkYSUyMGJhdGNoJTNBJTIwJTdCJTIyYiUyMiUzQSUyMGJhdGNoJTVCJTIyYSUyMiU1RCUyMColMjAyJTdEJTJDJTIwYmF0Y2hlZCUzRFRydWUpJTIwJTIwJTIzJTIwbmV3JTIwY29sdW1uJTIwd2l0aCUyMDYlMjBlbGVtZW50cyUzQSUyMCU1QjAlMkMlMjAxJTJDJTIwMiUyQyUyMDAlMkMlMjAxJTJDJTIwMiU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = Dataset.from_dict({<span class="hljs-string">&quot;a&quot;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">2</span>]})
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset.<span class="hljs-built_in">map</span>(<span class="hljs-keyword">lambda</span> batch: {<span class="hljs-string">&quot;b&quot;</span>: batch[<span class="hljs-string">&quot;a&quot;</span>] * <span class="hljs-number">2</span>}, batched=<span class="hljs-literal">True</span>)  <span class="hljs-comment"># new column with 6 elements: [0, 1, 2, 0, 1, 2]</span>
<span class="hljs-string">&#x27;ArrowInvalid: Column 1 named b expected length 3 but got length 6&#x27;</span>`,wrap:!1}}),J=new ht({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwRGF0YXNldCUwQWRhdGFzZXQlMjAlM0QlMjBEYXRhc2V0LmZyb21fZGljdCglN0IlMjJhJTIyJTNBJTIwJTVCMCUyQyUyMDElMkMlMjAyJTVEJTdEKSUwQWRhdGFzZXRfd2l0aF9kdXBsaWNhdGVzJTIwJTNEJTIwZGF0YXNldC5tYXAobGFtYmRhJTIwYmF0Y2glM0ElMjAlN0IlMjJiJTIyJTNBJTIwYmF0Y2glNUIlMjJhJTIyJTVEJTIwKiUyMDIlN0QlMkMlMjByZW1vdmVfY29sdW1ucyUzRCU1QiUyMmElMjIlNUQlMkMlMjBiYXRjaGVkJTNEVHJ1ZSklMEFsZW4oZGF0YXNldF93aXRoX2R1cGxpY2F0ZXMp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = Dataset.from_dict({<span class="hljs-string">&quot;a&quot;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">2</span>]})
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset_with_duplicates = dataset.<span class="hljs-built_in">map</span>(<span class="hljs-keyword">lambda</span> batch: {<span class="hljs-string">&quot;b&quot;</span>: batch[<span class="hljs-string">&quot;a&quot;</span>] * <span class="hljs-number">2</span>}, remove_columns=[<span class="hljs-string">&quot;a&quot;</span>], batched=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">len</span>(dataset_with_duplicates)
<span class="hljs-number">6</span>`,wrap:!1}}),$=new gt({props:{source:"https://github.com/huggingface/datasets/blob/main/docs/source/about_map_batch.mdx"}}),{c(){o=i("meta"),G=n(),U=i("p"),N=n(),v(h.$$.fragment),F=n(),m=i("p"),m.innerHTML=O,R=n(),v(u.$$.fragment),Y=n(),c=i("p"),c.innerHTML=tt,E=n(),v(d.$$.fragment),H=n(),f=i("p"),f.innerHTML=et,L=n(),w=i("ul"),w.innerHTML=st,Z=n(),b=i("p"),b.innerHTML=at,A=n(),g=i("p"),g.innerHTML=nt,Q=n(),y=i("p"),y.innerHTML=lt,X=n(),T=i("p"),T.innerHTML=it,q=n(),v(M.$$.fragment),W=n(),j=i("p"),j.textContent=pt,D=n(),v(J.$$.fragment),P=n(),v($.$$.fragment),V=n(),C=i("p"),this.h()},l(t){const e=wt("svelte-u9bgzb",document.head);o=p(e,"META",{name:!0,content:!0}),e.forEach(s),G=l(t),U=p(t,"P",{}),ot(U).forEach(s),N=l(t),I(h.$$.fragment,t),F=l(t),m=p(t,"P",{"data-svelte-h":!0}),r(m)!=="svelte-1klt1"&&(m.innerHTML=O),R=l(t),I(u.$$.fragment,t),Y=l(t),c=p(t,"P",{"data-svelte-h":!0}),r(c)!=="svelte-ftplyc"&&(c.innerHTML=tt),E=l(t),I(d.$$.fragment,t),H=l(t),f=p(t,"P",{"data-svelte-h":!0}),r(f)!=="svelte-catkrd"&&(f.innerHTML=et),L=l(t),w=p(t,"UL",{"data-svelte-h":!0}),r(w)!=="svelte-min2km"&&(w.innerHTML=st),Z=l(t),b=p(t,"P",{"data-svelte-h":!0}),r(b)!=="svelte-1t0thr9"&&(b.innerHTML=at),A=l(t),g=p(t,"P",{"data-svelte-h":!0}),r(g)!=="svelte-14uwqpb"&&(g.innerHTML=nt),Q=l(t),y=p(t,"P",{"data-svelte-h":!0}),r(y)!=="svelte-5jxdnp"&&(y.innerHTML=lt),X=l(t),T=p(t,"P",{"data-svelte-h":!0}),r(T)!=="svelte-1ac2mdx"&&(T.innerHTML=it),q=l(t),I(M.$$.fragment,t),W=l(t),j=p(t,"P",{"data-svelte-h":!0}),r(j)!=="svelte-b6xcy8"&&(j.textContent=pt),D=l(t),I(J.$$.fragment,t),P=l(t),I($.$$.fragment,t),V=l(t),C=p(t,"P",{}),ot(C).forEach(s),this.h()},h(){rt(o,"name","hf:doc:metadata"),rt(o,"content",Tt)},m(t,e){bt(document.head,o),a(t,G,e),a(t,U,e),a(t,N,e),_(h,t,e),a(t,F,e),a(t,m,e),a(t,R,e),_(u,t,e),a(t,Y,e),a(t,c,e),a(t,E,e),_(d,t,e),a(t,H,e),a(t,f,e),a(t,L,e),a(t,w,e),a(t,Z,e),a(t,b,e),a(t,A,e),a(t,g,e),a(t,Q,e),a(t,y,e),a(t,X,e),a(t,T,e),a(t,q,e),_(M,t,e),a(t,W,e),a(t,j,e),a(t,D,e),_(J,t,e),a(t,P,e),_($,t,e),a(t,V,e),a(t,C,e),B=!0},p:ut,i(t){B||(k(h.$$.fragment,t),k(u.$$.fragment,t),k(d.$$.fragment,t),k(M.$$.fragment,t),k(J.$$.fragment,t),k($.$$.fragment,t),B=!0)},o(t){x(h.$$.fragment,t),x(u.$$.fragment,t),x(d.$$.fragment,t),x(M.$$.fragment,t),x(J.$$.fragment,t),x($.$$.fragment,t),B=!1},d(t){t&&(s(G),s(U),s(N),s(F),s(m),s(R),s(Y),s(c),s(E),s(H),s(f),s(L),s(w),s(Z),s(b),s(A),s(g),s(Q),s(y),s(X),s(T),s(q),s(W),s(j),s(D),s(P),s(V),s(C)),s(o),z(h,t),z(u,t),z(d,t),z(M,t),z(J,t),z($,t)}}}const Tt='{"title":"Batch mapping","local":"batch-mapping","sections":[{"title":"Need for speed","local":"need-for-speed","sections":[],"depth":2},{"title":"Input size != output size","local":"input-size--output-size","sections":[],"depth":2}],"depth":1}';function Mt(K){return ct(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class It extends dt{constructor(o){super(),ft(this,o,Mt,yt,mt,{})}}export{It as component};
