import{s as oe,o as me,n as ie}from"../chunks/scheduler.bdbef820.js";import{S as ue,i as ce,g as i,s as p,r as T,A as fe,h as o,f as a,c as r,j as pe,u as J,x as g,k as G,y as he,a as s,v as y,d as b,t as U,w}from"../chunks/index.c0aea24a.js";import{T as re}from"../chunks/Tip.31005f7d.js";import{C as D}from"../chunks/CodeBlock.6ccca92e.js";import{H as de,E as $e}from"../chunks/EditOnGithub.725ee0c1.js";function Te(R){let l,m="The Beam API is deprecated and will be removed in the next major release.";return{c(){l=i("p"),l.textContent=m},l(n){l=o(n,"P",{"data-svelte-h":!0}),g(l)!=="svelte-h7wxa4"&&(l.textContent=m)},m(n,u){s(n,l,u)},p:ie,d(n){n&&a(l)}}}function Je(R){let l,m="When you run your pipeline, you can adjust the parameters to change the runner (Flink or Spark), output location (S3 bucket or HDFS), and the number of workers.";return{c(){l=i("p"),l.textContent=m},l(n){l=o(n,"P",{"data-svelte-h":!0}),g(l)!=="svelte-3buu9"&&(l.textContent=m)},m(n,u){s(n,l,u)},p:ie,d(n){n&&a(l)}}}function ye(R){let l,m,n,u,_,S,c,x,M,O='Some datasets are too large to be processed on a single machine. Instead, you can process them with <a href="https://beam.apache.org/" rel="nofollow">Apache Beam</a>, a library for parallel data processing. The processing pipeline is executed on a distributed processing backend such as <a href="https://flink.apache.org/" rel="nofollow">Apache Flink</a>, <a href="https://spark.apache.org/" rel="nofollow">Apache Spark</a>, or <a href="https://cloud.google.com/dataflow" rel="nofollow">Google Cloud Dataflow</a>.',H,V,K='We have already created Beam pipelines for some of the larger datasets like <a href="https://huggingface.co/datasets/wikipedia" rel="nofollow">wikipedia</a>, and <a href="https://huggingface.co/datasets/wiki40b" rel="nofollow">wiki40b</a>. You can load these normally with <a href="/docs/datasets/v2.21.0/en/package_reference/loading_methods#datasets.load_dataset">load_dataset()</a>. But if you want to run your own Beam pipeline with Dataflow, here is how:',W,j,ee="<li>Specify the dataset and configuration you want to process:</li>",B,C,Q,f,te="<li>Input your Google Cloud Platform information:</li>",Z,N,I,h,ae="<li>Specify your Python requirements:</li>",A,v,X,d,se="<li>Run the pipeline:</li>",Y,E,L,$,P,F,q,k,z;return _=new de({props:{title:"Beam Datasets",local:"beam-datasets",headingTag:"h1"}}),c=new re({props:{warning:!0,$$slots:{default:[Te]},$$scope:{ctx:R}}}),C=new D({props:{code:"REFUQVNFVF9OQU1FJTNEeW91cl9kYXRhc2V0X25hbWUlMjAlMjAlMjMlMjBleCUzQSUyMHdpa2lwZWRpYSUwQUNPTkZJR19OQU1FJTNEeW91cl9jb25maWdfbmFtZSUyMCUyMCUyMCUyMCUyMyUyMGV4JTNBJTIwMjAyMjAzMDEuZW4=",highlighted:`<span class="hljs-attr">DATASET_NAME</span>=your_dataset_name  <span class="hljs-comment"># ex: wikipedia</span>
<span class="hljs-attr">CONFIG_NAME</span>=your_config_name    <span class="hljs-comment"># ex: 20220301.en</span>`,wrap:!1}}),N=new D({props:{code:"UFJPSkVDVCUzRHlvdXJfcHJvamVjdCUwQUJVQ0tFVCUzRHlvdXJfYnVja2V0JTBBUkVHSU9OJTNEeW91cl9yZWdpb24=",highlighted:`<span class="hljs-attribute">PROJECT</span><span class="hljs-operator">=</span>your_project
<span class="hljs-attribute">BUCKET</span><span class="hljs-operator">=</span>your_bucket
<span class="hljs-attribute">REGION</span><span class="hljs-operator">=</span>your_region`,wrap:!1}}),v=new D({props:{code:"ZWNobyUyMCUyMmRhdGFzZXRzJTIyJTIwJTNFJTIwJTJGdG1wJTJGYmVhbV9yZXF1aXJlbWVudHMudHh0JTBBZWNobyUyMCUyMmFwYWNoZV9iZWFtJTIyJTIwJTNFJTNFJTIwJTJGdG1wJTJGYmVhbV9yZXF1aXJlbWVudHMudHh0",highlighted:`echo <span class="hljs-string">&quot;datasets&quot;</span> &gt; <span class="hljs-regexp">/tmp/</span>beam_requirements.txt
echo <span class="hljs-string">&quot;apache_beam&quot;</span> &gt;&gt; <span class="hljs-regexp">/tmp/</span>beam_requirements.txt`,wrap:!1}}),E=new D({props:{code:"ZGF0YXNldHMtY2xpJTIwcnVuX2JlYW0lMjBkYXRhc2V0cyUyRiUyNERBVEFTRVRfTkFNRSUyMCU1QyUwQS0tbmFtZSUyMCUyNENPTkZJR19OQU1FJTIwJTVDJTBBLS1zYXZlX2luZm8lMjAlNUMlMEEtLWNhY2hlX2RpciUyMGdzJTNBJTJGJTJGJTI0QlVDS0VUJTJGY2FjaGUlMkZkYXRhc2V0cyUyMCU1QyUwQS0tYmVhbV9waXBlbGluZV9vcHRpb25zJTNEJTVDJTBBJTIycnVubmVyJTNERGF0YWZsb3dSdW5uZXIlMkNwcm9qZWN0JTNEJTI0UFJPSkVDVCUyQ2pvYl9uYW1lJTNEJTI0REFUQVNFVF9OQU1FLWdlbiUyQyUyMiU1QyUwQSUyMnN0YWdpbmdfbG9jYXRpb24lM0RncyUzQSUyRiUyRiUyNEJVQ0tFVCUyRmJpbmFyaWVzJTJDdGVtcF9sb2NhdGlvbiUzRGdzJTNBJTJGJTJGJTI0QlVDS0VUJTJGdGVtcCUyQyUyMiU1QyUwQSUyMnJlZ2lvbiUzRCUyNFJFR0lPTiUyQ3JlcXVpcmVtZW50c19maWxlJTNEJTJGdG1wJTJGYmVhbV9yZXF1aXJlbWVudHMudHh0JTIy",highlighted:`datasets-cli run_beam datasets/<span class="hljs-variable">$DATASET_NAME</span> \\
--name <span class="hljs-variable">$CONFIG_NAME</span> \\
--save_info \\
--cache_dir gs://<span class="hljs-variable">$BUCKET</span>/cache/datasets \\
--beam_pipeline_options=\\
<span class="hljs-string">&quot;runner=DataflowRunner,project=<span class="hljs-variable">$PROJECT</span>,job_name=<span class="hljs-variable">$DATASET_NAME</span>-gen,&quot;</span>\\
<span class="hljs-string">&quot;staging_location=gs://<span class="hljs-variable">$BUCKET</span>/binaries,temp_location=gs://<span class="hljs-variable">$BUCKET</span>/temp,&quot;</span>\\
<span class="hljs-string">&quot;region=<span class="hljs-variable">$REGION</span>,requirements_file=/tmp/beam_requirements.txt&quot;</span>`,wrap:!1}}),$=new re({props:{$$slots:{default:[Je]},$$scope:{ctx:R}}}),F=new $e({props:{source:"https://github.com/huggingface/datasets/blob/main/docs/source/beam.mdx"}}),{c(){l=i("meta"),m=p(),n=i("p"),u=p(),T(_.$$.fragment),S=p(),T(c.$$.fragment),x=p(),M=i("p"),M.innerHTML=O,H=p(),V=i("p"),V.innerHTML=K,W=p(),j=i("ol"),j.innerHTML=ee,B=p(),T(C.$$.fragment),Q=p(),f=i("ol"),f.innerHTML=te,Z=p(),T(N.$$.fragment),I=p(),h=i("ol"),h.innerHTML=ae,A=p(),T(v.$$.fragment),X=p(),d=i("ol"),d.innerHTML=se,Y=p(),T(E.$$.fragment),L=p(),T($.$$.fragment),P=p(),T(F.$$.fragment),q=p(),k=i("p"),this.h()},l(e){const t=fe("svelte-u9bgzb",document.head);l=o(t,"META",{name:!0,content:!0}),t.forEach(a),m=r(e),n=o(e,"P",{}),pe(n).forEach(a),u=r(e),J(_.$$.fragment,e),S=r(e),J(c.$$.fragment,e),x=r(e),M=o(e,"P",{"data-svelte-h":!0}),g(M)!=="svelte-1lvp5mc"&&(M.innerHTML=O),H=r(e),V=o(e,"P",{"data-svelte-h":!0}),g(V)!=="svelte-1u11d4f"&&(V.innerHTML=K),W=r(e),j=o(e,"OL",{"data-svelte-h":!0}),g(j)!=="svelte-12t0ljj"&&(j.innerHTML=ee),B=r(e),J(C.$$.fragment,e),Q=r(e),f=o(e,"OL",{start:!0,"data-svelte-h":!0}),g(f)!=="svelte-zdhrpa"&&(f.innerHTML=te),Z=r(e),J(N.$$.fragment,e),I=r(e),h=o(e,"OL",{start:!0,"data-svelte-h":!0}),g(h)!=="svelte-8zj81h"&&(h.innerHTML=ae),A=r(e),J(v.$$.fragment,e),X=r(e),d=o(e,"OL",{start:!0,"data-svelte-h":!0}),g(d)!=="svelte-1wuz6qo"&&(d.innerHTML=se),Y=r(e),J(E.$$.fragment,e),L=r(e),J($.$$.fragment,e),P=r(e),J(F.$$.fragment,e),q=r(e),k=o(e,"P",{}),pe(k).forEach(a),this.h()},h(){G(l,"name","hf:doc:metadata"),G(l,"content",be),G(f,"start","2"),G(h,"start","3"),G(d,"start","4")},m(e,t){he(document.head,l),s(e,m,t),s(e,n,t),s(e,u,t),y(_,e,t),s(e,S,t),y(c,e,t),s(e,x,t),s(e,M,t),s(e,H,t),s(e,V,t),s(e,W,t),s(e,j,t),s(e,B,t),y(C,e,t),s(e,Q,t),s(e,f,t),s(e,Z,t),y(N,e,t),s(e,I,t),s(e,h,t),s(e,A,t),y(v,e,t),s(e,X,t),s(e,d,t),s(e,Y,t),y(E,e,t),s(e,L,t),y($,e,t),s(e,P,t),y(F,e,t),s(e,q,t),s(e,k,t),z=!0},p(e,[t]){const le={};t&2&&(le.$$scope={dirty:t,ctx:e}),c.$set(le);const ne={};t&2&&(ne.$$scope={dirty:t,ctx:e}),$.$set(ne)},i(e){z||(b(_.$$.fragment,e),b(c.$$.fragment,e),b(C.$$.fragment,e),b(N.$$.fragment,e),b(v.$$.fragment,e),b(E.$$.fragment,e),b($.$$.fragment,e),b(F.$$.fragment,e),z=!0)},o(e){U(_.$$.fragment,e),U(c.$$.fragment,e),U(C.$$.fragment,e),U(N.$$.fragment,e),U(v.$$.fragment,e),U(E.$$.fragment,e),U($.$$.fragment,e),U(F.$$.fragment,e),z=!1},d(e){e&&(a(m),a(n),a(u),a(S),a(x),a(M),a(H),a(V),a(W),a(j),a(B),a(Q),a(f),a(Z),a(I),a(h),a(A),a(X),a(d),a(Y),a(L),a(P),a(q),a(k)),a(l),w(_,e),w(c,e),w(C,e),w(N,e),w(v,e),w(E,e),w($,e),w(F,e)}}}const be='{"title":"Beam Datasets","local":"beam-datasets","sections":[],"depth":1}';function Ue(R){return me(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class je extends ue{constructor(l){super(),ce(this,l,Ue,ye,oe,{})}}export{je as component};
