import{s as O,n as tt,o as et}from"../chunks/scheduler.d6170356.js";import{S as at,i as st,g as r,s as l,r as U,A as lt,h as d,f as a,c as n,j as A,u as _,x as k,k as K,y as nt,a as s,v as R,d as T,t as X,w as b}from"../chunks/index.fcd4cc08.js";import{C as Q}from"../chunks/CodeBlock.7b16bdef.js";import{H as it}from"../chunks/Heading.8660bd0e.js";function rt(N){let i,w,G,j,o,Y,p,L=`<a href="https://github.com/dask/dask" rel="nofollow">Dask</a> is a parallel and distributed computing library that scales the existing Python and PyData ecosystem.
Since it uses <a href="https://filesystem-spec.readthedocs.io" rel="nofollow">fsspec</a> to read and write remote data, you can use the Hugging Face paths (<a href="https://huggingface.co/docs/huggingface_hub/guides/hf_file_system#integrations" rel="nofollow"><code>hf://</code></a>) to read and write data on the Hub:`,$,h,S='First you need to <a href="../huggingface_hub/quick-start#login">Login with your Hugging Face account</a>, for example using:',B,c,F,m,E='Then you can <a href="../huggingface_hub/quick-start#create-a-repository">Create a dataset repository</a>, for example using:',q,f,H,u,x='Finally, you can use <a href="https://huggingface.co/docs/huggingface_hub/guides/hf_file_system#integrations" rel="nofollow">Hugging Face paths</a> in Dask:',V,y,z,M,P=`This creates a dataset repository <code>username/my_dataset</code> containing your Dask dataset in Parquet format.
You can reload it later:`,C,g,v,J,D='For more information on the Hugging Face paths and how they are implemented, please refer to the <a href="https://huggingface.co/docs/huggingface_hub/guides/hf_file_system" rel="nofollow">the client library’s documentation on the HfFileSystem</a>.',W,Z,I;return o=new it({props:{title:"Dask",local:"dask",headingTag:"h1"}}),c=new Q({props:{code:"aHVnZ2luZ2ZhY2UtY2xpJTIwbG9naW4=",highlighted:'huggingface-<span class="hljs-keyword">cli</span> login',wrap:!1}}),f=new Q({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMEhmQXBpJTBBJTBBSGZBcGkoKS5jcmVhdGVfcmVwbyhyZXBvX2lkJTNEJTIydXNlcm5hbWUlMkZteV9kYXRhc2V0JTIyJTJDJTIwcmVwb190eXBlJTNEJTIyZGF0YXNldCUyMik=",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> HfApi

HfApi().create_repo(repo_id=<span class="hljs-string">&quot;username/my_dataset&quot;</span>, repo_type=<span class="hljs-string">&quot;dataset&quot;</span>)`,wrap:!1}}),y=new Q({props:{code:"aW1wb3J0JTIwZGFzay5kYXRhZnJhbWUlMjBhcyUyMGRkJTBBJTBBZGYudG9fcGFycXVldCglMjJoZiUzQSUyRiUyRmRhdGFzZXRzJTJGdXNlcm5hbWUlMkZteV9kYXRhc2V0JTIyKSUwQSUwQSUyMyUyMG9yJTIwd3JpdGUlMjBpbiUyMHNlcGFyYXRlJTIwZGlyZWN0b3JpZXMlMjBpZiUyMHRoZSUyMGRhdGFzZXQlMjBoYXMlMjB0cmFpbiUyRnZhbGlkYXRpb24lMkZ0ZXN0JTIwc3BsaXRzJTBBZGZfdHJhaW4udG9fcGFycXVldCglMjJoZiUzQSUyRiUyRmRhdGFzZXRzJTJGdXNlcm5hbWUlMkZteV9kYXRhc2V0JTJGdHJhaW4lMjIpJTBBZGZfdmFsaWQudG9fcGFycXVldCglMjJoZiUzQSUyRiUyRmRhdGFzZXRzJTJGdXNlcm5hbWUlMkZteV9kYXRhc2V0JTJGdmFsaWRhdGlvbiUyMiklMEFkZl90ZXN0JTIwLnRvX3BhcnF1ZXQoJTIyaGYlM0ElMkYlMkZkYXRhc2V0cyUyRnVzZXJuYW1lJTJGbXlfZGF0YXNldCUyRnRlc3QlMjIp",highlighted:`<span class="hljs-keyword">import</span> dask.dataframe <span class="hljs-keyword">as</span> dd

df.to_parquet(<span class="hljs-string">&quot;hf://datasets/username/my_dataset&quot;</span>)

<span class="hljs-comment"># or write in separate directories if the dataset has train/validation/test splits</span>
df_train.to_parquet(<span class="hljs-string">&quot;hf://datasets/username/my_dataset/train&quot;</span>)
df_valid.to_parquet(<span class="hljs-string">&quot;hf://datasets/username/my_dataset/validation&quot;</span>)
df_test .to_parquet(<span class="hljs-string">&quot;hf://datasets/username/my_dataset/test&quot;</span>)`,wrap:!1}}),g=new Q({props:{code:"aW1wb3J0JTIwZGFzay5kYXRhZnJhbWUlMjBhcyUyMGRkJTBBJTBBZGYlMjAlM0QlMjBkZC5yZWFkX3BhcnF1ZXQoJTIyaGYlM0ElMkYlMkZkYXRhc2V0cyUyRnVzZXJuYW1lJTJGbXlfZGF0YXNldCUyMiklMEElMEElMjMlMjBvciUyMHJlYWQlMjBmcm9tJTIwc2VwYXJhdGUlMjBkaXJlY3RvcmllcyUyMGlmJTIwdGhlJTIwZGF0YXNldCUyMGhhcyUyMHRyYWluJTJGdmFsaWRhdGlvbiUyRnRlc3QlMjBzcGxpdHMlMEFkZl90cmFpbiUyMCUzRCUyMGRkLnJlYWRfcGFycXVldCglMjJoZiUzQSUyRiUyRmRhdGFzZXRzJTJGdXNlcm5hbWUlMkZteV9kYXRhc2V0JTJGdHJhaW4lMjIpJTBBZGZfdmFsaWQlMjAlM0QlMjBkZC5yZWFkX3BhcnF1ZXQoJTIyaGYlM0ElMkYlMkZkYXRhc2V0cyUyRnVzZXJuYW1lJTJGbXlfZGF0YXNldCUyRnZhbGlkYXRpb24lMjIpJTBBZGZfdGVzdCUyMCUyMCUzRCUyMGRkLnJlYWRfcGFycXVldCglMjJoZiUzQSUyRiUyRmRhdGFzZXRzJTJGdXNlcm5hbWUlMkZteV9kYXRhc2V0JTJGdGVzdCUyMik=",highlighted:`<span class="hljs-keyword">import</span> dask.dataframe <span class="hljs-keyword">as</span> dd

df = dd.read_parquet(<span class="hljs-string">&quot;hf://datasets/username/my_dataset&quot;</span>)

<span class="hljs-comment"># or read from separate directories if the dataset has train/validation/test splits</span>
df_train = dd.read_parquet(<span class="hljs-string">&quot;hf://datasets/username/my_dataset/train&quot;</span>)
df_valid = dd.read_parquet(<span class="hljs-string">&quot;hf://datasets/username/my_dataset/validation&quot;</span>)
df_test  = dd.read_parquet(<span class="hljs-string">&quot;hf://datasets/username/my_dataset/test&quot;</span>)`,wrap:!1}}),{c(){i=r("meta"),w=l(),G=r("p"),j=l(),U(o.$$.fragment),Y=l(),p=r("p"),p.innerHTML=L,$=l(),h=r("p"),h.innerHTML=S,B=l(),U(c.$$.fragment),F=l(),m=r("p"),m.innerHTML=E,q=l(),U(f.$$.fragment),H=l(),u=r("p"),u.innerHTML=x,V=l(),U(y.$$.fragment),z=l(),M=r("p"),M.innerHTML=P,C=l(),U(g.$$.fragment),v=l(),J=r("p"),J.innerHTML=D,W=l(),Z=r("p"),this.h()},l(t){const e=lt("svelte-u9bgzb",document.head);i=d(e,"META",{name:!0,content:!0}),e.forEach(a),w=n(t),G=d(t,"P",{}),A(G).forEach(a),j=n(t),_(o.$$.fragment,t),Y=n(t),p=d(t,"P",{"data-svelte-h":!0}),k(p)!=="svelte-1nkggm5"&&(p.innerHTML=L),$=n(t),h=d(t,"P",{"data-svelte-h":!0}),k(h)!=="svelte-1hbzj8b"&&(h.innerHTML=S),B=n(t),_(c.$$.fragment,t),F=n(t),m=d(t,"P",{"data-svelte-h":!0}),k(m)!=="svelte-1tu70ef"&&(m.innerHTML=E),q=n(t),_(f.$$.fragment,t),H=n(t),u=d(t,"P",{"data-svelte-h":!0}),k(u)!=="svelte-qq5b5u"&&(u.innerHTML=x),V=n(t),_(y.$$.fragment,t),z=n(t),M=d(t,"P",{"data-svelte-h":!0}),k(M)!=="svelte-14rczfs"&&(M.innerHTML=P),C=n(t),_(g.$$.fragment,t),v=n(t),J=d(t,"P",{"data-svelte-h":!0}),k(J)!=="svelte-1k6ibc2"&&(J.innerHTML=D),W=n(t),Z=d(t,"P",{}),A(Z).forEach(a),this.h()},h(){K(i,"name","hf:doc:metadata"),K(i,"content",dt)},m(t,e){nt(document.head,i),s(t,w,e),s(t,G,e),s(t,j,e),R(o,t,e),s(t,Y,e),s(t,p,e),s(t,$,e),s(t,h,e),s(t,B,e),R(c,t,e),s(t,F,e),s(t,m,e),s(t,q,e),R(f,t,e),s(t,H,e),s(t,u,e),s(t,V,e),R(y,t,e),s(t,z,e),s(t,M,e),s(t,C,e),R(g,t,e),s(t,v,e),s(t,J,e),s(t,W,e),s(t,Z,e),I=!0},p:tt,i(t){I||(T(o.$$.fragment,t),T(c.$$.fragment,t),T(f.$$.fragment,t),T(y.$$.fragment,t),T(g.$$.fragment,t),I=!0)},o(t){X(o.$$.fragment,t),X(c.$$.fragment,t),X(f.$$.fragment,t),X(y.$$.fragment,t),X(g.$$.fragment,t),I=!1},d(t){t&&(a(w),a(G),a(j),a(Y),a(p),a($),a(h),a(B),a(F),a(m),a(q),a(H),a(u),a(V),a(z),a(M),a(C),a(v),a(J),a(W),a(Z)),a(i),b(o,t),b(c,t),b(f,t),b(y,t),b(g,t)}}}const dt='{"title":"Dask","local":"dask","sections":[],"depth":1}';function ot(N){return et(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ft extends at{constructor(i){super(),st(this,i,ot,rt,O,{})}}export{ft as component};
