import{s as X,n as Y,o as E}from"../chunks/scheduler.d6170356.js";import{S as P,i as D,g as h,s as l,r as Z,A as V,h as g,f as a,c as i,j as G,u as j,x as H,k as L,y as N,a as s,v as U,d as W,t as x,w as F}from"../chunks/index.fcd4cc08.js";import{C as z}from"../chunks/CodeBlock.7b16bdef.js";import{H as A}from"../chunks/Heading.8660bd0e.js";function B(I){let n,w,u,b,o,T,p,R=`<a href="https://github.com/webdataset/webdataset" rel="nofollow">WebDataset</a> is a library to write I/O pipelines for large datasets.
Since it supports streaming data using HTTP, you can use the Hugging Face data files URLs to stream a dataset in WebDataset format:`,y,r,v='First you need to <a href="../huggingface_hub/quick-start#login">Login with your Hugging Face account</a>, for example using:',M,m,J,c,C='And then you can stream Hugging Face datasets in WebDataset. Here is an example of how to load <a href="https://huggingface.co/datasets/timm/imagenet-12k-wds" rel="nofollow">timm/imagenet-12k-wds</a>:',$,d,k,f,_;return o=new A({props:{title:"WebDataset",local:"webdataset",headingTag:"h1"}}),m=new z({props:{code:"aHVnZ2luZ2ZhY2UtY2xpJTIwbG9naW4=",highlighted:'huggingface-<span class="hljs-keyword">cli</span> login',wrap:!1}}),d=new z({props:{code:"aW1wb3J0JTIwd2ViZGF0YXNldCUyMGFzJTIwd2RzJTBBZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMGdldF90b2tlbiUwQSUwQWhmX3Rva2VuJTIwJTNEJTIwZ2V0X3Rva2VuKCklMEFkYXRhc2V0JTIwJTNEJTIwd2RzLldlYkRhdGFzZXQoZiUyMnBpcGUlM0FjdXJsJTIwLXMlMjAtTCUyMGh0dHBzJTNBJTJGJTJGaHVnZ2luZ2ZhY2UuY28lMkZkYXRhc2V0cyUyRnRpbW0lMkZpbWFnZW5ldC0xMmstd2RzJTJGcmVzb2x2ZSUyRm1haW4lMkZpbWFnZW5ldDEyay10cmFpbi0lN0IlN0IwMDAwLi4xMDIzJTdEJTdELnRhciUyMC1IJTIwJ0F1dGhvcml6YXRpb24lM0FCZWFyZXIlMjAlN0JoZl90b2tlbiU3RCclMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> webdataset <span class="hljs-keyword">as</span> wds
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> get_token

<span class="hljs-meta">&gt;&gt;&gt; </span>hf_token = get_token()
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = wds.WebDataset(<span class="hljs-string">f&quot;pipe:curl -s -L https://huggingface.co/datasets/timm/imagenet-12k-wds/resolve/main/imagenet12k-train-{{0000..1023}}.tar -H &#x27;Authorization:Bearer <span class="hljs-subst">{hf_token}</span>&#x27;&quot;</span>)`,wrap:!1}}),{c(){n=h("meta"),w=l(),u=h("p"),b=l(),Z(o.$$.fragment),T=l(),p=h("p"),p.innerHTML=R,y=l(),r=h("p"),r.innerHTML=v,M=l(),Z(m.$$.fragment),J=l(),c=h("p"),c.innerHTML=C,$=l(),Z(d.$$.fragment),k=l(),f=h("p"),this.h()},l(t){const e=V("svelte-u9bgzb",document.head);n=g(e,"META",{name:!0,content:!0}),e.forEach(a),w=i(t),u=g(t,"P",{}),G(u).forEach(a),b=i(t),j(o.$$.fragment,t),T=i(t),p=g(t,"P",{"data-svelte-h":!0}),H(p)!=="svelte-1cfdl4w"&&(p.innerHTML=R),y=i(t),r=g(t,"P",{"data-svelte-h":!0}),H(r)!=="svelte-1hbzj8b"&&(r.innerHTML=v),M=i(t),j(m.$$.fragment,t),J=i(t),c=g(t,"P",{"data-svelte-h":!0}),H(c)!=="svelte-nxhj3"&&(c.innerHTML=C),$=i(t),j(d.$$.fragment,t),k=i(t),f=g(t,"P",{}),G(f).forEach(a),this.h()},h(){L(n,"name","hf:doc:metadata"),L(n,"content",S)},m(t,e){N(document.head,n),s(t,w,e),s(t,u,e),s(t,b,e),U(o,t,e),s(t,T,e),s(t,p,e),s(t,y,e),s(t,r,e),s(t,M,e),U(m,t,e),s(t,J,e),s(t,c,e),s(t,$,e),U(d,t,e),s(t,k,e),s(t,f,e),_=!0},p:Y,i(t){_||(W(o.$$.fragment,t),W(m.$$.fragment,t),W(d.$$.fragment,t),_=!0)},o(t){x(o.$$.fragment,t),x(m.$$.fragment,t),x(d.$$.fragment,t),_=!1},d(t){t&&(a(w),a(u),a(b),a(T),a(p),a(y),a(r),a(M),a(J),a(c),a($),a(k),a(f)),a(n),F(o,t),F(m,t),F(d,t)}}}const S='{"title":"WebDataset","local":"webdataset","sections":[],"depth":1}';function q(I){return E(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class et extends P{constructor(n){super(),D(this,n,q,B,X,{})}}export{et as component};
