import{s as re,n as me,o as ue}from"../chunks/scheduler.25b97de1.js";import{S as ce,i as de,g as o,s as n,r as C,A as Me,h as p,f as l,c as a,j as ae,u as J,x as r,k as oe,l as fe,y as he,a as s,v as $,d as B,t as W,w as x}from"../chunks/index.d9030fc9.js";import{C as pe}from"../chunks/CodeBlock.e6cd0d95.js";import{H as ie,E as ye}from"../chunks/EditOnGithub.91d95064.js";function be(A){let i,k,U,Z,u,E,c,N="Built-in Tensor Parallelism (TP) is now available with certain models using PyTorch. Tensor parallelism shards a model onto multiple GPUs, enabling larger model sizes, and parallelizes computations such as matrix multiplication.",I,d,K='To enable tensor parallel, pass the argument <code>tp_plan=&quot;auto&quot;</code> to <a href="/docs/transformers/main/en/model_doc/auto#transformers.AutoModel.from_pretrained">from_pretrained()</a>:',G,M,L,f,D="You can use <code>torchrun</code> to launch the above script with multiple processes, each mapping to a GPU:",H,h,Q,y,O="PyTorch tensor parallel is currently supported for the following models:",V,b,ee='<li><a href="https://huggingface.co/docs/transformers/model_doc/llama#transformers.LlamaModel" rel="nofollow">Llama</a></li>',z,w,te="You can request to add tensor parallel support for another model by opening a GitHub Issue or Pull Request.",R,j,X,T,le="You can benefit from considerable speedups for inference, especially for inputs with large batch size or long sequences.",Y,_,se='For a single forward pass on <a href="https://huggingface.co/docs/transformers/model_doc/llama#transformers.LlamaModel" rel="nofollow">Llama</a> with a sequence length of 512 and various batch sizes, the expected speedup is as follows:',q,m,ne='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/Meta-Llama-3-8B-Instruct%2C%20seqlen%20%3D%20512%2C%20python%2C%20w_%20compile.png"/>',P,g,F,v,S;return u=new ie({props:{title:"Multi-GPU inference",local:"multi-gpu-inference",headingTag:"h1"}}),M=new pe({props:{code:"aW1wb3J0JTIwb3MlMEFpbXBvcnQlMjB0b3JjaCUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUyQyUyMEF1dG9Ub2tlbml6ZXIlMEElMEFtb2RlbF9pZCUyMCUzRCUyMCUyMm1ldGEtbGxhbWElMkZNZXRhLUxsYW1hLTMtOEItSW5zdHJ1Y3QlMjIlMEElMEElMjMlMjBJbml0aWFsaXplJTIwZGlzdHJpYnV0ZWQlMEFyYW5rJTIwJTNEJTIwaW50KG9zLmVudmlyb24lNUIlMjJSQU5LJTIyJTVEKSUwQWRldmljZSUyMCUzRCUyMHRvcmNoLmRldmljZShmJTIyY3VkYSUzQSU3QnJhbmslN0QlMjIpJTBBdG9yY2guZGlzdHJpYnV0ZWQuaW5pdF9wcm9jZXNzX2dyb3VwKCUyMm5jY2wlMjIlMkMlMjBkZXZpY2VfaWQlM0RkZXZpY2UpJTBBJTBBJTIzJTIwUmV0cmlldmUlMjB0ZW5zb3IlMjBwYXJhbGxlbCUyMG1vZGVsJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwbW9kZWxfaWQlMkMlMEElMjAlMjAlMjAlMjB0cF9wbGFuJTNEJTIyYXV0byUyMiUyQyUwQSklMEElMEElMjMlMjBQcmVwYXJlJTIwaW5wdXQlMjB0b2tlbnMlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChtb2RlbF9pZCklMEFwcm9tcHQlMjAlM0QlMjAlMjJDYW4lMjBJJTIwaGVscCUyMiUwQWlucHV0cyUyMCUzRCUyMHRva2VuaXplcihwcm9tcHQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS5pbnB1dF9pZHMudG8oZGV2aWNlKSUwQSUwQSUyMyUyMERpc3RyaWJ1dGVkJTIwcnVuJTBBb3V0cHV0cyUyMCUzRCUyMG1vZGVsKGlucHV0cyk=",highlighted:`<span class="hljs-keyword">import</span> os
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

model_id = <span class="hljs-string">&quot;meta-llama/Meta-Llama-3-8B-Instruct&quot;</span>

<span class="hljs-comment"># Initialize distributed</span>
rank = <span class="hljs-built_in">int</span>(os.environ[<span class="hljs-string">&quot;RANK&quot;</span>])
device = torch.device(<span class="hljs-string">f&quot;cuda:<span class="hljs-subst">{rank}</span>&quot;</span>)
torch.distributed.init_process_group(<span class="hljs-string">&quot;nccl&quot;</span>, device_id=device)

<span class="hljs-comment"># Retrieve tensor parallel model</span>
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    tp_plan=<span class="hljs-string">&quot;auto&quot;</span>,
)

<span class="hljs-comment"># Prepare input tokens</span>
tokenizer = AutoTokenizer.from_pretrained(model_id)
prompt = <span class="hljs-string">&quot;Can I help&quot;</span>
inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).input_ids.to(device)

<span class="hljs-comment"># Distributed run</span>
outputs = model(inputs)`,wrap:!1}}),h=new pe({props:{code:"dG9yY2hydW4lMjAtLW5wcm9jLXBlci1ub2RlJTIwNCUyMGRlbW8ucHk=",highlighted:'torchrun --nproc-per-<span class="hljs-keyword">node</span> <span class="hljs-title">4</span> demo.py',wrap:!1}}),j=new ie({props:{title:"Expected speedups",local:"expected-speedups",headingTag:"h3"}}),g=new ye({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/perf_infer_gpu_multi.md"}}),{c(){i=o("meta"),k=n(),U=o("p"),Z=n(),C(u.$$.fragment),E=n(),c=o("p"),c.textContent=N,I=n(),d=o("p"),d.innerHTML=K,G=n(),C(M.$$.fragment),L=n(),f=o("p"),f.innerHTML=D,H=n(),C(h.$$.fragment),Q=n(),y=o("p"),y.textContent=O,V=n(),b=o("ul"),b.innerHTML=ee,z=n(),w=o("p"),w.textContent=te,R=n(),C(j.$$.fragment),X=n(),T=o("p"),T.textContent=le,Y=n(),_=o("p"),_.innerHTML=se,q=n(),m=o("div"),m.innerHTML=ne,P=n(),C(g.$$.fragment),F=n(),v=o("p"),this.h()},l(e){const t=Me("svelte-u9bgzb",document.head);i=p(t,"META",{name:!0,content:!0}),t.forEach(l),k=a(e),U=p(e,"P",{}),ae(U).forEach(l),Z=a(e),J(u.$$.fragment,e),E=a(e),c=p(e,"P",{"data-svelte-h":!0}),r(c)!=="svelte-18sueh2"&&(c.textContent=N),I=a(e),d=p(e,"P",{"data-svelte-h":!0}),r(d)!=="svelte-11ptn98"&&(d.innerHTML=K),G=a(e),J(M.$$.fragment,e),L=a(e),f=p(e,"P",{"data-svelte-h":!0}),r(f)!=="svelte-ey9nzh"&&(f.innerHTML=D),H=a(e),J(h.$$.fragment,e),Q=a(e),y=p(e,"P",{"data-svelte-h":!0}),r(y)!=="svelte-188qg3a"&&(y.textContent=O),V=a(e),b=p(e,"UL",{"data-svelte-h":!0}),r(b)!=="svelte-wihcs3"&&(b.innerHTML=ee),z=a(e),w=p(e,"P",{"data-svelte-h":!0}),r(w)!=="svelte-1s0fi8f"&&(w.textContent=te),R=a(e),J(j.$$.fragment,e),X=a(e),T=p(e,"P",{"data-svelte-h":!0}),r(T)!=="svelte-yctpd2"&&(T.textContent=le),Y=a(e),_=p(e,"P",{"data-svelte-h":!0}),r(_)!=="svelte-192kbjl"&&(_.innerHTML=se),q=a(e),m=p(e,"DIV",{style:!0,"data-svelte-h":!0}),r(m)!=="svelte-lg3kfi"&&(m.innerHTML=ne),P=a(e),J(g.$$.fragment,e),F=a(e),v=p(e,"P",{}),ae(v).forEach(l),this.h()},h(){oe(i,"name","hf:doc:metadata"),oe(i,"content",we),fe(m,"text-align","center")},m(e,t){he(document.head,i),s(e,k,t),s(e,U,t),s(e,Z,t),$(u,e,t),s(e,E,t),s(e,c,t),s(e,I,t),s(e,d,t),s(e,G,t),$(M,e,t),s(e,L,t),s(e,f,t),s(e,H,t),$(h,e,t),s(e,Q,t),s(e,y,t),s(e,V,t),s(e,b,t),s(e,z,t),s(e,w,t),s(e,R,t),$(j,e,t),s(e,X,t),s(e,T,t),s(e,Y,t),s(e,_,t),s(e,q,t),s(e,m,t),s(e,P,t),$(g,e,t),s(e,F,t),s(e,v,t),S=!0},p:me,i(e){S||(B(u.$$.fragment,e),B(M.$$.fragment,e),B(h.$$.fragment,e),B(j.$$.fragment,e),B(g.$$.fragment,e),S=!0)},o(e){W(u.$$.fragment,e),W(M.$$.fragment,e),W(h.$$.fragment,e),W(j.$$.fragment,e),W(g.$$.fragment,e),S=!1},d(e){e&&(l(k),l(U),l(Z),l(E),l(c),l(I),l(d),l(G),l(L),l(f),l(H),l(Q),l(y),l(V),l(b),l(z),l(w),l(R),l(X),l(T),l(Y),l(_),l(q),l(m),l(P),l(F),l(v)),l(i),x(u,e),x(M,e),x(h,e),x(j,e),x(g,e)}}}const we='{"title":"Multi-GPU inference","local":"multi-gpu-inference","sections":[{"title":"Expected speedups","local":"expected-speedups","sections":[],"depth":3}],"depth":1}';function je(A){return ue(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ve extends ce{constructor(i){super(),de(this,i,je,be,re,{})}}export{ve as component};
