import{s as ce,n as pe,o as me}from"../chunks/scheduler.25b97de1.js";import{S as de,i as fe,g as n,s as o,r as U,A as ue,h as r,f as l,c as a,j as ae,u as J,x as c,k as ne,l as he,y as Me,a as s,v as $,d as B,t as x,w as W}from"../chunks/index.d9030fc9.js";import{C as re}from"../chunks/CodeBlock.e6cd0d95.js";import{H as ie,E as ge}from"../chunks/EditOnGithub.91d95064.js";function we(A){let i,k,v,E,m,G,d,N="Built-in Tensor Parallelism (TP) is now available with certain models using PyTorch. Tensor parallelism shards a model onto multiple GPUs, enabling larger model sizes, and parallelizes computations such as matrix multiplication.",Z,f,K='To enable tensor parallel, pass the argument <code>tp_plan=&quot;auto&quot;</code> to <a href="/docs/transformers/main/en/model_doc/auto#transformers.AutoModel.from_pretrained">from_pretrained()</a>:',I,u,L,h,D="You can use <code>torchrun</code> to launch the above script with multiple processes, each mapping to a GPU:",Q,M,V,g,O="PyTorch tensor parallel is currently supported for the following models:",H,w,ee='<li><a href="https://huggingface.co/docs/transformers/model_doc/llama#transformers.LlamaModel" rel="nofollow">Llama</a></li> <li><a href="https://huggingface.co/docs/transformers/en/model_doc/gemma" rel="nofollow">Gemma</a>, <a href="https://huggingface.co/docs/transformers/en/model_doc/gemma2" rel="nofollow">Gemma2</a></li> <li><a href="https://huggingface.co/docs/transformers/en/model_doc/granite" rel="nofollow">Granite</a></li> <li><a href="https://huggingface.co/docs/transformers/en/model_doc/mistral" rel="nofollow">Mistral</a></li> <li><a href="https://huggingface.co/docs/transformers/en/model_doc/qwen2" rel="nofollow">Qwen2</a>, <a href="https://huggingface.co/docs/transformers/en/model_doc/qwen2_moe" rel="nofollow">Qwen2MoE</a>, <a href="https://huggingface.co/docs/transformers/v4.48.0/en/model_doc/qwen2_vl" rel="nofollow">Qwen2-VL</a></li> <li><a href="https://huggingface.co/docs/transformers/en/model_doc/starcoder2" rel="nofollow">Starcoder2</a></li> <li><a href="https://huggingface.co/docs/transformers/en/model_doc/cohere" rel="nofollow">Cohere</a>, <a href="https://huggingface.co/docs/transformers/en/model_doc/cohere2" rel="nofollow">Cohere2</a></li> <li><a href="https://huggingface.co/docs/transformers/en/model_doc/glm" rel="nofollow">GLM</a></li> <li><a href="https://huggingface.co/docs/transformers/en/model_doc/mixtral" rel="nofollow">Mixtral</a></li> <li><a href="https://huggingface.co/docs/transformers/en/model_doc/olmo" rel="nofollow">OLMo</a>, <a href="https://huggingface.co/docs/transformers/en/model_doc/olmo2" rel="nofollow">OLMo2</a></li> <li><a href="https://huggingface.co/docs/transformers/en/model_doc/phi" rel="nofollow">Phi</a>, <a href="https://huggingface.co/docs/transformers/en/model_doc/phi3" rel="nofollow">Phi-3</a></li>',q,y,te="You can request to add tensor parallel support for another model by opening a GitHub Issue or Pull Request.",z,_,R,b,le="You can benefit from considerable speedups for inference, especially for inputs with large batch size or long sequences.",P,j,se='For a single forward pass on <a href="https://huggingface.co/docs/transformers/model_doc/llama#transformers.LlamaModel" rel="nofollow">Llama</a> with a sequence length of 512 and various batch sizes, the expected speedup is as follows:',X,p,oe='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/Meta-Llama-3-8B-Instruct%2C%20seqlen%20%3D%20512%2C%20python%2C%20w_%20compile.png"/>',Y,T,S,C,F;return m=new ie({props:{title:"Multi-GPU inference",local:"multi-gpu-inference",headingTag:"h1"}}),u=new re({props:{code:"aW1wb3J0JTIwb3MlMEFpbXBvcnQlMjB0b3JjaCUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUyQyUyMEF1dG9Ub2tlbml6ZXIlMEElMEFtb2RlbF9pZCUyMCUzRCUyMCUyMm1ldGEtbGxhbWElMkZNZXRhLUxsYW1hLTMtOEItSW5zdHJ1Y3QlMjIlMEElMEElMjMlMjBJbml0aWFsaXplJTIwZGlzdHJpYnV0ZWQlMEFyYW5rJTIwJTNEJTIwaW50KG9zLmVudmlyb24lNUIlMjJSQU5LJTIyJTVEKSUwQWRldmljZSUyMCUzRCUyMHRvcmNoLmRldmljZShmJTIyY3VkYSUzQSU3QnJhbmslN0QlMjIpJTBBdG9yY2guZGlzdHJpYnV0ZWQuaW5pdF9wcm9jZXNzX2dyb3VwKCUyMm5jY2wlMjIlMkMlMjBkZXZpY2VfaWQlM0RkZXZpY2UpJTBBJTBBJTIzJTIwUmV0cmlldmUlMjB0ZW5zb3IlMjBwYXJhbGxlbCUyMG1vZGVsJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwbW9kZWxfaWQlMkMlMEElMjAlMjAlMjAlMjB0cF9wbGFuJTNEJTIyYXV0byUyMiUyQyUwQSklMEElMEElMjMlMjBQcmVwYXJlJTIwaW5wdXQlMjB0b2tlbnMlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChtb2RlbF9pZCklMEFwcm9tcHQlMjAlM0QlMjAlMjJDYW4lMjBJJTIwaGVscCUyMiUwQWlucHV0cyUyMCUzRCUyMHRva2VuaXplcihwcm9tcHQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS5pbnB1dF9pZHMudG8oZGV2aWNlKSUwQSUwQSUyMyUyMERpc3RyaWJ1dGVkJTIwcnVuJTBBb3V0cHV0cyUyMCUzRCUyMG1vZGVsKGlucHV0cyk=",highlighted:`<span class="hljs-keyword">import</span> os
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

model_id = <span class="hljs-string">&quot;meta-llama/Meta-Llama-3-8B-Instruct&quot;</span>

<span class="hljs-comment"># Initialize distributed</span>
rank = <span class="hljs-built_in">int</span>(os.environ[<span class="hljs-string">&quot;RANK&quot;</span>])
device = torch.device(<span class="hljs-string">f&quot;cuda:<span class="hljs-subst">{rank}</span>&quot;</span>)
torch.distributed.init_process_group(<span class="hljs-string">&quot;nccl&quot;</span>, device_id=device)

<span class="hljs-comment"># Retrieve tensor parallel model</span>
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    tp_plan=<span class="hljs-string">&quot;auto&quot;</span>,
)

<span class="hljs-comment"># Prepare input tokens</span>
tokenizer = AutoTokenizer.from_pretrained(model_id)
prompt = <span class="hljs-string">&quot;Can I help&quot;</span>
inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).input_ids.to(device)

<span class="hljs-comment"># Distributed run</span>
outputs = model(inputs)`,wrap:!1}}),M=new re({props:{code:"dG9yY2hydW4lMjAtLW5wcm9jLXBlci1ub2RlJTIwNCUyMGRlbW8ucHk=",highlighted:'torchrun --nproc-per-<span class="hljs-keyword">node</span> <span class="hljs-title">4</span> demo.py',wrap:!1}}),_=new ie({props:{title:"Expected speedups",local:"expected-speedups",headingTag:"h3"}}),T=new ge({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/perf_infer_gpu_multi.md"}}),{c(){i=n("meta"),k=o(),v=n("p"),E=o(),U(m.$$.fragment),G=o(),d=n("p"),d.textContent=N,Z=o(),f=n("p"),f.innerHTML=K,I=o(),U(u.$$.fragment),L=o(),h=n("p"),h.innerHTML=D,Q=o(),U(M.$$.fragment),V=o(),g=n("p"),g.textContent=O,H=o(),w=n("ul"),w.innerHTML=ee,q=o(),y=n("p"),y.textContent=te,z=o(),U(_.$$.fragment),R=o(),b=n("p"),b.textContent=le,P=o(),j=n("p"),j.innerHTML=se,X=o(),p=n("div"),p.innerHTML=oe,Y=o(),U(T.$$.fragment),S=o(),C=n("p"),this.h()},l(e){const t=ue("svelte-u9bgzb",document.head);i=r(t,"META",{name:!0,content:!0}),t.forEach(l),k=a(e),v=r(e,"P",{}),ae(v).forEach(l),E=a(e),J(m.$$.fragment,e),G=a(e),d=r(e,"P",{"data-svelte-h":!0}),c(d)!=="svelte-18sueh2"&&(d.textContent=N),Z=a(e),f=r(e,"P",{"data-svelte-h":!0}),c(f)!=="svelte-11ptn98"&&(f.innerHTML=K),I=a(e),J(u.$$.fragment,e),L=a(e),h=r(e,"P",{"data-svelte-h":!0}),c(h)!=="svelte-ey9nzh"&&(h.innerHTML=D),Q=a(e),J(M.$$.fragment,e),V=a(e),g=r(e,"P",{"data-svelte-h":!0}),c(g)!=="svelte-188qg3a"&&(g.textContent=O),H=a(e),w=r(e,"UL",{"data-svelte-h":!0}),c(w)!=="svelte-1ghi9eu"&&(w.innerHTML=ee),q=a(e),y=r(e,"P",{"data-svelte-h":!0}),c(y)!=="svelte-1s0fi8f"&&(y.textContent=te),z=a(e),J(_.$$.fragment,e),R=a(e),b=r(e,"P",{"data-svelte-h":!0}),c(b)!=="svelte-yctpd2"&&(b.textContent=le),P=a(e),j=r(e,"P",{"data-svelte-h":!0}),c(j)!=="svelte-192kbjl"&&(j.innerHTML=se),X=a(e),p=r(e,"DIV",{style:!0,"data-svelte-h":!0}),c(p)!=="svelte-lg3kfi"&&(p.innerHTML=oe),Y=a(e),J(T.$$.fragment,e),S=a(e),C=r(e,"P",{}),ae(C).forEach(l),this.h()},h(){ne(i,"name","hf:doc:metadata"),ne(i,"content",ye),he(p,"text-align","center")},m(e,t){Me(document.head,i),s(e,k,t),s(e,v,t),s(e,E,t),$(m,e,t),s(e,G,t),s(e,d,t),s(e,Z,t),s(e,f,t),s(e,I,t),$(u,e,t),s(e,L,t),s(e,h,t),s(e,Q,t),$(M,e,t),s(e,V,t),s(e,g,t),s(e,H,t),s(e,w,t),s(e,q,t),s(e,y,t),s(e,z,t),$(_,e,t),s(e,R,t),s(e,b,t),s(e,P,t),s(e,j,t),s(e,X,t),s(e,p,t),s(e,Y,t),$(T,e,t),s(e,S,t),s(e,C,t),F=!0},p:pe,i(e){F||(B(m.$$.fragment,e),B(u.$$.fragment,e),B(M.$$.fragment,e),B(_.$$.fragment,e),B(T.$$.fragment,e),F=!0)},o(e){x(m.$$.fragment,e),x(u.$$.fragment,e),x(M.$$.fragment,e),x(_.$$.fragment,e),x(T.$$.fragment,e),F=!1},d(e){e&&(l(k),l(v),l(E),l(G),l(d),l(Z),l(f),l(I),l(L),l(h),l(Q),l(V),l(g),l(H),l(w),l(q),l(y),l(z),l(R),l(b),l(P),l(j),l(X),l(p),l(Y),l(S),l(C)),l(i),W(m,e),W(u,e),W(M,e),W(_,e),W(T,e)}}}const ye='{"title":"Multi-GPU inference","local":"multi-gpu-inference","sections":[{"title":"Expected speedups","local":"expected-speedups","sections":[],"depth":3}],"depth":1}';function _e(A){return me(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ce extends de{constructor(i){super(),fe(this,i,_e,we,ce,{})}}export{Ce as component};
