import{s as de,o as ce,n as ue}from"../chunks/scheduler.01eeda35.js";import{S as fe,i as he,g as r,s,r as v,A as Me,h as i,f as l,c as o,j as me,u as U,x as p,k as pe,l as we,y as Te,a,v as g,d as x,t as E,w as W}from"../chunks/index.6dd51b66.js";import{T as ye}from"../chunks/Tip.de9bae2b.js";import{C as O}from"../chunks/CodeBlock.19ec9b8c.js";import{H as be,E as _e}from"../chunks/index.58fe8f9d.js";function Je(Z){let n,u="Expand the list below to see which models support tensor parallelism. Open a GitHub issue or pull request to add support for a model not currently below.";return{c(){n=r("p"),n.textContent=u},l(m){n=i(m,"P",{"data-svelte-h":!0}),p(n)!=="svelte-1awp5l7"&&(n.textContent=u)},m(m,L){a(m,n,L)},p:ue,d(m){m&&l(n)}}}function Ie(Z){let n,u,m,L,f,k,h,K='<a href="./perf_train_gpu_many#tensor-parallelism">Tensor parallelism</a> shards a model onto multiple GPUs and parallelizes computations such as matrix multiplication. It enables fitting larger model sizes into memory and is faster because each GPU can process a tensor slice.',B,d,X,M,ee='<summary>Supported models</summary> <ul><li><a href="./model_doc/cohere">Cohere</a> and <a href="./model_doc/cohere2">Cohere 2</a></li> <li><a href="./model_doc/gemma">Gemma</a> and <a href="./model_doc/gemma2">Gemma 2</a></li> <li><a href="./model_doc/glm">GLM</a></li> <li><a href="./model_doc/granite">Granite</a></li> <li><a href="./model_doc/llama">Llama</a></li> <li><a href="./model_doc/mistral">Mistral</a></li> <li><a href="./model_doc/mixtral">Mixtral</a></li> <li><a href="./model_doc/olmo">OLMo</a> and <a href="./model_doc/olmo2">OLMo2</a></li> <li><a href="./model_doc/phi">Phi</a> and <a href="./model_doc/phi3">Phi-3</a></li> <li><a href="./model_doc/qwen2">Qwen2</a>, <a href="./model_doc/qwen2_moe">Qwen2Moe</a>, and <a href="./model_doc/qwen2_5_vl">Qwen2-VL</a></li> <li><a href="./model_doc/starcoder2">Starcoder2</a></li></ul>',z,w,te='Set <code>tp_plan=&quot;auto&quot;</code> in <a href="/docs/transformers/main/en/model_doc/auto#transformers.AutoModel.from_pretrained">from_pretrained()</a> to enable tensor parallelism for inference.',R,T,A,y,le='Launch the inference script above on <a href="https://pytorch.org/docs/stable/elastic/run.html" rel="nofollow">torchrun</a> with 4 processes per GPU.',H,b,q,_,ae="For CPU, please binding different socket on each rank. For example, if you are using Intel 4th Gen Xeon:",P,J,Y,I,ne="The CPU benchmark data will be released soon.",F,$,se="You can benefit from considerable speed ups for inference, especially for inputs with large batch size or long sequences.",Q,j,oe='For a single forward pass on <a href="./model_doc/llama">Llama</a> with a sequence length of 512 and various batch sizes, you can expect the following speed ups.',S,c,re='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/Meta-Llama-3-8B-Instruct%2C%20seqlen%20%3D%20512%2C%20python%2C%20w_%20compile.png"/>',V,C,N,G,D;return f=new be({props:{title:"Distributed GPU inference",local:"distributed-gpu-inference",headingTag:"h1"}}),d=new ye({props:{warning:!1,$$slots:{default:[Je]},$$scope:{ctx:Z}}}),T=new O({props:{code:"aW1wb3J0JTIwb3MlMEFpbXBvcnQlMjB0b3JjaCUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUyQyUyMEF1dG9Ub2tlbml6ZXIlMEElMEElMEElMjMlMjBlbmFibGUlMjB0ZW5zb3IlMjBwYXJhbGxlbGlzbSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMm1ldGEtbGxhbWElMkZNZXRhLUxsYW1hLTMtOEItSW5zdHJ1Y3QlMjIlMkMlMEElMjAlMjAlMjAlMjB0cF9wbGFuJTNEJTIyYXV0byUyMiUyQyUwQSklMEElMEElMjMlMjBwcmVwYXJlJTIwaW5wdXQlMjB0b2tlbnMlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJtZXRhLWxsYW1hJTJGTWV0YS1MbGFtYS0zLThCLUluc3RydWN0JTIyKSUwQXByb21wdCUyMCUzRCUyMCUyMkNhbiUyMEklMjBoZWxwJTIyJTBBaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHByb21wdCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLmlucHV0X2lkcy50byhtb2RlbC5kZXZpY2UpJTBBJTBBJTIzJTIwZGlzdHJpYnV0ZWQlMjBydW4lMEFvdXRwdXRzJTIwJTNEJTIwbW9kZWwoaW5wdXRzKQ==",highlighted:`<span class="hljs-keyword">import</span> os
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer


<span class="hljs-comment"># enable tensor parallelism</span>
model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;meta-llama/Meta-Llama-3-8B-Instruct&quot;</span>,
    tp_plan=<span class="hljs-string">&quot;auto&quot;</span>,
)

<span class="hljs-comment"># prepare input tokens</span>
tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;meta-llama/Meta-Llama-3-8B-Instruct&quot;</span>)
prompt = <span class="hljs-string">&quot;Can I help&quot;</span>
inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).input_ids.to(model.device)

<span class="hljs-comment"># distributed run</span>
outputs = model(inputs)`,wrap:!1}}),b=new O({props:{code:"dG9yY2hydW4lMjAtLW5wcm9jLXBlci1ub2RlJTIwNCUyMGRlbW8ucHk=",highlighted:"torchrun --nproc-per-node 4 demo.py",wrap:!1}}),J=new O({props:{code:"ZXhwb3J0JTIwT01QX05VTV9USFJFQURTJTNENTYlMEFudW1hY3RsJTIwLUMlMjAwLTU1JTIwLW0lMjAwJTIwdG9yY2hydW4lMjAtLW5ub2RlcyUzRDIlMjAtLW5vZGVfcmFuayUzRDAlMjAtLW1hc3Rlcl9hZGRyJTNEJTIyMTI3LjAuMC4xJTIyJTIwLS1tYXN0ZXJfcG9ydCUzRDI5NTAwJTIwLS1ucHJvYy1wZXItbm9kZSUyMDElMjBkZW1vLnB5JTIwJTI2JTIwbnVtYWN0bCUyMC1DJTIwNTYtMTExJTIwLW0lMjAxJTIwdG9yY2hydW4lMjAtLW5ub2RlcyUzRDIlMjAtLW5vZGVfcmFuayUzRDElMjAtLW1hc3Rlcl9hZGRyJTNEJTIyMTI3LjAuMC4xJTIyJTIwLS1tYXN0ZXJfcG9ydCUzRDI5NTAwJTIwLS1ucHJvYy1wZXItbm9kZSUyMDElMjBkZW1vLnB5JTIwJTI2JTIwd2FpdA==",highlighted:`<span class="hljs-built_in">export</span> OMP_NUM_THREADS=56
numactl -C 0-55 -m 0 torchrun --nnodes=2 --node_rank=0 --master_addr=<span class="hljs-string">&quot;127.0.0.1&quot;</span> --master_port=29500 --nproc-per-node 1 demo.py &amp; numactl -C 56-111 -m 1 torchrun --nnodes=2 --node_rank=1 --master_addr=<span class="hljs-string">&quot;127.0.0.1&quot;</span> --master_port=29500 --nproc-per-node 1 demo.py &amp; <span class="hljs-built_in">wait</span>`,wrap:!1}}),C=new _e({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/perf_infer_gpu_multi.md"}}),{c(){n=r("meta"),u=s(),m=r("p"),L=s(),v(f.$$.fragment),k=s(),h=r("p"),h.innerHTML=K,B=s(),v(d.$$.fragment),X=s(),M=r("details"),M.innerHTML=ee,z=s(),w=r("p"),w.innerHTML=te,R=s(),v(T.$$.fragment),A=s(),y=r("p"),y.innerHTML=le,H=s(),v(b.$$.fragment),q=s(),_=r("p"),_.textContent=ae,P=s(),v(J.$$.fragment),Y=s(),I=r("p"),I.textContent=ne,F=s(),$=r("p"),$.textContent=se,Q=s(),j=r("p"),j.innerHTML=oe,S=s(),c=r("div"),c.innerHTML=re,V=s(),v(C.$$.fragment),N=s(),G=r("p"),this.h()},l(e){const t=Me("svelte-u9bgzb",document.head);n=i(t,"META",{name:!0,content:!0}),t.forEach(l),u=o(e),m=i(e,"P",{}),me(m).forEach(l),L=o(e),U(f.$$.fragment,e),k=o(e),h=i(e,"P",{"data-svelte-h":!0}),p(h)!=="svelte-hxh0ev"&&(h.innerHTML=K),B=o(e),U(d.$$.fragment,e),X=o(e),M=i(e,"DETAILS",{"data-svelte-h":!0}),p(M)!=="svelte-svh7x"&&(M.innerHTML=ee),z=o(e),w=i(e,"P",{"data-svelte-h":!0}),p(w)!=="svelte-1cfuw7o"&&(w.innerHTML=te),R=o(e),U(T.$$.fragment,e),A=o(e),y=i(e,"P",{"data-svelte-h":!0}),p(y)!=="svelte-968uev"&&(y.innerHTML=le),H=o(e),U(b.$$.fragment,e),q=o(e),_=i(e,"P",{"data-svelte-h":!0}),p(_)!=="svelte-97lh6d"&&(_.textContent=ae),P=o(e),U(J.$$.fragment,e),Y=o(e),I=i(e,"P",{"data-svelte-h":!0}),p(I)!=="svelte-10l7ig9"&&(I.textContent=ne),F=o(e),$=i(e,"P",{"data-svelte-h":!0}),p($)!=="svelte-8zwnyi"&&($.textContent=se),Q=o(e),j=i(e,"P",{"data-svelte-h":!0}),p(j)!=="svelte-bszveo"&&(j.innerHTML=oe),S=o(e),c=i(e,"DIV",{style:!0,"data-svelte-h":!0}),p(c)!=="svelte-lg3kfi"&&(c.innerHTML=re),V=o(e),U(C.$$.fragment,e),N=o(e),G=i(e,"P",{}),me(G).forEach(l),this.h()},h(){pe(n,"name","hf:doc:metadata"),pe(n,"content",$e),we(c,"text-align","center")},m(e,t){Te(document.head,n),a(e,u,t),a(e,m,t),a(e,L,t),g(f,e,t),a(e,k,t),a(e,h,t),a(e,B,t),g(d,e,t),a(e,X,t),a(e,M,t),a(e,z,t),a(e,w,t),a(e,R,t),g(T,e,t),a(e,A,t),a(e,y,t),a(e,H,t),g(b,e,t),a(e,q,t),a(e,_,t),a(e,P,t),g(J,e,t),a(e,Y,t),a(e,I,t),a(e,F,t),a(e,$,t),a(e,Q,t),a(e,j,t),a(e,S,t),a(e,c,t),a(e,V,t),g(C,e,t),a(e,N,t),a(e,G,t),D=!0},p(e,[t]){const ie={};t&2&&(ie.$$scope={dirty:t,ctx:e}),d.$set(ie)},i(e){D||(x(f.$$.fragment,e),x(d.$$.fragment,e),x(T.$$.fragment,e),x(b.$$.fragment,e),x(J.$$.fragment,e),x(C.$$.fragment,e),D=!0)},o(e){E(f.$$.fragment,e),E(d.$$.fragment,e),E(T.$$.fragment,e),E(b.$$.fragment,e),E(J.$$.fragment,e),E(C.$$.fragment,e),D=!1},d(e){e&&(l(u),l(m),l(L),l(k),l(h),l(B),l(X),l(M),l(z),l(w),l(R),l(A),l(y),l(H),l(q),l(_),l(P),l(Y),l(I),l(F),l($),l(Q),l(j),l(S),l(c),l(V),l(N),l(G)),l(n),W(f,e),W(d,e),W(T,e),W(b,e),W(J,e),W(C,e)}}}const $e='{"title":"Distributed GPU inference","local":"distributed-gpu-inference","sections":[],"depth":1}';function je(Z){return ce(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class xe extends fe{constructor(n){super(),he(this,n,je,Ie,de,{})}}export{xe as component};
