import{s as mt,n as rt,o as ct}from"../chunks/scheduler.25b97de1.js";import{S as ut,i as Ut,g as o,s as n,r as R,A as Mt,h as p,f as e,c as a,j as at,u as _,x as m,k as ot,l as dt,y as Tt,a as s,v,d as w,t as $,w as k}from"../chunks/index.d9030fc9.js";import{C as pt}from"../chunks/CodeBlock.b38ef023.js";import{H as it,E as ft}from"../chunks/index.676ef4be.js";function yt(W){let i,Q,g,B,c,L,u,F="某些模型现已支持内置的<strong>张量并行</strong>（Tensor Parallelism, TP），并通过 PyTorch 实现。张量并行技术将模型切分到多个 GPU 上，从而支持更大的模型尺寸，并对诸如矩阵乘法等计算任务进行并行化。",I,U,A="要启用张量并行，只需在调用 <code>from_pretrained()</code> 时传递参数 <code>tp_plan=&quot;auto&quot;</code>：",Z,M,G,d,D="您可以使用 <code>torchrun</code> 命令启动上述脚本，多进程模式会自动将每个进程映射到一张 GPU：",H,T,N,f,K="目前，PyTorch 张量并行支持以下模型：",V,y,tt='<li><a href="https://huggingface.co/docs/transformers/model_doc/llama#transformers.LlamaModel" rel="nofollow">Llama</a></li>',x,h,lt="如果您希望对其他模型添加张量并行支持，可以通过提交 GitHub Issue 或 Pull Request 来提出请求。",O,C,z,J,et="对于推理场景（尤其是处理大批量或长序列的输入），张量并行可以显著提升计算速度。",Y,b,st='以下是 <a href="https://huggingface.co/docs/transformers/model_doc/llama#transformers.LlamaModel" rel="nofollow">Llama</a> 模型在序列长度为 512 且不同批量大小情况下的单次前向推理的预期加速效果：',S,r,nt='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/Meta-Llama-3-8B-Instruct%2C%20seqlen%20%3D%20512%2C%20python%2C%20w_%20compile.png"/>',X,j,q,E,P;return c=new it({props:{title:"多GPU推理",local:"多gpu推理",headingTag:"h1"}}),M=new pt({props:{code:"aW1wb3J0JTIwb3MlMEFpbXBvcnQlMjB0b3JjaCUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUyQyUyMEF1dG9Ub2tlbml6ZXIlMEElMEFtb2RlbF9pZCUyMCUzRCUyMCUyMm1ldGEtbGxhbWElMkZNZXRhLUxsYW1hLTMtOEItSW5zdHJ1Y3QlMjIlMEElMEElMjMlMjAlRTUlODglOUQlRTUlQTclOEIlRTUlOEMlOTYlRTUlODglODYlRTUlQjglODMlRTUlQkMlOEYlRTclOEUlQUYlRTUlQTIlODMlMEFyYW5rJTIwJTNEJTIwaW50KG9zLmVudmlyb24lNUIlMjJSQU5LJTIyJTVEKSUwQWRldmljZSUyMCUzRCUyMHRvcmNoLmRldmljZShmJTIyY3VkYSUzQSU3QnJhbmslN0QlMjIpJTBBdG9yY2guY3VkYS5zZXRfZGV2aWNlKGRldmljZSklMEF0b3JjaC5kaXN0cmlidXRlZC5pbml0X3Byb2Nlc3NfZ3JvdXAoJTIybmNjbCUyMiUyQyUyMGRldmljZV9pZCUzRGRldmljZSklMEElMEElMjMlMjAlRTglOEUlQjclRTUlOEYlOTYlRTYlOTQlQUYlRTYlOEMlODElRTUlQkMlQTAlRTklODclOEYlRTUlQjklQjYlRTglQTElOEMlRTclOUElODQlRTYlQTglQTElRTUlOUUlOEIlMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjBtb2RlbF9pZCUyQyUwQSUyMCUyMCUyMCUyMHRwX3BsYW4lM0QlMjJhdXRvJTIyJTJDJTBBKSUwQSUwQSUyMyUyMCVFNSU4NyU4NiVFNSVBNCU4NyVFOCVCRSU5MyVFNSU4NSVBNXRva2VucyUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkKSUwQXByb21wdCUyMCUzRCUyMCUyMkNhbiUyMEklMjBoZWxwJTIyJTBBaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHByb21wdCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLmlucHV0X2lkcy50byhkZXZpY2UpJTBBJTBBJTIzJTIwJUU1JTg4JTg2JUU1JUI4JTgzJUU1JUJDJThGJUU4JUJGJTkwJUU4JUExJThDJTBBb3V0cHV0cyUyMCUzRCUyMG1vZGVsKGlucHV0cyk=",highlighted:`<span class="hljs-keyword">import</span> os
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

model_id = <span class="hljs-string">&quot;meta-llama/Meta-Llama-3-8B-Instruct&quot;</span>

<span class="hljs-comment"># 初始化分布式环境</span>
rank = <span class="hljs-built_in">int</span>(os.environ[<span class="hljs-string">&quot;RANK&quot;</span>])
device = torch.device(<span class="hljs-string">f&quot;cuda:<span class="hljs-subst">{rank}</span>&quot;</span>)
torch.cuda.set_device(device)
torch.distributed.init_process_group(<span class="hljs-string">&quot;nccl&quot;</span>, device_id=device)

<span class="hljs-comment"># 获取支持张量并行的模型</span>
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    tp_plan=<span class="hljs-string">&quot;auto&quot;</span>,
)

<span class="hljs-comment"># 准备输入tokens</span>
tokenizer = AutoTokenizer.from_pretrained(model_id)
prompt = <span class="hljs-string">&quot;Can I help&quot;</span>
inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).input_ids.to(device)

<span class="hljs-comment"># 分布式运行</span>
outputs = model(inputs)`,wrap:!1}}),T=new pt({props:{code:"dG9yY2hydW4lMjAtLW5wcm9jLXBlci1ub2RlJTIwNCUyMGRlbW8ucHk=",highlighted:'torchrun --nproc-per-<span class="hljs-keyword">node</span> <span class="hljs-title">4</span> demo.py',wrap:!1}}),C=new it({props:{title:"预期性能提升",local:"预期性能提升",headingTag:"h3"}}),j=new ft({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/zh/perf_infer_gpu_multi.md"}}),{c(){i=o("meta"),Q=n(),g=o("p"),B=n(),R(c.$$.fragment),L=n(),u=o("p"),u.innerHTML=F,I=n(),U=o("p"),U.innerHTML=A,Z=n(),R(M.$$.fragment),G=n(),d=o("p"),d.innerHTML=D,H=n(),R(T.$$.fragment),N=n(),f=o("p"),f.textContent=K,V=n(),y=o("ul"),y.innerHTML=tt,x=n(),h=o("p"),h.textContent=lt,O=n(),R(C.$$.fragment),z=n(),J=o("p"),J.textContent=et,Y=n(),b=o("p"),b.innerHTML=st,S=n(),r=o("div"),r.innerHTML=nt,X=n(),R(j.$$.fragment),q=n(),E=o("p"),this.h()},l(t){const l=Mt("svelte-u9bgzb",document.head);i=p(l,"META",{name:!0,content:!0}),l.forEach(e),Q=a(t),g=p(t,"P",{}),at(g).forEach(e),B=a(t),_(c.$$.fragment,t),L=a(t),u=p(t,"P",{"data-svelte-h":!0}),m(u)!=="svelte-1rnooy8"&&(u.innerHTML=F),I=a(t),U=p(t,"P",{"data-svelte-h":!0}),m(U)!=="svelte-1lcqrkq"&&(U.innerHTML=A),Z=a(t),_(M.$$.fragment,t),G=a(t),d=p(t,"P",{"data-svelte-h":!0}),m(d)!=="svelte-1u51d1q"&&(d.innerHTML=D),H=a(t),_(T.$$.fragment,t),N=a(t),f=p(t,"P",{"data-svelte-h":!0}),m(f)!=="svelte-15hz8rz"&&(f.textContent=K),V=a(t),y=p(t,"UL",{"data-svelte-h":!0}),m(y)!=="svelte-wihcs3"&&(y.innerHTML=tt),x=a(t),h=p(t,"P",{"data-svelte-h":!0}),m(h)!=="svelte-dhflz7"&&(h.textContent=lt),O=a(t),_(C.$$.fragment,t),z=a(t),J=p(t,"P",{"data-svelte-h":!0}),m(J)!=="svelte-sjssgg"&&(J.textContent=et),Y=a(t),b=p(t,"P",{"data-svelte-h":!0}),m(b)!=="svelte-1gd4c5g"&&(b.innerHTML=st),S=a(t),r=p(t,"DIV",{style:!0,"data-svelte-h":!0}),m(r)!=="svelte-lg3kfi"&&(r.innerHTML=nt),X=a(t),_(j.$$.fragment,t),q=a(t),E=p(t,"P",{}),at(E).forEach(e),this.h()},h(){ot(i,"name","hf:doc:metadata"),ot(i,"content",ht),dt(r,"text-align","center")},m(t,l){Tt(document.head,i),s(t,Q,l),s(t,g,l),s(t,B,l),v(c,t,l),s(t,L,l),s(t,u,l),s(t,I,l),s(t,U,l),s(t,Z,l),v(M,t,l),s(t,G,l),s(t,d,l),s(t,H,l),v(T,t,l),s(t,N,l),s(t,f,l),s(t,V,l),s(t,y,l),s(t,x,l),s(t,h,l),s(t,O,l),v(C,t,l),s(t,z,l),s(t,J,l),s(t,Y,l),s(t,b,l),s(t,S,l),s(t,r,l),s(t,X,l),v(j,t,l),s(t,q,l),s(t,E,l),P=!0},p:rt,i(t){P||(w(c.$$.fragment,t),w(M.$$.fragment,t),w(T.$$.fragment,t),w(C.$$.fragment,t),w(j.$$.fragment,t),P=!0)},o(t){$(c.$$.fragment,t),$(M.$$.fragment,t),$(T.$$.fragment,t),$(C.$$.fragment,t),$(j.$$.fragment,t),P=!1},d(t){t&&(e(Q),e(g),e(B),e(L),e(u),e(I),e(U),e(Z),e(G),e(d),e(H),e(N),e(f),e(V),e(y),e(x),e(h),e(O),e(z),e(J),e(Y),e(b),e(S),e(r),e(X),e(q),e(E)),e(i),k(c,t),k(M,t),k(T,t),k(C,t),k(j,t)}}}const ht='{"title":"多GPU推理","local":"多gpu推理","sections":[{"title":"预期性能提升","local":"预期性能提升","sections":[],"depth":3}],"depth":1}';function Ct(W){return ct(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Et extends ut{constructor(i){super(),Ut(this,i,Ct,yt,mt,{})}}export{Et as component};
