import{s as zt,o as At,n as qt}from"../chunks/scheduler.9991993c.js";import{S as Qt,i as Nt,g as p,s as a,r as c,A as Yt,h as i,f as s,c as n,j as Vt,u as o,x as r,k as Ft,y as Kt,a as l,v as d,d as h,t as f,w as y}from"../chunks/index.7fc9a5e7.js";import{T as Dt}from"../chunks/Tip.9de92fc6.js";import{C as w}from"../chunks/CodeBlock.e11cba92.js";import{H as bt}from"../chunks/Heading.e3de321f.js";function Ot(N){let m,_="请注意，随机创建的模型使用“空”张量进行初始化，这些张量占用内存空间但不填充它（因此随机值是给定时间内该内存块中的任何内容）。在第3步之后，对未初始化的权重执行适合模型/参数种类的随机初始化（例如正态分布），以尽可能提高速度！";return{c(){m=p("p"),m.textContent=_},l(M){m=i(M,"P",{"data-svelte-h":!0}),r(m)!=="svelte-1s8s2m3"&&(m.textContent=_)},m(M,q){l(M,m,q)},p:qt,d(M){M&&s(m)}}}function te(N){let m,_,M,q,$,Y,g,xt="当你想使用一个非常大的预训练模型时，一个挑战是尽量减少对内存的使用。通常从PyTorch开始的工作流程如下：",K,u,kt="<li>用随机权重创建你的模型。</li> <li>加载你的预训练权重。</li> <li>将这些预训练权重放入你的随机模型中。</li>",D,T,Ct="步骤1和2都需要完整版本的模型在内存中，这在大多数情况下不是问题，但如果你的模型开始达到几个GB的大小，这两个副本可能会让你超出内存的限制。更糟糕的是，如果你使用<code>torch.distributed</code>来启动分布式训练，每个进程都会加载预训练模型并将这两个副本存储在内存中。",O,j,tt,U,vt="在本指南中，我们将探讨 Transformers 提供的解决方案来处理这个问题。请注意，这是一个积极开发的领域，因此这里解释的API在将来可能会略有变化。",et,b,st,x,Jt="自4.18.0版本起，占用空间超过10GB的模型检查点将自动分成较小的片段。在使用<code>model.save_pretrained(save_dir)</code>时，您最终会得到几个部分<code>checkpoints</code>（每个的大小都小于10GB）以及一个索引，该索引将参数名称映射到存储它们的文件。",lt,k,Bt="您可以使用<code>max_shard_size</code>参数来控制分片之前的最大大小。为了示例的目的，我们将使用具有较小分片大小的普通大小的模型：让我们以传统的BERT模型为例。",at,C,nt,v,Zt='如果您使用 <a href="%E6%A8%A1%E5%9E%8B%E9%A2%84%E8%AE%AD%E7%BB%83%E4%BF%9D%E5%AD%98"><code>PreTrainedModel.save_pretrained</code></a> 进行保存，您将得到一个新的文件夹，其中包含两个文件：模型的配置和权重：',pt,J,it,B,Gt="现在让我们使用最大分片大小为200MB：",mt,Z,rt,G,Rt="在模型配置文件最上方，我们可以看到三个不同的权重文件，以及一个<code>index.json</code>索引文件。这样的<code>checkpoint</code>可以使用<code>[~PreTrainedModel.from_pretrained]</code>方法完全重新加载：",ct,R,ot,X,Xt="对于大型模型来说，这样做的主要优点是在上述工作流程的步骤2中，每个<code>checkpoint</code>的分片在前一个分片之后加载，从而将内存中的内存使用限制在模型大小加上最大分片的大小。",dt,H,Ht="在后台，索引文件用于确定<code>checkpoint</code>中包含哪些键以及相应的权重存储在哪里。我们可以像加载任何json一样加载该索引，并获得一个字典：",ht,I,ft,W,It="目前元数据仅包括模型的总大小。我们计划在将来添加其他信息：",yt,L,Mt,S,Wt="权重映射是该索引的主要部分，它将每个参数的名称（通常在PyTorch模型的<code>state_dict</code>中找到）映射到存储该参数的文件：",jt,E,wt,P,Lt='如果您想直接在模型内部加载这样的分片<code>checkpoint</code>，而不使用 [<code>PreTrainedModel.from_pretrained</code>](就像您会为完整<code>checkpoint</code>执行 <code>model.load_state_dict()</code> 一样)，您应该使用 <a href="/docs/transformers/main/zh/main_classes/model#transformers.modeling_utils.load_sharded_checkpoint">modeling_utils.load_sharded_checkpoint()</a>：',_t,V,$t,F,gt,z,St="分片<code>checkpoints</code>在上述工作流的第2步中降低了内存使用，但为了在低内存环境中使用该模型，我们建议使用基于 Accelerate 库的工具。",ut,A,Et='请阅读以下指南以获取更多信息：<a href="./main_classes/model#large-model-loading">使用 Accelerate 进行大模型加载</a>',Tt,Q,Ut;return $=new bt({props:{title:"实例化大型模型",local:"实例化大型模型",headingTag:"h1"}}),j=new Dt({props:{$$slots:{default:[Ot]},$$scope:{ctx:N}}}),b=new bt({props:{title:"分片checkpoints",local:"分片checkpoints",headingTag:"h2"}}),C=new w({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbCUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsLmZyb21fcHJldHJhaW5lZCglMjJiZXJ0LWJhc2UtY2FzZWQlMjIp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModel

model = AutoModel.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)`,wrap:!1}}),J=new w({props:{code:"aW1wb3J0JTIwb3MlMEFpbXBvcnQlMjB0ZW1wZmlsZSUwQSUwQXdpdGglMjB0ZW1wZmlsZS5UZW1wb3JhcnlEaXJlY3RvcnkoKSUyMGFzJTIwdG1wX2RpciUzQSUwQSUyMCUyMCUyMCUyMG1vZGVsLnNhdmVfcHJldHJhaW5lZCh0bXBfZGlyKSUwQSUyMCUyMCUyMCUyMHByaW50KHNvcnRlZChvcy5saXN0ZGlyKHRtcF9kaXIpKSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> os
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> tempfile

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">with</span> tempfile.TemporaryDirectory() <span class="hljs-keyword">as</span> tmp_dir:
<span class="hljs-meta">... </span>    model.save_pretrained(tmp_dir)
<span class="hljs-meta">... </span>    <span class="hljs-built_in">print</span>(<span class="hljs-built_in">sorted</span>(os.listdir(tmp_dir)))
[<span class="hljs-string">&#x27;config.json&#x27;</span>, <span class="hljs-string">&#x27;pytorch_model.bin&#x27;</span>]`,wrap:!1}}),Z=new w({props:{code:"d2l0aCUyMHRlbXBmaWxlLlRlbXBvcmFyeURpcmVjdG9yeSgpJTIwYXMlMjB0bXBfZGlyJTNBJTBBJTIwJTIwJTIwJTIwbW9kZWwuc2F2ZV9wcmV0cmFpbmVkKHRtcF9kaXIlMkMlMjBtYXhfc2hhcmRfc2l6ZSUzRCUyMjIwME1CJTIyKSUwQSUyMCUyMCUyMCUyMHByaW50KHNvcnRlZChvcy5saXN0ZGlyKHRtcF9kaXIpKSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">with</span> tempfile.TemporaryDirectory() <span class="hljs-keyword">as</span> tmp_dir:
<span class="hljs-meta">... </span>    model.save_pretrained(tmp_dir, max_shard_size=<span class="hljs-string">&quot;200MB&quot;</span>)
<span class="hljs-meta">... </span>    <span class="hljs-built_in">print</span>(<span class="hljs-built_in">sorted</span>(os.listdir(tmp_dir)))
[<span class="hljs-string">&#x27;config.json&#x27;</span>, <span class="hljs-string">&#x27;pytorch_model-00001-of-00003.bin&#x27;</span>, <span class="hljs-string">&#x27;pytorch_model-00002-of-00003.bin&#x27;</span>, <span class="hljs-string">&#x27;pytorch_model-00003-of-00003.bin&#x27;</span>, <span class="hljs-string">&#x27;pytorch_model.bin.index.json&#x27;</span>]`,wrap:!1}}),R=new w({props:{code:"d2l0aCUyMHRlbXBmaWxlLlRlbXBvcmFyeURpcmVjdG9yeSgpJTIwYXMlMjB0bXBfZGlyJTNBJTBBJTIwJTIwJTIwJTIwbW9kZWwuc2F2ZV9wcmV0cmFpbmVkKHRtcF9kaXIlMkMlMjBtYXhfc2hhcmRfc2l6ZSUzRCUyMjIwME1CJTIyKSUwQSUyMCUyMCUyMCUyMG5ld19tb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQodG1wX2Rpcik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">with</span> tempfile.TemporaryDirectory() <span class="hljs-keyword">as</span> tmp_dir:
<span class="hljs-meta">... </span>    model.save_pretrained(tmp_dir, max_shard_size=<span class="hljs-string">&quot;200MB&quot;</span>)
<span class="hljs-meta">... </span>    new_model = AutoModel.from_pretrained(tmp_dir)`,wrap:!1}}),I=new w({props:{code:"aW1wb3J0JTIwanNvbiUwQSUwQXdpdGglMjB0ZW1wZmlsZS5UZW1wb3JhcnlEaXJlY3RvcnkoKSUyMGFzJTIwdG1wX2RpciUzQSUwQSUyMCUyMCUyMCUyMG1vZGVsLnNhdmVfcHJldHJhaW5lZCh0bXBfZGlyJTJDJTIwbWF4X3NoYXJkX3NpemUlM0QlMjIyMDBNQiUyMiklMEElMjAlMjAlMjAlMjB3aXRoJTIwb3Blbihvcy5wYXRoLmpvaW4odG1wX2RpciUyQyUyMCUyMnB5dG9yY2hfbW9kZWwuYmluLmluZGV4Lmpzb24lMjIpJTJDJTIwJTIyciUyMiklMjBhcyUyMGYlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpbmRleCUyMCUzRCUyMGpzb24ubG9hZChmKSUwQSUwQXByaW50KGluZGV4LmtleXMoKSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> json

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">with</span> tempfile.TemporaryDirectory() <span class="hljs-keyword">as</span> tmp_dir:
<span class="hljs-meta">... </span>    model.save_pretrained(tmp_dir, max_shard_size=<span class="hljs-string">&quot;200MB&quot;</span>)
<span class="hljs-meta">... </span>    <span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(os.path.join(tmp_dir, <span class="hljs-string">&quot;pytorch_model.bin.index.json&quot;</span>), <span class="hljs-string">&quot;r&quot;</span>) <span class="hljs-keyword">as</span> f:
<span class="hljs-meta">... </span>        index = json.load(f)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(index.keys())
dict_keys([<span class="hljs-string">&#x27;metadata&#x27;</span>, <span class="hljs-string">&#x27;weight_map&#x27;</span>])`,wrap:!1}}),L=new w({props:{code:"aW5kZXglNUIlMjJtZXRhZGF0YSUyMiU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>index[<span class="hljs-string">&quot;metadata&quot;</span>]
{<span class="hljs-string">&#x27;total_size&#x27;</span>: <span class="hljs-number">433245184</span>}`,wrap:!1}}),E=new w({props:{code:"aW5kZXglNUIlMjJ3ZWlnaHRfbWFwJTIyJTVE",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>index[<span class="hljs-string">&quot;weight_map&quot;</span>]
{<span class="hljs-string">&#x27;embeddings.LayerNorm.bias&#x27;</span>: <span class="hljs-string">&#x27;pytorch_model-00001-of-00003.bin&#x27;</span>,
 <span class="hljs-string">&#x27;embeddings.LayerNorm.weight&#x27;</span>: <span class="hljs-string">&#x27;pytorch_model-00001-of-00003.bin&#x27;</span>,
 ...`,wrap:!1}}),V=new w({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5tb2RlbGluZ191dGlscyUyMGltcG9ydCUyMGxvYWRfc2hhcmRlZF9jaGVja3BvaW50JTBBJTBBd2l0aCUyMHRlbXBmaWxlLlRlbXBvcmFyeURpcmVjdG9yeSgpJTIwYXMlMjB0bXBfZGlyJTNBJTBBJTIwJTIwJTIwJTIwbW9kZWwuc2F2ZV9wcmV0cmFpbmVkKHRtcF9kaXIlMkMlMjBtYXhfc2hhcmRfc2l6ZSUzRCUyMjIwME1CJTIyKSUwQSUyMCUyMCUyMCUyMGxvYWRfc2hhcmRlZF9jaGVja3BvaW50KG1vZGVsJTJDJTIwdG1wX2Rpcik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers.modeling_utils <span class="hljs-keyword">import</span> load_sharded_checkpoint

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">with</span> tempfile.TemporaryDirectory() <span class="hljs-keyword">as</span> tmp_dir:
<span class="hljs-meta">... </span>    model.save_pretrained(tmp_dir, max_shard_size=<span class="hljs-string">&quot;200MB&quot;</span>)
<span class="hljs-meta">... </span>    load_sharded_checkpoint(model, tmp_dir)`,wrap:!1}}),F=new bt({props:{title:"低内存加载",local:"低内存加载",headingTag:"h2"}}),{c(){m=p("meta"),_=a(),M=p("p"),q=a(),c($.$$.fragment),Y=a(),g=p("p"),g.textContent=xt,K=a(),u=p("ol"),u.innerHTML=kt,D=a(),T=p("p"),T.innerHTML=Ct,O=a(),c(j.$$.fragment),tt=a(),U=p("p"),U.textContent=vt,et=a(),c(b.$$.fragment),st=a(),x=p("p"),x.innerHTML=Jt,lt=a(),k=p("p"),k.innerHTML=Bt,at=a(),c(C.$$.fragment),nt=a(),v=p("p"),v.innerHTML=Zt,pt=a(),c(J.$$.fragment),it=a(),B=p("p"),B.textContent=Gt,mt=a(),c(Z.$$.fragment),rt=a(),G=p("p"),G.innerHTML=Rt,ct=a(),c(R.$$.fragment),ot=a(),X=p("p"),X.innerHTML=Xt,dt=a(),H=p("p"),H.innerHTML=Ht,ht=a(),c(I.$$.fragment),ft=a(),W=p("p"),W.textContent=It,yt=a(),c(L.$$.fragment),Mt=a(),S=p("p"),S.innerHTML=Wt,jt=a(),c(E.$$.fragment),wt=a(),P=p("p"),P.innerHTML=Lt,_t=a(),c(V.$$.fragment),$t=a(),c(F.$$.fragment),gt=a(),z=p("p"),z.innerHTML=St,ut=a(),A=p("p"),A.innerHTML=Et,Tt=a(),Q=p("p"),this.h()},l(t){const e=Yt("svelte-u9bgzb",document.head);m=i(e,"META",{name:!0,content:!0}),e.forEach(s),_=n(t),M=i(t,"P",{}),Vt(M).forEach(s),q=n(t),o($.$$.fragment,t),Y=n(t),g=i(t,"P",{"data-svelte-h":!0}),r(g)!=="svelte-f980nr"&&(g.textContent=xt),K=n(t),u=i(t,"OL",{"data-svelte-h":!0}),r(u)!=="svelte-qi8gup"&&(u.innerHTML=kt),D=n(t),T=i(t,"P",{"data-svelte-h":!0}),r(T)!=="svelte-3m51fj"&&(T.innerHTML=Ct),O=n(t),o(j.$$.fragment,t),tt=n(t),U=i(t,"P",{"data-svelte-h":!0}),r(U)!=="svelte-1fsody1"&&(U.textContent=vt),et=n(t),o(b.$$.fragment,t),st=n(t),x=i(t,"P",{"data-svelte-h":!0}),r(x)!=="svelte-p12rar"&&(x.innerHTML=Jt),lt=n(t),k=i(t,"P",{"data-svelte-h":!0}),r(k)!=="svelte-fnq70n"&&(k.innerHTML=Bt),at=n(t),o(C.$$.fragment,t),nt=n(t),v=i(t,"P",{"data-svelte-h":!0}),r(v)!=="svelte-1vlh5wl"&&(v.innerHTML=Zt),pt=n(t),o(J.$$.fragment,t),it=n(t),B=i(t,"P",{"data-svelte-h":!0}),r(B)!=="svelte-15702pw"&&(B.textContent=Gt),mt=n(t),o(Z.$$.fragment,t),rt=n(t),G=i(t,"P",{"data-svelte-h":!0}),r(G)!=="svelte-jhz32j"&&(G.innerHTML=Rt),ct=n(t),o(R.$$.fragment,t),ot=n(t),X=i(t,"P",{"data-svelte-h":!0}),r(X)!=="svelte-bnpz0s"&&(X.innerHTML=Xt),dt=n(t),H=i(t,"P",{"data-svelte-h":!0}),r(H)!=="svelte-18it1xo"&&(H.innerHTML=Ht),ht=n(t),o(I.$$.fragment,t),ft=n(t),W=i(t,"P",{"data-svelte-h":!0}),r(W)!=="svelte-26dikn"&&(W.textContent=It),yt=n(t),o(L.$$.fragment,t),Mt=n(t),S=i(t,"P",{"data-svelte-h":!0}),r(S)!=="svelte-1lig6ro"&&(S.innerHTML=Wt),jt=n(t),o(E.$$.fragment,t),wt=n(t),P=i(t,"P",{"data-svelte-h":!0}),r(P)!=="svelte-8dbmah"&&(P.innerHTML=Lt),_t=n(t),o(V.$$.fragment,t),$t=n(t),o(F.$$.fragment,t),gt=n(t),z=i(t,"P",{"data-svelte-h":!0}),r(z)!=="svelte-1qqd7tm"&&(z.innerHTML=St),ut=n(t),A=i(t,"P",{"data-svelte-h":!0}),r(A)!=="svelte-kz6dhx"&&(A.innerHTML=Et),Tt=n(t),Q=i(t,"P",{}),Vt(Q).forEach(s),this.h()},h(){Ft(m,"name","hf:doc:metadata"),Ft(m,"content",ee)},m(t,e){Kt(document.head,m),l(t,_,e),l(t,M,e),l(t,q,e),d($,t,e),l(t,Y,e),l(t,g,e),l(t,K,e),l(t,u,e),l(t,D,e),l(t,T,e),l(t,O,e),d(j,t,e),l(t,tt,e),l(t,U,e),l(t,et,e),d(b,t,e),l(t,st,e),l(t,x,e),l(t,lt,e),l(t,k,e),l(t,at,e),d(C,t,e),l(t,nt,e),l(t,v,e),l(t,pt,e),d(J,t,e),l(t,it,e),l(t,B,e),l(t,mt,e),d(Z,t,e),l(t,rt,e),l(t,G,e),l(t,ct,e),d(R,t,e),l(t,ot,e),l(t,X,e),l(t,dt,e),l(t,H,e),l(t,ht,e),d(I,t,e),l(t,ft,e),l(t,W,e),l(t,yt,e),d(L,t,e),l(t,Mt,e),l(t,S,e),l(t,jt,e),d(E,t,e),l(t,wt,e),l(t,P,e),l(t,_t,e),d(V,t,e),l(t,$t,e),d(F,t,e),l(t,gt,e),l(t,z,e),l(t,ut,e),l(t,A,e),l(t,Tt,e),l(t,Q,e),Ut=!0},p(t,[e]){const Pt={};e&2&&(Pt.$$scope={dirty:e,ctx:t}),j.$set(Pt)},i(t){Ut||(h($.$$.fragment,t),h(j.$$.fragment,t),h(b.$$.fragment,t),h(C.$$.fragment,t),h(J.$$.fragment,t),h(Z.$$.fragment,t),h(R.$$.fragment,t),h(I.$$.fragment,t),h(L.$$.fragment,t),h(E.$$.fragment,t),h(V.$$.fragment,t),h(F.$$.fragment,t),Ut=!0)},o(t){f($.$$.fragment,t),f(j.$$.fragment,t),f(b.$$.fragment,t),f(C.$$.fragment,t),f(J.$$.fragment,t),f(Z.$$.fragment,t),f(R.$$.fragment,t),f(I.$$.fragment,t),f(L.$$.fragment,t),f(E.$$.fragment,t),f(V.$$.fragment,t),f(F.$$.fragment,t),Ut=!1},d(t){t&&(s(_),s(M),s(q),s(Y),s(g),s(K),s(u),s(D),s(T),s(O),s(tt),s(U),s(et),s(st),s(x),s(lt),s(k),s(at),s(nt),s(v),s(pt),s(it),s(B),s(mt),s(rt),s(G),s(ct),s(ot),s(X),s(dt),s(H),s(ht),s(ft),s(W),s(yt),s(Mt),s(S),s(jt),s(wt),s(P),s(_t),s($t),s(gt),s(z),s(ut),s(A),s(Tt),s(Q)),s(m),y($,t),y(j,t),y(b,t),y(C,t),y(J,t),y(Z,t),y(R,t),y(I,t),y(L,t),y(E,t),y(V,t),y(F,t)}}}const ee='{"title":"实例化大型模型","local":"实例化大型模型","sections":[{"title":"分片checkpoints","local":"分片checkpoints","sections":[],"depth":2},{"title":"低内存加载","local":"低内存加载","sections":[],"depth":2}],"depth":1}';function se(N){return At(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class me extends Qt{constructor(m){super(),Nt(this,m,se,te,zt,{})}}export{me as component};
