import{s as j,n as D,o as I}from"../chunks/scheduler.25b97de1.js";import{S as K,i as W,g as u,s as l,r as C,A as J,h as g,f as n,c as s,j as U,u as E,x as L,k as A,y as N,a,v as M,d as k,t as H,w as q}from"../chunks/index.d9030fc9.js";import{H as O,E as Q}from"../chunks/EditOnGithub.91d95064.js";function V(S){let i,v,h,w,o,B,r,_,c,z="Falcon3 represents a natural evolution from previous releases, emphasizing expanding the models’ science, math, and code capabilities. This iteration includes five base models: Falcon3-1B-Base, Falcon3-3B-Base, Falcon3-Mamba-7B-Base, Falcon3-7B-Base, and Falcon3-10B-Base. In developing these models, we incorporated several key innovations aimed at improving the models’ performances while reducing training costs:",b,m,G=`One pre-training: We conducted a single large-scale pretraining run on the 7B model, using 2048 H100 GPU chips, leveraging 14 trillion tokens featuring web, code, STEM, and curated high-quality and multilingual data.
Depth up-scaling for improved reasoning: Building on recent studies on the effects of model depth, we upscaled the 7B model to a 10B parameters model by duplicating the redundant layers and continuing pre-training with 2TT of high-quality data. This yielded Falcon3-10B-Base which achieves state-of-the-art zero-shot and few-shot performance for models under 13B parameters.
Knowledge distillation for better tiny models: To provide compact and efficient alternatives, we developed Falcon3-1B-Base and Falcon3-3B-Base by leveraging pruning and knowledge distillation techniques, using less than 100GT of curated high-quality data, thereby redefining pre-training efficiency.`,y,d,T,f,R='<li><a href="https://huggingface.co/blog/falcon3" rel="nofollow">Blog post</a></li> <li><a href="https://huggingface.co/collections/tiiuae/falcon3-67605ae03578be86e4e87026" rel="nofollow">Models on Huggingface</a></li>',x,p,F,$,P;return o=new O({props:{title:"Falcon3",local:"falcon3",headingTag:"h1"}}),r=new O({props:{title:"Overview",local:"overview",headingTag:"h2"}}),d=new O({props:{title:"Resources",local:"resources",headingTag:"h2"}}),p=new Q({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/model_doc/falcon3.md"}}),{c(){i=u("meta"),v=l(),h=u("p"),w=l(),C(o.$$.fragment),B=l(),C(r.$$.fragment),_=l(),c=u("p"),c.textContent=z,b=l(),m=u("p"),m.textContent=G,y=l(),C(d.$$.fragment),T=l(),f=u("ul"),f.innerHTML=R,x=l(),C(p.$$.fragment),F=l(),$=u("p"),this.h()},l(e){const t=J("svelte-u9bgzb",document.head);i=g(t,"META",{name:!0,content:!0}),t.forEach(n),v=s(e),h=g(e,"P",{}),U(h).forEach(n),w=s(e),E(o.$$.fragment,e),B=s(e),E(r.$$.fragment,e),_=s(e),c=g(e,"P",{"data-svelte-h":!0}),L(c)!=="svelte-324e7g"&&(c.textContent=z),b=s(e),m=g(e,"P",{"data-svelte-h":!0}),L(m)!=="svelte-1p3hmsh"&&(m.textContent=G),y=s(e),E(d.$$.fragment,e),T=s(e),f=g(e,"UL",{"data-svelte-h":!0}),L(f)!=="svelte-1kksbbi"&&(f.innerHTML=R),x=s(e),E(p.$$.fragment,e),F=s(e),$=g(e,"P",{}),U($).forEach(n),this.h()},h(){A(i,"name","hf:doc:metadata"),A(i,"content",X)},m(e,t){N(document.head,i),a(e,v,t),a(e,h,t),a(e,w,t),M(o,e,t),a(e,B,t),M(r,e,t),a(e,_,t),a(e,c,t),a(e,b,t),a(e,m,t),a(e,y,t),M(d,e,t),a(e,T,t),a(e,f,t),a(e,x,t),M(p,e,t),a(e,F,t),a(e,$,t),P=!0},p:D,i(e){P||(k(o.$$.fragment,e),k(r.$$.fragment,e),k(d.$$.fragment,e),k(p.$$.fragment,e),P=!0)},o(e){H(o.$$.fragment,e),H(r.$$.fragment,e),H(d.$$.fragment,e),H(p.$$.fragment,e),P=!1},d(e){e&&(n(v),n(h),n(w),n(B),n(_),n(c),n(b),n(m),n(y),n(T),n(f),n(x),n(F),n($)),n(i),q(o,e),q(r,e),q(d,e),q(p,e)}}}const X='{"title":"Falcon3","local":"falcon3","sections":[{"title":"Overview","local":"overview","sections":[],"depth":2},{"title":"Resources","local":"resources","sections":[],"depth":2}],"depth":1}';function Y(S){return I(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ne extends K{constructor(i){super(),W(this,i,Y,V,j,{})}}export{ne as component};
