import{s as _s,o as ws,n as vs}from"../chunks/scheduler.bdbef820.js";import{S as ys,i as Ms,g,s as n,r,A as ks,h as f,f as e,c as l,j as js,u as p,x as d,k as bs,y as Ts,a,v as c,d as o,t as m,w as u}from"../chunks/index.c0aea24a.js";import{T as qs}from"../chunks/Tip.31005f7d.js";import{C as z}from"../chunks/CodeBlock.6ccca92e.js";import{H as X,E as Js}from"../chunks/EditOnGithub.725ee0c1.js";function Cs(N){let i,$='Metrics is deprecated in 🤗 Datasets. To learn more about how to use metrics, take a look at the library 🤗 <a href="https://huggingface.co/docs/evaluate/index" rel="nofollow">Evaluate</a>! In addition to metrics, you can find more tools for evaluating models and datasets.';return{c(){i=g("p"),i.innerHTML=$},l(h){i=f(h,"P",{"data-svelte-h":!0}),d(i)!=="svelte-77nnuj"&&(i.innerHTML=$)},m(h,P){a(h,i,P)},p:vs,d(h){h&&e(i)}}}function Hs(N){let i,$,h,P,j,F,x,B,b,os='🤗 Datasets provides various common and NLP-specific <a href="https://huggingface.co/metrics" rel="nofollow">metrics</a> for you to measure your models performance. In this section of the tutorials, you will load a metric and use it to evaluate your models predictions.',S,_,ms='You can see what metrics are available with <a href="/docs/datasets/v2.21.0/en/package_reference/loading_methods#datasets.list_metrics">list_metrics()</a>:',U,w,V,v,A,y,us='It is very easy to load a metric with 🤗 Datasets. In fact, you will notice that it is very similar to loading a dataset! Load a metric from the Hub with <a href="/docs/datasets/v2.21.0/en/package_reference/loading_methods#datasets.load_metric">load_metric()</a>:',Y,M,Q,k,gs="This will load the metric associated with the MRPC dataset from the GLUE benchmark.",D,T,K,q,fs="If you are using a benchmark dataset, you need to select a metric that is associated with the configuration you are using. Select a metric configuration by providing the configuration name:",O,J,ss,C,ts,H,hs='Before you begin using a <a href="/docs/datasets/v2.21.0/en/package_reference/main_classes#datasets.Metric">Metric</a> object, you should get to know it a little better. As with a dataset, you can return some basic information about a metric. For example, access the <code>inputs_description</code> parameter in <a href="/docs/datasets/v2.21.0/en/package_reference/main_classes#datasets.MetricInfo">datasets.MetricInfo</a> to get more information about a metrics expected input format and some usage examples:',es,L,as,Z,ds='Notice for the MRPC configuration, the metric expects the input format to be zero or one. For a complete list of attributes you can return with your metric, take a look at <a href="/docs/datasets/v2.21.0/en/package_reference/main_classes#datasets.MetricInfo">MetricInfo</a>.',ns,G,ls,R,xs='Once you have loaded a metric, you are ready to use it to evaluate a models predictions. Provide the model predictions and references to <a href="/docs/datasets/v2.21.0/en/package_reference/main_classes#datasets.Metric.compute">compute()</a>:',is,W,rs,E,ps,I,cs;return j=new X({props:{title:"Evaluate predictions",local:"evaluate-predictions",headingTag:"h1"}}),x=new qs({props:{warning:!0,$$slots:{default:[Cs]},$$scope:{ctx:N}}}),w=new z({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbGlzdF9tZXRyaWNzJTBBbWV0cmljc19saXN0JTIwJTNEJTIwbGlzdF9tZXRyaWNzKCklMEFsZW4obWV0cmljc19saXN0KSUwQXByaW50KG1ldHJpY3NfbGlzdCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> list_metrics
<span class="hljs-meta">&gt;&gt;&gt; </span>metrics_list = list_metrics()
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">len</span>(metrics_list)
<span class="hljs-number">28</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(metrics_list)
[<span class="hljs-string">&#x27;accuracy&#x27;</span>, <span class="hljs-string">&#x27;bertscore&#x27;</span>, <span class="hljs-string">&#x27;bleu&#x27;</span>, <span class="hljs-string">&#x27;bleurt&#x27;</span>, <span class="hljs-string">&#x27;cer&#x27;</span>, <span class="hljs-string">&#x27;comet&#x27;</span>, <span class="hljs-string">&#x27;coval&#x27;</span>, <span class="hljs-string">&#x27;cuad&#x27;</span>, <span class="hljs-string">&#x27;f1&#x27;</span>, <span class="hljs-string">&#x27;gleu&#x27;</span>, <span class="hljs-string">&#x27;glue&#x27;</span>, <span class="hljs-string">&#x27;indic_glue&#x27;</span>, <span class="hljs-string">&#x27;matthews_correlation&#x27;</span>, <span class="hljs-string">&#x27;meteor&#x27;</span>, <span class="hljs-string">&#x27;pearsonr&#x27;</span>, <span class="hljs-string">&#x27;precision&#x27;</span>, <span class="hljs-string">&#x27;recall&#x27;</span>, <span class="hljs-string">&#x27;rouge&#x27;</span>, <span class="hljs-string">&#x27;sacrebleu&#x27;</span>, <span class="hljs-string">&#x27;sari&#x27;</span>, <span class="hljs-string">&#x27;seqeval&#x27;</span>, <span class="hljs-string">&#x27;spearmanr&#x27;</span>, <span class="hljs-string">&#x27;squad&#x27;</span>, <span class="hljs-string">&#x27;squad_v2&#x27;</span>, <span class="hljs-string">&#x27;super_glue&#x27;</span>, <span class="hljs-string">&#x27;wer&#x27;</span>, <span class="hljs-string">&#x27;wiki_split&#x27;</span>, <span class="hljs-string">&#x27;xnli&#x27;</span>]`,wrap:!1}}),v=new X({props:{title:"Load metric",local:"load-metric",headingTag:"h2"}}),M=new z({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9tZXRyaWMlMEFtZXRyaWMlMjAlM0QlMjBsb2FkX21ldHJpYygnZ2x1ZSclMkMlMjAnbXJwYycp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_metric
<span class="hljs-meta">&gt;&gt;&gt; </span>metric = load_metric(<span class="hljs-string">&#x27;glue&#x27;</span>, <span class="hljs-string">&#x27;mrpc&#x27;</span>)`,wrap:!1}}),T=new X({props:{title:"Select a configuration",local:"select-a-configuration",headingTag:"h2"}}),J=new z({props:{code:"bWV0cmljJTIwJTNEJTIwbG9hZF9tZXRyaWMoJ2dsdWUnJTJDJTIwJ21ycGMnKQ==",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>metric = load_metric(<span class="hljs-string">&#x27;glue&#x27;</span>, <span class="hljs-string">&#x27;mrpc&#x27;</span>)',wrap:!1}}),C=new X({props:{title:"Metrics object",local:"metrics-object",headingTag:"h2"}}),L=new z({props:{code:"cHJpbnQobWV0cmljLmlucHV0c19kZXNjcmlwdGlvbik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(metric.inputs_description)
Compute GLUE evaluation metric associated to each GLUE dataset.
Args:
    predictions: <span class="hljs-built_in">list</span> of predictions to score.
        Each translation should be tokenized into a <span class="hljs-built_in">list</span> of tokens.
    references: <span class="hljs-built_in">list</span> of lists of references <span class="hljs-keyword">for</span> each translation.
        Each reference should be tokenized into a <span class="hljs-built_in">list</span> of tokens.
Returns: depending on the GLUE subset, one <span class="hljs-keyword">or</span> several of:
    <span class="hljs-string">&quot;accuracy&quot;</span>: Accuracy
    <span class="hljs-string">&quot;f1&quot;</span>: F1 score
    <span class="hljs-string">&quot;pearson&quot;</span>: Pearson Correlation
    <span class="hljs-string">&quot;spearmanr&quot;</span>: Spearman Correlation
    <span class="hljs-string">&quot;matthews_correlation&quot;</span>: Matthew Correlation
Examples:
    &gt;&gt;&gt; glue_metric = datasets.load_metric(<span class="hljs-string">&#x27;glue&#x27;</span>, <span class="hljs-string">&#x27;sst2&#x27;</span>)  <span class="hljs-comment"># &#x27;sst2&#x27; or any of [&quot;mnli&quot;, &quot;mnli_mismatched&quot;, &quot;mnli_matched&quot;, &quot;qnli&quot;, &quot;rte&quot;, &quot;wnli&quot;, &quot;hans&quot;]</span>
    &gt;&gt;&gt; references = [<span class="hljs-number">0</span>, <span class="hljs-number">1</span>]
    &gt;&gt;&gt; predictions = [<span class="hljs-number">0</span>, <span class="hljs-number">1</span>]
    &gt;&gt;&gt; results = glue_metric.compute(predictions=predictions, references=references)
    &gt;&gt;&gt; <span class="hljs-built_in">print</span>(results)
    {<span class="hljs-string">&#x27;accuracy&#x27;</span>: <span class="hljs-number">1.0</span>}
    ...
    &gt;&gt;&gt; glue_metric = datasets.load_metric(<span class="hljs-string">&#x27;glue&#x27;</span>, <span class="hljs-string">&#x27;mrpc&#x27;</span>)  <span class="hljs-comment"># &#x27;mrpc&#x27; or &#x27;qqp&#x27;</span>
    &gt;&gt;&gt; references = [<span class="hljs-number">0</span>, <span class="hljs-number">1</span>]
    &gt;&gt;&gt; predictions = [<span class="hljs-number">0</span>, <span class="hljs-number">1</span>]
    &gt;&gt;&gt; results = glue_metric.compute(predictions=predictions, references=references)
    &gt;&gt;&gt; <span class="hljs-built_in">print</span>(results)
    {<span class="hljs-string">&#x27;accuracy&#x27;</span>: <span class="hljs-number">1.0</span>, <span class="hljs-string">&#x27;f1&#x27;</span>: <span class="hljs-number">1.0</span>}
    ...`,wrap:!1}}),G=new X({props:{title:"Compute metric",local:"compute-metric",headingTag:"h2"}}),W=new z({props:{code:"bW9kZWxfcHJlZGljdGlvbnMlMjAlM0QlMjBtb2RlbChtb2RlbF9pbnB1dHMpJTBBZmluYWxfc2NvcmUlMjAlM0QlMjBtZXRyaWMuY29tcHV0ZShwcmVkaWN0aW9ucyUzRG1vZGVsX3ByZWRpY3Rpb25zJTJDJTIwcmVmZXJlbmNlcyUzRGdvbGRfcmVmZXJlbmNlcyk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>model_predictions = model(model_inputs)
<span class="hljs-meta">&gt;&gt;&gt; </span>final_score = metric.compute(predictions=model_predictions, references=gold_references)`,wrap:!1}}),E=new Js({props:{source:"https://github.com/huggingface/datasets/blob/main/docs/source/metrics.mdx"}}),{c(){i=g("meta"),$=n(),h=g("p"),P=n(),r(j.$$.fragment),F=n(),r(x.$$.fragment),B=n(),b=g("p"),b.innerHTML=os,S=n(),_=g("p"),_.innerHTML=ms,U=n(),r(w.$$.fragment),V=n(),r(v.$$.fragment),A=n(),y=g("p"),y.innerHTML=us,Y=n(),r(M.$$.fragment),Q=n(),k=g("p"),k.textContent=gs,D=n(),r(T.$$.fragment),K=n(),q=g("p"),q.textContent=fs,O=n(),r(J.$$.fragment),ss=n(),r(C.$$.fragment),ts=n(),H=g("p"),H.innerHTML=hs,es=n(),r(L.$$.fragment),as=n(),Z=g("p"),Z.innerHTML=ds,ns=n(),r(G.$$.fragment),ls=n(),R=g("p"),R.innerHTML=xs,is=n(),r(W.$$.fragment),rs=n(),r(E.$$.fragment),ps=n(),I=g("p"),this.h()},l(s){const t=ks("svelte-u9bgzb",document.head);i=f(t,"META",{name:!0,content:!0}),t.forEach(e),$=l(s),h=f(s,"P",{}),js(h).forEach(e),P=l(s),p(j.$$.fragment,s),F=l(s),p(x.$$.fragment,s),B=l(s),b=f(s,"P",{"data-svelte-h":!0}),d(b)!=="svelte-1rpmogw"&&(b.innerHTML=os),S=l(s),_=f(s,"P",{"data-svelte-h":!0}),d(_)!=="svelte-1p915ax"&&(_.innerHTML=ms),U=l(s),p(w.$$.fragment,s),V=l(s),p(v.$$.fragment,s),A=l(s),y=f(s,"P",{"data-svelte-h":!0}),d(y)!=="svelte-12chbwz"&&(y.innerHTML=us),Y=l(s),p(M.$$.fragment,s),Q=l(s),k=f(s,"P",{"data-svelte-h":!0}),d(k)!=="svelte-lnaepx"&&(k.textContent=gs),D=l(s),p(T.$$.fragment,s),K=l(s),q=f(s,"P",{"data-svelte-h":!0}),d(q)!=="svelte-ufgbiu"&&(q.textContent=fs),O=l(s),p(J.$$.fragment,s),ss=l(s),p(C.$$.fragment,s),ts=l(s),H=f(s,"P",{"data-svelte-h":!0}),d(H)!=="svelte-ow84lu"&&(H.innerHTML=hs),es=l(s),p(L.$$.fragment,s),as=l(s),Z=f(s,"P",{"data-svelte-h":!0}),d(Z)!=="svelte-1612rq"&&(Z.innerHTML=ds),ns=l(s),p(G.$$.fragment,s),ls=l(s),R=f(s,"P",{"data-svelte-h":!0}),d(R)!=="svelte-1tfhcb9"&&(R.innerHTML=xs),is=l(s),p(W.$$.fragment,s),rs=l(s),p(E.$$.fragment,s),ps=l(s),I=f(s,"P",{}),js(I).forEach(e),this.h()},h(){bs(i,"name","hf:doc:metadata"),bs(i,"content",Ls)},m(s,t){Ts(document.head,i),a(s,$,t),a(s,h,t),a(s,P,t),c(j,s,t),a(s,F,t),c(x,s,t),a(s,B,t),a(s,b,t),a(s,S,t),a(s,_,t),a(s,U,t),c(w,s,t),a(s,V,t),c(v,s,t),a(s,A,t),a(s,y,t),a(s,Y,t),c(M,s,t),a(s,Q,t),a(s,k,t),a(s,D,t),c(T,s,t),a(s,K,t),a(s,q,t),a(s,O,t),c(J,s,t),a(s,ss,t),c(C,s,t),a(s,ts,t),a(s,H,t),a(s,es,t),c(L,s,t),a(s,as,t),a(s,Z,t),a(s,ns,t),c(G,s,t),a(s,ls,t),a(s,R,t),a(s,is,t),c(W,s,t),a(s,rs,t),c(E,s,t),a(s,ps,t),a(s,I,t),cs=!0},p(s,[t]){const $s={};t&2&&($s.$$scope={dirty:t,ctx:s}),x.$set($s)},i(s){cs||(o(j.$$.fragment,s),o(x.$$.fragment,s),o(w.$$.fragment,s),o(v.$$.fragment,s),o(M.$$.fragment,s),o(T.$$.fragment,s),o(J.$$.fragment,s),o(C.$$.fragment,s),o(L.$$.fragment,s),o(G.$$.fragment,s),o(W.$$.fragment,s),o(E.$$.fragment,s),cs=!0)},o(s){m(j.$$.fragment,s),m(x.$$.fragment,s),m(w.$$.fragment,s),m(v.$$.fragment,s),m(M.$$.fragment,s),m(T.$$.fragment,s),m(J.$$.fragment,s),m(C.$$.fragment,s),m(L.$$.fragment,s),m(G.$$.fragment,s),m(W.$$.fragment,s),m(E.$$.fragment,s),cs=!1},d(s){s&&(e($),e(h),e(P),e(F),e(B),e(b),e(S),e(_),e(U),e(V),e(A),e(y),e(Y),e(Q),e(k),e(D),e(K),e(q),e(O),e(ss),e(ts),e(H),e(es),e(as),e(Z),e(ns),e(ls),e(R),e(is),e(rs),e(ps),e(I)),e(i),u(j,s),u(x,s),u(w,s),u(v,s),u(M,s),u(T,s),u(J,s),u(C,s),u(L,s),u(G,s),u(W,s),u(E,s)}}}const Ls='{"title":"Evaluate predictions","local":"evaluate-predictions","sections":[{"title":"Load metric","local":"load-metric","sections":[],"depth":2},{"title":"Select a configuration","local":"select-a-configuration","sections":[],"depth":2},{"title":"Metrics object","local":"metrics-object","sections":[],"depth":2},{"title":"Compute metric","local":"compute-metric","sections":[],"depth":2}],"depth":1}';function Zs(N){return ws(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Is extends ys{constructor(i){super(),Ms(this,i,Zs,Hs,_s,{})}}export{Is as component};
