Update docs from 3c5588d

microsoft · Dec 5, 2023 · 034db9b · 034db9b
1 parent e2c061f
commit 034db9b
Show file tree

Hide file tree

Showing 7 changed files with 23 additions and 23 deletions.
diff --git a/_sources/features/custom_scripts.md.txt b/_sources/features/custom_scripts.md.txt
@@ -51,8 +51,8 @@ Use `my_script.py` with Olive workflow configuration json file(sub_types name sh
         "type": "accuracy",
         "sub_types": [
             {"name": "accuracy_score", "priority": 1, "goal": {"type": "max-degradation", "value": 0.01}},
-            {"name": "f1_score", "metric_config": {"multiclass": false}},
-            {"name": "auroc", "metric_config": {"num_classes": 2}}
+            {"name": "f1_score"},
+            {"name": "auroc"}
         ],
         "user_config":{
             "post_processing_func": "post_process",
@@ -103,8 +103,8 @@ Use `script_dir` and `my_script.py` with Olive workflow configuration json file:
         "type": "accuracy",
         "sub_types": [
             {"name": "accuracy_score", "priority": 1, "goal": {"type": "max-degradation", "value": 0.01}},
-            {"name": "f1_score", "metric_config": {"multiclass": false}},
-            {"name": "auroc", "metric_config": {"num_classes": 2}}
+            {"name": "f1_score"},
+            {"name": "auroc"}
         ]
         "user_config":{
             "post_processing_func": "post_process",

diff --git a/_sources/overview/options.md.txt b/_sources/overview/options.md.txt
@@ -233,7 +233,7 @@ information of the evaluator contains following items:
     - `type: [str]` The type of the metric. The supported types are `accuracy`, `latency`, `throughput` and `custom`.
 
     - `backend: [str]` The type of metrics' backend. Olive implement `torch_metrics` and `huggingface_metrics` backends. The default value is `torch_metrics`.
-        - `torch_metrics` backend uses `torchmetrics` library to compute metrics. It supports `accuracy_score`, `f1_score`, `precision`, `recall` and `auc` metrics.
+        - `torch_metrics` backend uses `torchmetrics`(>=0.1.0) library to compute metrics. It supports `accuracy_score`, `f1_score`, `precision`, `recall` and `auroc` metrics which are used for `binary` task (equal to `metric_config:{"task": "binary"}`) by default. You need alter the `task` if needed. Please refer to [torchmetrics](https://lightning.ai/docs/torchmetrics/stable/) for more details.
         - `huggingface_metrics` backend uses huggingface `evaluate` library to compute metrics. The supported metrics can be found at [huggingface metrics](https://huggingface.co/metrics).
 
     - `subtypes: [List[Dict]]` The subtypes of the metric. Cannot be null or empty. Each subtype is a dictionary that contains following items:
@@ -311,8 +311,8 @@ information of the evaluator contains following items:
                 "type": "accuracy",
                 "sub_types": [
                     {"name": "accuracy_score", "priority": 1, "goal": {"type": "max-degradation", "value": 0.01}},
-                    {"name": "f1_score", "metric_config": {"multiclass": false}},
-                    {"name": "auroc", "metric_config": {"num_classes": 2}}
+                    {"name": "f1_score"},
+                    {"name": "auroc"}
                 ],
                 "user_config":{
                     "post_processing_func": "post_process",

diff --git a/_sources/tutorials/configure_metrics.rst.txt b/_sources/tutorials/configure_metrics.rst.txt
@@ -19,8 +19,8 @@ Accuracy Metric
                 "type": "accuracy",
                 "sub_types": [
                     {"name": "accuracy_score", "priority": 1, "goal": {"type": "max-degradation", "value": 0.01}},
-                    {"name": "f1_score", "metric_config": {"multiclass": false}},
-                    {"name": "auroc", "metric_config": {"num_classes": 2}}
+                    {"name": "f1_score"},
+                    {"name": "auroc"}
                 ],
                 "user_config": {
                     "post_processing_func": "post_process",
@@ -227,8 +227,8 @@ If you have multiple metrics to evaluate, you can configure them in the followin
                     "type": "accuracy",
                     "sub_types": [
                         {"name": "accuracy_score", "priority": 1, "goal": {"type": "max-degradation", "value": 0.01}},
-                        {"name": "f1_score", "metric_config": {"multiclass": false}},
-                        {"name": "auroc", "metric_config": {"num_classes": 2}}
+                        {"name": "f1_score"},
+                        {"name": "auroc"}
                     ]
                 },
                 {

diff --git a/features/custom_scripts.html b/features/custom_scripts.html
@@ -184,8 +184,8 @@ <h3>Examples<a class="headerlink" href="#examples" title="Permalink to this head
         <span class="s2">&quot;type&quot;</span><span class="p">:</span> <span class="s2">&quot;accuracy&quot;</span><span class="p">,</span>
         <span class="s2">&quot;sub_types&quot;</span><span class="p">:</span> <span class="p">[</span>
             <span class="p">{</span><span class="s2">&quot;name&quot;</span><span class="p">:</span> <span class="s2">&quot;accuracy_score&quot;</span><span class="p">,</span> <span class="s2">&quot;priority&quot;</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span> <span class="s2">&quot;goal&quot;</span><span class="p">:</span> <span class="p">{</span><span class="s2">&quot;type&quot;</span><span class="p">:</span> <span class="s2">&quot;max-degradation&quot;</span><span class="p">,</span> <span class="s2">&quot;value&quot;</span><span class="p">:</span> <span class="mf">0.01</span><span class="p">}},</span>
-            <span class="p">{</span><span class="s2">&quot;name&quot;</span><span class="p">:</span> <span class="s2">&quot;f1_score&quot;</span><span class="p">,</span> <span class="s2">&quot;metric_config&quot;</span><span class="p">:</span> <span class="p">{</span><span class="s2">&quot;multiclass&quot;</span><span class="p">:</span> <span class="n">false</span><span class="p">}},</span>
-            <span class="p">{</span><span class="s2">&quot;name&quot;</span><span class="p">:</span> <span class="s2">&quot;auroc&quot;</span><span class="p">,</span> <span class="s2">&quot;metric_config&quot;</span><span class="p">:</span> <span class="p">{</span><span class="s2">&quot;num_classes&quot;</span><span class="p">:</span> <span class="mi">2</span><span class="p">}}</span>
+            <span class="p">{</span><span class="s2">&quot;name&quot;</span><span class="p">:</span> <span class="s2">&quot;f1_score&quot;</span><span class="p">},</span>
+            <span class="p">{</span><span class="s2">&quot;name&quot;</span><span class="p">:</span> <span class="s2">&quot;auroc&quot;</span><span class="p">}</span>
         <span class="p">],</span>
         <span class="s2">&quot;user_config&quot;</span><span class="p">:{</span>
             <span class="s2">&quot;post_processing_func&quot;</span><span class="p">:</span> <span class="s2">&quot;post_process&quot;</span><span class="p">,</span>
@@ -230,8 +230,8 @@ <h3>Examples<a class="headerlink" href="#id1" title="Permalink to this heading">
         <span class="s2">&quot;type&quot;</span><span class="p">:</span> <span class="s2">&quot;accuracy&quot;</span><span class="p">,</span>
         <span class="s2">&quot;sub_types&quot;</span><span class="p">:</span> <span class="p">[</span>
             <span class="p">{</span><span class="s2">&quot;name&quot;</span><span class="p">:</span> <span class="s2">&quot;accuracy_score&quot;</span><span class="p">,</span> <span class="s2">&quot;priority&quot;</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span> <span class="s2">&quot;goal&quot;</span><span class="p">:</span> <span class="p">{</span><span class="s2">&quot;type&quot;</span><span class="p">:</span> <span class="s2">&quot;max-degradation&quot;</span><span class="p">,</span> <span class="s2">&quot;value&quot;</span><span class="p">:</span> <span class="mf">0.01</span><span class="p">}},</span>
-            <span class="p">{</span><span class="s2">&quot;name&quot;</span><span class="p">:</span> <span class="s2">&quot;f1_score&quot;</span><span class="p">,</span> <span class="s2">&quot;metric_config&quot;</span><span class="p">:</span> <span class="p">{</span><span class="s2">&quot;multiclass&quot;</span><span class="p">:</span> <span class="n">false</span><span class="p">}},</span>
-            <span class="p">{</span><span class="s2">&quot;name&quot;</span><span class="p">:</span> <span class="s2">&quot;auroc&quot;</span><span class="p">,</span> <span class="s2">&quot;metric_config&quot;</span><span class="p">:</span> <span class="p">{</span><span class="s2">&quot;num_classes&quot;</span><span class="p">:</span> <span class="mi">2</span><span class="p">}}</span>
+            <span class="p">{</span><span class="s2">&quot;name&quot;</span><span class="p">:</span> <span class="s2">&quot;f1_score&quot;</span><span class="p">},</span>
+            <span class="p">{</span><span class="s2">&quot;name&quot;</span><span class="p">:</span> <span class="s2">&quot;auroc&quot;</span><span class="p">}</span>
         <span class="p">]</span>
         <span class="s2">&quot;user_config&quot;</span><span class="p">:{</span>
             <span class="s2">&quot;post_processing_func&quot;</span><span class="p">:</span> <span class="s2">&quot;post_process&quot;</span><span class="p">,</span>

diff --git a/overview/options.html b/overview/options.html
@@ -433,7 +433,7 @@ <h2>Evaluators Information<a class="headerlink" href="#evaluators-information" t
 <li><p><code class="docutils literal notranslate"><span class="pre">type:</span> <span class="pre">[str]</span></code> The type of the metric. The supported types are <code class="docutils literal notranslate"><span class="pre">accuracy</span></code>, <code class="docutils literal notranslate"><span class="pre">latency</span></code>, <code class="docutils literal notranslate"><span class="pre">throughput</span></code> and <code class="docutils literal notranslate"><span class="pre">custom</span></code>.</p></li>
 <li><p><code class="docutils literal notranslate"><span class="pre">backend:</span> <span class="pre">[str]</span></code> The type of metrics’ backend. Olive implement <code class="docutils literal notranslate"><span class="pre">torch_metrics</span></code> and <code class="docutils literal notranslate"><span class="pre">huggingface_metrics</span></code> backends. The default value is <code class="docutils literal notranslate"><span class="pre">torch_metrics</span></code>.</p>
 <ul>
-<li><p><code class="docutils literal notranslate"><span class="pre">torch_metrics</span></code> backend uses <code class="docutils literal notranslate"><span class="pre">torchmetrics</span></code> library to compute metrics. It supports <code class="docutils literal notranslate"><span class="pre">accuracy_score</span></code>, <code class="docutils literal notranslate"><span class="pre">f1_score</span></code>, <code class="docutils literal notranslate"><span class="pre">precision</span></code>, <code class="docutils literal notranslate"><span class="pre">recall</span></code> and <code class="docutils literal notranslate"><span class="pre">auc</span></code> metrics.</p></li>
+<li><p><code class="docutils literal notranslate"><span class="pre">torch_metrics</span></code> backend uses <code class="docutils literal notranslate"><span class="pre">torchmetrics</span></code>(&gt;=0.1.0) library to compute metrics. It supports <code class="docutils literal notranslate"><span class="pre">accuracy_score</span></code>, <code class="docutils literal notranslate"><span class="pre">f1_score</span></code>, <code class="docutils literal notranslate"><span class="pre">precision</span></code>, <code class="docutils literal notranslate"><span class="pre">recall</span></code> and <code class="docutils literal notranslate"><span class="pre">auroc</span></code> metrics which are used for <code class="docutils literal notranslate"><span class="pre">binary</span></code> task (equal to <code class="docutils literal notranslate"><span class="pre">metric_config:{&quot;task&quot;:</span> <span class="pre">&quot;binary&quot;}</span></code>) by default. You need alter the <code class="docutils literal notranslate"><span class="pre">task</span></code> if needed. Please refer to <a class="reference external" href="https://lightning.ai/docs/torchmetrics/stable/">torchmetrics</a> for more details.</p></li>
 <li><p><code class="docutils literal notranslate"><span class="pre">huggingface_metrics</span></code> backend uses huggingface <code class="docutils literal notranslate"><span class="pre">evaluate</span></code> library to compute metrics. The supported metrics can be found at <a class="reference external" href="https://huggingface.co/metrics">huggingface metrics</a>.</p></li>
 </ul>
 </li>
@@ -507,8 +507,8 @@ <h3>Example<a class="headerlink" href="#id3" title="Permalink to this heading">
 <span class="w">                </span><span class="nt">&quot;type&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;accuracy&quot;</span><span class="p">,</span>
 <span class="w">                </span><span class="nt">&quot;sub_types&quot;</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
 <span class="w">                    </span><span class="p">{</span><span class="nt">&quot;name&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;accuracy_score&quot;</span><span class="p">,</span><span class="w"> </span><span class="nt">&quot;priority&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">1</span><span class="p">,</span><span class="w"> </span><span class="nt">&quot;goal&quot;</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="nt">&quot;type&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;max-degradation&quot;</span><span class="p">,</span><span class="w"> </span><span class="nt">&quot;value&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.01</span><span class="p">}},</span>
-<span class="w">                    </span><span class="p">{</span><span class="nt">&quot;name&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;f1_score&quot;</span><span class="p">,</span><span class="w"> </span><span class="nt">&quot;metric_config&quot;</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="nt">&quot;multiclass&quot;</span><span class="p">:</span><span class="w"> </span><span class="kc">false</span><span class="p">}},</span>
-<span class="w">                    </span><span class="p">{</span><span class="nt">&quot;name&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;auroc&quot;</span><span class="p">,</span><span class="w"> </span><span class="nt">&quot;metric_config&quot;</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="nt">&quot;num_classes&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">2</span><span class="p">}}</span>
+<span class="w">                    </span><span class="p">{</span><span class="nt">&quot;name&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;f1_score&quot;</span><span class="p">},</span>
+<span class="w">                    </span><span class="p">{</span><span class="nt">&quot;name&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;auroc&quot;</span><span class="p">}</span>
 <span class="w">                </span><span class="p">],</span>
 <span class="w">                </span><span class="nt">&quot;user_config&quot;</span><span class="p">:{</span>
 <span class="w">                    </span><span class="nt">&quot;post_processing_func&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;post_process&quot;</span><span class="p">,</span>

diff --git a/searchindex.js b/searchindex.js