src/documentation/how_tos/how_to_aggregate_evaluations.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from example_data import DummyAggregationLogic, example_data\n",
    "\n",
    "from intelligence_layer.evaluation.aggregation.aggregator import Aggregator\n",
    "from intelligence_layer.evaluation.aggregation.in_memory_aggregation_repository import (\n",
    "    InMemoryAggregationRepository,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# How to aggregate evaluations\n",
    "0. Run the evaluations of all your tasks and datasets (see [here](./how_to_evaluate_runs.ipynb)).\n",
    "    - When aggregating multiple evaluations, all of them need the same data types \n",
    "1. Initialize all necessary repositories for the `Aggregator`, and an `AggregationLogic`\n",
    "2. Run the `Aggregator` to aggregate all examples and create a single `AggregationOverview`\n",
    "3. (Optional) Save the `AggregationOverview.id` for later retrieval\n",
    "\n",
    "### Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 0\n",
    "\n",
    "\n",
    "my_example_data = example_data()\n",
    "print()\n",
    "\n",
    "evaluation_ids = [\n",
    "    my_example_data.evaluation_overview_1.id,\n",
    "    my_example_data.evaluation_overview_2.id,\n",
    "]\n",
    "\n",
    "# Step 1\n",
    "evaluation_repository = my_example_data.evaluation_repository\n",
    "aggregation_repository = InMemoryAggregationRepository()\n",
    "aggregation_logic = DummyAggregationLogic()\n",
    "\n",
    "# Step 2\n",
    "aggregator = Aggregator(\n",
    "    evaluation_repository,\n",
    "    aggregation_repository,\n",
    "    \"MyAggregationDescription\",\n",
    "    aggregation_logic,\n",
    ")\n",
    "aggregation_overview = aggregator.aggregate_evaluation(\n",
    "    *evaluation_ids, labels=set([\"label_a\"]), metadata=dict({\"key\": \"value\"})\n",
    ")\n",
    "\n",
    "# Step 3\n",
    "print(aggregation_overview.id)\n",
    "print(aggregation_overview.labels)\n",
    "print(aggregation_overview.metadata)"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython"
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}