introduction_to_rlberry.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Introduction to rlberry.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3.7.11 64-bit ('rlberry': conda)"
    },
    "language_info": {
      "name": "python",
      "version": "3.7.11",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "interpreter": {
      "hash": "77626d6da4f24721c6cde6592fa6521e12f75aa06b21b1ac25df0859aa7f2530"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/rlberry-py/notebooks/blob/main/introduction_to_rlberry.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gVtvJ8vqQehR"
      },
      "source": [
        "# Introduction to\n",
        "<img src=\"https://raw.githubusercontent.com/rlberry-py/rlberry/main/assets/logo_wide.svg\" width=\"700\"/>\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BR9N_lSyTh-E"
      },
      "source": [
        "# Colab setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j2VU8Y7xQbSg",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "72e71bdd-9bc0-4330-c717-3c549653fc0e"
      },
      "source": [
        "from IPython import get_ipython\n",
        "COLAB = False\n",
        "if 'google.colab' in str(get_ipython()):\n",
        "    COLAB = True\n",
        "\n",
        "if COLAB:\n",
        "    # install rlberry library\n",
        "    !pip install git+https://github.com/rlberry-py/rlberry.git@v0.2.1#egg=rlberry[default] > /dev/null 2>&1\n",
        "    !apt-get install -y xvfb python-opengl ffmpeg > /dev/null 2>&1\n",
        "\n",
        "    print(\"\")\n",
        "    print(\" ~~~  Libraries installed, please restart the runtime! ~~~ \")\n",
        "    print(\"\")\n"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            " ~~~  Libraries installed, please restart the runtime! ~~~ \n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rvkt6z9pVUnv"
      },
      "source": [
        "# Create directory for saving videos\n",
        "!mkdir videos > /dev/null 2>&1\n",
        "\n",
        "# Initialize display and import function to show videos\n",
        "import rlberry.colab_utils.display_setup\n",
        "from rlberry.colab_utils.display_setup import show_video"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BCCyreAiWikz"
      },
      "source": [
        "# Interacting with a simple environment"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 512
        },
        "id": "M5qrjKk3WiHE",
        "outputId": "6790fa7f-9b43-4e89-eab2-e44d97a8d977"
      },
      "source": [
        "from rlberry.envs import GridWorld\n",
        "\n",
        "# A grid world is a simple environment with finite states and actions, on which \n",
        "# we can test simple algorithms.\n",
        "# -> The reward function can be accessed by: env.R[state, action]\n",
        "# -> And the transitions: env.P[state, action, next_state]\n",
        "env_ctor = GridWorld\n",
        "env_kwargs =dict(nrows=3, ncols=10,\n",
        "                reward_at = {(1,1):0.1, (2, 9):1.0},\n",
        "                walls=((1,4),(2,4), (1,5)),\n",
        "                success_probability=0.9)\n",
        "env = env_ctor(**env_kwargs)\n",
        "\n",
        "# Let's visuzalize a random policy in this environment!\n",
        "env.enable_rendering()\n",
        "env.reset()\n",
        "for tt in range(20):\n",
        "    action = env.action_space.sample()\n",
        "    next_state, reward, is_terminal, info = env.step(action)\n",
        "\n",
        "# save video and clear buffer\n",
        "env.save_video('./videos/gw.mp4', framerate=5)\n",
        "env.clear_render_buffer()\n",
        "# show video\n",
        "show_video('./videos/gw.mp4')"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[INFO] OpenGL_accelerate module loaded \n",
            "[INFO] Using accelerated ArrayDatatype \n",
            "[INFO] Generating grammar tables from /usr/lib/python3.7/lib2to3/Grammar.txt \n",
            "[INFO] Generating grammar tables from /usr/lib/python3.7/lib2to3/PatternGrammar.txt \n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<video alt=\"videos/gw.mp4\" autoplay\n",
              "                      loop controls style=\"height: 400px;\">\n",
              "                    <source src=\"data:video/mp4;base64,AAAAIGZ0eXBpc29tAAACAGlzb21pc28yYXZjMW1wNDEAAAAIZnJlZQAAI5ZtZGF0AAACrQYF//+p3EXpvebZSLeWLNgg2SPu73gyNjQgLSBjb3JlIDE1MiByMjg1NCBlOWE1OTAzIC0gSC4yNjQvTVBFRy00IEFWQyBjb2RlYyAtIENvcHlsZWZ0IDIwMDMtMjAxNyAtIGh0dHA6Ly93d3cudmlkZW9sYW4ub3JnL3gyNjQuaHRtbCAtIG9wdGlvbnM6IGNhYmFjPTEgcmVmPTMgZGVibG9jaz0xOjA6MCBhbmFseXNlPTB4MzoweDExMyBtZT1oZXggc3VibWU9NyBwc3k9MSBwc3lfcmQ9MS4wMDowLjAwIG1peGVkX3JlZj0xIG1lX3JhbmdlPTE2IGNocm9tYV9tZT0xIHRyZWxsaXM9MSA4eDhkY3Q9MSBjcW09MCBkZWFkem9uZT0yMSwxMSBmYXN0X3Bza2lwPTEgY2hyb21hX3FwX29mZnNldD0tMiB0aHJlYWRzPTMgbG9va2FoZWFkX3RocmVhZHM9MSBzbGljZWRfdGhyZWFkcz0wIG5yPTAgZGVjaW1hdGU9MSBpbnRlcmxhY2VkPTAgYmx1cmF5X2NvbXBhdD0wIGNvbnN0cmFpbmVkX2ludHJhPTAgYmZyYW1lcz0zIGJfcHlyYW1pZD0yIGJfYWRhcHQ9MSBiX2JpYXM9MCBkaXJlY3Q9MSB3ZWlnaHRiPTEgb3Blbl9nb3A9MCB3ZWlnaHRwPTIga2V5aW50PTI1MCBrZXlpbnRfbWluPTUgc2NlbmVjdXQ9NDAgaW50cmFfcmVmcmVzaD0wIHJjX2xvb2thaGVhZD00MCByYz1jcmYgbWJ0cmVlPTEgY3JmPTIzLjAgcWNvbXA9MC42MCBxcG1pbj0wIHFwbWF4PTY5IHFwc3RlcD00IGlwX3JhdGlvPTEuNDAgYXE9MToxLjAwAIAAAAZVZYiEADP//u4ePgUzcA5qZD3tioYCDH4JLTiO5G7RPVkoQbH1RujSQWAAEYixCJZiv+mVPSyKwxjjABSjBaevgRBhNqBIPNIy6vKn7GBXZG94Ft4RlAnKDmNTsqmgL4pZVkAAAAMAAGdWWMm5du3gCNQYg8TQoASMRLlFAABPcQSC0G/9puFmiz7auRukMMl2PNlvBl3E5m8ZxAJ0rPX4Pq59meTXcWiN3DkWKAzktQ4CL9ocJ+Pj//MGxgKoDdUhgqCWgs0VurroIxHOufLnq5KLCf6xgDQIapmgBqWyROs5iB30dXTkn59uNe6X81SklTZ2Ti3lkLMTrNo5njnxDM5VkJvFpJekcx8yUhp/YdEwo/RduA/3PfM6Ap15LSdAX17lis5RfXEXcPgWqAAAYwFsxStZ/3KPClZhFOW8x2pgMTdLMHh3oh0y8KnE+az/JsOIWNot5N2rSnYtuSQTSCEjNOJu2tPDPdRHbasRgXyoGijYlSoHghYvQwk5g77PQnWpH3yFqfyzXP5/S/gKYaLJVKtxZzHDMqGNTeoVv5zOUfoswpX8/jeu3HW2VlBjCVgjKJaqd9Mi7vkQJsuL3cxJtYt39C9Sc2ZArVIZVAbs67czeCpMq7mB80EK/Yz/vqhO+rE/i8wRZhxLvDRiVKejeY/Bbtj+wrr044/NJVGHq+poAFYUg4iGI5vZSIc4Ijdy7HM/q3xOZIzm+Uf5M9HHJp4ALcL4J73OfJKD29sldBAlB+THI8yK2fv+wV6/iUzIsYwhiylvEyJSWXG6SVafSZkKhNgRVTOcl+6qDBuXq7MmcOlvJWQxRlB14GvC7aO/7ecc/iCab746+Onf5Btkymnm53xH3DqkovzVROw0iH0qchGg8w9nS/PWDU6gcyZuaMwI0o3QsRUNdmDYBeQOXaRXPR0AfukkqVRa/GBpjivUS8bSuU+MYU8bBP4l3gmovpfEga+IPfSkIlkH0RUH0LAKYsE5zhO356RL3hNT4FE/Y11eZJbR2iVF3MXY4FjxgPOfvggyNYIcsK/Y1MfAH6KOvd+gy30Px9CTS2bhMV3vSHgEL/H6H/vbuNSDjcu8qaBedHGqIDa/ZtBJA66q404/NeKQz/fM/T0yaMbtds48+OPe/6ZEMxwWdAsw2NCXFJ8+RibdmXntBoFq0i4LDGHQtDoPIYT0rwM6ECxwb0YcNW3fmL7Qa6QcvosWrrl3DMewbZFldsNSkb83sBn1FN01D/pldFKQL+/cFuHZT98fF1DUJrQXFL5HLO6cBftq0xUyrxD+C4f5eMXBFGz1zyzl/tiWSqIeuEBYS5kfTzNqcntVPKmSepxrWSmdohBNgYPAD80jQvtDtWBMecT/DqR2uzIIpczWVv3i5u+QpdbeGmxws1iMCIWhjDVmlZx5Ex81Zz+VwbE4LkQWgJepdiGuEgnjV0mMBTmsZHElu3ff6B6f+5QgE1bShUlGqBO2CnzmPcp9sOUNAQetPgjHvSowy8dwCYaKsaT//8gMSgZexG0P86fl0pvIzXtMIF3XmDq9W76/ioY5dJtiLnb0+27cYiULA947E0ARP3r+zOOwvQAHPBV8pQ9k5QXsGfbs/D65OScEX1o1uhxdPQViLCPfB54lpPc9/AKWqhX22kU6f2zGvR2E+/Lbru0xnP3xLbGhu5Htow2mxJmt/BGOZ4Bw7sCXlcoASsbnUZwwSpg1HD814V4KGEKkI8SGynHbx9OQEAmNqdAVFgbEnL4bSN5RPOgCYaCehNh7BoP3QFIzvyMne1Fy2y9kS1zRU6k71moC6sTRwTt2k9uNQ+klSpCHwqTum3+d6UHoHBJl3gRC+SIbSmBUhkne+iLjoqaUz61Fn9/60aKx9z0ZmEFBhKZ4R+YCAquHtxUhaPCFBpaGoUF6N+rnn4Yi4stPQOBd93i04cK9IZHald9FkKV8QQd7gl1EU5POpX2iS/Td6uz3lcUeoSP40c/OM6wzQHLTfzew3NZJzHri9oPrWMU7aaPMCO8E8o8rupTo78KvsAUk0BH7vKi+6A7TQHBiglS/eFXGXY/N0XvCG6xz+LRiRU1FwI0W1rhQBk4exVZk7m8+5EowcTNdsOaoLh8nHmp3iJKDeH6+chxsFaWzrmFe1m/FBzvz4YOtS2hl0AAAAwAB+QAAAaZBmiJsQv/+jLAAqkYzFnrJ9bvkBCq/ygx6Clp3/uWAdo5ZWxWLibixfC1IwgKS6XTqF4S2xoPSeVaM/lhBbUdd+ZfXrQRNYldfj74p5spC9aL8ekp/4RqmhRBzoOv6AoKLHWmNiamoyVagyNqQc3u7ynaXFOGx/YKt6V4qY7WxvSYPKWv8TkmAHWHAN7MlYyy+ijn5DCiw+gC90Dr28aKTNbUbjIpGHOAAKh/ZfX//s+9ZIMPDCZrV3p7SfSeNQUO7EjgNaBeoiLcdyvvjoey0/ACcN1vM2Pq91bR0J3SlGBExlDWMthWGb2Rqj40oIu2aQkog0A5o6GFpPrzTtLGQQsiQeOOqG7NyyUMWnAU9yc+dgKsLRs+ibv/11rJoaHdWgbFiHA2EZoJeHs+QC5KIl6x4PtP+pEPWUdY6yjUI00Rlgnyn4Njq8Xcs0VcspyzYHhxuNwPzKMFUMItudOW3iHrUlwZyyUwU9h5uMfvPcdA29vvy488g7PXEVsSYZphnCaaVnTC0VzjMvlQEUdSKWaJeSneKLanlNDCCW8AXhpCTrvDyiAAAACoBnkF5C/8AGcDPr0s5c0+Duu1+rrkbaE8AUi1jvpg1OIMl+U4jiU96FSkAAAHsQZpDPCGTKYQ3//6nhALf5q4D/tCK5f7XqMhbzyIF4ggT216aoH9pwMM3rprRuOS9SkXC5OcSWDB5qgcD976y8zbXxkVdRl5DkMFSu/weargP55znrrcPS1b62nAyL8qAE2JT4BieNRfTLWdWo1YcxsQqv8+ZixQXX21oirjnPDMaFDapoeXc8m3xtTH8A2dTo0LOIRh3l24ulhBYmJMKLZbr6xXAF+xRHZCKiiG3kidPdaOrQIXIpas/RuQuKEEXZfKmzrvCqQn8eF2pmnwmzclrWVPQQdvt4YMrDckSvSH1G2LGrh8v4uJVpcsEuBD5GXDkfFre/R65mBAhOYyMA5z5mSv2yQPMahDG3DpHzwQsmoSWq7/dmR1NIO3dyN9asOPf6m47FxZr4oDatxsTe2TIR+sJomofNBHkAG6TGuw/jApCgiBv+6L+49JnDL+FkTiRSr5IkhqTPwWKVTyCAh8BY7ySW/7r4m7epH+vgCTe5JK7fZIG36OAwW1DINPk8pLuNRHHuMAbhyq9knQQNoOkofsU9Aqn0s6yEEsJDCgpSxiDrWlH3kKHSUwT9+7JbvNGAhsrW8Myl5S19WDKk96XbDeDHcqF8EksXtCwW3/X/ggjisXDL+DQ5qAvH6F7ILUew4z/FabGWH+AAAAALEGaZ0nhDyZTAhn//p4QCv2no9EvyzNJY6GNg2t298Tgkkiyi4BQvHqM9FbBAAAAKkGehUURPDP/AOKXegA+ep+l335PKK0CGgeCZSHY/RhW/gcOIUehlbuN6QAAAYYBnqR0Qv8BNrMR/XmMLwCFV/lBj0FLTv/csA7RyytisXE3Fi+FqRgZP7pdOoYx2NjQek8q0Z/LCC2o678y+vWgiaxK6/H3xTzZSF60X49JT/wjVNCiDnQdf0BQUWOtMbE1NRkq1BkbUg5vd3lOkXZUN2+wVb0rxUx2tjekweUtf4nJMANvuD0PCgkN2EuWWX0Uc/IYUWH0AXugde3jQuaQIOlzuXqGZGw65dCuobUHuuLXY02AJ9J41BQzx8oyQNMXnoZFjOp7LT7DokMwrqoMkCo10AChFdhQOLwr9w7QVHOhZPa2/s7uZjC1vi0LGIWknVZMPCKnTwU8lJ2ZeD9TROvSuB+tChaJAVpp1PG/FIlVzDtH2S0hjIe/deYlC9iMJnW18GqrmKhXvsKLwbxgoh54PE2wLx0CQLyAp3RpCEQxCWanCY16s50YXAZ/xX0iZhLzRHJDJlSFdUQCbUb317cvvDND+ev+38kI4R179hfbx8hQPmlCVwqXmxW0Vf26ozAAdMEAAAAcAZ6makL/ATbOSjRGqu/W7xVyEumoiYjHjoWKGQAAAY1BmqlJqEFomUwU8O/+qZYBE/PlADpWnwbbr2adqbq0ApfcA0WeyhqXF8VQaFtPbpHG6SPKa52nIrTOOvbiKV5NpajQuyDkMnk4dPlj+OMdT7M/xUA+/hGqcjdaJqFJNqVROKlNEDFdXSrJ88FIq9WvCbG3YaG4gY1k+5oohCj6THyR1oQnUNq8APaBQXO0hFyOxEyl+00BvA8Bw6uA1T49VOJM7c8eTLAAhJ1yU03LIqkPIbEu9P93uGo6Koqjd+joaXUgx6McZMsdHpGTsRk9VBkgVIhWQnxXYUDz8K+yb23MNDwX2aZuUMbmMJnd/OV80F/E66jd3HdmBvD5t4vGZifY69DrgzqYD4kU9w3U8M1O5XASMJSiXzV6SKoygqaDVmu/BqrrcxXvsKNvJDSKBVC4ItjISkhIHmUx/H3JVNVT1aw+ag6e5dRcpqjds/oQnmOOYiICaK448A2o3dDuDYqt1f0R7fsoRCLm7GzaGQAs4dLuWkd5tgNoq6Fi5F+nrs15e0OlBEP87cEjAAABkgGeyGpC/wEOzjp72lG+WRrPBetyPs6C20jgARkah2DsAX8iulaSJbkjuOhXuSeOaWjMWJf0RzoA1iIJNtfVcm7fe5RUVhQFvZyFEkfxUeAiCs34YgLccjakpEaWEzSUa21mI9IsjJD2LihfiUVOjqjX4LDs1kW8t/+qrfB5vx0+HtYEcGUGmDlOBSOSEmcVqaJvID8Niajn9Tm0ejhiIFSidHOCVTGIgrz7ERm1SNQ7IR0a6fz8BnKbVrB5Y6+X52+zXlrG1oxjmLl0ML4GtputCHmiuCfoO+YIQvBQq0NnS2Lm7pzj0pSP+pto3KzJOqprgf91NtZYB+gRk1w+MGKD7al3dF1WWB1XrV2ZVHPCXa1jVZSoRccxkqAR6zA2hzjDSbYtMSQxHsTwYxdUbor+2Z8Gqqo8xsfZBVtE53qBao+UWufhDxjk9diBKkOnK+sGbIVj/e9gUU9PBoEpAl9edQihkeQarnAoONJmSaYzdv9M3Mxsv5LFIcKG/jEhZ8QkRPX+4X8cm5ilhTX9u0XBIwAAAZNBmspJ4QpSZTAh3/6plgEX8lDfpnJk+Op40FmeiTINvOjT1N1aAUvuAaLPZQ1Li+KoNCrO1KRbo7d+QZScitM469uIpXk2lqNC7IOQyeTh0+WP44x1Psz/FQD7+EapyN1omoUk2pVE4qU0QMV1dKsnzwUir1a8JsbQLN3FjOsn3NFEIUfSY+SOtCE6htXgB4R1qu9eJwR5uRNsRS1qOaX7TQG8DwHDq4DVPj1U4kztzx5MsACEnXJTTcsiqQ8hsS70/3e4ajoqiqN36OhpdSDHoxxinZbZjJ2H/VNiQhYlwnYXwla6DY8K+yb23MNDwX2aZuUMbmMJnd/OV80F/E66jd3HdmBvD5t4vGZifY69DrgzqYD4kU9w3U8M1OC0YJhn040rZRL5q9JFUZQVNBqzXfg1Vdo6+e9J9odstQasFlAn/3yokJA8ymP4+5KpqqerWHzUHT3LqLlNUbtn9CE8xxzERATRXHHgG1G5MAkztRcKqElJJ+vlm5uxs2hkALOHS7lpHebYDaKuhYmv6D29QAM/AAABkkGa60nhDomUwId//qmWANx4Bs3Xl1wqdrl+T7NHaisQlNZORAviBNY7MSDbzo09TdWgFL7gGiz2UNS4viqDQoxbpHG7HJ/kGUnIrTOOvbiKV5NpajQuyDkMnk4dPlj+OMdT7M/xUA+/hGqcjdaJqFJNqVROKlNEDFdXSrJ88FIq9WvCbGxvp7iEnWT7miiEKPpMfJHWhCdQ2rwA3cFyhTNHtPtRzS/aaA3geA4dXAap8eqnEmduePJlgAQk65KablkVSHkNiXen+73DUdFUVRu/R0NLqQY9GNS3DKXAydhjMZQZIFSV1kJ8V2FA8/Cvsm9tzDQ8F9mmblDG5jCZ3fzlfNBfxOuo3dx3Zgbw+beLxmYn2OvQ64M6mA+JFPcN1PDNTFWwSMiyiXzV6SKoygqaDVmu/BqrGZCvfYUb8GHNsCqFwRbGQlJCQPMpj+PuSqaqnq1h81B09y6i5TVG7Z/QhPMccxEQE0Vxx4BtRufxuX3hmqzMj2/ZQiEXN2Nm0MgBZw6XctI7zbAbRV0LEctHhAAAAZZBmwxJ4Q8mUwId//6plgAEJ+SjQpfN6BbN7x7veVAeMdmJBt50aepurQCl9wDRZ7KGpcXxVBoVaOpSLdGiSSDKTkVpnHXtxFK8m0tRoXZByGTycOnyx/HGOp9mf4qAffwjVORutE1Ckm1KonFSmiBiurpVk+eCkVerXhNjY35u4g41k+5oohCj6THyR1oQnUNq8AOb/5J4cz+u5uRNmuq7ETKX7TQG8DwHDq4DVPj1U4kztzx5MsACEnXJTTcsiqQ8hsS70/3e4ajoqiqN36OhpdSDHoxxh7J5HYmDbl6KDV/zJlWQnxXYUDz8K+yb23MNDwX2aZuUMbmMJnd/OV80F/E66jd3HdmBvD5t4vGZifY69DrgzqYD4kU9w3U8M1NSK+4CYixpbSiXzV6SKoygqaDVmu/Bqq751896T7ULZag1YLKBP/vlRISB5lMfx9yVTVU9WsPmoOnuXUXKao3bP6EJ5jjmIiAmiuOPANqNybeVnai4VdI9v2UIhFzdjZtDIAWcOl3LSO82wG0VdCxFuGDgPsaAAAABkkGbLUnhDyZTAh3//qmWAABKfkdaCx3jPJ2uX5Ps0dqKxCU1k5EC+IE1jsxINvOjT1N1aAUvuAaLPZQ1Li+KoNCjFukcbscn+QZScitM469uIpXk2lqNC7IOQyeTh0+WP44x1Psz/FQD7+EapyN1omoUk2pVE4qU0QMV1dKsnzwUir1a8JsbG+nuISdZPuaKIQo+kx8kdaEJ1DavADdwXKFM0e0+1HNL9poDeB4Dh1cBqnx6qcSZ2548mWABCTrkppuWRVIeQ2Jd6f7vcNR0VRVG79HQ0upBj0Y1LcMpcDJ2GMxlBkgVJXWQnxXYUDz8K+yb23MNDwX2aZuUMbmMJnd/OV80F/E66jd3HdmBvD5t4vGZifY69DrgzqYD4kU9w3U8M1MVbBIyLKJfNXpIqjKCpoNWa78GqsZkK99hRvwYc2wKoXBFsZCUkJA8ymP4+5KpqqerWHzUHT3LqLlNUbtn9CE8xxzERATRXHHgG1G5/G5feGarMyPb9lCIRc3Y2bQyAFnDpdy0jvNsBtFXQsR4BQAAAZVBm05J4Q8mUwId//6plgAAAwGM9uAlbghyLuAZJafBtuvZp2purQCl9wDRZ7KGpcXxVBoVjLdI43SUvTXO05FaZx17cRSvJtLUaF2Qchk8nDp8sfxxjqfZn+KgH38I1TkbrRNQpJtSqJxUpogYrq6VZPngpFXq14TY2hYVuISdZPuaKIQo+kx8kdaEJ1DavAD5Z2yWkJFTSjnNdcHeCJlL9poDeB4Dh1cBqnx6qcSZ2548mWABCTrkppuWRVIeQ2Jd6f7vcNR0VRVG79HQ0upBj0Y4xgLn2YydiF1RQav+ZR52F8JWug2PCvsm9tzDQ8F9mmblDG5jCZ3fzlfNBfxOuo3dx3Zgbw+beLxmYn2OvQ64M6mA+JFPcN1PDNTqnMHiuB5C6hpqXzV6SKoygqaDVmu/Bqq4bl896T7ULZag1YLKBP/vlRISB5lMfx9yVTVU9WsPmoOnuXUXKao3bP6EJ5jjmIiAmiuOPANqNyLhJnai4VYJKST9fLNzdjZtDIAWcOl3LSO82wG0VdCxEYABoBt8FOEAAAGQQZtvSeEPJlMCHf/+qZYAAFS4wSAL/afBtuvZp2purQCl9wDRZ7KGpcXxVBoW09ukcbpI8prnacitM469uIpXk2lqNC7IOQyeTh0+WP44x1Psz/FQD7+EapyN1omoUk2pVE4qU0QMV1dKsnzwUir1a8JsbdhobiBjWT7miiEKPpMfJHWhCdQ2rwA9r4/KFMI8y1qOaX7TQG8DwHDq4DVPj1U4kztzx5MsACEnXJTTcsiqQ8hsS70/3e4ajoqiqN36OhpdSDHoxxkyx0ekZOxGT1UGSBUiFZCfFdhQPPwr7Jvbcw0PBfZpm5QxuYwmd385XzQX8TrqN3cd2YG8Pm3i8ZmJ9jr0OuDOpgPiRT3DdTwzU7lcBIwlKJfNXpIqjKCpoNWa78GqutzFe+wo28kNIoFULgi2MhKSEgeZTH8fclU1VPVrD5qDp7l1FymqN2z+hCeY45iIgJorjjwDajd0O4Niq3V/RHt+yhEIubsbNoZACzh0u5aR3m2A2iroWLkXj+uzXl7Q6TuvTT62EHt3gQAAAZVBm5FJ4Q8mUwURPDv//qmWAABVPgUJWXRmTn6u6vUggNzG6C6LDZ9Cp6wCXx2YkG3nRp6m6tAKX3ANFnsoalxfFUGhQ63SON2RgHIMpORWmcde3EUrybS1GhdkHIZPJw6fLH8cY6n2Z/ioB9/CNU5G60TUKSbUqicVKaIGK6ulWT54KRV6teE2NjIT3FjOsn3NFEIUfSY+SOtCE6htXgBtJlfdpSToPtRzS/aaA3geA4dXAap8eqnEmduePJlgAQk65KablkVSHkNiXen+73DUdFUVRu/R0NLqQY9GNTrrKXIydhkUZQZIFSJ1kJ8V2FA8/Cvsm9tzDQ8F9mmblDG5jCZ3fzlfNBfxOuo3dx3Zgbw+beLxmYn2OvQ64M6mA+JFPcN1PDNTCvI+Qi01L5q9JFUZQVNBqzXfg1Vk6hXvsKNpJDSKBVC4ItjISkhIHmUx/H3JVNVT1aw+ag6e5dRcpqjds/oQnmOOYiICaK448A2o3Qo3L7wzVaAm5kn6+Wbm7GzaGQAs4dLuWkd5tgNoq6FiG54AAAGTAZ+wakL/AABxjPwAcevTsHYAv5FdK0kS3JHcVBp3+/y1JmLEv6I50AaxEEm2vquTdvvcoqKwoC3s5CiSP4qPARBWb8MQFuORtSUiNLCZpKNbazEekWRkh7FxQvxKP5RpY1+Cw7NZFvLf/qq3web8dP9HCIZwim6huL244FI5ISZxWpom8gPw2JqOf1ObR6OGIgVKJ0c4JVMYiCvPsRGbVI1DshHRrp/PwGcptWsHljr5fnb7NeWsbTXTOoVuIobfltN1kGYBCkJ9B3zBCF4KFWhs6Wxc3dOcelKR/1NtG5WZJ1VNcD/uptrLAP0CMmuHxgxQfbUu7ouqywOq9auzKo54S7WsarKVCLjmMlQCPWdzmOl9k1jttYtMSQxHsTwYxdUbor+2Z8GqpCxLIPZBVtE53qBao+UWufhDxjk9diBKkOnK+sGbIVj/e9gUU9PBoEpAl9edQihkeQarnAoONJmSaYzgfYs3Mxsy5LFIcKG/jEhZ8QkRPX+4X8cm5ilhTX9vKpAJZnAskXKN+eATmlUDzgAAAZFBm7NJ4Q8mUwU8N//+p4QCIcqiAKDaEVy+vZp2pg7wCWdwDRUvMnAnXxVBoWyXqUi3SFy8SWC0HKu20ywhPbVtLUaF2QcmwErfn8sPxxCCuZn+KgH58I1TgAmxIDF9WIxNio7RAxW81KycPBSKXfpimxtzShtNwsqjGaKIQpGkx8aayDrhDavAD2Dn24QgOsR1flgTyb9Fh9AHNq3/VHjozdcztzx5MoQIinXJOwaSmoH0gURjTX/4cNB0VSLWi7pF93VfAYNqnYxMydhu9z5GUGSBUdqABUp/YoH34V9eFsm2KHgvsbfNR4FvWKafXeV80F82q2ZaIT9jhreQpeFIVweg69Di1EVJAp8FKlXcQ5cKgizgfdSiYKt6SKQygqXyK7PfiVV1SYr32FGgkhgg1YJtNP/vlRISBfrsfxkHgEuMNDnILSp2eQKdx3W3bVZcI5jjlhJ/miuGR/9qN3LLg2KrdWhJ04Z+sld5uNcKheMKN80pDErHdsBtFXQsXIqAhwk35cYD273VYCd11lE7gQAAAZUBn9JqQv8BDs46e9pRvlkazwXrciTu24AiY1DsHYAv5FdK0kS3JHcSXW7/f5opMxYl/RHOgDWIgk219Vybt97lFRWFAW9nIUSR/FR4CIKzfhiAtxyNqSkRpYTNJRrbWYj0iyMkPYuKF+JQ+yv+oY2q7jZwS5b/9VW+Dzfjp8E4YlBpfbPEQbkhJnFamibyA/DYmo5/U5tHo4YiBUonRzglUxiIK8+xEZtUjUOyEdGun8/AZym1aweWOvl+dvs15axtSPTOoVuIobfltN1kGQBCkJ9B3zBCF4KFWhs6Wxc3dOcelKR/1NtG5WZJ1VNcD/uptrLAP0CMmuHxgxQfbUu7ouqywOq9auzKo54S7WsarKVCLjmMlQCPWXtdsaN2w2xJDEexPBjF1Ruiv7ZnwaqlsH3g9kFW0TneoFqj5Ra5+EPGOT12IEqQ6cr6wZshWP972BRT08GgSkCX151CKGR5BqucCg40mZJpjOTZZX+LGzDEsUhwob+MSFnxCRE9f7hfxybmKWFNf27Izwb0DwhJqYBoaigAAAGhQZvUSeEPJlMCF//+jLAIRpvTXfHL9hBwiu8HTRoE83RIJuvicApJrZQY9BS07/3LAO0csrYrFxNxYvhakYLWNyA8ffCltjQek8q0Z/LCC2o678y+vWgiaxK6/H3xTzZSF60X49JT/wjVNCiDnQdf0BQUWOtMbE1NRkq1BkbUg5vd3lOfD9xM1bHNK8VMdrY3pMHlLX+JyTADXe+l5zg0y6yKOfkMKLD6AL3QOvbxopM1tRuMikYc4AAqH9l9f/+z71kgw8MJmtXentJ9J41BQ7sSOA1oF6iItx3K+9qsSSkiy0GzJAVZ23+raOhO6UowImMoaxlsKwzeyNUfGlBF2zSElEGgHNHQwtJ9eadpYyCFkSDxx1Q3ZuWShi04CnuTnzsBVhaNnnTLI6U5NTNBLw9nyAXJREvWPB9p/3HHMyjrHWTsYNzWSYqcp+DY6vF3LNFXLKcs2B4cbjcD8yjBVDCLbnTlt4h61Zc9cXFYF4tI5Ar5pJqyZkCjdCCtwdnlyVsSYZphvLVUrOmFornGZfKgIo6kUs0isttskW1PDhjgAAAD5m1vb3YAAABsbXZoZAAAAAAAAAAAAAAAAAAAA+gAABBoAAEAAAEAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAMQdHJhawAAAFx0a2hkAAAAAwAAAAAAAAAAAAAAAQAAAAAAABBoAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAQAAAAAMgAAAA8AAAAAAAJGVkdHMAAAAcZWxzdAAAAAAAAAABAAAQaAAAEAAAAQAAAAACiG1kaWEAAAAgbWRoZAAAAAAAAAAAAAAAAAAAKAAAAKgAVcQAAAAAAC1oZGxyAAAAAAAAAAB2aWRlAAAAAAAAAAAAAAAAVmlkZW9IYW5kbGVyAAAAAjNtaW5mAAAAFHZtaGQAAAABAAAAAAAAAAAAAAAkZGluZgAAABxkcmVmAAAAAAAAAAEAAAAMdXJsIAAAAAEAAAHzc3RibAAAAJdzdHNkAAAAAAAAAAEAAACHYXZjMQAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAMgAPAASAAAAEgAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABj//wAAADFhdmNDAWQAFf/hABhnZAAVrNlAyH6EAAADAAQAAAMAKDxYtlgBAAZo6+PLIsAAAAAYc3R0cwAAAAAAAAABAAAAFQAACAAAAAAUc3RzcwAAAAAAAAABAAAAAQAAAJBjdHRzAAAAAAAAABAAAAABAAAQAAAAAAEAABgAAAAAAQAACAAAAAABAAAQAAAAAAEAACgAAAAAAQAAEAAAAAABAAAAAAAAAAEAAAgAAAAAAQAAGAAAAAABAAAIAAAAAAYAABAAAAAAAQAAGAAAAAABAAAIAAAAAAEAABgAAAAAAQAACAAAAAABAAAQAAAAABxzdHNjAAAAAAAAAAEAAAABAAAAFQAAAAEAAABoc3RzegAAAAAAAAAAAAAAFQAACQoAAAGqAAAALgAAAfAAAAAwAAAALgAAAYoAAAAgAAABkQAAAZYAAAGXAAABlgAAAZoAAAGWAAABmQAAAZQAAAGZAAABlwAAAZUAAAGZAAABpQAAABRzdGNvAAAAAAAAAAEAAAAwAAAAYnVkdGEAAABabWV0YQAAAAAAAAAhaGRscgAAAAAAAAAAbWRpcmFwcGwAAAAAAAAAAAAAAAAtaWxzdAAAACWpdG9vAAAAHWRhdGEAAAABAAAAAExhdmY1Ny44My4xMDA=\" type=\"video/mp4\" />\n",
              "                 </video>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jWgoh5yoYuDE"
      },
      "source": [
        "# Creating an agent\n",
        "\n",
        "Let's create an agent that runs value iteration to find a near-optimal policy.\n",
        "This is possible in our GridWorld, because we have access to the transitions `env.P` and the rewards `env.R`.\n",
        "\n",
        "\n",
        "An `Agent` must implement at least two methods, **fit()** and **eval()**. These methods train and evaluate the agent, respectively. A convenient subclass of `Agent`, `AgentWithSimplePolicy`, requires a **policy()** method instead of **eval()** and implements a default `eval()` method that performs Monte-Carlo policy evaluation.\n",
        "\n",
        "It can also implement **sample_parameters()** used for hyperparameter optimization with [Optuna](https://optuna.org/).\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AJVMjW7WYspi"
      },
      "source": [
        "import numpy as np\n",
        "from rlberry.agents import AgentWithSimplePolicy\n",
        "\n",
        "class ValueIterationAgent(AgentWithSimplePolicy):\n",
        "    name = 'ValueIterationAgent'\n",
        "    def __init__(self, env, gamma=0.99, epsilon=1e-5, **kwargs):   # it's important to put **kwargs to ensure compatibility with the base class \n",
        "        \"\"\"\n",
        "        gamma: discount factor\n",
        "        episilon: precision of value iteration\n",
        "        \"\"\"\n",
        "        AgentWithSimplePolicy.__init__(self, env, **kwargs) # self.env is initialized in the base class\n",
        "\n",
        "        self.gamma = gamma\n",
        "        self.epsilon = epsilon \n",
        "        self.Q = None  # Q function to be computed in fit()\n",
        "    \n",
        "    def fit(self, budget=None, **kwargs):  \n",
        "        \"\"\"\n",
        "        Run value iteration.\n",
        "        \"\"\"\n",
        "        S, A = env.observation_space.n, env.action_space.n \n",
        "        Q = np.zeros((S, A))\n",
        "        V = np.zeros(S)\n",
        "\n",
        "        while True:\n",
        "            TQ = np.zeros((S, A))\n",
        "            for ss in range(S):\n",
        "                for aa in range(A):\n",
        "                    TQ[ss, aa] = env.R[ss, aa] + self.gamma*env.P[ss, aa, :].dot(V)\n",
        "            V = TQ.max(axis=1)\n",
        "\n",
        "            if np.abs(TQ-Q).max() < self.epsilon:\n",
        "                break\n",
        "            Q = TQ\n",
        "        self.Q = Q\n",
        "          \n",
        "    def policy(self, observation):\n",
        "        return self.Q[observation, :].argmax()\n",
        "\n",
        "    \n",
        "    @classmethod\n",
        "    def sample_parameters(cls, trial):\n",
        "      \"\"\"\n",
        "      Sample hyperparameters for hyperparam optimization using Optuna (https://optuna.org/)\n",
        "      \"\"\"\n",
        "      gamma = trial.suggest_categorical('gamma', [0.1, 0.25, 0.5, 0.75, 0.99])\n",
        "      return {'gamma':gamma}\n"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 441
        },
        "id": "W3r5WHYZctYN",
        "outputId": "cca04fcf-eec6-4048-c1ba-fd9d09f26608"
      },
      "source": [
        "# Now, let's fit and test the agent!\n",
        "agent = ValueIterationAgent(env)\n",
        "agent.fit()\n",
        "\n",
        "\n",
        "# Run agent's policy\n",
        "env.enable_rendering()\n",
        "state = env.reset()\n",
        "for tt in range(20):\n",
        "    action = agent.policy(state)\n",
        "    state, reward, is_terminal, info = env.step(action)\n",
        "\n",
        "# save video and clear buffer\n",
        "env.save_video('./videos/gw.mp4', framerate=5)\n",
        "env.clear_render_buffer()\n",
        "# show video\n",
        "show_video('./videos/gw.mp4')\n"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<video alt=\"videos/gw.mp4\" autoplay\n",
              "                      loop controls style=\"height: 400px;\">\n",
              "                    <source src=\"data:video/mp4;base64,AAAAIGZ0eXBpc29tAAACAGlzb21pc28yYXZjMW1wNDEAAAAIZnJlZQAAHmZtZGF0AAACrQYF//+p3EXpvebZSLeWLNgg2SPu73gyNjQgLSBjb3JlIDE1MiByMjg1NCBlOWE1OTAzIC0gSC4yNjQvTVBFRy00IEFWQyBjb2RlYyAtIENvcHlsZWZ0IDIwMDMtMjAxNyAtIGh0dHA6Ly93d3cudmlkZW9sYW4ub3JnL3gyNjQuaHRtbCAtIG9wdGlvbnM6IGNhYmFjPTEgcmVmPTMgZGVibG9jaz0xOjA6MCBhbmFseXNlPTB4MzoweDExMyBtZT1oZXggc3VibWU9NyBwc3k9MSBwc3lfcmQ9MS4wMDowLjAwIG1peGVkX3JlZj0xIG1lX3JhbmdlPTE2IGNocm9tYV9tZT0xIHRyZWxsaXM9MSA4eDhkY3Q9MSBjcW09MCBkZWFkem9uZT0yMSwxMSBmYXN0X3Bza2lwPTEgY2hyb21hX3FwX29mZnNldD0tMiB0aHJlYWRzPTMgbG9va2FoZWFkX3RocmVhZHM9MSBzbGljZWRfdGhyZWFkcz0wIG5yPTAgZGVjaW1hdGU9MSBpbnRlcmxhY2VkPTAgYmx1cmF5X2NvbXBhdD0wIGNvbnN0cmFpbmVkX2ludHJhPTAgYmZyYW1lcz0zIGJfcHlyYW1pZD0yIGJfYWRhcHQ9MSBiX2JpYXM9MCBkaXJlY3Q9MSB3ZWlnaHRiPTEgb3Blbl9nb3A9MCB3ZWlnaHRwPTIga2V5aW50PTI1MCBrZXlpbnRfbWluPTUgc2NlbmVjdXQ9NDAgaW50cmFfcmVmcmVzaD0wIHJjX2xvb2thaGVhZD00MCByYz1jcmYgbWJ0cmVlPTEgY3JmPTIzLjAgcWNvbXA9MC42MCBxcG1pbj0wIHFwbWF4PTY5IHFwc3RlcD00IGlwX3JhdGlvPTEuNDAgYXE9MToxLjAwAIAAAAYPZYiEAC///usu/Mslf7EY+0wWu0XeO+3lU4xWQ+Zr1Q6D/+IN8BMXmwABOD+V5LMV/xqY8b5hSgxt3nT4LS08EP0gHadNiD16pdQaqbJ5ZCnyTcTcN7nWOmcmgZFedEAAAAMAALNGgocxSlL5f5gxB4mhQAkgqRgAAUipuYNBvsbVfrZ802Ra83TzAIpfXilDLUOZvGcP2e8z1+D6uff8dYCmvxtYQ9w0LFKQCEE/U+JPAY0/+gNms5PE7r1FJ21iy9/AUbOxM7UcZfELvqpvP6ZeBlsNWntx4ioUc0GaA76OxDl1YYnGvdL+apSQ26OqD/amXXDmctbT458QzOVagMkaRbVN/y6ryuQm+tTKI2LtwEbqnLwxri3Uu+TirFtEHvxUIbd99NJeAAAGR8MijWmh3dfrPRJgDfamGTBS+gD/ToR4UXVrbnIT9nQP1nTuEZ0pLsAeFDhgFeMbR88cGQiwPAlxBXlvkSbeqtoZlEqVUg2tHpuajdTGuTxfgiOhnoDwu7PWZIPZJM97SXLaft84DlAdhgPO9ZnpQSw8KRgsZKGsS4LW4eIUsxAMIzpxMXzeM8zFinw7NncpB3MpH1EVWgy3egCoBDRb8xhpGY40UuK4ya9XFima972nJ7VTyq1HNTfEfiaWK+26RSAz/tUn+PkbIDHHN2ssNU6tlsyeALaPvNCYAm5FZEUE3RRrkQ776a70EXWmMaLvcgMyGrDK8USp6IDxzL8hyMq0+uJ1rObi2msBBLF+NrB1SN1F6OMxof4KzO4PmFiv1REW26Hs0vJGNXSV20d/km415nnrhvomCaaHt7l5cl/f/PmOn/W+xnHgj9dbB5DQ/sh7qBGqLrSoYfPiqKa/2SQGZeeyE9XkL436FMgAbSSE/u1idjzn1FMnjf4tZ0xJub3bJbkfa9Ob4mQHJ59JPwG19L4hNjL+66FJDHY6KaNdEhvjbO4GcFlXFSsMFmE9wxfhnLCGZOk7HfMKnJBODLbBUiV+eCDI1gh1cFaI9Y+AP0UddxRYVAv5LR5mOHOwF79jNE+cZ+VOhz19JLRcYcpSFn6s0+jOjulujHigq4fLSiK8fm4FIcV4QPUHMmjG7X3MHnxx73/SnZmOCzoFmGxoOS6SuQP5KoEDDXi9FRSKbAvysiOxiYYCfZAzv5w81ufjhvVs/LsLDfwXPN7sWivX+3wZIPZKHrFGUrjIg2Az6inJVQ/6ZXRSkC/tNSm+7Kfvj4uoQIBJC4pfI5Z3SeeJ4A6OK4l7OIn11EqrRgyOTAbHAV5NhEheQhkRrzvoOnJ7VTyp3cyyscCobsV0uTku88z3hmj/wTWxtXsuyeDngvP8QlOmbNqXLBmQRs2eyvB5af0TTYjbcisBBC8jBOZTu4/JQUTiBYci4oWSDaoy6w2WU9Vk1UTCZ7+yiFHTeflBoOrZAtjgfJdwpgLR4STGO3nsRWPH4VdCODwQuCF0HORFuhMbBUJ8yToh///IEpu7stKuZowzMjhddkl9oLHznz0MxzuRxBUVZ8ddP4O+VwsNraHmOxu6HKFtvrrBEq0toNQAC2LXk/591zGaeY8TWp8mtV0mGYfzkQmMpIpLCPfQ+EbOMmJAN+kwVvDROZUP9Z7v+wPuj/eKTU1/4dQJtq9pJTzq17iEn6F+kh+P+jkjdYTb7j/C8RZHPG+GSeI/2b8ApZX0eMo7p0gmXH4BZ5A88SRRQeo5ouMb6wedv8fxke1NSIL5aeFs+rKNaPsL/JztNNkMWKJEL4tez+J6fe4UMHsJ4HK97XREK58bA+yB6sWYcmGPbMvrz8eQFWduJmoNT46PtggpRrRtkK/fBHAHfAuHe8HzMB6jn4pxRpayopmJ1SdyZQRPglHBVZgadESB3FtU9e2nGhaZtJ3AWD1RpnUG3phYuIr0TCMRfzQ7wQGLYF37JKPYayjHxpojLgy+sJyKwfX8QTiH4b+DXGUsU1bLRHwG7Ejwzjk4IosbWj5qswK6OGE+TTIWQHRlWfeRqqB74rfOjh5vbLKzF7rkkrAiOCCOEFc/tVCT4s4Qg9grJN090Y0BgFM3MwJvAAABkEGaIWxDv/6plgAAGWeXCn38ef2wCU30QHK+fdwWZ6JMg286NPU3VoBS+4Bos9lDUuL4qg0KurdI43R+e8gyk5FaZx17cRSvJtLUaF2Qchk8nDp8sfxxjqfZn+KgH38I1TkbrRNQpJtSqJxUpogYrq6VZPngpFXq14TY2U+huOa9ZPuaKIQo+kx8kdaEJ1DavADnEG1WV7/WebkTbTDvBEyl+00BvA8Bw6uA1T49VOJM7c8eTLAAhJ1yU03LIqkPIbEu9P93uGo6Koqjd+joaXUgx6McYp2W2YydiANTYkIWJcJ2F8JWug2PCvsm9tzDQ8F9mmblDG5jCZ3fzlfNBfxOuo3dx3Zgbw+beLxmYn2OvQ64M6mA+JFPcN1PDNTUw1w97/lY2yiXzV6SKoygqaDVmu/Bqq2BsXT0n2h2y1BqwWUCf/fKiQkDzKY/j7kqmqp6tYfNQdPcuouU1Ru2f0ITzHHMREBNFcceAbUbj2CTO1FwqgSUkn6+Wbm7GzaGQAs4dLuWkd5tgNoq6FieNPQAAAGTQZpCPCGTKYQ7//6plgAAAwAoXwKErLozJ82JJoLM9EmQbedGnqbq0ApfcA0WeyhqXF8VQaFWdqUi3R278gyk5FaZx17cRSvJtLUaF2Qchk8nDp8sfxxjqfZn+KgH38I1TkbrRNQpJtSqJxUpogYrq6VZPngpFXq14TY2gWbuLGdZPuaKIQo+kx8kdaEJ1DavADw5wLPF5eKDzcibYilrUc0v2mgN4HgOHVwGqfHqpxJnbnjyZYAEJOuSmm5ZFUh5DYl3p/u9w1HRVFUbv0dDS6kGPRjjFOy2zGTsP+qbEhCxLhOwvhK10Gx4V9k3tuYaHgvs0zcoY3MYTO7+cr5oL+J11G7uO7MDeHzbxeMzE+x16HXBnUwHxIp7hup4ZqcFowTDPpxpWyiXzV6SKoygqaDVmu/Bqq7R1896T7Q7Zag1YLKBP/vlRISB5lMfx9yVTVU9WsPmoOnuXUXKao3bP6EJ5jjmIiAmiuOPANqNyYBJnai4VUJKST9fLNzdjZtDIAWcOl3LSO82wG0VdCxQRLwWKwAAAZRBmmNJ4Q8mUwId//6plgAAAwAjPyUN+mcmT5sSTQWZ6JMg286NPU3VoBS+4Bos9lDUuL4qg0Ks7UpFujt35BlJyK0zjr24ileTaWo0Lsg5DJ5OHT5Y/jjHU+zP8VAPv4RqnI3WiahSTalUTipTRAxXV0qyfPBSKvVrwmxtAs3cWM6yfc0UQhR9Jj5I60ITqG1eAHhzgWeLy8UHm5E2xFLWo5pftNAbwPAcOrgNU+PVTiTO3PHkywAISdclNNyyKpDyGxLvT/d7hqOiqKo3fo6Gl1IMejHGKdltmMnYf9U2JCFiXCdhfCVroNjwr7Jvbcw0PBfZpm5QxuYwmd385XzQX8TrqN3cd2YG8Pm3i8ZmJ9jr0OuDOpgPiRT3DdTwzU4LRgmGfTjStlEvmr0kVRlBU0GrNd+DVV2jr570n2h2y1BqwWUCf/fKiQkDzKY/j7kqmqp6tYfNQdPcuouU1Ru2f0ITzHHMREBNFcceAbUbkwCTO1FwqoSUkn6+Wbm7GzaGQAs4dLuWkd5tgNoq6FigiXgtDAAAAZRBmoRJ4Q8mUwId//6plgAAAwAer24CVl0Zk+bEk0FmeiTINvOjT1N1aAUvuAaLPZQ1Li+KoNCrO1KRbo7d+QZScitM469uIpXk2lqNC7IOQyeTh0+WP44x1Psz/FQD7+EapyN1omoUk2pVE4qU0QMV1dKsnzwUir1a8JsbQLN3FjOsn3NFEIUfSY+SOtCE6htXgB4c4Fni8vFB5uRNsRS1qOaX7TQG8DwHDq4DVPj1U4kztzx5MsACEnXJTTcsiqQ8hsS70/3e4ajoqiqN36OhpdSDHoxxinZbZjJ2H/VNiQhYlwnYXwla6DY8K+yb23MNDwX2aZuUMbmMJnd/OV80F/E66jd3HdmBvD5t4vGZifY69DrgzqYD4kU9w3U8M1OC0YJhn040rZRL5q9JFUZQVNBqzXfg1Vdo6+e9J9odstQasFlAn/3yokJA8ymP4+5KpqqerWHzUHT3LqLlNUbtn9CE8xxzERATRXHHgG1G5MAkztRcKqElJJ+vlm5uxs2hkALOHS7lpHebYDaKuhYoIl4LdQAAAZRBmqVJ4Q8mUwId//6plgAAAwAbT24CVl0Zk+bEk0FmeiTINvOjT1N1aAUvuAaLPZQ1Li+KoNCrO1KRbo7d+QZScitM469uIpXk2lqNC7IOQyeTh0+WP44x1Psz/FQD7+EapyN1omoUk2pVE4qU0QMV1dKsnzwUir1a8JsbQLN3FjOsn3NFEIUfSY+SOtCE6htXgB4c4Fni8vFB5uRNsRS1qOaX7TQG8DwHDq4DVPj1U4kztzx5MsACEnXJTTcsiqQ8hsS70/3e4ajoqiqN36OhpdSDHoxxinZbZjJ2H/VNiQhYlwnYXwla6DY8K+yb23MNDwX2aZuUMbmMJnd/OV80F/E66jd3HdmBvD5t4vGZifY69DrgzqYD4kU9w3U8M1OC0YJhn040rZRL5q9JFUZQVNBqzXfg1Vdo6+e9J9odstQasFlAn/3yokJA8ymP4+5KpqqerWHzUHT3LqLlNUbtn9CE8xxzERATRXHHgG1G5MAkztRcKqElJJ+vlm5uxs2hkALOHS7lpHebYDaKuhYoIl4LqwAAAZRBmsZJ4Q8mUwId//6plgAAAwAYL24CVl0Zk+bEk0FmeiTINvOjT1N1aAUvuAaLPZQ1Li+KoNCrO1KRbo7d+QZScitM469uIpXk2lqNC7IOQyeTh0+WP44x1Psz/FQD7+EapyN1omoUk2pVE4qU0QMV1dKsnzwUir1a8JsbQLN3FjOsn3NFEIUfSY+SOtCE6htXgB4c4Fni8vFB5uRNsRS1qOaX7TQG8DwHDq4DVPj1U4kztzx5MsACEnXJTTcsiqQ8hsS70/3e4ajoqiqN36OhpdSDHoxxinZbZjJ2H/VNiQhYlwnYXwla6DY8K+yb23MNDwX2aZuUMbmMJnd/OV80F/E66jd3HdmBvD5t4vGZifY69DrgzqYD4kU9w3U8M1OC0YJhn040rZRL5q9JFUZQVNBqzXfg1Vdo6+e9J9odstQasFlAn/3yokJA8ymP4+5KpqqerWHzUHT3LqLlNUbtn9CE8xxzERATRXHHgG1G5MAkztRcKqElJJ+vlm5uxs2hkALOHS7lpHebYDaKuhYoIl4L7wAAAZRBmudJ4Q8mUwId//6plgAAAwAVT4FCVl0Zk+bEk0FmeiTINvOjT1N1aAUvuAaLPZQ1Li+KoNCrO1KRbo7d+QZScitM469uIpXk2lqNC7IOQyeTh0+WP44x1Psz/FQD7+EapyN1omoUk2pVE4qU0QMV1dKsnzwUir1a8JsbQLN3FjOsn3NFEIUfSY+SOtCE6htXgB4c4Fni8vFB5uRNsRS1qOaX7TQG8DwHDq4DVPj1U4kztzx5MsACEnXJTTcsiqQ8hsS70/3e4ajoqiqN36OhpdSDHoxxinZbZjJ2H/VNiQhYlwnYXwla6DY8K+yb23MNDwX2aZuUMbmMJnd/OV80F/E66jd3HdmBvD5t4vGZifY69DrgzqYD4kU9w3U8M1OC0YJhn040rZRL5q9JFUZQVNBqzXfg1Vdo6+e9J9odstQasFlAn/3yokJA8ymP4+5KpqqerWHzUHT3LqLlNUbtn9CE8xxzERATRXHHgG1G5MAkztRcKqElJJ+vlm5uxs2hkALOHS7lpHebYDaKuhYoIl4MPwAAAZRBmwhJ4Q8mUwId//6plgAAAwASn5KG/TOTJ82JJoLM9EmQbedGnqbq0ApfcA0WeyhqXF8VQaFWdqUi3R278gyk5FaZx17cRSvJtLUaF2Qchk8nDp8sfxxjqfZn+KgH38I1TkbrRNQpJtSqJxUpogYrq6VZPngpFXq14TY2gWbuLGdZPuaKIQo+kx8kdaEJ1DavADw5wLPF5eKDzcibYilrUc0v2mgN4HgOHVwGqfHqpxJnbnjyZYAEJOuSmm5ZFUh5DYl3p/u9w1HRVFUbv0dDS6kGPRjjFOy2zGTsP+qbEhCxLhOwvhK10Gx4V9k3tuYaHgvs0zcoY3MYTO7+cr5oL+J11G7uO7MDeHzbxeMzE+x16HXBnUwHxIp7hup4ZqcFowTDPpxpWyiXzV6SKoygqaDVmu/Bqq7R1896T7Q7Zag1YLKBP/vlRISB5lMfx9yVTVU9WsPmoOnuXUXKao3bP6EJ5jjmIiAmiuOPANqNyYBJnai4VUJKST9fLNzdjZtDIAWcOl3LSO82wG0VdCxQRLwZNAAAAcNBmylJ4Q8mUwIf//6plgAAAwAQH5lNCAjebyATSIDOLXkLp0EzT1Pdy+ZekGb6x++n119JdvfIVde/W8EF6bsrmGOrOUGdHN/ooqvmbfBL5sM0zOHvqzXApPhov0D2IWbUlAVlwSoV7aVgK4cYEuXpDDtZjlXWLQeMSyc/lMwP/MUxKK5XipzaLawJBgOFXobV4AeKuxOIJjoZHg8rbHKt+/+l+00BvA8Bvo2A1T49XHkM7c8eTLAAmh1yU0xkpqEDEhIY02Rl4rpWSVeDPA0jAZ3szrWRtdULsxig25qig1f8ynTsNYStdCBeFfXKN7mGh4L7UA9SGNzGE0Cj/K+aC/irTRu7e+xw5yekvCkMd8VuXx0wYoz9k6/KXncQ5cJ0J8xE//8vZsonXUOR4+AxvJTIG7Xwaqu0dfPek+1y2UblWDHK1/75USEgd5TH8bUQg3uPVq9sSAWnl0kN2qN24zo3OY45BUAJori0ujajcmASZ2ouNaCTov8xXNU3Y3kjsJjXN6UhiS4OWA2iroWMxH5Tdv0Z0fj9W5BS0mVX94gY1gJhohE8vTvVMDKdPM23Dx1NxjC7aTByPcY1reV3u/EfAAABlkGbSknhDyZTAgh//qpVAADFT9UAE1JVPZ7NKg/YBw7zo09T3cvlVIGnYOeEoUGXxXb3yLU2OSbwex88Q9rwvNFyhnDodA1V8zb4Je5xGTh1++r8dOeX3kw7TvoOCxY6S5uQUS9HxOgYF8EuXpHe+vnausWlE7Yr+6lqOpxMC1iBJoogsCNTF9NdlK8vnjpw1JQGvlMLYl72QgTyb9Fh5AP8oJft8HwK8P07c8eTLAAdXGgSAgAm1TtAReY02jCYLxWSVmepgjobT+PcEG1V0YlhmxAG6EjKDJAqQ6oBiiV4QNg1PkB7IVHg2zvtXxtIY3MYOORZi/uIL60cCAu2q7MEJJeWWW9W2BDr0OqZC00CnsFmajrimxXGRnzJ2mpXNXpKbTKD5y7Wa78GquwLFe+wo7+2TzlWDIp1/75USEgXq7H8qYQgpSUw56IEpZ7l1jc31t25govOY45KGOXI4eHJCnLTEzcvvDNeHJEsf+YrLKp1e0ZYASsh0u5aRJMLQYU1/b05mhgJijrXBe/B3jN3mC7if8EAAAIqQZtuSeEPJlMCH//+qZYAUnp1wAJK2MHB+2fAK2eP3rz8YZhdGoVzZGJG0cF6mrAoha3ZeCzNsCTHwpSfYQPW36sLDfmPyoWdb8wBwtbPezxI3HEc0p9owJqy7uaNqK++ShuCPa4GD6otq/4n3jladtCMk+sKASeT0K/jpjXLNTa1fCsSqSpaYXzFhDWslbY5lYExtzGk3ft9ScqHlxpgYdOkTZIn3SBMVsowdeQuny60NrEHWXxnknQEdQHepF0Z8+pvh/CVmgttqcOhig0BuKvPc9DyAw5GuU33NTFXLmuLodceD7s9D1NBnr3Rw0z4a8kvO0Z4XJEwvZ2/QcDWchSdAYE3Tg/cx10OcSUB7K/b9Aayg6+NudKMs5o2kCSTMd8/eZUrN6P/nQ1sR28gLV1l0O+aScJrCS5KbdvXSH060NGINUqr1iBHZ6bxFmx6q//cL4J82EPXsKjBvVeQyJPGqZ3o1yu/fcLM2Q2SsVl1F/ArWcO7fiaZ2km/+8mvO7P1gEuN2QPDbdqwxSaspSuAjLZfoK3Sy55sELf1LAqJIbVVCBqMTBZ81FAK6EiOVD/8jvGNr/19z9BRAMJI5HbNBbRpug0R8TXSmHH3Lcn0WsPs0QOqId6N1zAOvHnopKbUk2jycN/NWcAjA2KrdXiJGJ8+1/rbea2bHJI/2GQpvsKvQdOwmfYNPLMKHAjfNlN/s8L9hGRNzF0eGhL/6KljSGqAbLVxH+AAAAAgQZ+MRRE8N/8AOf2sIybC4OysJu8343yiC0YFqkcBBcAAAAAcAZ+rdEL/AGH70n2Ji8dqDV+Eut8XRHEgRYe7uQAAAA4Bn61qQv8AAfxMPMAGBQAAABZBm7JJqEFomUwId//+qZYAAAMAAEbBAAAAEUGf0EURLDf/AAE1yMcaAIOAAAAADgGf73RC/wAB+9ZjoAdMAAAADgGf8WpC/wAB/Ew8wAYFAAACdUGb9EmoQWyZTBRML//+jLACgYfYAVhHUUxJOf7llHkkE8Amrc7f2HuQj7+REAQJ2Se+hRgtRdbLLDj/ROks4EJFDwfzp+O3lCMPYhddGS/U2lPQZE1cMELQDjQBUKrxQzttEYRy4YwvNnkxFFqwgFq2LT0/mbIDZrvNJuDbcSesY8lse6hWVd0cPhdNi2WXVIFxqpcvlhC6CxZ9zyj6GGhVcQcxNZYwY9NApINEaI02196JH/xACqMv6GniZzJYtXJxuFqTAMbdJa9r33oy8IROA6MeYW8LQQKRFigRsgWI3DtiHfnVRwJNq1z+zuQjJIatfKIoldcnYOXZFiRZiz3kSYiOzJoPnM5G1gEf3lHGIOL+TPCDeksv+cgczpbmAF1hLbSUE9xe2RnaYEnAmihP1cr82Rsg0rkQ05RvGHkfJG/ChygnOVHIdZTA2FRbPMg1jARORc/JcV4CAdr9tSuk45rDdMyHk5gBKeCk+2l3s/NA8z8ifq06VOIotv5QG8GMfSZ2TA6WqRViW1YuWxnX/e5f/DLfFmYvaVpBpIbaXW1KfTcjIlz7pQuyCbXLjiodybflURlq1NopUI0zt4hDuGVJfBJ/Vmj6Rnj946MwA9abQL0DJOXctXc3cro3eU8PH8s7rplJe14lPtCzhJIoBsuA9g0DUn0uaW7IC7I6blxqC1glWt/XdzeCt7GOZRRWs84z0so1YkDTJMTftHhE0Ug1MPL8axEe871WBXRiDJCP+tteFD3LlaaQT2DMhFJvVZXhvcNwLxK+qW+x1hWUVb3WKe646Dt2CTu3xUROO27JnRWZB9EzjU8XQvetk3T3NmOAAAAAKgGeE2pC/wBiQHgCkEbL/gU1AlwJIXRr1vidG6tj55s43AhO4ddFdzowPAAAA75tb292AAAAbG12aGQAAAAAAAAAAAAAAAAAAAPoAAAQaAABAAABAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAC6HRyYWsAAABcdGtoZAAAAAMAAAAAAAAAAAAAAAEAAAAAAAAQaAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAEAAAAADIAAAAPAAAAAAACRlZHRzAAAAHGVsc3QAAAAAAAAAAQAAEGgAABAAAAEAAAAAAmBtZGlhAAAAIG1kaGQAAAAAAAAAAAAAAAAAACgAAACoAFXEAAAAAAAtaGRscgAAAAAAAAAAdmlkZQAAAAAAAAAAAAAAAFZpZGVvSGFuZGxlcgAAAAILbWluZgAAABR2bWhkAAAAAQAAAAAAAAAAAAAAJGRpbmYAAAAcZHJlZgAAAAAAAAABAAAADHVybCAAAAABAAABy3N0YmwAAACXc3RzZAAAAAAAAAABAAAAh2F2YzEAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAADIADwAEgAAABIAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAY//8AAAAxYXZjQwFkABX/4QAYZ2QAFazZQMh+hAAAAwAEAAADACg8WLZYAQAGaOvjyyLAAAAAGHN0dHMAAAAAAAAAAQAAABUAAAgAAAAAFHN0c3MAAAAAAAAAAQAAAAEAAABoY3R0cwAAAAAAAAALAAAACwAAEAAAAAABAAAoAAAAAAEAABAAAAAAAQAAAAAAAAABAAAIAAAAAAEAACgAAAAAAQAAEAAAAAABAAAAAAAAAAEAAAgAAAAAAQAAGAAAAAABAAAIAAAAABxzdHNjAAAAAAAAAAEAAAABAAAAFQAAAAEAAABoc3RzegAAAAAAAAAAAAAAFQAACMQAAAGUAAABlwAAAZgAAAGYAAABmAAAAZgAAAGYAAABmAAAAccAAAGaAAACLgAAACQAAAAgAAAAEgAAABoAAAAVAAAAEgAAABIAAAJ5AAAALgAAABRzdGNvAAAAAAAAAAEAAAAwAAAAYnVkdGEAAABabWV0YQAAAAAAAAAhaGRscgAAAAAAAAAAbWRpcmFwcGwAAAAAAAAAAAAAAAAtaWxzdAAAACWpdG9vAAAAHWRhdGEAAAABAAAAAExhdmY1Ny44My4xMDA=\" type=\"video/mp4\" />\n",
              "                 </video>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7XV-Z1Znen44"
      },
      "source": [
        "# `AgentManager`: A powerfull class for hyperparameter optimization, training and evaluating agents."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l3OCUH2YfhOQ"
      },
      "source": [
        "# Create random agent as a baseline\n",
        "class RandomAgent(AgentWithSimplePolicy):\n",
        "    name = 'RandomAgent'\n",
        "    def __init__(self, env, **kwargs):\n",
        "        AgentWithSimplePolicy.__init__(self, env, **kwargs)\n",
        "\n",
        "    def fit(self, budget=None, **kwargs):  \n",
        "        pass\n",
        "          \n",
        "    def policy(self, observation):\n",
        "        return self.env.action_space.sample()"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 741
        },
        "id": "ePl7Bqq2eniv",
        "outputId": "df02d6e2-f046-4abf-89b3-e9f89ee1e75f"
      },
      "source": [
        "from rlberry.manager import AgentManager, evaluate_agents\n",
        "\n",
        "# Define parameters\n",
        "vi_params = {'gamma':0.1, 'epsilon':1e-3}\n",
        "\n",
        "# Create AgentManager to fit 4 agents using 1 job\n",
        "vi_stats = AgentManager(\n",
        "    ValueIterationAgent,\n",
        "    (env_ctor, env_kwargs),\n",
        "    fit_budget=0,\n",
        "    eval_kwargs=dict(eval_horizon=20),\n",
        "    init_kwargs=vi_params,\n",
        "    n_fit=4)\n",
        "vi_stats.fit()\n",
        "\n",
        "# Create AgentManager for baseline\n",
        "baseline_stats = AgentManager(\n",
        "    RandomAgent,\n",
        "    (env_ctor, env_kwargs),\n",
        "    fit_budget=0,\n",
        "    eval_kwargs=dict(eval_horizon=20),\n",
        "    n_fit=1)\n",
        "baseline_stats.fit()\n",
        "\n",
        "# Compare policies using 10 Monte Carlo simulations\n",
        "output = evaluate_agents([vi_stats, baseline_stats], n_simulations=10)\n"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[INFO] Running AgentManager fit() for ValueIterationAgent...  \n",
            "[INFO] ... trained! \n",
            "[INFO] Running AgentManager fit() for RandomAgent...  \n",
            "[INFO] ... trained! \n",
            "[INFO] Evaluating ValueIterationAgent... \n",
            "[INFO] [eval]... simulation 1/10 \n",
            "[INFO] [eval]... simulation 2/10 \n",
            "[INFO] [eval]... simulation 3/10 \n",
            "[INFO] [eval]... simulation 4/10 \n",
            "[INFO] [eval]... simulation 5/10 \n",
            "[INFO] [eval]... simulation 6/10 \n",
            "[INFO] [eval]... simulation 7/10 \n",
            "[INFO] [eval]... simulation 8/10 \n",
            "[INFO] [eval]... simulation 9/10 \n",
            "[INFO] [eval]... simulation 10/10 \n",
            "[INFO] Evaluating RandomAgent... \n",
            "[INFO] [eval]... simulation 1/10 \n",
            "[INFO] [eval]... simulation 2/10 \n",
            "[INFO] [eval]... simulation 3/10 \n",
            "[INFO] [eval]... simulation 4/10 \n",
            "[INFO] [eval]... simulation 5/10 \n",
            "[INFO] [eval]... simulation 6/10 \n",
            "[INFO] [eval]... simulation 7/10 \n",
            "[INFO] [eval]... simulation 8/10 \n",
            "[INFO] [eval]... simulation 9/10 \n",
            "[INFO] [eval]... simulation 10/10 \n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEICAYAAABS0fM3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAc1ElEQVR4nO3de5gdVZnv8W9ooUPLNVy8kIDRhB8dEcGOIOgMqGiHqETESxL1nCDq8UBilJE5OF6GiY4yOIoR4zUPB45nJAKOGMZoj2cEZBA1bAE12bzYBiHJ+HhJws2GhoQ+f1Q1qWz27q7udO3dnfp9nidP71q1atXbndr73bVW1apJAwMDmJlZee3V6gDMzKy1nAjMzErOicDMrOScCMzMSs6JwMys5JwIzMxK7hlFNi5pDrAcaANWRsQlNeuPAq4ADgO2Au+IiE1FxmRmZruaVNR9BJLagHuA1wCbgLXAgohYn6lzLfBvEXGVpFcB50TEO4dq98477xxob28vJGYzsz1VX1/fn7u6ug6rt67IM4ITgd6I2AAgaRUwD1ifqTMLuCB9fSNw/XCNtre309nZOcahmpnt2SqVyn2N1hU5RnAEsDGzvCkty7oLeFP6+ixgf0mHFBiTmZnVKHSMIIcPAV+UtAj4MbAZ2DHUBv39/VSr1SaEZmZWDkUmgs3AtMzy1LTsKRHxX6RnBJL2A86OiAeGatRdQ2ZmI1epVBquKzIRrAVmSppOkgDmAwuzFSQdCmyNiCeBD5NcQWRmZk1U2BhBRGwHFgM9QBW4JiLWSVom6cy02mlASLoHeBbwj0XFY2Zm9RV2+WhRqtXqgLuGzMxGplKpVLq6umbXW+c7i83MSq7VVw2VUk9PD2vWrGl1GGzbtg2Agw8+uKVxzJ07l+7u7pbGYFZmTgQltmXLFqD1icDMWsuJoAW6u7vHxTfgpUuXArB8+fIWR2JmreQxAjOzknMiMDMrOScCM7OSK90YweWXX05vb2+rwxgXBv8Og2MFZTdjxgyWLFnS6jDMmq50iaC3t5c7f11lR8eUVofScpN2JP/9lQ1/aHEkrdfWt7XVIZi1TOkSAcCOjik8eszcVodh48i+d7f+vg6zVvEYgZlZyTkRmJmVnBOBmVnJlW6MYOvWrbT1bXGfsO2irW8LW7fu3eowzFrCZwRmZiVXujOCKVOmcO8DT/iqIdvFvnevYcoUX1Js5VRoIpA0B1gOtAErI+KSmvVHAlcBB6V1LooI99mYmTVRYV1DktqAFcAZwCxggaRZNdU+SvIIyxNInmn8paLiMTOz+oocIzgR6I2IDRHxOLAKmFdTZwA4IH19IPBfBcZjZmZ1FNk1dASwMbO8CTipps7FwL9LWgI8Ezi9wHjMzKyOVg8WLwCujIjPSjoZ+IakYyPiyUYb9Pf3U61WR73Dvr6+UW9re7a+vr7dOrbMJqoiE8FmYFpmeWpalnUuMAcgIm6TNBk4FPhjo0bb29vp7OwcdVAdHR209d3n+wiASU88CsDA3vu2OJLWa+vbSkfHs3br2DIbzyqVSsN1RSaCtcBMSdNJEsB8YGFNnfuBVwNXSuoEJgN/KjAmZsyYUWTzE8rgNNQznv+sFkcyHjzLx4aVVmGJICK2S1oM9JBcGnpFRKyTtAy4PSJWA38DfF3SB0kGjhdFxEBRMQGebz7Dzyw2M4BJAwOFfu6OuWq1OjDRT997enpYs6b1XVNPnRG0+Jvw3Llz6e7ubmkMZnu6SqVS6erqml1vXasHi62FDjnkkFaHYGbjgBNBC3R3d/sbsJmNG550zsys5JwIzMxKzonAzKzknAjMzErOicDMrOScCMzMSs6JwMys5JwIzMxKzonAzKzknAjMzErOicDMrOScCMzMSs6JwMys5JwIzMxKrtBpqCXNAZaTPKFsZURcUrP+MuCV6WIHcHhEHFRkTGZmtqvCEoGkNmAF8BpgE7BW0uqIWD9YJyI+mKm/BDihqHjMzKy+IruGTgR6I2JDRDwOrALmDVF/AXB1gfGYmVkdRSaCI4CNmeVNadnTSDoKmA78qMB4zMysjvHyqMr5wHURsWO4iv39/VSr1SaEZGZWDkUmgs3AtMzy1LSsnvnA+XkabW9vp7OzczdDMzMrl0ql0nBdkYlgLTBT0nSSBDAfWFhbSdIxwMHAbQXGYmZmDRQ2RhAR24HFQA9QBa6JiHWSlkk6M1N1PrAqIgaKisXMzBqbNDAwsT5/q9XqgLuGzMxGplKpVLq6umbXW+c7i83MSs6JwMys5JwIzMxKzonAzKzknAjMzErOicDMrOScCMzMSs6JwMys5IZNBJJenqfMzMwmpjxnBJfnLDMzswmo4aRzkk4GTgEOk3RBZtUBJI+eNDOzPcBQs4/uA+yX1tk/U/4Q8OYigzIzs+ZpmAgi4mbgZklXRsR9TYzJzMyaKM/zCK6U9LQpSiPiVQXEY2ZmTZYnEXwo83oycDawvZhwzMys2YZNBBFR+3yzWyX9vKB4zMysyYZNBJKmZBb3ArqAA/M0LmkOsJzkKqOVEXFJnTpvBS4GBoC7IuJpj7M0M7Pi5OkaqpB8SE8i6RK6Fzh3uI0ktQErgNcAm4C1klZHxPpMnZnAh4GXR8Q2SYeP/FcwM7PdkadraPoo2z4R6I2IDQCSVgHzgPWZOu8BVkTEtnRffxzlvszMbJTydA1NBs4DXkFyZnAL8JWIeGyYTY8ANmaWNwEn1dQ5Ot3HrSTdRxdHxA/yhW5mZmMhT9fQ/wEeZue0EguBbwBvGaP9zwROA6YCP5b0ooh4oNEG/f39VKvVMdi1mZlBvkRwbETMyizfKGl9w9o7bQamZZanpmVZm4CfRcQTwL2S7iFJDGsbNdre3k5nZ2eO3ZuZ2aBKpfYC0J3yTDr3C0kvG1yQdBJwe47t1gIzJU2XtA8wH1hdU+d6krMBJB1K0lW0IUfbZmY2RvKcEXQBP5F0f7p8JBCSfgUMRMRx9TaKiO2SFgM9JP3/V0TEOknLgNsjYnW67rXpGcYO4MKI2LKbv5OZmY1AnkQwZ7SNR8QaYE1N2cczrweAC9J/ZmbWAnkSwScj4p3ZAknfqC0zM7OJKc8YwQuzC5KeQdJdZGZme4ChHkzzYeDvgH0lPURyZzHA48DXmhCbmZk1wVDPI/g08GlJn46IDzcxJjMza6I8YwTfl/TXtYUR8eMC4jEzsybLkwguzLyeTDKHUAXwg2nMzPYAeSade0N2WdI04POFRWRmZk2V56qhWpsAz/FgZraHyDP76OUks45CkjiOB35RZFBmZtY8ecYIsvMKbQeujohbC4rHzMyaLM8YwVXppHFHDxYVG5KZmTXTsGMEkk4DfkPy2MkvAffUu5zUzMwmpjxdQ58FXhsRASDpaOBqPM2EmdkeIc9VQ3sPJgGAiLgH2Lu4kMzMrJlyDRZLWgn833T57eR7MI2ZmU0AeRLB/wTOB96fLt9CMlZgZmZ7gDxXDfUDn0v/jYikOcBykieUrYyIS2rWLwI+w85nGX8xIlaOdD9mZjZ6ec4IRkVSG8mVRq8huRt5raTVEVH74PtvRcTiouIwM7OhjWaKibxOBHojYkNEPA6sAuYVuD8zMxuFws4IgCOAjZnlTcBJdeqdnd6XcA/wwYjYWKeOmZkVJM9cQ0eTTEV9VLZ+RIzFNNQ3kExZ0S/pfwBXMcz01v39/VSr1THYtZmZQb4zgmuBrwBfB3aMoO3NwLTM8lR2DgoDEBFbMosrgUuHa7S9vZ3OTk9+amY2EpVKpeG6PIlge0R8eRT7XQvMlDSdJAHMBxZmK0h6TkT8Pl08E/BXfTOzJsuTCG6QdB7wHaB/sDAitg61UURsl7QY6CG5fPSKiFgnaRlwe0SsBt4v6UySWU23AotG92uYmdloTRoYGBiygqR76xQPRMTziwlpaNVqdcBdQ2ZmI1OpVCpdXV2z663Lc0PZ9LEPyczMxos8Vw3tTTLNxODU0zcBX42IJwqMy8zMmiTPGMGXSWYbHZxf6J1p2buLCsrMzJonTyJ4aUS8OLP8I0l3FRWQmZk1V54pJnZIesHggqTnM7L7CczMbBzLc0ZwIXCjpA3AJJI7jM8pNCozM2uaPFcN/YekmYB2FkX/UNuYmdnE0bBrSNKr0p9vAl4HzEj/vS4tMzOzPcBQZwSnAj8C3lBn3QDwr4VEZGZmTdUwEUTE36cvl0XELncXp/MHmZnZHiDPVUPfrlN23VgHYmZmrdHwjEDSMcALgQNrxgQOACYXHZiZmTXHUGMEAl4PHMSu4wQPA+8pMigzM2ueocYIvgt8V9LJEXFbE2MyM7MmynND2R2SzifpJnqqSygi3lVYVGZm1jR5Bou/ATwb6AZuJnnk5MNFBmVmZs2TJxHMiIiPAX+JiKtIbi47KU/jkuZICkm9ki4aot7ZkgYk1X1ogpmZFSdPIhh87sADko4FDgQOH24jSW3ACuAMYBawQNKsOvX2B5YCP8sbtJmZjZ08ieBrkg4GPgasBtYDl+bY7kSgNyI2RMTjwCpgXp16nwD+CXgsX8hmZjaW8kw6tzJ9eTMwkucUHwFszCxvoqZLSdJLgGkR8T1JF46gbTMzGyN5HlX58XrlEbFsd3YsaS/gc8CikWzX399PtVrdnV2bmVlGnstH/5J5PZnkJrM8n8SbgWmZ5alp2aD9gWOBmyRBcmXSaklnRsTtjRptb2+ns7Mzx+7NzGxQpVJpuC5P19Bns8uS/hnoybHftcDMdIK6zcB8YGGm3QeBQzPt3gR8aKgkYGZmYy/PYHGtDpJv90OKiO3AYpKkUQWuiYh1kpZJOnMU+zUzswLkGSP4FcnzBwDagMOAXOMDEbEGWFNT1mjM4bQ8bZqZ2djKM0bw+szr7cAf0m/7Zma2BxhqGuop6cva6SQOkEREbC0uLDMza5ahzggqJF1Ck+qsG2Bk9xSYmdk4NdQ01H4cpZlZCeQZIyCdYmImu05D/eOigjIzs+bJc9XQu0kmhZsK3Am8DLgNeFWxoZmZWTPkuY9gKfBS4L6IeCVwAvBAoVGZmVnT5EkEj0XEYwCS2iPibpLnGZuZ2R4gzxjBJkkHAdcDP5S0Dbiv2LDMzKxZ8sw1dFb68mJJN5I8mOYHhUZlZmZNk2ew+AvAqoj4SUTc3ISYzMysifJ0DVWAjyqZK/o7JEnBM4Same0hhh0sjoirImIuyZVDAfyTpN8UHpmZmTXFSKahngEcAxwF3F1MOGZm1mx5xgguBc4CfkvyAPpPRITvIzAz20PkGSP4LXByRPy56GDMzKz58iSCrwMLJT0/IpZJOhJ4dkT8fLgNJc0BlpM80GZlRFxSs/59wPnADuAR4L0RsX6kv4SZmY1enjGCFcDJwIJ0+eG0bEiS2tJ6ZwCzgAWSZtVU+2ZEvCgijgcuBT6XN3AzMxsbeRLBSRFxPvAYQERsA/bJsd2JQG9EbIiIx0nGF+ZlK0TEQ5nFZ7LzkZhmZtYkebqGnki/3Q8ASDoMeDLHdkcAGzPLm4CTaitJOh+4gCS5eEZTM7Mmy5MIvkByI9nhkv4ReDPw0bEKICJWACskLUzb/e9D1e/v76darY7V7s3MSi/PXEP/IqkCvJrksZVvjIg8n8SbgWmZ5alpWSOrgC8P12h7ezudnZ05dm9mZoMqlUrDdbmeUJZOPT3Sm8jWAjMlTSdJAPOBhdkKkmZGxOBdyq8DfMeymVmT5UoEoxER2yUtBnpILh+9IiLWSVoG3B4Rq4HFkk4HngC2MUy3kJmZjb1JAwMT60KdarU64K4hM7ORqVQqla6urtn11o1kriEzM9sDORGYmZWcE4GZWck5EZiZlVxhVw2Z2cTU09PDmjVrWhrDtm3bADj44INbGgfA3Llz6e7ubnUYhXIiMLNxZ8uWLcD4SARl4ERgZrvo7u5u+TfgpUuXArB8+fKWxlEWTgRm48Tll19Ob29vq8MYFwb/DoMJoexmzJjBkiVLCmvficBsnOjt7eU36+7gyP12tDqUljtgYBIA/ffd3uJIWu/+R9oK34cTgdk4cuR+O/i7lzw0fEUrjU/94oDC9+HLR83MSs6JwMys5JwIzMxKzonAzKzknAjMzErOicDMrOQKvXxU0hxgOckTylZGxCU16y8A3g1sB/4EvCsi7isyJrPxauvWrfz54bamXC5oE8d9D7dx6Nathe6jsDMCSW3ACuAMYBawQNKsmmp3ALMj4jjgOuDSouIxM7P6ijwjOBHojYgNAJJWAfOA9YMVIuLGTP2fAu8oMB6zcW3KlCk88+ENvqHMdvGpXxxA+5Qphe6jyDGCI4CNmeVNaVkj5wLfLzAeMzOrY1xMMSHpHcBs4NTh6vb391OtVosPyqzJ+vr6KH5WGZuI+vr6Cv3cKzIRbAamZZanpmW7kHQ68BHg1IjoH67R9vZ2Ojs7xyxIs/Gio6ODYd8AVkodHR27/blXqVQarisyEawFZkqaTpIA5gMLsxUknQB8FZgTEX8sMBYzM2ugsDGCiNgOLAZ6gCpwTUSsk7RM0plptc8A+wHXSrpT0uqi4jEzs/oKHSOIiDXAmpqyj2den17k/s3MbHi+s9jMrOScCMzMSs6JwMys5JwIzMxKzonAzKzknAjMzErOicDMrOScCMzMSs6JwMys5JwIzMxKzonAzKzkxsXzCMwscf8jfmYxwIOPTwLgwH0GWhxJ693/SBszC96HE4HZODFjxoxWhzBuPNTbC8DhR/lvMpPijw0nArNxYsmSJa0OYdxYunQpAMuXL29xJOXgMQIzs5JzIjAzK7lCu4YkzQGWA23Ayoi4pGb9XwOfB44D5kfEdUXGY2ZmT1fYGYGkNmAFcAYwC1ggaVZNtfuBRcA3i4rDzMyGVuQZwYlAb0RsAJC0CpgHrB+sEBG/S9c9WWAcZmY2hCLHCI4ANmaWN6VlZmY2jky4y0f7+/upVqutDsPMCtTX1wfg93qTFJkINgPTMstT07Ld0t7eTmdn5+42Y2bjWEdHB4Df62OoUqk0XFdkIlgLzJQ0nSQBzAcWFrg/MzMbhcLGCCJiO7AY6AGqwDURsU7SMklnAkh6qaRNwFuAr0paV1Q8ZmZW36SBgYk1qVO1Wh3w6aJZcXp6elizZk1LY+hN5xoaD/MvzZ07l+7u7laHsdsqlUqlq6trdr11E26w2Mz2fIccckirQygVJwIz20V3d/ce8Q3Y8vNcQ2ZmJedEYGZWck4EZmYl50RgZlZyTgRmZiXnRGBmVnJOBGZmJedEYGZWchPuhrK+vr4/VyqV+1odh5nZBHNUoxUTbq4hMzMbW+4aMjMrOScCM7OScyIwMys5JwIzs5JzIjAzKzknglGSdKOk7pqyD0j6coP6N0mq+3SgHPtaJOmL6es3Spo1mnYatH28pLmZ5TMlXbSbbX5A0mOSDtz9COu2f5qkU4po20DSDkl3Svq1pBskHTRG7T51HI9Re8dLGpA0Z6zarGn/eZJK8Zx1J4LRuxqYX1M2Py0v0huBESUCSUPdL3I88FQiiIjVEXHJKGMbtABYC7xpN9tp5DTAiaA4j0bE8RFxLLAVOL/VATWwAPjP9GcRngeUIhFMuBvKxpHrgE9K2iciHpf0POC5wAJJnwP2Ba6LiL+v3VDSIxGxX/r6zcDrI2KRpMOArwBHplU/EBG3ZrY7BTgTOFXSR4Gz01UrgMOAPuA9EXG3pCuBx4ATgFslrQKWA5OBR4FzgHuBZcC+kl4BfDqNe3ZELE5/pyuAQ4E/AedExP1p2w8Bs4FnA38bEdelMb4A2A84D/gI8L/T8g7gSuBYINK/1fkRcbuk1wL/ALQDv03384ik3wFXAW8A9gbekv5O7wN2SHoHsCQibhnm/8pG7zbgOABJJ1JzDEVESFpEclx2AC8AvhMRf5tucw7wYeAB4C6gPy1/Ho2PrUdJjtvDgXcB/w04GfhZRCxKt59Ecjy8BrhF0uSIeCxd9zHgHWm7G4FKRPxzemw2eq/UO54vATol3QlcFRGXjdlfdZzxGcEoRcRW4OfAGWnRfOAa4CMRMZvkzXOqpONG0Oxy4LKIeCnJh/zKmn3+BFgNXJh+Y/st8DWSD8Mu4EPAlzKbTAVOiYgLgLuBv4qIE4CPA5+KiMfT199K2/tWTTyXk7wBjgP+BfhCZt1zgFcAryd5wwyaD6wCbgEk6Vlp+XnAtoiYBXwM6CKpcCjwUeD0iHgJcDtwQaa9P6flXwY+FBG/I0mWl6UxOwkURFIb8GqSYw7qHEOZ6scDbwNeBLxN0jRJzyFJ8C8nOVayZ7JDHVsHk3zwfzDd92XAC4EXSTo+rXMKcG/6HrgJeF0a8+B758Uk781sd+xQ75V6x/NFwC3pcbbHJgHwGcHuGuwe+m7681zgrZLeS/K3fQ7Jwf/LnO2dDsySNLh8gKT9GlVO150CXJvZpj1T5dqI2JG+PhC4StJMYIDkG/ZwTmZn9843gEsz666PiCeB9ZkPe0hO08+KiCclfZvkW9sXSd5kywEi4teSBv8mLyP5G92a/g77kHwLHfSv6c8KxXU12a72Tb8FHwFUgR+m5UMdQ/8REQ8CSFpPMp3BocBNEfGntPxbwNFp/aGOrRsiYkDSr4A/RMSv0u3XkXTX3ElynK1K668iOWv4NknS+W56dvCYpBvSbYd7rzQ6nkvBiWD3fBe4TNJLSE6Lt5J803hpRGxLTzkn19kuO69Hdv1ewMsGT3EHZQ7cWnsBD0TE8Q3W/yXz+hPAjRFxVnpaflOjRnPqz7yeBCDpRcBM4IeZD/V7SRJBI5OAH0ZEo37ewf3swMdrszwaEcen3Xk9JGMEX2DoYyh7POzu/9VgW0/WtPsk8Iz0TOVsYJ6kj5AcQ4dI2n+INod7rzzteC4Tdw3thoh4BLiRpK/zauAAkg/fB9NvFWc02PQPkjol7QWclSn/d2DJ4ELmNDjrYWD/dP8PAfdKektaf5KkFzfY54HA5vT1onrt1fETdg6Iv52ku2coC4CLI+J56b/nAs+VdBRwK/DWNM5ZJF0IAD8FXi5pRrrumZKOrtN21lAx2xiJiD7g/cDfpBccNDqGGvkZSffoIZIGx3gGjfTYyno18MuImJYeZ0eRnA2cRXKcvUHS5PQs4PXp7zKS98qg0hxnTgS772qS/sirI+Iu4A6SvtRvkhyU9VwE/BvJm+H3mfL3A7Ml/TI9vX5fnW1XARdKuiMd/Ho7cK6ku4B1wLwG+7wU+LSkO9j129qNJN1Rd0p6W802S4Bz0m6cdwJLG7Q9aD7wnZqy76TlXwIOS3+vT6axPph2GywCrk73cxtwzDD7uQE4K435r4apa7shIu4g6dpcQONjqNG2vwcuJvk/vZWkm2nQSI+trAU8/Tj7NrAgItaSjCv8Evg+8CvgwbRO3vfKoF+SXJRwl6QPjiC+Ccezj1pTpKfze0fEY2kC+3+A0gFrszEjab/0qrMO4MfAeyPiF62Oazxzn6s1SwdwY9pFMAk4z0nACvK1tPtxMsmVSU4Cw/AZgZlZyXmMwMys5JwIzMxKzonAzKzknAjMWkDSQZLOa3UcZuBEYNYqB5HMv2TWcr581KwBSdcD00guQ1weEV+TdC7wv8jMppnO1Fp35lhJF6dlz09/fj4ivkAysdkL0jl9fhgRFzbzdzPLciIwa+xdEbFV0r7AWknfI5k59SUk0w/8iCQZwM6ZY/9T0pEkc/R0puuOAV5JMl1BKHl40UXAsUPMfWPWNE4EZo29X9LgXFDTSKZCuDmdghxJ17JzNs2hZo79XkT0A/2S/giUbnZLG9+cCMzqkHQayYf7yRHRJ+kmkjmkOhtsMtTMsWM5M6fZmPNgsVl9B5I8SKdP0jEkz014Jslsmgens3GenamfZ+bYrNLMbGnjnxOBWX0/IJn7vkoysPtTkimYP0XyZLpbgd+xc2bLPDPHPiUitpA8jOfXkj5TzK9glo/nGjIbgczMls8gmQr5ioionRLZbELxGYHZyFycXvL5a5Knr13f4njMdpvPCMzMSs5nBGZmJedEYGZWck4EZmYl50RgZlZyTgRmZiXnRGBmVnL/H9cceTO7zU4GAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "slTijRljiamE",
        "outputId": "e5513671-498d-42b5-ff8b-714180465629"
      },
      "source": [
        "# The value of gamma above makes our VI agent quite bad! Let's optimize it.\n",
        "vi_stats.optimize_hyperparams(n_trials=30, timeout=30, n_fit=1, sampler_method='random', pruner_method='none')\n",
        "\n",
        "# fit with optimized params\n",
        "vi_stats.fit()\n",
        "\n",
        "# ... and see the results\n",
        "output = evaluate_agents([vi_stats, baseline_stats])\n"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Connected to rlberry_data/temp/manager_data/ValueIterationAgent_488f4904b7988b6c1320a4a725034f64/optuna_data.db (sqlite3 version = 2.6.0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[32m[I 2022-02-20 21:23:37,109]\u001b[0m A new study created in RDB with name: no-name-649024b4-f9bc-4132-8d4d-0b1fbcdad28a\u001b[0m\n",
            "[INFO] Saved AgentManager(ValueIterationAgent) using pickle. \n",
            "[INFO] Running AgentManager fit() for optim...  \n",
            "[INFO] ... trained! \n",
            "[INFO] [eval]... simulation 1/2 \n",
            "[INFO] [eval]... simulation 2/2 \n",
            "\u001b[32m[I 2022-02-20 21:23:37,456]\u001b[0m Trial 0 finished with value: 0.7449999999999999 and parameters: {'gamma': 0.75}. Best is trial 0 with value: 0.7449999999999999.\u001b[0m\n",
            "[INFO] Running AgentManager fit() for optim...  \n",
            "[INFO] ... trained! \n",
            "[INFO] [eval]... simulation 1/2 \n",
            "[INFO] [eval]... simulation 2/2 \n",
            "\u001b[32m[I 2022-02-20 21:23:38,308]\u001b[0m Trial 1 finished with value: 6.715 and parameters: {'gamma': 0.99}. Best is trial 1 with value: 6.715.\u001b[0m\n",
            "[INFO] Running AgentManager fit() for optim...  \n",
            "[INFO] ... trained! \n",
            "[INFO] [eval]... simulation 1/2 \n",
            "[INFO] [eval]... simulation 2/2 \n",
            "\u001b[32m[I 2022-02-20 21:23:39,204]\u001b[0m Trial 2 finished with value: 6.344999999999999 and parameters: {'gamma': 0.99}. Best is trial 1 with value: 6.715.\u001b[0m\n",
            "[INFO] Running AgentManager fit() for optim...  \n",
            "[INFO] ... trained! \n",
            "[INFO] [eval]... simulation 1/2 \n",
            "[INFO] [eval]... simulation 2/2 \n",
            "\u001b[32m[I 2022-02-20 21:23:39,830]\u001b[0m Trial 3 finished with value: 1.3249999999999997 and parameters: {'gamma': 0.75}. Best is trial 1 with value: 6.715.\u001b[0m\n",
            "[INFO] Running AgentManager fit() for optim...  \n",
            "[INFO] ... trained! \n",
            "[INFO] [eval]... simulation 1/2 \n",
            "[INFO] [eval]... simulation 2/2 \n",
            "\u001b[32m[I 2022-02-20 21:23:40,460]\u001b[0m Trial 4 finished with value: 0.8099999999999999 and parameters: {'gamma': 0.25}. Best is trial 1 with value: 6.715.\u001b[0m\n",
            "[INFO] Running AgentManager fit() for optim...  \n",
            "[INFO] ... trained! \n",
            "[INFO] [eval]... simulation 1/2 \n",
            "[INFO] [eval]... simulation 2/2 \n",
            "\u001b[32m[I 2022-02-20 21:23:41,083]\u001b[0m Trial 5 finished with value: 0.8199999999999998 and parameters: {'gamma': 0.5}. Best is trial 1 with value: 6.715.\u001b[0m\n",
            "[INFO] Running AgentManager fit() for optim...  \n",
            "[INFO] ... trained! \n",
            "[INFO] [eval]... simulation 1/2 \n",
            "[INFO] [eval]... simulation 2/2 \n",
            "\u001b[32m[I 2022-02-20 21:23:41,909]\u001b[0m Trial 6 finished with value: 0.815 and parameters: {'gamma': 0.5}. Best is trial 1 with value: 6.715.\u001b[0m\n",
            "[INFO] Running AgentManager fit() for optim...  \n",
            "[INFO] ... trained! \n",
            "[INFO] [eval]... simulation 1/2 \n",
            "[INFO] [eval]... simulation 2/2 \n",
            "\u001b[32m[I 2022-02-20 21:23:42,576]\u001b[0m Trial 7 finished with value: 0.84 and parameters: {'gamma': 0.5}. Best is trial 1 with value: 6.715.\u001b[0m\n",
            "[INFO] Running AgentManager fit() for optim...  \n",
            "[INFO] ... trained! \n",
            "[INFO] [eval]... simulation 1/2 \n",
            "[INFO] [eval]... simulation 2/2 \n",
            "\u001b[32m[I 2022-02-20 21:23:43,219]\u001b[0m Trial 8 finished with value: 0.815 and parameters: {'gamma': 0.5}. Best is trial 1 with value: 6.715.\u001b[0m\n",
            "[INFO] Running AgentManager fit() for optim...  \n",
            "[INFO] ... trained! \n",
            "[INFO] [eval]... simulation 1/2 \n",
            "[INFO] [eval]... simulation 2/2 \n",
            "\u001b[32m[I 2022-02-20 21:23:43,965]\u001b[0m Trial 9 finished with value: 0.78 and parameters: {'gamma': 0.25}. Best is trial 1 with value: 6.715.\u001b[0m\n",
            "[INFO] Running AgentManager fit() for optim...  \n",
            "[INFO] ... trained! \n",
            "[INFO] [eval]... simulation 1/2 \n",
            "[INFO] [eval]... simulation 2/2 \n",
            "\u001b[32m[I 2022-02-20 21:23:44,799]\u001b[0m Trial 10 finished with value: 0.815 and parameters: {'gamma': 0.5}. Best is trial 1 with value: 6.715.\u001b[0m\n",
            "[INFO] Running AgentManager fit() for optim...  \n",
            "[INFO] ... trained! \n",
            "[INFO] [eval]... simulation 1/2 \n",
            "[INFO] [eval]... simulation 2/2 \n",
            "\u001b[32m[I 2022-02-20 21:23:45,492]\u001b[0m Trial 11 finished with value: 0.815 and parameters: {'gamma': 0.5}. Best is trial 1 with value: 6.715.\u001b[0m\n",
            "[INFO] Running AgentManager fit() for optim...  \n",
            "[INFO] ... trained! \n",
            "[INFO] [eval]... simulation 1/2 \n",
            "[INFO] [eval]... simulation 2/2 \n",
            "\u001b[32m[I 2022-02-20 21:23:46,154]\u001b[0m Trial 12 finished with value: 1.65 and parameters: {'gamma': 0.75}. Best is trial 1 with value: 6.715.\u001b[0m\n",
            "[INFO] Running AgentManager fit() for optim...  \n",
            "[INFO] ... trained! \n",
            "[INFO] [eval]... simulation 1/2 \n",
            "[INFO] [eval]... simulation 2/2 \n",
            "\u001b[32m[I 2022-02-20 21:23:46,899]\u001b[0m Trial 13 finished with value: 0.825 and parameters: {'gamma': 0.1}. Best is trial 1 with value: 6.715.\u001b[0m\n",
            "[INFO] Running AgentManager fit() for optim...  \n",
            "[INFO] ... trained! \n",
            "[INFO] [eval]... simulation 1/2 \n",
            "[INFO] [eval]... simulation 2/2 \n",
            "\u001b[32m[I 2022-02-20 21:23:47,499]\u001b[0m Trial 14 finished with value: 0.8199999999999998 and parameters: {'gamma': 0.5}. Best is trial 1 with value: 6.715.\u001b[0m\n",
            "[INFO] Running AgentManager fit() for optim...  \n",
            "[INFO] ... trained! \n",
            "[INFO] [eval]... simulation 1/2 \n",
            "[INFO] [eval]... simulation 2/2 \n",
            "\u001b[32m[I 2022-02-20 21:23:48,057]\u001b[0m Trial 15 finished with value: 1.04 and parameters: {'gamma': 0.75}. Best is trial 1 with value: 6.715.\u001b[0m\n",
            "[INFO] Running AgentManager fit() for optim...  \n",
            "[INFO] ... trained! \n",
            "[INFO] [eval]... simulation 1/2 \n",
            "[INFO] [eval]... simulation 2/2 \n",
            "\u001b[32m[I 2022-02-20 21:23:48,668]\u001b[0m Trial 16 finished with value: 1.0949999999999998 and parameters: {'gamma': 0.75}. Best is trial 1 with value: 6.715.\u001b[0m\n",
            "[INFO] Running AgentManager fit() for optim...  \n",
            "[INFO] ... trained! \n",
            "[INFO] [eval]... simulation 1/2 \n",
            "[INFO] [eval]... simulation 2/2 \n",
            "\u001b[32m[I 2022-02-20 21:23:49,736]\u001b[0m Trial 17 finished with value: 6.619999999999999 and parameters: {'gamma': 0.99}. Best is trial 1 with value: 6.715.\u001b[0m\n",
            "[INFO] Running AgentManager fit() for optim...  \n",
            "[INFO] ... trained! \n",
            "[INFO] [eval]... simulation 1/2 \n",
            "[INFO] [eval]... simulation 2/2 \n",
            "\u001b[32m[I 2022-02-20 21:23:50,365]\u001b[0m Trial 18 finished with value: 1.495 and parameters: {'gamma': 0.75}. Best is trial 1 with value: 6.715.\u001b[0m\n",
            "[INFO] Running AgentManager fit() for optim...  \n",
            "[INFO] ... trained! \n",
            "[INFO] [eval]... simulation 1/2 \n",
            "[INFO] [eval]... simulation 2/2 \n",
            "\u001b[32m[I 2022-02-20 21:23:50,751]\u001b[0m Trial 19 finished with value: 0.83 and parameters: {'gamma': 0.5}. Best is trial 1 with value: 6.715.\u001b[0m\n",
            "[INFO] Running AgentManager fit() for optim...  \n",
            "[INFO] ... trained! \n",
            "[INFO] [eval]... simulation 1/2 \n",
            "[INFO] [eval]... simulation 2/2 \n",
            "\u001b[32m[I 2022-02-20 21:23:51,130]\u001b[0m Trial 20 finished with value: 0.8099999999999999 and parameters: {'gamma': 0.25}. Best is trial 1 with value: 6.715.\u001b[0m\n",
            "[INFO] Running AgentManager fit() for optim...  \n",
            "[INFO] ... trained! \n",
            "[INFO] [eval]... simulation 1/2 \n",
            "[INFO] [eval]... simulation 2/2 \n",
            "\u001b[32m[I 2022-02-20 21:23:51,507]\u001b[0m Trial 21 finished with value: 0.82 and parameters: {'gamma': 0.5}. Best is trial 1 with value: 6.715.\u001b[0m\n",
            "[INFO] Running AgentManager fit() for optim...  \n",
            "[INFO] ... trained! \n",
            "[INFO] [eval]... simulation 1/2 \n",
            "[INFO] [eval]... simulation 2/2 \n",
            "\u001b[32m[I 2022-02-20 21:23:51,894]\u001b[0m Trial 22 finished with value: 1.045 and parameters: {'gamma': 0.75}. Best is trial 1 with value: 6.715.\u001b[0m\n",
            "[INFO] Running AgentManager fit() for optim...  \n",
            "[INFO] ... trained! \n",
            "[INFO] [eval]... simulation 1/2 \n",
            "[INFO] [eval]... simulation 2/2 \n",
            "\u001b[32m[I 2022-02-20 21:23:52,291]\u001b[0m Trial 23 finished with value: 0.7849999999999999 and parameters: {'gamma': 0.5}. Best is trial 1 with value: 6.715.\u001b[0m\n",
            "[INFO] Running AgentManager fit() for optim...  \n",
            "[INFO] ... trained! \n",
            "[INFO] [eval]... simulation 1/2 \n",
            "[INFO] [eval]... simulation 2/2 \n",
            "\u001b[32m[I 2022-02-20 21:23:52,678]\u001b[0m Trial 24 finished with value: 0.785 and parameters: {'gamma': 0.25}. Best is trial 1 with value: 6.715.\u001b[0m\n",
            "[INFO] Running AgentManager fit() for optim...  \n",
            "[INFO] ... trained! \n",
            "[INFO] [eval]... simulation 1/2 \n",
            "[INFO] [eval]... simulation 2/2 \n",
            "\u001b[32m[I 2022-02-20 21:23:53,199]\u001b[0m Trial 25 finished with value: 7.0249999999999995 and parameters: {'gamma': 0.99}. Best is trial 25 with value: 7.0249999999999995.\u001b[0m\n",
            "[INFO] Running AgentManager fit() for optim...  \n",
            "[INFO] ... trained! \n",
            "[INFO] [eval]... simulation 1/2 \n",
            "[INFO] [eval]... simulation 2/2 \n",
            "\u001b[32m[I 2022-02-20 21:23:53,580]\u001b[0m Trial 26 finished with value: 0.815 and parameters: {'gamma': 0.25}. Best is trial 25 with value: 7.0249999999999995.\u001b[0m\n",
            "[INFO] Running AgentManager fit() for optim...  \n",
            "[INFO] ... trained! \n",
            "[INFO] [eval]... simulation 1/2 \n",
            "[INFO] [eval]... simulation 2/2 \n",
            "\u001b[32m[I 2022-02-20 21:23:53,955]\u001b[0m Trial 27 finished with value: 0.8049999999999999 and parameters: {'gamma': 0.1}. Best is trial 25 with value: 7.0249999999999995.\u001b[0m\n",
            "[INFO] Running AgentManager fit() for optim...  \n",
            "[INFO] ... trained! \n",
            "[INFO] [eval]... simulation 1/2 \n",
            "[INFO] [eval]... simulation 2/2 \n",
            "\u001b[32m[I 2022-02-20 21:23:54,334]\u001b[0m Trial 28 finished with value: 0.845 and parameters: {'gamma': 0.25}. Best is trial 25 with value: 7.0249999999999995.\u001b[0m\n",
            "[INFO] Running AgentManager fit() for optim...  \n",
            "[INFO] ... trained! \n",
            "[INFO] [eval]... simulation 1/2 \n",
            "[INFO] [eval]... simulation 2/2 \n",
            "\u001b[32m[I 2022-02-20 21:23:54,719]\u001b[0m Trial 29 finished with value: 0.8049999999999999 and parameters: {'gamma': 0.1}. Best is trial 25 with value: 7.0249999999999995.\u001b[0m\n",
            "[INFO] Number of finished trials: 30 \n",
            "[INFO] Best trial: \n",
            "[INFO] Value: 7.0249999999999995 \n",
            "[INFO] Params: \n",
            "[INFO]     gamma: 0.99 \n",
            "[INFO] Running AgentManager fit() for ValueIterationAgent...  \n",
            "[INFO] ... trained! \n",
            "[INFO] Evaluating ValueIterationAgent... \n",
            "[INFO] [eval]... simulation 1/5 \n",
            "[INFO] [eval]... simulation 2/5 \n",
            "[INFO] [eval]... simulation 3/5 \n",
            "[INFO] [eval]... simulation 4/5 \n",
            "[INFO] [eval]... simulation 5/5 \n",
            "[INFO] Evaluating RandomAgent... \n",
            "[INFO] [eval]... simulation 1/5 \n",
            "[INFO] [eval]... simulation 2/5 \n",
            "[INFO] [eval]... simulation 3/5 \n",
            "[INFO] [eval]... simulation 4/5 \n",
            "[INFO] [eval]... simulation 5/5 \n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEGCAYAAABvtY4XAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAXbElEQVR4nO3de5hddX3v8XcIYQIiV9GqQVADXwZRosELYOuF6mDEC7VWUvUckLZPDxSwVixUsdRaaW0rBopayrFQa0EFEWmjKadyF6luCBcZvi0XqeFYFRIEDAwkmf6x1pCdOHvPysxesydr3q/nmWf2Xrffd2bW/szav73Wb80ZHR1FktQ82/S7AElSPQx4SWooA16SGsqAl6SGMuAlqaG27XcB7VauXDk6MDDQ7zIkaauxdu3aBxYvXrzHePNmVMAPDAwwODjY7zIkaavRarXu6zTPLhpJaigDXpIayoCXpIYy4CWpoQx4SWooA16SGsqAl6SGmlHnwTfBihUrWL58eV9rWLNmDQC77rprX+sAWLJkCUNDQ/0uQ5qVDPgGevDBB4GZEfCS+seA77GhoaG+H7GedNJJACxbtqyvdUjqL/vgJamhDHhJaigDXpIaqjF98GeffTZ33XVXv8uYEcZ+D2N98bPdwoULOeGEE/pdhjTtGhPwd911FytvH2b9Drv1u5S+m7O++LO27vlxnyvpv7lrV/e7BKlvGhPwAOt32I3H9lvS7zI0g2x/Z3+vSZD6yT54SWooA16SGsqAl6SGMuAlqaEMeElqqNrOoomIAL7UNukFwEcz89N1tSlJ2qi2gM/MBBYBRMRc4H7g0rraW716NXPXPuhpcdrE3LUPsnr1vH6XIfXFdJ0Hfxhwd2beV2sr69cxd+2DtTaxVdiwofi+jT1wrF/X7wqkvpmugD8KuHCihUZGRhgeHp5UAwsXLmT+/PmTWrdpVq1aBcCCBQv6XMnMsGDBgknvV9LWbM7o6GitDUTEdsD/B16UmV2vnR8eHh4dHBystZ7ZwPHgpdmj1Wq1Fi9efNB486bjPfybgJsmCndJUm9NR8AvpUL3jCSpt2oN+Ih4GvAG4Kt1tiNJ+kW1fsiamT8Hdq+zDUnS+DyPTpIayoCXpIYy4CWpoRp1R6eZYMWKFSxf3t/hEmbSPVmXLFnC0NBQv8uQZiUDvoF2393PtSUZ8D03NDTkEaukGcE+eElqKANekhrKgJekhjLgJamhDHhJaigDXpIayoCXpIYy4CWpoQx4SWooA16SGsqAl6SGqnUsmojYBTgPOAAYBd6XmTfU2aYkqVD3Efwy4JuZuR9wIDBcc3uSpFJtR/ARsTPwK8DRAJn5BPBEXe1JkjZVZxfN84GfAn8fEQcCLeCk8kbckqSa1Rnw2wIvA07IzBsjYhlwCnBapxVGRkYYHrYXR5J6oc6AXwWsyswby+cXUwR8RwMDAwwODtZYkiQ1S6vV6jivtg9ZM/O/gR9GRJSTDgPuqKs9SdKm6r5l3wnAFyNiO+Ae4Jia25MklWoN+MxcCRxUZxuSpPF5JaskNZQBL0kNZcBLUkMZ8JLUUAa8JDWUAS9JDWXAS1JDGfCS1FATBnxEHFplmiRpZqlyBH92xWmSpBmk41AFEXEwcAiwR0R8oG3WTsDcuguTJE1Nt7FotgN2LJd5etv0h4Ffr7MoSdLUdQz4zLwauDoizs/M+6axJklSD1QZTfL8iBjdfGJmvr6GeiRJPVIl4D/Y9ng+8A5gXT3lSJJ6ZcKAz8zN7wd1fUT8e031SJJ6ZMKAj4jd2p5uAywGdq6tIklST1TpomkBo8Aciq6Ze4Fj6yxKkjR1Vbponj/ZjUfED4BHgPXAusz09n2SNE2qdNHMB44DXk1xJH8t8LnMfLxiG6/LzAcmX6IkaTKqdNH8A8VR+NjwBL8JfAF4Z11FSZKmrkrAH5CZ+7c9vzIi7qi4/VHgX8vz6P82M8/ttvDIyAjDw8MVNy1J6qZKwN8UEa/KzO8ARMQrge9V3P6rM/P+iHgmcEVE3JmZ13RaeGBggMHBwYqbliS1Wpufyb5RlYBfDHw7Iv6rfP48ICPiNmA0M1/SacXMvL/8/pOIuBR4BdAx4CVJvVMl4A+fzIYj4mnANpn5SPn4jcDHJrMtSdKWqxLwH8/M97ZPiIgvbD5tHM8CLo2IsXb+KTO/ObkyJUlbqkrAv6j9SURsS9Ft01Vm3gMcOMm6JElT1O2GH6cCfwRsHxEPU1zJCvAE0PVsGElS/3UbD/4M4IyIOCMzT53GmiRJPVCli+YbEfErm0/sdrqjJKn/qgT8yW2P51Oc6tgCvOGHJM1gVQYbe0v784jYE/h0bRVJknpim0msswrwclNJmuGqjCZ5NsWYMlD8Q1gE3FRnUZKkqavSB98+7sw64MLMvL6meiRJPVKlD/6CiNgO2HdsUr0lSZJ6YcI++Ih4LfCfwDnAZ4D/GO+0SUnSzFKli+avgTdmZgJExL7AhVQYrkCS1D9VzqKZNxbuAJn5H8C8+kqSJPVCpQ9ZI+I84B/L5++m+g0/JEl9UiXg/w9wPHBi+fxair54SdIMVuUsmhHgU+WXJGkrMZkrWSVJWwEDXpIaqkof/JRExFyKD2Xvz8wj6m5PklSoMhbNvhRDBu/VvnxmVh0u+CRgGNhpMgVKkianyhH8V4DPAX8HrN+SjUfEAuDNwJ8BH9ji6iRJk1Yl4Ndl5mcnuf1PAx8Cnj7J9SVJk1Ql4C+PiOOAS4GRsYmZubrbShFxBPCTzGyV49lMaGRkhOHh4SqLSpImUCXg/3f5vf3WfaPACyZY71DgrRGxhOJWfztFxD9m5ns6rTAwMMDgoPcSkaSqWq1Wx3lVLnR6/mQazcxTgVPhqREpP9gt3CVJvVXlLJp5FMMVjA0RfBXwt5n5ZI11SZKmqEoXzWcpRo8cG3/mveW036raSGZeRfGPQZI0TaoE/Msz88C259+KiFvqKkiS1BtVhipYHxEvHHsSES9gC8+HlyRNvypH8CcDV0bEPcAciitaj6m1KknSlFU5i+bfImIfIDZOypFu60iS+q9jF01EvL78/msUww0sLL/eXE6TJM1g3Y7gXwN8C3jLOPNGga/WUpEkqSc6Bnxm/nH58GOZeW/7vIiY1MVPkqTpU+UsmkvGmXZxrwuRJPVWxyP4iNgPeBGw82Z97jtRjC0jSZrBuvXBB3AEsAub9sM/Avx2nUVJkqauWx/8ZcBlEXFwZt4wjTVJknqgyoVON0fE8RTdNU91zWTm+2qrSpI0ZVU+ZP0C8EvAEHA1sICim0aSNINVCfiFmXka8PPMvIDioqdX1luWJGmqqgT82LjvD0XEAcDOwDPrK0mS1AtV+uDPjYhdgdOArwM7Ah+ttSpJ0pRVGWzsvPLh1Ux8H1ZJ0gxR5ZZ94x6tZ+bHel+OJKlXqnTR/Lzt8XyKi5+GJ1opIuYD1wADZTsXt41vI0mqWZUumr9ufx4RfwWsqLDtEeD1mfloeePu6yLiG5n5ncmVKknaElWO4De3A8W58F1l5ijwaPl0Xvk1Oon2JEmTUKUP/jY2BvNcYA+gUv97RMwFWhQ3CjknM2/stvzIyAjDwxP2/kiSKqhyBH9E2+N1wI8zc12VjWfmemBRROwCXBoRB2Tm7Z2WHxgYYHBwsMqmJUlAq9XqOK/bLft2i4jdKIYlGPt6DNipnF5ZZj4EXAkcviXrSZImr9sRfIuia2bOOPNGmeCc+IjYA3gyMx+KiO2BNwB/MdlCJUlbpttwwVO9Ld+zgQvKfvhtgC9n5j9PcZuSpIoqnUVTDlWwD5sOF3xNt3Uy81bgpVOqTpI0aVXOovkt4CSKUyNXAq8CbgBeX29pkqSpqDKa5EnAy4H7MvN1FEflD9ValSRpyqoE/OOZ+ThARAxk5p0U92uVJM1gVfrgV5XnsX8NuCIi1gD31VuWJGmqqoxFc2T58PSIuJLihh/frLUqSdKUVfmQ9Szgosz8dmZePQ01SZJ6oEoXTQv4SEQEcClF2H+v3rIkSVM14YesmXlBZi6hOJMmgb+IiP+svTJJ0pRUOYtmzEJgP2Av4M56ypEk9UqVPvhPAkcCdwMXAX9aDh4mSZrBqvTB3w0cnJkP1F2MJKl3qnTR/B1w+NjNtyPieRHxinrLkiRNVZWAPwc4GFhaPn+knCZJmsGqBPwrM/N44HGAzFwDbFdrVZKkKasS8E+WY7qPwlM38thQa1WSpCmrEvBnUVzg9MyI+DPgOuATtVYlSZqyKmPRfDEiWsBhFLfve3tmDtdemSRpSird0akcIniLLm6KiD2BfwCeRdG9c25mLtviCiVJk7IlV7JuqXXAH2Tm/hR3gTo+IvavsT1JUpvaAj4zf5SZN5WPHwGGgefW1Z4kaVOVumimKiL2prjV343dlhsZGWF42O59SeqF2gM+InYELgHen5kPd1t2YGCAwcHBukuSpMZotVod59XZB09EzKMI9y9m5lfrbEuStKnaAj4i5gD/FxjOzE/V1Y4kaXx1dtEcCrwXuC0iVpbT/igzl9fYpiSpVFvAZ+Z1FBdGSZL6oNY+eElS/xjwktRQBrwkNZQBL0kNZcBLUkMZ8JLUUAa8JDWUAS9JDWXAS1JDGfCS1FAGvCQ1lAEvSQ1lwEtSQxnwktRQBrwkNZQBL0kNZcBLUkMZ8JLUULXdsi8iPg8cAfwkMw+oqx1J0vjqPII/Hzi8xu1LkrqoLeAz8xpgdV3blyR1V1sXzWSMjIwwPDzc7zIkqRFmVMAPDAwwODjY7zIkaavRarU6zvMsGklqKANekhqqtoCPiAuBG4qHsSoijq2rLUnSL6qtDz4zl9a1bUnSxOyikaSGMuAlqaEMeElqKANekhrKgJekhjLgJamhDHhJaigDXpIayoCXpIYy4CWpoQx4SWooA16SGsqAl6SGMuAlqaEMeElqqBl1T1apqVasWMFZZ53V1xrWrVvHk08+2dcaAEZHRwGYM2dOX+uYN28e227b/wg88cQTGRoaqmXb/f/pJE2L9evXs2HDhn6X8ZSxoO+X9evXz4iAr1OtP11EHA4sA+YC52Xmn9fZnjRTDQ0N1XaUVtWKFStYvnx5X2sAWLNmDQC77rprX+tYsmRJ3/8mdast4CNiLnAO8AZgFfDdiPh6Zt5RV5uSOpsJ/2Q0ver8kPUVwF2ZeU9mPgFcBLytxvYkSW3q7KJ5LvDDtuergFd2W2FkZITh4eEaS5Kk2WNGfcIwMDDA4OBgv8uQpK1Gq9XqOK/OLpr7gT3bni8op0mSpkGdR/DfBfaJiOdTBPtRwG/W2J4kqU1tR/CZuQ74PWAFMAx8OTO/X1d7kqRN1doHn5nLgf6feCtJs5Bj0UhSQ82os2jWrl37QKvVuq/fdUjSVmSvTjPm9Hs8CElSPeyikaSGMuAlqaEMeElqKANekhrKgJekhjLgJamhDPhxRMSVETG02bT3R8RnOyx/VUQcNMm2jo6Ivykfvz0i9p/Mdjpse1FELGl7/taIOGWK23x/RDweETtPvcJxt//aiDikjm3PdhGxPiJWRsTtEXF5ROzSo+0+tQ/3aHuLImK0vCNcz0XE3hExK8bFMuDHdyHF4Gjtjiqn1+ntwBYFfER0u1htEfBUwGfm13tw28SlFAPJ/doUt9PJawEDvh6PZeaizDwAWA0c3++COlgKXFd+r8PezJKBD2fUlawzyMXAxyNiu8x8IiL2Bp4DLI2ITwHbAxdn5h9vvmJEPJqZO5aPfx04IjOPjog9gM8BzysXfX9mXt+23iHAW4HXRMRHgHeUs84B9gDWAr+dmXdGxPnA48BLgesj4iKKe9/OBx4DjgHuBT4GbB8RrwbOKOs+KDN/r/yZPg88A/gpcExm/le57YeBg4BfAj6UmReXNb4Q2BE4Dvgw8Pfl9B2A84EDgCx/V8dn5vci4o3AnwADwN1lO49GxA+AC4C3APOAd5Y/0+8C6yPiPcAJmXntBH8rTc4NwEsAIuIVbLb/ZGZGxNEU++QOwAuBSzPzQ+U6xwCnAg8BtwAj5fS96bxfPUaxzz4TeB/wv4CDgRsz8+hy/TkU+8IbgGsjYn5mPl7OOw14T7ndHwKtzPyrcr/s9DoZb1/+c2AwIlYCF2TmmT37rc4wHsGPIzNXA/8OvKmcdBTwZeDDmXkQxQvjNRHxki3Y7DLgzMx8OUV4n7dZm98Gvg6cXB5l3Q2cSxFyi4EPAp9pW2UBcEhmfgC4E/jlzHwp8FHgE+VtEj8KfKnc3pc2q+dsip37JcAXgbPa5j0beDVwBMWLYcxRFLdevBaIiHhWOf04YE1m7g+cBiymWOAZwEeAX83MlwHfAz7Qtr0HyumfBT6YmT+g+Cd4Zlmz4V6D8n7Jh1HsbzDO/tO2+CLgXcCLgXdFxJ4R8WyKf9qHUuwn7e86u+1Xu1IE+u+XbZ8JvAh4cUQsKpc5BLi33P+vAt5c1jz2ujmQ4nXZ3iXa7XUy3r58CnBtuY81NtzBI/huxrppLiu/Hwv8RkT8DsXv7dkUO/atFbf3q8D+ETH2fKeI2LHTwuW8Q4CvtK0z0LbIVzJzffl4Z+CCiNgHGKU4Ip7IwWzsZvkC8Mm2eV/LzA3AHW0hDsVb5iMzc0NEXEJxpPU3FC+gZQCZeXtEjP1OXkXxO7q+/Bm2ozhyHPPV8nuL+rp8tNH25VHrcymG8L6inN5t//m3zPwZQETcQTHuyTOAqzLzp+X0LwH7lst3268uz8zRiLgN+HFm3lau/32KbpOVFPvYReXyF1Ec5V9C8c/ksvJo/vGIuLxcd6LXSad9eVYw4Du7DDgzIl5G8RZ1NcXRwcszc0359m/+OOu1D+7TPn8b4FVjbzfHtO2Um9sGeCgzF3WY//O2x38KXJmZR5Zvka/qtNGKRtoezwGIiBcD+wBXtIX1vRQB38kc4IrM7NSXOtbOetwXp8Njmbmo7FJbQdEHfxbd95/2fWGqf6exbW3YbLsbgG3LdxbvAN4WER+m2H92j4ind9nmRK+TX9iXZxO7aDrIzEeBKyn6Ey8EdqII1Z+VRwJv6rDqjyNiMCK2AY5sm/6vwAljT9rekrZ7BHh62f7DwL0R8c5y+TkRcWCHNndm4+0Qjx5ve+P4Nhs/SH43RbdLN0uB0zNz7/LrOcBzImIv4HrgN8o696d4Ow/wHeDQiFhYzntaROw7zrbbdatZPZCZa4ETgT8oP6TvtP90ciNFF+XuETH2+cmYLd2v2h0G3JqZe5b72F4UR+9HUuxjb4mI+eVR+xHlz7Ilr5Mxs2YfM+C7u5Ciz+/CzLwFuJmiv/KfKHa48ZwC/DPFjv6jtuknAgdFxK3lW93fHWfdi4CTI+Lm8oOjdwPHRsQtwPeBt3Vo85PAGRFxM5seYV1J0S20MiLetdk6JwDHlN0p7wVO6rDtMUcBl2427dJy+meAPcqf6+NlrT8r38IfDVxYtnMDsN8E7VwOHFnW/MsTLKtJysybKboXl9J5/+m07o+A0yn+ntdTdPeM2dL9qt1SfnEfuwRYmpnfpei3vxX4BnAb8LNymaqvkzG3UnyQf0tE/P4W1LfVcbhgTVn51npeZj5e/mP6f0CUH/RKPRERO5ZnYO0AXAP8Tmbe1O+6ZjL7PdULOwBXlm/X5wDHGe6qwbllF+B8ijN1DPcJeAQvSQ1lH7wkNZQBL0kNZcBLUkMZ8FIPRcQuEXFcv+uQwICXem0XirF5pL7zNEnNOhHxNWBPitPtlmXmuRFxLPCHtI2OWI66Oe4ooBFxejntBeX3T2fmWRQDWr2wHPPlisw8eTp/NqmdAa/Z6H2ZuToitge+GxH/QjEK5ssoLmP/FkXIw8ZRQK+LiOdRjOEyWM7bD3gdxWXvGcUNYU4BDugyNoo0bQx4zUYnRsTYOEF7UlxSf3U5TDQR8RU2jo7YbRTQf8nMEWAkIn4CzLrRCjWzGfCaVSLitRShfXBmro2IqyjGFxrssEq3UUB7OdKi1HN+yKrZZmeKm5OsjYj9KMasfxrF6Ii7lqMrvqNt+SqjgLabNSMVauYz4DXbfJNi7PFhig9Ev0MxVO4nKO7idT3wAzaOVFhlFNCnZOaDFDc4uT0i/rKeH0GqxrFoJDYZqXBbiiFrP5+Zmw9dK21VPIKXCqeXpzbeTnGnqq/1uR5pyjyCl6SG8ghekhrKgJekhjLgJamhDHhJaigDXpIa6n8ArnGQ+wSGmdYAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NsJrVDT6hpeB"
      },
      "source": [
        "# clear generated data\n",
        "vi_stats.clear_output_dir()\n",
        "baseline_stats.clear_output_dir()"
      ],
      "execution_count": 9,
      "outputs": []
    }
  ]
}