mode_training.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "colab": {
      "name": "mode_training.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "ZEAMTtE4ZG9M"
      ],
      "toc_visible": true,
      "include_colab_link": true
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/gongx030/dream_PGE/blob/main/mode_training.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M0jDe-m_5aq4"
      },
      "source": [
        "# Model training"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "RUNTIME = 'msi'"
      ],
      "metadata": {
        "id": "5VA-6Q6ciFVs"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "obcf6pAcw6TQ"
      },
      "source": [
        "### Install external packages"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%bash -s \"$RUNTIME\"\n",
        "if [[ \"$1\" == \"colab\" ]]; then\n",
        "  pip install -U tensorflow-addons\n",
        "  pip install -U session_info\n",
        "  pip install -U biopython\n",
        "else\n",
        "  pip install tensorflow==2.6 keras==2.6\n",
        "  pip install tensorflow-addons==0.14.0\n",
        "  pip install tensorflow_probability==0.14.1\n",
        "  pip install session_info\n",
        "  pip install biopython\n",
        "  pip install sklearn\n",
        "  pip install matplotlib\n",
        "  pip install tensorflow_io\n",
        "fi"
      ],
      "metadata": {
        "id": "jmnenNTxiL2f",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1b9f551c-a84e-4f53-98ee-e3ed6a9a1ab0"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: tensorflow==2.6 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (2.6.0)\n",
            "Requirement already satisfied: keras==2.6 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (2.6.0)\n",
            "Requirement already satisfied: keras-preprocessing~=1.1.2 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorflow==2.6) (1.1.2)\n",
            "Requirement already satisfied: tensorflow-estimator~=2.6 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorflow==2.6) (2.9.0)\n",
            "Requirement already satisfied: opt-einsum~=3.3.0 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorflow==2.6) (3.3.0)\n",
            "Requirement already satisfied: typing-extensions~=3.7.4 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorflow==2.6) (3.7.4.3)\n",
            "Requirement already satisfied: gast==0.4.0 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorflow==2.6) (0.4.0)\n",
            "Requirement already satisfied: absl-py~=0.10 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorflow==2.6) (0.15.0)\n",
            "Requirement already satisfied: termcolor~=1.1.0 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorflow==2.6) (1.1.0)\n",
            "Requirement already satisfied: numpy~=1.19.2 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorflow==2.6) (1.19.5)\n",
            "Requirement already satisfied: astunparse~=1.6.3 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorflow==2.6) (1.6.3)\n",
            "Requirement already satisfied: wheel~=0.35 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorflow==2.6) (0.37.1)\n",
            "Requirement already satisfied: grpcio<2.0,>=1.37.0 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorflow==2.6) (1.47.0)\n",
            "Requirement already satisfied: protobuf>=3.9.2 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorflow==2.6) (3.19.4)\n",
            "Requirement already satisfied: google-pasta~=0.2 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorflow==2.6) (0.2.0)\n",
            "Requirement already satisfied: wrapt~=1.12.1 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorflow==2.6) (1.12.1)\n",
            "Requirement already satisfied: h5py~=3.1.0 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorflow==2.6) (3.1.0)\n",
            "Requirement already satisfied: tensorboard~=2.6 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorflow==2.6) (2.9.1)\n",
            "Requirement already satisfied: flatbuffers~=1.12.0 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorflow==2.6) (1.12)\n",
            "Requirement already satisfied: six~=1.15.0 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorflow==2.6) (1.15.0)\n",
            "Requirement already satisfied: clang~=5.0 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorflow==2.6) (5.0)\n",
            "Requirement already satisfied: cached-property in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from h5py~=3.1.0->tensorflow==2.6) (1.5.2)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorboard~=2.6->tensorflow==2.6) (0.4.6)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorboard~=2.6->tensorflow==2.6) (0.6.1)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorboard~=2.6->tensorflow==2.6) (1.8.1)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorboard~=2.6->tensorflow==2.6) (63.2.0)\n",
            "Requirement already satisfied: werkzeug>=1.0.1 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorboard~=2.6->tensorflow==2.6) (2.2.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorboard~=2.6->tensorflow==2.6) (3.4.1)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorboard~=2.6->tensorflow==2.6) (2.9.1)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorboard~=2.6->tensorflow==2.6) (2.28.1)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from google-auth<3,>=1.6.3->tensorboard~=2.6->tensorflow==2.6) (5.2.0)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from google-auth<3,>=1.6.3->tensorboard~=2.6->tensorflow==2.6) (0.2.8)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from google-auth<3,>=1.6.3->tensorboard~=2.6->tensorflow==2.6) (4.9)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.6->tensorflow==2.6) (1.3.1)\n",
            "Requirement already satisfied: importlib-metadata>=4.4 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from markdown>=2.6.8->tensorboard~=2.6->tensorflow==2.6) (4.12.0)\n",
            "Requirement already satisfied: charset-normalizer<3,>=2 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard~=2.6->tensorflow==2.6) (2.1.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard~=2.6->tensorflow==2.6) (2022.6.15)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard~=2.6->tensorflow==2.6) (1.26.11)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard~=2.6->tensorflow==2.6) (3.3)\n",
            "Requirement already satisfied: MarkupSafe>=2.1.1 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from werkzeug>=1.0.1->tensorboard~=2.6->tensorflow==2.6) (2.1.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard~=2.6->tensorflow==2.6) (3.8.1)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard~=2.6->tensorflow==2.6) (0.4.8)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.6->tensorflow==2.6) (3.2.0)\n",
            "Requirement already satisfied: tensorflow-addons==0.14.0 in /panfs/jay/groups/18/garrydj/gongx030/.local/lib/python3.7/site-packages (0.14.0)\n",
            "Requirement already satisfied: typeguard>=2.7 in /panfs/jay/groups/18/garrydj/gongx030/.local/lib/python3.7/site-packages (from tensorflow-addons==0.14.0) (2.13.3)\n",
            "Requirement already satisfied: tensorflow_probability==0.14.1 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (0.14.1)\n",
            "Requirement already satisfied: decorator in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorflow_probability==0.14.1) (5.1.1)\n",
            "Requirement already satisfied: cloudpickle>=1.3 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorflow_probability==0.14.1) (2.1.0)\n",
            "Requirement already satisfied: numpy>=1.13.3 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorflow_probability==0.14.1) (1.19.5)\n",
            "Requirement already satisfied: six>=1.10.0 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorflow_probability==0.14.1) (1.15.0)\n",
            "Requirement already satisfied: dm-tree in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorflow_probability==0.14.1) (0.1.7)\n",
            "Requirement already satisfied: gast>=0.3.2 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorflow_probability==0.14.1) (0.4.0)\n",
            "Requirement already satisfied: absl-py in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorflow_probability==0.14.1) (0.15.0)\n",
            "Requirement already satisfied: session_info in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (1.0.0)\n",
            "Requirement already satisfied: stdlib-list in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from session_info) (0.8.0)\n",
            "Requirement already satisfied: biopython in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (1.79)\n",
            "Requirement already satisfied: numpy in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from biopython) (1.19.5)\n",
            "Requirement already satisfied: sklearn in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (0.0)\n",
            "Requirement already satisfied: scikit-learn in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from sklearn) (1.0.2)\n",
            "Requirement already satisfied: joblib>=0.11 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from scikit-learn->sklearn) (1.1.0)\n",
            "Requirement already satisfied: numpy>=1.14.6 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from scikit-learn->sklearn) (1.19.5)\n",
            "Requirement already satisfied: scipy>=1.1.0 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from scikit-learn->sklearn) (1.7.3)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from scikit-learn->sklearn) (3.1.0)\n",
            "Requirement already satisfied: matplotlib in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (3.5.2)\n",
            "Requirement already satisfied: pyparsing>=2.2.1 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from matplotlib) (3.0.9)\n",
            "Requirement already satisfied: numpy>=1.17 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from matplotlib) (1.19.5)\n",
            "Requirement already satisfied: pillow>=6.2.0 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from matplotlib) (9.2.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from matplotlib) (21.3)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from matplotlib) (1.4.4)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from matplotlib) (4.34.4)\n",
            "Requirement already satisfied: python-dateutil>=2.7 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from matplotlib) (2.8.2)\n",
            "Requirement already satisfied: cycler>=0.10 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from matplotlib) (0.11.0)\n",
            "Requirement already satisfied: typing-extensions in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from kiwisolver>=1.0.1->matplotlib) (3.7.4.3)\n",
            "Requirement already satisfied: six>=1.5 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from python-dateutil>=2.7->matplotlib) (1.15.0)\n",
            "Requirement already satisfied: tensorflow_io in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (0.26.0)\n",
            "Requirement already satisfied: tensorflow-io-gcs-filesystem==0.26.0 in /panfs/jay/groups/18/garrydj/gongx030/.conda/envs/tf26_py37_a100/lib/python3.7/site-packages (from tensorflow_io) (0.26.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Load Python packages"
      ],
      "metadata": {
        "id": "wH9oF2RleEYl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "from tensorflow import keras\n",
        "from tensorflow.keras import layers\n",
        "from scipy import stats\n",
        "import session_info\n",
        "import tensorflow_probability as tfp\n",
        "import pdb\n",
        "from sklearn.metrics import r2_score\n",
        "import shutil\n",
        "import pandas as pd\n",
        "import re\n",
        "import os\n",
        "from Bio.Seq import Seq\n",
        "import gc\n",
        "import matplotlib.pyplot as plt\n",
        "import datetime\n",
        "import socket\n",
        "import time\n",
        "import json\n",
        "from collections import OrderedDict\n",
        "import random as python_random"
      ],
      "metadata": {
        "id": "XamZpd5EeGHR"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "if RUNTIME == 'colab':\n",
        "  from google.colab import drive"
      ],
      "metadata": {
        "id": "bHHclmdZm_T3"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j-TPW4ME18Vg"
      },
      "source": [
        "### TensorFlow setup"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "gpus = tf.config.list_physical_devices('GPU')\n",
        "for gpu in gpus:\n",
        "  tf.config.experimental.set_memory_growth(gpu, True)\n",
        "logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n",
        "print('Physical GPUs: {}'.format(len(gpus)))\n",
        "print('Logical GPUs: {}'.format(len(logical_gpus)))"
      ],
      "metadata": {
        "id": "Wh7r7D7SeMEq",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "97cdad41-0c3b-4b0f-a814-5310f06bb455"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Physical GPUs: 4\n",
            "Logical GPUs: 4\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "2022-08-07 14:34:27.263063: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
            "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2022-08-07 14:34:31.875947: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 38427 MB memory:  -> device: 0, name: NVIDIA A100-SXM4-40GB, pci bus id: 0000:06:00.0, compute capability: 8.0\n",
            "2022-08-07 14:34:31.881587: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /job:localhost/replica:0/task:0/device:GPU:1 with 38427 MB memory:  -> device: 1, name: NVIDIA A100-SXM4-40GB, pci bus id: 0000:47:00.0, compute capability: 8.0\n",
            "2022-08-07 14:34:31.883593: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /job:localhost/replica:0/task:0/device:GPU:2 with 38427 MB memory:  -> device: 2, name: NVIDIA A100-SXM4-40GB, pci bus id: 0000:85:00.0, compute capability: 8.0\n",
            "2022-08-07 14:34:31.885714: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /job:localhost/replica:0/task:0/device:GPU:3 with 38427 MB memory:  -> device: 3, name: NVIDIA A100-SXM4-40GB, pci bus id: 0000:c7:00.0, compute capability: 8.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "socket.gethostname()"
      ],
      "metadata": {
        "id": "dYGE9B-1A25u",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "724a6880-9a78-4088-9694-5ec328d6b3b7"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'aga03'"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hl2lXOAv9ljx"
      },
      "source": [
        "## Parameters"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ARGS = {\n",
        "  'model_id' : 'm20220727e',\n",
        "  'global_seed' : 123,\n",
        "  'shuffle_size' : 1000,\n",
        "  'max_width' : 100,\n",
        "  'head_len' : 17,\n",
        "  'tail_len' : 13,\n",
        "  'pct_ds' : 1, # % of total data for training/testing,\n",
        "  'train_split' : 0.95,\n",
        "  'alphabets' : {'A' : 0, 'C' : 1, 'G' : 2, 'T' : 3, 'N' : 4, 'M' : 5},\n",
        "  'learning_rate' : 0.001,\n",
        "  'initial_epoch': 0,\n",
        "  'epochs' : 20,\n",
        "  'batch_size' : 512,\n",
        "  'dropout_rate' : 0.1,\n",
        "  'kmer': 10,\n",
        "  'strides' : 1,\n",
        "  'embedding_dim' : 512,\n",
        "  'num_heads' : 8,\n",
        "  'ff_mult' : 4,\n",
        "  'num_projectors' : 32,\n",
        "  'n_blocks_regressor' : 4,\n",
        "  'warmup_steps' : 12500, # ~ 1 epoch\n",
        "  'mask_ratio' : 0.05,\n",
        "  'remote_sample_submission_file' : 'https://raw.githubusercontent.com/de-Boer-Lab/DREAM-2022/main/sample_submission.json',\n",
        "}\n",
        "if RUNTIME == 'msi':\n",
        "  ARGS['remote_data_dir'] = 'https://s3.msi.umn.edu/gongx030/projects/dream_PGE/training_data/pct_ds=' + str(ARGS['pct_ds']) + '/'\n",
        "  ARGS['local_data_dir'] = re.sub('https://', './', ARGS['remote_data_dir'])\n",
        "  ARGS['remote_checkpoint_dir'] = 'https://s3.msi.umn.edu/gongx030/projects/dream_PGE/notebooks_msi/' + ARGS['model_id'] + '/tf_ckpts/'\n",
        "  ARGS['remote_log_dir'] = 'https://s3.msi.umn.edu/gongx030/projects/dream_PGE/notebooks_msi/' + ARGS['model_id'] + '/log/'\n",
        "  ARGS['local_checkpoint_dir'] = re.sub('https://', './', ARGS['remote_checkpoint_dir'])\n",
        "  ARGS['local_log_dir'] = re.sub('https://', './', ARGS['remote_log_dir'])\n",
        "  ARGS['remote_test_data'] = 'https://s3.msi.umn.edu/gongx030/projects/dream_PGE/test_sequences.txt.gz'\n",
        "  ARGS['local_test_data'] = re.sub('https://', './', ARGS['remote_test_data'])\n",
        "  ARGS['local_sample_submission_file'] = re.sub('https://', './', ARGS['remote_sample_submission_file'])\n",
        "  ARGS['remote_prediction_file'] = 'https://s3.msi.umn.edu/gongx030/projects/dream_PGE/predictions/' + ARGS['model_id'] + '/pred.json'\n",
        "  ARGS['local_prediction_file'] = re.sub('https://', './', ARGS['remote_prediction_file'])\n",
        "  ARGS['s3_prediction_file'] = re.sub('https://s3.msi.umn.edu', 's3://', ARGS['remote_prediction_file'])\n",
        "  ARGS['remote_prediction_tsv_file'] = 'https://s3.msi.umn.edu/gongx030/projects/dream_PGE/predictions/' + ARGS['model_id'] + '/pred.tsv'\n",
        "  ARGS['local_prediction_tsv_file'] = re.sub('https://', './', ARGS['remote_prediction_tsv_file'])\n",
        "  ARGS['s3_prediction_tsv_file'] = re.sub('https://s3.msi.umn.edu', 's3://', ARGS['remote_prediction_tsv_file'])\n",
        "else:\n",
        "  ARGS['local_data_dir'] = '/content/drive/MyDrive/training_data/pct_ds=' + str(ARGS['pct_ds']) + '/'\n",
        "  ARGS['local_checkpoint_dir'] = '/content/drive/MyDrive/' + ARGS['model_id'] + '/tf_ckpts/'\n",
        "  ARGS['local_log_dir'] = '/content/drive/MyDrive/' + ARGS['model_id'] + '/log/'"
      ],
      "metadata": {
        "id": "D0pjVLfcfMl7"
      },
      "execution_count": 68,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Set seeds"
      ],
      "metadata": {
        "id": "5i5XrE1H2Iwq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# The below is necessary for starting Numpy generated random numbers\n",
        "# in a well-defined initial state.\n",
        "np.random.seed(ARGS['global_seed'])\n",
        "\n",
        "# The below is necessary for starting core Python generated random numbers\n",
        "# in a well-defined state.\n",
        "tf.random.set_seed(ARGS['global_seed'])\n",
        "\n",
        "# The below set_seed() will make random number generation\n",
        "# in the TensorFlow backend have a well-defined initial state.\n",
        "# For further details, see:\n",
        "# https://www.tensorflow.org/api_docs/python/tf/random/set_seed\n",
        "python_random.seed(ARGS['global_seed'])"
      ],
      "metadata": {
        "id": "5LTFxNSIWQID"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hkXc5vbaiJGB"
      },
      "source": [
        "## Functions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "5AHxqjy7iRlJ"
      },
      "outputs": [],
      "source": [
        "mse_loss = keras.losses.MeanSquaredError(reduction = 'none')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "n69Cc4Yd76rm"
      },
      "outputs": [],
      "source": [
        "scc_loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits = True, reduction = 'none')"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### pearson_r"
      ],
      "metadata": {
        "id": "oPZLHuW06Tcm"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "https://www.tutorialexample.com/implement-pearson-correlation-coefficient-loss-in-tensorflow-tensorflow-tutorial/"
      ],
      "metadata": {
        "id": "TH7CX-Wk7Y2e"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def pearson_r(x, y):\n",
        "  \n",
        "  mx = tf.reduce_mean(x, axis = 0, keepdims = True)\n",
        "  my = tf.reduce_mean(y, axis = 0, keepdims = True)\n",
        "  xm = x - mx\n",
        "  ym = y - my\n",
        "  t1_norm = tf.nn.l2_normalize(xm, axis = 0)\n",
        "  t2_norm = tf.nn.l2_normalize(ym, axis = 0)\n",
        "  return tf.reduce_sum(tf.multiply(t1_norm, t2_norm))"
      ],
      "metadata": {
        "id": "bDou1j1A6S2j"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x = np.random.rand(100)\n",
        "y = np.random.rand(100)\n",
        "print('pearson r (stats.pearsonr): {}'.format(stats.pearsonr(x, y)[0]))\n",
        "print('pearson r (pearson_r): {}'.format(pearson_r(tf.expand_dims(x, 1), tf.expand_dims(y, 1))))"
      ],
      "metadata": {
        "id": "4E4kvC_v6oYG",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "13b6f761-b005-4e10-f2b8-b2c5703a0826"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "pearson r (stats.pearsonr): -0.09270195576139686\n",
            "pearson r (pearson_r): -0.09270195576139688\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### GLULayer"
      ],
      "metadata": {
        "id": "mrQ9T4-oSk8X"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class GLULayer(layers.Layer):\n",
        "  def __init__(self, dim):\n",
        "    super(GLULayer, self).__init__()\n",
        "    self.dim = dim\n",
        "\n",
        "  def call(self, x):\n",
        "    out, gate = tf.split(x, 2, axis = self.dim)\n",
        "    return out * tf.sigmoid(gate)"
      ],
      "metadata": {
        "id": "oMX5LC1aSkRK"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "layer = GLULayer(dim = 2)\n",
        "x = tf.random.uniform([3, 10, 6])\n",
        "layer(x).shape"
      ],
      "metadata": {
        "id": "ivpHjDlQSxUU",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6f1bd14c-44b6-4ba0-9493-f9ef6b173840"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TensorShape([3, 10, 3])"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### SwiGLU"
      ],
      "metadata": {
        "id": "tHdNKhzVvsMf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class SwiGLULayer(layers.Layer):\n",
        "  def __init__(self, dim):\n",
        "    super(SwiGLULayer, self).__init__()\n",
        "    self.dim = dim\n",
        "    self.swish = layers.Activation('swish')\n",
        "\n",
        "  def call(self, x):\n",
        "    out, gate = tf.split(x, 2, axis = self.dim)\n",
        "    return out * self.swish(gate)"
      ],
      "metadata": {
        "id": "WeSyNETvvuex"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "layer = GLULayer(dim = 2)\n",
        "x = tf.random.uniform([3, 10, 6])\n",
        "layer(x).shape"
      ],
      "metadata": {
        "id": "JuwI-YXjvuex",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6c807edc-6005-4304-cc04-226fb250c663"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TensorShape([3, 10, 3])"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### FeedForwardSwiGLU"
      ],
      "metadata": {
        "id": "i7BVgkw-w4UO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class FeedForwardSwiGLU(layers.Layer):\n",
        "  def __init__(self, embedding_dim, mult=4, rate = 0.0, use_bias = False):\n",
        "    super(FeedForwardSwiGLU, self).__init__()\n",
        "    self.net = tf.keras.Sequential(\n",
        "      [\n",
        "        layers.LayerNormalization(epsilon = 1e-6),\n",
        "        layers.Dense(embedding_dim * mult, use_bias = use_bias),\n",
        "        SwiGLULayer(dim = 2),\n",
        "        layers.Dropout(rate),\n",
        "        layers.Dense(embedding_dim, use_bias = use_bias),\n",
        "        layers.Dropout(rate),\n",
        "      ]\n",
        "    )\n",
        "\n",
        "  def call(self, inputs):\n",
        "    return self.net(inputs)"
      ],
      "metadata": {
        "id": "eVdwwIxbw4UP"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "ffn = FeedForwardSwiGLU(embedding_dim = 5, mult = 4)\n",
        "x = tf.random.uniform([3, 10, 5])\n",
        "print(ffn(x).shape)"
      ],
      "metadata": {
        "id": "bdHfQ6HHw4UP",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8bd98526-f169-4a52-a611-ac95c23d11f6"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(3, 10, 5)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "2022-08-07 14:34:37.384740: I tensorflow/stream_executor/cuda/cuda_blas.cc:1760] TensorFloat-32 will be used for the matrix multiplication. This will only be logged once.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### CustomSchedule"
      ],
      "metadata": {
        "id": "rke3X-N6-n9S"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n",
        "  def __init__(self, d_model, warmup_steps=4000):\n",
        "    super(CustomSchedule, self).__init__()\n",
        "\n",
        "    self.d_model = d_model\n",
        "    self.d_model = tf.cast(self.d_model, tf.float32)\n",
        "\n",
        "    self.warmup_steps = warmup_steps\n",
        "\n",
        "  def __call__(self, step):\n",
        "    arg1 = tf.math.rsqrt(step)\n",
        "    arg2 = step * (self.warmup_steps ** -1.5)\n",
        "\n",
        "    return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)"
      ],
      "metadata": {
        "id": "hNYjuoTd-jsC"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "d_model = 128\n",
        "learning_rate = CustomSchedule(d_model)\n",
        "temp_learning_rate_schedule = CustomSchedule(d_model, warmup_steps=4000)"
      ],
      "metadata": {
        "id": "eCHvjE5N-s3-"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.plot(temp_learning_rate_schedule(tf.range(40000, dtype=tf.float32)))\n",
        "plt.ylabel('Learning Rate')\n",
        "plt.xlabel('Train Step')"
      ],
      "metadata": {
        "id": "HYFkjhvb-2OG",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 297
        },
        "outputId": "259f001e-3a12-416a-dfc1-f1ad4254cc98"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 0, 'Train Step')"
            ]
          },
          "metadata": {},
          "execution_count": 21
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEGCAYAAACtqQjWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAA0FUlEQVR4nO3deXxV9Zn48c+Tfd9DWAKEJSxBKWpEca+4oO2UaYsj6m9qW6vTVttOl7H66/wcf/7qTO2mtdV23JdRgVJbsXWjWreqQFxQQJDkghC23ASIJBBCkuf3x/kGLuEmuUnuzb3Jfd6vV14593vO+Z7n3kCenPP9nueIqmKMMcaEQ0K0AzDGGDN8WFIxxhgTNpZUjDHGhI0lFWOMMWFjScUYY0zYJEU7gGgqKirSsrKyaIdhjDFDyttvv12vqsXB1sV1UikrK6OqqiraYRhjzJAiIh93t84ufxljjAkbSyrGGGPCxpKKMcaYsLGkYowxJmwsqRhjjAmbiCYVEZknIhtEpFpEbgiyPlVEFrv1K0SkLGDdja59g4hcGND+gIjUiciabo75fRFRESmKyJsyxhjTrYglFRFJBO4CLgIqgMtEpKLLZlcBe1R1MnA7cJvbtwJYCMwA5gF3u/4AHnJtwY45FrgA2BLWN2OMMSYkkTxTmQ1Uq6pPVVuBRcD8LtvMBx52y0uBuSIirn2Rqh5U1U1AtesPVX0V2N3NMW8HrgeGZT1/VWXJqq00HWyLdijGGBNUJJPKGGBrwOta1xZ0G1VtAxqBwhD3PYqIzAe2qerqXra7RkSqRKTK7/eH8j5ixntb93L9H97nh0vfj3YoxhgT1LAYqBeRDOB/Azf1tq2q3qOqlapaWVwctMpAzNqyez8Ayz/cFeVIjDEmuEgmlW3A2IDXpa4t6DYikgTkAg0h7htoEjABWC0im93274jIyAHEH3Nq/M0AtLZ1sNUlGGOMiSWRTCqrgHIRmSAiKXgD78u6bLMMuNItLwBeUu/5xsuAhW522ASgHFjZ3YFU9QNVHaGqZapahne57ERV3RnetxRdNf4mRLzlZ9fsiG4wxhgTRMSSihsjuQ54HvgQWKKqa0XkFhH5nNvsfqBQRKqB7wE3uH3XAkuAdcBzwLWq2g4gIk8AbwJTRaRWRK6K1HuINT5/M2dPKWbG6ByeXTOs8qUxZpiIaJViVX0GeKZL200Byy3AJd3seytwa5D2y0I4bllfY411HR3KpvomTptUyMllBfzs+Q3saDzAqNz0aIdmjDGHDYuB+niwvfEALYc6mFicybzjvKGi5+xsxRgTYyypDBE+N0g/qTiLScVZTBuZzdOrt0c5KmOMOZollSGixt8EwMTiTADmzxrDO1v28nFDczTDMsaYo1hSGSJ8/may05IozkoFYP6s0YjAn961sxVjTOywpDJE1PibmFichbg5xaPz0jl1QiF/fLcWbxa2McZEnyWVIcLnb2ZSUeZRbZ8/cQybG/bz7ta90QnKGGO6sKQyBDQdbGPnJy1MGpF1VPtFx40kNSmBP73bU7EBY4wZPJZUhoBNbubXxC5nKtlpyZxfUcLTq7dzsK09GqEZY8xRLKkMAb56b+ZX1zMVgEsqx7Jn/yFeWGtFJo0x0WdJZQioqWsiQWB8YcYx686cXERpfjqPr7Dnkhljos+SyhBQU99MaX4GqUmJx6xLSBAumz2ON30N+Ny9LMYYEy2WVIaAmromJhVndrv+kspSkhKERau2druNMcYMBksqMa6jQ9nc0MzE4mPHUzqNyE7jvOklLH271gbsjTFRZUklxnUWkpzUQ1IBuPyUcexubrUik8aYqLKkEuM6n/Y4sYfLXwBnTC5iQlEmD/x9s91hb4yJGksqMa5z8L23M5WEBOErp5exeute3tmyZzBCM8aYY1hSiXE1/iay05IoykrpddsFJ5WSm57Mfa9tGoTIjDHmWJZUYpzP33xUIcmeZKQkcdnscTy/didbd+8fhOiMMeZollRinM/f3ON04q6uPG08CSI89MbmyAVljDHdiGhSEZF5IrJBRKpF5IYg61NFZLFbv0JEygLW3ejaN4jIhQHtD4hInYis6dLXz0RkvYi8LyJ/FJG8SL63wXC4kGQv4ymBRuWmc/Hxo1i8aiuN+w9FMDpjjDlWxJKKiCQCdwEXARXAZSJS0WWzq4A9qjoZuB24ze1bASwEZgDzgLtdfwAPubaulgPHqepM4CPgxrC+oSjYdPgRwqGfqQB845xJNB1s48E3bGzFGDO4InmmMhuoVlWfqrYCi4D5XbaZDzzslpcCc8UbPJgPLFLVg6q6Cah2/aGqrwK7ux5MVV9Q1Tb38i2gNNxvaLAdeYRw6GcqANNH5XDe9BIe/Ptm9rXY2YoxZvBEMqmMAQLrhtS6tqDbuITQCBSGuG9Pvgo8G2yFiFwjIlUiUuX3+/vQ5eDz+bsvJNmbb8+dTOOBQzz61scRiMwYY4IbdgP1IvIjoA14LNh6Vb1HVStVtbK4uHhwg+ujGn8zYwuCF5LszczSPM6eUsx9r21if2tb7zsYY0wYRDKpbAPGBrwudW1BtxGRJCAXaAhx32OIyJeBzwJX6DC4rbzG33TMg7n64lvnTmZ3cyuPvWVl8Y0xgyOSSWUVUC4iE0QkBW/gfVmXbZYBV7rlBcBLLhksAxa62WETgHJgZU8HE5F5wPXA51R1yN+k0dGhbKpv7tPMr64qywo4Y3IRv32lxsZWjDGDImJJxY2RXAc8D3wILFHVtSJyi4h8zm12P1AoItXA94Ab3L5rgSXAOuA54FpVbQcQkSeAN4GpIlIrIle5vn4DZAPLReQ9EfldpN7bYNi29wAH2zr6PEjf1Q/nTWN3cyv3vuoLU2TGGNO9pEh2rqrPAM90abspYLkFuKSbfW8Fbg3Sflk3208eULAxxlffv+nEXR1fmstnZo7ivtc38c9zyijOTg1HeMYYE9SwG6gfLmrq+jedOJgfXDCV1rYOfv3SxgH3ZYwxPbGkEqN89aEXkuzNhKJMLj15LI+v2MJmdwZkjDGRYEklRnk1v0IrJBmK78wtJzUpgR//5cOw9GeMMcFYUolRNf6mXh/M1RcjctL41txy/vrhLl7eUBe2fo0xJpAllRjUdLCNXZ8cHNB04mC+cnoZE4oyueXpdbS2dYS1b2OMAUsqMenI0x7Dd6YCkJqUyE3/UIGvvpmHrNikMSYCLKnEIN/h59KH90wF4NNTRzB32gh+9deN7GxsCXv/xpj4ZkklBtUMoJBkKG76hwraVfk/T61hGFSzMcbEEEsqMcg3gEKSoRhfmMl3z5vC8nW7eHbNzogcwxgTnyypxKAaf1PYB+m7uuqMCRw3JoebnlprT4g0xoSNJZUY01lIciDViUORlJjAbV+cyZ79rdz6zLqIHssYEz8sqcSYzkKSk0ZE9kwFYMboXK45ayJLqmr5m927YowJA0sqMebwI4QjfKbS6Ttzy5laks31S9+noengoBzTGDN8WVKJMZGcThxMWnIidyycReP+Q9z45Ac2G8wYMyCWVGKMr76JnDAVkgzV9FE5XD9vKi+s28WSqq2DdlxjzPBjSSXG1NQ1MzGMhSRD9dXTJ3DapEL+79PrDt/Rb4wxfWVJJcb46iM/nTiYhAThF//0KVKTEvjmY+9woLV90GMwxgx9llRiyL6WQ+z65GBYqxP3xajcdG6/dBYbdu3j3/9kd9sbY/rOkkoM2RSmRwgPxDlTR/Ctc8v5wzu1LF5l4yvGmL6JaFIRkXkiskFEqkXkhiDrU0VksVu/QkTKAtbd6No3iMiFAe0PiEidiKzp0leBiCwXkY3ue34k31sk1ByuTjz4l78CfWduOWeWF3HTsrWs2dYY1ViMMUNLxJKKiCQCdwEXARXAZSJS0WWzq4A9qjoZuB24ze1bASwEZgDzgLtdfwAPubaubgBeVNVy4EX3ekjx+ZtJEBgXoUKSoUpMEO64dBZFmSlc/UgVdfusmrExJjSRPFOZDVSrqk9VW4FFwPwu28wHHnbLS4G54k17mg8sUtWDqroJqHb9oaqvAruDHC+wr4eBfwzjexkUPn8z4yJYSLIvCrNSuffKSvbuP8Q1j7xNyyEbuDfG9C6SSWUMEHhRvta1Bd1GVduARqAwxH27KlHVHW55J1ASbCMRuUZEqkSkyu/3h/I+Bo33COHoXvoKNGN0LncsnMV7W/dy/dL3beDeGNOrYTlQr95vv6C/AVX1HlWtVNXK4uLiQY6se+2ukGQ0B+mDuXDGSK6fN5Vlq7fz65eqox2OMSbGRTKpbAPGBrwudW1BtxGRJCAXaAhx3652icgo19coYEhVSNzuCknG0plKp2+cPYkvnDiGXy7/iMWrtkQ7HGNMDItkUlkFlIvIBBFJwRt4X9Zlm2XAlW55AfCSO8tYBix0s8MmAOXAyl6OF9jXlcBTYXgPg2awC0n2hYjwky/M5Kwpxdz45Ae8sNYe7GWMCS5iScWNkVwHPA98CCxR1bUicouIfM5tdj9QKCLVwPdwM7ZUdS2wBFgHPAdcq6rtACLyBPAmMFVEakXkKtfXT4DzRWQjcJ57PWR0FpIcjJL3/ZGSlMBvrziR40vz+NYT77JyU7C5EsaYeCfxPPhaWVmpVVVV0Q4DgB/98QOeXr2d1f9xwaDX/eqL3c2tLPjdG/j3HWTxNXOoGJ0T7ZCMMYNMRN5W1cpg64blQP1Q5PM3M2nE4BeS7KuCzBQeveoUslKTuOK+t/hwxyfRDskYE0MsqcSIGn8TE4ti89JXV2Py0nni6lNJTUrkivtWsGHnvmiHZIyJEZZUYsC+lkPU7YteIcn+KCvK5IlrTiU5Ubj83rf4aJclFmOMJZWYcHiQPganE/dkQlEmT1x9KokJXmKxS2HGGEsqMcBX31lIcuicqXSaWJzFE9ecSlJCApf+95u8/bHNCjMmnvWaVERkioi82FkVWERmisi/Rz60+OHzN5OYIFEvJNlfk4qzWPqNORRmpXLFfSt4ecOQuu/UGBNGoZyp3AvcCBwCUNX38W5kNGFS429ibH56TBSS7K/S/AyW/MscJhZlcfUjVTy9enu0QzLGREEoSSVDVbvezd4WiWDilc/fPOTGU4Ipzk5l0b+cyglj8/n2onf571dqrAilMXEmlKRSLyKTcAUaRWQBsKPnXUyo2jsUX33zkJr51ZOctGQeuWo2Fx83iv96dj3/+48fcKi9I9phGWMGSVII21wL3ANME5FtwCbgiohGFUe27z1Aa4wWkuyvtOREfn3ZCZQVZXDX32rYuvsAd11xIrnpydEOzRgTYaGcqaiqngcUA9NU9YwQ9zMhiJVHCIdbQoLwbxdO42cLZrJiUwNf/O0bbKpvjnZYxpgICyU5/AFAVZtVtfMOt6WRCym+1Lh7VIbL5a+uLqkcyyNfPYX6poN87jev89d1u6IdkjEmgrpNKiIyTUS+COSKyBcCvr4MpA1ahMOcz99EbnoyhZkp0Q4lYuZMKuTp686grDCTrz1SxS9e2EB7hw3gGzMc9TSmMhX4LJAH/ENA+z7g6gjGFFe8RwhnxnwhyYEaW5DB778+h5ueWsOvX6pmdW0jv7p0FvnDOJkaE4+6TSqq+hTwlIjMUdU3BzGmuOLzN3Nmeew81jiS0pITue2LM5k1Np+bl63l4jtf445LZ3HKxMJoh2aMCZNQxlTeFZFrReRuEXmg8yvikcWBzkKSk0YMz/GUYESEy08Zx9JvzCE1KYHL7n2LX76wgTabdmzMsBBKUnkUGAlcCLyC97x4K0kbBp2FJIdKyftwmlmax5+/fSZfOLGUO1+q5tJ73mLr7v3RDssYM0ChJJXJqvp/gGZVfRj4DHBKZMOKD52FJCfH0ZlKoKzUJH5+yae487IT+GjnPi7+1Wssqdpqd+EbM4SFklQOue97ReQ4IBcYEbmQ4kdNnSskWRCfSaXT5z41mme+cyYVo3O4fun7fPnBVexoPBDtsIwx/RBKUrlHRPKBfweWAeuA2yIaVZzw1TcxriCDlCS7l3RsQQZPXH0qt8yfwcpNu7ngl6+yZJWdtRgz1PT620xV71PVPar6qqpOVNURwLOhdC4i80Rkg4hUi8gNQdanishit36FiJQFrLvRtW8QkQt761NE5orIOyLynoi8LiKTQ4kxmmrqmplYFN9nKYESEoQvzSnj+X89ixljcrj+D+/zpQdW8nGD3YlvzFDRY1IRkTkiskBERrjXM0XkceDvvXUsIonAXcBFQAVwmYhUdNnsKmCPqk4GbsedAbntFgIzgHnA3SKS2EufvwWuUNVZwON4Z1Yxq71D2dQwfApJhtO4wgwe/9qp/L/5M3h3y14uuP1V7nxxIwfb2qMdmjGmFz3dUf8z4AHgi8BfROTHwAvACqA8hL5nA9Wq6lPVVmARML/LNvOBh93yUmCueHcBzgcWqepBVd0EVLv+eupTgRy3nAvE9AM9OgtJDreaX+GSkCD885wyXvz+2ZxfUcIvl3/EvDte4/WN9dEOzRjTg57uqP8McIKqtrgxla3Acaq6OcS+x7h9OtVy7Kyxw9uoapuINAKFrv2tLvuOccvd9fk14BkROQB8ApwaLCgRuQa4BmDcuHEhvpXwq3aFJIdTdeJIKMlJ4zeXn8g/Vfq56ak1/K/7V/DZmaO48eLpjMlLj3Z4xpguerr81aKqLQCqugfY2IeEEg3fBS5W1VLgQeCXwTZS1XtUtVJVK4uLo3cne+c9KkPxufTRcNaUYp7717P47nlTWL5uF+f+/GV+/vwGmg7a8+KMiSU9nalMFJFlAa8nBL5W1c/10vc2YGzA61LXFmybWhFJwrts1dDLvse0i0gx8ClVXeHaFwPP9RJfVNW4QpIFVvsqZGnJiXznvHIWVJbys+fW85u/VbNo1VZ+cMEULqkcS2LC8K6fZsxQ0FNS6Tr+8Ys+9r0KKBeRCXgJYSFweZdtlgFXAm8CC4CXVFVd8npcRH4JjMYbw1kJSDd97sGrpjxFVT8Czgc+7GO8g8oXJ4UkI2FMXjp3LDyBL58+gR//eR03PPkBD72xmRsumsbZU4rtMzUminoqKPnKQDp2YyTXAc8DicADqrpWRG4BqlR1GXA/8KiIVAO78ZIEbrslePfEtAHXqmo7QLA+XfvVwB9EpAMvyXx1IPFHWo2/mbOnxEchyUiZNTaP3399Ds+u2cl/PfshX35wFSeX5fODC6ZakUpjokTi+eayyspKraqqGvTj7ms5xPE3v8D186byzXNi/naaIaG1rYPFVVv59Ysbqdt3kDPLi/jBBVP51Ni8aIdmzLAjIm+ramWwdXYrdxQcGaS3mV/hkpKUwD+fOp5Xr/80P7p4Omu2NTL/rr9z9SNVvF+7N9rhGRM3ehpTMRFy5Ln0NvMr3NKSE7n6rIlcdso4Hnh9E/e+5mP5ul2cWV7EtZ+ezCkTCmzMxZgI6jWpiMjTeDcWBmoEqoD/7px2bELn81shyUjLSk3i23PL+crpZfzPW1u4/3UfC+95i5PG53Ptpyfx6akjLLkYEwGhXP7yAU3Ave7rE7znqUxxr00f1fitkORgyU5L5hvnTOL1H57LLfNnsLOxha8+VMVFv3qNP75bS2ubPRzMmHAK5fLXaap6csDrp0VklaqeLCJrIxXYcObzWyHJwZaWnMiX5pRx2exxPPXedn77cjXfXbya/3xmPV86dTyXnzKOwqzUaIdpzJAXyp/KWSJyuJ6JW+4cYW6NSFTDWGchyUkjbJA+GpITE1hwUinLv3s2D33lZKaPyuEXyz9izk9e4odL32f9zk+iHaIxQ1ooZyrfB14XkRq8mw8nAN8UkUyOFIM0Idq2xyskaWcq0ZWQIJwzdQTnTB3Bxl37ePCNzTz5Ti2Lq7Zy2qRC/tep4zm/ooTkRLtEaUxf9JpUVPUZESkHprmmDQGD83dEKrDhqsY9QtjOVGJHeUk2//n54/m3C6byxKot/M+bH/PNx96hKCuVf6os5bLZ4xhbkBHtMI0ZEkKdUnwSUOa2/5SIoKqPRCyqYaymzlUntjOVmJOfmcI3z5nMv5w1iVc+quPxFVv43Ss1/PaVGs4sL+by2eOYO32Enb0Y04NQphQ/CkwC3gM6n5KkgCWVfvDVN5OXYYUkY1lignDutBLOnVbC9r0HWLxqK4tXbeXr//M2xdmp/OOs0XzhxFKmj8rpvTNj4kwoZyqVQIXGcz2XMKqpa2JikRWSHCpG56Xz3fOn8K1zJ/O3DX5+X7WVh97YzL2vbaJiVA5fOHEM82eNoTjbZo4ZA6EllTXASGBHhGOJC756KyQ5FCUlJnB+RQnnV5Swu7mVp1dv58l3avnxXz7kv55dz9lTivnCiWM4b3oJacmJ0Q7XmKgJJakUAetEZCVwsLMxhOepmC4+aTmEf99Bq/k1xBVkpnDlaWVceVoZG3ft48l3t/HHd7bx0vo6MlMSOa+ihM8cP4qzpxaTmmQJxsSXUJLKzZEOIl50FpKcaDW/ho3ykmx+OG8aP7hgKm/5Gvjz+9t5ds1OnnpvO9mpSZw/o4TPzhzFGZOLrYKCiQuhTCke0HNVzBG+w4Uk7UxluElMEE6fXMTpk4u4Zf5xvFHTwJ9Xb+f5tTt58p1t5KQlceGMkVx0/EhOm1Rkl8jMsNVtUhGR11X1DBHZx9EFJQVQVbWpL31U429yhSTtnofhLDkxgbOnFHP2lGJu/fzxvF7t58+rd/Dsmp38/u1aMlISOXtKMedXlHDutBHkZdhMQDN89PTkxzPc9+zBC2d48/mbrZBknElJSjg8PflgWztv1jTwwrpd/HXdLp5ds5PEBGF2WcHhSQB2k6UZ6kJ68qOIJAIlBCQhVd0SwbgGxWA/+fGC219hXEEG9115cu8bm2Gto0N5f1sjy9ft5IW1u9joboqdNjLblY8p5qTx+XajpYlJPT35MZSbH78F/AewC+isE67AzLBFGAfaO5TNDfs5Z+qIaIdiYkBCgjBrbB6zxubxbxdOY3N9M8vX7eKvH+7ivtd8/O6VGrJSkzh9ciHnTB3B2VOKGZ2XHu2wjelVKLO/vgNMVdWGvnYuIvOAXwGJwH2q+pMu61Px7sw/CWgALlXVzW7djcBVeHfxf1tVn++pT/HuJvwxcInb57eqemdfY46UzkKS9rRHE0xZUSZXnzWRq8+ayL6WQ/y9uoFXPvLzyoY6nl+7C4ApJVmcM3UEZ5UXU1mWb4P9JiaFklS24j3psU/cJbO7gPOBWmCViCxT1XUBm10F7FHVySKyELgNuFREKoCFwAxgNPBXEZni9umuzy8DY4FpqtohIjF1StD5COGJNvPL9CI7LZl5x41k3nEjUVU21jXx8oY6XvnIz4N/38Q9r/pISUqgcnw+p00q5LTJRcwck0uSXSozMSCUpOIDXhaRv3D0zY+/7GW/2UC1qvoARGQRMB8ITCrzOXIfzFLgN+6MYz6wSFUPAptEpNr1Rw99fgO4XFU7XHx1Iby3QVNj04lNP4gIU0qymVKSzTVnTaL5YBtv+Rp4o8b7+vkLH8ELH5GVmsQpEwqYM6mQ0ycXMbUkm4QEKwVkBl8oSWWL+0pxX6Eag3eW06kWOKW7bVS1TUQagULX/laXfce45e76nIR3lvN5wI93yWxj16BE5BrgGoBx48Z1XR0xNX4rJGkGLjM1ibnTS5g7vQSA3c2tvFnTwBs19bxR08CL672/pQozUzhlYgEnl3lf00flkGhJxgyCHpOKu4Q1RVWvGKR4BiIVaFHVShH5AvAAcGbXjVT1HuAe8GZ/DVZwPn+Tlbs3YVeQmcJnZo7iMzNHAbB97wHerGng7zX1rPDt5pkPdgKQlZrEiePzmV2Wz8llBXxqbJ6NyZiI6DGpqGq7iIwXkRRV7eujg7fhjXF0KnVtwbapFZEkIBdvwL6nfbtrrwWedMt/BB7sY7wR5atv5hwrJGkibHReOl88qZQvnlQKeElm1ebd3temPd7lMiAlMYGZpblUlhUwe0I+s8bm21m0CYtQx1T+LiLLgObOxhDGVFYB5SIyAe8X/0Lg8i7bLAOuBN4EFgAvqaq6Yz0uIr/EG6gvB1bi3c3fXZ9/Aj4NbALOBj4K4b0Nis5CkjZIbwbb6Lx05s/yyvMD7N3fStXmPazavJuVm3e76cveCXtZYQazxuZxwrh8Zo3NY/qoHLtR1/RZKEmlxn0lACHfXe/GSK4Dnseb/vuAqq4VkVuAKlVdBtwPPOoG4nfjJQncdkvwBuDbgGtVtR0gWJ/ukD8BHhOR7wJNwNdCjTXSOgtJ2nRiE215GSmcV1HCeRXemMyB1nZW1+7lva17eW/LXt6oaeBP720HvGoAx4/JdYnGu6dmTF66PQvI9CikO+qHq8G6o/4Pb9fy/d+v5q/fO5vJ9mx6E8NUlR2NLby3dS/vbtnDu1v28sG2Rg62efc9F2enMnNMLjPG5HK8+yrJSbVEE2cGekd9MXA93j0jaZ3tqnpu2CIc5nz1VkjSDA0iwui8dEbnpXPx8d7g/6H2Dtbv2Me7W/fwnksyf9tQR4f7e7QoK4XjxuRy3OhcjhuTy/GluYzOTbNEE6dCufz1GLAY+CzwdbwxEH8kgxpuauqaGW+FJM0QlZyYwPGlXrL40hyvbX9rGx/u+IQPahtZs/0T1mxr5LWN9bS7TFOQmcKM0TmHk830UdmML8y0ac1xIJSkUqiq94vId9yzVV4RkVWRDmw48dU32YO5zLCSkZLESeMLOGl8weG2lkPtfLjDSzAfbGtkzbZPuPdVH20u0aQlJzC1JJtpI3OYNsp9H5lNvs06G1ZCSSqH3PcdIvIZYDtQ0MP2JkB7h7K5fj+ftkKSZphLS07khHH5nDAu/3Bby6F2Nu5qYv3OT1i/cx/rd37C8g93sbjqyD3MI3PSDieZ6e77xOJMq9A8RIWSVH4sIrnA94FfAznAdyMa1TBSu2c/re0ddqZi4lJacuLhS2edVBV/00HW7/CSzPod+/hw5z7+Xu3jULt3VpOcKEwoyqR8RDaTR2RRXpJF+YhsyooySE2ymzZjWSiPE/6zW2zEuw/E9MGR6cQ268sY8CYDjMhOY0R2GmcF3BB8qL0Dn7+Z9Ts/4cMd+6iua2Lt9kaeWbODzkmqiQnC+IKMoxLN5BFZTCrOIj3Fkk0sCGX21xTgt0CJqh4nIjOBz6nqjyMe3TBg1YmNCU1yYgJTR2YzdWQ282cdaW851I7P38zGOi/RbNzVxMa6fby4vu7wxAARKM1PZ3JxFhOKsphYnMnEokwmFGcyMsdmog2mUC5/3Qv8G/DfAKr6vog8jvfsEtMLKyRpzMCkJSdSMTqHitE5R7W3tnXwcUMzG12i+ahuHz5/M2/6Gmg51HF4u/TkRCa4BDOxKJOJxZlMKMpiQlEmuenJg/12hr1QkkqGqq7skunbIhTPsOPzN9mlL2MiICUpgfKSbMpLsuH4I+0dHcrOT1rYVN+Mr76ZTf5mNtU3sWZbI89+sOPw/TXgVXP2kkwm4wszGV+YwbiCDMYXZJKbYQmnP0JJKvUiMgnvEcKIyAJgR0SjGkZq/M18eqoVkjRmsCQkHLmB8/TJRUeta23rYMvu/Wyq9xKNz+8lnpfW+6lvqj1q29z05MNJZlxBhlv2Es/InDR7Xk03Qkkq1+KVip8mItvwCjYOhVL4Udd44BD1TQeZZKVZjIkJKUkJTB6R5collRy1rvlgG1t27+fjhv1s3b2fj3c383HDfj7Y1shza3Yevt8GvCrPpQXpjC/IYHxhJmNd4hmTl05pQTo5afF7lhPK7C8fcJ6IZAIJqrpPRP4VuCPCsQ15vs5BenuOijExLzM1iemjcpg+KueYdW3tHexobOHjBi/ZbGnwks+W3ftZtXkPTQePHhHITkuiNN8lmfwjX2PyMijNTycvI3nYTh4I5UwFAFVtDnj5PSyp9KpzOrHN/DJmaEtKTGBsQQZjCzI4g6Mvqakqu5tbqd1zgNo9B9i2d7/3fc8Btu7ez1u+hmOSTkZKoksy6V7yOZx00hmTn05RZuqQvbwWclLpYmi+20FW428iKUEYX2iFJI0ZrkSEwqxUCrNS+dTYvGPWqyqNBw4FJJ0D1O7Zzza3/M6WvTQeOHTUPsmJQklOGqNy0xiVm+6+pzEqL/1wW2FmSkwmnv4mlfitl98HPn8z4woyrNyEMXFMRMjLSCEvw6vmHMy+lkNestl9gB2NB9je2MLOxha27z3A6tq9PLe2hda2jqP2SUlMoCQ3lVE56YzKS2Nkbhqjc48knVF5aRRkDH7i6TapiMg+gicPAdIjFtEw4hWStEtfxpieZaclM21kMtNGHjueA0cuse1obHFfB9i+t4WdLgG9u2UvOxtbaG0/OvEkJ3rVC0bmplGSk0pJThojc9IoyUnjtEmFjMhJC3q8geg2qahqyE95NMeyQpLGmHAJvMTW3dlOR4eye38rO/Z6SWdHYws7P2lhl/u+fuc+Xtngp7m1HYBHvjp7cJOKGZjOQpJ246MxZjAkJAhFWakUZaUeVcCzq6aDbexsbGFUbvgTClhSiZgjNb9sOrExJnZkpSZF9LHmER1BFpF5IrJBRKpF5IYg61NFZLFbv0JEygLW3ejaN4jIhX3o804RaYrYmwqRTSc2xsSjiCUVEUkE7gIuAiqAy0SkostmVwF7VHUycDtwm9u3AlgIzADmAXeLSGJvfYpIJZBPDKjxN5NvhSSNMXEmkmcqs4FqVfWpaiuwCJjfZZv5wMNueSkwV7zbTOcDi1T1oKpuAqpdf9326RLOz4DrI/ieQlbjt5lfxpj4E8mkMgbYGvC61rUF3UZV2/AeBFbYw7499XkdsExVeyx2KSLXiEiViFT5/f4+vaG+8PmbmWTjKcaYODMs7soTkdHAJXiPO+6Rqt6jqpWqWllcHJnqwZ2FJO1MxRgTbyKZVLYBYwNel7q2oNuISBKQCzT0sG937ScAk4FqEdkMZIhIdbjeSF9ZIUljTLyKZFJZBZSLyAQRScEbeF/WZZtlwJVueQHwkqqqa1/oZodNAMqBld31qap/UdWRqlqmqmXAfjf4HxU1nc+lt5L3xpg4E7H7VFS1TUSuA54HEoEHVHWtiNwCVKnqMuB+4FF3VrEbL0ngtlsCrMN7yuS1qtoOEKzPSL2H/vK5QpLjCqyQpDEmvkT05kdVfQZ4pkvbTQHLLXhjIcH2vRW4NZQ+g2wT1VMEn7+ZcYVWSNIYE3/st14E1PibmFhkl76MMfHHkkqYtbV38HHDfiaNsEF6Y0z8saQSZrV7DniFJO1MxRgThyyphJmv3gpJGmPilyWVMOssJGkl740x8ciSSpjV+JvIz0gm3wpJGmPikCWVMKvxN9tZijEmbllSCTOfv8nGU4wxccuSShg17j9EfVOrFZI0xsQtSyphVONmftnlL2NMvLKkEkZHHiFsl7+MMfHJkkoYWSFJY0y8s6QSRjX+JiskaYyJa/bbL4x8Np3YGBPnLKmESVt7B5sbmm08xRgT1yyphEntngMcalcrJGmMiWuWVMKks5Cklbw3xsQzSyphUlPnphPbmYoxJo5ZUgkTX30TBZkpVkjSGBPXIppURGSeiGwQkWoRuSHI+lQRWezWrxCRsoB1N7r2DSJyYW99ishjrn2NiDwgIsmRfG9d1dQ1M7HILn0ZY+JbxJKKiCQCdwEXARXAZSJS0WWzq4A9qjoZuB24ze1bASwEZgDzgLtFJLGXPh8DpgHHA+nA1yL13oLx1VshSWOMieSZymygWlV9qtoKLALmd9lmPvCwW14KzBURce2LVPWgqm4Cql1/3fapqs+oA6wESiP43o7SWUjS7lExxsS7SCaVMcDWgNe1ri3oNqraBjQChT3s22uf7rLXPwPPDfgdhKjm8COELakYY+LbcByovxt4VVVfC7ZSRK4RkSoRqfL7/WE54JFHCNvlL2NMfItkUtkGjA14Xeragm4jIklALtDQw7499iki/wEUA9/rLihVvUdVK1W1sri4uI9vKbgaV0hyrBWSNMbEuUgmlVVAuYhMEJEUvIH3ZV22WQZc6ZYXAC+5MZFlwEI3O2wCUI43TtJtnyLyNeBC4DJV7Yjg+zqGz9/EeCskaYwxJEWqY1VtE5HrgOeBROABVV0rIrcAVaq6DLgfeFREqoHdeEkCt90SYB3QBlyrqu0Awfp0h/wd8DHwpjfWz5Oqekuk3l+gGn+zjacYYwwRTCrgzcgCnunSdlPAcgtwSTf73grcGkqfrj2i76U7be0dfNzQzNzpI6JxeGOMiSl2vWaADheStDMVY4yxpDJQNf7O59LbzC9jjLGkMkCHn0tvhSSNMcaSykDV+K2QpDHGdLKkMkA+vxWSNMaYTpZUBqjG32SD9MYY41hSGYDG/YdoaG616sTGGONYUhmAzkKSdqZijDEeSyoDUFPXWZ3YzlSMMQYsqQyIr76Z5EQrJGmMMZ0sqQxATV0T4wqskKQxxnSy34YD4Ku3QpLGGBPIkko/dRaStEF6Y4w5wpJKP211hSRtkN4YY46wpNJPPr9NJzbGmK4sqfSTVSc2xphjWVLpJ5+/mcLMFPIyrJCkMcZ0sqTSTzX+JhtPMcaYLiyp9JNXndjGU4wxJpAllX7Yu7+VhuZWJo2wMxVjjAkU0aQiIvNEZIOIVIvIDUHWp4rIYrd+hYiUBay70bVvEJELe+tTRCa4PqpdnxEb7Kixpz0aY0xQEUsqIpII3AVcBFQAl4lIRZfNrgL2qOpk4HbgNrdvBbAQmAHMA+4WkcRe+rwNuN31tcf1HRGHpxOPsKRijDGBInmmMhuoVlWfqrYCi4D5XbaZDzzslpcCc0VEXPsiVT2oqpuAatdf0D7dPue6PnB9/mOk3liN3xWSzE+P1CGMMWZIimRSGQNsDXhd69qCbqOqbUAjUNjDvt21FwJ7XR/dHQsAEblGRKpEpMrv9/fjbUFZYQafP2EMSVZI0hhjjhJ3vxVV9R5VrVTVyuLi4n71sXD2OH664FNhjswYY4a+SCaVbcDYgNelri3oNiKSBOQCDT3s2117A5Dn+ujuWMYYYyIskkllFVDuZmWl4A28L+uyzTLgSre8AHhJVdW1L3SzwyYA5cDK7vp0+/zN9YHr86kIvjdjjDFBJPW+Sf+oapuIXAc8DyQCD6jqWhG5BahS1WXA/cCjIlIN7MZLErjtlgDrgDbgWlVtBwjWpzvkD4FFIvJj4F3XtzHGmEEk3h/58amyslKrqqqiHYYxxgwpIvK2qlYGWxd3A/XGGGMix5KKMcaYsLGkYowxJmwsqRhjjAmbuB6oFxE/8HE/dy8C6sMYTrhYXH1jcfWNxdU3sRoXDCy28aoa9O7xuE4qAyEiVd3Nfogmi6tvLK6+sbj6JlbjgsjFZpe/jDHGhI0lFWOMMWFjSaX/7ol2AN2wuPrG4uobi6tvYjUuiFBsNqZijDEmbOxMxRhjTNhYUjHGGBM2llT6QUTmicgGEakWkRsG4XibReQDEXlPRKpcW4GILBeRje57vmsXEbnTxfa+iJwY0M+VbvuNInJld8frJZYHRKRORNYEtIUtFhE5yb3XarevDCCum0Vkm/vc3hORiwPW3eiOsUFELgxoD/qzdY9bWOHaF7tHL/QW01gR+ZuIrBORtSLynVj4vHqIK6qfl9svTURWishqF9v/7ak/8R6Psdi1rxCRsv7G3M+4HhKRTQGf2SzXPpj/9hNF5F0R+XMsfFaoqn314Quv5H4NMBFIAVYDFRE+5magqEvbT4Eb3PINwG1u+WLgWUCAU4EVrr0A8Lnv+W45vx+xnAWcCKyJRCx4z8051e3zLHDRAOK6GfhBkG0r3M8tFZjgfp6JPf1sgSXAQrf8O+AbIcQ0CjjRLWcDH7ljR/Xz6iGuqH5eblsBstxyMrDCvb+g/QHfBH7nlhcCi/sbcz/jeghYEGT7wfy3/z3gceDPPX32g/VZ2ZlK380GqlXVp6qtwCJgfhTimA887JYfBv4xoP0R9byF90TMUcCFwHJV3a2qe4DlwLy+HlRVX8V79k3YY3HrclT1LfX+tT8S0Fd/4urOfGCRqh5U1U1ANd7PNejP1v3FeC6wNMh77CmmHar6jlveB3wIjCHKn1cPcXVnUD4vF4+qapN7mey+tIf+Aj/LpcBcd/w+xTyAuLozKD9LESkFPgPc51739NkPymdlSaXvxgBbA17X0vN/yHBQ4AUReVtErnFtJaq6wy3vBEp6iS+ScYcrljFuOZwxXucuPzwg7jJTP+IqBPaqalt/43KXGk7A+ws3Zj6vLnFBDHxe7nLOe0Ad3i/dmh76OxyDW9/ojh/2/wdd41LVzs/sVveZ3S4iqV3jCvH4/f1Z3gFcD3S41z199oPyWVlSGRrOUNUTgYuAa0XkrMCV7i+bmJgbHkuxAL8FJgGzgB3AL6IRhIhkAX8A/lVVPwlcF83PK0hcMfF5qWq7qs4CSvH+Wp4WjTi66hqXiBwH3IgX38l4l7R+OFjxiMhngTpVfXuwjhkKSyp9tw0YG/C61LVFjKpuc9/rgD/i/Ufb5U6Zcd/reokvknGHK5ZtbjksMarqLveLoAO4F+9z609cDXiXL5K6tPdKRJLxfnE/pqpPuuaof17B4oqFzyuQqu4F/gbM6aG/wzG49bnu+BH7fxAQ1zx3KVFV9SDwIP3/zPrzszwd+JyIbMa7NHUu8Cui/Vn1NuhiX8cMiiXhDa5N4Mjg1YwIHi8TyA5YfgNvLORnHD3Y+1O3/BmOHiBc6doLgE14g4P5brmgnzGVcfSAeNhi4djByosHENeogOXv4l03BpjB0QOTPrxByW5/tsDvOXrw85shxCN418bv6NIe1c+rh7ii+nm5bYuBPLecDrwGfLa7/oBrOXrweUl/Y+5nXKMCPtM7gJ9E6d/+ORwZqI/uZ9WfXyrx/oU3s+MjvGu9P4rwsSa6H+ZqYG3n8fCuhb4IbAT+GvAPU4C7XGwfAJUBfX0VbxCuGvhKP+N5Au/SyCG8a6xXhTMWoBJY4/b5Da7qQz/jetQd931gGUf/0vyRO8YGAmbZdPezdT+HlS7e3wOpIcR0Bt6lrfeB99zXxdH+vHqIK6qfl9tvJvCui2ENcFNP/QFp7nW1Wz+xvzH3M66X3Ge2BvgfjswQG7R/+27fcziSVKL6WVmZFmOMMWFjYyrGGGPCxpKKMcaYsLGkYowxJmwsqRhjjAkbSyrGGGPCxpKKMX0kIoUBVWl3ytGVfXusxisilSJyZx+P91VXvfZ9EVkjIvNd+5dFZPRA3osx4WZTio0ZABG5GWhS1Z8HtCXpkdpLA+2/FHgFr6pwoyutUqyqm0TkZbyqwlXhOJYx4WBnKsaEgXuuxu9EZAXwUxGZLSJvuudcvCEiU9125wQ89+JmV7jxZRHxici3g3Q9AtgHNAGoapNLKAvwbpZ7zJ0hpbvncbziCo8+H1AK5mUR+ZXbbo2IzA5yHGPCwpKKMeFTCpymqt8D1gNnquoJwE3Af3azzzS8cuizgf9wNbkCrQZ2AZtE5EER+QcAVV0KVAFXqFfksA34Nd6zPU4CHgBuDegnw233TbfOmIhI6n0TY0yIfq+q7W45F3hYRMrxSqJ0TRad/qJeMcKDIlKHVwb/cAl0VW0XkXl4VXDnAreLyEmqenOXfqYCxwHLvUdkkIhXtqbTE66/V0UkR0Ty1CuMaExYWVIxJnyaA5b/H/A3Vf28e2bJy93sczBguZ0g/yfVG/hcCawUkeV41XBv7rKZAGtVdU43x+k6eGqDqSYi7PKXMZGRy5Ey4V/ubyciMloCnm+O96yTj93yPrzHAYNXCLBYROa4/ZJFZEbAfpe69jOARlVt7G9MxvTEzlSMiYyf4l3++nfgLwPoJxn4uZs63AL4ga+7dQ8BvxORA3jPHFkA3CkiuXj/t+/Aq2wN0CIi77r+vjqAeIzpkU0pNmaYs6nHZjDZ5S9jjDFhY2cqxhhjwsbOVIwxxoSNJRVjjDFhY0nFGGNM2FhSMcYYEzaWVIwxxoTN/wf8cK+ZiXorMQAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### ConformerSASwiGLULayer"
      ],
      "metadata": {
        "id": "wGqeHeQRxxLY"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "saxlznhLxxLY"
      },
      "outputs": [],
      "source": [
        "class ConformerSASwiGLULayer(layers.Layer):\n",
        "  def __init__(self, embedding_dim,  ff_mult = 4, kernel_size = 15, rate = 0.2, num_heads = 4, use_bias = False):\n",
        "    super(ConformerSASwiGLULayer, self).__init__()\n",
        "    self.ff1 = FeedForwardSwiGLU(embedding_dim = embedding_dim, mult = ff_mult, rate = rate, use_bias = use_bias)\n",
        "    self.conv = tf.keras.Sequential([\n",
        "      layers.LayerNormalization(epsilon = 1e-6),            \n",
        "      layers.SeparableConv1D(filters = embedding_dim, kernel_size = kernel_size, padding = 'same', activation = 'relu'),\n",
        "      layers.Dropout(rate),\n",
        "    ])\n",
        "    self.layernorm = layers.LayerNormalization(epsilon = 1e-6)        \n",
        "    self.attn = layers.MultiHeadAttention(num_heads = num_heads, key_dim = embedding_dim)\n",
        "    self.ff2 = FeedForwardSwiGLU(embedding_dim = embedding_dim, mult = ff_mult, rate = rate, use_bias = use_bias)\n",
        "\n",
        "  def call(self, x):\n",
        "    x = tf.cast(x, dtype = tf.dtypes.float32)\n",
        "    x = x + 0.5 * self.ff1(x)\n",
        "    x = x + self.conv(x)\n",
        "    x = self.layernorm(x)\n",
        "    x = x + self.attn(x, x)\n",
        "    x = x + 0.5 * self.ff2(x)\n",
        "    return x"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "layer = ConformerSASwiGLULayer(embedding_dim = 14)\n",
        "x = tf.random.uniform([3, 10, 14])\n",
        "print(layer(x).shape)"
      ],
      "metadata": {
        "id": "I5mMf8GPxxLZ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "52b3a289-ae04-4e8b-a6ee-e694465bab0b"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(3, 10, 14)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### SequenceMaskLayer"
      ],
      "metadata": {
        "id": "CKbk2vhOTgA5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class SequenceMaskLayer(layers.Layer):\n",
        "  def __init__(self, n_positions, ratio = 0.2):\n",
        "    super(SequenceMaskLayer, self).__init__()\n",
        "    self.ratio = ratio\n",
        "    self.n_positions = n_positions\n",
        "    self.N = 4\n",
        "    self.M = 5\n",
        "\n",
        "  def call(self, x):\n",
        "\n",
        "    if self.ratio > 0:\n",
        "      m = tf.random.uniform(tf.shape(x)) < self.ratio\n",
        "      m = tf.cast(m, dtype = tf.uint8)\n",
        "      is_valid = tf.cast(tf.math.not_equal(x, self.N), tf.uint8)\n",
        "      m = m * is_valid\n",
        "      x0 = tf.ones(tf.shape(x), dtype = tf.uint8) * self.M\n",
        "\n",
        "      x = m * x0 + (1 - m) * x\n",
        "      m = tf.cast(m, dtype = tf.float32)\n",
        "    else:\n",
        "      m = tf.zeros(tf.shape(x))\n",
        "    \n",
        "    return x, m"
      ],
      "metadata": {
        "id": "UPGXsnfmTgA6"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "layer = SequenceMaskLayer(n_positions = 20, ratio = 0.2)\n",
        "x = tf.random.uniform([3, 20], maxval = 5)\n",
        "x = tf.cast(x, tf.uint8)\n",
        "print(x)\n",
        "x, m = layer(x)\n",
        "print(x)\n",
        "print(m)"
      ],
      "metadata": {
        "id": "TOj2tUQmTgA6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ba16bad2-43d7-4907-a1aa-bcc0bc174687"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tf.Tensor(\n",
            "[[4 2 4 2 2 3 2 0 1 4 3 0 0 2 1 3 0 0 4 0]\n",
            " [2 3 2 0 0 1 2 2 1 1 0 0 3 2 4 0 1 3 3 1]\n",
            " [1 1 0 4 3 4 0 4 2 2 2 0 0 0 4 1 4 4 3 0]], shape=(3, 20), dtype=uint8)\n",
            "tf.Tensor(\n",
            "[[4 2 4 2 2 3 2 5 1 4 3 0 0 5 1 3 0 0 4 0]\n",
            " [2 5 5 0 0 1 5 2 1 1 0 0 3 5 4 0 5 3 3 1]\n",
            " [1 5 0 4 3 4 5 4 2 2 2 5 0 0 4 5 4 4 5 0]], shape=(3, 20), dtype=uint8)\n",
            "tf.Tensor(\n",
            "[[0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]\n",
            " [0. 1. 1. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 1. 0. 0. 1. 0. 0. 0.]\n",
            " [0. 1. 0. 0. 0. 0. 1. 0. 0. 0. 0. 1. 0. 0. 0. 1. 0. 0. 1. 0.]], shape=(3, 20), dtype=float32)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jGFLv3peCCKS"
      },
      "source": [
        "## Read data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "id": "om_YoxPICCKS"
      },
      "outputs": [],
      "source": [
        "input_dim = int(6) # A,C,G,T,N,M\n",
        "#input_dim = int(5) # A,C,G,T,N\n",
        "n_positions = ARGS['max_width'] * 2"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I9CPXMei2W9E"
      },
      "source": [
        "The tfdataset was prepraed by this notebook: https://github.com/ikwak2/dream_PGE/blob/main/notebooks/prepare_tfdatasets.ipynb"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "id": "wOqk480wuFye"
      },
      "outputs": [],
      "source": [
        "ds = tf.data.experimental.load(ARGS['local_data_dir'])"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!ls"
      ],
      "metadata": {
        "id": "lv4VYzc9Ytie",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e977b87c-f528-4504-86c7-f0e5688194f5"
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "LICENSE        pred.json\t\t  s3.msi.umn.edu\r\n",
            "notebooks      raw.githubusercontent.com  test_sequences.txt.gz\r\n",
            "notebooks_msi  README.md\t\t  train_sequences.txt.gz\r\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "id": "VbU-MCtK1B7k",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c837fa26-cc67-4806-b4de-9c277eb899df"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "downsampled dataset size: 6737568\n",
            "training dataset size: 6400689\n"
          ]
        }
      ],
      "source": [
        "n = int(ds.cardinality())\n",
        "n_train = int(n * ARGS['train_split'])\n",
        "print('downsampled dataset size: %d' % (n))\n",
        "print('training dataset size: %d' % (n_train))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {
        "id": "KtNM9BPoCCKT"
      },
      "outputs": [],
      "source": [
        "ds = ds.shuffle(ARGS['shuffle_size'], seed = 1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "metadata": {
        "id": "zQBqfMrpCCKT",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4d19b13e-13fd-458a-ced7-5c697e89160d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "# training samples: 6400689\n",
            "# test samples: 336879\n"
          ]
        }
      ],
      "source": [
        "train_ds = ds.take(n_train)\n",
        "test_ds = ds.skip(n_train)\n",
        "print('# training samples: %d' % (train_ds.cardinality()))\n",
        "print('# test samples: %d' % (test_ds.cardinality()))"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_ds = train_ds.batch(ARGS['batch_size'])\n",
        "test_ds = test_ds.batch(ARGS['batch_size'])"
      ],
      "metadata": {
        "id": "Cy45m_ZgwmBt"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_ds.element_spec"
      ],
      "metadata": {
        "id": "MNyS46F8TFRB",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e113ceb4-823c-408e-a65f-66fbde53ab70"
      },
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'base': TensorSpec(shape=(None, 200), dtype=tf.uint8, name=None),\n",
              " 'seq': TensorSpec(shape=(None, 200), dtype=tf.uint8, name=None),\n",
              " 'expression': TensorSpec(shape=(None,), dtype=tf.float32, name=None)}"
            ]
          },
          "metadata": {},
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## The regressor"
      ],
      "metadata": {
        "id": "3KwOLqNOwU3U"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class Regressor(keras.Model):\n",
        "  def __init__(self, n_positions, kmer = 3, embedding_dim = 32, input_dim = 5, n_blocks = 4, kernel_size =15, rate = 0.2, strides = 2, ratio = 0.2, num_heads = 4, ff_mult = 4, use_bias = False, num_projectors = 8):\n",
        "    super(Regressor, self).__init__()\n",
        "    self.n_positions = int(n_positions / strides)\n",
        "    self.input_dim = input_dim\n",
        "    self.kmer = kmer\n",
        "    self.strides = strides\n",
        "    self.num_projectors = num_projectors\n",
        "    self.n_blocks = n_blocks\n",
        "    self.masking = SequenceMaskLayer(n_positions = n_positions, ratio = ratio)\n",
        "    self.pos_embedding = layers.Embedding(self.n_positions, embedding_dim)\n",
        "    self.strand_embedding = layers.Embedding(2, embedding_dim) # plus/minus strands\n",
        "    self.expression_embedding = layers.Dense(embedding_dim)\n",
        "    self.kmer_dense = layers.Dense(embedding_dim)\n",
        "    self.blocks = keras.Sequential([ConformerSASwiGLULayer(embedding_dim = embedding_dim, kernel_size = kernel_size, rate = rate, num_heads = num_heads) for _ in range(n_blocks)])\n",
        "    self.dropout = layers.Dropout(rate)\n",
        "    self.expression_dense = layers.Dense(1)\n",
        "    self.nucleotide_dense = layers.Dense(input_dim)\n",
        "\n",
        "  def call(self, x):\n",
        "\n",
        "    batch_size = tf.shape(x['seq'])[0]\n",
        "\n",
        "    x = tf.one_hot(x['seq'], self.input_dim)\n",
        "\n",
        "    x = tf.expand_dims(x, 2)\n",
        "    x = tf.image.extract_patches(x, sizes = [1, self.kmer, 1, 1], strides = [1, self.strides, 1, 1], rates = [1, 1, 1, 1], padding = 'SAME')\n",
        "    x = tf.squeeze(x, 2)\n",
        "\n",
        "    x = self.kmer_dense(x)\n",
        "\n",
        "    pos = tf.range(start=0, limit=self.n_positions, delta=1)\n",
        "    pos = tf.expand_dims(pos, 0)\n",
        "    pos = self.pos_embedding(pos)\n",
        "\n",
        "    strand = tf.repeat([0,1], repeats = int(self.n_positions / 2))\n",
        "    strand = tf.expand_dims(strand, 0)\n",
        "    strand = self.strand_embedding(strand)\n",
        "\n",
        "    x = x + pos + strand\n",
        "\n",
        "    expression = tf.zeros((batch_size, self.num_projectors, 1), dtype = tf.float32)\n",
        "    expression = self.expression_embedding(expression)\n",
        "\n",
        "    x = tf.concat([expression, x], axis = 1)\n",
        "    x = self.blocks(x)\n",
        "\n",
        "    expression = x[:,:self.num_projectors,:]\n",
        "    x = x[:, -self.n_positions:, :]\n",
        "\n",
        "    expression = self.dropout(expression)\n",
        "    expression = self.expression_dense(expression)\n",
        "    expression = tf.reduce_mean(expression, 1)\n",
        "\n",
        "    x = self.nucleotide_dense(x)\n",
        "\n",
        "    return expression, x\n",
        "\n",
        "  def train_step(self, batch):\n",
        "\n",
        "    seq, mask = self.masking(batch['seq'])\n",
        "    \n",
        "    with tf.GradientTape() as tape:\n",
        "      expression, seq_pred = self({'seq' : seq})\n",
        "      loss_expression = mse_loss(batch['expression'], expression)\n",
        "      loss_expression = tf.reduce_mean(loss_expression)\n",
        "      loss_seq = mask * scc_loss(batch['seq'], seq_pred)\n",
        "      loss_seq = tf.reduce_sum(loss_seq) / (tf.reduce_sum(mask) + 1)\n",
        "      loss = loss_expression + loss_seq\n",
        "\n",
        "    gradients = tape.gradient(loss, self.trainable_variables)\n",
        "    self.optimizer.apply_gradients(zip(gradients, self.trainable_variables))\n",
        "\n",
        "    self.compiled_metrics.update_state(batch['expression'], expression)\n",
        "    res = {'loss' : loss, 'loss_seq' : loss_seq, 'loss_expression' : loss_expression}\n",
        "    res.update({m.name: m.result() for m in self.metrics})\n",
        "    return res\n",
        "\n",
        "  def test_step(self, batch):\n",
        "    expression, _ = self(batch)\n",
        "    self.compiled_metrics.update_state(batch['expression'], expression)\n",
        "    return {m.name: m.result() for m in self.metrics}"
      ],
      "metadata": {
        "id": "DjpQ5MFlpZ5v"
      },
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = Regressor(embedding_dim = 12, n_positions = 120, kernel_size = 3, strides = 1, num_projectors = ARGS['num_projectors'])\n",
        "x = tf.random.uniform((3, 120), maxval = 5)\n",
        "x = tf.cast(x, dtype = tf.uint8)\n",
        "expression_pred, x = model({'seq' : x})\n",
        "print(expression_pred.shape)\n",
        "print(x.shape)"
      ],
      "metadata": {
        "id": "JHA8irohpZ5w",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bbeddcff-35c4-42db-90a8-c06a72bf2e95"
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(3, 1)\n",
            "(3, 120, 5)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "strategy = tf.distribute.MirroredStrategy()\n",
        "#strategy = tf.distribute.experimental.CentralStorageStrategy()\n",
        "#strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy()"
      ],
      "metadata": {
        "id": "t_QJg04-pZ5w",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9ffbd269-2b5c-4653-d155-1fb6c49211b2"
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:Using MirroredStrategy with devices ('/job:localhost/replica:0/task:0/device:GPU:0', '/job:localhost/replica:0/task:0/device:GPU:1', '/job:localhost/replica:0/task:0/device:GPU:2', '/job:localhost/replica:0/task:0/device:GPU:3')\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "with strategy.scope():\n",
        "  model = Regressor(\n",
        "    n_positions = n_positions,\n",
        "    embedding_dim = ARGS['embedding_dim'], \n",
        "    n_blocks = ARGS['n_blocks_regressor'],\n",
        "    kmer = ARGS['kmer'],\n",
        "    input_dim = input_dim,\n",
        "    strides = ARGS['strides'],\n",
        "    ratio = ARGS['mask_ratio'],\n",
        "    num_heads = ARGS['num_heads'],\n",
        "    rate = ARGS['dropout_rate'],\n",
        "    num_projectors = ARGS['num_projectors'],\n",
        "  )"
      ],
      "metadata": {
        "id": "1s-DI91ppZ5w"
      },
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "learning_rate = CustomSchedule(ARGS['embedding_dim'], warmup_steps = ARGS['warmup_steps'])\n",
        "optimizer = tf.keras.optimizers.Adam(learning_rate, beta_1 = 0.9, beta_2 = 0.98, epsilon=1e-9, clipnorm=1., clipvalue=0.5)"
      ],
      "metadata": {
        "id": "b9CPAqzqbS4C"
      },
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model.compile(\n",
        "  optimizer = optimizer,\n",
        "  metrics = [pearson_r],\n",
        ")"
      ],
      "metadata": {
        "id": "7KW-9ktVbiak",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "76aff734-eac9-4790-db5f-e64939bdb8a6"
      },
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
            "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
            "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
            "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "if ARGS['train_split'] < 1:\n",
        "  monitor = 'val_pearson_r'\n",
        "else:\n",
        "  monitor = 'pearson_r'"
      ],
      "metadata": {
        "id": "mokLgrTWV8i8"
      },
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(\n",
        "  filepath = ARGS['local_checkpoint_dir'],\n",
        "  save_weights_only = True,\n",
        "  monitor = monitor,\n",
        "  mode='max',\n",
        "  save_best_only = True,\n",
        "  save_freq = 'epoch',\n",
        ")"
      ],
      "metadata": {
        "id": "vBB1dWoGL-CR"
      },
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Model fitting"
      ],
      "metadata": {
        "id": "JOtXgSxYvy0F"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Epoch 1 \n",
        "\n",
        "initial epoch: 1"
      ],
      "metadata": {
        "id": "zGLEOzKhpPE2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "if ARGS['train_split'] < 1:\n",
        "  model.fit(\n",
        "    train_ds,\n",
        "    validation_data = test_ds,\n",
        "    epochs = ARGS['epochs'],\n",
        "    callbacks = [model_checkpoint_callback],\n",
        "    initial_epoch = 1 - 1, \n",
        "  )\n",
        "else:\n",
        "  model.fit(\n",
        "    train_ds,\n",
        "    epochs = ARGS['epochs'],\n",
        "    callbacks = [model_checkpoint_callback],\n",
        "    initial_epoch = 1 - 1, \n",
        "  )\n"
      ],
      "metadata": {
        "id": "spFmkqwZDwDb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9d0ec544-2599-4aaa-9b04-42262cc64105"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "2022-07-27 16:09:17.679247: W tensorflow/core/grappler/optimizers/data/auto_shard.cc:695] AUTO sharding policy will apply DATA sharding policy as it failed to apply FILE sharding policy because of the following reason: Did not find a shardable source, walked to a node which is not a dataset: name: \"LoadDataset/_1\"\n",
            "op: \"LoadDataset\"\n",
            "input: \"Const/_0\"\n",
            "attr {\n",
            "  key: \"Treader_func_args\"\n",
            "  value {\n",
            "    list {\n",
            "    }\n",
            "  }\n",
            "}\n",
            "attr {\n",
            "  key: \"compression\"\n",
            "  value {\n",
            "    s: \"\"\n",
            "  }\n",
            "}\n",
            "attr {\n",
            "  key: \"output_shapes\"\n",
            "  value {\n",
            "    list {\n",
            "      shape {\n",
            "        dim {\n",
            "          size: 200\n",
            "        }\n",
            "      }\n",
            "      shape {\n",
            "      }\n",
            "      shape {\n",
            "        dim {\n",
            "          size: 200\n",
            "        }\n",
            "      }\n",
            "    }\n",
            "  }\n",
            "}\n",
            "attr {\n",
            "  key: \"output_types\"\n",
            "  value {\n",
            "    list {\n",
            "      type: DT_UINT8\n",
            "      type: DT_FLOAT\n",
            "      type: DT_UINT8\n",
            "    }\n",
            "  }\n",
            "}\n",
            "attr {\n",
            "  key: \"reader_func\"\n",
            "  value {\n",
            "    func {\n",
            "      name: \"__inference_load_lambda_4809\"\n",
            "    }\n",
            "  }\n",
            "}\n",
            ". Consider either turning off auto-sharding or switching the auto_shard_policy to DATA to shard this dataset. You can do this by creating a new `tf.data.Options()` object then setting `options.experimental_distribute.auto_shard_policy = AutoShardPolicy.DATA` before applying the options object to the dataset via `dataset.with_options(options)`.\n",
            "2022-07-27 16:09:17.768048: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:185] None of the MLIR Optimization Passes are enabled (registered 2)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/20\n",
            "INFO:tensorflow:batch_all_reduce: 100 all-reduces with algorithm = nccl, num_packs = 1\n",
            "WARNING:tensorflow:Efficient allreduce is not supported for 2 IndexedSlices\n",
            "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:GPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:GPU:0', '/job:localhost/replica:0/task:0/device:GPU:1', '/job:localhost/replica:0/task:0/device:GPU:2', '/job:localhost/replica:0/task:0/device:GPU:3').\n",
            "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:GPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:GPU:0', '/job:localhost/replica:0/task:0/device:GPU:1', '/job:localhost/replica:0/task:0/device:GPU:2', '/job:localhost/replica:0/task:0/device:GPU:3').\n",
            "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
            "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
            "INFO:tensorflow:batch_all_reduce: 100 all-reduces with algorithm = nccl, num_packs = 1\n",
            "WARNING:tensorflow:Efficient allreduce is not supported for 2 IndexedSlices\n",
            "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:GPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:GPU:0', '/job:localhost/replica:0/task:0/device:GPU:1', '/job:localhost/replica:0/task:0/device:GPU:2', '/job:localhost/replica:0/task:0/device:GPU:3').\n",
            "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:GPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:GPU:0', '/job:localhost/replica:0/task:0/device:GPU:1', '/job:localhost/replica:0/task:0/device:GPU:2', '/job:localhost/replica:0/task:0/device:GPU:3').\n",
            "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
            "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "2022-07-27 16:09:51.268588: I tensorflow/stream_executor/cuda/cuda_dnn.cc:369] Loaded cuDNN version 8200\n",
            "2022-07-27 16:09:53.166674: I tensorflow/stream_executor/cuda/cuda_dnn.cc:369] Loaded cuDNN version 8200\n",
            "2022-07-27 16:09:54.403054: I tensorflow/stream_executor/cuda/cuda_dnn.cc:369] Loaded cuDNN version 8200\n",
            "2022-07-27 16:09:55.267105: W tensorflow/stream_executor/gpu/asm_compiler.cc:113] *** WARNING *** You are using ptxas 9.0.176, which is older than 9.2.88. ptxas 9.x before 9.2.88 is known to miscompile XLA code, leading to incorrect results or invalid-address errors.\n",
            "\n",
            "You do not need to update to CUDA 9.2.88; cherry-picking the ptxas binary is sufficient.\n",
            "2022-07-27 16:09:55.378306: W tensorflow/stream_executor/gpu/asm_compiler.cc:231] Falling back to the CUDA driver for PTX compilation; ptxas does not support CC 8.0\n",
            "2022-07-27 16:09:55.378353: W tensorflow/stream_executor/gpu/asm_compiler.cc:234] Used ptxas at ptxas\n",
            "2022-07-27 16:09:55.381254: W tensorflow/stream_executor/gpu/redzone_allocator.cc:314] Unimplemented: ptxas ptxas too old. Falling back to the driver to compile.\n",
            "Relying on driver to perform ptx compilation. \n",
            "Modify $PATH to customize ptxas location.\n",
            "This message will be only logged once.\n",
            "2022-07-27 16:09:55.398162: I tensorflow/stream_executor/cuda/cuda_dnn.cc:369] Loaded cuDNN version 8200\n",
            "2022-07-27 16:09:55.558522: W tensorflow/stream_executor/gpu/asm_compiler.cc:113] *** WARNING *** You are using ptxas 9.0.176, which is older than 9.2.88. ptxas 9.x before 9.2.88 is known to miscompile XLA code, leading to incorrect results or invalid-address errors.\n",
            "\n",
            "You do not need to update to CUDA 9.2.88; cherry-picking the ptxas binary is sufficient.\n",
            "2022-07-27 16:09:55.952617: W tensorflow/stream_executor/gpu/asm_compiler.cc:113] *** WARNING *** You are using ptxas 9.0.176, which is older than 9.2.88. ptxas 9.x before 9.2.88 is known to miscompile XLA code, leading to incorrect results or invalid-address errors.\n",
            "\n",
            "You do not need to update to CUDA 9.2.88; cherry-picking the ptxas binary is sufficient.\n",
            "2022-07-27 16:09:56.826903: W tensorflow/stream_executor/gpu/asm_compiler.cc:113] *** WARNING *** You are using ptxas 9.0.176, which is older than 9.2.88. ptxas 9.x before 9.2.88 is known to miscompile XLA code, leading to incorrect results or invalid-address errors.\n",
            "\n",
            "You do not need to update to CUDA 9.2.88; cherry-picking the ptxas binary is sufficient.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "12502/12502 [==============================] - ETA: 0s - loss: 0.8371 - loss_seq: 0.2761 - loss_expression: 0.5610 - pearson_r: 0.6785"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "2022-07-27 17:26:07.810499: W tensorflow/core/grappler/optimizers/data/auto_shard.cc:695] AUTO sharding policy will apply DATA sharding policy as it failed to apply FILE sharding policy because of the following reason: Did not find a shardable source, walked to a node which is not a dataset: name: \"LoadDataset/_1\"\n",
            "op: \"LoadDataset\"\n",
            "input: \"Const/_0\"\n",
            "attr {\n",
            "  key: \"Treader_func_args\"\n",
            "  value {\n",
            "    list {\n",
            "    }\n",
            "  }\n",
            "}\n",
            "attr {\n",
            "  key: \"compression\"\n",
            "  value {\n",
            "    s: \"\"\n",
            "  }\n",
            "}\n",
            "attr {\n",
            "  key: \"output_shapes\"\n",
            "  value {\n",
            "    list {\n",
            "      shape {\n",
            "        dim {\n",
            "          size: 200\n",
            "        }\n",
            "      }\n",
            "      shape {\n",
            "      }\n",
            "      shape {\n",
            "        dim {\n",
            "          size: 200\n",
            "        }\n",
            "      }\n",
            "    }\n",
            "  }\n",
            "}\n",
            "attr {\n",
            "  key: \"output_types\"\n",
            "  value {\n",
            "    list {\n",
            "      type: DT_UINT8\n",
            "      type: DT_FLOAT\n",
            "      type: DT_UINT8\n",
            "    }\n",
            "  }\n",
            "}\n",
            "attr {\n",
            "  key: \"reader_func\"\n",
            "  value {\n",
            "    func {\n",
            "      name: \"__inference_load_lambda_4809\"\n",
            "    }\n",
            "  }\n",
            "}\n",
            ". Consider either turning off auto-sharding or switching the auto_shard_policy to DATA to shard this dataset. You can do this by creating a new `tf.data.Options()` object then setting `options.experimental_distribute.auto_shard_policy = AutoShardPolicy.DATA` before applying the options object to the dataset via `dataset.with_options(options)`.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
            "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n",
            "12502/12502 [==============================] - 4785s 379ms/step - loss: 0.8371 - loss_seq: 0.2761 - loss_expression: 0.5610 - pearson_r: 0.6785 - val_pearson_r: 0.7267\n",
            "Epoch 2/20\n",
            "12502/12502 [==============================] - 4737s 379ms/step - loss: 0.5439 - loss_seq: 0.0687 - loss_expression: 0.4752 - pearson_r: 0.7279 - val_pearson_r: 0.7411\n",
            "Epoch 3/20\n",
            "12502/12502 [==============================] - 4748s 380ms/step - loss: 0.5259 - loss_seq: 0.0682 - loss_expression: 0.4576 - pearson_r: 0.7388 - val_pearson_r: 0.7463\n",
            "Epoch 4/20\n",
            "12502/12502 [==============================] - 4738s 379ms/step - loss: 0.5168 - loss_seq: 0.0686 - loss_expression: 0.4483 - pearson_r: 0.7449 - val_pearson_r: 0.7493\n",
            "Epoch 5/20\n",
            "12502/12502 [==============================] - 4735s 379ms/step - loss: 0.5092 - loss_seq: 0.0684 - loss_expression: 0.4408 - pearson_r: 0.7493 - val_pearson_r: 0.7500\n",
            "Epoch 6/20\n",
            "12502/12502 [==============================] - 4739s 379ms/step - loss: 0.5028 - loss_seq: 0.0682 - loss_expression: 0.4346 - pearson_r: 0.7530 - val_pearson_r: 0.7501\n",
            "Epoch 7/20\n",
            "12502/12502 [==============================] - 4751s 380ms/step - loss: 0.4971 - loss_seq: 0.0683 - loss_expression: 0.4288 - pearson_r: 0.7570 - val_pearson_r: 0.7491\n",
            "Epoch 8/20\n",
            "12502/12502 [==============================] - 4743s 379ms/step - loss: 0.4900 - loss_seq: 0.0682 - loss_expression: 0.4218 - pearson_r: 0.7611 - val_pearson_r: 0.7480\n",
            "Epoch 9/20\n",
            "12502/12502 [==============================] - 4750s 380ms/step - loss: 0.4839 - loss_seq: 0.0683 - loss_expression: 0.4156 - pearson_r: 0.7658 - val_pearson_r: 0.7445\n",
            "Epoch 10/20\n",
            "12502/12502 [==============================] - 4741s 379ms/step - loss: 0.4752 - loss_seq: 0.0683 - loss_expression: 0.4069 - pearson_r: 0.7712 - val_pearson_r: 0.7415\n",
            "Epoch 11/20\n",
            "12502/12502 [==============================] - 4739s 379ms/step - loss: 0.4660 - loss_seq: 0.0684 - loss_expression: 0.3975 - pearson_r: 0.7769 - val_pearson_r: 0.7388\n",
            "Epoch 12/20\n",
            "12502/12502 [==============================] - 4746s 380ms/step - loss: 0.4567 - loss_seq: 0.0683 - loss_expression: 0.3883 - pearson_r: 0.7833 - val_pearson_r: 0.7315\n",
            "Epoch 13/20\n",
            "12502/12502 [==============================] - 4741s 379ms/step - loss: 0.4452 - loss_seq: 0.0685 - loss_expression: 0.3767 - pearson_r: 0.7900 - val_pearson_r: 0.7257\n",
            "Epoch 14/20\n",
            "12502/12502 [==============================] - 4744s 379ms/step - loss: 0.4341 - loss_seq: 0.0684 - loss_expression: 0.3658 - pearson_r: 0.7967 - val_pearson_r: 0.7228\n",
            "Epoch 15/20\n",
            "  728/12502 [>.............................] - ETA: 1:11:39 - loss: 0.4252 - loss_seq: 0.0686 - loss_expression: 0.3565 - pearson_r: 0.8006"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ArD-gBiIjgYD"
      },
      "source": [
        "## Read testing data"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "os.system('wget -cx ' + ARGS['remote_test_data'])"
      ],
      "metadata": {
        "id": "veXsv1c0KPx_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "61b50061-7ef8-4c16-98dc-c66b7db39c6f"
      },
      "execution_count": 88,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "--2022-08-07 17:29:51--  https://s3.msi.umn.edu/gongx030/projects/dream_PGE/test_sequences.txt.gz\n",
            "Resolving s3.msi.umn.edu (s3.msi.umn.edu)... 128.101.189.144\n",
            "Connecting to s3.msi.umn.edu (s3.msi.umn.edu)|128.101.189.144|:443... connected.\n",
            "HTTP request sent, awaiting response... 416 Requested Range Not Satisfiable\n",
            "\n",
            "    The file is already fully retrieved; nothing to do.\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {},
          "execution_count": 88
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.read_csv(ARGS['local_test_data'], compression = 'gzip', sep = '\\t', header = None)\n",
        "df.columns = ['dna', 'expression']\n",
        "df['dna'] = df['dna'].astype('string')\n",
        "df['dna0'] = df['dna']\n",
        "print('test dataset size: %d' % len(df))"
      ],
      "metadata": {
        "id": "ZAaoU0rwKbIE",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d8e69ffd-9b3d-4df7-f076-5d5f62f90d08"
      },
      "execution_count": 89,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "test dataset size: 71103\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df['len'] = df['dna'].str.len()\n",
        "plt.hist(df['len'])\n",
        "print('min len: {}'.format(df['len'].min()))\n",
        "print('max len: {}'.format(df['len'].max()))"
      ],
      "metadata": {
        "id": "PxfUTp-KHR7U",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "outputId": "0630852a-77c9-49f8-8f8c-2a7e89a440e3"
      },
      "execution_count": 90,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "min len: 110\n",
            "max len: 110\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAD4CAYAAAAO9oqkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAUcUlEQVR4nO3df6zd9X3f8eerOFCWDWzCnUdtNJPhNSNsIXBlHHXa1ljxDzrVtEoYqIpvkYOrQbpsmtQ5XVVrECSiSWVhSpC84tqu0gBlpViNiec5ibJKM3AJhJ+hviFBvh7gW2xgKQoZyXt/nI/JyeVc32P73nPt+PmQvjqf7/v7+X7P52PweZ3v93zPcaoKSdLp7efmegCSpLlnGEiSDANJkmEgScIwkCQB8+Z6AMfr/PPPryVLlsz1MCTplPHoo4/+dVUN9dp2yobBkiVLGB0dnethSNIpI8kLU23zMpEkyTCQJBkGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkujjG8hJfhG4p6v0XuD3ge2tvgT4HnBNVR1OEuBzwFXAG8BvVtU327FGgN9rx/lMVW1r9SuArcDZwE7gU+W/uqNT1JKNX56T5/3ebb8yJ8+rnw3TnhlU1XNVdVlVXQZcQecF/n5gI7CnqpYCe9o6wBpgaVs2AHcCJDkP2ARcCSwDNiVZ0Pa5E7iha7/VMzE5SVJ/jvUy0QrgO1X1ArAW2Nbq24CrW3stsL069gLzk1wArAJ2V9WhqjoM7AZWt23nVNXedjawvetYkqQBONYwuBb4UmsvrKoXW/slYGFrLwL2d+0z3mpHq4/3qL9Dkg1JRpOMTkxMHOPQJUlT6TsMkpwJ/Crwp5O3tXf0s36Nv6o2V9VwVQ0PDfX8FVZJ0nE4ljODNcA3q+rltv5yu8RDezzY6geAC7v2W9xqR6sv7lGXJA3IsYTBdfzkEhHADmCktUeAB7rq69KxHHitXU7aBaxMsqB9cLwS2NW2vZ5kebsTaV3XsSRJA9DXP26T5N3AR4Df6irfBtybZD3wAnBNq++kc1vpGJ07j64HqKpDSW4BHmn9bq6qQ619Iz+5tfTBtkiSBqSvMKiqvwHeM6n2Cp27iyb3LeCmKY6zBdjSoz4KXNrPWCRJM89vIEuSDANJkmEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJNFnGCSZn+S+JN9O8mySDyU5L8nuJPva44LWN0nuSDKW5Ikkl3cdZ6T135dkpKt+RZIn2z53JMnMT1WSNJV+zww+B3ylqt4HfAB4FtgI7KmqpcCetg6wBljalg3AnQBJzgM2AVcCy4BNRwKk9bmha7/VJzYtSdKxmDYMkpwL/DPgLoCq+mFVvQqsBba1btuAq1t7LbC9OvYC85NcAKwCdlfVoao6DOwGVrdt51TV3qoqYHvXsSRJA9DPmcFFwATwR0keS/KHSd4NLKyqF1ufl4CFrb0I2N+1/3irHa0+3qP+Dkk2JBlNMjoxMdHH0CVJ/egnDOYBlwN3VtUHgb/hJ5eEAGjv6Gvmh/fTqmpzVQ1X1fDQ0NBsP50knTb6CYNxYLyqHmrr99EJh5fbJR7a48G2/QBwYdf+i1vtaPXFPeqSpAGZNgyq6iVgf5JfbKUVwDPADuDIHUEjwAOtvQNY1+4qWg681i4n7QJWJlnQPjheCexq215PsrzdRbSu61iSpAGY12e/3wa+mORM4HngejpBcm+S9cALwDWt707gKmAMeKP1paoOJbkFeKT1u7mqDrX2jcBW4GzgwbZIkgakrzCoqseB4R6bVvToW8BNUxxnC7ClR30UuLSfsUiSZp7fQJYkGQaSJMNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgSaLPMEjyvSRPJnk8yWirnZdkd5J97XFBqyfJHUnGkjyR5PKu44y0/vuSjHTVr2jHH2v7ZqYnKkma2rGcGfxyVV1WVcNtfSOwp6qWAnvaOsAaYGlbNgB3Qic8gE3AlcAyYNORAGl9bujab/Vxz0iSdMxO5DLRWmBba28Dru6qb6+OvcD8JBcAq4DdVXWoqg4Du4HVbds5VbW3qgrY3nUsSdIA9BsGBfyPJI8m2dBqC6vqxdZ+CVjY2ouA/V37jrfa0erjPervkGRDktEkoxMTE30OXZI0nXl99vunVXUgyd8Fdif5dvfGqqokNfPD+2lVtRnYDDA8PDzrzydJp4u+zgyq6kB7PAjcT+ea/8vtEg/t8WDrfgC4sGv3xa12tPriHnVJ0oBMGwZJ3p3k7xxpAyuBp4AdwJE7gkaAB1p7B7Cu3VW0HHitXU7aBaxMsqB9cLwS2NW2vZ5kebuLaF3XsSRJA9DPZaKFwP3tbs95wJ9U1VeSPALcm2Q98AJwTeu/E7gKGAPeAK4HqKpDSW4BHmn9bq6qQ619I7AVOBt4sC2SpAGZNgyq6nngAz3qrwAretQLuGmKY20BtvSojwKX9jFeSdIs8BvIkiTDQJJkGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJHEMYJDkjyWNJ/qKtX5TkoSRjSe5Jcmarn9XWx9r2JV3H+HSrP5dkVVd9dauNJdk4g/OTJPXhWM4MPgU827X+WeD2qroYOAysb/X1wOFWv731I8klwLXA+4HVwBdawJwBfB5YA1wCXNf6SpIGpK8wSLIY+BXgD9t6gA8D97Uu24CrW3ttW6dtX9H6rwXurqo3q+q7wBiwrC1jVfV8Vf0QuLv1lSQNSL9nBv8F+B3gx239PcCrVfVWWx8HFrX2ImA/QNv+Wuv/dn3SPlPV3yHJhiSjSUYnJib6HLokaTrThkGSfwkcrKpHBzCeo6qqzVU1XFXDQ0NDcz0cSfqZMa+PPr8E/GqSq4CfB84BPgfMTzKvvftfDBxo/Q8AFwLjSeYB5wKvdNWP6N5nqrokaQCmPTOoqk9X1eKqWkLnA+CvVtVvAF8DPtq6jQAPtPaOtk7b/tWqqla/tt1tdBGwFHgYeARY2u5OOrM9x44ZmZ0kqS/9nBlM5T8Adyf5DPAYcFer3wX8cZIx4BCdF3eq6ukk9wLPAG8BN1XVjwCSfBLYBZwBbKmqp09gXJKkY3RMYVBVXwe+3trP07kTaHKfHwAfm2L/W4Fbe9R3AjuPZSySpJnjN5AlSYaBJMkwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJLoIwyS/HySh5N8K8nTSf5Tq1+U5KEkY0nuSXJmq5/V1sfa9iVdx/p0qz+XZFVXfXWrjSXZOAvzlCQdRT9nBm8CH66qDwCXAauTLAc+C9xeVRcDh4H1rf964HCr3976keQS4Frg/cBq4AtJzkhyBvB5YA1wCXBd6ytJGpBpw6A6vt9W39WWAj4M3Nfq24CrW3ttW6dtX5EkrX53Vb1ZVd8FxoBlbRmrquer6ofA3a2vJGlA+vrMoL2Dfxw4COwGvgO8WlVvtS7jwKLWXgTsB2jbXwPe012ftM9U9V7j2JBkNMnoxMREP0OXJPWhrzCoqh9V1WXAYjrv5N83m4M6yjg2V9VwVQ0PDQ3NxRAk6WfSMd1NVFWvAl8DPgTMTzKvbVoMHGjtA8CFAG37ucAr3fVJ+0xVlyQNSD93Ew0lmd/aZwMfAZ6lEwofbd1GgAdae0dbp23/alVVq1/b7ja6CFgKPAw8AixtdyedSedD5h0zMDdJUp/mTd+FC4Bt7a6fnwPuraq/SPIMcHeSzwCPAXe1/ncBf5xkDDhE58Wdqno6yb3AM8BbwE1V9SOAJJ8EdgFnAFuq6ukZm6EkaVrThkFVPQF8sEf9eTqfH0yu/wD42BTHuhW4tUd9J7Czj/FKkmaB30CWJBkGkiTDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEmijzBIcmGSryV5JsnTST7V6ucl2Z1kX3tc0OpJckeSsSRPJLm861gjrf++JCNd9SuSPNn2uSNJZmOykqTe+jkzeAv491V1CbAcuCnJJcBGYE9VLQX2tHWANcDStmwA7oROeACbgCuBZcCmIwHS+tzQtd/qE5+aJKlf04ZBVb1YVd9s7f8LPAssAtYC21q3bcDVrb0W2F4de4H5SS4AVgG7q+pQVR0GdgOr27ZzqmpvVRWwvetYkqQBOKbPDJIsAT4IPAQsrKoX26aXgIWtvQjY37XbeKsdrT7eoy5JGpC+wyDJ3wb+O/Bvq+r17m3tHX3N8Nh6jWFDktEkoxMTE7P9dJJ02ugrDJK8i04QfLGq/qyVX26XeGiPB1v9AHBh1+6LW+1o9cU96u9QVZurariqhoeGhvoZuiSpD/3cTRTgLuDZqvqDrk07gCN3BI0AD3TV17W7ipYDr7XLSbuAlUkWtA+OVwK72rbXkyxvz7Wu61iSpAGY10efXwI+DjyZ5PFW+13gNuDeJOuBF4Br2radwFXAGPAGcD1AVR1KcgvwSOt3c1Udau0bga3A2cCDbZEkDci0YVBVfwlMdd//ih79C7hpimNtAbb0qI8Cl043FknS7PAbyJIkw0CSZBhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiT6CIMkW5IcTPJUV+28JLuT7GuPC1o9Se5IMpbkiSSXd+0z0vrvSzLSVb8iyZNtnzuSZKYnKUk6un7ODLYCqyfVNgJ7qmopsKetA6wBlrZlA3AndMID2ARcCSwDNh0JkNbnhq79Jj+XJGmWTRsGVfUN4NCk8lpgW2tvA67uqm+vjr3A/CQXAKuA3VV1qKoOA7uB1W3bOVW1t6oK2N51LEnSgBzvZwYLq+rF1n4JWNjai4D9Xf3GW+1o9fEe9Z6SbEgymmR0YmLiOIcuSZrshD9Abu/oawbG0s9zba6q4aoaHhoaGsRTStJp4XjD4OV2iYf2eLDVDwAXdvVb3GpHqy/uUZckDdDxhsEO4MgdQSPAA131de2uouXAa+1y0i5gZZIF7YPjlcCutu31JMvbXUTruo4lSRqQedN1SPIl4F8A5ycZp3NX0G3AvUnWAy8A17TuO4GrgDHgDeB6gKo6lOQW4JHW7+aqOvKh9I107lg6G3iwLZKkAZo2DKrquik2rejRt4CbpjjOFmBLj/oocOl045AkzR6/gSxJMgwkSYaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAksRJFAZJVid5LslYko1zPR5JOp2cFGGQ5Azg88Aa4BLguiSXzO2oJOn0cVKEAbAMGKuq56vqh8DdwNo5HpMknTbmzfUAmkXA/q71ceDKyZ2SbAA2tNXvJ3luAGObSecDfz3Xgxgw5zwg+eygn/Gn+N/51PD3p9pwsoRBX6pqM7B5rsdxvJKMVtXwXI9jkJzz6cE5n/pOlstEB4ALu9YXt5okaQBOljB4BFia5KIkZwLXAjvmeEySdNo4KS4TVdVbST4J7ALOALZU1dNzPKzZcMpe4joBzvn04JxPcamquR6DJGmOnSyXiSRJc8gwkCQZBscryZYkB5M81VU7L8nuJPva44JWX5Dk/iRPJHk4yaVTHDNJbk3yV0meTfJvBjWffszSnFck+WaSx5P8ZZKLBzWffkwx548leTrJj5MMT+r/6faTKs8lWTXFMS9K8lDrd0+7aeKkMUtz/mLb/lQ7/rtmex7HYjbm3NX3jiTfn62xzxTD4PhtBVZPqm0E9lTVUmBPWwf4XeDxqvonwDrgc1Mc8zfp3GL7vqr6R3S+iX0y2crMz/lO4Deq6jLgT4Dfm+Exn6itvHPOTwG/Dnyju9h+QuVa4P1tny+0n1qZ7LPA7VV1MXAYWD/DYz5RW5n5OX8ReB/wj4GzgU/M7JBP2FZmfs60EFkw04OdDYbBcaqqbwCHJpXXAttaextwdWtfAny17fdtYEmShT0O+6+Bm6vqx63vwRke9gmZpTkXcE5rnwv8nxkc8gnrNeeqeraqen37fS1wd1W9WVXfBcbo/NTK25IE+DBwXyt1/5mdFGZ6zm3/ndUAD9P5LtFJYzbm3ALiPwO/MwtDnnGGwcxaWFUvtvZLwJEXv2/ReYdBkmV0vhLe6y/DPwD+VZLRJA8mWTrbA54BJzrnTwA7k4wDHwdum93hzqpeP6uyaFKf9wCvVtVbR+lzKulnzm9rl4c+Dnxllsc1m/qd8yeBHV1/P05qhsEsae+Ajty3exswP8njwG8DjwE/6rHbWcAP2lfc/xuwZQBDnTHHOed/B1xVVYuBPwL+YABD1dz5AvCNqvpfcz2Q2ZTkF4CPAf91rsfSr5PiS2c/Q15OckFVvZjkAuAgQFW9DlwPb18m+C7wfI/9x4E/a+376bw4nuyOe85JhoAPVNVDrXQPp/Y7xn5+VuUVOiE5r50dnOo/vdL3T8kk2QQMAb81gHHNpn7m/EHgYmCs878/fyvJWPuc6KTkmcHM2gGMtPYI8ABAkvldd4x8gs47o9d77P/nwC+39j8H/mr2hjpjTmTOh4Fzk/zDtv4R4NlZHu9s2gFcm+SsJBcBS+lcH39bO3v6GvDRVnr7z+wUNe2cAZJ8AlgFXHfkM7FTWD//nb9cVX+vqpZU1RLgjZM5CACoKpfjWIAvAS8C/4/OO/r1dK4H7wH2Af8TOK/1/RCdF/bn6LzzX9B1nJ3AL7T2fODLwJPA/6bzrnnO5zrLc/61Nt9vAV8H3jvX8+xjzr/W2m8CLwO7uvr/R+A7bd5rppjze+m8eIwBfwqcNdfzHMCc32p9Hm/L78/1PGd7zpOO//25nuN0iz9HIUnyMpEkyTCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJ+P8aOT58V/L+GQAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print('number of unique sequences in the first {} positions: {}'.format(ARGS['head_len'], len(df['dna'].str[:ARGS['head_len']].unique())))\n",
        "print('number of unique sequences in the last {} positions: {}'.format(ARGS['tail_len'], len(df['dna'].str[-ARGS['tail_len']:].unique())))"
      ],
      "metadata": {
        "id": "dD49DoSqLBUF",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8d18a5bb-3a4c-441e-9d0b-77dd0409aae1"
      },
      "execution_count": 91,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "number of unique sequences in the first 17 positions: 1\n",
            "number of unique sequences in the last 13 positions: 1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df['dna'] = df['dna'].str[ARGS['head_len']:]\n",
        "df['dna'] = df['dna'].str[:-ARGS['tail_len']]"
      ],
      "metadata": {
        "id": "5DinLJpsD-VH"
      },
      "execution_count": 92,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df['len'] = df['dna'].str.len()\n",
        "plt.hist(df['len'])\n",
        "print('min len: {}'.format(df['len'].min()))\n",
        "print('max len: {}'.format(df['len'].max()))"
      ],
      "metadata": {
        "id": "xC_9TprOF7h0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "outputId": "e9191f8b-fcd0-4212-f116-787f458fafa0"
      },
      "execution_count": 93,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "min len: 80\n",
            "max len: 80\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAD4CAYAAAAO9oqkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAUY0lEQVR4nO3df6zd9X3f8ecruBBKCzbhxiI2m13VTeUghR8WOMvWbXgxNpliS00QaJtdynA1yJROq1Znq+YGkolM02hQUyYruNhZF+KyZnitqWs5qapWNfElUIj5MS4Q5usBvsUGlqCASN/743w8DuZc33Ov7z3Xjp8P6avz+b6/n+/3fD66cF/nfM/nXKeqkCSd3t4z2wOQJM0+w0CSZBhIkgwDSRKGgSQJmDPbA5iqCy64oBYtWjTbw5CkU8ZDDz3011U11OvYKRsGixYtYnh4eLaHIUmnjCTPj3fM20SSJMNAkmQYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgSaKPbyAn+SDw9a7SzwD/HtjW6ouA7wHXVtWRJAG+BFwDvA78UlV9p11rPfAb7Tqfr6qtrX45cA9wNrAT+Ez5r+7oFLVo4x/NyvN+7/aPz8rz6sfDhO8Mquqpqrqkqi4BLqfzC/4bwEZgT1UtAfa0fYDVwJK2bQDuAkhyPrAJuBK4AtiUZF475y7gpq7zVk3H5CRJ/ZnsbaIVwDNV9TywBtja6luBta29BthWHXuBuUkuBK4GdlfV4ao6AuwGVrVj51bV3vZuYFvXtSRJAzDZMLgO+Fprz6+qF1r7RWB+ay8ADnSdM9pqx6uP9qi/S5INSYaTDI+NjU1y6JKk8fQdBknOBD4B/P6xx9or+hm/x19Vm6tqWVUtGxrq+VdYJUlTMJl3BquB71TVS23/pXaLh/Z4qNUPAhd1nbew1Y5XX9ijLkkakMmEwfW8fYsIYAewvrXXA/d31delYznwarudtAtYmWRe++B4JbCrHXstyfK2Emld17UkSQPQ1z9uk+Qc4GPAr3SVbwe2J7kReB64ttV30llWOkJn5dENAFV1OMltwL7W79aqOtzaN/P20tIH2iZJGpC+wqCqfgC875jay3RWFx3bt4BbxrnOFmBLj/owcHE/Y5EkTT+/gSxJMgwkSYaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkkSfYZBkbpL7kjyZ5IkkH0lyfpLdSZ5uj/Na3yS5M8lIkkeTXNZ1nfWt/9NJ1nfVL0/yWDvnziSZ/qlKksbT7zuDLwF/XFU/D3wYeALYCOypqiXAnrYPsBpY0rYNwF0ASc4HNgFXAlcAm44GSOtzU9d5q05sWpKkyZgwDJKcB/wCcDdAVb1ZVa8Aa4CtrdtWYG1rrwG2VcdeYG6SC4Grgd1VdbiqjgC7gVXt2LlVtbeqCtjWdS1J0gD0885gMTAG/G6Sh5N8Jck5wPyqeqH1eRGY39oLgANd54+22vHqoz3q75JkQ5LhJMNjY2N9DF2S1I9+wmAOcBlwV1VdCvyAt28JAdBe0df0D++dqmpzVS2rqmVDQ0Mz/XSSdNroJwxGgdGqerDt30cnHF5qt3hoj4fa8YPARV3nL2y149UX9qhLkgZkwjCoqheBA0k+2EorgMeBHcDRFUHrgftbewewrq0qWg682m4n7QJWJpnXPjheCexqx15LsrytIlrXdS1J0gDM6bPfvwR+L8mZwLPADXSCZHuSG4HngWtb353ANcAI8HrrS1UdTnIbsK/1u7WqDrf2zcA9wNnAA22TJA1IX2FQVY8Ay3ocWtGjbwG3jHOdLcCWHvVh4OJ+xiJJmn5+A1mSZBhIkgwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJIk+wyDJ95I8luSRJMOtdn6S3Umebo/zWj1J7kwykuTRJJd1XWd96/90kvVd9cvb9UfauZnuiUqSxjeZdwb/sKouqaplbX8jsKeqlgB72j7AamBJ2zYAd0EnPIBNwJXAFcCmowHS+tzUdd6qKc9IkjRpJ3KbaA2wtbW3Amu76tuqYy8wN8mFwNXA7qo6XFVHgN3Aqnbs3KraW1UFbOu6liRpAPoNgwL+JMlDSTa02vyqeqG1XwTmt/YC4EDXuaOtdrz6aI/6uyTZkGQ4yfDY2FifQ5ckTWROn/3+blUdTPJ+YHeSJ7sPVlUlqekf3jtV1WZgM8CyZctm/Pkk6XTR1zuDqjrYHg8B36Bzz/+ldouH9niodT8IXNR1+sJWO159YY+6JGlAJgyDJOck+emjbWAl8F1gB3B0RdB64P7W3gGsa6uKlgOvtttJu4CVSea1D45XArvasdeSLG+riNZ1XUuSNAD93CaaD3yjrfacA/y3qvrjJPuA7UluBJ4Hrm39dwLXACPA68ANAFV1OMltwL7W79aqOtzaNwP3AGcDD7RNkjQgE4ZBVT0LfLhH/WVgRY96AbeMc60twJYe9WHg4j7GK0maAX4DWZJkGEiSDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkiUmEQZIzkjyc5A/b/uIkDyYZSfL1JGe2+lltf6QdX9R1jc+2+lNJru6qr2q1kSQbp3F+kqQ+TOadwWeAJ7r2vwjcUVU/CxwBbmz1G4EjrX5H60eSpcB1wIeAVcDvtIA5A/gysBpYClzf+kqSBqSvMEiyEPg48JW2H+Aq4L7WZSuwtrXXtH3a8RWt/xrg3qp6o6qeA0aAK9o2UlXPVtWbwL2tryRpQPp9Z/BbwL8B/qbtvw94pareavujwILWXgAcAGjHX239/3/9mHPGq79Lkg1JhpMMj42N9Tl0SdJEJgyDJP8YOFRVDw1gPMdVVZurallVLRsaGprt4UjSj405ffT5KPCJJNcA7wXOBb4EzE0yp736XwgcbP0PAhcBo0nmAOcBL3fVj+o+Z7y6JGkAJnxnUFWfraqFVbWIzgfA36yqfwJ8C/hk67YeuL+1d7R92vFvVlW1+nVttdFiYAnwbWAfsKStTjqzPceOaZmdJKkv/bwzGM+vA/cm+TzwMHB3q98NfDXJCHCYzi93qmp/ku3A48BbwC1V9SOAJJ8GdgFnAFuqav8JjEuSNEmTCoOq+lPgT1v7WTorgY7t80PgU+Oc/wXgCz3qO4GdkxmLJGn6+A1kSZJhIEkyDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIk+giDJO9N8u0kf5Vkf5LPtfriJA8mGUny9SRntvpZbX+kHV/Uda3PtvpTSa7uqq9qtZEkG2dgnpKk4+jnncEbwFVV9WHgEmBVkuXAF4E7qupngSPAja3/jcCRVr+j9SPJUuA64EPAKuB3kpyR5Azgy8BqYClwfesrSRqQCcOgOr7fdn+ibQVcBdzX6luBta29pu3Tjq9Ikla/t6reqKrngBHgiraNVNWzVfUmcG/rK0kakL4+M2iv4B8BDgG7gWeAV6rqrdZlFFjQ2guAAwDt+KvA+7rrx5wzXr3XODYkGU4yPDY21s/QJUl96CsMqupHVXUJsJDOK/mfn8lBHWccm6tqWVUtGxoamo0hSNKPpUmtJqqqV4BvAR8B5iaZ0w4tBA629kHgIoB2/Dzg5e76MeeMV5ckDUg/q4mGksxt7bOBjwFP0AmFT7Zu64H7W3tH26cd/2ZVVatf11YbLQaWAN8G9gFL2uqkM+l8yLxjGuYmSerTnIm7cCGwta36eQ+wvar+MMnjwL1JPg88DNzd+t8NfDXJCHCYzi93qmp/ku3A48BbwC1V9SOAJJ8GdgFnAFuqav+0zVCSNKEJw6CqHgUu7VF/ls7nB8fWfwh8apxrfQH4Qo/6TmBnH+OVJM0Av4EsSTIMJEmGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJJEH2GQ5KIk30ryeJL9ST7T6ucn2Z3k6fY4r9WT5M4kI0keTXJZ17XWt/5PJ1nfVb88yWPtnDuTZCYmK0nqrZ93Bm8B/7qqlgLLgVuSLAU2Anuqagmwp+0DrAaWtG0DcBd0wgPYBFwJXAFsOhogrc9NXeetOvGpSZL6NWEYVNULVfWd1v6/wBPAAmANsLV12wqsbe01wLbq2AvMTXIhcDWwu6oOV9URYDewqh07t6r2VlUB27quJUkagEl9ZpBkEXAp8CAwv6peaIdeBOa39gLgQNdpo612vPpoj7okaUD6DoMkPwX8d+BXq+q17mPtFX1N89h6jWFDkuEkw2NjYzP9dJJ02ugrDJL8BJ0g+L2q+oNWfqnd4qE9Hmr1g8BFXacvbLXj1Rf2qL9LVW2uqmVVtWxoaKifoUuS+tDPaqIAdwNPVNV/7jq0Azi6Img9cH9XfV1bVbQceLXdTtoFrEwyr31wvBLY1Y69lmR5e651XdeSJA3AnD76fBT4Z8BjSR5ptX8L3A5sT3Ij8DxwbTu2E7gGGAFeB24AqKrDSW4D9rV+t1bV4da+GbgHOBt4oG2SpAGZMAyq6s+B8db9r+jRv4BbxrnWFmBLj/owcPFEY5EkzQy/gSxJMgwkSYaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEmijzBIsiXJoSTf7aqdn2R3kqfb47xWT5I7k4wkeTTJZV3nrG/9n06yvqt+eZLH2jl3Jsl0T1KSdHz9vDO4B1h1TG0jsKeqlgB72j7AamBJ2zYAd0EnPIBNwJXAFcCmowHS+tzUdd6xzyVJmmEThkFV/Rlw+JjyGmBra28F1nbVt1XHXmBukguBq4HdVXW4qo4Au4FV7di5VbW3qgrY1nUtSdKATPUzg/lV9UJrvwjMb+0FwIGufqOtdrz6aI96T0k2JBlOMjw2NjbFoUuSjnXCHyC3V/Q1DWPp57k2V9Wyqlo2NDQ0iKeUpNPCVMPgpXaLh/Z4qNUPAhd19VvYaserL+xRlyQN0FTDYAdwdEXQeuD+rvq6tqpoOfBqu520C1iZZF774HglsKsdey3J8raKaF3XtSRJAzJnog5Jvgb8A+CCJKN0VgXdDmxPciPwPHBt674TuAYYAV4HbgCoqsNJbgP2tX63VtXRD6VvprNi6WzggbZJkgZowjCoquvHObSiR98CbhnnOluALT3qw8DFE41DkjRz/AayJMkwkCQZBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kSJ1EYJFmV5KkkI0k2zvZ4JOl0clKEQZIzgC8Dq4GlwPVJls7uqCTp9HFShAFwBTBSVc9W1ZvAvcCaWR6TJJ025sz2AJoFwIGu/VHgymM7JdkAbGi730/y1ADGNp0uAP56tgcxYM55QPLFQT/jO/hzPjX87fEOnCxh0Jeq2gxsnu1xTFWS4apaNtvjGCTnfHpwzqe+k+U20UHgoq79ha0mSRqAkyUM9gFLkixOciZwHbBjlsckSaeNk+I2UVW9leTTwC7gDGBLVe2f5WHNhFP2FtcJcM6nB+d8iktVzfYYJEmz7GS5TSRJmkWGgSTJMJgOST6Y5JGu7bUkv5rkw0n+MsljSf5nknPHOX9ukvuSPJnkiSQfGfQcJmsa5vyvkuxP8t0kX0vy3kHPYSp6jbstfHiw/SmVr7dFEL3O/Wzr81SSqwc99qmY6nyTfCzJQ+2/g4eSXDUb45+KE/kZt/P/VpLvJ/m1QY77hFWV2zRudD4Af5HOlzv2AX+/1X8ZuG2cc7YC/7y1zwTmzvY8ZnLOdL5k+BxwdtvfDvzSbM+jj3n2HHd7vK7V/gvwL3qcuxT4K+AsYDHwDHDGbM9pBud7KfCB1r4YODjb85npOXdd4z7g94Ffm+35TGbzncH0WwE8U1XPAz8H/Fmr7wZ+8djOSc4DfgG4G6Cq3qyqVwYz1GkzqTk3c4Czk8wBfhL4PzM+yulx7LhfAK6i8wsAOsG+tsd5a4B7q+qNqnoOGKHzZ1hOdlOab1U9XFVHf6b72zXOmvnhToup/oxJspZOmJxyqyENg+l3HfC11t7P239j6VO884t1Ry0GxoDfTfJwkq8kOWfmhzmtJjXnqjoI/Cfgf9P5H+3VqvqTAYzzhPQaN/AQ8EpVvdW6jdJ5dXmsXn9ypVe/k8YJzrfbLwLfqao3Zmqs0+VE5pzkp4BfBz43mNFOL8NgGrX7iJ+g8xYROrdJbk7yEPDTwJs9TpsDXAbcVVWXAj8ATpk/4T2VOSeZRycwFgMfAM5J8k8HM+Kp6zVuYNWsDmoGTcd8k3wI+CLwK9M+wBlwgnP+TeCOqvr+zIxuZp0UXzr7MbKaziuglwCq6klgJUCSnwM+3uOcUWC0qh5s+/dxCoUBU5vzPwKeq6qx1u8PgL8D/NeBjHjqeo37o8DcJHPaK8fx/pTKqfgnV05kviRZCHwDWFdVzwxozCfqROZ8JfDJJP8RmAv8TZIfVtVvD2boJ8Z3BtPret6+XUKS97fH9wC/QeeDp3eoqheBA0k+2EorgMdnfqjTZtJzpvMWfHmSn0wSOnN+YgBjPVG9xv048C3gk63PeuD+HufuAK5LclaSxcAS4NsDGPOJmPJ8k8wF/gjYWFV/MZjhTospz7mq/l5VLaqqRcBvAf/hVAkCwNVE07XReTv5MnBeV+0zwP9q2+28/Y3vDwA7u/pdAgwDjwL/A5g32/MZwJw/BzwJfBf4KnDWbM+nzzm/a9zAz9D5xT5C53bZWa3vJ4Bbu879d3RWET0FrJ7tuczkfOm8EPgB8EjX9v7Zns9M/4y7rvGbnGKrifxzFJIkbxNJkgwDSRKGgSQJw0CShGEgScIwkCRhGEiSgP8HK0YCkouTmwEAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "assert all(df['len'] <= ARGS['max_width'])"
      ],
      "metadata": {
        "id": "TY-qUq68Kf77"
      },
      "execution_count": 94,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df['dna'] = df['dna'].str.pad(width = ARGS['max_width'], side = 'both', fillchar = 'N')\n",
        "df['dna'] = df['dna'] + df['dna'].apply(lambda x: str(Seq(x).reverse_complement())).astype('string')"
      ],
      "metadata": {
        "id": "Mh7hqqG5K1J8"
      },
      "execution_count": 95,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 96,
      "metadata": {
        "id": "siz7Lbd3jgYG"
      },
      "outputs": [],
      "source": [
        "#input_dim = int(5) # A,C,G,T,N\n",
        "input_dim = int(6) # A,C,G,T,N,M\n",
        "n_positions = ARGS['max_width'] * 2"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "n_positions = int(ARGS['max_width'] * 2)\n",
        "print('# positions=%d' % (n_positions))"
      ],
      "metadata": {
        "id": "o7kPkdY3K8X0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fee2a033-957a-4067-c0a3-9c0366384752"
      },
      "execution_count": 97,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "# positions=200\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dna = np.empty((0, n_positions), np.uint8)\n",
        "for x in np.array_split(df['dna'], 10): # split data into chunks\n",
        "  y = np.array(x.apply(list))\n",
        "  y = np.vstack(y)\n",
        "  y = np.vectorize(ARGS['alphabets'].get)(y)\n",
        "  y = y.astype(np.uint8)\n",
        "  print(y.shape)\n",
        "  dna = np.append(dna, y, axis = 0)"
      ],
      "metadata": {
        "id": "lREKXNHuLDA9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d57ff3c1-68a7-4544-e3fa-29c83ca83187"
      },
      "execution_count": 98,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(7111, 200)\n",
            "(7111, 200)\n",
            "(7111, 200)\n",
            "(7110, 200)\n",
            "(7110, 200)\n",
            "(7110, 200)\n",
            "(7110, 200)\n",
            "(7110, 200)\n",
            "(7110, 200)\n",
            "(7110, 200)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "base = np.zeros(dna.shape, dtype = np.uint8)\n",
        "base[dna == 4] = 0 #\n",
        "base[(dna == 0) | (dna == 2)] = 1 # A/G\n",
        "base[(dna == 1) | (dna == 3)] = 2 # C/T"
      ],
      "metadata": {
        "id": "CRrmdcsSzlj4"
      },
      "execution_count": 99,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "ds = tf.data.Dataset.from_tensor_slices({\n",
        "  'seq' : dna,\n",
        "  'expression' : tf.zeros([len(df)]),\n",
        "  'base' : base,\n",
        "})\n",
        "ds.element_spec"
      ],
      "metadata": {
        "id": "MLaEYm4PLMp1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d2d8494a-07f5-4f41-d749-477b53be2f9c"
      },
      "execution_count": 100,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'seq': TensorSpec(shape=(200,), dtype=tf.uint8, name=None),\n",
              " 'expression': TensorSpec(shape=(), dtype=tf.float32, name=None),\n",
              " 'base': TensorSpec(shape=(200,), dtype=tf.uint8, name=None)}"
            ]
          },
          "metadata": {},
          "execution_count": 100
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Prediction"
      ],
      "metadata": {
        "id": "9FnLlgwJKQ7-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model.load_weights(ARGS['local_checkpoint_dir'])"
      ],
      "metadata": {
        "id": "GJhPSCiRKPX7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "50592492-de53-47f0-aac3-c24129cbfb4d"
      },
      "execution_count": 102,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.training.tracking.util.CheckpointLoadStatus at 0x7fe868978310>"
            ]
          },
          "metadata": {},
          "execution_count": 102
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "expression = []\n",
        "pbar = keras.utils.Progbar(len(ds.batch(ARGS['batch_size'])))\n",
        "for step, batch in enumerate(ds.batch(ARGS['batch_size'])):\n",
        "  pbar.update(step)\n",
        "  expression_pred, _ = model(batch)\n",
        "  expression.append(expression_pred)\n",
        "expression = tf.concat(expression, axis = 0)"
      ],
      "metadata": {
        "id": "JMnvDzXWtrVs",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f39ec8f0-b945-47fd-9f26-2e1c7d327488"
      },
      "execution_count": 103,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "138/139 [============================>.] - ETA: 0s"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df['expression'] = expression.numpy()"
      ],
      "metadata": {
        "id": "42FU_zm_uYrW"
      },
      "execution_count": 104,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('mean predicted expression: {}'.format(df['expression'].mean()))\n",
        "print('std predicted expression: {}'.format(df['expression'].std()))"
      ],
      "metadata": {
        "id": "pM6G_VO0GsAL",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "adf5bb9f-3f95-4e23-a811-c1e2e8e44047"
      },
      "execution_count": 105,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "mean predicted expression: 0.29278337955474854\n",
            "std predicted expression: 1.0623327493667603\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "p = plt.hist(df['expression'], bins = 100)"
      ],
      "metadata": {
        "id": "l894HCQ7OVj8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "outputId": "d9c567bf-d352-4517-fdab-8a4c319fe583"
      },
      "execution_count": 106,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAR4klEQVR4nO3df6ykV33f8fcndnAKUbDBW8fZH1mrWRG5iBb3ynZFVTnZ1PgHZUkViFEVNs5WK1TTkIIENpayUhCSEVEciFJLG9bFliwMAiqvEitma0BOpdr1mhCDbRLfOja7Kxsv+EfSOKm74ds/5pgMl7u+e2fmzty55/2SRvd5znNm5vvI1mfOnjnPM6kqJEl9+JFZFyBJmh5DX5I6YuhLUkcMfUnqiKEvSR05fdYFvJyzzz67tm/fPusyJGmuPPDAA9+pqk3LHVvXob99+3YOHz486zIkaa4keeJkx1ac3klyc5Knk3xjmWPvT1JJzm77SfKJJItJHkxywVDf3UkebY/do56MJGl0pzKn/yngsqWNSbYClwLfGmq+HNjRHnuBm1rf1wD7gIuAC4F9Sc4ap3BJ0uqtGPpVdQ/wzDKHbgQ+AAxf0rsLuLUG7gXOTHIu8GbgUFU9U1XPAodY5oNEkrS2Rlq9k2QXcKyq/mzJoc3AkaH9o63tZO3LvfbeJIeTHD5+/Pgo5UmSTmLVoZ/klcCHgN+cfDlQVfuraqGqFjZtWvbLZ0nSiEYZ6f8T4Dzgz5I8DmwBvprkJ4FjwNahvlta28naJUlTtOrQr6qvV9U/rqrtVbWdwVTNBVX1FHAQeFdbxXMx8HxVPQncBVya5Kz2Be6lrU2SNEWnsmTz08D/BF6X5GiSPS/T/U7gMWAR+APgPwJU1TPAh4H72+O3WpskaYqynu+nv7CwUF6cJUmrk+SBqlpY7ti6viJXktbK9mv/6Af2H7/hyhlVMl3ecE2SOmLoS1JHDH1J6oihL0kdMfQlqSOu3pGmZHi1SC8rRbT+ONKXpI4Y+pLUEad3pDnmlJFWy9CXNFN+cE2XoS/NgY0QjBvhHDYCQ19aR8YJxkmG6kYK6I10LpNg6EsTdiohs/RmX9K0GPrSnPEDY2308i8CQ1/S1PnBNTuGvrSG5inc5qlWjc7QlzagXqYqtHqGviQtsZE/NL0NgyR1xNCXpI6sOL2T5GbgLcDTVfX61vYx4N8CLwL/G7i6qp5rx64D9gB/D/x6Vd3V2i8DPg6cBnyyqm6Y+NlIG8ikvlid5Q+A++Xw+nMqI/1PAZctaTsEvL6q3gD8BXAdQJLzgauAf9qe81+SnJbkNOD3gcuB84F3tr6SpClaMfSr6h7gmSVtX6yqE233XmBL294F3F5V/7eq/hJYBC5sj8WqeqyqXgRub30lSVM0idU7vwZ8pm1vZvAh8JKjrQ3gyJL2i5Z7sSR7gb0A27Ztm0B5kiZlXla1OK10cmOFfpLrgRPAbZMpB6pqP7AfYGFhoSb1upIG5iW4tTZGDv0kv8rgC96dVfVSOB8Dtg5129LaeJl2ae45stS8GGnJZluJ8wHgrVX1wtChg8BVSc5Ich6wA/hfwP3AjiTnJXkFgy97D45XuiRptU5lyeangUuAs5McBfYxWK1zBnAoCcC9VfXuqnooyWeBhxlM+1xTVX/fXuc9wF0MlmzeXFUPrcH5SJpjTj2tvRVDv6reuUzzgZfp/xHgI8u03wncuarqJGmCnIbzilxJ6oo3XJOW0cuvX63VVb9avxzpS1JHHOlLzamMVh3Rat4Z+pJG4gfgfDL0Jb0sw31jcU5fkjpi6EtSRwx9SeqIoS9JHfGLXGkFfpGpjcSRviR1xNCXpI4Y+pLUEef0JW0IfvdyahzpS1JHHOmra44O1RtH+pLUEUNfkjpi6EtSR5zTl7QuncpPVmr1HOlLUkdWDP0kNyd5Osk3htpek+RQkkfb37Nae5J8IslikgeTXDD0nN2t/6NJdq/N6Uja6LZf+0fff2j1TmWk/yngsiVt1wJ3V9UO4O62D3A5sKM99gI3weBDAtgHXARcCOx76YNCWiuGg/TDVgz9qroHeGZJ8y7glrZ9C/C2ofZba+Be4Mwk5wJvBg5V1TNV9SxwiB/+IJEkrbFRv8g9p6qebNtPAee07c3AkaF+R1vbydp/SJK9DP6VwLZt20YsTzo5R/7q2dird6qqktQkimmvtx/YD7CwsDCx15U0v/ygnpxRV+98u03b0P4+3dqPAVuH+m1pbSdrlyRN0aihfxB4aQXObuCOofZ3tVU8FwPPt2mgu4BLk5zVvsC9tLVJkqZoxemdJJ8GLgHOTnKUwSqcG4DPJtkDPAG8o3W/E7gCWAReAK4GqKpnknwYuL/1+62qWvrlsCRpja0Y+lX1zpMc2rlM3wKuOcnr3AzcvKrqpAlxTlij2mhXBntFriR1xHvvaMNwNN8f/5uvniN9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOuL99DXXvJ+6tDqO9CWpI4a+JHXE0Jekjow1p5/kPwP/ASjg68DVwLnA7cBrgQeAX6mqF5OcAdwK/Avgu8AvV9Xj47y/+jE8d//4DVfOsBJpvo080k+yGfh1YKGqXg+cBlwFfBS4sap+BngW2NOesgd4trXf2PpJkqZo3Omd04F/lOR04JXAk8DPA59rx28B3ta2d7V92vGdSTLm+0uSVmHk6Z2qOpbkt4FvAX8LfJHBdM5zVXWidTsKbG7bm4Ej7bknkjzPYAroO8Ovm2QvsBdg27Zto5anDcxlmtLoxpneOYvB6P084KeAVwGXjVtQVe2vqoWqWti0adO4LydJGjLO9M4vAH9ZVcer6v8BXwDeBJzZpnsAtgDH2vYxYCtAO/5qBl/oSpKmZJzQ/xZwcZJXtrn5ncDDwJeBX2p9dgN3tO2DbZ92/EtVVWO8vyRplUYO/aq6j8EXsl9lsFzzR4D9wAeB9yVZZDBnf6A95QDw2tb+PuDaMeqWJI1grHX6VbUP2Lek+THgwmX6/h3w9nHeT5I0Hq/IlaSOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOjLWvXekSfO3cKW1ZehrJk4l3P2FLGnynN6RpI440tfUOHKXZs+RviR1xJG+1pSje2l9MfQ1c34wSNPj9I4kdcTQl6SOOL0jSadoI1w8ONZIP8mZST6X5JtJHknyL5O8JsmhJI+2v2e1vknyiSSLSR5McsFkTkGSdKrGnd75OPDHVfWzwD8DHgGuBe6uqh3A3W0f4HJgR3vsBW4a870lSas0cugneTXwr4EDAFX1YlU9B+wCbmndbgHe1rZ3AbfWwL3AmUnOHfX9JUmrN85I/zzgOPBfk/xpkk8meRVwTlU92fo8BZzTtjcDR4aef7S1SZKmZJzQPx24ALipqt4I/A3/MJUDQFUVUKt50SR7kxxOcvj48eNjlCdJWmqc0D8KHK2q+9r+5xh8CHz7pWmb9vfpdvwYsHXo+Vta2w+oqv1VtVBVC5s2bRqjPEnSUiOHflU9BRxJ8rrWtBN4GDgI7G5tu4E72vZB4F1tFc/FwPND00CSpCkYd53+fwJuS/IK4DHgagYfJJ9Nsgd4AnhH63sncAWwCLzQ+kqSpmis0K+qrwELyxzauUzfAq4Z5/0kSePxNgyS1BFDX5I6YuhLUkcMfUnqiHfZ1ET4QyjSfHCkL0kdcaSvkTm6l+aPI31J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEZdsalVcpinNN0NfkkYwPAB6/IYrZ1jJ6ji9I0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSR8YO/SSnJfnTJH/Y9s9Lcl+SxSSfSfKK1n5G219sx7eP+96SpNWZxEj/vcAjQ/sfBW6sqp8BngX2tPY9wLOt/cbWT5I0RWOFfpItwJXAJ9t+gJ8HPte63AK8rW3vavu04ztbf0nSlIw70v9d4APA99r+a4HnqupE2z8KbG7bm4EjAO34863/D0iyN8nhJIePHz8+ZnmSpGEj33snyVuAp6vqgSSXTKqgqtoP7AdYWFioSb2uRudN1qSNY5wbrr0JeGuSK4AfA34C+DhwZpLT22h+C3Cs9T8GbAWOJjkdeDXw3THeX5K0SiNP71TVdVW1paq2A1cBX6qqfw98Gfil1m03cEfbPtj2ace/VFWO5CVpitZinf4HgfclWWQwZ3+gtR8AXtva3wdcuwbvLUl6GRO5n35VfQX4Stt+DLhwmT5/B7x9Eu8nSRqNV+RKUkcMfUnqiKEvSR3xN3K1LNfmSxuTI31J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIV+RK0piGr2B//IYrZ1jJygx9fZ+3XpA2Pqd3JKkjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkdGDv0kW5N8OcnDSR5K8t7W/pokh5I82v6e1dqT5BNJFpM8mOSCSZ2EJOnUjDPSPwG8v6rOBy4GrklyPnAtcHdV7QDubvsAlwM72mMvcNMY7y1JGsHIF2dV1ZPAk237r5M8AmwGdgGXtG63AF8BPtjab62qAu5NcmaSc9vraEa8IEvqy0Tm9JNsB94I3AecMxTkTwHntO3NwJGhpx1tbUtfa2+Sw0kOHz9+fBLlSZKasUM/yY8Dnwd+o6r+avhYG9XXal6vqvZX1UJVLWzatGnc8iRJQ8YK/SQ/yiDwb6uqL7Tmbyc5tx0/F3i6tR8Dtg49fUtrkyRNychz+kkCHAAeqarfGTp0ENgN3ND+3jHU/p4ktwMXAc87ny9po1nvd9wc5y6bbwJ+Bfh6kq+1tg8xCPvPJtkDPAG8ox27E7gCWAReAK4e470lSSMYZ/XO/wByksM7l+lfwDWjvp8kaXxekStJHTH0Jakj/nJWh7wgS+qXI31J6oihL0kdMfQlqSOGviR1xC9yJWmNrMercw39TrhiRxI4vSNJXTH0Jakjhr4kdcTQl6SO+EXuBrMeVwtIWj8M/Q3MFTuSlnJ6R5I6YuhLUkcMfUnqiHP6c855e0mrYejPCcNdmm/rZWWdob+OGfSSJs05fUnqyNRH+kkuAz4OnAZ8sqpumHYN64GjeEmzMNXQT3Ia8PvAvwGOAvcnOVhVD0+zjrVwsvk6w13SUrOc35/2SP9CYLGqHgNIcjuwC1gXoT+pgDboJZ2qk+XFWn0YTDv0NwNHhvaPAhcNd0iyF9jbdv9Pkj+fUm0nczbwnRnXMEmez/rm+axvUzuffHSsp//0yQ6su9U7VbUf2D/rOl6S5HBVLcy6jknxfNY3z2d92wjnM+3VO8eArUP7W1qbJGkKph369wM7kpyX5BXAVcDBKdcgSd2a6vROVZ1I8h7gLgZLNm+uqoemWcMI1s1U04R4Puub57O+zf35pKpmXYMkaUq8IleSOmLoS1JHDP0VJPlwkgeTfC3JF5P81KxrGkeSjyX5Zjun/5bkzFnXNK4kb0/yUJLvJZnL5XRJLkvy50kWk1w763rGleTmJE8n+casa5mEJFuTfDnJw+3/tffOuqZRGfor+1hVvaGq/jnwh8BvzriecR0CXl9VbwD+ArhuxvVMwjeAfwfcM+tCRjF0e5LLgfOBdyY5f7ZVje1TwGWzLmKCTgDvr6rzgYuBa+b1v5Ghv4Kq+quh3VcBc/3Nd1V9sapOtN17GVwrMdeq6pGqmvWV2+P4/u1JqupF4KXbk8ytqroHeGbWdUxKVT1ZVV9t238NPMLgDgNzZ91dkbseJfkI8C7geeDnZlzOJP0a8JlZF6GVb0+i9SPJduCNwH0zLmUkhj6Q5L8DP7nMoeur6o6quh64Psl1wHuAfVMtcJVWOp/W53oG/2S9bZq1jepUzklaa0l+HPg88BtLZgHmhqEPVNUvnGLX24A7Weehv9L5JPlV4C3AzpqTCzVW8d9oHnl7kjmQ5EcZBP5tVfWFWdczKuf0V5Bkx9DuLuCbs6plEtqP2HwAeGtVvTDregR4e5J1L0mAA8AjVfU7s65nHF6Ru4IknwdeB3wPeAJ4d1XN7SgsySJwBvDd1nRvVb17hiWNLckvAr8HbAKeA75WVW+eaVGrlOQK4Hf5h9uTfGS2FY0nyaeBSxjcivjbwL6qOjDTosaQ5F8BfwJ8nUEWAHyoqu6cXVWjMfQlqSNO70hSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1JH/D4qsCsH8gmfWAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Prepare submission file"
      ],
      "metadata": {
        "id": "19E3HT1ESJ4A"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model.summary()"
      ],
      "metadata": {
        "id": "1k--o7NCg8oz",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fed10846-0229-4e83-edd4-04bc8b17b2fd"
      },
      "execution_count": 110,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"regressor_1\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "sequence_mask_layer_2 (Seque multiple                  0 (unused)\n",
            "_________________________________________________________________\n",
            "embedding_2 (Embedding)      multiple                  102400    \n",
            "_________________________________________________________________\n",
            "embedding_3 (Embedding)      multiple                  1024      \n",
            "_________________________________________________________________\n",
            "dense_26 (Dense)             multiple                  1024      \n",
            "_________________________________________________________________\n",
            "dense_27 (Dense)             multiple                  31232     \n",
            "_________________________________________________________________\n",
            "sequential_29 (Sequential)   (None, 232, 512)          47286272  \n",
            "_________________________________________________________________\n",
            "dropout_49 (Dropout)         multiple                  0         \n",
            "_________________________________________________________________\n",
            "dense_44 (Dense)             multiple                  513       \n",
            "_________________________________________________________________\n",
            "dense_45 (Dense)             multiple                  3078      \n",
            "=================================================================\n",
            "Total params: 47,425,543\n",
            "Trainable params: 47,425,543\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Modified from https://github.com/de-Boer-Lab/DREAM-2022/blob/main/How%20to%20submit%20predictions%20to%20the%20public%20leaderboard.pdf"
      ],
      "metadata": {
        "id": "Mr9rV0nDWx2u"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "os.system('wget -cx ' + ARGS['remote_sample_submission_file'])"
      ],
      "metadata": {
        "id": "BYr7HS0iSL32",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "58235dee-3aeb-4556-b890-9b5e4798fbe5"
      },
      "execution_count": 111,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "--2022-08-07 17:32:22--  https://raw.githubusercontent.com/de-Boer-Lab/DREAM-2022/main/sample_submission.json\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.110.133, 185.199.108.133, 185.199.109.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.110.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 416 Range Not Satisfiable\n",
            "\n",
            "    The file is already fully retrieved; nothing to do.\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {},
          "execution_count": 111
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "with open(ARGS['local_sample_submission_file'], 'r') as f:\n",
        "  ground = json.load(f)"
      ],
      "metadata": {
        "id": "TM86DEDHSRPg"
      },
      "execution_count": 112,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "indices = np.array([int(indice) for indice in list(ground.keys())])"
      ],
      "metadata": {
        "id": "KBD_KIirSYad"
      },
      "execution_count": 113,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "PRED_DATA = OrderedDict()\n",
        "Y_Pred = df['expression'].to_numpy()"
      ],
      "metadata": {
        "id": "KM5aQCeYSYf7"
      },
      "execution_count": 114,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for i in indices:\n",
        "  PRED_DATA[str(i)] = float(Y_Pred[i])"
      ],
      "metadata": {
        "id": "-JOyfzpgTBT9"
      },
      "execution_count": 115,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "local_prediction_dir = re.sub('(.+/)..+', '\\\\1', ARGS['local_prediction_file'])\n",
        "if not os.path.exists(local_prediction_dir):\n",
        "  os.makedirs(local_prediction_dir)\n",
        "  print('creating {}'.format(local_prediction_dir))"
      ],
      "metadata": {
        "id": "94ub4d1qVlv0"
      },
      "execution_count": 116,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "with open(ARGS['local_prediction_file'], 'w') as f:\n",
        "  json.dump(PRED_DATA, f)\n",
        "print('writing {}'.format(ARGS['local_prediction_file']))"
      ],
      "metadata": {
        "id": "tVP8DI4HTTNY",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "83630539-6df7-4bc8-f623-7f875ad15666"
      },
      "execution_count": 117,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "writing ./s3.msi.umn.edu/gongx030/projects/dream_PGE/predictions/m20220727e/pred.json\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Only if the notebook runs on MSI, otherwise, ignore this cell."
      ],
      "metadata": {
        "id": "uZWiJ3TPNcdC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!s3cmd sync {ARGS['local_prediction_file']} {ARGS['s3_prediction_file']}\n",
        "!s3cmd setacl -P {ARGS['s3_prediction_file']}"
      ],
      "metadata": {
        "id": "Mc91PmIJWRoK"
      },
      "execution_count": 118,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('prediction file: {}'.format(ARGS['remote_prediction_file']))"
      ],
      "metadata": {
        "id": "CUsACX1WWiND",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a50324f1-2df8-4458-b836-9da188e23b11"
      },
      "execution_count": 119,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "prediction file: https://s3.msi.umn.edu/gongx030/projects/dream_PGE/predictions/m20220727e/pred.json\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df[['dna0', 'expression']].to_csv(ARGS['local_prediction_tsv_file'], header = False, sep = '\\t', index = False)"
      ],
      "metadata": {
        "id": "SiZ9hbVXaAx3"
      },
      "execution_count": 120,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!s3cmd sync {ARGS['local_prediction_tsv_file']} {ARGS['s3_prediction_tsv_file']}\n",
        "!s3cmd setacl -P {ARGS['s3_prediction_tsv_file']}"
      ],
      "metadata": {
        "id": "CkL8h8FGadY6"
      },
      "execution_count": 121,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('prediction file(tsv): {}'.format(ARGS['remote_prediction_tsv_file']))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sQX_3risahNN",
        "outputId": "1405c624-34b0-4903-a418-f2439d229979"
      },
      "execution_count": 122,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "prediction file(tsv): https://s3.msi.umn.edu/gongx030/projects/dream_PGE/predictions/m20220727e/pred.tsv\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# SessionInfo"
      ],
      "metadata": {
        "id": "h6z-1F1D4owK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "session_info.show()"
      ],
      "metadata": {
        "id": "53wVoagb4tD8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "74fa0970-dc30-492f-9cc9-76ce90418ec9"
      },
      "execution_count": 123,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "<details>\n",
              "<summary>Click to view session information</summary>\n",
              "<pre>\n",
              "-----\n",
              "Bio                         1.79\n",
              "keras                       2.6.0\n",
              "matplotlib                  3.5.2\n",
              "numpy                       1.19.5\n",
              "pandas                      1.3.5\n",
              "scipy                       1.7.3\n",
              "session_info                1.0.0\n",
              "sklearn                     1.0.2\n",
              "tensorflow                  2.6.0\n",
              "tensorflow_probability      0.14.1\n",
              "-----\n",
              "</pre>\n",
              "<details>\n",
              "<summary>Click to view modules imported as dependencies</summary>\n",
              "<pre>\n",
              "2f7ece400a652629565c523b34ee61b04afa385c    NA\n",
              "PIL                                         9.2.0\n",
              "absl                                        NA\n",
              "astunparse                                  1.6.3\n",
              "backcall                                    0.2.0\n",
              "beta_ufunc                                  NA\n",
              "binom_ufunc                                 NA\n",
              "cached_property                             1.5.2\n",
              "certifi                                     2022.06.15\n",
              "cffi                                        1.15.1\n",
              "charset_normalizer                          2.1.0\n",
              "cloudpickle                                 2.1.0\n",
              "cycler                                      0.10.0\n",
              "cython_runtime                              NA\n",
              "dateutil                                    2.8.2\n",
              "debugpy                                     1.6.2\n",
              "decorator                                   5.1.1\n",
              "defusedxml                                  0.7.1\n",
              "entrypoints                                 0.4\n",
              "flatbuffers                                 NA\n",
              "gast                                        NA\n",
              "google                                      NA\n",
              "h5py                                        3.1.0\n",
              "idna                                        3.3\n",
              "importlib_metadata                          NA\n",
              "ipykernel                                   6.15.1\n",
              "ipython_genutils                            0.2.0\n",
              "jedi                                        0.18.1\n",
              "jinja2                                      3.1.2\n",
              "joblib                                      1.1.0\n",
              "keras_preprocessing                         1.1.2\n",
              "kiwisolver                                  1.4.4\n",
              "markupsafe                                  2.1.1\n",
              "matplotlib_inline                           NA\n",
              "mpl_toolkits                                NA\n",
              "nbinom_ufunc                                NA\n",
              "opt_einsum                                  v3.3.0\n",
              "packaging                                   21.3\n",
              "parso                                       0.8.3\n",
              "pexpect                                     4.8.0\n",
              "pickleshare                                 0.7.5\n",
              "pkg_resources                               NA\n",
              "prompt_toolkit                              3.0.30\n",
              "psutil                                      5.9.1\n",
              "ptyprocess                                  0.7.0\n",
              "pydev_ipython                               NA\n",
              "pydevconsole                                NA\n",
              "pydevd                                      2.8.0\n",
              "pydevd_file_utils                           NA\n",
              "pydevd_plugins                              NA\n",
              "pydevd_tracing                              NA\n",
              "pygments                                    2.12.0\n",
              "pyparsing                                   3.0.9\n",
              "pytz                                        2022.1\n",
              "requests                                    2.28.1\n",
              "setuptools                                  63.2.0\n",
              "six                                         1.15.0\n",
              "storemagic                                  NA\n",
              "tensorboard                                 2.9.1\n",
              "termcolor                                   1.1.0\n",
              "threadpoolctl                               3.1.0\n",
              "tmp2xi5csl9                                 NA\n",
              "tmp9rv1w0t_                                 NA\n",
              "tmpbaeatp84                                 NA\n",
              "tornado                                     6.2\n",
              "traitlets                                   5.3.0\n",
              "tree                                        0.1.7\n",
              "typing_extensions                           NA\n",
              "urllib3                                     1.26.11\n",
              "wcwidth                                     0.2.5\n",
              "wrapt                                       1.12.1\n",
              "zipp                                        NA\n",
              "zmq                                         23.2.0\n",
              "</pre>\n",
              "</details> <!-- seems like this ends pre, so might as well be explicit -->\n",
              "<pre>\n",
              "-----\n",
              "IPython             7.34.0\n",
              "jupyter_client      7.3.4\n",
              "jupyter_core        4.11.1\n",
              "notebook            6.4.12\n",
              "-----\n",
              "Python 3.7.12 | packaged by conda-forge | (default, Oct 26 2021, 06:08:21) [GCC 9.4.0]\n",
              "Linux-3.10.0-1160.71.1.el7.x86_64-x86_64-with-centos-7.9.2009-Core\n",
              "-----\n",
              "Session information updated at 2022-08-07 17:32\n",
              "</pre>\n",
              "</details>"
            ]
          },
          "metadata": {},
          "execution_count": 123
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "tFWCf4x-5bTE"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}