Dropout.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import struct as st\n",
    "import matplotlib.pyplot as plt\n",
    "import copy\n",
    "%matplotlib inline\n",
    "np.set_printoptions(threshold=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "(x_train,y_train), (x_test,y_test) = tf.keras.datasets.mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0xb3daf88d0>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAN+UlEQVR4nO3dX4hcdZrG8edRZ4jGkeimjUFlMxtzI6tR6QRBGV0kQ/SmlagY4p8F2Ygm4GBAxYXohRdBdAaFVYz/JiOzjiOjmAvdnayOyIiMdkI0UXE10qIx6XSIMAmCWZN3L/pk6Ildv2qrTv1J3u8Hmqo6b506L4c8OVXnV6d+jggBOPod0+sGAHQHYQeSIOxAEoQdSIKwA0kc182NzZw5M+bMmdPNTQKpjIyMaPfu3Z6s1lbYbS+W9LCkYyU9GRFrSs+fM2eOhoeH29kkgILBwcGGtZbfxts+VtJ/SLpc0tmSlto+u9XXA9BZ7XxmXyjp04j4LCL2S/qdpKF62gJQt3bCfrqkLyY8/rJa9ndsL7c9bHt4bGysjc0BaEfHz8ZHxNqIGIyIwYGBgU5vDkAD7YR9u6QzJzw+o1oGoA+1E/Z3Jc2z/VPbP5Z0naT19bQFoG4tD71FxHe2V0r6b40PvT0dER/U1hmAWrU1zh4Rr0h6paZeAHQQX5cFkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgibZmcQV6af/+/cX64sWLG9a2bdtWXPe9994r1mfMmFGs96O2wm57RNJeSQckfRcRg3U0BaB+dRzZ/yUidtfwOgA6iM/sQBLthj0k/dH2RtvLJ3uC7eW2h20Pj42Ntbk5AK1qN+wXR8QFki6XtML2zw5/QkSsjYjBiBgcGBhoc3MAWtVW2CNie3W7S9JLkhbW0RSA+rUcdtvTbf/k0H1JP5e0ta7GANSrnbPxsyS9ZPvQ6/xnRPxXLV3hiLF379626iXTp08v1jdu3Fisv/HGGw1r8+fPL657/PHHF+tHopbDHhGfSSrvMQB9g6E3IAnCDiRB2IEkCDuQBGEHkuAS16PAjh07GtYeeeSR4rojIyNtbbvZ8FezS0lLHnrooWJ9y5YtxXpENKzNmzevuO7BgweL9SMRR3YgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIJx9qPAW2+91bD2wAMPdHTb06ZNK9Zvv/32hrUXX3yxuO6qVata6umQ6vLrSa1YsaK47tF4iStHdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgnH2I8Cjjz5arN95550tv/Ydd9xRrM+aNatYv+2224r1E044oWGt2Tj6ggULivXR0dFi/bTTTmtYu+iii4rrHo04sgNJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoyzHwH27dtXrH/zzTcNa2eddVZx3XvvvbdYP/HEE4v1Zvbs2dOwdv/99xfX3blzZ7HebErnxx57rGHtuOPy/dNvemS3/bTtXba3Tlh2iu0Ntj+pbk/ubJsA2jWVt/G/lrT4sGV3S3otIuZJeq16DKCPNQ17RLwp6fD3YkOS1lX310m6sua+ANSs1RN0syLi0ARjOyU1/AK17eW2h20Pj42Ntbg5AO1q+2x8jM+e13AGvYhYGxGDETE4MDDQ7uYAtKjVsI/ani1J1e2u+loC0Amthn29pJuq+zdJermedgB0StPBRtvPSbpU0kzbX0q6V9IaSb+3fbOkzyVd28kms7v22vLufeGFFxrWNm3aVFx39erVxfqaNWuK9W+//bZYL10v/+yzzxbXbfax7+GHHy7Wh4aGivVsmoY9IpY2KF1Wcy8AOoivywJJEHYgCcIOJEHYgSQIO5BEvuv8jkBnnHFGsX7ZZY0HRpoNvTWbNnnp0kaDMeOWLVtWrG/btq1YL2n2E9pLlixp+bUz4sgOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kwzn4EaPazxzNmzGj5tb/44oti/cILLyzWx3+oqDHbDWvNpppetGhRsY4fhiM7kARhB5Ig7EAShB1IgrADSRB2IAnCDiTBOPtRoNm0zL10/fXXN6ytWrWquO5JJ51UdzupcWQHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQYZz8CHDx4sFjfsGFDw1qz683bdcMNNxTr69at6+j2MXVNj+y2n7a9y/bWCcvus73d9ubq74rOtgmgXVN5G/9rSYsnWf6riDiv+nul3rYA1K1p2CPiTUl7utALgA5q5wTdStvvV2/zT270JNvLbQ/bHh4bG2tjcwDa0WrYH5M0V9J5knZIeqjREyNibUQMRsTgwMBAi5sD0K6Wwh4RoxFxICIOSnpC0sJ62wJQt5bCbnv2hIdXSdra6LkA+kPTcXbbz0m6VNJM219KulfSpbbPkxSSRiTd0sEe07v11luL9SeffLJhrfS77XXo9OujPk3DHhFLJ1n8VAd6AdBBfF0WSIKwA0kQdiAJwg4kQdiBJLjEtQv27t1brD///PPF+hNPPFGsl4a/LrnkkuK6CxYsKNYffPDBYv2rr74q1tE/OLIDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKMs3fBxo0bi/VbbmnvCuHSOPyyZcuK67799tvFerNx9vnz5xfr6B8c2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcbZa/Dxxx8X60uWLGnr9ZuN059zzjkNa/v27Suuu2LFipZ6OmTu3LltrY/u4cgOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kwzl6DV199tVj/+uuvi/WrrrqqWD///POL9QMHDjSsvf7668V19+zZU6xHRLE+e/bsYh39o+mR3faZtv9k+0PbH9i+vVp+iu0Ntj+pbk/ufLsAWjWVt/HfSVoVEWdLulDSCttnS7pb0msRMU/Sa9VjAH2qadgjYkdEbKru75X0kaTTJQ1JWlc9bZ2kKzvVJID2/aATdLbnSDpf0l8kzYqIHVVpp6RZDdZZbnvY9vDY2FgbrQJox5TDbvtESX+Q9IuI+OvEWoyfxZn0TE5ErI2IwYgYHBgYaKtZAK2bUtht/0jjQf9tRLxYLR61Pbuqz5a0qzMtAqhD06E3j88H/JSkjyLilxNK6yXdJGlNdftyRzo8AhxzTPn/zNKUylOpl4bWJOmdd95pWLvmmmuK686cObNYv+uuu4r1oaGhYh39Yyrj7BdJukHSFtubq2X3aDzkv7d9s6TPJV3bmRYB1KFp2CPiz5IaHXouq7cdAJ3C12WBJAg7kARhB5Ig7EAShB1IgktcazA6OtrW+qeeemqxfvXVVxfr69evb3nbzS7PveCCC1p+bfQXjuxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kATj7DU499xz21r/8ccfL9ab/Zxz6ReAVq9eXVy3NN0zji4c2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcbZa9Dst9OfeeaZYn3lypXF+qJFi4r10m/DX3fddcV1kQdHdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IYirzs58p6TeSZkkKSWsj4mHb90n6N0lj1VPviYhXOtVoP5s2bVqxfuONN7ZVB+owlS/VfCdpVURssv0TSRttb6hqv4qIBzvXHoC6TGV+9h2SdlT399r+SNLpnW4MQL1+0Gd223MknS/pL9Wilbbft/207ZMbrLPc9rDt4bGxscmeAqALphx22ydK+oOkX0TEXyU9JmmupPM0fuR/aLL1ImJtRAxGxGDpt9IAdNaUwm77RxoP+m8j4kVJiojRiDgQEQclPSFpYefaBNCupmG3bUlPSfooIn45YfnsCU+7StLW+tsDUJepnI2/SNINkrbY3lwtu0fSUtvnaXw4bkTSLR3pEEAtpnI2/s+SPEkp5Zg6cKTiG3RAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkHBHd25g9JunzCYtmStrdtQZ+mH7trV/7kuitVXX29o8RMenvv3U17N/buD0cEYM9a6CgX3vr174kemtVt3rjbTyQBGEHkuh12Nf2ePsl/dpbv/Yl0VurutJbTz+zA+ieXh/ZAXQJYQeS6EnYbS+2/bHtT23f3YseGrE9YnuL7c22h3vcy9O2d9neOmHZKbY32P6kup10jr0e9Xaf7e3Vvtts+4oe9Xam7T/Z/tD2B7Zvr5b3dN8V+urKfuv6Z3bbx0r6X0mLJH0p6V1JSyPiw6420oDtEUmDEdHzL2DY/pmkfZJ+ExH/XC17QNKeiFhT/Ud5ckTc1Se93SdpX6+n8a5mK5o9cZpxSVdK+lf1cN8V+rpWXdhvvTiyL5T0aUR8FhH7Jf1O0lAP+uh7EfGmpD2HLR6StK66v07j/1i6rkFvfSEidkTEpur+XkmHphnv6b4r9NUVvQj76ZK+mPD4S/XXfO8h6Y+2N9pe3utmJjErInZU93dKmtXLZibRdBrvbjpsmvG+2XetTH/eLk7Qfd/FEXGBpMslrajervalGP8M1k9jp1OaxrtbJplm/G96ue9anf68Xb0I+3ZJZ054fEa1rC9ExPbqdpekl9R/U1GPHppBt7rd1eN+/qafpvGebJpx9cG+6+X0570I+7uS5tn+qe0fS7pO0voe9PE9tqdXJ05ke7qkn6v/pqJeL+mm6v5Nkl7uYS9/p1+m8W40zbh6vO96Pv15RHT9T9IVGj8jv03Sv/eihwZ9/ZOk96q/D3rdm6TnNP627v80fm7jZkn/IOk1SZ9I+h9Jp/RRb89K2iLpfY0Ha3aPertY42/R35e0ufq7otf7rtBXV/YbX5cFkuAEHZAEYQeSIOxAEoQdSIKwA0kQdiAJwg4k8f8/3S/CFebGuwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "image_index = 17\n",
    "print(y_train[image_index])\n",
    "plt.imshow(x_train[image_index],cmap = 'Greys')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train_flattened = np.reshape(x_train,(x_train.shape[0],-1))\n",
    "x_test_flattened = np.reshape(x_test,(x_test.shape[0],-1))\n",
    "y_train = y_train.reshape(y_train.shape[0],1)\n",
    "y_test = y_test.reshape(y_test.shape[0],1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train_flattened = x_train_flattened.astype('float32')\n",
    "x_test_flattened = x_test_flattened.astype('float32')\n",
    "y_train = y_train.astype('float32')\n",
    "y_test = y_test.astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train_flattened = x_train_flattened/255\n",
    "x_test_flattened = x_test_flattened/255\n",
    "X_train = x_train_flattened.T\n",
    "X_test = x_test_flattened.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_labels = 10\n",
    "Y_train = np.zeros((n_labels,y_train.shape[0]))\n",
    "Y_test = np.zeros((n_labels,y_test.shape[0]))\n",
    "\n",
    "for i in range(len(y_train)):\n",
    "    Y_train[int(y_train[i])][i] = 1\n",
    "    \n",
    "for i in range(len(y_test)):\n",
    "    Y_test[int(y_test[i])][i] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def relu(z):\n",
    "    return np.maximum(z,0)\n",
    "\n",
    "def sigmoid(z):\n",
    "    return 1.0/(1.0+np.exp(-z))\n",
    "\n",
    "def tanh(z):\n",
    "    return np.tanh(z)\n",
    "\n",
    "def atan(z):\n",
    "    return np.arctan(z)\n",
    "\n",
    "def softmax(z):\n",
    "    return np.divide(np.exp(z-np.max(z,axis=0)),np.sum(np.exp(z-np.max(z,axis=0)),axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def activation_function(z,activation):\n",
    "    if activation == 'relu':\n",
    "        return relu(z)\n",
    "    elif activation == 'tanh':\n",
    "        return tanh(z)\n",
    "    elif activation == 'atan':\n",
    "        return atan(z)\n",
    "    elif activation == 'sigmoid':\n",
    "        return sigmoid(z)\n",
    "    elif activation == 'softmax':\n",
    "        return softmax(z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def initialize_parameters(X,nn_architecture,nn_activations,learning_rate,**kwargs):\n",
    "    cache = {}\n",
    "    h_params = {'nn_arch': nn_architecture,\n",
    "                'lr': learning_rate,\n",
    "                'nn_acti': nn_activations,\n",
    "                'reg' : kwargs.get('reg',None),\n",
    "                'lam': kwargs.get('lam',1),\n",
    "                'kp': kwargs.get('kp',1)}\n",
    "    W = np.random.rand(nn_architecture[0],X.shape[0]) * 0.01\n",
    "    B = np.random.rand(nn_architecture[0],1)\n",
    "    cache.update({'W'+str(0): W, 'B'+str(0):B})\n",
    "    for i in range(1,len(nn_architecture)):\n",
    "        W = np.random.rand(nn_architecture[i],nn_architecture[i-1]) * 0.01\n",
    "        B = np.random.rand(nn_architecture[i],1)\n",
    "        cache.update({'W'+str(i): W, 'B'+str(i):B})\n",
    "        \n",
    "    return cache,h_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward_nn_network(X,cache,h_params):\n",
    "    A = X\n",
    "    if h_params['reg'] != 'dropout':\n",
    "        for i in range(len(h_params['nn_arch'])):\n",
    "            X = A\n",
    "            Z = np.matmul(cache['W'+str(i)],X) + cache['B'+str(i)]\n",
    "            A = activation_function(Z,h_params['nn_acti'][i])\n",
    "            cache.update({'A'+str(i): A,\n",
    "                         'Z'+str(i): Z})\n",
    "    else:\n",
    "        for i in range(len(h_params['nn_arch'])):\n",
    "            X = A\n",
    "            mask = (np.random.standard_normal(cache['W'+str(i)].shape) >= h_params['kp']) * 1\n",
    "            Z = np.matmul(np.multiply(cache['W'+str(i)],mask),X) + cache['B'+str(i)]\n",
    "            A = activation_function(Z,h_params['nn_acti'][i]) / (1-h_params['kp'])\n",
    "            cache.update({'A'+str(i): A,\n",
    "                         'Z'+str(i): Z,\n",
    "                         'D'+str(i): mask})\n",
    "    return A,cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cost_function(Y_pred,Y_true,cost):\n",
    "    if cost == 'mae':\n",
    "        return 1.0/Y_pred.shape[1] * np.sum(np.abs(Y_pred-Y_true),axis = 1,keepdims = True)\n",
    "    elif cost == 'mse':\n",
    "        return 0.5/Y_pred.shape[1] * np.sum((Y_pred-Y_true)**2,axis = 1,keepdims = True)\n",
    "    elif cost == 'cross-entropy':\n",
    "        if Y_pred.shape[0] == 2:\n",
    "            return -1.0/Y_pred.shape[1] * np.sum(np.multiply(Y_true,np.log(Y_pred))+np.multiply((1.0-Y_true),np.log(1.0-Y_pred)),axis = 1, keepdims = True)\n",
    "        else:\n",
    "            return -1.0/Y_pred.shape[1] * np.sum(np.multiply(Y_true,np.log(Y_pred)),axis = 1,keepdims = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cost_function_derivative(Y_pred,Y_train,cost):\n",
    "    if cost == 'mae':\n",
    "        return 1.0/Y_pred.shape[1]\n",
    "    elif cost == 'mse':\n",
    "        return 1.0/Y_pred.shape[1] * (Y_pred - Y_train)\n",
    "    elif cost == 'cross-entropy':\n",
    "        if Y_pred.shape[0] == 2:\n",
    "            return 1.0/Y_pred.shape[1] * np.divide((Y_pred-Y_train),((1-Y_pred)*(Y_pred)))\n",
    "        else:\n",
    "            return 1.0/Y_pred.shape[1] * np.divide(Y_train,Y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def activation_derivative(Z,activation):\n",
    "    if activation == 'relu':\n",
    "        return (lambda X: X>=0)(Z) * 1\n",
    "    elif activation == 'atan':\n",
    "        return 1.0/(Z**2 +1.0)\n",
    "    elif activation == 'tanh':\n",
    "        return 1.0-(tanh(Z))**2\n",
    "    elif activation == 'sigmoid':\n",
    "        return np.multiply(sigmoid(Z),1.0-sigmoid(Z))\n",
    "    elif activation == 'softmax':\n",
    "        return np.multiply(softmax(Z),np.eye(Z.shape[0])-softmax(Z))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "def backward_nn_network(X,Y_pred,Y_true,cost,cache,h_params):\n",
    "    \n",
    "    if h_params['reg'] != 'dropout':\n",
    "        # Obtain the pre-multiplier (also dB)\n",
    "        if h_params[\"nn_acti\"][-1] == 'softmax' and cost == 'cross-entropy':\n",
    "            pre_mul = Y_pred-Y_true\n",
    "        else:\n",
    "            # First find the derivatives for the cost function\n",
    "            dJ = cost_function_derivative(Y_pred,Y_true,cost)\n",
    "            # Then find the activation derivative for the last layer\n",
    "            dA_dZ = activation_derivative(cache['A'+str(len(h_params['nn_arch'])-1)],h_params['nn_acti'][len(h_params['nn_arch'])-1])\n",
    "            pre_mul = np.matmul(dJ.T,dA_dZ)\n",
    "\n",
    "        # Obtain the dW and dB\n",
    "        dW = np.matmul(pre_mul,cache['A'+str(len(h_params['nn_arch'])-2)].T)\n",
    "        dB = np.sum(pre_mul,axis=1,keepdims=True)\n",
    "        # Store it in a cache\n",
    "        cache.update({'dW'+str(len(h_params['nn_arch'])-1): dW,\n",
    "                     'dB'+str(len(h_params['nn_arch'])-1): dB})\n",
    "\n",
    "        # Similarly for the middle layers\n",
    "\n",
    "        for i in range(len(h_params['nn_arch'])-2,0,-1):\n",
    "            pre_mul = np.matmul(cache['W'+str(i+1)].T,pre_mul)\n",
    "            dA_dZ = activation_derivative(cache['A'+str(i)],h_params['nn_acti'][i])\n",
    "            pre_mul = np.multiply(pre_mul,dA_dZ)\n",
    "            dW = np.matmul(pre_mul,cache['A'+str(i-1)].T)\n",
    "            dB = np.sum(pre_mul,axis=1,keepdims=True)\n",
    "            cache.update({'dW'+str(i): dW,\n",
    "                         'dB'+str(i): dB})\n",
    "\n",
    "        # Similarly for the first layer\n",
    "        pre_mul = np.matmul(cache['W'+str(1)].T,pre_mul)\n",
    "        dA_dZ = activation_derivative(cache['A'+str(0)],h_params['nn_acti'][0])\n",
    "        pre_mul = np.multiply(pre_mul,dA_dZ)\n",
    "        dW = np.matmul(pre_mul,X.T)\n",
    "        dB = np.sum(pre_mul,axis=1,keepdims=True)\n",
    "        cache.update({'dW'+str(0): dW,\n",
    "                     'dB'+str(0): dB})\n",
    "        \n",
    "        \n",
    "    else:\n",
    "        # Obtain the pre-multiplier (also dB)\n",
    "        if h_params[\"nn_acti\"][-1] == 'softmax' and cost == 'cross-entropy':\n",
    "            pre_mul = Y_pred-Y_true\n",
    "        else:\n",
    "            # First find the derivatives for the cost function\n",
    "            dJ = cost_function_derivative(Y_pred,Y_true,cost)\n",
    "            # Then find the activation derivative for the last layer\n",
    "            dA_dZ = activation_derivative(cache['A'+str(len(h_params['nn_arch'])-1)],h_params['nn_acti'][len(h_params['nn_arch'])-1])\n",
    "            pre_mul = np.matmul(dJ.T,dA_dZ)\n",
    "\n",
    "        # Obtain the dW and dB\n",
    "        dW = np.matmul(pre_mul,cache['A'+str(len(h_params['nn_arch'])-2)].T) / (1-h_params['kp'])\n",
    "        dB = np.sum(pre_mul,axis=1,keepdims=True)\n",
    "        # Store it in a cache\n",
    "        cache.update({'dW'+str(len(h_params['nn_arch'])-1): dW,\n",
    "                     'dB'+str(len(h_params['nn_arch'])-1): dB})\n",
    "\n",
    "        # Similarly for the middle layers\n",
    "\n",
    "        for i in range(len(h_params['nn_arch'])-2,0,-1):\n",
    "            pre_mul = np.matmul(np.multiply(cache['D'+str(i+1)],cache['W'+str(i+1)]).T,pre_mul)\n",
    "            dA_dZ = activation_derivative(cache['A'+str(i)],h_params['nn_acti'][i])\n",
    "            pre_mul = np.multiply(pre_mul,dA_dZ)\n",
    "            dW = np.matmul(pre_mul,cache['A'+str(i-1)].T) / (1-h_params['kp'])\n",
    "            dB = np.sum(pre_mul,axis=1,keepdims=True)\n",
    "            cache.update({'dW'+str(i): dW,\n",
    "                         'dB'+str(i): dB})\n",
    "\n",
    "        # Similarly for the first layer\n",
    "        pre_mul = np.matmul(cache['W'+str(1)].T,pre_mul)\n",
    "        dA_dZ = activation_derivative(cache['A'+str(0)],h_params['nn_acti'][0])\n",
    "        pre_mul = np.multiply(pre_mul,dA_dZ)\n",
    "        dW = np.matmul(pre_mul,X.T) / (1-h_params['kp'])\n",
    "        dB = np.sum(pre_mul,axis=1,keepdims=True)\n",
    "        cache.update({'dW'+str(0): dW,\n",
    "                     'dB'+str(0): dB})\n",
    "    \n",
    "    return cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_weights(cache,h_params):\n",
    "    if h_params['reg'] == None:\n",
    "        for i in range(len(h_params['nn_arch'])):\n",
    "            cache['W'+str(i)] -= cache['dW'+str(i)] * h_params['lr']\n",
    "            cache['B'+str(i)] -= cache['dB'+str(i)] * h_params['lr']\n",
    "    elif h_params['reg'] == 'L2':\n",
    "        for i in range(len(h_params['nn_arch'])):\n",
    "            cache['W'+str(i)] -= (cache['dW'+str(i)] + h_params['lam'] * cache['W'+str(i)]/cache['A0'].shape[1]) * h_params['lr']\n",
    "            cache['B'+str(i)] -= cache['dB'+str(i)] * h_params['lr']\n",
    "    elif h_params['reg'] == 'dropout':\n",
    "        for i in range(len(h_params['nn_arch'])):\n",
    "            cache['W'+str(i)] -= np.multiply(cache['D'+str(i)],cache['dW'+str(i)]) * h_params['lr']\n",
    "            cache['B'+str(i)] -= cache['dB'+str(i)] * h_params['lr']\n",
    "    return cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def nn_network(X,Y_true,nn_architecture,nn_activations,cost,learning_rate,n_epoch,regularization,lam,keep_probs):\n",
    "    cache,h_params = initialize_parameters(X,nn_architecture,nn_activations,learning_rate,lam,keep_probs)\n",
    "    cost = []\n",
    "    if regularization == None:\n",
    "        for i in range(n_epoch):\n",
    "            Y_pred,cache = forward_nn_network(X,cache,h_params,regularization)\n",
    "            loss = cost_function(Y_pred,Y_true,cost)\n",
    "            print(np.sum(loss))\n",
    "            cache = backward_nn_network(X,Y_pred,Y_true,cost,cache,h_params)\n",
    "            cache = update_weights(cache,h_params)\n",
    "            cost.append(np.sum(loss))\n",
    "    elif regularization == 'L2':\n",
    "        for i in range(n_epoch):\n",
    "            Y_pred,cache = forward_nn_network(X,cache,h_params,regularization)\n",
    "            loss = cost_function(Y_pred,Y_true,cost)\n",
    "            for j in range(len(h_params['nn_arch'])):\n",
    "                loss += 0.5 * h_params['lam'] * np.sum(np.square(cache['W'+str(j)])) / Y_pred.shape[1]\n",
    "            print(np.sum(loss))\n",
    "            cache = backward_nn_network(X,Y_pred,Y_true,cost,cache,h_params)\n",
    "            cache = update_weights(cache,h_params,regularization)\n",
    "            cost.append(np.sum(loss))\n",
    "    elif regularization == 'dropout':\n",
    "        for i in range(n_epoch):\n",
    "            Y_pred,cache = forward_nn_network(X,cache,h_params,regularization)\n",
    "            loss = cost_function(Y_pred,Y_true,cost)\n",
    "            print(np.sum(loss))\n",
    "            cache = backward_nn_network(X,Y_pred,Y_true,cost,cache,h_params)\n",
    "            cache = update_weights(cache,h_params)\n",
    "            cost.append(np.sum(loss))\n",
    "            \n",
    "    return cache,h_params,cost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "cache,h_params = initialize_parameters(X_train,[25,10],['sigmoid','softmax'],1e-4,reg = 'dropout',kp = 0.7)\n",
    "cost = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.7077650782678746\n",
      "1.918114035293459\n",
      "1.7588308923593803\n",
      "2.068988293533276\n",
      "1.7177449380717187\n",
      "1.8125111554395354\n",
      "1.6157331777932524\n",
      "1.7979730368768847\n",
      "1.9763448891217936\n",
      "1.672094062650999\n",
      "1.722523366497826\n",
      "1.7636647394997735\n",
      "1.9535667553283904\n",
      "2.0724560003384798\n",
      "1.728716885117035\n",
      "1.6033189868107456\n",
      "1.7334027910449699\n",
      "1.512303816960783\n",
      "1.9327542398879731\n",
      "1.7581826749408487\n",
      "1.6320930742369715\n",
      "1.9500054326212366\n",
      "1.8743779607744109\n",
      "1.9307318673926037\n",
      "1.7833756351461543\n",
      "2.0040644828652043\n",
      "1.8789485818917568\n",
      "1.6561932205693557\n",
      "1.7805581094783536\n",
      "1.7768015058107232\n",
      "1.5854108655299017\n",
      "1.7090961200355748\n",
      "1.7304447726645724\n",
      "1.8041933100543674\n",
      "1.7665703432611486\n",
      "1.717098924322307\n",
      "2.060102651112225\n",
      "1.5800464644131444\n",
      "1.8279949691960429\n",
      "1.7539888138366717\n",
      "1.8565525509175556\n",
      "1.6380727085053857\n",
      "1.7334944372774173\n",
      "1.9605729732754429\n",
      "1.6596388559611257\n",
      "1.7054342576046604\n",
      "1.6898820527362521\n",
      "1.9802285223253562\n",
      "1.5837693840292972\n",
      "1.7795834216293047\n",
      "1.8084911078500572\n",
      "1.8288056667209764\n",
      "1.8307545681366868\n",
      "1.9359787323770705\n",
      "1.734428870652787\n",
      "1.6213177340133904\n",
      "1.8974634110906092\n",
      "2.0255936828329433\n",
      "1.9077032612006697\n",
      "1.874775232052144\n",
      "1.8188250561532255\n",
      "1.9845781716609139\n",
      "1.8596936518535512\n",
      "1.6976993067843278\n",
      "1.8485767599593315\n",
      "1.7866901791905796\n",
      "1.7263991049306648\n",
      "1.7331024124178311\n",
      "1.5519948294876829\n",
      "1.716789144117924\n",
      "1.7024917203311836\n",
      "1.7826386920263666\n",
      "1.8135224737195421\n",
      "1.922652782047992\n",
      "2.1264992169171855\n",
      "1.830487789286791\n",
      "1.7533845702095974\n",
      "1.9590843942845175\n",
      "1.8076833225486757\n",
      "1.7360091157307318\n",
      "1.6409348120731666\n",
      "1.7304560812177374\n",
      "1.8256509259552753\n",
      "2.036237822817809\n",
      "1.6830296811122385\n",
      "1.8448333987048193\n",
      "1.7380829782489826\n",
      "2.1117203775090645\n",
      "1.8100997797004088\n",
      "1.766206670253025\n",
      "1.9815245140105202\n",
      "1.9401276108527337\n",
      "1.775384415541058\n",
      "1.7791671256961175\n",
      "1.8282764695612692\n",
      "1.7445849143827643\n",
      "1.8399398767508892\n",
      "1.8085230851846987\n",
      "1.7495657634211628\n",
      "1.8125303125105066\n"
     ]
    }
   ],
   "source": [
    "for i in range(100):\n",
    "    Y_pred,cache = forward_nn_network(X_train,cache,h_params)\n",
    "    loss = cost_function(Y_pred,Y_train,'cross-entropy')\n",
    "    print(np.sum(loss))\n",
    "    cache = backward_nn_network(X_train,Y_pred,Y_train,'cross-entropy',cache,h_params)\n",
    "    cache = update_weights(cache,h_params)\n",
    "    cost.append(np.sum(loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0xb42896cc0>]"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO2debwcRbXHf6dn5s5ds69kZV9DCAQkgKwqIaCIooIKiiiI6AMfT0F8gIoL7uIDRQRFFEEEFEX2PawhCQnZwxZIyHaz332WrvdHd3VXV1cvM3fmznLr+/nkkzvTPd3V3dW/OnXq1ClijEGj0Wg0tY9R6QJoNBqNpjRoQddoNJo6QQu6RqPR1Ala0DUajaZO0IKu0Wg0dUKyUiceNWoUmzp1aqVOr9FoNDXJggULtjDGRqu2VUzQp06divnz51fq9BqNRlOTENE7Qdu0y0Wj0WjqBC3oGo1GUydoQddoNJo6QQu6RqPR1Ala0DUajaZO0IKu0Wg0dYIWdI1Go6kTalbQMzkTd89fC53+V6PRaCwqNrGov1z/xGrc+NSbaE0nMWfa+EoXR6PRaCpOzVroG3f2AQC6+nIVLolGo9FUBzUr6DnTBAAkE1Thkmg0Gk11UMOCbvnOk0bNXoJGo9GUlJpVw1zettANbaFrNBoNUMOCnrct9IQWdI1GowFQw4LuuFy0D12j0WgA1LCgcwvdIC3oGo1GA9SwoOfylqDreUUajUZjUbuCboctcktdo9FoBjs1LOiWkOe1ia7RaDQAaljQuWVuagtdo9FoAMQQdCKaRERPEdFyIlpGRJcE7Hc8ES2y93mm9EX1ks1rC12j0WhE4iTnygG4jDG2kIjaACwgoscYY8v5DkQ0DMBvAMxmjL1LRGPKVF6HvPahazQajYdIC50xtoExttD+uwPACgATpN0+DeA+xti79n6bS11QGe5D1wa6RqPRWBTkQyeiqQBmAHhZ2rQPgOFE9DQRLSCicwN+fwERzSei+e3t7cWU14Fb5tpC12g0GovYgk5ErQDuBXApY2yXtDkJ4DAApwI4GcBVRLSPfAzG2M2MsZmMsZmjR4/uR7HdOHTtQ9doNBqLWAtcEFEKlpjfwRi7T7HLOgBbGWNdALqI6FkA0wGsLllJJXgcuo5y0Wg0Gos4US4E4FYAKxhjvwjY7X4AxxBRkoiaAbwPlq+9bOR1HLpGo9F4iGOhHw3gHABLiGiR/d2VACYDAGPsJsbYCiJ6GMBrAEwAtzDGlpajwBxumGsLXaPRaCwiBZ0x9hyAyAxYjLGfAvhpKQoVB5PpQVGNRqMRqdmZotzTovVco9FoLGpW0LmFbmofukaj0QCoYUHnOq5dLhqNRmNRs4Lu+NC1ha7RaDQA6kDQdZSLRqPRWNSwoFv/583KlkOj0WiqhZoVdKYHRTUajcZDTQo6Y8zJh64FXaPRaCxqUtDFyBYd5aLRaDQWNSnoOVHQtYWu0Wg0AOpA0HWUi0aj0VjUpqALoS06ykWj0WgsalPQRQs9hstl8dodeHDJhnIWSaPRaCpOrAUuqg2+WhEQT9BPv/F5AMCa604tW5k0Go2m0tSkhZ71uFy0D12j0WiAGhX0fIEuF41GoxkM1KSg8/VEAW2hazQaDacmBT2bFycWVbAgGo1GU0XUpKBrl4tGo9H4qUlBFwdFtaBrNBqNRU0Kek7nctFoNBoftSnoBcahazQazWAgUtCJaBIRPUVEy4loGRFdErLv4USUI6IzS1tML8VGuei8LxqNpp6JM1M0B+AyxthCImoDsICIHmOMLRd3IqIEgB8DeLQM5fQWqMgolzxjMEBlKJFGo9FUnkgLnTG2gTG20P67A8AKABMUu34NwL0ANpe0hAq4D52oMJeL9rdrNJp6piAfOhFNBTADwMvS9xMAnAHgtxG/v4CI5hPR/Pb29sJKKsCjXFrTSfRm87F/pwVdo9HUM7EFnYhaYVnglzLGdkmbfwXgcsZYqAOEMXYzY2wmY2zm6NGjCy+tTXfGEvExbWl09OZi/y6nBV2j0dQxsbItElEKlpjfwRi7T7HLTAB3EREAjAIwh4hyjLF/lqykAj22VT52SCPW7+iJ/Ts9KKrRaOqZSEEnS6VvBbCCMfYL1T6Msd2F/W8D8EC5xBwAegULfdXGjti/0xa6RqOpZ+JY6EcDOAfAEiJaZH93JYDJAMAYu6lMZQvEcbkMaSzI5aJ96BqNpp6JFHTG2HNA/Fg/xtjn+1OgOPRk82hIGBjalEImb6I3m0djKhH5O72gtEajqWdqcqaoJeAGhjRa7VFcKz2f14Ku0Wjql5oU9O5MDs0NSbQ1pgAAHb3ZWL8TZ5hqNBpNvVFza4o+umwj7p6/DgmD0GZb6J19wRa6qVPtajSaQULNWeitaUvE8yYTLPRgQRcjW3SUi0ajqWdqTtAnjWh2/ubiHuZyEd0sOe1D12g0dUzNCfq4oY3O39zlsivEQs/qVLsajWaQUHOCnkq4RR4Sw+WS1y4XjUYzSKi5QVEA+M1nDsWIlga0NsZwueSLy52u0Wg0tUZNCvqcaeOdv1saEqEWelYvV6fRaAYJNedykWlrTIVa6Pm8FnSNRjM4qANBT0aELQpRLlrQNRpNHVPzgt7amAyfWCREtuj0uRqNpp6peUFvSBjOCkYqRA3XFrpGo6lnal/Qk4Yn1lwm7xkU1blcNJpqIG8y7OyOl4NJE5+aF/RUhIXuFfSBKJFGo4ni6vuXYvr3Hi1oTWBNNHUg6IRMLszlIk4sqj9Ff2Z1O55dXfyC2xpNJbh/0XoAQEZbWSWlDgQ9vg+9HsMWP/eHeTj3D/MqXYx+c/3jr2P+mm2VLoZmgIm9co4mFjUv6NagaDwfuh4UrV5++fhqnHnTi5UuhmaAYDqvUlmoeUGPttDditMX4prRaDQDB38rrTXoNaWi9gU9SbEHRfvqbABGx9VrahVuZ2lLvbTUvqAnjFDLW7TQezL1JegdIROqNJpqhtk2urZJSkvNC3rkxCJhU2+uvgRdx/FqahXHztKCXlIiBZ2IJhHRU0S0nIiWEdElin0+Q0SvEdESInqBiKaXp7h+UgGDou9u7caKDbuQFyz03mx9+dB39GQqXQSNpl8wreglJU763ByAyxhjC4moDcACInqMMbZc2OdtAMcxxrYT0SkAbgbwvjKU10cqYSBvMuRNhoThDrAc+9OnAAB//Pzhznf1NokhLClZLaH9qIMP/sS1y6W0RFrojLENjLGF9t8dAFYAmCDt8wJjbLv98SUAE0td0CBSSUvEg9wupmShd/blMPWK/+D2F9cMQOnKS72EYdbJZWgKQQ+KloWCfOhENBXADAAvh+x2PoCHii9SYTTYS9IFCboY5dKby2Pzrl4AwB+ee7v8hSsz9RLlotd6HbzoJ19aYgs6EbUCuBfApYyxXQH7nABL0C8P2H4BEc0novnt7aWZrs7XGF22XlkkRywMssIW6yn+tV5mvmo9H3y4US764ZeSWIJORClYYn4HY+y+gH0OBnALgNMZY1tV+zDGbmaMzWSMzRw9enSxZfbABf2sm1/CQ0s2+LZzw72lIYkewYde+3JeTy6X+rgOTXx0lEt5iBPlQgBuBbCCMfaLgH0mA7gPwDmMsdWlLWI4qYQrzas2dfi28yiXpoYEerNmXVmD9SKEdXIZmgLQel4e4kS5HA3gHABLiGiR/d2VACYDAGPsJgBXAxgJ4De2KyPHGJtZ+uL6aUi6bZJKGHgO9JZ0Er3ZvCuCdWCic5dLrXuP6qVh0hSOfvalJVLQGWPPIUL+GGNfBPDFUhWqELjLJQgee96STqAnk3cGT2tcAwG4L0Pca+nqy+HAax7B/566P774/j3KV7AC0S/14INHt+hHX1pqfqZolKB329P9mxuS6M2ayOW5VVv7ks6vxYh5LVs7rYlIt72wplxFKgr9Tg8+tMulPNSBoIeLGZ9M1JRKIJM362qRizwrTND5btVmFbH6eSSaAtFx6KWl5gU9nUw4f6t0jSfkakxZOV94moDat8+FOPQavxjtchl8uNkWK1uOeqPmBb2t0R0GUFUOHqrYmEogkxNdLgNSvLLCwxaNmNfiWujV9RZpQR+86EdfWmpe0FvTrqCr3Cnch96YTFgWuskHRatb0Ret3YFN9qzWIMwCXS6canuHqq08moFDJ+cqLbUv6IKFrsqm2JsVXS4M2QFatYgxhttfXIOuInOWf/TG53Hiz54O3ccJW4xdJu//1YK20AcvtT43bsWGXXjpLeU8yooQJw69qhEt9L5cHjc+9QYygmi7PnTL185dMOV2uTy9uh1X378MqzZ24AdnTCvqGF0RC3LkzcIsdL5/tQlolRVHM4BUm/uvUE65fi4AYM11p1a4JBY1L+hpYWJRT8bETx9Z5dnebQt42hb07gFatYgvPtFZxlWFCp1YxKNiqu0VqrYGRjNw1POT396VwZCmlCetd7mpeZeLGE++9L2dvu29QpQL4Ap6uePQeS+hISJOvj9wgY57LdU6maPayqMZOGrdQg9iZ3cWM659DD95eOWAnrfmBV1kw84e33fdWctCbuIul8zALArRZ89IFVMTlBqzwCgXnqis2l4ibaEPXur10fPVxB5c6k8YWE7qStB3KVbwWfqelVa3UXK5FGuf3/rc23hQkdVRxrHQYwr6Uys3Y+PO8KgWmZxZmIXOXTTV9g7V60utiUY/+tJSV4IeBve1uy6X4o5z7QPL8ZU7Fkbu12cvSB1X0M+77RV8/LcvFFSWQhe4MB2XS3W9RqKFXi853jXxqNfeGQ+LHujLqwtBf/nKkzBtwtDQfVwL3bLiyx3lwi10cSZrEFzE3tthuYziCm6+QIE2q3ZQ1P07aOWpSjPv7W1YHrCIiqZ46lTPK5Zmoy4EfeyQRkwe2Ry6Dx+cVEW5/Hvxesy5fm5JLVdX0KNvsSxicaxUxpjjE49r1Doulyp7icT73t7RV5WLeX/ydy9izq/nVroYdUe11cVapy4EHQBaGsIt4ZQtrD2OD9010b9256tYvmEX+ko46YgLepyQJS7ofNe8UMufWe1fqu/pVZux+7cedKJ64nZbufBXn8vF/fv9P3kKZ938UuUKoxlQCnW5LF+/y9dTuugvC3DcT58qZbFqlroR9OaG8JB6npWxS+Fy4aLboRhULZaMLdJxKixPGMbLIWYw+Nwf5vn2f3zFJgCWGwBAbB+KWbVhi94CLVq7o0Il0VQ7c34919dTemjpRryztbtCJaou6kbQW9LhFnpastB5ki4ASNjqXuw0fRXcQs/no9Uz51joVjnyEYrLexc8L01cK6dao1z0OOjgpRLGxdQr/oPvP7C8rOeoVABC3Qh6tIXu9aFv7uhzp87bdyFqVmcuYMCuozeL//7bImd26L8Xr8ddr6wFEC3OgGvNcws9yocuD7jEFcRqjXLRCZoGL5V69rc893ZZj18pI6VuBD3Shy4J+pbOPvzkEWsWF7fQowQ9Iwj6I8s2OsJ4+4vv4L5X38NNz76Jua+342t3vursF2eA0+9yibLQvcR9KcwCB1EHijpac6QsMMawYkN9RtgMdF0cKGOmUuG3dSPozelwC73BiUN3RfvpldaAo2ELaWeED11M+nXhnxfgyZWbrd+TK8Tn3Or1ecd5sNzyTxoxXS5SzKXqFCs27MLzb2zxlqXKLHNOtcciFxJKuWFnT8lF44/Pr8Ep1891x0zqiIHuLQ6Ezq7cuMuZhzLQNbt+BD3CQudhiz1CSNyw5hQA1zLe3p0Jna0pR8Fst10sPF1LTlFb4gi67HIpdMKQ6qU45fq5+MwtL3u+c+PQq0tAq1zPY0c/rd7UgVk/ehJ/fH5NSc/Po5ne3VZ/A39xHv2b7Z04+ronsTlifYA4lNt4WLutG7N/NRfX2j56HYdeJC0RPnQ+KCrmTB/e3ADAdbl8457XcOSPnnAGTmUy0ovNLWpuoasag0JdLl/80yu4d+F7ofvLk6LiVhreUJTTSunJ5HHB7fOxbnt88am2BkamL2Zc/Ab7+T+1anNZylHdS7IURxwL/Y/Pv433dvTg4WUb+32+QgR9V28WP3tkVUE9tG1dVg6Xhe9UJlKrbgR9SFMqdLvKJcMtdEOKFe8OSOAlW2rcoub/r9vhTw6mstp9+zguFwOPr9iMH0dkaJNXWyo0yqWc+vnEyk14dPkm/PDBFbF/U20+fZnemBY6H8cpZ8rkekOuui+8uQUHXv0wdvZkffuWokErZLzmZ4+swg1PvYEHXlsfuS9vmORe8EAbK5GCTkSTiOgpIlpORMuI6BLFPkREvyaiN4joNSI6tDzFDeaQScMAAJNHNONnn5iOyz64j2d7c8rvkuGNgFypgkQ4yELngi5vB+KJLXe5GDGbV9lCjx/lYv1fzkrGezuFDApVuw89roXupJfoq76ZrtWK/OR//cTr6MrksWy9mwq7lNWjkLrGe+rZXPRv8lLvt1JVOo6E5ABcxhg7AMCRAC4mogOkfU4BsLf97wIAvy1pKWOQMAiLr/4Q7r/4aJx52ERceNyenu2GQYGRMPILqxJmwBvlws8JuC6XTM7/Isex0LnLJRlT0VWWSpyuq8m8la4c8AHbQs5RbWGUMoXOIO4qcYrmctydr/51IR5eugFTr/gPflRAbyqM7kwOH73x+YJy3sjjRbwqEAh/e+VdPLWyePdVTybv87sXExgQxwByj+ud61F1PnTG2AbG2EL77w4AKwBMkHY7HcDtzOIlAMOIaHzJSxvB0OYUhrfYfnHFlPsW2+0ydWQz2tJJZPMmGGPOqkacPoUwA37hTya858gqJhHFGeDM5eOnCQDUicXiVBw3l0sZLXSDZ5krxEKP3ueWuW/hgdfW44YnX8efX3qn2OIVRVxB55dc6lWx+L3sb0K5vMnQk8mDMYYHXtuAL//Fyhr6u2ff6m8RAQCvrNmORWt34EcPxW8g5EfPPxMBl9+7BOfd9krR5Tn79y/hiB8+4T1+AW1zIQm2uCvHXbe3BsIWiWgqgBkAXpY2TQCwVvi8Dn7RBxFdQETziWh+e7s/R0kpEbXx4hMsa52vP2oYhGSCkDcZMnnT5x5QLTYNqCx06/ZxQVYNnsQbFPXmcgmCn0eV/zxO9Skk2+Jb7Z34n78vDpxMFYSTj6YAEz1O3f/+f1bgq399FT97dDWu+ufSgsrUX+ImC+P3t1p96JfdvQj7X/1w2WKkea2c+/oW7PGt/8T6TZxn7+yiqvchB1ClkCile088d9DyjlUbtkhErQDuBXApY6yoWQ6MsZsZYzMZYzNHjx5dzCFiI4reaQfvBsC10BNESBgGciZDb8YvWEEWmeyK4WfglrnKVRMvbNG2wEKGfR5fvgl7ffshvL6pQ7mXWFFVvYI1W7qccsap05fctQj3LFiHZQWmjDWcWPr4v6l6H3pMC51fRyZn4smVm/Dbp98M3b+9ow+PlCByIy7/XGQN7pVrPoKot3HbDJ8gF1i0OC5NkUKu3clpDmt9UBnx1I47U0qvUXUuFwAgohQsMb+DMXafYpf3AEwSPk+0v6sKuBuA53tJGISkQcjlTWeJOpEgl4ss2Pwh8pwqSkGP8UTjWMHz1liTSh5cslHpRBdFsUeyKNfv6MHxP3u6oMgT7jcstJvPB0WjupyZnIktnX0AqlPQN+3qRUevFWnBXW08wVtPJo/rHlrps9zFF/wLt82PjFY69w/zcOGfFwRGVckQWdkGV2/qiHsZSsploRtF+IT8Fi03boTvHL+6n6DxLhHRwAmqax29WfxrsTqa5bHlmzDj2sd8E7tyQsiMHBLMT7Olsw83PvVGZBlLRZwoFwJwK4AVjLFfBOz2LwDn2tEuRwLYyRgb2MX0QuAVzXG5kOVyydk+RZm+AJdLVnoRHEG3R8FllwwQd1A0ulJOsfO9r9iwS2nJi/VUHpTjsbHbFFZGEOLgVCEYMaNc/uvOVzHz+497zlUsr63bUfJZlO/74RM45Xorq5888ev3c9/CTc+8idteWOP5TaF+07X2RKGoeyVunfPrufjQL58t6DwycaI2iqEYF3/QLYtaVnHzrl4c8YPHY6VEEN/LoLDFb97zGv7rzlexaqPbWPIivPTWVgDAYsmFIz43N/Gd/4J++siqAUsFEMdCPxrAOQBOJKJF9r85RPRlIvqyvc+DAN4C8AaA3wP4SnmKWxzcr+u4XGwLPW8y5QBWkIWel2oD/8gFWdU1jzMoqhpMDTrO1q6+yEHRUoTNOYJe5ECcWIHXbuvGi29u9Wznk0RMk/Vb0D9yw/P45O9e7N9BFKzbbs0r4NeSlMZM5Ia/2Mso5nd3znu3yLMBffnSDtqu3tSBBe9sK0rRuVG0cWcv/u+J1wPqgr+3+NDSjdjc0Yffz40e0PUIekBl46uFyb1bwDXKZLEWjTV39TB1GXYp4urLQfj0SgCMsecQ8aiYZZpcXKpClRpuWfH0AAZZ3+XyzOk2j2ptwJZOy4IN8pnKhnRecrmoKMRCDxNPXiYKsJnFitorNEiFphHoL6qc6xfdsQBL39uFV779AYxuS3v2zzMW+JKt3tSB7/57GW793OFlK28ceKpl/nzc0Eypx1bgvebPsZDIC8637luCs4+YXPgPEc9NUQi8x3Dnl44s+Lf8jn3tzoV4Zc12J7RY9S6INZ/f+1wMY0i8XvmZdWdyeG3dTvknnjJknbUNvNvF1Ng7u7MY3ZoOtMS3dWecCLxyUjczRcPgbgBuYRkGIZUwkDNNx0If2eIKTZDLxR8z63W5xPmNijgul6wgKipf5aK1O/CK7WcXK3mxA2DFNgP8fOJ5+f15cuUm3/65fLCgf+dfy/D8G1ux4J3tRZamNHBfqTzvQHaxFNt2Rj2jYt1fQZRa0DnF9Ob4PeTvocoAckIBhVrJd5PdoCqyIS6Xy+5ejLNufslJ2+C9BO9Cz2ETED/4y2dxx8vvBj7LHd3x3Z39YXAIujSj04pysVwuvIs1Qmg9e7J55WIX8sPKm1b3+08vrgk8d85k+O+7F+HAqx8O3CeOy4W/hETqF+czt7yMT9z0onNOt4ysKP8df9EKHehyB4fcc7Y1Wh1B1XhFzjQjXS6lGjPdsLMn1gC0fL/4/eQDvtyFJ9/WYmOPB3pQWDXWUwqKGhRl3t/K0SIi4nf8XucVA5NzX2/3RBipLHReVO6D7+5TLR6vHjPjyPXkiRWbAhe02dY1MC6XQSHoCcdCt19I24eezbuDoiNbXUH/2SOrcOA1j/jyScgP0GQMd857N1QwTcZw38L30BUy2cQV6+AXwun2mdGuSrGS50xWUHIhTrE+dGcKtHBPuIAEZaMMmonX34k0Ip19Ocz60ZO4+l/LIveVLVh+Tfz5uKGZ/bTQecMQc1C0VPejqix0yT/Ob4V4b7m7Uaw/piPo/u/OuXWeJ8JIrP/8uEFFFXtBGannLTfYOcncZwh2sarCHsvBoBB0blGpLHTe1eOZFwGgw26t5WnDPp8pY5ExynF8fNyPLw+6ivBKmTXNyDdHtPjzeabsAciVc/6abbjg9vmho/UiQcv1qV5ILiAqSzSbZwOywAUPPbxv4brofaVBcX7/eD2SrUlOsZb2QOepL5egF3MZckgir5fisbjR5RVv7/9A8H0U31F31q0bYy5CZPnV73j5HV9PxudD9xl4wRFL2wfI5RI5KFoP+FwuBiFp+9C5y2VIk/9WyBEwKgs9KstjnJeVC3qY+DtWSp7FsNC9PnSVhZ4zmRNXDQBf/stCbOnsw9bOPowZ0uh8rxKpJ1Zswvl/mo97LzoKh00Z7tnmdpnd7zIKC0ss60C4HPhzCJoFvH5HDx5auhHnH7O7r5HO+3zo1vdysVXXYZrMl82Tw78d6NVtVILOGIsMFYzimhi9H5nz/zTf89kxCIR7wt9RlYUu9m7aO/pwzI+f8p3D63LxbnN7ou61X/vActw5b63jKpTPyZHrM2PMZ7WrylBOBoWFHuRyyeUZeuyY7SGNfmHe0ZPF9q4M/vTCGjDm90XnTWBIY3ibGOdl5RU2LFomK6QXCHvvejJ5j4DnTFNZyWSRb0xZVYGLmTgQ9NJbW/HGZjc+9wU7BHGhYrDSzTrn7zKrXAs504x0VZQiO6TqOSx8dzv+959LwBjDF/80H9c+sBzv7ejxDIpn8yaeXW2t/MQtcwpwlajapThRTnF7KKVq+PoUDXyhCchUlHKZPNEQUlnofLN4T17f1Kk8lqfHyt1n/DiKurVplzXhrUNawUy+/XKdYiEWepzB21IwOCx04pa5JVoEO2zRZNjZk0VD0lCueLSjO4Nv3bcEDy/biOmThim72EEDQWOHpLHXmFZ0xYgJ51ZjmIUuWrlh0Q77X/0wPnHYROdz3mQ+XyBgR54IUVR8ARDeuPBfmIzhrJtfAgCsue5UAG5SMi5WH/vN80gYhL9/+Sgho6Pfh65y5efyLHAwsVRRHfw8Mp/63YvI5hmuOu0AdPRZ4yX5PPO4XH752Go8Zy/lx5Nhui4X7/FUghvWoHOrMDrKxd/r6Q8qa7EvZzrpf0tJbzaPzr4cRrWmo3cWEOsED8P9+/y1vu1xUkyEDYqqGoagRliup7JRxBAcgFBoTqRiGRQWOn8RuRCZjCGVMJA3GVZv6sSeo1udRaRFdnRnsc32fXX35XyCxEJiqI/ZazQa7HNE4VjoIQ+db4tTMR5b4YYH5k11N3BXb9aZqQi4ubx5Yik3Yb//+Lynk8ubeGz5Jix8dwdeWbPd3t8/WKUaI+AvVM6Mtr9LYZiqngM/7uf/8ArWbrMmlsjjIm9v6XL+NpwoF7UPXW2hBz8vfg/iulxK5WtX5u0vkwX52VtedmYEF4JYzXl512x166s7xT663GFhixxvZFhA2LJ0KrlXY5rBz7LQnDPFMigsdHllIf53zmRYtbEDs/YcqRT07d0ZZy3S3lxeEbYU/KCsyUvxBL3XEfQQC91xuXijQhoShm/wJid1MVXW6WdvfRnvbO12rO4mW9C7HEF3fy/jZJk0Gb50u9cHyosi/oxf36J1O/HnF9fgrS1dnuNHuRLivgqL1u5wFjqRUcY32/+/+JY7i1UWdNH/zRiwuaMX33PWi4weFI3z/OO6UooNi2SM4aZn3BmVKvdKOQZmGWOYX+QcAvG+KefUIMQAACAASURBVGdgKwyHoA5dn8pCl2LM3dTS8UVZbhgZWKAeFBNpVgyDQtAdlwu5DzFpEHoyOWzc1YvdR7X4cpsDloXOv7/wzwt8fnZT4VcXz5kw/PkeVDnPe2NY6NxtIvucG5J+QRfTt+ZMpow7fse2dvhgWKMk6ByViKR42J7JMKo1jS2dfW5stuRDz+ZNp6F6dnU7nl3tTZsc5EMXzxvXevzojc87DZSM2kJXDGIyWNPYbRKCS81kDN/4+2uefeXfyoRZZnEHRV33V+hugWza1RcYxscpx8Bsf9oIsWejSl+sinIJIqMIW4TjcvE2DGHvtCzgPgs9rDEoJP1oPxgcLhfyW+jJhOH4rhuShmOJi+RM05ldms0zbJViSU3B5XLTZw/z/T5pGFglZMZTdb93dmfR2Rcd5eK6XLw+55SiIRIJstA5727rxj0L1jk+dO7zDxORhH3OrGliwjArIoZP6ecvDL8vqtwYIkE+9LzJfFOv+4PqGKq7YjKGHz5oiV/SHjx39mdea162alXXsa0rgzW22yabNzH1iv/gpme8aXVju1wUURVxkMMwi03zXCj9sfrFOhuWIymWy0URthiEyeJb2b6lCVlwAx7meislg0TQrf+5tc3AkDTIabmTdhijjCgqKvqyJm57fg0AN0rEc17JGpdfmkzOxPTvPepEB4RFufCXMJs3PV10latIPmeYIJ5z6zz8z98XOy8fz9QoL3orwkUun3crf58Ua84vpTdi9Z6cqU7OxQcigXBXVFxR88QwC91rmc12hANg1RfxGZqMecQwztT/2b96Fsf/7GkAboKmG5600qkG5YTxEeAvjjPDWLVfsYK+fkcPZv/qWd/8jCDE6yrUXSRa1SoLnde7eOsN+MMW3SgXL2FWtnzf5J6vNSiqftfiPqv+MigEXfahM2b9zVtYg0hp6W7Y2YvHlvvzj3Cuf+J1LLYT+6giBORjyq13e2ef53NYnc8KPnTZ5RJGlKDznN98VqwzKGpvV4mN6EPnlhQP9TOlFy1qmnmQD/3zf3SXHgsrf1zLUpUZT4XYo0oZhsfl4os7l4qljEO3v+rO5JwwOGdyCy9PyDW82d6JTbaAyvvF7bnI+6meSd5pmPOBqYjvePkdrNzYgb+9sla5XUa8HYX2AEQLXe1ysetZDNecZ6YoD1t0Zqb6ez1BPVq/he53uQRa6DrKpXSQFIfOmCW23KpMJkhp6T69KnyZPNFX3aQQ9LQktnJF2RTT0gHcl7Anm/fkZVa5ijznNNUzRTnD7Bmy7R1W4yIPiqoMjpQTtmi6i3s4oYneFy3Kd5jLR8ehh03KiGv5iIISViYx3j7PmM9CFylkpugBVz/iCHqccEfOST9/xhlYlH8XFeZ6zq0vY+l7O32C6HMVwBWiH/xnBT75uxc994HD63hvQHppmaBQwKlXRC9Pl1VY1art4q0LqgvKsMWAEVTZQhffLzl+32ehMxaYy0XHoZcBblkyMCfKBeAWev/aNrWFLgm6pI5xu66MMU+I4QtCbvFol4sZah0Ma7YGenk+aD6uwCNpwsRGTPxl+epNJy6YW6FRvsNcgIU+aUST83eYlR/mpvKex5T+Vsdc7+pxG+kpI1s8PvSoQdAor8Iuuzckx0LnTWv1ojkHjcO27gwOGD8Ex+87xvd7n8su5L6s3LgLc1/fgh3dS3DlnP0921QTi3iZuPtve7c/mRSv4z2KZRtViMUV60kconofvDFTWd8yXnebd5uv18WYx+of2drgZGL0DYoqVqyqtIU+qASdv5wmc1Pp8u9VUS6FoPKhy2IrV7hNu7wulyCeXt2ufMGAOC6X8JeD57BRhW+Jn73HdK1vefDK53KJWB0nb6oHRQ+eOMzxOYda6DFnOKpWl1HBrdkj9xiB7V1ZKSrJ3z33fI4IsOTuLXmvvMl8EUBrrjvV50KQBTzsuYorR8kWepgPXQ7nE0mXwEKP7yaKqDeM1y+5oQ7eVyyTM7FILrPJPPd9aFPKEXSfy8UXtqijXAYUp/vM4LG8DINCXRd7jm6JPLbKQpfFNptjuPr+pXhjszVFeWNMC32jXaGmTxzq2xYV5WK5RYIrkxxFKQt6WIibPGmpL+cO2DqiH2FBZ/Pq9Lmm6eYWCZuWHnfChrhf2G+eWLkZgNUYyzOBo1wlUZ0Fbv1ncqYtstaxH13uXyg6lzcdi979jik/M8bw6LKNnmclTn6SBT1O2KKqkW2067PKp63CVDSisZ9XpIXuBgm43wVY6HmFoMNy/XBXo7vdW8ahQq4m36CoLOhhETID5HIZVIIuinhCEMKkEexyaUwZzlqkYTQmo10uKzbuwu0vvoOv3LEAQLgPXWwMeMU5cs+RofupyJss1IqVk1XJFVJlKYkvZy7PnPval8v7luKKtLRMFuAjZcLf0dE/HNNkeGjJBp/oiC91nG5/OmkgzxjER+ibWFaALxyAJ+z1V4+/7vz9RztSSqS9s8/XKwsa3Fy2fhcu+PMCfPsfS5xtvAOaN5nPoo4T5aIaOOYWetACMDLiIXnDHuRjlokSQC7e4rXEmRAU9YzkOHTukgQsV+dfXnrH+eyz0FnwLNO4Pcn+MqgE3YlyAfOKe4jLJZ1MxEr0rPp9g/RdrxBVA3hD5GTE3DI8jlg1kBMnbDHMKpJjlOUKmVGsP+kMeJomciZzytqXNT0Ti/Im84iMCpUP3UqZ4JYjzOUiX9vcN7bgojsW4tRfzw3sjsexEhuSBkyTOeMugDXRTCTO1H+RbV3u897ZkwmtVrk886VclQV9+YZd2O+qh5yFje+e708NbDLm83mrejzOPbHLxO8d7yV09uWcENTuTA43P/sm9rvqIby+yT94Kp6bw0Uy7phHlGuGN2bitQQdWywH3yXoUVlx6O5xhkrZVP/3n0t9ZfD+NqhRMbG9K4OX39qq3F4qBpWgJz1hi+6lJ0JcLumkEStFlGoGqGw9c8uGi3CYy0U8WphFFBXlIoctysUMstDDltfjom350E1n8e2+nOlM/c8zhoXvbsfKjcEvvHU+E0vf867p2JA0PC9G2Mstd825aL7Z3oVr7Sn64nXx30R16RsSfgtdJs7Uf5Ht3VnHRabK7imSNxl2Sg2ILCC/enw1erMmnlq1Wfl7q4x+F8n9i9b79hddEYBVL+5ZsA4Hf+dRrNrYgYOueQTfvNeaJfvUqnb88MGVzj5BeHzoeeb5P4ooi1aVklm0rMXGUhWyGlSn5Kn/qug1jjwoylhY6CTDhX9egE/d/FJsl1UxDKpBUddCd6evA9b0btnCbk0n0dmXi3RpcFRZF2XrmXd9eTnihi325Uw73a+/EkaVL2cyz3T+plTCs3qSzzXBhcD+rIqkECd1ZE0mCLqb7ybuxLhcnuEuKa65IWkgl2doSEb70P0Dhe4L9epaN4+I+JJ+457XAmOtOQnDsFeHCm7O+TVeeterWLe9Bx87dGLgvoA1saitMYVs3oxeGMVkPh+6LEJvtVszUGUrEvCGj8YZxJSFti+XxzP2IG1YWtx0iOCJ7Vvhg6LxLHQR8RrSSXcmuDftLi+HWnjX7+xxFosH/JMDw8oQZqG/2d7phK22d/Rh0ojmwOP2h0Fmodthi4x5fOgJyYe+x6gW7DmmFYBtoccw0VUWuk/QbfFMJQjvbO3y5VsOoi+Xtyqo4sWMEvTebB7rtvc4n5savG24LCw5wWUCqF8sboVk8ibyJnNWav/Jw6vc35lmrJ6N6gVIJw1PtzfcQpd8v8LxtgvrOIr7RYk5ACcPT9gkJH6t/1y0HvPf2R5poXf05tBop5nI5sPvj2rCVdBi5KrVoxwBzZk+S1+FHP3x5uZOx+gJsyjluRYiL7zpzvblLrS4g6JRPvSocYC0MKaV8wyKhp/3gcUbPJ8TIS+/qucc1AMR3/W4wRDFECnoRPQHItpMREsDtg8lon8T0WIiWkZE55W+mKVBtNBlH3pKcME8+T/HOyP66WQiljCpGnJZbLnFkDQM/OPV9zy/EevNB/Yfg92GuXHYfTkT6VRCudpOlA/9krsWeTLeNTWoGxlO3omcsD4HrXYEWBVaTDj23BtbArPWBUXjqMSiIWl4LKhHlvmjQNyyBA/qiiutFzpTMWEQ8oyFJgbzx6GHn6OzL4fGVMJKqBZpobvuK05Qw9apyLnPr3f9zl787tm3Qs9lnc9b9l8/+YYzN2H9zmABChP0r/9tse/4ceOxo/YLHQeQyqXy5Qfx+uYOT11VGWr8OfflTM+qRlamzuje0KvvbvdMSiwlcSz02wDMDtl+MYDljLHpAI4H8HMiagjZv2LwhyPHoScMQirpfXDcBZNWxJerUC3fJfu3eaKqZIKwrSuDIU0pJ35drIBXztnfI/B9WRPppIG97V6DSNwJUfz4cjROkIXOUVpCdoXuzlqV8uCJw3zbAG+3VhUFBKiTd6WT3rTDYhdYRu46i0LAXUsdvVnH/xsXg0i5SpVInFwuIh29WaS5oEesPJU3/Y1J0ESizj6/Bb4xRIRV9GTyWPreTo+LiT+bddu7g35WQBgiw/auDF6KOSgYFR21QJGWV3z24nshNvpRvagtnRnsO67N+WwYhCcuOw5HTB3hfMcNq55sHiNaXKmzYv6jG6wfPrgS//fE65H7FUOkGjDGngUQ1kdlANrIUrRWe9/yND/9xJttUYpyMby3gvvELZdLcZOOZLHttlvlVMJAdyaP5lTCsYRF8U8Y5PE/cpfLl96/B47dZ7TnmHIkTRDcZydbHH4fOp8papFRvFhcaHhmxrFDGvHRQ3bD5BHNnnJ3Z9xqENQwqpJ3WRZ6uOC55Q12uXDk6JQ4GESxXS7yuYMGqjt6c2hMWS6XTM4M9c/nFOcOstDFtW9Nk+Gt9k587c5XA4+t4tv/WILT/u85z7gO980/E5ICI+4AX95kuP6J13HV/fHWHS0mw2YuoEfIn8u8t7fhyvvCo64AYO8xrqAniLDn6FbsPsqdi8JXt9rc0YvJgi88kzdjT7pqiREKXQyl8KHfAGB/AOsBLAFwCWNsYIIuC8TRMubNS2750CUL3eCCHs/lokJ2ufCuccIg9GTyaGpIOMIpDi7JA6y9WRPpZAKGQThiqndR5riDtiOa1Z2mIAs9jsuly2mgyHEjiIIqCk26AAs9zkpPbmoB6/+LT9jT81lE1W2OImGQ0koWkds63o0Oarx29WbRmEw4DVYYYloFTpB/Vuy+Z/ImXlkTPUYgw2PkxYFYXg/ltNEiUemROaff+HxB5SlG0MX7JT5z/v1ZN7/oCQgIQhyw5IOiogHIDZmNO/swQXCN9mTysRu4ahb0kwEsArAbgEMA3EBEQ1Q7EtEFRDSfiOa3t4cnvioH3NK2olwEi5jI2cZnhSYM10IvBitfjPc7brGmEoTuTA5NQqy5bKGLPLxso7MUniyMcZNTDVFEQqhwXwp3UI3DmGX9cZHjgp0wyHEjBAp6gMipXS4Je+3UYDZ39OHye17DZXdbftpme7A3LFVBIRhEVqKmMAtdOi5PVRC0Nmc2z9CYMpBKGOjLhfdAcnn/oKjocjlmr1HO3+KgaF/W9CzVVihigxinHYxKjywSNavZU44ipsqLcehel4t1rLhrpu42tNH5mw+Kisfr7M0hmzextasPY4a4+3Zn8rFcLgDQmi79+q1AaQT9PAD3MYs3ALwNYD/VjoyxmxljMxljM0ePHq3apawIBrrS/XLPl2fh7gtnARAEPZWI1fVXn8/7Q24dJA3uckk6vg1R8FQWJZ+iLAvjWKFC/ddJe2PZd09WlkUV2qZi6Xs7sb0r4/iDRRG5c95anPjzZ/DSW5YFyC3DZMJAQyKBTM6bq110uQT50FWTqxrsKJcwHX7fD5/A3+avxRY7BTF/WVWDZcVYe06Ui/3TFsUi4vJxuXWryuvDKWxQVO1y+e1nDsUBu7k2U7cwKNqXy3sSuRWK2ICH9WzGD23EuCGNsS10oLB1NaPSLqsQZ6GKZed1QrUQvIrxw5qcd57ruNgYdfblsLmjD4wB43yCXvsW+rsATgIAIhoLYF8A0cPqA4CYsQ8QE/IwTxeKdy1nTh2Bkfbq5NynLlvoE4Y14VMzJ0WeW+Uf5dkVkwahJ8tdLsw+j9flEmQYcku+MWXgte98CAdNcF/sloYEWtJJfP6oqT5fuziFOYxdvTl8+IbnnM+iaC2xJwCtkmYHpgzX5SIKepcgNEEi97Qd6/yjj01zr9GOQy/EsnZnqwanhi0EQ4hyGTskjS++fw/fPnLjwXO1BDVegFWf0nxQNOT8OYXLhffGEgZ5BKtTaDj7cv6GoBB6Y0ylB6z73dSQQE9MixQobCHqYix08TmLvu3/vLYB//P3xZ4ecRjjhzY6z8ZJvS1a6H05bLUNCb5SF0ee3RtExQSdiO4E8CKAfYloHRGdT0RfJqIv27tcC+AoIloC4AkAlzPGtgQdbyB57OvHYalgsYqZ5LzZFoNXG7Jmirovz8jWBvz4zIOV5zt0shvtocq8x2dNJhOWD725QRgUTUqDogHXxC30bJ5hSGPKUzb+kn/nIwfi9i8c4fkdt9CjpqcDwLrtPY6PWsyWGOR+CnS5ZIVB0QCR45bqMKEHMbw5pRS0MLigK1e3KUIcEkQw7YHJBJFy4ph8Lj5Y1tSQCHRXNKYSbhx6SNcvn/ev5MQb14RiaTxOXy6vLGtcxHseZiU3NyTtyTvlsdDl3k/UjGjr+O5vWtNJrLnuVIwdYgnuPQvWWT3iGLSmk86z4e+UeL+7+nKOO1HuuYlzH6LOUQ7iRLmczRgbzxhLMcYmMsZuZYzdxBi7yd6+njH2IcbYNMbYQYyxv5SlpEXQmEp4bpxjoUsuF4WeOw+wIWl45uGHhT3d95Wj8eOPTwvczrl7/jq8vrnTOyiaDHe5wNnPqkD8xRN3DftdmIWusp55ucQXK2gANpUwnPKLky26Y1joHNECSicTdm71+ALAXS4qH2Yx6zmKceiGQUq3m+xu6Ox1LfSmVAI/PMNfFxpTCaQScVwu/iiX1+zVsWQLXaQ3a0am8Y2CD/SFlZFb6J0xJ8cBhY1lyBOL4ljXYsPtpg8u7BiA9e7zu6uaWNTRl3OefaN0TFUjuMeoFvz1i+/zfNfSUB5BH1RT/znyxCKlhe6ELXofWJQ2eN7BCEOpKZVwLGFZ0OUY51GtVpSKbKmIVl6YoIf50FOGgV74M8cBkqAHWEliLhxR5Loy0Ra6UwYx+2WCrEHBAgQgbDWdYtZzJNvtlbMnTqlubY80IMjPk04Z1sLjigaQb4sSdFWUC0e20EX663LhZeTHCqIlnUQim/csthJF1DWLyBOLmhsSzjKJgb8RB3SdOSfud5Frt9okhAZcNfW/qy/nPPuwXC8AcOkH9sbph0zw9W7FCUmlZFBN/XctdCbFofv35c9RTs4Vt1IQCLP28Ke7FWkWLHQuiNMnDlUOwD1y6bFWeSRLV6xvYV3tMIsgpRAe1+Xivlg3B8w45GGLAPCosAarKHiJiAgHsbFI2qtJFWKhB7lc3mzvxJMrg9eFDYJbZjnTtF9wf/llC533BPjAp0rQedhinEHRoAYtYRA+dfhkHDZluK/n0JfL91/Qk9FpcptSidjjMpxCBjpll0uYdc3FUbxu/l6I3+2KaBA4yYTrZk04muFu7+x1BT1qoPXSD+yD3Ue1+IS/mgdFa4a9xrRi5pTh+MEZ06SZov7bwCtCOuXN5RJXYxgYGlMJ/PHzhwfuI+dVAYCTDxoHIvI88InDm5zBWtnSFfcLs9DDVmRSWXvuxCIhjW3AC5kwXPESXyAx5jcsJ4ZVPgOnH7IbfvHJ6UgmDGWURxhNDWqXy0k/fwY3PvVm7OM45bWrRCYX7EPv6M1hzysfdD7zLn9bOonmhqSyR+NEueRZ5EzRoAYtQYTRbWnce9FRmDTcm+SpL2eG+qq/+5EDg09qw63JMAFOJgiTR0Qv/CJSiIUu96r4ylpHKdYE4LM1xUbAmRUuCnpM91BSsNBV71Sn4HJpSiXwpy8cgTMPC0/MJjdILVUctlgzpJMJ3HPRUThsynDvxCLFm8VfCp/LJULRfYcKeWlbhEFR/vK02QJ9w6dnKNMCyF03MQ1rmGjyOFqVf1WZPiBkYpFMMqFOP9wjuFyCXARuGQjXnzUDHzt0omuhF+NyCRmkEyeBRMG72tm8GdpQqgYRL/nA3vj5J6crB5Fb0wl7pmg+PA49ZFKTGEkhx3bz/DpBTBkZneWP1zsxYkgua8Igxw0Yl0LCR+V9x9mx4arnywXda6HbPnThfe3ojWehWy426/e8HojvTafgcmlsSOC4fUbjvKOnhh6T14UvHL07nrzsuEgXZLEMKkEX8STgUViveYVvG4gWdHlzmIxNGO5NwAUArXb3cfzQJvzXSXvbZRAmIEnlaRV8cUGpPh/8r/d7BFXMSwGorfcOO8Y8jv85ZajdC2LYoqps0ya4S+qlPC4Xw/JfF+D75hOLekOswKtOOyD28XjjmM2b9iSj6LLkbKt7ysgWHDp5uPKetDYmnYigMJ5/YwveaO9EwiCsvNZNpXT2EZM8Ian8vvFGpzuT8wjbfV85ynPc5hiDcY7LRbiXzZLLIGUYvpC9KPrjchli13PVoDefBf3QUjeJm7gEn3vMePXJY6ErWt17FqzDFjvvPjckogY5iQhrrjsVV3/4AOwx2p+TqVQMWkGPtNDtCiWHLQYt1hxEWGja1JFulzXrWOiuxe0MzKaCLXRxcCUosmv/8W2eKJLbvnA4nv3GCe7vQsoYx6riYYsy3QEWejpp4JlvHI9bPjfT+c4j6HYDEydzHYd3aRev3RG4T9xEa4BbP6IsdBEu/vIxRNrSKTQkrFzda7f1+LZz7l+0Hg8u2YgEkWeG448+drBHlPl9H2OL686erJRG1nvNcSbXOC4XQdBl92AyQTjt4N0ijyUSlP5XhdyY8/uqGvQe3uLvKfBbHxbEEFTtEwY574RjoUtFf2zZJqQSbtpt1SzUxdd8KPjkZWLQCrqcbVHG9aEn8KVjd3e+3xaS1wLwV5IwKZgsdH8zkoUOeAdmOfKCAqIQBg2KEnlz1TQ3JDF5ZDOusyfz5EyG0w/xvpzc8ojj92xJJzyCfvKBYwHAmYI+fdIwj4VuEGHKyBaManUtPE+Ui7NGqXvuo/cKH2COCosEwif8yPB7mclbYYtxIm4sQXc/q+oVt9BLBX/++4+3Jpjt6PYK+tSRLR7/bixBV8y6ldMupxIGEgbh7COiJ9lxCrHQ5X35vVQN1Kp6CvwOhIWsBkVtEZHTaw8ydnIm84i4qv7FiZ0vNYNW0OXkXDJOCFrSwIn7jcXq758S67g+l0uIoov+b8flkhYF3R86GVZJQgdFFQO/XARyeRPXnzUD0ye6LpBDJlmTpOJM7W5JJ5EWynXR8Xt5tt9/8dGecvNiiuX1Wuj+sLnxQ8P933FenjiiL5cxmzORoOA1KEWyeebpkfFjzBAmnLUVKOhRIsgFZ5+xbWhrTFoWulAJG5IGfvaJ6c7nsBWGnH0Ug6JylAZvdFX1KojCBkXdfQ+aMMR5FyYrVvoZ3eoXdN6oye2w+D6GjevwbfI7NWfaOADAu9u6PYtWqHziYYEI5WLQCnpUEnvHQrcrN38JRyi6d2EEpUjlEw14MjBe2Vs8E6GEyU02YW6DMNdJQtF15BYGn8QhCgEXoTh5tVsavCLFZ+eJDGn0N1QiKSlsEbDuyRkzJuDtH80JjaMPCiuUKWR2Hr9fPGwxjg+9J5v3KL/jxxVUpS2dip3DPg481n/P0S0Y2pTyuVzkOhEnQZZqMFd2uTgWbIQ7at6VJzl/FzIoyn3lj339WPz7q8c49/Kk/cfgtIPHe/Ydo6hvQZb5c5ef6Pwd9kR5QyWP/QQ1YKp7FhUIUA4GraBHW+hWhRAtv7nfPAGPft2KB7/0A3vjm7P39f3O53IJeKbcCr3rgln48/lHKKcS82KJqW9VFYe/pGHrH6rKwS1WPl4gvm8fnm65YKLWvgSA5nTC0/0cLpT3qydY1rqY7VFVFnliEWAJJBfrMD2N6+NuTiedXkkUbpQLczIvAl5rW0VGFTonlD3KQi/UHfOV4/fC6YfshuP3HYNhzX5B59fxyKXH4scfn+bJMhqEytpskgwJfpyoBmLMkEZ8/6MHASgu4VZjKgEiwueOmoJJI5rw4em7eQbTgXALXWbCsCbsO9bKdx7WSPNnxxtEvmdQY6x694pdR6E/DFpB9+ZyCbbQRbGYNKLZ8fte+oF98BXJtaAi6JHyF2F0Wxrv33s0Pn7YBADAUGGyBk/2JPoIVa6FYbaAytbY4VLudBkuwnwAiluSPznz4NjCB1gWujgeIDY6c6ZZ1pTHlaS43+LkpkYhvQE/Vth09riWUMog/OurR+OBrx3jfBdk+fN7mcmZnpm7J+w7Jta5ADfUbvZB45zvZPfUoZOHeXo0Qflybv/CEbj3oqN835968Hhcf9YMjG5LY1hTA3Z0Z5Ritu+4Nnzq8MnKSWQyKteUHB1zwn5WpI1qDgeHPxfursnmTBw2JbxOir8D3MHuKSNbMPebJ2LskEZfAy6msOWERUg5C5mHGAlJpwdifR5rv4PjhdS6cZL0DTSDV9AV2RZFeBx6sp/d44MmDsWQxiQ+Kg06yi395Sfvh5XXzvZYRzw1rBjvq2r1ufjK1vRdF8zC6z8I9v1z4eSWE3e5yBZQlO85YZBnPEAsI4/CaVGMDYiIlqN3sMn6mxtTQxRTpuNa6MmElYt8ohAueu9Fs3D+Mbv79uWNTibvdbkU0ose09aIxdd8CF85fk/H8pZnkFpZF8UIILWP+9h9RkeK4dCmFHZIFrpMnMZPbaG73739ozk4bIoV+hpmoTtWrnAv0wEzaEWGCT081SCu/LxVg6Jh98CxzMME3T4Hr6vnzJqK/zt7Bs4SBoHPKmBAeKAYvILuyeWiEnQzcFsYB9lieNw+liU3pDGF175zZN44fQAAFMJJREFUMo4WFiQA/AMmhkG+0CeeA310m98CETl2b+vY8gCmtRKT9Yi5JXr47m4MOvfH88bLWUJNeuHet7s3wuSmzx6KP5/vzeYYBHe1iDPjxCv//FFTfecUIyrkxkT1nsb1SXMhEPffa0wb/vuD+/j25QLW1ZfzpDMWG6s44ylDm1IgIjx66bH43TmHAfBeqxy10R/7oTWdRFdfLlTMeA/v0BDXkWqcRvwubv4gvs1pHHPxQkBHtLjGgSoySfaUqFJlhKWN4FvCXC68By82Sh+evpunh1zKsZBSMSiTcwFytkWFoOe5hR5eAX/xyemeYx2421As/97Jvi6qfJw40QFDmqxjyHndZb5w9O4YM6QRc4SuvczYIY149OvHemLf00kDbY1JXD7bWo/EcTNJFvS5s6bgmdXuClOzD/IOSnEe/+/jfDP5uKtFnHghCsLVpx2AK07Zz3MPxZeY/81dHiqximuhc2vS9ywUz5j3LLozedtCt74XexcvXHEi9rvq4VjnnjqqBVPtdSlFIcjkTXzx/bvj+/9ZASB4ED0OrY1JdPbmQlfmMQzCI5cei7FD0jjke48BAP543uE474+vOPs0KizooAH3MINH9kNnpDj9ILiF3tyQUL6bcg1Q9VrDXC4xDHRfYyR/DwRrw0/PPBhrtwfPMSgng1bQo1wpXDiihPdjh/pzOKhm48m+xjhhdleddgCO33cMDp4YPhBnGISPTI+e5LHP2DbPZyLCku+4+eJV4wZA/NWO9hrjnwHHj+V1ubjbDYPQaHgFSExJ6rhceBkVVlXcXhR/lvze/++p+3u+F/FO2CLHhy9qR2PKynteaC4s2UL/4vv3QDbP8OOHVxa9OhZg3eOuTB5DIway9x3X5slmeMK+Y/C7cw7DhX9eAEAd2hj0voS9R651a33O5EzrWYXcr8OnDsfuI1sw7+1tsVcYUhEWf+760IMLkgqIQ096orHU137yQeM8LsiBpPr6DANE1Oj8r8+egdMP2c0JK+z3+eTwpxjhY22NKWdQcSDgVrBslcRdizEM0W8d5arwWOhS9z+uhf4JRbIkLvx8GjZfhUg5o1PKkcMcC10+ZuGvkNiY89mwpQhx43mAOmIkoZKvecYk12hQDcwGDdbGieV2p+GHR2IBwN+/fBSG2S6XoAyLvJ4etedI/OMr/oFiINyH7ljoMSKn5PskXm+QhhQyga3UDFoLvbkhievPOkRpVQLWQOP1Z80o2fl8FaMCkw6i+MN5h+OvL7+L8VLUQFBZr/vYtMAX9JZzZ2LN1i7n87DmBqz43mzcPX8tPnjA2NByiC9yWhoUPWzycMxbsw0HTxzqLPig8mUetddI/H3BOs93UWICALMPHIfj9h3ts9B5BJDsMijGovZY6Dl3FSIgMoV+KLwXxPPwHLdP8Lq9sptCrJ+qQdGw1aqCUIlinIaLrywUlcBq33FtmDFZPVAclnFSZZknDPI0Aryhlp9v0hNeq74nhSyGXWoGrYUOAKcfMgEH7jY0escSIItif6YF//JT03GrkAelVOw3bgi+d/pBiskU6gp61hGT8cmA0K0PHDDWtw5nU0MCnztqKnaLyHooWuWuy8V62U6bPh4vfeskXHyCGzKqEpViBqwSBuGmcw7D2UdM9ljohuEuCRgmhHFJKwTdST9cQA54GTFtxGePnIw/fSHewDXg7WmI5eMpIYIiU8S6MWVkM34qLM+ompxjGIQzZkwILQt3tfRHFrk4X2InuBORb/EPzjgIc795guc71UQ8wBuNJfe6OZWIP+cMakEfSGQfen/CIc+YMREn7R9u5ZaSsFjjciCGyPEBOjHKZNzQRpx84DicarujVA1OoYL+m88cisfsSWOAFTnB38uGhBEYtljMqyuWjQsPFzGeciLuuIVIqxBJFJV/XkbMOCq62D7zvikAgq39pBT1oVraUSxL0iD84IyDsPjq4MRVcZeKCxtA5vf16x/cxzPvAIBvNbBZe4z0GRncAJMtfbFx6m9IczkYtC6XgUYWnUpMCy6WgS6rKg6dv1diSSba0T8ql1ChMy7lsQoiQms6iY7eHNIpw21QpN8VsyCzWLZrT7cWnHAm3+RNrPjebLy3owcf+MUzBR1XjCSK0wh/8ICx+JhtLYuiK4YoHrH7CKy57tTApHTighOphOG5H9xCF+tPggjJhIGhzcHl4+6uTbui004EIQqx3IuS+0Cqxp+XWV4Kz7OPL3KNCloIuxxUXxNTp8iiWI0xrEEMtL9f7PK74u6PMuHiwcVr3rfdvCGlyHTHIxU8Frr0HPvrQz9n1lQAbmRPNm+iqSFRUCIxTmuMVMoivz93Jk6xGzKvD12RyyVgYHzKyBbce9EsAEBDgqTnY/8fECIcNFHqhP2sORyzFKsTAfFWDfP6w70PiT/Ln3z8YBy5xwjP7E8Or1NhAi2nUXj+ihPx0CXvjy5cGdEW+gAhi2IxvtdKMdBlFX2QXNhcC9lr7QHuCztGmIBVijI7K0alDHTZA43yUYs5j6qxaU55XS7FNPie1asKdJN589XHHxQF3N5TMmF4np08OUc+z70XHYXbX1yDq+9f5jnekMYUXr3qgwXlrxc5ddp4fF2YLCa7Rnh5Z+05Ep88XD0GxAc2Q2fdSu/02CGNGKtIQzCQ1I6ZWOMMtB+6lBQTmlcq5Kn/KgtQJapO1Eg/dJ2fO50wXJePdMCiXC4KseZ+Y3f+Q+HHFROgFdoeiJazyl0VFiHEE9mlEqTMBy/eI/k4586aiuevONF3zOEtDZGrKwXd+hs+PcMTvSbfS16Xwhpjvlxh2GLO1eg2jbTQiegPAE4DsJkxdlDAPscD+BWAFIAtjLHjSlnIeqAaH34Ut3/hCOw2rKmiIZbOTFHuchG2kWShi3ARaUgYsTJGquDHTacSQhy67HLpnw+d48s3XoSF3pZOgsgSrP4YEIXWVbFXIfag3ARX4WGLhb4afOb0HgFzRIIikfh5+KBomKB/Y/a+mDZxqJNWI855qoE4LpfbANwA4HbVRiIaBuA3AGYzxt4lovjp6AYRxVhylYavXSkuJTdQPPb1Y/HXee961l2VUVmAMlzQi4kNJqFR4D58WQOKcdVzQRdFXI7sKKa8huEO5BYa5aIqX1yyOTfVtHh/vvMRa8BXLIvqWRX6bpx84DjcfeGsyGyiHN6wyOuMhp03nUzg9EPCwyurkUhBZ4w9S0RTQ3b5NID7GGPv2vtvLk3R6pP9xrXh4UuPjd6xiqiEv3/vsW245sMHOp/DXC7q99K2GpMG0Bc8qBcGP246ZThrU8oiUExDnTAIl8/ez0lBC/jLV+w9b7MFvT+9qpEFLuLiulwMx6XywQPG4lB70o/YWQjrTcWFiHDE7t6FzptSicDVteR4eO4Vr6VxrLiUwjm6D4DhRPQ0ES0gonODdiSiC4hoPhHNb29vD9qtLgnL513tVNKHznEm9ghdem5BqwTBGahT5JKJi+i2+eqJe+GI3Udg9rRxyn0K5aLj98R+49yc8z4Lvch7ziNd+tMjLNTdc8Bu1nV8dMYE5+mIsd5Ri8mUQldVfnj5nPw87qzf/p+32ihFlEsSwGEATgLQBOBFInqJMbZa3pExdjOAmwFg5syZtatw/aAa/W5RVIMlw1xFd5BfVNX+3HosZPk5Dj9sOmVg0ohm3H2hFZ73gzMOcqJQ5HsTd+1ZGTn/R5w0BSr4dZZrzEbV05kysgVrrjsVAPD48k0AvKGFostFLej9L2tYfiAnfFJafagW38UoSiHo6wBsZYx1AegiomcBTAfgE/TBTD9mdGugHhSVX1QR7ifdc3Qr9h8/BOcdPbXgc7oWulfE+OxJABjenMK729xthfqfnXOVSIBb7dDFYo83aw917DcALP3uyZFWLX8UYr4UI9JCL6+w8jDMzx5pPbegRGtAZfOwlIJSCPr9AG4goiSABgDvA/DLEhy3ruBL14WNmmtCUIQNciEIs7TSKQPXflQZnBUN96GHiHShi4ZHMTPGEm1hcP93MRb6ymtnh/bG4vRyxMyKnKhBUSqzR68haWD1909xxPpbc/bDt/+x1NfbePjS93vW761F4oQt3gngeACjiGgdgGtghSeCMXYTY2wFET0M4DUAJoBbGGNLy1fk2mTc0EY8d/kJGD80PDGVRo2cWxtwxUGlQYdPHYHzjp6KL0kJwgqBHzfM6h6pWKC4WF696oOx85gEwQU9aIAwjFKkSVZZ6FFhi/2JyImL+Aw/874pnl4WRxzTqFXiRLmcHWOfnwL4aUlKVMdMHN4cvZNGyZVz9kdLOolTp7kLeYS5XBIGeaJkisEIGXTlFBoREsbwEhxrlL2+5lZ7PdqBRhmWKM5CVcz+rMWQ3mpFT/3X1ATDWxqcuGYOF4pyBeG4g2jBAyBTRpZmAZRSwV17WzrVybTKjdJC96zD6u/RaD0vHZWPR9NoiiSOD70/uOIUvM+nDp/kZEysBvYbZy0zOHlEZXqDTiMo3DOxwRUXgJZ/o+k/2kLX1CxxXCL9gecoDxtfTBiEc2ZNxVVSgqlSUejg5kEThuLfXz0G+41vi945gmIahSgLfbhi0LEawmILYe43T3DCYasNLeiamsWZKVqm4//gjGnYa0wrjtozOjLpga8dg86+0qZIeOYbxxc1IWraxP6vwrX4mg8VlYKYu3zEAUZRsFVRQTWm55hUod5PHLSga2oWI2RiUSkY1ZrGN07eL9a+B00o/VKGlfTPF7NiEgDsM7YN9150FKYJ98OIEPR6nOBTKbQPXROLGz99aKWL4IPLgCgY8759Ep75xvEVKY/G4rApwz1hgqLLpZgZu5r46LuricWpB4/HxOFHe1bFqTSq/C7iIhea6qAlncQ3Tt4XB4wfoq3xMlM9b6em6pk+aVili+AlZAq3prq4+IS9Kl2EQYEWdE3Nko+R11ozeLj3ollYu62n0sWoKFrQNTWLu3BzhQuiqQoOmzICh/ln9A8q9KugqVmC1vnUaAYrWtA1NQtj9btQgUZTDFrQNTULX3lmILL1aTS1gBZ0Tc2iXS4ajRc9KKqpWdylxCpaDE0JOPuIyThyjxHRO2pC0YKuqVncxX61otc6P/rYtEoXoS7QLhdNzWLqQVGNxoMWdE3Nwl0u2kLXaCy0oGtqlrztctGDohqNhRZ0Tc3CV3FPhyzirNEMJvSgqKZmOXfWVGzpzODC4/aodFE0mqpAC7qmZmlMJXDlnP0rXQyNpmqI7KsS0R+IaDMRLY3Y73AiyhHRmaUrnkaj0WjiEsf5eBuA2WE7EFECwI8BPFqCMmk0Go2mCCIFnTH2LIBtEbt9DcC9ADaXolAajUajKZx+hwcQ0QQAZwD4bf+Lo9FoNJpiKUW8168AXM4YM6N2JKILiGg+Ec1vb28vwak1Go1GwylFlMtMAHfZkztGAZhDRDnG2D/lHRljNwO4GQBmzpzJ5O0ajUajKZ5+CzpjbHf+NxHdBuABlZhrNBqNprxECjoR3QngeACjiGgdgGsApACAMXZTWUun0Wg0mtgQX8ZrwE9M1A7gnSJ/PgrAlhIWp5Loa6lO9LVUH/VyHUD/rmUKY2y0akPFBL0/ENF8xtjMSpejFOhrqU70tVQf9XIdQPmuRWc10mg0mjpBC7pGo9HUCbUq6DdXugAlRF9LdaKvpfqol+sAynQtNelD12g0Go2fWrXQNRqNRiOhBV2j0WjqhJoTdCKaTUSriOgNIrqi0uWJQpVPnohGENFjRPS6/f9w+3siol/b1/YaER1auZJ7IaJJRPQUES0nomVEdIn9fS1eSyMRzSOixfa1fNf+fncietku89+IqMH+Pm1/fsPePrWS5VdBRAkiepWIHrA/1+S1ENEaIlpCRIuIaL79Xc3VMQAgomFEdA8RrSSiFUQ0q9zXUlOCbuddvxHAKQAOAHA2ER1Q2VJFchv8+eSvAPAEY2xvAE/YnwHruva2/12A6spgmQNwGWPsAABHArjYvve1eC19AE5kjE0HcAiA2UR0JKyc/r9kjO0FYDuA8+39zwew3f7+l/Z+1cYlAFYIn2v5Wk5gjB0ixGnXYh0DgOsBPMwY2w/AdFjPp7zXwhirmX8AZgF4RPj8LQDfqnS5YpR7KoClwudVAMbbf48HsMr++3cAzlbtV23/ANwP4IO1fi0AmgEsBPA+WDP3knJdA/AIgFn230l7P6p02YVrmGiLw4kAHgBANXwtawCMkr6ruToGYCiAt+V7W+5rqSkLHcAEAGuFz+vs72qNsYyxDfbfGwGMtf+uieuzu+kzALyMGr0W20WxCNaiLI8BeBPADsZYzt5FLK9zLfb2nQBGDmyJQ/kVgG8C4CmsR6J2r4UBeJSIFhDRBfZ3tVjHdgfQDuCPtivsFiJqQZmvpdYEve5gVnNcM7GjRNQKa3WqSxlju8RttXQtjLE8Y+wQWNbtEQD2q3CRioKITgOwmTG2oNJlKRHHMMYOheWCuJiIjhU31lAdSwI4FMBvGWMzAHTBda8AKM+11JqgvwdgkvB5ov1drbGJiMYDgP0/X7qvqq+PiFKwxPwOxth99tc1eS0cxtgOAE/BcksMIyKegVQsr3Mt9vahALYOcFGDOBrAR4hoDYC7YLldrkdtXgsYY+/Z/28G8A9YjW0t1rF1ANYxxl62P98DS+DLei21JuivANjbHsFvAHAWgH9VuEzF8C8An7P//hwsfzT//lx7xPtIADuF7llFISICcCuAFYyxXwibavFaRhPRMPvvJlhjAStgCfuZ9m7ytfBrPBPAk7Z1VXEYY99ijE1kjE2F9T48yRj7DGrwWoiohYja+N8APgRgKWqwjjHGNgJYS0T72l+dBGA5yn0tlR48KGKwYQ6A1bB8nt+udHlilPdOABsAZGG12ufD8lk+AeB1AI8DGGHvS7CieN4EsATAzEqXX7iOY2B1D18DsMj+N6dGr+VgAK/a17IUwNX293sAmAfgDQB/B5C2v2+0P79hb9+j0tcQcF3Hw1pgpiavxS7zYvvfMv5+12Ids8t3CID5dj37J4Dh5b4WPfVfo9Fo6oRac7loNBqNJgAt6BqNRlMnaEHXaDSaOkELukaj0dQJWtA1Go2mTtCCrtFoNHWCFnSNRqOpE/4f2t2UhN5c8vwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "36.934999999999995\n"
     ]
    }
   ],
   "source": [
    "accuracy_train = (1-(np.sum(np.abs((Y_pred >= np.max(Y_pred,axis=0)) * 1 - Y_train)))/2/Y_train.shape[1]) * 100\n",
    "print(accuracy_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}