PyTorchCropImages.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "PyTorch Dataset: https://pytorch.org/tutorials/beginner/data_loading_tutorial.html\n",
    "\n",
    "NOTE:\n",
    "1. The validation range is changed to small size for debug.\n",
    "2. Mean:  [0.90960454 0.81946206 0.87811487]\n",
    "   Std:  [0.13244118 0.24944844 0.16392948]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F # stateless functions\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torch.utils.data import sampler\n",
    "\n",
    "import os\n",
    "import pandas as pd\n",
    "from skimage import io, transform\n",
    "import numpy as np\n",
    "import copy\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "#import torchvision.datasets as dset\n",
    "import torchvision.transforms as T\n",
    "import torchvision.models as models\n",
    "\n",
    "import time\n",
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "using device: cpu\n"
     ]
    }
   ],
   "source": [
    "image_cropped_dir = '../yi_data/train_crop_128'\n",
    "NUM_TRAIN = 6800 #8492\n",
    "\n",
    "USE_GPU = False\n",
    "dtype = torch.float32   # use float throughout the training\n",
    "print_every = 100\n",
    "\n",
    "SCALE_SZ = 128\n",
    "BATCH_SZ = 8\n",
    "\n",
    "if USE_GPU and torch.cuda.is_available():\n",
    "    device = torch.device('cuda')\n",
    "else:\n",
    "    device = torch.device('cpu')\n",
    "    \n",
    "print('using device:', device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Customized Dataset\n",
    "class ProstateCancerDataset(Dataset):\n",
    "    \"\"\"Prostate Cancer Biopsy Dataset\"\"\"\n",
    "    \n",
    "    def __init__(self, csv_file, root_dir, transform=None):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            csv_file (string): Path to the csv file\n",
    "            root_dir (string): Path to the directory with all images\n",
    "            transform (callable, optional): Optional transform to be applied on an image sample\n",
    "        \"\"\"\n",
    "        # Shuffle dataframes with fixed seed; otherwise, validation set only get cancerous samples\n",
    "        self.cancer_df = pd.read_csv(csv_file).sample(frac=1, random_state=1)\n",
    "        self.root_dir = root_dir\n",
    "        self.transform = transform\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.cancer_df)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        img_path = os.path.join(self.root_dir, f'{self.cancer_df.iloc[idx, 0]}_0.png')\n",
    "        # (D,W,H)\n",
    "        img = io.imread(img_path)\n",
    "        isup = self.cancer_df.iloc[idx, 2]\n",
    "        gleason = self.cancer_df.iloc[idx, 3]\n",
    "        \n",
    "        if self.transform:\n",
    "            img = self.transform(img)\n",
    "        sample = {'image': img, 'isup_grade': isup, 'gleason_score': gleason}\n",
    "        return sample        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##Customize Transforms\n",
    "class Rescale(object):\n",
    "    \"\"\"Rescale the image sample to the given size\n",
    "    Args:\n",
    "        output_size (tuple): Desired output size. Output is matched to output_size.\n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self, output_size):\n",
    "        assert isinstance(output_size, tuple)\n",
    "        self.output_size = output_size\n",
    "        \n",
    "    def __call__(self, sample):\n",
    "        img = transform.resize(sample['image'], self.output_size)\n",
    "        return {'image': img, 'isup_grade': sample['isup_grade'], 'gleason_score': sample['gleason_score']}\n",
    "    \n",
    "\"\"\"\n",
    "class ToTensor(object):\n",
    "    \"\"\"Convert ndarrays in sample to Tensors.\"\"\"\n",
    "    \n",
    "    def __call__(self, sample):\n",
    "        img = sample['image']        \n",
    "        # Swap color axis to [C,H,W]\n",
    "        img = img.transpose(2,0,1)\n",
    "        return {'image': img, 'isup_grade': sample['isup_grade'], 'gleason_score': sample['gleason_score']}\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compse tranforms of totensor\n",
    "# More transformer to try: crop, normalize, etc.\n",
    "# And transformer is a useful tool for data augmentation\n",
    "biopsy_train = ProstateCancerDataset(csv_file='train_512.csv',\n",
    "                                     root_dir=image_cropped_dir,\n",
    "                                     transform=T.Compose([\n",
    "                                                 T.ToTensor(),\n",
    "                                                 T.Normalize((0.90960454,0.81946206,0.87811487),\n",
    "                                                             (0.13244118,0.24944844,0.16392948)),\n",
    "                                     ]))\n",
    "\n",
    "loader_train = DataLoader(biopsy_train, batch_size=BATCH_SZ, num_workers=4,\n",
    "                          sampler=sampler.SubsetRandomSampler(range(NUM_TRAIN)))\n",
    "\n",
    "loader_val = DataLoader(biopsy_train, batch_size=BATCH_SZ, num_workers=4,\n",
    "                        sampler=sampler.SubsetRandomSampler(range(NUM_TRAIN, 8492)))  #NUM_TRAIN+200  Use smaller size for debug"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "isup_distr = defaultdict(int)\n",
    "for batch_i, batch_sample in enumerate(loader_train):\n",
    "    #for grade in batch_sample['isup_grade']:\n",
    "        #print(int(grade))\n",
    "    #    isup_distr[int(grade)] += 1\n",
    "    print(batch_sample['image'][0].shape)\n",
    "    #for s_i, sample in enumerate(batch_sample['isup_grade']):\n",
    "    #    print(sample)\n",
    "    print(batch_sample['isup_grade'])\n",
    "    print(batch_sample['image'][0])\n",
    "    plt.imshow(batch_sample['image'][0].permute(1,2,0))\n",
    "    assert False\n",
    "print(isup_distr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Two-Layer Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_accuracy(loader, model):\n",
    "    num_correct = 0\n",
    "    num_samples = 0\n",
    "    model.eval() # Set model to evaluation mode\n",
    "    with torch.no_grad():\n",
    "        for batch in loader:\n",
    "            x = batch['image'].to(device=device, dtype=dtype)\n",
    "            y = batch['isup_grade'].to(device=device, dtype=torch.long)\n",
    "            scores = model(x)\n",
    "            _, preds = scores.max(1)\n",
    "            num_correct += (preds==y).sum()\n",
    "            num_samples += preds.size(0)\n",
    "            #print(batch['isup_grade'].unique(), batch['isup_grade'][preds==y].unique())\n",
    "            #print(batch['isup_grade'])\n",
    "        acc = float(num_correct) / num_samples\n",
    "        print('Got {:d}/{:d} correct {:.2f}'.format(num_correct, num_samples, acc*100))\n",
    "\n",
    "def flatten(x):\n",
    "    N = x.shape[0] # read in N, C, H, W\n",
    "    return x.view(N, -1)\n",
    "\n",
    "def train_sequential(model, optimizer, scheduler, epochs=1):\n",
    "    \"\"\"\n",
    "    Train a model using PyTorch Sequential API.\n",
    "    \n",
    "    Inputs:\n",
    "    - model: A PyTorch Module giving the model to train.\n",
    "    - optimizer: An Optimizer object we will use to train the model.\n",
    "    - epochs: The expected usage number of each image.\n",
    "    \n",
    "    Output: Print model accuracies.\n",
    "    \"\"\"\n",
    "    model = model.to(device=device)\n",
    "    for e in range(epochs):\n",
    "        print(f'Epoch {e}')\n",
    "        for t, batch in enumerate(loader_train):\n",
    "            x = batch['image'].to(device=device, dtype=dtype)\n",
    "            y = batch['isup_grade'].to(device=device, dtype=torch.long)\n",
    "            \n",
    "            scores = model(x)\n",
    "            loss = F.cross_entropy(scores, y)\n",
    "            \n",
    "            # Zero out all of the gradients for the variables which the optimizer\n",
    "            # will update.\n",
    "            optimizer.zero_grad()\n",
    "            \n",
    "            # Backward pass: compute the gradient of the loss with respect to\n",
    "            # each parameter of the model.\n",
    "            loss.backward()\n",
    "            \n",
    "            # Update the parameters of the model using the gradients computed by\n",
    "            # the backward pass.\n",
    "            optimizer.step()\n",
    "            \n",
    "            if t % print_every == 0:\n",
    "                print('Iteration {:d}, loss = {:.4f}'.format(t, loss.item()))\n",
    "                check_accuracy(loader_val, model)\n",
    "                # Decay learning rate after each validation check\n",
    "        scheduler.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0\n",
      "Iteration 0, loss = 1.7992\n",
      "Got 436/1692 correct 25.77\n",
      "Iteration 100, loss = 2.9562\n",
      "Got 453/1692 correct 26.77\n",
      "Epoch 1\n",
      "Iteration 0, loss = 1.9555\n",
      "Got 443/1692 correct 26.18\n",
      "Iteration 100, loss = 1.6102\n",
      "Got 450/1692 correct 26.60\n",
      "Epoch 2\n",
      "Iteration 0, loss = 2.2169\n",
      "Got 452/1692 correct 26.71\n",
      "Iteration 100, loss = 1.5697\n",
      "Got 454/1692 correct 26.83\n",
      "Epoch 3\n",
      "Iteration 0, loss = 1.6583\n",
      "Got 450/1692 correct 26.60\n",
      "Iteration 100, loss = 1.7210\n",
      "Got 459/1692 correct 27.13\n",
      "Epoch 4\n",
      "Iteration 0, loss = 1.5495\n",
      "Got 460/1692 correct 27.19\n",
      "Iteration 100, loss = 1.5922\n",
      "Got 463/1692 correct 27.36\n",
      "Epoch 5\n",
      "Iteration 0, loss = 1.5628\n",
      "Got 461/1692 correct 27.25\n",
      "Iteration 100, loss = 1.5773\n",
      "Got 461/1692 correct 27.25\n",
      "Epoch 6\n",
      "Iteration 0, loss = 1.5719\n",
      "Got 461/1692 correct 27.25\n",
      "Iteration 100, loss = 1.6990\n",
      "Got 463/1692 correct 27.36\n",
      "Epoch 7\n",
      "Iteration 0, loss = 1.6314\n",
      "Got 463/1692 correct 27.36\n",
      "Iteration 100, loss = 1.5721\n",
      "Got 464/1692 correct 27.42\n",
      "Epoch 8\n",
      "Iteration 0, loss = 1.6030\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 100, loss = 1.5235\n",
      "Got 464/1692 correct 27.42\n",
      "Epoch 9\n",
      "Iteration 0, loss = 1.5961\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 100, loss = 1.5918\n",
      "Got 464/1692 correct 27.42\n",
      "Epoch 10\n",
      "Iteration 0, loss = 1.5810\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 100, loss = 1.6121\n",
      "Got 464/1692 correct 27.42\n",
      "Epoch 11\n",
      "Iteration 0, loss = 1.6059\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 100, loss = 1.6501\n",
      "Got 464/1692 correct 27.42\n",
      "Epoch 12\n",
      "Iteration 0, loss = 1.5700\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 100, loss = 1.6175\n",
      "Got 464/1692 correct 27.42\n",
      "Epoch 13\n",
      "Iteration 0, loss = 1.5490\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 100, loss = 1.6505\n",
      "Got 464/1692 correct 27.42\n",
      "Epoch 14\n",
      "Iteration 0, loss = 1.6672\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 100, loss = 1.5217\n",
      "Got 464/1692 correct 27.42\n",
      "Epoch 15\n",
      "Iteration 0, loss = 1.6055\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 100, loss = 1.6484\n",
      "Got 464/1692 correct 27.42\n",
      "Epoch 16\n",
      "Iteration 0, loss = 1.5884\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 100, loss = 1.6557\n",
      "Got 464/1692 correct 27.42\n",
      "Epoch 17\n",
      "Iteration 0, loss = 1.5626\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 100, loss = 1.6475\n",
      "Got 464/1692 correct 27.42\n",
      "Epoch 18\n",
      "Iteration 0, loss = 1.5855\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 100, loss = 1.5072\n",
      "Got 464/1692 correct 27.42\n",
      "Epoch 19\n",
      "Iteration 0, loss = 1.5856\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 100, loss = 1.4874\n",
      "Got 464/1692 correct 27.42\n",
      "Epoch 20\n",
      "Iteration 0, loss = 1.4962\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 100, loss = 1.6280\n",
      "Got 464/1692 correct 27.42\n",
      "Epoch 21\n",
      "Iteration 0, loss = 1.6494\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 100, loss = 1.5916\n",
      "Got 464/1692 correct 27.42\n",
      "Epoch 22\n",
      "Iteration 0, loss = 1.6643\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 100, loss = 1.6766\n",
      "Got 464/1692 correct 27.42\n",
      "Epoch 23\n",
      "Iteration 0, loss = 1.4305\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 100, loss = 1.5423\n",
      "Got 464/1692 correct 27.42\n",
      "Epoch 24\n",
      "Iteration 0, loss = 1.6202\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 100, loss = 1.4705\n",
      "Got 464/1692 correct 27.42\n",
      "Epoch 25\n",
      "Iteration 0, loss = 1.5314\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 100, loss = 1.4792\n",
      "Got 464/1692 correct 27.42\n",
      "Epoch 26\n",
      "Iteration 0, loss = 1.5988\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 100, loss = 1.5303\n",
      "Got 464/1692 correct 27.42\n",
      "Epoch 27\n",
      "Iteration 0, loss = 1.6203\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 100, loss = 1.5199\n",
      "Got 464/1692 correct 27.42\n",
      "Epoch 28\n",
      "Iteration 0, loss = 1.7177\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 100, loss = 1.6355\n",
      "Got 464/1692 correct 27.42\n",
      "Epoch 29\n",
      "Iteration 0, loss = 1.6581\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 100, loss = 1.6317\n",
      "Got 464/1692 correct 27.42\n"
     ]
    }
   ],
   "source": [
    "class Flatten(nn.Module):\n",
    "    def forward(self, x):\n",
    "        return flatten(x)\n",
    "    \n",
    "hidden_layer_size = 100\n",
    "learning_rate = 3e-3\n",
    "\n",
    "model = nn.Sequential(\n",
    "    Flatten(),\n",
    "    nn.Linear(3*SCALE_SZ*SCALE_SZ, hidden_layer_size),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(hidden_layer_size, 6)\n",
    ")\n",
    "\n",
    "# Use Nesterov momentum\n",
    "\"\"\"\n",
    "optimizer = optim.SGD(model.parameters(),\n",
    "                      lr=learning_rate,\n",
    "                      momentum=.9,\n",
    "                      nesterov=True)\n",
    "\"\"\"\n",
    "optimizer = optim.Adam(model.parameters(),\n",
    "                       lr=learning_rate)\n",
    "\n",
    "# Learning Rate scheduler\n",
    "scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=0.5)\n",
    "\n",
    "#train_sequential(model, optimizer, scheduler, 30)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CNN\n",
    "Reference: https://pytorch.org/tutorials/beginner/finetuning_torchvision_models_tutorial.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model, dataloaders, criterion, optimizer, scheduler, num_epochs=25):\n",
    "    since = time.time()\n",
    "    \n",
    "    train_acc_history = []\n",
    "    val_acc_history = []\n",
    "    loss_history = []\n",
    "    \n",
    "    best_model_wts = copy.deepcopy(model.state_dict())\n",
    "    best_acc = 0.0\n",
    "    \n",
    "    for epoch in range(num_epochs):\n",
    "        print(f'Epoch {epoch}/{num_epochs-1}')\n",
    "        print('-'*10)\n",
    "        \n",
    "        # Each epoch has a training and validation phase\n",
    "        for phase in ['train', 'val']:\n",
    "            if phase == 'train':\n",
    "                model.train()   # Set model to training phase\n",
    "            else:\n",
    "                model.eval()    # Set model to evaluate phase\n",
    "                \n",
    "            running_loss = 0.0\n",
    "            running_corrects = 0\n",
    "            \n",
    "            for batch in dataloaders[phase]:\n",
    "                inputs = batch['image'].to(device=device, dtype=dtype)\n",
    "                labels = batch['isup_grade'].to(device=device, dtype=torch.long)\n",
    "                \n",
    "                # Zero the parameter gradients\n",
    "                optimizer.zero_grad()\n",
    "                \n",
    "                # Forward, track history if only in training\n",
    "                with torch.set_grad_enabled(phase == 'train'):\n",
    "                    outputs = model(inputs)\n",
    "                    loss = criterion(outputs, labels)\n",
    "                    \n",
    "                    _, preds = torch.max(outputs, 1)\n",
    "\n",
    "                    # Backward + optimize only if in training phase\n",
    "                    if phase == 'train':\n",
    "                        loss.backward()\n",
    "                        optimizer.step()\n",
    "                    \n",
    "                # Statistics\n",
    "                running_loss += loss.item() * inputs.size(0)\n",
    "                #print(loss.item())\n",
    "                running_corrects += torch.sum(preds == labels)\n",
    "            \n",
    "            # End of epoch\n",
    "            epoch_loss = running_loss / len(dataloaders[phase].dataset)\n",
    "            epoch_acc = running_corrects.double() / len(dataloaders[phase].dataset)\n",
    "            print('{} Loss: {:4f} Acc: {:4f}'.format(phase, epoch_loss, epoch_acc))\n",
    "            \n",
    "            # deep copy the model\n",
    "            if phase == 'val' and epoch_acc > best_acc:\n",
    "                best_acc = epoch_acc\n",
    "                best_model_wts = copy.deepcopy(model.state_dict())\n",
    "            if phase == 'val':\n",
    "                val_acc_history.append(epoch_acc)\n",
    "            else:\n",
    "                train_acc_history.append(epoch_acc)\n",
    "                loss_history.append(epoch_loss)\n",
    "                \n",
    "            if scheduler is not None and phase == 'train':\n",
    "                scheduler.step()\n",
    "                \n",
    "        print()\n",
    "    \n",
    "    time_elapsed = time.time() - since\n",
    "    print('Training complete in {:.0f}m {:0f}s'.format(time_elapsed//60, time_elapsed%60))\n",
    "    print('Best val Acc: {:4f}'.format(best_acc))\n",
    "    \n",
    "    model.load_state_dict(best_model_wts)\n",
    "    return model, loss_history, train_acc_history, val_acc_history\n",
    "\n",
    "\n",
    "def set_parameter_requires_grad(model, feature_extracting):\n",
    "    if feature_extracting:\n",
    "        for param in model.parameters():\n",
    "            param.requires_grad = False\n",
    "            \n",
    "\n",
    "def initialize_model(num_classes, feature_extract=False, use_pretrained=False):\n",
    "    model_ft = None\n",
    "    input_size = 0\n",
    "    \n",
    "    model_ft = models.alexnet(pretrained=use_pretrained)\n",
    "    set_parameter_requires_grad(model_ft, feature_extract)\n",
    "    num_ftrs = model_ft.classifier[6].in_features\n",
    "    model_ft.classifier[6] = nn.Linear(num_ftrs, num_classes)\n",
    "    #input_size = 128\n",
    "    \n",
    "    return model_ft, input_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0/29\n",
      "----------\n",
      "train Loss: 1.362890 Acc: 0.214555\n",
      "val Loss: 0.332934 Acc: 0.055817\n",
      "\n",
      "Epoch 1/29\n",
      "----------\n",
      "train Loss: 1.311477 Acc: 0.234691\n",
      "val Loss: 0.314514 Acc: 0.065120\n",
      "\n",
      "Epoch 2/29\n",
      "----------\n",
      "train Loss: 1.290490 Acc: 0.248234\n",
      "val Loss: 0.344744 Acc: 0.051225\n",
      "\n",
      "Epoch 3/29\n",
      "----------\n",
      "train Loss: 1.268827 Acc: 0.266251\n",
      "val Loss: 0.324312 Acc: 0.066415\n",
      "\n",
      "Epoch 4/29\n",
      "----------\n",
      "train Loss: 1.258370 Acc: 0.272138\n",
      "val Loss: 0.360057 Acc: 0.039331\n",
      "\n",
      "Epoch 5/29\n",
      "----------\n",
      "train Loss: 1.251786 Acc: 0.275553\n",
      "val Loss: 0.306454 Acc: 0.072068\n",
      "\n",
      "Epoch 6/29\n",
      "----------\n",
      "train Loss: 1.237759 Acc: 0.286387\n",
      "val Loss: 0.308663 Acc: 0.069948\n",
      "\n",
      "Epoch 7/29\n",
      "----------\n",
      "train Loss: 1.228292 Acc: 0.282854\n",
      "val Loss: 0.307566 Acc: 0.071008\n",
      "\n",
      "Epoch 8/29\n",
      "----------\n",
      "train Loss: 1.210777 Acc: 0.298045\n",
      "val Loss: 0.315462 Acc: 0.070537\n",
      "\n",
      "Epoch 9/29\n",
      "----------\n",
      "train Loss: 1.205048 Acc: 0.300871\n",
      "val Loss: 0.305038 Acc: 0.071715\n",
      "\n",
      "Epoch 10/29\n",
      "----------\n",
      "train Loss: 1.189986 Acc: 0.308526\n",
      "val Loss: 0.304645 Acc: 0.074894\n",
      "\n",
      "Epoch 11/29\n",
      "----------\n",
      "train Loss: 1.173302 Acc: 0.314767\n",
      "val Loss: 0.302926 Acc: 0.075483\n",
      "\n",
      "Epoch 12/29\n",
      "----------\n",
      "train Loss: 1.159267 Acc: 0.316769\n",
      "val Loss: 0.307633 Acc: 0.075247\n",
      "\n",
      "Epoch 13/29\n",
      "----------\n",
      "train Loss: 1.148727 Acc: 0.322657\n",
      "val Loss: 0.319600 Acc: 0.065709\n",
      "\n",
      "Epoch 14/29\n",
      "----------\n",
      "train Loss: 1.130319 Acc: 0.332077\n",
      "val Loss: 0.310169 Acc: 0.072657\n",
      "\n",
      "Epoch 15/29\n",
      "----------\n",
      "train Loss: 1.108913 Acc: 0.341380\n",
      "val Loss: 0.309515 Acc: 0.067358\n",
      "\n",
      "Epoch 16/29\n",
      "----------\n",
      "train Loss: 1.084451 Acc: 0.348210\n",
      "val Loss: 0.315951 Acc: 0.070301\n",
      "\n",
      "Epoch 17/29\n",
      "----------\n",
      "train Loss: 1.062710 Acc: 0.350447\n",
      "val Loss: 0.314466 Acc: 0.070890\n",
      "\n",
      "Epoch 18/29\n",
      "----------\n",
      "train Loss: 1.040476 Acc: 0.370584\n",
      "val Loss: 0.349078 Acc: 0.058408\n",
      "\n",
      "Epoch 19/29\n",
      "----------\n",
      "train Loss: 1.011542 Acc: 0.384244\n",
      "val Loss: 0.338469 Acc: 0.066886\n",
      "\n",
      "Epoch 20/29\n",
      "----------\n",
      "train Loss: 0.985555 Acc: 0.399906\n",
      "val Loss: 0.331643 Acc: 0.067358\n",
      "\n",
      "Epoch 21/29\n",
      "----------\n",
      "train Loss: 0.952919 Acc: 0.409444\n",
      "val Loss: 0.340843 Acc: 0.070772\n",
      "\n",
      "Epoch 22/29\n",
      "----------\n",
      "train Loss: 0.934656 Acc: 0.416274\n",
      "val Loss: 0.346903 Acc: 0.067711\n",
      "\n",
      "Epoch 23/29\n",
      "----------\n",
      "train Loss: 0.893968 Acc: 0.434998\n",
      "val Loss: 0.368848 Acc: 0.066651\n",
      "\n",
      "Epoch 24/29\n",
      "----------\n",
      "train Loss: 0.874460 Acc: 0.446420\n",
      "val Loss: 0.402958 Acc: 0.061470\n",
      "\n",
      "Epoch 25/29\n",
      "----------\n",
      "train Loss: 0.864889 Acc: 0.453368\n",
      "val Loss: 0.373555 Acc: 0.054286\n",
      "\n",
      "Epoch 26/29\n",
      "----------\n",
      "train Loss: 0.804959 Acc: 0.476448\n",
      "val Loss: 0.422391 Acc: 0.062647\n",
      "\n",
      "Epoch 27/29\n",
      "----------\n",
      "train Loss: 0.784739 Acc: 0.492817\n",
      "val Loss: 0.438983 Acc: 0.064414\n",
      "\n",
      "Epoch 28/29\n",
      "----------\n",
      "train Loss: 0.764436 Acc: 0.500707\n",
      "val Loss: 0.431818 Acc: 0.056406\n",
      "\n",
      "Epoch 29/29\n",
      "----------\n",
      "train Loss: 0.730713 Acc: 0.516722\n",
      "val Loss: 0.373487 Acc: 0.064885\n",
      "\n",
      "Training complete in 342m 20.773282s\n",
      "Best val Acc: 0.075483\n",
      "[1.3628902690229807, 1.3114773116468879, 1.2904904980845908, 1.2688274681147012, 1.2583701434811498, 1.251786026199977, 1.2377591009584734, 1.2282922054538736, 1.210777206068268, 1.2050480546672784, 1.189986279142167, 1.1733020416588589, 1.159266971778151, 1.1487269380212786, 1.13031868961584, 1.1089128336982799, 1.0844507543084985, 1.0627104649833503, 1.0404758913810108, 1.0115423351596449, 0.9855547343791795, 0.9529193795855967, 0.9346560984920119, 0.8939681303035073, 0.8744598218168339, 0.8648892399616753, 0.8049592892628372, 0.784739344343508, 0.764435925561754, 0.7307131956460843]\n",
      "[tensor(0.2146, dtype=torch.float64), tensor(0.2347, dtype=torch.float64), tensor(0.2482, dtype=torch.float64), tensor(0.2663, dtype=torch.float64), tensor(0.2721, dtype=torch.float64), tensor(0.2756, dtype=torch.float64), tensor(0.2864, dtype=torch.float64), tensor(0.2829, dtype=torch.float64), tensor(0.2980, dtype=torch.float64), tensor(0.3009, dtype=torch.float64), tensor(0.3085, dtype=torch.float64), tensor(0.3148, dtype=torch.float64), tensor(0.3168, dtype=torch.float64), tensor(0.3227, dtype=torch.float64), tensor(0.3321, dtype=torch.float64), tensor(0.3414, dtype=torch.float64), tensor(0.3482, dtype=torch.float64), tensor(0.3504, dtype=torch.float64), tensor(0.3706, dtype=torch.float64), tensor(0.3842, dtype=torch.float64), tensor(0.3999, dtype=torch.float64), tensor(0.4094, dtype=torch.float64), tensor(0.4163, dtype=torch.float64), tensor(0.4350, dtype=torch.float64), tensor(0.4464, dtype=torch.float64), tensor(0.4534, dtype=torch.float64), tensor(0.4764, dtype=torch.float64), tensor(0.4928, dtype=torch.float64), tensor(0.5007, dtype=torch.float64), tensor(0.5167, dtype=torch.float64)]\n",
      "[tensor(0.0558, dtype=torch.float64), tensor(0.0651, dtype=torch.float64), tensor(0.0512, dtype=torch.float64), tensor(0.0664, dtype=torch.float64), tensor(0.0393, dtype=torch.float64), tensor(0.0721, dtype=torch.float64), tensor(0.0699, dtype=torch.float64), tensor(0.0710, dtype=torch.float64), tensor(0.0705, dtype=torch.float64), tensor(0.0717, dtype=torch.float64), tensor(0.0749, dtype=torch.float64), tensor(0.0755, dtype=torch.float64), tensor(0.0752, dtype=torch.float64), tensor(0.0657, dtype=torch.float64), tensor(0.0727, dtype=torch.float64), tensor(0.0674, dtype=torch.float64), tensor(0.0703, dtype=torch.float64), tensor(0.0709, dtype=torch.float64), tensor(0.0584, dtype=torch.float64), tensor(0.0669, dtype=torch.float64), tensor(0.0674, dtype=torch.float64), tensor(0.0708, dtype=torch.float64), tensor(0.0677, dtype=torch.float64), tensor(0.0667, dtype=torch.float64), tensor(0.0615, dtype=torch.float64), tensor(0.0543, dtype=torch.float64), tensor(0.0626, dtype=torch.float64), tensor(0.0644, dtype=torch.float64), tensor(0.0564, dtype=torch.float64), tensor(0.0649, dtype=torch.float64)]\n"
     ]
    }
   ],
   "source": [
    "model_ft, input_size = initialize_model(6)\n",
    "#print(model_ft)\n",
    "# Send the model to GPU/CPU\n",
    "model_ft = model_ft.to(device)\n",
    "\n",
    "optimizer = optim.SGD(model_ft.parameters(),\n",
    "                      lr=3e-3,\n",
    "                      momentum=.9,\n",
    "                      nesterov=True)\n",
    "\n",
    "best_model, loss_history, train_acc_history, val_acc_history = train_model(model_ft, {'train': loader_train, 'val': loader_val}, F.cross_entropy, optimizer, None, 30)\n",
    "print(loss_history)\n",
    "print(train_acc_history)\n",
    "print(val_acc_history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEGCAYAAACHGfl5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXxcdb3/8dcnS5t0TfclLW2BWpCCrYRF64IgUlAEEbmA+FCuWu9VFBUrxatQ0StcuK6/q3JRUfSyFVo2ActWNmVr6QqltoWWJl2SLmmbNmmzfH5/nEma5UxyJpnpZGbez8cjj8ycOXPm+82ZnM85n+9yzN0REZHclpfuAoiISPopGIiIiIKBiIgoGIiICAoGIiICFKS7AIkaPny4T5w4Md3FEBHJKEuWLNnu7iPivZ5xwWDixIksXrw43cUQEckoZraxs9czLhh0xwNLK7h54Ro2V9cytqSY2WdN4fzppekulohIr5H1weCBpRVcs2AltfWNAFRU13LNgpUACggiIjFZ34B888I1LYGgWW19IzcvXJOmEomI9D5ZHww2V9cmtFxEJBdlfTAYW1Kc0HIRkVyU9cFg9llTKC7Mb7OsuDCf2WdNSVOJRER6n6xvQG5uJFZvIhGR+LI+GEAQEKIe/NUNVURyUU4Eg6jUDVVEclXWtxkkQt1QRSRX6cqglUS6oSqdJCLZRFcGrUTthtqcTqqorsU5lE56YGnFYSiliEjyKRi0ErUbqtJJIpJtlCZqJWo31ERHNSulJCK9nYJBO1G6oY4tKaYi5MAflmZSDyURyQRKE3VDIqOaE0kpPbC0ghk3Ps2kOY8w48an1QYhIoeNrgy6IZFRzVFTSoleQSj1JCLJpGDQTVFHNUdNKXV2BdH+c5R6EpFkU5ooxaKmlBJplFbqSUSSTVcGKRY1pZRIo7RSTyKSbAoGh0GUlNLss6a0OXBD/EZppZ5EJNmUJuolzp9eyg0XHE9pSTEGlJYUc8MFx4ceiJV6EpFk05VBLxK1UVqpJxFJtpQFAzO7DfgEUOnuU0Ne/yxwdexpDfDv7r48VeXJNko9iUgypTJN9CdgZievvw182N1PAH4E3JrCsuQkpZ5EJKqUXRm4+3NmNrGT1//R6ulLwLhUlSWXZWPqSUSSr7e0GXwReCzei2Y2C5gFcMQRRxyuMuWcTEg9iUhqpL03kZl9hCAYXB1vHXe/1d3L3L1sxIgRh69w0kG6U0+glJJIKqT1ysDMTgB+D5zt7jvSWRaJLp2pJ6WURFIjbcHAzI4AFgCfc/d/pqscklrJTj0l2ptJ3VpFokll19K7gNOA4WZWDlwHFAK4+y3AtcAw4DdmBtDg7mWpKo/0XpoFViT9Utmb6JIuXv8S8KVUfb5kFs0CK5JeaW9AFklEusdDiGSr3tK1VCSSdI6HAKWTJHspGEjGSdd4CKWTJJspTSRZKRXjIZROkmymKwPJWskeD9GdwXFKKUmmUDAQIVrg0OA4yWZKE4lEFDWdBEopSebRlYFIRKkYHAdKJ0nvoGAgkoBkD45TOkl6i0hpIjO70swGWeAPZvaamX0s1YUTyVTqoSSZJmqbwb+6+x7gY8AI4HLgxpSVSiTDRe3ammgPJZFUiZomstjvc4A/uvtyi80uJyLhkt1DCdS+IKkT9cpgiZk9ThAMFprZQKApdcUSyQ2J9FBqbl+oqK7FOdS+oJv7SDJEDQZfBOYAJ7n7foKpqC9PWalEckQiI6XVviCpFDVN9D5gmbvvM7PLgPcCv0xdsURyR9QeSuquKqkU9crgt8B+M3sP8F1gI/DnlJVKRDqI144Qr7uq0kmSiKjBoMHdHTgP+KW7/xIYmLpiiUh76q4qqRQ1TbTXzK4BPgd80Mzyid3CUkQOj1RNqCcC0YPBvwCXEow32Bq7mf3NqSuWiIRRd1VJlUhpInffCtwBDDazTwB17t5pm4GZ3WZmlWa2Ks7rZma/MrN1ZrbCzN6bcOlFpAN1V5XuiDodxUXAK8BngIuAl83swi7e9idgZievnw1Mjv3MImikFpEeUndV6Y6oaaL/IBhjUAlgZiOAJ4H74r3B3Z8zs4mdbPM84M+xhumXzKzEzMa4+5aIZRKRONRdVRIVtTdRXnMgiNmRwHvjKQU2tXpeHlvWgZnNMrPFZra4qqqqhx8rIs3UXVWaRT2g/83MFprZF8zsC8AjwKM9/OywuY08bEV3v9Xdy9y9bMSIET38WBFppu6q0ixSmsjdZ5vZp4EZBAfxW939/h5+djkwvtXzccDmHm5TRBKg7qrSLPLNbdx9PjA/iZ/9EHCFmd0NnALsVnuByOGXiu6qknk6DQZmtpfw1I0B7u6DOnnvXcBpwHAzKweuIzZQzd1vIUgznQOsA/ajie9Eeq3ZZ01pc0c2iN9dFdTYnIk6DQbu3u0pJ9z9ki5ed+Br3d2+iBw+idz/WbfyzEy6B7KIRBK1u2pnjc0KBr2XgoGIJJXGLmSmno4VEBFpQ2MXMpOCgYgklcYuZCaliUQkqTR2ITMpGIhI0mnsQuZRmkhE0iKRqbYl9XRlICJpkcjYBVDPo1RTMBCRtIk6dkED2VJPwUBEer1EBrLpCqJ7FAxEpNeL2vNIVxDdpwZkEen1og5kS3TswgNLK5hx49NMmvMIM258OqcHvCkYiEivF7XnUaJTYWgE9CEKBiLS650/vZQbLjie0pJiDCgtKeaGC47vkPqJegUBGgHdntoMRCQjROl5lMh9FzQCui1dGYhI1oh6BQGJXUXkAl0ZiEhWiTp2IdG7t2U7BQMRyUmJjoDOdgoGIpKzol5F5AIFAxGRLuTCqOaUNiCb2UwzW2Nm68xsTsjrR5jZIjNbamYrzOycVJZHRCRRuTIeIWXBwMzygV8DZwPvBi4xs3e3W+37wDx3nw5cDPwmVeUREemOXBmPkMo00cnAOnd/C8DM7gbOA95otY4Dg2KPBwObU1geEZGEJToeIVNTSqlME5UCm1o9L48ta20ucJmZlQOPAl8P25CZzTKzxWa2uKqqKhVlFREJlch4hExOKaUyGFjIMm/3/BLgT+4+DjgH+IuZdSiTu9/q7mXuXjZixIgUFFVEJFwid2TL5JRSKtNE5cD4Vs/H0TEN9EVgJoC7v2hmRcBwoDKF5RIRiSyR8QiJTpTXm9JJqQwGrwKTzWwSUEHQQHxpu3XeAc4A/mRmxwJFgPJAItKrRB2PMLakmIqQA3/7lFJvvO9CytJE7t4AXAEsBFYT9Bp63cyuN7NPxla7CviymS0H7gK+4O7tU0kiIhkhakqpN6aTUjrozN0fJWgYbr3s2laP3wBmpLIMIiKHS9SUUm+cMVUjkEVEkihKSilqOulw0hTWIiKHWSI9lODw3J5TVwYiIodZIj2UDldjs4KBiEgaRO2h1FljczKDgdJEIiK92OFqbFYwEBHpxQ7X7TkVDEREerFEG5u7S20GIiK92OG6Padl2oBfM6sCNnbz7cOB7UksTm+QbXXKtvpA9tUp2+oD2VensPpMcPe4M31mXDDoCTNb7O5l6S5HMmVbnbKtPpB9dcq2+kD21ak79VGbgYiIKBiIiEjuBYNb012AFMi2OmVbfSD76pRt9YHsq1PC9cmpNgMREQmXa1cGIiISQsFARERyJxiY2UwzW2Nm68xsTrrLkwxmtsHMVprZMjNbnO7yJMrMbjOzSjNb1WrZUDN7wszWxn4PSWcZExWnTnPNrCK2n5aZ2TnpLGMizGy8mS0ys9Vm9rqZXRlbnpH7qZP6ZPI+KjKzV8xseaxOP4wtn2RmL8f20T1m1qfT7eRCm4GZ5QP/BM4Eygnuz3xJ7E5rGcvMNgBl7p6Rg2XM7ENADfBnd58aW3YTsNPdb4wF7SHufnU6y5mIOHWaC9S4+3+ns2zdYWZjgDHu/pqZDQSWAOcDXyAD91Mn9bmIzN1HBvR39xozKwReAK4Evg0scPe7zewWYLm7/zbednLlyuBkYJ27v+XuB4G7gfPSXKac5+7PATvbLT4PuD32+HaCf9SMEadOGcvdt7j7a7HHewnuZ15Khu6nTuqTsTxQE3taGPtx4HTgvtjyLvdRrgSDUmBTq+flZPgXIMaBx81siZnNSndhkmSUu2+B4B8XGJnm8iTLFWa2IpZGyoiUSntmNhGYDrxMFuyndvWBDN5HZpZvZsuASuAJYD1Q7e4NsVW6POblSjCwkGXZkB+b4e7vBc4GvhZLUUjv81vgKGAasAX4aXqLkzgzGwDMB77p7nvSXZ6eCqlPRu8jd29092nAOIJMyLFhq3W2jVwJBuXA+FbPxwGb01SWpHH3zbHflcD9BF+CTLctltdtzu9Wprk8Pebu22L/rE3A78iw/RTLQ88H7nD3BbHFGbufwuqT6fuombtXA88ApwIlZtY8M3WXx7xcCQavApNjret9gIuBh9Jcph4xs/6xBjDMrD/wMWBV5+/KCA8Bn489/jzwYBrLkhTNB82YT5FB+ynWOPkHYLW7/6zVSxm5n+LVJ8P30QgzK4k9LgY+StAWsgi4MLZal/soJ3oTAcS6iv0CyAduc/f/THOResTMjiS4GoDgvhR3ZlqdzOwu4DSC6Xa3AdcBDwDzgCOAd4DPuHvGNMjGqdNpBOkHBzYAX2nOt/d2ZvYB4HlgJdAUW/w9gjx7xu2nTupzCZm7j04gaCDOJzjBn+fu18eOEXcDQ4GlwGXufiDudnIlGIiISHy5kiYSEZFOKBiIiIiCgYiIBA2PGWX48OE+ceLEdBdDRCSjLFmyZHtn90DOuGAwceJEFi/OuDnZRES67YGlFdy8cA2bq2sZW1LM7LOmcP70xCZRMLONnb2eccFARCSXPLC0gmsWrKS2vhGAiuparlmwEiDhgNAZtRmIiPRiNy9c0xIImtXWN3LzwjVJ/RxdGYiIpEFXqZ9te+p4Zk0lFdW1oe/fHGd5d2VFMKivr6e8vJy6urp0FyXlioqKGDduHIWFhekuioh0U1jqZ86CFayvqqGxyVm0porVW4L5APMMmkLGBo8tKU5qmbIiGJSXlzNw4EAmTpxIMPVIdnJ3duzYQXl5OZMmTUp3cUQkRJTG3rDUT119E//v6XXk5xllE4Zw9cxj+MgxI1i9eQ/fu39Vm/WLC/OZfdaUpJY7K4JBXV1d1gcCADNj2LBhVFVVpbsoIhIi7Iz/6vkrWPrOLkYOKmLjjn1s2LE/buoHYOm1ZzKo6NCV/zGjB2FmPe5N1JWsCAZA1geCZrlST5FM4+785NHVHc74DzQ0cfuLQa/O4QP6Mml4P/r1yWf/wcYO2ygtKW4TCJqdP7006Qf/9rImGIiIpEq81E9dfSMvv72TRW9W8tSb26jcGz4pqAErf3gWA/oWtGyv9RUEpCb1k4icDAbJGMDRWnV1NXfeeSdf/epXE3rfOeecw5133klJSUm3P1tEUiss9TP7vuX87vm3eKtqH7X1jfQtyGPG0cPZU9vA7tr6DtsYW1LcEgjg0PiAVKd+EpFzwSAVAziqq6v5zW9+0yEYNDY2kp+fH/d9jz76aLc+T0QOn7DG3vpG580te7nklPGcccwo3nfUMIoK8xM64z8cqZ9EZF0w+OHDr/PG5vi3aF36TjUHG5vaLKutb+S7963grlfeCX3Pu8cO4rpzj4u7zTlz5rB+/XqmTZtGYWEhAwYMYMyYMSxbtow33niD888/n02bNlFXV8eVV17JrFnBveubp9aoqanh7LPP5gMf+AD/+Mc/KC0t5cEHH6S4OLldx0QkMcs3Vcdt7G1y58fnH99mWW88448q64JBV9oHgq6WR3HjjTeyatUqli1bxjPPPMPHP/5xVq1a1dL987bbbmPo0KHU1tZy0kkn8elPf5phw4a12cbatWu56667+N3vfsdFF13E/Pnzueyyy7pdJhHpWljK+KzjRvPwis3830sbWVG+GyP8TvLx+vn3tjP+qLIuGHR2Bg8w48anQyN9aUkx93zlfUkpw8knn9xmHMCvfvUr7r8/uEPlpk2bWLt2bYdgMGnSJKZNmwbAiSeeyIYNG5JSFhEJF5Yyvure5VyzYAW19U1MHjmA6887jj75xg8fXt2rGntTIeuCQVdmnzUl5a34/fv3b3n8zDPP8OSTT/Liiy/Sr18/TjvttNCR0n379m15nJ+fT21tcoeai+SKKB1E3J0bH3uzQ1tAY5NDvnH3rFM5ZdLQlq7cRYUFGZn6SUTOBYNU5PQGDhzI3r17Q1/bvXs3Q4YMoV+/frz55pu89NJL3f4cEelcvJ4/T67eyqDiPpTvqqV8134qdtVyoCE8NVxX38SpR7a9cs/U1E8ici4YQPJ37LBhw5gxYwZTp06luLiYUaNGtbw2c+ZMbrnlFk444QSmTJnCqaeemrTPFcklnZ3xV+6pY3n5bn7wwKrQnj9/XbGVof37UFpSzJRRAznjmJHMW1wetxtoLjL3sKaR3qusrMzb39xm9erVHHvssWkq0eGXa/UVCeuyWZBnHDN6IFU1B9i2J3ywVzMD3r7x411us7gwnxsuOD4rrwLMbIm7l8V7PSevDEQkc8Sb5qGhyXlz614+ccIYjh9XwnvGDebrdy1ly+6ObXJhZ/uZ3A00FRQMRCRt4qV+3J012/byyIotPLJiS9xpHhqbnF9cPL3l+dUzj0mog0gutAVEpWAgImkRb4bPx1ZuYV1VDeur9pFncOqRw9i57yDVEfL7OtvvPgUDEUm6KN07/+tvHbt2HmhoYuEb2zj1yKF8YcYkZh43mhED+2b0NA+ZIqXBwMxmAr8E8oHfu/uNcda7ELgXOMndF4etIyLpFXWCx3jdOxe8Vk5RYT7lu2qpqK4N7ckDQWPv3bPaDgDVGX/qpSwYmFk+8GvgTKAceNXMHnL3N9qtNxD4BvByqsoiIj3T2QSPHztuFGu31fDPbXv557a9/PnFjR368Nc3Os+v3c7kUQMoLSnmxAlDeHBZBXvqGjp8VrZN85ApUnllcDKwzt3fAjCzu4HzgDfarfcj4CbgOyksS1sr5sFT18Puchg8Ds64Fk646LB9/IABA6ipqTlsnyfSU2Ezd9bWN3LVvctpmuc091DvU5DHwTiDuQAe/9aHWx6fOGFIr5vTP5elMhiUAptaPS8HTmm9gplNB8a7+1/NLG4wMLNZwCyAI444omelWjEPHv4G1Meme9i9KXgOhzUgiPQWnaV/6uobWbxhV9yZOxubnG+f+S7eNWogU0YP5Iih/fjQTYtC11djb++WymAQdn/GlhFuZpYH/Bz4QlcbcvdbgVshGHTW6cqPzYGtK+O/Xv4qNLbrplZfCw9eAUtuD3/P6OPh7NDmDgCuvvpqJkyY0HI/g7lz52JmPPfcc+zatYv6+np+/OMfc95553VadJHDLV6PnufWVlG9v54X1+/ocEXQWmlJMd84Y3KbZYnM/6XUT++Rl8JtlwPjWz0fB2xu9XwgMBV4xsw2AKcCD5lZ3BFySdE+EHS1PIKLL76Ye+65p+X5vHnzuPzyy7n//vt57bXXWLRoEVdddRWZNtpbsl9Y+udAQxMLXqvgraoaLiobx21fKOOmTx9PcWHbGzV1doC/4YLjKS0pxggCRraO6s0mqbwyeBWYbGaTgArgYuDS5hfdfTcwvPm5mT0DfKfHvYk6OYMH4OdTg9RQe4PHw+WPdOsjp0+fTmVlJZs3b6aqqoohQ4YwZswYvvWtb/Hcc8+Rl5dHRUUF27ZtY/To0d36DJFEdJb62V5zgFff3snLb++Mm/4x4JnZH2mzrE9BfuSUjs74M0+kYGBm84HbgMfcPdJdYNy9wcyuABYSdC29zd1fN7PrgcXu/lB3C90jZ1zbts0AoLA4WN4DF154Iffddx9bt27l4osv5o477qCqqoolS5ZQWFjIxIkTQ6euFkm2eF075y1+h8q9B1lXGXReKCrMo29BXujsnfGmb9ABPntFvTL4LXA58Cszuxf4k7u/2dWb3P1R4NF2y0KPuu5+WsSy9ExzI3GSexNdfPHFfPnLX2b79u08++yzzJs3j5EjR1JYWMiiRYvYuHFjEgov0rV49+x9cf1OPjxlBBe8t5RTJg3j+NLBPLpyi3r0CBAxGLj7k8CTZjYYuAR4wsw2Ab8D/s/dw0eP9FYnXJT0nkPHHXcce/fupbS0lDFjxvDZz36Wc889l7KyMqZNm8YxxxyT1M8TaW/r7joeXFYRN/UD8KfLT27zXD16pFnkNgMzGwZcBnwOWArcAXwA+DxwWioKl2lWrjzUi2n48OG8+OKLoetpjIG0FnVkb9i6Xz/9aAry87h/aTn/WL8DdyjMN+obO3ZW0GAu6UzUNoMFwDHAX4Bz3X1L7KV7zEzTR4iEiHKQ72xkb5R158TWPWJoP75++mQ+Nb2U5ZuqlfqRhEW9Mvgfd3867IXObpYgkqu6Osg3Njnb9tTx40feCB3Ze/X8Fdz96jvU1jdRe7CB2vpGKnbV0hTSO3n4gL48O/u0lvv1Thoe3INbqR9JRNRgcKyZvebu1QBmNgS4xN1/k7qiJcbdW/4ZspnGKqRX1JROvOkbvnvfCv778TVs3V1HQ9iRPeZAQxNNDiXFhYwZVES/Pvls2lkRuu6OmgMdvvtK/UiiogaDL7v7r5ufuPsuM/sy0CuCQVFRETt27GDYsGFZHRDcnR07dlBUVJTuouSkeGf7NQfqOXrkQNZu28s/t9WwtnJv3Ebcg41NnDhhCOOGFFNa0o+fPr6GHfsOdlivtKSYeV9pO3NnvHEBuXrPXkmuqMEgz8zMY6elsRlJ+6SuWIkZN24c5eXlVFVVpbsoKVdUVMS4cePSXYys093592vrG/n+A6+3PB/Yt4DJowbQr08++w92nMahtKSYX7a6M1e/PvmR8/uJTPMgkqiowWAhMM/MbiGYX+jfgL+lrFQJKiwsZNKkSekuhhwmicyr39359787fwV/X7+d/n0KWFu5lzVba9heE3/Kktv/9WTeNWoAowcVYWaRb8aSSNdOdQOVVLIoOejYpHJfAc4gGKn+OMHNauLPYJUiZWVlvnixOjDlqngH2fZz38Rb7z/PP473Hz2CrXvq2Lq7jm176rh54RpqDnScVx+CM/fJIwcwedRAHn99a+j8+6Ulxfx9zumhZdWBW3oLM1vSWYefSMGgN1EwyE5RD5zvv+EpNu/uOK3H4OJCvvaRo6irb6K2vpE/v7iBfQd6dq5iwPqfnENenrWUMUogEumNugoGUccZTAZuAN4NtLReuvuRPS6hZLXu9rWffd9ynly9jaH9+7C5uo6te2rZUl0X2tgKsLu2np88GsyQUpBnnfbU+dH5UxkzqIjRg4sYNaiI8379ApurOwaYsSXFLYEAlKaR7Ba1zeCPwHUE9x/4CME8RdnbbUc61ZNc/JwFK9i8u5ajRgzgnR372bhzH/cuLg+9TeJfV2xhUFEBY0uKGT24iONLS/jris3sDUnVjB5cxJPf/jBFBXkU5Ocx48anQ3velJYU87lTJ7RZ9t2zjtH8+5LzogaDYnd/KtajaCMw18yeJwgQkkPiNbauqtjN8eMGs+9AI/sONFBzoIE/vPBWh943dfVN3PS3NS3PBxcXhs6aCcHZxoq5Z7VZdsqkoaEH7jkzj2FA30Nf50RvsAI645fcFjUY1MUakdfGpqWuAEamrliSLD2Z96b9upura5n70OsdDvAHG5r4/QtvJ1Suh66YwYSh/RncrzDuWXy8aZSh6wN3ogd4nfFLrovam+gkYDVQQnAD+0HAze7+UmqL15EakKNLpMEzbN2iwjw+d+oE6hudF9Ztb5kHP4wBT3z7wwzoW0D/vvn071PAB+PcC7d97xs1zIqkXo8bkGMDzC5y99lADUF7gaRAss7i6+ob+ee2vaFn8c1TIjyycguDigoZXFzIoOIC/vj3t0NTOr97/m36FuRxypHDuPik8dz63FtU7u3Y335sSTFHjxzQZlnUVI3SNCLp12UwcPdGMzux9QhkSb6ezl45+77l3PnyRvbUNbCusqbT3jQHG5vYtHM/e2rr2VPXELePfbPl132Motj9b4cP6JuSXLzSNCLpFbXNYCnwYOwuZ/uaF7r7gpSUKstEOeO/eWH4VAfXPriKdZU17KmrZ29dA3tq63l+7XYONnbsfbN44y4+OHkEZxw7kqljBzP34dfZtqfjWXxpSTF/++aHWp43NDbxwZsWsSWk/35pSXFLIADl4kWyVdRgMBTYAbQeZumAgkEX4p3FP7SsgoHFhS2jYCtC+rkD7Klr4DfPrGNgUZDOGdi3sEMgaOYeTIvQ7EBDU6Sz+IL8PK6eqe6VIrks6m0v1U7QDVt313FdSN6+vtF5ek0V44cWM3pQEVNLB7O95gA1ISNmxwwu4h9zTm8zG2vU3jea90ZEooo6AvmPBFcCbbj7vya9RBkiXupn0879/G3VVh5btYXX3qmO+34Dnv9u1z1qrp55TIdpuRPtQx/1gK4zfpHcFTVN9NdWj4uATwGbk1+czBCW+vnOvcu5eeGbLemed48ZxFVnvou/vLQxbu+b1nQWLyLp1K2J6mID0J50945TNaZYbxhncOoNT7E1pLG1MN+YfdYUZh43hiOG9QPUh15EeoekTFQXYjJwRDff26uFpX9OOXIor7y9k5ff3skrb+8MDQQADY3OrA8d1WaZzuJFJBNEbTPYS9s2g63A1SkpURqFpX++dc+ylooP7FtA2cQhVO6pC53XPt7tB5WLF5HeLmpvooGpLki67a6tDx2x68Dg4gLu+NKpHDtmEPl50e9iJSKSKaJeGXwKeNrdd8eelwCnufsDqSxcssTr+XOwoYln1lTywLIKnlxdycE4s2fuqW1gaungludK/YhItok6Ud0yd5/WbtlSd58e7z2pkmgDcthZfN+CPMomlPD6lr1U769nWP8+nPuesTyyYgtVIfe5jXdbQxGRTJGsBuS8Hrw3rW5euKZD6udAQxN/X7+Tc98zlguml/KBycMpzM9j2vgSpX9EJCeFHeTDLDazn5nZUWZ2pJn9HFjS1ZvMbKaZrTGzdWY2J+T1b5vZG2a2wsyeMrMJYdvpic0hI3UhGPT1/y6ZzkeOGUlhfvBnOH96KTdccDylJcUYwRWBuoCKSC6Ienb/deAHwD2x548D3+/sDXWdQuQAAA4+SURBVLGpr38NnAmUA6+a2UPu/kar1ZYCZe6+38z+HbgJ+JcEyt+lsSXFkW+cAur5IyK5KdKVgbvvc/c57l4W+/meu+/r4m0nA+vc/S13PwjcDZzXbruL3H1/7OlLwLhEK9CV2WdNobjVrJug1I+ISHuRgoGZPRHrQdT8fIiZLezibaXAplbPy2PL4vki8Ficz59lZovNbHFVVVWUIrdQ6kdEpGtR00TD3b1l1jV332VmXd0D2UKWhXZdMrPLgDLgw2Gvu/utwK0Q9CaKVOJWlPoREelc1AbkJjNrmX7CzCYS58DeSjkwvtXzcYRMbmdmHwX+A/iku3fs1ykiIikX9crgP4AXzOzZ2PMPAbO6eM+rwGQzmwRUABcDl7ZewcymA/8LzHT3ysilFhGRpIragPw3gjTOGoIeRVcB4X02D72nAbgCWAisBua5++tmdr2ZfTK22s3AAOBeM1tmZg91rxoiItITUaej+BJwJUGqZxlwKvAibW+D2YG7Pwo82m7Zta0efzTB8oqISApEbTO4EjgJ2OjuHwGmA4l16xERkV4rajCoc/c6ADPr6+5vAuqoLyKSJaI2IJfHxhk8ADxhZrvI4dteiohkm6j3M/hU7OFcM1sEDAb+lrJSiYjIYZXwzKPu/mzXa4mISCaJ2mYgIiJZTMFAREQUDERERMFARERQMBARERQMREQEBQMREUHBQEREUDAQEREUDEREBAWDzLViHvx8KswtCX6vmHf4tpnIZ2fKNqNK9zZT8fmpkCnllBbmnvD95dOqrKzMFy9enLoPWDEPnroedpfD4HFwxrVwwkW9a5sr5sHD34D6VjebKyyGc3/VcbtRPzvqNhP97KjbfOgb0NBqvYIi+PAcmPxRaGoEbwp+1j4OL/wCGlvdLju/L8y4Eo4+AzAwg3VPwws/a7tevHJG/TslUvdUbDPRz48qke9IMr9LiWwz0XWzTRLqbmZL3L0s7usKBq2k4p8tFdv8+VTYvanj8qLBcNo1wfYL+0HFElj8x44HzlP/HSbMgKZ6aKyHpgZ4dDbU7uy4zeIhcNYNYHnBz2NXQ+2O8PVO/0GwrcaDwXZf+Dkc2NNx3fy+MGJK8NqBvbA/ZHupUlAM7/86jDwWRr4bhh0Fr98ffx8ddwHs3QLV78A9l4X/jYpK4JO/ggGjYMBIGDAa3vxr5/u9sT6o+2/fH2y/vX7D4dxfxAJhIzQ1wWPfDf/8wePhW6u69/dIxknA8Z8JltftDvbpnz4B+0JuaT5gNMxaBH0HQmF/WHVfagJhuk/o0nny1wkFA4i+c+IdZHvyz/bz44LPTdY23eGHJd0rS2/yrpnQd1BwYFj8h/jrXfQXyMuPBaN8uPMiIOw7a3DZ/OA1B+64MM56BNvypuBxXmGwXlNDyHr5wVVG2GtdsvDPtzzI7wMNdd3YZic+dSuMPwmGTArK3NV3vqkRqjfC78+E/ds7bq+gGCafGbsic1j/VHiZm/dLU32CBbbY75C/UUFfmPThYN/k5UN+Iax5DOr3d1y3/wi4/LHgd9FgWHnv4b96c4eGA8HfZ+W98Pj32/6tehq0knRc6ioYJDyFdcZpvyN3bwqeH6yBEcdA5WqoejP4CfuDN79n2+sw4ljIyzu03Xg7cs8WeGsRrH86PBBA/OWdqVwdnJnHM2gc/NvzQV3ra+F/yoh74PziE5BfEPzD5RfCn88LP0MdOCb4Z2s+KPzp41CzNXy9Wc8G28orCA54/3Ni/EB46T2Hnq99PP6X/d2fbLdsXJx1x8XSRF2tNx6uWAzb/xns88o3giuYMN4IH/gOlIyHkiPgga+G/40GjYVL74WabYd+npwbZ5tNcMpXoM/AIBA+eyPU7uq43oBRQXCz/EPB8PZzwz8fg/tnBQ/7j4CBY4N6NR+gd2+CB78Gq+YH+2bHOtj5VnAFF09DLWxfe+iKMF7w8qYgRVc0OPjpOyj4joYFmH7Dgv+TA3uDn2f/K85nH4B9VdDYcOjqNSwQQLDe/8SOb/l9g8DtjW3Xqa+Fx/8DjjoD+g0NgiXEPzYATL0weL5jXXDV3DoQNG9zwSx48Iq2V95h6mvh4Sthx/rgSnToUbB1BSy8puNn790Cw44O/vY71sL2dZ0cl7pxDOlE9l8ZxIuqrfUZGKQtKldD/b746xUPCdIrffrDGw8EX9pm+X1h4odgTzlUrQ6W9R8BB/eFf5Hz8uHjP4P3XAoFfTovX2118I/z8v8GB5ApZwef39XZTyJnFOluM0jnNqP+ndK9zXjrfuIXMGoqlL8Cm14Jzk7jXc0MfxcMmxwclIZPhqd+FJ7SaV/OVHyXEtlmvHX7jwjSmM1B+B+/6rhOa30GQMkEGDIB3n4uOClsL68gCMJdHeQhCIQFRcHVTEERLPxeJyvHuVqMp//IYB9tXhZ+XNKVQYI6i56fnR8EgcHjDl1ah32Jz7guyAtveAE2PB9cXrfXeADWPwFHngbTLoGjToeRx4XnRfP7BGfSD18Jz94EM74J7/0crH647dXG6T8ItvvkD4O8etnl8JHvQ/9hwfa7usQ849o49bm2Y/mb39vVNqOul0nbjPp3Svc2u1p39FQo+1dYfnfH9wJgcMWrbRcVFEUrZyq+S4lsM966Z/0ETvjMoWWv3x8eNPoNhw9eFfzv7toIuzaEBwIIAun7/i04EA87GuZ/GfaG3OV38Hg48/q2y176bedXpLs2wM71cPel4Z8N8KWng2BdHEsHxz0uhfydeiB3rwziRdUoeby5JcRNv8ytjrbN4z8T5GGfvRk2vRRcXtfXts29mgWpmSPeB2f/F4x5T/R6J1IfSX+jYzKl4jufyHqJSHbDbG+9euvJVVHUundBDciHszdPdxqF3WHj3+H/LmibdmpWPBS++9ahPKdIV1LVBTVTZEIX2DTsIwUD6LVdvdpI9GpDpDO6IowmnX+nw/zZCgapkuwdmYpurSIiMWpATpUTLkpuFE+kMU1EJMk0N1FvccJFQZpp8HjAgt+5kuMVkbTTlUFvkuyrDRGRiDKuzcDMqoCQjv6RDAdChkZmtGyrU7bVB7KvTtlWH8i+OoXVZ4K7j4j3howLBj1hZos7a0DJRNlWp2yrD2RfnbKtPpB9depOfdRmICIiCgYiIpJ7weDWdBcgBbKtTtlWH8i+OmVbfSD76pRwfXKqzUBERMLl2pWBiIiEUDAQEZHcCQZmNtPM1pjZOjObk+7yJIOZbTCzlWa2zMx6wYRNiTGz28ys0sxWtVo21MyeMLO1sd9D0lnGRMWp01wzq4jtp2Vmdk46y5gIMxtvZovMbLWZvW5mV8aWZ+R+6qQ+mbyPiszsFTNbHqvTD2PLJ5nZy7F9dI+ZdXoXrZxoMzCzfOCfwJlAOfAqcIm7v5HWgvWQmW0Aytw9IwfLmNmHgBrgz+4+NbbsJmCnu98YC9pD3L2Te332LnHqNBeocff/TmfZusPMxgBj3P01MxsILAHOB75ABu6nTupzEZm7jwzo7+41ZlYIvABcCXwbWODud5vZLcByd/9tvO3kypXBycA6d3/L3Q8CdwPnpblMOc/dnwN2tlt8HnB77PHtBP+oGSNOnTKWu29x99dij/cCq4FSMnQ/dVKfjOWB5tu2FcZ+HDgduC+2vMt9lCvBoBRoPT90ORn+BYhx4HEzW2Jms9JdmCQZ5e5bIPjHBUamuTzJcoWZrYilkTIipdKemU0EpgMvkwX7qV19IIP3kZnlm9kyoBJ4AlgPVLt7882wuzzm5UowCLtNWDbkx2a4+3uBs4GvxVIU0vv8FjgKmAZsAX6a3uIkzswGAPOBb7r7nnSXp6dC6pPR+8jdG919GjCOIBNybNhqnW0jV4JBOTC+1fNxQMgdrjOLu2+O/a4E7if4EmS6bbG8bnN+tzLN5ekxd98W+2dtAn5Hhu2nWB56PnCHuy+ILc7Y/RRWn0zfR83cvRp4BjgVKDGz5pmpuzzm5UoweBWYHGtd7wNcDDyU5jL1iJn1jzWAYWb9gY8B2XBLtIeAz8cefx54MI1lSYrmg2bMp8ig/RRrnPwDsNrdf9bqpYzcT/Hqk+H7aISZlcQeFwMfJWgLWQRcGFuty32UE72JAGJdxX4B5AO3uft/prlIPWJmRxJcDUBwX4o7M61OZnYXcBrBdLvbgOuAB4B5wBHAO8Bn3D1jGmTj1Ok0gvSDAxuArzTn23s7M/sA8DywEmiKLf4eQZ494/ZTJ/W5hMzdRycQNBDnE5zgz3P362PHiLuBocBS4DJ3PxB3O7kSDEREJL5cSROJiEgnFAxERETBQEREFAxERAQFAxERQcFA5LAys9PM7K/pLodIewoGIiKiYCASxswui80Rv8zM/jc2EViNmf3UzF4zs6fMbERs3Wlm9lJskrP7myc5M7OjzezJ2Dzzr5nZUbHNDzCz+8zsTTO7IzYqViStFAxE2jGzY4F/IZgIcBrQCHwW6A+8Fpsc8FmC0cUAfwaudvcTCEa2Ni+/A/i1u78HeD/BBGgQzJT5TeDdwJHAjJRXSqQLBV2vIpJzzgBOBF6NnbQXE0zE1gTcE1vn/4AFZjYYKHH3Z2PLbwfujc0bVeru9wO4ex1AbHuvuHt57PkyYCLBDUlE0kbBQKQjA25392vaLDT7Qbv1OpvLpbPUT+v5YRrR/6H0AkoTiXT0FHChmY2Elvv9TiD4f2meBfJS4AV33w3sMrMPxpZ/Dng2Nkd+uZmdH9tGXzPrd1hrIZIAnZGItOPub5jZ9wnuIpcH1ANfA/YBx5nZEmA3QbsCBNMD3xI72L8FXB5b/jngf83s+tg2PnMYqyGSEM1aKhKRmdW4+4B0l0MkFZQmEhERXRmIiIiuDEREBAUDERFBwUBERFAwEBERFAxERAT4/+i+dzEFzZn2AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "\"\"\"\n",
    "Graphs\n",
    "1. loss vs. iterations\n",
    "2. Train/Validation accuracy along epoch\n",
    "\"\"\"\n",
    "plt.subplot(2,1,1)\n",
    "plt.plot(loss_history, 'o')\n",
    "plt.xlabel('epoch')\n",
    "plt.ylabel('loss')\n",
    "\n",
    "plt.subplot(2,1,2)\n",
    "plt.plot(train_acc_history, '-o')\n",
    "plt.plot(val_acc_history, '-o')\n",
    "plt.legend(['train', 'val'], loc='upper left')\n",
    "plt.xlabel('epoch')\n",
    "plt.ylabel('accuracy')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AlexNet(\n",
      "  (features): Sequential(\n",
      "    (0): Conv2d(3, 64, kernel_size=(11, 11), stride=(4, 4), padding=(2, 2))\n",
      "    (1): ReLU(inplace=True)\n",
      "    (2): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "    (3): Conv2d(64, 192, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))\n",
      "    (4): ReLU(inplace=True)\n",
      "    (5): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "    (6): Conv2d(192, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (7): ReLU(inplace=True)\n",
      "    (8): Conv2d(384, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (9): ReLU(inplace=True)\n",
      "    (10): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (11): ReLU(inplace=True)\n",
      "    (12): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "  )\n",
      "  (avgpool): AdaptiveAvgPool2d(output_size=(6, 6))\n",
      "  (classifier): Sequential(\n",
      "    (0): Dropout(p=0.5, inplace=False)\n",
      "    (1): Linear(in_features=9216, out_features=4096, bias=True)\n",
      "    (2): ReLU(inplace=True)\n",
      "    (3): Dropout(p=0.5, inplace=False)\n",
      "    (4): Linear(in_features=4096, out_features=4096, bias=True)\n",
      "    (5): ReLU(inplace=True)\n",
      "    (6): Linear(in_features=4096, out_features=6, bias=True)\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(best_model)"
   ]
  }
 ],
 "metadata": {
  "CodeCell": {
   "cm_config": {
    "lineWrapping": true
   }
  },
  "MarkdownCell": {
   "cm_config": {
    "lineWrapping": true
   }
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}