Polly Voice Detection With Gluon MXNet and Sagemaker.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 16.6 ms, sys: 6.07 ms, total: 22.7 ms\n",
      "Wall time: 6.8 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "%%capture\n",
    "!pip install pydub > /dev/null"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.5 s, sys: 377 ms, total: 2.88 s\n",
      "Wall time: 50.9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import random\n",
    "import base64\n",
    "import json\n",
    "import tarfile\n",
    "import wave\n",
    "from contextlib import closing\n",
    "from os import listdir, makedirs\n",
    "from os.path import isfile, join\n",
    "from pickle import dump\n",
    "from sagemaker.mxnet import MXNet\n",
    "from shutil import rmtree, copy2\n",
    "from urllib.request import urlretrieve\n",
    "from tempfile import gettempdir\n",
    "\n",
    "import boto3\n",
    "import cv2\n",
    "import matplotlib\n",
    "matplotlib.use(\"agg\")\n",
    "import matplotlib.pyplot as plt\n",
    "import mxnet as mx\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sagemaker\n",
    "from pydub import AudioSegment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "sagemaker_session = sagemaker.Session()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 457.37 s, sys: 56.00 s, total: 513.37 s\n",
      "Wall time: 356.27 s\n"
     ]
    }
   ],
   "source": [
    "rmtree(\"data/sentences\", True)\n",
    "makedirs(\"data/sentences\")\n",
    "\n",
    "urlretrieve(\"http://www.cs.cornell.edu/people/pabo/movie-review-data/rotten_imdb.tar.gz\",\n",
    "            \"data/sentences/sentences.tar.gz\")\n",
    "\n",
    "tar = tarfile.open(\"data/sentences/sentences.tar.gz\")\n",
    "tar.extractall(\"data/sentences\")\n",
    "tar.close()\n",
    "\n",
    "with open(\"data/sentences/plot.tok.gt9.5000\", \"r\", encoding = \"ISO-8859-1\") as first_file:\n",
    "    first_sentences = first_file.read().split(\"\\n\")[0:5000]\n",
    "with open(\"data/sentences/quote.tok.gt9.5000\", \"r\", encoding = \"ISO-8859-1\") as second_file:\n",
    "    second_sentences = second_file.read().split(\"\\n\")[0:5000]\n",
    "    \n",
    "rmtree(\"data/sentences\", True)\n",
    "makedirs(\"data/sentences\")\n",
    "\n",
    "with open(\"data/sentences/sentences.txt\", \"w\") as sentences_file:\n",
    "    for sentence in first_sentences + second_sentences:\n",
    "        sentences_file.write(\"{}\\n\".format(sentence))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"data/sentences/sentences.txt\", \"r\", encoding = \"ISO-8859-1\") as sentences_file:\n",
    "    sentences = sentences_file.read().split(\"\\n\")[:-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "voices = [\"Ivy\", \"Joanna\", \"Joey\", \"Justin\", \"Kendra\", \"Kimberly\", \"Matthew\", \"Salli\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "client = boto3.client(\"polly\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "i = 1\n",
    "\n",
    "random.seed(42)\n",
    "\n",
    "rmtree(\"data/mp3\", True)\n",
    "makedirs(\"data/mp3\")\n",
    "\n",
    "for sentence in sentences:\n",
    "    voice = random.choice(voices)\n",
    "    file_mask = \"data/mp3/sample-{:05}-{}.mp3\".format(i, voice)\n",
    "    i += 1\n",
    "    response = client.synthesize_speech(\n",
    "        OutputFormat=\"mp3\",\n",
    "        Text=sentence,\n",
    "        TextType=\"text\",\n",
    "        VoiceId=voice\n",
    "    )\n",
    "    with open(file_mask, \"wb\") as out:\n",
    "        with closing(response[\"AudioStream\"]) as stream:\n",
    "            out.write(stream.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "mp3_files = sorted([f for f in listdir(\"data/mp3\") if isfile(join(\"data/mp3\", f))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 457.37 s, sys: 56.00 s, total: 513.37 s\n",
      "Wall time: 356.27 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "rmtree(\"data/wav\", True)\n",
    "makedirs(\"data/wav\")\n",
    "\n",
    "sample_start = random.randint(500, 1000)\n",
    "sample_finish = sample_start + 2000\n",
    "\n",
    "for mp3 in mp3_files:\n",
    "    sound = AudioSegment.from_mp3(\"data/mp3/{}\".format(mp3))[sample_start:sample_finish]\n",
    "    sound.export(\"data/wav/{}wav\".format(mp3[:-3]), format=\"wav\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def graph_spectrogram(wav_file, out):\n",
    "    wav = wave.open(wav_file, \"r\")\n",
    "    frames = wav.readframes(-1)\n",
    "    sound_info = np.frombuffer(frames, \"int16\")\n",
    "    frame_rate = wav.getframerate()\n",
    "    wav.close()\n",
    "    fig = plt.figure()\n",
    "    fig.set_size_inches((1.4, 1.4))\n",
    "    ax = plt.Axes(fig, [0., 0., 1., 1.])\n",
    "    ax.set_axis_off()\n",
    "    fig.add_axes(ax)\n",
    "    plt.set_cmap(\"hot\")\n",
    "    plt.specgram(sound_info, Fs=frame_rate)\n",
    "    plt.savefig(out, format=\"png\")\n",
    "    plt.close(fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "wav_files = sorted([f for f in listdir(\"data/wav/\") if isfile(join(\"data/wav/\", f))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 457.37 s, sys: 56.00 s, total: 513.37 s\n",
      "Wall time: 356.27 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "%%capture --no-stdout --no-display\n",
    "rmtree(\"data/spectrograms\", True)\n",
    "makedirs(\"data/spectrograms\")\n",
    "\n",
    "for wav in wav_files:\n",
    "    graph_spectrogram(\"data/wav/{}\".format(wav), \"data/spectrograms/{}png\".format(wav[:-3]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "spectrograms = sorted([join(\"data/spectrograms/\", f) for f in listdir(\"data/spectrograms/\") if isfile(join(\"data/spectrograms/\", f))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame({\n",
    "    \"wav\": [join(\"data/wav/\", f) for f in wav_files],\n",
    "    \"mp3\": [join(\"data/mp3/\", f) for f in mp3_files],\n",
    "    \"spectrogram\": spectrograms\n",
    "})\n",
    "df[\"label\"] = df.spectrogram.str.extract(\"sample-\\\\d+-(\\\\w+)\\\\.png\", expand=False).apply(lambda x: voices.index(x))\n",
    "df[\"voice\"] = df.spectrogram.str.extract('sample-\\\\d+-(\\\\w+)\\\\.png', expand=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = df.groupby(\"voice\").apply(lambda x: x.sample(frac=.8)).reset_index(0, drop=True)\n",
    "validation = df.loc[~df.index.isin(train.index), :].groupby(\"voice\").apply(lambda x: x.sample(frac=.5)).reset_index(0, drop=True)\n",
    "test = df.loc[np.logical_not(np.logical_xor(~df.index.isin(train.index), ~df.index.isin(validation.index))), :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def transform(row):\n",
    "    img = cv2.imread(row[\"spectrogram\"])\n",
    "    img = mx.nd.array(img)\n",
    "    img = img.astype(np.float32)\n",
    "    img = mx.nd.transpose(img, (2, 0, 1))\n",
    "    img = img / 255\n",
    "    label = np.float32(row[\"label\"])\n",
    "    return img, label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 26.8 s, sys: 2.54 s, total: 29.3 s\n",
      "Wall time: 13.6 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train_nd = [transform(row) for _, row in train.iterrows()]\n",
    "validation_nd = [transform(row) for _, row in validation.iterrows()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_to_disk(data, type):\n",
    "    makedirs(\"{}/pvdwgmas/data/pickles/{}\".format(gettempdir(), type))\n",
    "    with open(\"{}/pvdwgmas/data/pickles/{}/data.p\".format(gettempdir(), type), \"wb\") as out:\n",
    "        dump(data, out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.85 s, sys: 4.25 s, total: 6.1 s\n",
      "Wall time: 7.44 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "rmtree(\"{}/pvdwgmas\".format(gettempdir()), True)\n",
    "\n",
    "save_to_disk(train_nd, \"train\")\n",
    "save_to_disk(validation_nd, \"validation\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 18.9 s, sys: 7.9 s, total: 26.8 s\n",
      "Wall time: 26.9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "inputs = sagemaker_session.upload_data(path=\"{}/pvdwgmas/data/pickles\".format(gettempdir()),\n",
    "                                       bucket=\"redacted\", key_prefix=\"cosmin/sagemaker/demo\")\n",
    "rmtree(\"{}/pvdwgmas\".format(gettempdir()), True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "rmtree(\"data/test\", True)\n",
    "makedirs(\"data/test\")\n",
    "for _, row in test.iterrows():\n",
    "    makedirs(\"data/test/{}\".format(row[\"voice\"]), exist_ok=True)\n",
    "    copy2(row[\"mp3\"], \"data/test/{}\".format(row[\"voice\"]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "estimator = MXNet(\"voice-recognition-sagemaker-script.py\", \n",
    "          role=sagemaker.get_execution_role(), \n",
    "          train_instance_count=1, \n",
    "          train_instance_type=\"ml.p2.xlarge\",\n",
    "          hyperparameters={\"epochs\": 5},\n",
    "          py_version=\"py3\",\n",
    "          framework_version=\"1.1.0\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:sagemaker:Created S3 bucket: redacted\n",
      "INFO:sagemaker:Creating training-job with name: sagemaker-mxnet-2018-05-29-10-33-20-284\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "..............................\n",
      "\u001b[31m2018-05-29 10:38:13,223 INFO - root - running container entrypoint\u001b[0m\n",
      "\u001b[31m2018-05-29 10:38:13,224 INFO - root - starting train task\u001b[0m\n",
      "\u001b[31m2018-05-29 10:38:13,243 INFO - container_support.training - Training starting\u001b[0m\n",
      "\u001b[31m2018-05-29 10:38:14,949 INFO - mxnet_container.train - MXNetTrainingEnvironment: {'user_requirements_file': None, 'model_dir': '/opt/ml/model', 'input_dir': '/opt/ml/input', '_scheduler_ip': '10.32.0.4', 'output_dir': '/opt/ml/output', 'hosts': ['algo-1'], 'channel_dirs': {'training': '/opt/ml/input/data/training'}, 'code_dir': '/opt/ml/code', 'sagemaker_region': 'us-east-1', 'input_config_dir': '/opt/ml/input/config', '_scheduler_host': 'algo-1', 'container_log_level': 20, 'job_name': 'sagemaker-mxnet-2018-05-29-10-33-20-284', 'output_data_dir': '/opt/ml/output/data/', 'enable_cloudwatch_metrics': False, 'base_dir': '/opt/ml', 'user_script_name': 'voice-recognition-sagemaker-script.py', 'user_script_archive': 's3://redacted/sagemaker-mxnet-2018-05-29-10-33-20-284/source/sourcedir.tar.gz', 'resource_config': {'hosts': ['algo-1'], 'current_host': 'algo-1'}, 'current_host': 'algo-1', '_ps_verbose': 0, 'available_gpus': 1, '_ps_port': 8000, 'hyperparameters': {'sagemaker_enable_cloudwatch_metrics': False, 'epochs': 5, 'sagemaker_container_log_level': 20, 'sagemaker_program': 'voice-recognition-sagemaker-script.py', 'sagemaker_region': 'us-east-1', 'sagemaker_submit_directory': 's3://redacted/sagemaker-mxnet-2018-05-29-10-33-20-284/source/sourcedir.tar.gz', 'sagemaker_job_name': 'sagemaker-mxnet-2018-05-29-10-33-20-284'}, 'channels': {'training': {'TrainingInputMode': 'File', 'RecordWrapperType': 'None', 'S3DistributionType': 'FullyReplicated'}}, 'available_cpus': 4}\u001b[0m\n",
      "\u001b[31mDownloading s3://redacted/sagemaker-mxnet-2018-05-29-10-33-20-284/source/sourcedir.tar.gz to /tmp/script.tar.gz\u001b[0m\n",
      "\u001b[31m2018-05-29 10:38:15,061 INFO - botocore.vendored.requests.packages.urllib3.connectionpool - Starting new HTTP connection (1): 169.254.170.2\u001b[0m\n",
      "\u001b[31m2018-05-29 10:38:15,137 INFO - botocore.vendored.requests.packages.urllib3.connectionpool - Starting new HTTPS connection (1): redacted.s3.amazonaws.com\u001b[0m\n",
      "\u001b[31m2018-05-29 10:38:15,224 INFO - mxnet_container.train - Starting distributed training task\u001b[0m\n",
      "\u001b[31mCollecting opencv-python\n",
      "  Downloading https://files.pythonhosted.org/packages/8e/ad/b33f9389626709f20c69e6152b886c27e41da16b5c34470481cf609d5a06/opencv_python-3.4.1.15-cp35-cp35m-manylinux1_x86_64.whl (24.9MB)\u001b[0m\n",
      "\u001b[31mRequirement already satisfied: numpy>=1.11.1 in /usr/local/lib/python3.5/dist-packages (from opencv-python) (1.13.3)\u001b[0m\n",
      "\u001b[31mInstalling collected packages: opencv-python\u001b[0m\n",
      "\u001b[31mSuccessfully installed opencv-python-3.4.1.15\u001b[0m\n",
      "\u001b[31mCollecting pydub\u001b[0m\n",
      "\u001b[31m  Downloading https://files.pythonhosted.org/packages/c4/f5/1ed76122c43b1a8db51f0b31ee4f0e0c0d66dfaf5f192a11a164013e5239/pydub-0.22.0-py2.py3-none-any.whl\u001b[0m\n",
      "\u001b[31mInstalling collected packages: pydub\u001b[0m\n",
      "\u001b[31mSuccessfully installed pydub-0.22.0\u001b[0m\n",
      "\u001b[31mCollecting matplotlib\n",
      "  Downloading https://files.pythonhosted.org/packages/81/31/4e261379e0cd4e9bbacfc96b124ebac0706b44374bd1d34ef899796f741b/matplotlib-2.2.2-cp35-cp35m-manylinux1_x86_64.whl (12.6MB)\u001b[0m\n",
      "\u001b[31mCollecting kiwisolver>=1.0.1 (from matplotlib)\n",
      "  Downloading https://files.pythonhosted.org/packages/7e/31/d6fedd4fb2c94755cd101191e581af30e1650ccce7a35bddb7930fed6574/kiwisolver-1.0.1-cp35-cp35m-manylinux1_x86_64.whl (949kB)\u001b[0m\n",
      "\u001b[31mCollecting pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 (from matplotlib)\n",
      "  Downloading https://files.pythonhosted.org/packages/6a/8a/718fd7d3458f9fab8e67186b00abdd345b639976bc7fb3ae722e1b026a50/pyparsing-2.2.0-py2.py3-none-any.whl (56kB)\u001b[0m\n",
      "\u001b[31mRequirement already satisfied: six>=1.10 in /usr/local/lib/python3.5/dist-packages (from matplotlib) (1.11.0)\u001b[0m\n",
      "\u001b[31mRequirement already satisfied: numpy>=1.7.1 in /usr/local/lib/python3.5/dist-packages (from matplotlib) (1.13.3)\u001b[0m\n",
      "\u001b[31mCollecting cycler>=0.10 (from matplotlib)\n",
      "  Downloading https://files.pythonhosted.org/packages/f7/d2/e07d3ebb2bd7af696440ce7e754c59dd546ffe1bbe732c8ab68b9c834e61/cycler-0.10.0-py2.py3-none-any.whl\u001b[0m\n",
      "\u001b[31mRequirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.5/dist-packages (from matplotlib) (2.7.3)\u001b[0m\n",
      "\u001b[31mCollecting pytz (from matplotlib)\n",
      "  Downloading https://files.pythonhosted.org/packages/dc/83/15f7833b70d3e067ca91467ca245bae0f6fe56ddc7451aa0dc5606b120f2/pytz-2018.4-py2.py3-none-any.whl (510kB)\u001b[0m\n",
      "\u001b[31mRequirement already satisfied: setuptools in /usr/local/lib/python3.5/dist-packages (from kiwisolver>=1.0.1->matplotlib) (39.2.0)\u001b[0m\n",
      "\u001b[31mInstalling collected packages: kiwisolver, pyparsing, cycler, pytz, matplotlib\u001b[0m\n",
      "\u001b[31mSuccessfully installed cycler-0.10.0 kiwisolver-1.0.1 matplotlib-2.2.2 pyparsing-2.2.0 pytz-2018.4\u001b[0m\n",
      "\u001b[31m2018-05-29 10:38:21,160 INFO - matplotlib.font_manager - font search path ['/usr/local/lib/python3.5/dist-packages/matplotlib/mpl-data/fonts/ttf', '/usr/local/lib/python3.5/dist-packages/matplotlib/mpl-data/fonts/afm', '/usr/local/lib/python3.5/dist-packages/matplotlib/mpl-data/fonts/pdfcorefonts']\u001b[0m\n",
      "\u001b[31m2018-05-29 10:38:21,410 INFO - matplotlib.font_manager - generated new fontManager\u001b[0m\n",
      "\u001b[31m/usr/local/lib/python3.5/dist-packages/mxnet_container/train.py:178: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead\n",
      "  train_args = inspect.getargspec(user_module.train)\u001b[0m\n",
      "\u001b[31m[10:38:32] src/operator/nn/./cudnn/./cudnn_algoreg-inl.h:107: Running performance tests to find the best convolution algorithm, this can take a while... (setting env variable MXNET_CUDNN_AUTOTUNE_DEFAULT to 0 to disable)\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ec2-user/anaconda3/envs/mxnet_p36/lib/python3.6/site-packages/sagemaker/session.py:630: DeprecationWarning: generator 'multi_stream_iter' raised StopIteration\n",
      "  for idx, event in sagemaker.logs.multi_stream_iter(client, log_group, stream_names, positions):\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31mEpoch 0. Loss: 1.19020674213, Train_acc 0.927615951994, Test_acc 0.924924924925\u001b[0m\n",
      "\u001b[31mEpoch 1. Loss: 0.0955917794597, Train_acc 0.910488811101, Test_acc 0.904904904905\u001b[0m\n",
      "\u001b[31mEpoch 2. Loss: 0.0780380586131, Train_acc 0.982872859107, Test_acc 0.967967967968\u001b[0m\n",
      "\u001b[31mEpoch 3. Loss: 0.0515212092374, Train_acc 0.987123390424, Test_acc 0.95995995996\u001b[0m\n",
      "\u001b[31mEpoch 4. Loss: 0.0513322874282, Train_acc 0.995874484311, Test_acc 0.978978978979\u001b[0m\n",
      "===== Job Complete =====\n",
      "Billable seconds: 337\n"
     ]
    }
   ],
   "source": [
    "estimator.fit(inputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Deployment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:sagemaker:Creating model with name: sagemaker-mxnet-2018-05-29-10-33-20-284\n",
      "INFO:sagemaker:Creating endpoint with name sagemaker-mxnet-2018-05-29-10-33-20-284\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------------------------------!"
     ]
    }
   ],
   "source": [
    "predictor = estimator.deploy(instance_type=\"ml.m4.xlarge\", initial_instance_count=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "sagemaker_runtime_client = boto3.client(\"sagemaker-runtime\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Single Sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Kimberly predicted as Joanna\n"
     ]
    }
   ],
   "source": [
    "with open(\"Kimberly recites some shameless self promotion ad.mp3\", \"rb\") as audio_file:\n",
    "    payload = base64.b64encode(audio_file.read()).decode(\"utf-8\")\n",
    "    response = sagemaker_runtime_client.invoke_endpoint(\n",
    "        EndpointName=predictor.endpoint,\n",
    "        Body=payload,\n",
    "        ContentType=\"audio/mp3\",\n",
    "        Accept=\"application/json\"\n",
    "    )[\"Body\"].read()\n",
    "    print(\"Kimberly predicted as {}\".format(json.loads(response, encoding=\"utf-8\")))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Batch Mode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recordings with Salli:\n",
      "        Total: 125\n",
      "        Detected: 121\n",
      "        Accuracy: 0.97\n",
      "    \n",
      "Recordings with Kimberly:\n",
      "        Total: 120\n",
      "        Detected: 115\n",
      "        Accuracy: 0.96\n",
      "    \n",
      "Recordings with Joey:\n",
      "        Total: 125\n",
      "        Detected: 125\n",
      "        Accuracy: 1.00\n",
      "    \n",
      "Recordings with Justin:\n",
      "        Total: 120\n",
      "        Detected: 111\n",
      "        Accuracy: 0.93\n",
      "    \n",
      "Recordings with Matthew:\n",
      "        Total: 125\n",
      "        Detected: 125\n",
      "        Accuracy: 1.00\n",
      "    \n",
      "Recordings with Kendra:\n",
      "        Total: 120\n",
      "        Detected: 117\n",
      "        Accuracy: 0.97\n",
      "    \n",
      "Recordings with Ivy:\n",
      "        Total: 115\n",
      "        Detected: 111\n",
      "        Accuracy: 0.97\n",
      "    \n",
      "Recordings with Joanna:\n",
      "        Total: 130\n",
      "        Detected: 128\n",
      "        Accuracy: 0.98\n",
      "    \n"
     ]
    }
   ],
   "source": [
    "for directory in listdir(\"data/test\"):\n",
    "    batch = []\n",
    "    cnt = 0\n",
    "    total = 0\n",
    "    detected = 0\n",
    "    for file in listdir(\"data/test/{}\".format(directory)):\n",
    "        with open(\"data/test/{}/{}\".format(directory, file), \"rb\") as audio_file:\n",
    "            batch.append(base64.b64encode(audio_file.read()).decode(\"utf-8\"))\n",
    "            cnt += 1\n",
    "            if cnt == 5:\n",
    "                binary_json = json.dumps(batch).encode(\"utf-8\")\n",
    "                response = sagemaker_runtime_client.invoke_endpoint(\n",
    "                    EndpointName=predictor.endpoint,\n",
    "                    Body=binary_json,\n",
    "                    ContentType=\"application/json\",\n",
    "                    Accept=\"application/json\"\n",
    "                )[\"Body\"].read()\n",
    "                individual_predictions = json.loads(response, encoding=\"utf-8\")\n",
    "                for prediction in individual_predictions:\n",
    "                    total += 1 \n",
    "                    if prediction == directory:\n",
    "                        detected += 1\n",
    "                cnt = 0\n",
    "                batch = []\n",
    "    print(\"\"\"Recordings with {}:\n",
    "        Total: {}\n",
    "        Detected: {}\n",
    "        Accuracy: {:0.2f}\n",
    "    \"\"\".format(directory, str(total), str(detected), detected/total))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "What phrase to pronounce?Storm is what we're in\n",
      "Ivy, Joanna, Joey, Justin, Kendra, Kimberly, Matthew or Salli?Salli\n",
      "Salli predicted as Salli\n"
     ]
    }
   ],
   "source": [
    "test_phrase = input(\"What phrase to pronounce?\")\n",
    "test_voice = input(\"Ivy, Joanna, Joey, Justin, Kendra, Kimberly, Matthew or Salli?\")\n",
    "response = client.synthesize_speech(\n",
    "    OutputFormat=\"mp3\",\n",
    "    Text=test_phrase,\n",
    "    TextType=\"text\",\n",
    "    VoiceId=test_voice\n",
    ")\n",
    "with closing(response[\"AudioStream\"]) as stream:\n",
    "    payload = base64.b64encode(stream.read()).decode(\"utf-8\")\n",
    "    response = sagemaker_runtime_client.invoke_endpoint(\n",
    "        EndpointName=predictor.endpoint,\n",
    "        Body=payload,\n",
    "        ContentType=\"audio/mp3\",\n",
    "        Accept=\"application/json\"\n",
    "    )[\"Body\"].read()\n",
    "    print(\"{} predicted as {}\".format(test_voice, json.loads(response, encoding=\"utf-8\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_mxnet_p36",
   "language": "python",
   "name": "conda_mxnet_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}