4.1 Example - Supervised Spam Classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Spam Classification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Some background information for text processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bag of words? Bag of whaa...?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"figures/bowjpg\" width=\"100%\">\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = [\"It was a bright cold day in April, and the clocks were striking thirteen\",\n",
    "    \"The sky above the port was the color of television, tuned to a dead channel\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CountVectorizer(analyzer=u'word', binary=False, decode_error=u'strict',\n",
       "        dtype=<type 'numpy.int64'>, encoding=u'utf-8', input=u'content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 1), preprocessor=None, stop_words=None,\n",
       "        strip_accents=None, token_pattern=u'(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
       "        tokenizer=None, vocabulary=None)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "vectorizer = CountVectorizer()\n",
    "vectorizer.fit(X)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{u'above': 0,\n",
       " u'and': 1,\n",
       " u'april': 2,\n",
       " u'bright': 3,\n",
       " u'channel': 4,\n",
       " u'clocks': 5,\n",
       " u'cold': 6,\n",
       " u'color': 7,\n",
       " u'day': 8,\n",
       " u'dead': 9,\n",
       " u'in': 10,\n",
       " u'it': 11,\n",
       " u'of': 12,\n",
       " u'port': 13,\n",
       " u'sky': 14,\n",
       " u'striking': 15,\n",
       " u'television': 16,\n",
       " u'the': 17,\n",
       " u'thirteen': 18,\n",
       " u'to': 19,\n",
       " u'tuned': 20,\n",
       " u'was': 21,\n",
       " u'were': 22}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer.vocabulary_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_bag_of_words = vectorizer.transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<2x23 sparse matrix of type '<type 'numpy.int64'>'\n",
       "\twith 25 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_bag_of_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 23)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_bag_of_words.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1],\n",
       "       [1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 3, 0, 1, 1, 1, 0]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_bag_of_words.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'above',\n",
       " u'and',\n",
       " u'april',\n",
       " u'bright',\n",
       " u'channel',\n",
       " u'clocks',\n",
       " u'cold',\n",
       " u'color',\n",
       " u'day',\n",
       " u'dead',\n",
       " u'in',\n",
       " u'it',\n",
       " u'of',\n",
       " u'port',\n",
       " u'sky',\n",
       " u'striking',\n",
       " u'television',\n",
       " u'the',\n",
       " u'thirteen',\n",
       " u'to',\n",
       " u'tuned',\n",
       " u'was',\n",
       " u'were']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([u'and', u'april', u'bright', u'clocks', u'cold', u'day', u'in',\n",
       "        u'it', u'striking', u'the', u'thirteen', u'was', u'were'], \n",
       "       dtype='<U10'),\n",
       " array([u'above', u'channel', u'color', u'dead', u'of', u'port', u'sky',\n",
       "        u'television', u'the', u'to', u'tuned', u'was'], \n",
       "       dtype='<U10')]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer.inverse_transform(X_bag_of_words)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TF-IDF\n",
    "In information retrieval, tf–idf, short for term frequency–inverse document frequency, is a numerical statistic that is intended to reflect how important a word is to a document in a collection or corpus.\n",
    "\n",
    "The importance increases proportionally to the number of times a word appears in the document but is offset by the frequency of the word in the corpus. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Typically, the tf-idf weight is composed by two terms: the first computes the normalized Term Frequency (TF), aka. the number of times a word appears in a document, divided by the total number of words in that document; the second term is the Inverse Document Frequency (IDF), computed as the logarithm of the number of the documents in the corpus divided by the number of documents where the specific term appears.\n",
    "\n",
    "**TF**: Term Frequency, which measures how frequently a term occurs in a document. Since every document is different in length, it is possible that a term would appear much more times in long documents than shorter ones. Thus, the term frequency is often divided by the document length (aka. the total number of terms in the document) as a way of normalization: \n",
    "\n",
    "**TF(t) = (Number of times term t appears in a document) / (Total number of terms in the document).**\n",
    "\n",
    "**IDF**: Inverse Document Frequency, which measures how important a term is. While computing TF, all terms are considered equally important. However it is known that certain terms, such as \"is\", \"of\", and \"that\", may appear a lot of times but have little importance. Thus we need to weigh down the frequent terms while scale up the rare ones, by computing the following: \n",
    "\n",
    "**IDF(t) = log_e(Total number of documents / Number of documents with term t in it).**\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TfidfVectorizer(analyzer=u'word', binary=False, decode_error=u'strict',\n",
       "        dtype=<type 'numpy.int64'>, encoding=u'utf-8', input=u'content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 1), norm=u'l2', preprocessor=None, smooth_idf=True,\n",
       "        stop_words=None, strip_accents=None, sublinear_tf=False,\n",
       "        token_pattern=u'(?u)\\\\b\\\\w\\\\w+\\\\b', tokenizer=None, use_idf=True,\n",
       "        vocabulary=None)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "tfidf_vectorizer = TfidfVectorizer()\n",
    "tfidf_vectorizer.fit(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.    0.29  0.29  0.29  0.    0.29  0.29  0.    0.29  0.    0.29  0.29\n",
      "   0.    0.    0.    0.29  0.    0.21  0.29  0.    0.    0.21  0.29]\n",
      " [ 0.26  0.    0.    0.    0.26  0.    0.    0.26  0.    0.26  0.    0.\n",
      "   0.26  0.26  0.26  0.    0.26  0.55  0.    0.26  0.26  0.18  0.  ]]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "np.set_printoptions(precision=2)\n",
    "\n",
    "print(tfidf_vectorizer.transform(X).toarray())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bigrams and N-Grams\n",
    "Entirely discarding word order is not always a good idea, as composite phrases often have specific meaning, and modifiers like \"not\" can invert the meaning of words.\n",
    "A simple way to include some word order are n-grams, which don't only look at a single token, but at all pairs of neighborhing tokens:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CountVectorizer(analyzer=u'word', binary=False, decode_error=u'strict',\n",
       "        dtype=<type 'numpy.int64'>, encoding=u'utf-8', input=u'content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(2, 2), preprocessor=None, stop_words=None,\n",
       "        strip_accents=None, token_pattern=u'(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
       "        tokenizer=None, vocabulary=None)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# look at sequences of tokens of minimum length 2 and maximum length 2\n",
    "bigram_vectorizer = CountVectorizer(ngram_range=(2, 2))\n",
    "bigram_vectorizer.fit(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'above the',\n",
       " u'and the',\n",
       " u'april and',\n",
       " u'bright cold',\n",
       " u'clocks were',\n",
       " u'cold day',\n",
       " u'color of',\n",
       " u'day in',\n",
       " u'dead channel',\n",
       " u'in april',\n",
       " u'it was',\n",
       " u'of television',\n",
       " u'port was',\n",
       " u'sky above',\n",
       " u'striking thirteen',\n",
       " u'television tuned',\n",
       " u'the clocks',\n",
       " u'the color',\n",
       " u'the port',\n",
       " u'the sky',\n",
       " u'to dead',\n",
       " u'tuned to',\n",
       " u'was bright',\n",
       " u'was the',\n",
       " u'were striking']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bigram_vectorizer.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0,\n",
       "        1, 0, 1],\n",
       "       [1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1,\n",
       "        0, 1, 0]])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bigram_vectorizer.transform(X).toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CountVectorizer(analyzer=u'word', binary=False, decode_error=u'strict',\n",
       "        dtype=<type 'numpy.int64'>, encoding=u'utf-8', input=u'content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 2), preprocessor=None, stop_words=None,\n",
       "        strip_accents=None, token_pattern=u'(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
       "        tokenizer=None, vocabulary=None)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gram_vectorizer = CountVectorizer(ngram_range=(1, 2))\n",
    "gram_vectorizer.fit(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'above',\n",
       " u'above the',\n",
       " u'and',\n",
       " u'and the',\n",
       " u'april',\n",
       " u'april and',\n",
       " u'bright',\n",
       " u'bright cold',\n",
       " u'channel',\n",
       " u'clocks',\n",
       " u'clocks were',\n",
       " u'cold',\n",
       " u'cold day',\n",
       " u'color',\n",
       " u'color of',\n",
       " u'day',\n",
       " u'day in',\n",
       " u'dead',\n",
       " u'dead channel',\n",
       " u'in',\n",
       " u'in april',\n",
       " u'it',\n",
       " u'it was',\n",
       " u'of',\n",
       " u'of television',\n",
       " u'port',\n",
       " u'port was',\n",
       " u'sky',\n",
       " u'sky above',\n",
       " u'striking',\n",
       " u'striking thirteen',\n",
       " u'television',\n",
       " u'television tuned',\n",
       " u'the',\n",
       " u'the clocks',\n",
       " u'the color',\n",
       " u'the port',\n",
       " u'the sky',\n",
       " u'thirteen',\n",
       " u'to',\n",
       " u'to dead',\n",
       " u'tuned',\n",
       " u'tuned to',\n",
       " u'was',\n",
       " u'was bright',\n",
       " u'was the',\n",
       " u'were',\n",
       " u'were striking']"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gram_vectorizer.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1,\n",
       "        1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1,\n",
       "        1, 0, 1, 1],\n",
       "       [1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 0,\n",
       "        0, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 3, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1,\n",
       "        0, 1, 0, 0]])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gram_vectorizer.transform(X).toarray()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Character N-grams"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sometimes it is also helpful to not look at words, but instead single character.\n",
    "That is particularly useful if you have very noisy data, want to identify the language, or we want to predict something about a single word.\n",
    "We can simply look at characters instead of words by setting ``analyzer=\"char\"``.\n",
    "Looking at single characters is usually not very informative, but looking at longer n-grams of characters can be:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CountVectorizer(analyzer='char', binary=False, decode_error=u'strict',\n",
       "        dtype=<type 'numpy.int64'>, encoding=u'utf-8', input=u'content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(2, 2), preprocessor=None, stop_words=None,\n",
       "        strip_accents=None, token_pattern=u'(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
       "        tokenizer=None, vocabulary=None)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "char_vectorizer = CountVectorizer(ngram_range=(2, 2), analyzer=\"char\")\n",
    "char_vectorizer.fit(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[u' a', u' b', u' c', u' d', u' i', u' o', u' p', u' s', u' t', u' w', u', ', u'a ', u'ab', u'ad', u'an', u'ap', u'as', u'ay', u'bo', u'br', u'ch', u'ck', u'cl', u'co', u'd ', u'da', u'de', u'e ', u'ea', u'ed', u'ee', u'el', u'en', u'er', u'ev', u'f ', u'g ', u'gh', u'ha', u'he', u'hi', u'ht', u'ig', u'ik', u'il', u'in', u'io', u'ir', u'is', u'it', u'ki', u'ks', u'ky', u'l,', u'ld', u'le', u'lo', u'n ', u'n,', u'nd', u'ne', u'ng', u'nn', u'o ', u'oc', u'of', u'ol', u'on', u'or', u'ov', u'po', u'pr', u'r ', u're', u'ri', u'rt', u's ', u'si', u'sk', u'st', u't ', u'te', u'th', u'to', u'tr', u'tu', u'un', u've', u'vi', u'wa', u'we', u'y ']\n"
     ]
    }
   ],
   "source": [
    "print(char_vectorizer.get_feature_names())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Moving on to the problem at hand"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "with open(os.path.join(\"data\",\"SMSSpamCollection\")) as f:\n",
    "    lines = [line.strip().split(\"\\t\") for line in f.readlines()]\n",
    "text = [x[1] for x in lines]\n",
    "y = [x[0] == \"ham\" for x in lines]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "\n",
    "text_train, text_test, y_train, y_test = train_test_split(text, y, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "vectorizer = CountVectorizer()\n",
    "vectorizer.fit(text_train)\n",
    "\n",
    "X_train = vectorizer.transform(text_train)\n",
    "X_test = vectorizer.transform(text_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7464\n"
     ]
    }
   ],
   "source": [
    "print(len(vectorizer.vocabulary_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[u'00', u'000', u'000pes', u'008704050406', u'0089', u'0121', u'01223585236', u'01223585334', u'02', u'0207', u'02072069400', u'02073162414', u'02085076972', u'03', u'04', u'0430', u'05', u'050703', u'0578', u'06']\n"
     ]
    }
   ],
   "source": [
    "print(vectorizer.get_feature_names()[:20])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[u'getting', u'getzed', u'gf', u'ghodbandar', u'ghost', u'gibbs', u'gibe', u'gift', u'gifted', u'gifts', u'giggle', u'gimme', u'gimmi', u'gin', u'girl', u'girlfrnd', u'girlie', u'girls', u'gist', u'giv']\n"
     ]
    }
   ],
   "source": [
    "print(vectorizer.get_feature_names()[3000:3020])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SGDClassifier(alpha=0.0001, average=False, class_weight=None, epsilon=0.1,\n",
       "       eta0=0.0, fit_intercept=True, l1_ratio=0.15,\n",
       "       learning_rate='optimal', loss='hinge', n_iter=5, n_jobs=1,\n",
       "       penalty='l2', power_t=0.5, random_state=None, shuffle=True,\n",
       "       verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "clf = SGDClassifier()\n",
    "clf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SGDClassifier(alpha=0.0001, average=False, class_weight=None, epsilon=0.1,\n",
       "       eta0=0.0, fit_intercept=True, l1_ratio=0.15,\n",
       "       learning_rate='optimal', loss='hinge', n_iter=5, n_jobs=1,\n",
       "       penalty='l2', power_t=0.5, random_state=None, shuffle=True,\n",
       "       verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9813486370157819"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.99880382775119614"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.score(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}