Linear Regression Example PySpark.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Linear Regression Example PySpark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "spark = SparkSession.builder.appName('lr_example').getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.regression import LinearRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use Spark to read in the Ecommerce Customers csv file.\n",
    "data = spark.read.csv(\"Ecommerce_Customers.csv\",inferSchema=True,header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- Email: string (nullable = true)\n",
      " |-- Address: string (nullable = true)\n",
      " |-- Avatar: string (nullable = true)\n",
      " |-- Avg Session Length: double (nullable = true)\n",
      " |-- Time on App: double (nullable = true)\n",
      " |-- Time on Website: double (nullable = true)\n",
      " |-- Length of Membership: double (nullable = true)\n",
      " |-- Yearly Amount Spent: double (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Print the Schema of the DataFrame\n",
    "data.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+--------------------+----------------+------------------+------------------+------------------+--------------------+-------------------+\n",
      "|               Email|             Address|          Avatar|Avg Session Length|       Time on App|   Time on Website|Length of Membership|Yearly Amount Spent|\n",
      "+--------------------+--------------------+----------------+------------------+------------------+------------------+--------------------+-------------------+\n",
      "|mstephenson@ferna...|835 Frank TunnelW...|          Violet| 34.49726772511229| 12.65565114916675| 39.57766801952616|  4.0826206329529615|  587.9510539684005|\n",
      "|   hduke@hotmail.com|4547 Archer Commo...|       DarkGreen| 31.92627202636016|11.109460728682564|37.268958868297744|    2.66403418213262|  392.2049334443264|\n",
      "|    pallen@yahoo.com|24645 Valerie Uni...|          Bisque|33.000914755642675|11.330278057777512|37.110597442120856|   4.104543202376424| 487.54750486747207|\n",
      "|riverarebecca@gma...|1414 David Throug...|     SaddleBrown| 34.30555662975554|13.717513665142507| 36.72128267790313|   3.120178782748092|  581.8523440352177|\n",
      "|mstephens@davidso...|14023 Rodriguez P...|MediumAquaMarine| 33.33067252364639|12.795188551078114| 37.53665330059473|   4.446308318351434|  599.4060920457634|\n",
      "|alvareznancy@luca...|645 Martha Park A...|     FloralWhite|33.871037879341976|12.026925339755056| 34.47687762925054|   5.493507201364199|   637.102447915074|\n",
      "|katherine20@yahoo...|68388 Reyes Light...|   DarkSlateBlue| 32.02159550138701|11.366348309710526| 36.68377615286961|   4.685017246570912|  521.5721747578274|\n",
      "|  awatkins@yahoo.com|Unit 6538 Box 898...|            Aqua|32.739142938380326| 12.35195897300293| 37.37335885854755|  4.4342734348999375|  549.9041461052942|\n",
      "|vchurch@walter-ma...|860 Lee KeyWest D...|          Salmon| 33.98777289568564|13.386235275676436|37.534497341555735|  3.2734335777477144|  570.2004089636196|\n",
      "|    bonnie69@lin.biz|PSC 2734, Box 525...|           Brown|31.936548618448917|11.814128294972196| 37.14516822352819|   3.202806071553459|  427.1993848953282|\n",
      "|andrew06@peterson...|26104 Alexander G...|          Tomato|33.992572774953736|13.338975447662113| 37.22580613162114|   2.482607770510596|  492.6060127179966|\n",
      "|ryanwerner@freema...|Unit 2413 Box 034...|          Tomato| 33.87936082480498|11.584782999535266| 37.08792607098381|    3.71320920294043|  522.3374046069357|\n",
      "|   knelson@gmail.com|6705 Miller Orcha...|       RoyalBlue|29.532428967057943|10.961298400154098| 37.42021557502538|   4.046423164299585|  408.6403510726275|\n",
      "|wrightpeter@yahoo...|05302 Dunlap Ferr...|          Bisque| 33.19033404372265|12.959226091609382|36.144666700041924|   3.918541839158999|  573.4158673313865|\n",
      "|taylormason@gmail...|7773 Powell Sprin...|        DarkBlue|32.387975853153876|13.148725692056516| 36.61995708279922|   2.494543646659249|  470.4527333009554|\n",
      "| jstark@anderson.com|49558 Ramirez Roa...|            Peru|30.737720372628182|12.636606052000127|36.213763093698624|  3.3578468423262944|  461.7807421962299|\n",
      "| wjennings@gmail.com|6362 Wilson Mount...|      PowderBlue| 32.12538689728784|11.733861690857394|  34.8940927514398|  3.1361327164897803| 457.84769594494855|\n",
      "|rebecca45@hale-ba...|8982 Burton RowWi...|       OliveDrab|32.338899323067196|12.013194694014402| 38.38513659413844|   2.420806160901484| 407.70454754954415|\n",
      "|alejandro75@hotma...|64475 Andre Club ...|            Cyan|32.187812045932155|  14.7153875441565| 38.24411459434352|   1.516575580831944|  452.3156754800354|\n",
      "|samuel46@love-wes...|544 Alexander Hei...|   LightSeaGreen| 32.61785606282345|13.989592555825254|37.190503800397956|   4.064548550437977|   605.061038804892|\n",
      "+--------------------+--------------------+----------------+------------------+------------------+------------------+--------------------+-------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "data.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Row(Email='mstephenson@fernandez.com', Address='835 Frank TunnelWrightmouth, MI 82180-9605', Avatar='Violet', Avg Session Length=34.49726772511229, Time on App=12.65565114916675, Time on Website=39.57766801952616, Length of Membership=4.0826206329529615, Yearly Amount Spent=587.9510539684005)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mstephenson@fernandez.com\n",
      "835 Frank TunnelWrightmouth, MI 82180-9605\n",
      "Violet\n",
      "34.49726772511229\n",
      "12.65565114916675\n",
      "39.57766801952616\n",
      "4.0826206329529615\n",
      "587.9510539684005\n"
     ]
    }
   ],
   "source": [
    "for item in data.head():\n",
    "    print(item)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setting Up DataFrame for Machine Learning "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A few things we need to do before Spark can accept the data!\n",
    "# It needs to be in the form of two columns\n",
    "# (\"label\",\"features\")\n",
    "\n",
    "# Import VectorAssembler and Vectors\n",
    "from pyspark.ml.linalg import Vectors\n",
    "from pyspark.ml.feature import VectorAssembler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Email',\n",
       " 'Address',\n",
       " 'Avatar',\n",
       " 'Avg Session Length',\n",
       " 'Time on App',\n",
       " 'Time on Website',\n",
       " 'Length of Membership',\n",
       " 'Yearly Amount Spent']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "assembler = VectorAssembler(\n",
    "    inputCols=[\"Avg Session Length\", \"Time on App\", \n",
    "               \"Time on Website\",'Length of Membership'],\n",
    "    outputCol=\"features\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "output = assembler.transform(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+\n",
      "|            features|\n",
      "+--------------------+\n",
      "|[34.4972677251122...|\n",
      "|[31.9262720263601...|\n",
      "|[33.0009147556426...|\n",
      "|[34.3055566297555...|\n",
      "|[33.3306725236463...|\n",
      "|[33.8710378793419...|\n",
      "|[32.0215955013870...|\n",
      "|[32.7391429383803...|\n",
      "|[33.9877728956856...|\n",
      "|[31.9365486184489...|\n",
      "|[33.9925727749537...|\n",
      "|[33.8793608248049...|\n",
      "|[29.5324289670579...|\n",
      "|[33.1903340437226...|\n",
      "|[32.3879758531538...|\n",
      "|[30.7377203726281...|\n",
      "|[32.1253868972878...|\n",
      "|[32.3388993230671...|\n",
      "|[32.1878120459321...|\n",
      "|[32.6178560628234...|\n",
      "+--------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "output.select(\"features\").show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+--------------------+----------------+------------------+------------------+------------------+--------------------+-------------------+--------------------+\n",
      "|               Email|             Address|          Avatar|Avg Session Length|       Time on App|   Time on Website|Length of Membership|Yearly Amount Spent|            features|\n",
      "+--------------------+--------------------+----------------+------------------+------------------+------------------+--------------------+-------------------+--------------------+\n",
      "|mstephenson@ferna...|835 Frank TunnelW...|          Violet| 34.49726772511229| 12.65565114916675| 39.57766801952616|  4.0826206329529615|  587.9510539684005|[34.4972677251122...|\n",
      "|   hduke@hotmail.com|4547 Archer Commo...|       DarkGreen| 31.92627202636016|11.109460728682564|37.268958868297744|    2.66403418213262|  392.2049334443264|[31.9262720263601...|\n",
      "|    pallen@yahoo.com|24645 Valerie Uni...|          Bisque|33.000914755642675|11.330278057777512|37.110597442120856|   4.104543202376424| 487.54750486747207|[33.0009147556426...|\n",
      "|riverarebecca@gma...|1414 David Throug...|     SaddleBrown| 34.30555662975554|13.717513665142507| 36.72128267790313|   3.120178782748092|  581.8523440352177|[34.3055566297555...|\n",
      "|mstephens@davidso...|14023 Rodriguez P...|MediumAquaMarine| 33.33067252364639|12.795188551078114| 37.53665330059473|   4.446308318351434|  599.4060920457634|[33.3306725236463...|\n",
      "|alvareznancy@luca...|645 Martha Park A...|     FloralWhite|33.871037879341976|12.026925339755056| 34.47687762925054|   5.493507201364199|   637.102447915074|[33.8710378793419...|\n",
      "|katherine20@yahoo...|68388 Reyes Light...|   DarkSlateBlue| 32.02159550138701|11.366348309710526| 36.68377615286961|   4.685017246570912|  521.5721747578274|[32.0215955013870...|\n",
      "|  awatkins@yahoo.com|Unit 6538 Box 898...|            Aqua|32.739142938380326| 12.35195897300293| 37.37335885854755|  4.4342734348999375|  549.9041461052942|[32.7391429383803...|\n",
      "|vchurch@walter-ma...|860 Lee KeyWest D...|          Salmon| 33.98777289568564|13.386235275676436|37.534497341555735|  3.2734335777477144|  570.2004089636196|[33.9877728956856...|\n",
      "|    bonnie69@lin.biz|PSC 2734, Box 525...|           Brown|31.936548618448917|11.814128294972196| 37.14516822352819|   3.202806071553459|  427.1993848953282|[31.9365486184489...|\n",
      "|andrew06@peterson...|26104 Alexander G...|          Tomato|33.992572774953736|13.338975447662113| 37.22580613162114|   2.482607770510596|  492.6060127179966|[33.9925727749537...|\n",
      "|ryanwerner@freema...|Unit 2413 Box 034...|          Tomato| 33.87936082480498|11.584782999535266| 37.08792607098381|    3.71320920294043|  522.3374046069357|[33.8793608248049...|\n",
      "|   knelson@gmail.com|6705 Miller Orcha...|       RoyalBlue|29.532428967057943|10.961298400154098| 37.42021557502538|   4.046423164299585|  408.6403510726275|[29.5324289670579...|\n",
      "|wrightpeter@yahoo...|05302 Dunlap Ferr...|          Bisque| 33.19033404372265|12.959226091609382|36.144666700041924|   3.918541839158999|  573.4158673313865|[33.1903340437226...|\n",
      "|taylormason@gmail...|7773 Powell Sprin...|        DarkBlue|32.387975853153876|13.148725692056516| 36.61995708279922|   2.494543646659249|  470.4527333009554|[32.3879758531538...|\n",
      "| jstark@anderson.com|49558 Ramirez Roa...|            Peru|30.737720372628182|12.636606052000127|36.213763093698624|  3.3578468423262944|  461.7807421962299|[30.7377203726281...|\n",
      "| wjennings@gmail.com|6362 Wilson Mount...|      PowderBlue| 32.12538689728784|11.733861690857394|  34.8940927514398|  3.1361327164897803| 457.84769594494855|[32.1253868972878...|\n",
      "|rebecca45@hale-ba...|8982 Burton RowWi...|       OliveDrab|32.338899323067196|12.013194694014402| 38.38513659413844|   2.420806160901484| 407.70454754954415|[32.3388993230671...|\n",
      "|alejandro75@hotma...|64475 Andre Club ...|            Cyan|32.187812045932155|  14.7153875441565| 38.24411459434352|   1.516575580831944|  452.3156754800354|[32.1878120459321...|\n",
      "|samuel46@love-wes...|544 Alexander Hei...|   LightSeaGreen| 32.61785606282345|13.989592555825254|37.190503800397956|   4.064548550437977|   605.061038804892|[32.6178560628234...|\n",
      "+--------------------+--------------------+----------------+------------------+------------------+------------------+--------------------+-------------------+--------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "output.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_data = output.select(\"features\",'Yearly Amount Spent')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data,test_data = final_data.randomSplit([0.7,0.3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-------------------+\n",
      "|summary|Yearly Amount Spent|\n",
      "+-------+-------------------+\n",
      "|  count|                362|\n",
      "|   mean| 497.11216990024263|\n",
      "| stddev|  80.02100842452546|\n",
      "|    min| 256.67058229005585|\n",
      "|    max|  765.5184619388373|\n",
      "+-------+-------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "train_data.describe().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-------------------+\n",
      "|summary|Yearly Amount Spent|\n",
      "+-------+-------------------+\n",
      "|  count|                138|\n",
      "|   mean|  505.0899538073016|\n",
      "| stddev|   77.4197933344791|\n",
      "|    min| 302.18954780965197|\n",
      "|    max|  725.5848140556806|\n",
      "+-------+-------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "test_data.describe().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a Linear Regression Model object\n",
    "lr = LinearRegression(labelCol='Yearly Amount Spent')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fit the model to the data and call this model lrModel\n",
    "lrModel = lr.fit(train_data,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coefficients: [26.016036525346106,38.97201084192561,0.6656029642873608,61.581612753204766] Intercept: -1072.2049057694205\n"
     ]
    }
   ],
   "source": [
    "# Print the coefficients and intercept for linear regression\n",
    "print(\"Coefficients: {} Intercept: {}\".format(lrModel.coefficients,lrModel.intercept))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_results = lrModel.evaluate(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------------+\n",
      "|          residuals|\n",
      "+-------------------+\n",
      "| 10.952431989913407|\n",
      "| -3.986398412750475|\n",
      "|-3.8802272472486834|\n",
      "| 22.307541517197933|\n",
      "| 18.676190336837237|\n",
      "|  3.987571728017315|\n",
      "| 0.6777094826013581|\n",
      "| -5.305016122824441|\n",
      "|-10.971194870711543|\n",
      "|-1.7262798268065467|\n",
      "| -9.568127823435987|\n",
      "|  -5.36272941356259|\n",
      "|-17.290392090575835|\n",
      "| 11.994905227627498|\n",
      "|  -8.71142583136384|\n",
      "| 16.707709339578003|\n",
      "|  4.252731738041803|\n",
      "| -5.802694072146778|\n",
      "| -3.180210699745089|\n",
      "|  4.973513209019984|\n",
      "+-------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Interesting results....\n",
    "test_results.residuals.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "unlabeled_data = test_data.select('features')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = lrModel.transform(unlabeled_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------------------+\n",
      "|            features|        prediction|\n",
      "+--------------------+------------------+\n",
      "|[30.7377203726281...| 450.8283102063165|\n",
      "|[30.8364326747734...| 471.4882988397401|\n",
      "|[30.8794843441274...|494.08682723210336|\n",
      "|[31.2834474760581...| 569.4735479084695|\n",
      "|[31.3123495994443...| 444.9152276911034|\n",
      "|[31.3584771924370...| 491.1883787214581|\n",
      "|[31.3895854806643...|409.39190157738153|\n",
      "|[31.5257524169682...| 449.2706429327063|\n",
      "|[31.8093003166791...| 547.7430942335527|\n",
      "|[31.8186165667690...| 448.1449531969422|\n",
      "|[31.8279790554652...| 449.5708753703775|\n",
      "|[31.8745516945853...| 397.6479736598301|\n",
      "|[31.9048571310136...|  491.240249513392|\n",
      "|[31.9262720263601...| 380.2100282166989|\n",
      "|[32.0085045178551...|451.90864686011923|\n",
      "|[32.0478146331398...| 480.6818484192654|\n",
      "|[32.0542618511847...| 557.6219259309412|\n",
      "|[32.0883806304482...| 517.9685604603048|\n",
      "|[32.1223647957977...| 532.1138292647654|\n",
      "|[32.1977237974218...| 509.1154445375521|\n",
      "+--------------------+------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 11.08458090598401\n",
      "MSE: 122.8679338613053\n"
     ]
    }
   ],
   "source": [
    "print(\"RMSE: {}\".format(test_results.rootMeanSquaredError))\n",
    "print(\"MSE: {}\".format(test_results.meanSquaredError))"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}