Skip to content

Commit d3e1391

Browse files
maciewarmaciewar
committed
Add ML2 workshop notebook
1 parent 53ade5b commit d3e1391

File tree

2 files changed

+21965
-0
lines changed

2 files changed

+21965
-0
lines changed

‎ml2/MLworkshop2.ipynb

+351
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,351 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "markdown",
5+
"metadata": {
6+
"collapsed": true
7+
},
8+
"source": [
9+
"# https://github.com/apisarek/MLworkshops"
10+
]
11+
},
12+
{
13+
"cell_type": "code",
14+
"execution_count": null,
15+
"metadata": {
16+
"collapsed": false
17+
},
18+
"outputs": [],
19+
"source": [
20+
"import matplotlib.pyplot as plt\n",
21+
"import pandas as pd\n",
22+
"import numpy as np\n",
23+
"\n",
24+
"np.random.seed(42)\n",
25+
"%matplotlib inline"
26+
]
27+
},
28+
{
29+
"cell_type": "code",
30+
"execution_count": null,
31+
"metadata": {
32+
"collapsed": false
33+
},
34+
"outputs": [],
35+
"source": [
36+
"houses = pd.read_csv('./kc_house_data.csv')\n",
37+
"sqft_living = houses[['sqft_living']].values[:, 0]\n",
38+
"price = np.array(houses['price'])\n",
39+
"houses.head()"
40+
]
41+
},
42+
{
43+
"cell_type": "code",
44+
"execution_count": null,
45+
"metadata": {
46+
"collapsed": false
47+
},
48+
"outputs": [],
49+
"source": [
50+
"plt.scatter(sqft_living, price)\n",
51+
"plt.show()"
52+
]
53+
},
54+
{
55+
"cell_type": "markdown",
56+
"metadata": {},
57+
"source": [
58+
"$h_W(X) = WX$"
59+
]
60+
},
61+
{
62+
"cell_type": "code",
63+
"execution_count": null,
64+
"metadata": {
65+
"collapsed": false
66+
},
67+
"outputs": [],
68+
"source": [
69+
"def predict(weights, feature_vector):\n",
70+
" return feature_vector @ weights"
71+
]
72+
},
73+
{
74+
"cell_type": "markdown",
75+
"metadata": {},
76+
"source": [
77+
"$J(W) = \\sum (WX - Y)^2$"
78+
]
79+
},
80+
{
81+
"cell_type": "code",
82+
"execution_count": null,
83+
"metadata": {
84+
"collapsed": true
85+
},
86+
"outputs": [],
87+
"source": [
88+
"def RSS(weights, feature_matrix, target):\n",
89+
" prediction = predict(weights, feature_matrix)\n",
90+
" return ((prediction - target) ** 2).sum()"
91+
]
92+
},
93+
{
94+
"cell_type": "markdown",
95+
"metadata": {},
96+
"source": [
97+
"$J(W) = \\sum (WX - Y)^2 + \\lambda \\sum W^2$"
98+
]
99+
},
100+
{
101+
"cell_type": "code",
102+
"execution_count": null,
103+
"metadata": {
104+
"collapsed": true
105+
},
106+
"outputs": [],
107+
"source": [
108+
"def regularized_RSS(weights, feature_matrix, target, regularization_size):\n",
109+
" pass"
110+
]
111+
},
112+
{
113+
"cell_type": "markdown",
114+
"metadata": {},
115+
"source": [
116+
"$\\frac{\\partial J}{\\partial W} = (WX - Y) X$"
117+
]
118+
},
119+
{
120+
"cell_type": "code",
121+
"execution_count": null,
122+
"metadata": {
123+
"collapsed": true
124+
},
125+
"outputs": [],
126+
"source": [
127+
"def gradient_descent(feature_matrix, target, initialize_weights=lambda x: (np.zeros(x.shape[1])),\n",
128+
" alpha=1e-12, iterations=1000):\n",
129+
" \n",
130+
" current_weights = initialize_weights(feature_matrix)\n",
131+
" for i in range(1, iterations):\n",
132+
" prediction = predict(current_weights, feature_matrix)\n",
133+
" error = prediction - target\n",
134+
" cost = RSS(current_weights, feature_matrix, target)\n",
135+
" \n",
136+
" if i % (iterations / 10) == 0:\n",
137+
" print('iteration: ', i, 'cost: ', cost)\n",
138+
" \n",
139+
" gradient = (error @ feature_matrix)\n",
140+
" current_weights -= alpha * gradient\n",
141+
" \n",
142+
" print('optimization done')\n",
143+
" return current_weights"
144+
]
145+
},
146+
{
147+
"cell_type": "markdown",
148+
"metadata": {},
149+
"source": [
150+
"$\\frac{\\partial J}{\\partial W} = (WX - Y)X + \\lambda W$"
151+
]
152+
},
153+
{
154+
"cell_type": "code",
155+
"execution_count": null,
156+
"metadata": {
157+
"collapsed": true
158+
},
159+
"outputs": [],
160+
"source": [
161+
"def regularized_gradient_descent(feature_matrix, target, initialize_weights=lambda x: (np.zeros(x.shape[1])),\n",
162+
" alpha=1e-12, regularization_size=0.01, iterations=1000):\n",
163+
" pass"
164+
]
165+
},
166+
{
167+
"cell_type": "code",
168+
"execution_count": null,
169+
"metadata": {
170+
"collapsed": true
171+
},
172+
"outputs": [],
173+
"source": [
174+
"def momentum_gradient_descent(feature_matrix, target, initialize_weights=lambda x: (np.zeros(x.shape[1])),\n",
175+
" alpha=1e-12, regularization_size=0.01, iterations=1000, momentum=0.9):\n",
176+
" pass"
177+
]
178+
},
179+
{
180+
"cell_type": "markdown",
181+
"metadata": {},
182+
"source": [
183+
"$W=(X^T X)^{-1} X^T Y$"
184+
]
185+
},
186+
{
187+
"cell_type": "code",
188+
"execution_count": null,
189+
"metadata": {
190+
"collapsed": true
191+
},
192+
"outputs": [],
193+
"source": [
194+
"def closed_form_solution(feature_matrix, target):\n",
195+
" return np.linalg.pinv(feature_matrix.T @ feature_matrix) @ feature_matrix.T @ target"
196+
]
197+
},
198+
{
199+
"cell_type": "markdown",
200+
"metadata": {},
201+
"source": [
202+
"$E=\\begin{bmatrix}\n",
203+
"1 & 0 & \\dots & 0\\\\ \n",
204+
"0 & 1 & \\dots & 0\\\\\n",
205+
"\\vdots & \\vdots & \\ddots & \\vdots\\\\ \n",
206+
"0 & 0 & \\dots & 1\n",
207+
"\\end{bmatrix}\\\\\n",
208+
"W=(X^T X + \\lambda E)^{-1} X^T Y$"
209+
]
210+
},
211+
{
212+
"cell_type": "code",
213+
"execution_count": null,
214+
"metadata": {
215+
"collapsed": false
216+
},
217+
"outputs": [],
218+
"source": [
219+
"def regularized_closed_form_solution(feature_matrix, target, regularization_size=0.0):\n",
220+
" pass"
221+
]
222+
},
223+
{
224+
"cell_type": "code",
225+
"execution_count": null,
226+
"metadata": {
227+
"collapsed": false
228+
},
229+
"outputs": [],
230+
"source": [
231+
"def prepare_dataset(X1, Y):\n",
232+
" X = np.column_stack((X1, np.ones_like(X1), X1 ** 2, ..., np.log(X1)))\n",
233+
" X = X / X.max(axis=0)\n",
234+
" Y = Y / Y.max()\n",
235+
" return X, Y"
236+
]
237+
},
238+
{
239+
"cell_type": "code",
240+
"execution_count": null,
241+
"metadata": {
242+
"collapsed": true
243+
},
244+
"outputs": [],
245+
"source": [
246+
"def plot_closed_form_regularization(dataset, regularizations, colors): \n",
247+
"# plt.figure(figsize=(30,20))\n",
248+
" X, Y = dataset\n",
249+
" plt.scatter(X1, Y)\n",
250+
" for regularization, color in zip(regularizations, colors):\n",
251+
" weights = regularized_closed_form_solution(X, Y, regularization_size=regularization)\n",
252+
" print('loss', regularized_RSS(weights, X, Y, regularization))\n",
253+
" plt.plot(X1, predict(weights, X), color) \n",
254+
" plt.show()"
255+
]
256+
},
257+
{
258+
"cell_type": "code",
259+
"execution_count": null,
260+
"metadata": {
261+
"collapsed": true
262+
},
263+
"outputs": [],
264+
"source": [
265+
"def plot_gradient_regularization(dataset, regularizations, colors, gradient_parameters): \n",
266+
"# plt.figure(figsize=(30,20))\n",
267+
" X, Y = dataset\n",
268+
" plt.scatter(X1, Y)\n",
269+
" for regularization, color in zip(regularizations, colors):\n",
270+
" weights = regularized_gradient_descent(X, Y, regularization_size=regularization, **gradient_parameters)\n",
271+
" plt.plot(X1, predict(weights, X), color) \n",
272+
" plt.show()"
273+
]
274+
},
275+
{
276+
"cell_type": "code",
277+
"execution_count": null,
278+
"metadata": {
279+
"collapsed": false
280+
},
281+
"outputs": [],
282+
"source": [
283+
"order = sqft_living.argsort()\n",
284+
"X1 = sqft_living[order]\n",
285+
"Y = price[order]\n",
286+
"dataset = prepare_dataset(X1, Y)\n",
287+
"\n",
288+
"gradient_parameters = {'alpha': 0, 'iterations': 1}\n",
289+
"regularization_values = []\n",
290+
"colors = ['r', 'g', 'y', 'k']\n",
291+
"plot_closed_form_regularization(dataset, regularization_values, colors)\n",
292+
"# plot_gradient_regularization(dataset, regularization_values, colors, gradient_parameters)"
293+
]
294+
},
295+
{
296+
"cell_type": "code",
297+
"execution_count": null,
298+
"metadata": {
299+
"collapsed": false
300+
},
301+
"outputs": [],
302+
"source": [
303+
"X1 = np.linspace(1, 15, 20)\n",
304+
"Y = 2 * np.sin(X1) + X1\n",
305+
"dataset = prepare_dataset(X1, Y)\n",
306+
"\n",
307+
"gradient_parameters = {'alpha': 0, 'iterations': 1}\n",
308+
"regularization_values = []\n",
309+
"colors = ['r', 'g', 'y', 'k']\n",
310+
"plot_closed_form_regularization(dataset, regularization_values, colors)\n",
311+
"# plot_gradient_regularization(dataset, regularization_values, colors, gradient_parameters)"
312+
]
313+
},
314+
{
315+
"cell_type": "markdown",
316+
"metadata": {},
317+
"source": [
318+
"# classification example: http://cs.stanford.edu/people/karpathy/convnetjs/demo/classify2d.html"
319+
]
320+
},
321+
{
322+
"cell_type": "markdown",
323+
"metadata": {},
324+
"source": [
325+
"- train, val, test split\n",
326+
"- minibatches"
327+
]
328+
}
329+
],
330+
"metadata": {
331+
"kernelspec": {
332+
"display_name": "Python 3",
333+
"language": "python",
334+
"name": "python3"
335+
},
336+
"language_info": {
337+
"codemirror_mode": {
338+
"name": "ipython",
339+
"version": 3
340+
},
341+
"file_extension": ".py",
342+
"mimetype": "text/x-python",
343+
"name": "python",
344+
"nbconvert_exporter": "python",
345+
"pygments_lexer": "ipython3",
346+
"version": "3.5.1"
347+
}
348+
},
349+
"nbformat": 4,
350+
"nbformat_minor": 0
351+
}

0 commit comments

Comments
 (0)