Skip to content

Commit cb8954f

Browse files
cezanneccezannec
committed
Existing DL material
1 parent 77331e4 commit cb8954f

34 files changed

+87623
-0
lines changed

‎intro-neural-networks/gradient-descent/GradientDescent.ipynb

Lines changed: 244 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,244 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "markdown",
5+
"metadata": {},
6+
"source": [
7+
"# Implementing the Gradient Descent Algorithm\n",
8+
"\n",
9+
"In this lab, we'll implement the basic functions of the Gradient Descent algorithm to find the boundary in a small dataset. First, we'll start with some functions that will help us plot and visualize the data."
10+
]
11+
},
12+
{
13+
"cell_type": "code",
14+
"execution_count": null,
15+
"metadata": {
16+
"collapsed": true
17+
},
18+
"outputs": [],
19+
"source": [
20+
"import matplotlib.pyplot as plt\n",
21+
"import numpy as np\n",
22+
"import pandas as pd\n",
23+
"\n",
24+
"#Some helper functions for plotting and drawing lines\n",
25+
"\n",
26+
"def plot_points(X, y):\n",
27+
" admitted = X[np.argwhere(y==1)]\n",
28+
" rejected = X[np.argwhere(y==0)]\n",
29+
" plt.scatter([s[0][0] for s in rejected], [s[0][1] for s in rejected], s = 25, color = 'blue', edgecolor = 'k')\n",
30+
" plt.scatter([s[0][0] for s in admitted], [s[0][1] for s in admitted], s = 25, color = 'red', edgecolor = 'k')\n",
31+
"\n",
32+
"def display(m, b, color='g--'):\n",
33+
" plt.xlim(-0.05,1.05)\n",
34+
" plt.ylim(-0.05,1.05)\n",
35+
" x = np.arange(-10, 10, 0.1)\n",
36+
" plt.plot(x, m*x+b, color)"
37+
]
38+
},
39+
{
40+
"cell_type": "markdown",
41+
"metadata": {},
42+
"source": [
43+
"## Reading and plotting the data"
44+
]
45+
},
46+
{
47+
"cell_type": "code",
48+
"execution_count": null,
49+
"metadata": {
50+
"collapsed": true
51+
},
52+
"outputs": [],
53+
"source": [
54+
"data = pd.read_csv('data.csv', header=None)\n",
55+
"X = np.array(data[[0,1]])\n",
56+
"y = np.array(data[2])\n",
57+
"plot_points(X,y)\n",
58+
"plt.show()"
59+
]
60+
},
61+
{
62+
"cell_type": "markdown",
63+
"metadata": {},
64+
"source": [
65+
"## TODO: Implementing the basic functions\n",
66+
"Here is your turn to shine. Implement the following formulas, as explained in the text.\n",
67+
"- Sigmoid activation function\n",
68+
"\n",
69+
"$$\\sigma(x) = \\frac{1}{1+e^{-x}}$$\n",
70+
"\n",
71+
"- Output (prediction) formula\n",
72+
"\n",
73+
"$$\\hat{y} = \\sigma(w_1 x_1 + w_2 x_2 + b)$$\n",
74+
"\n",
75+
"- Error function\n",
76+
"\n",
77+
"$$Error(y, \\hat{y}) = - y \\log(\\hat{y}) - (1-y) \\log(1-\\hat{y})$$\n",
78+
"\n",
79+
"- The function that updates the weights\n",
80+
"\n",
81+
"$$ w_i \\longrightarrow w_i + \\alpha (y - \\hat{y}) x_i$$\n",
82+
"\n",
83+
"$$ b \\longrightarrow b + \\alpha (y - \\hat{y})$$"
84+
]
85+
},
86+
{
87+
"cell_type": "code",
88+
"execution_count": null,
89+
"metadata": {
90+
"collapsed": true
91+
},
92+
"outputs": [],
93+
"source": [
94+
"# Implement the following functions\n",
95+
"\n",
96+
"# Activation (sigmoid) function\n",
97+
"def sigmoid(x):\n",
98+
" pass\n",
99+
"\n",
100+
"# Output (prediction) formula\n",
101+
"def output_formula(features, weights, bias):\n",
102+
" pass\n",
103+
"\n",
104+
"# Error (log-loss) formula\n",
105+
"def error_formula(y, output):\n",
106+
" pass\n",
107+
"\n",
108+
"# Gradient descent step\n",
109+
"def update_weights(x, y, weights, bias, learnrate):\n",
110+
" pass"
111+
]
112+
},
113+
{
114+
"cell_type": "markdown",
115+
"metadata": {},
116+
"source": [
117+
"## Training function\n",
118+
"This function will help us iterate the gradient descent algorithm through all the data, for a number of epochs. It will also plot the data, and some of the boundary lines obtained as we run the algorithm."
119+
]
120+
},
121+
{
122+
"cell_type": "code",
123+
"execution_count": null,
124+
"metadata": {
125+
"collapsed": true
126+
},
127+
"outputs": [],
128+
"source": [
129+
"np.random.seed(44)\n",
130+
"\n",
131+
"epochs = 100\n",
132+
"learnrate = 0.01\n",
133+
"\n",
134+
"def train(features, targets, epochs, learnrate, graph_lines=False):\n",
135+
" \n",
136+
" errors = []\n",
137+
" n_records, n_features = features.shape\n",
138+
" last_loss = None\n",
139+
" weights = np.random.normal(scale=1 / n_features**.5, size=n_features)\n",
140+
" bias = 0\n",
141+
" for e in range(epochs):\n",
142+
" del_w = np.zeros(weights.shape)\n",
143+
" for x, y in zip(features, targets):\n",
144+
" output = output_formula(x, weights, bias)\n",
145+
" error = error_formula(y, output)\n",
146+
" weights, bias = update_weights(x, y, weights, bias, learnrate)\n",
147+
" \n",
148+
" # Printing out the log-loss error on the training set\n",
149+
" out = output_formula(features, weights, bias)\n",
150+
" loss = np.mean(error_formula(targets, out))\n",
151+
" errors.append(loss)\n",
152+
" if e % (epochs / 10) == 0:\n",
153+
" print(\"\\n========== Epoch\", e,\"==========\")\n",
154+
" if last_loss and last_loss < loss:\n",
155+
" print(\"Train loss: \", loss, \" WARNING - Loss Increasing\")\n",
156+
" else:\n",
157+
" print(\"Train loss: \", loss)\n",
158+
" last_loss = loss\n",
159+
" predictions = out > 0.5\n",
160+
" accuracy = np.mean(predictions == targets)\n",
161+
" print(\"Accuracy: \", accuracy)\n",
162+
" if graph_lines and e % (epochs / 100) == 0:\n",
163+
" display(-weights[0]/weights[1], -bias/weights[1])\n",
164+
" \n",
165+
"\n",
166+
" # Plotting the solution boundary\n",
167+
" plt.title(\"Solution boundary\")\n",
168+
" display(-weights[0]/weights[1], -bias/weights[1], 'black')\n",
169+
"\n",
170+
" # Plotting the data\n",
171+
" plot_points(features, targets)\n",
172+
" plt.show()\n",
173+
"\n",
174+
" # Plotting the error\n",
175+
" plt.title(\"Error Plot\")\n",
176+
" plt.xlabel('Number of epochs')\n",
177+
" plt.ylabel('Error')\n",
178+
" plt.plot(errors)\n",
179+
" plt.show()"
180+
]
181+
},
182+
{
183+
"cell_type": "markdown",
184+
"metadata": {},
185+
"source": [
186+
"## Time to train the algorithm!\n",
187+
"When we run the function, we'll obtain the following:\n",
188+
"- 10 updates with the current training loss and accuracy\n",
189+
"- A plot of the data and some of the boundary lines obtained. The final one is in black. Notice how the lines get closer and closer to the best fit, as we go through more epochs.\n",
190+
"- A plot of the error function. Notice how it decreases as we go through more epochs."
191+
]
192+
},
193+
{
194+
"cell_type": "code",
195+
"execution_count": null,
196+
"metadata": {
197+
"collapsed": true
198+
},
199+
"outputs": [],
200+
"source": [
201+
"train(X, y, epochs, learnrate, True)"
202+
]
203+
},
204+
{
205+
"cell_type": "code",
206+
"execution_count": null,
207+
"metadata": {
208+
"collapsed": true
209+
},
210+
"outputs": [],
211+
"source": []
212+
},
213+
{
214+
"cell_type": "code",
215+
"execution_count": null,
216+
"metadata": {
217+
"collapsed": true
218+
},
219+
"outputs": [],
220+
"source": []
221+
}
222+
],
223+
"metadata": {
224+
"kernelspec": {
225+
"display_name": "Python 3",
226+
"language": "python",
227+
"name": "python3"
228+
},
229+
"language_info": {
230+
"codemirror_mode": {
231+
"name": "ipython",
232+
"version": 3
233+
},
234+
"file_extension": ".py",
235+
"mimetype": "text/x-python",
236+
"name": "python",
237+
"nbconvert_exporter": "python",
238+
"pygments_lexer": "ipython3",
239+
"version": "3.6.1"
240+
}
241+
},
242+
"nbformat": 4,
243+
"nbformat_minor": 2
244+
}

‎intro-neural-networks/gradient-descent/GradientDescentSolutions.ipynb

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "markdown",
5+
"metadata": {},
6+
"source": [
7+
"# Solutions"
8+
]
9+
},
10+
{
11+
"cell_type": "code",
12+
"execution_count": null,
13+
"metadata": {
14+
"collapsed": true
15+
},
16+
"outputs": [],
17+
"source": [
18+
"# Activation (sigmoid) function\n",
19+
"def sigmoid(x):\n",
20+
" return 1 / (1 + np.exp(-x))\n",
21+
"\n",
22+
"def output_formula(features, weights, bias):\n",
23+
" return sigmoid(np.dot(features, weights) + bias)\n",
24+
"\n",
25+
"def error_formula(y, output):\n",
26+
" return - y*np.log(output) - (1 - y) * np.log(1-output)\n",
27+
"\n",
28+
"def update_weights(x, y, weights, bias, learnrate):\n",
29+
" output = output_formula(x, weights, bias)\n",
30+
" d_error = -(y - output)\n",
31+
" weights -= learnrate * d_error * x\n",
32+
" bias -= learnrate * d_error\n",
33+
" return weights, bias"
34+
]
35+
}
36+
],
37+
"metadata": {
38+
"kernelspec": {
39+
"display_name": "Python 3",
40+
"language": "python",
41+
"name": "python3"
42+
},
43+
"language_info": {
44+
"codemirror_mode": {
45+
"name": "ipython",
46+
"version": 3
47+
},
48+
"file_extension": ".py",
49+
"mimetype": "text/x-python",
50+
"name": "python",
51+
"nbconvert_exporter": "python",
52+
"pygments_lexer": "ipython3",
53+
"version": "3.6.1"
54+
}
55+
},
56+
"nbformat": 4,
57+
"nbformat_minor": 2
58+
}

0 commit comments

Comments
 (0)