Skip to content

Commit 87f9bf8

Browse files
SuperSecureHumanSuperSecureHuman
committed
Chapter 05 complete
1 parent dd70a19 commit 87f9bf8

File tree

3 files changed

+353
-0
lines changed

3 files changed

+353
-0
lines changed

‎Chapter 05/1.Complete_Code.ipynb

+237
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,237 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": 2,
6+
"metadata": {},
7+
"outputs": [],
8+
"source": [
9+
"import numpy as np\n",
10+
"import nnfs\n",
11+
"from nnfs.datasets import spiral_data\n",
12+
"nnfs.init()"
13+
]
14+
},
15+
{
16+
"cell_type": "code",
17+
"execution_count": 3,
18+
"metadata": {},
19+
"outputs": [],
20+
"source": [
21+
"# Dense layer\n",
22+
"class Layer_Dense:\n",
23+
"\n",
24+
" # Layer initialization\n",
25+
" def __init__(self, n_inputs, n_neurons):\n",
26+
" # Initialize weights and biases\n",
27+
" self.weights = 0.01 * np.random.randn(n_inputs, n_neurons)\n",
28+
" self.biases = np.zeros((1, n_neurons))\n",
29+
"\n",
30+
" # Forward pass\n",
31+
" def forward(self, inputs):\n",
32+
" # Calculate output values from inputs, weights and biases\n",
33+
" self.output = np.dot(inputs, self.weights) + self.biases"
34+
]
35+
},
36+
{
37+
"cell_type": "code",
38+
"execution_count": 4,
39+
"metadata": {},
40+
"outputs": [],
41+
"source": [
42+
"# ReLU activation\n",
43+
"class Activation_ReLU:\n",
44+
"\n",
45+
" # Forward pass\n",
46+
" def forward(self, inputs):\n",
47+
" # Calculate output values from inputs\n",
48+
" self.output = np.maximum(0, inputs)\n",
49+
"\n",
50+
"\n",
51+
"\n",
52+
"# Softmax activation\n",
53+
"class Activation_Softmax:\n",
54+
"\n",
55+
" # Forward pass\n",
56+
" def forward(self, inputs):\n",
57+
"\n",
58+
" # Get unnormalized probabilities\n",
59+
" exp_values = np.exp(inputs - np.max(inputs, axis=1,\n",
60+
" keepdims=True))\n",
61+
" # Normalize them for each sample\n",
62+
" probabilities = exp_values / np.sum(exp_values, axis=1,\n",
63+
" keepdims=True)\n",
64+
"\n",
65+
" self.output = probabilities"
66+
]
67+
},
68+
{
69+
"cell_type": "code",
70+
"execution_count": 5,
71+
"metadata": {},
72+
"outputs": [],
73+
"source": [
74+
"# Common loss class\n",
75+
"class Loss:\n",
76+
"\n",
77+
" # Calculates the data and regularization losses\n",
78+
" # given model output and ground truth values\n",
79+
" def calculate(self, output, y):\n",
80+
"\n",
81+
" # Calculate sample losses\n",
82+
" sample_losses = self.forward(output, y)\n",
83+
"\n",
84+
" # Calculate mean loss\n",
85+
" data_loss = np.mean(sample_losses)\n",
86+
"\n",
87+
" # Return loss\n",
88+
" return data_loss\n",
89+
"\n",
90+
"\n",
91+
"# Cross-entropy loss\n",
92+
"class Loss_CategoricalCrossentropy(Loss):\n",
93+
"\n",
94+
" # Forward pass\n",
95+
" def forward(self, y_pred, y_true):\n",
96+
"\n",
97+
" # Number of samples in a batch\n",
98+
" samples = len(y_pred)\n",
99+
"\n",
100+
" # Clip data to prevent division by 0\n",
101+
" # Clip both sides to not drag mean towards any value\n",
102+
" y_pred_clipped = np.clip(y_pred, 1e-7, 1 - 1e-7)\n",
103+
"\n",
104+
"\n",
105+
" # Probabilities for target values -\n",
106+
" # only if categorical labels\n",
107+
" if len(y_true.shape) == 1:\n",
108+
" correct_confidences = y_pred_clipped[\n",
109+
" range(samples),\n",
110+
" y_true\n",
111+
" ]\n",
112+
"\n",
113+
" # Mask values - only for one-hot encoded labels\n",
114+
" elif len(y_true.shape) == 2:\n",
115+
" correct_confidences = np.sum(\n",
116+
" y_pred_clipped*y_true,\n",
117+
" axis=1\n",
118+
" )\n",
119+
"\n",
120+
" # Losses\n",
121+
" negative_log_likelihoods = -np.log(correct_confidences)\n",
122+
" return negative_log_likelihoods"
123+
]
124+
},
125+
{
126+
"cell_type": "code",
127+
"execution_count": 6,
128+
"metadata": {},
129+
"outputs": [],
130+
"source": [
131+
"# Create dataset\n",
132+
"X, y = spiral_data(samples=100, classes=3)\n",
133+
"\n",
134+
"# Create Dense layer with 2 input features and 3 output values\n",
135+
"dense1 = Layer_Dense(2, 3)\n",
136+
"\n",
137+
"# Create ReLU activation (to be used with Dense layer):\n",
138+
"activation1 = Activation_ReLU()\n",
139+
"\n",
140+
"# Create second Dense layer with 3 input features (as we take output\n",
141+
"# of previous layer here) and 3 output values\n",
142+
"dense2 = Layer_Dense(3, 3)\n",
143+
"\n",
144+
"# Create Softmax activation (to be used with Dense layer):\n",
145+
"activation2 = Activation_Softmax()\n",
146+
"\n",
147+
"# Create loss function\n",
148+
"loss_function = Loss_CategoricalCrossentropy()\n"
149+
]
150+
},
151+
{
152+
"cell_type": "code",
153+
"execution_count": 8,
154+
"metadata": {},
155+
"outputs": [
156+
{
157+
"name": "stdout",
158+
"output_type": "stream",
159+
"text": [
160+
"[[0.33333334 0.33333334 0.33333334]\n",
161+
" [0.3333332 0.3333332 0.33333364]\n",
162+
" [0.3333329 0.33333293 0.3333342 ]\n",
163+
" [0.3333326 0.33333263 0.33333477]\n",
164+
" [0.33333233 0.3333324 0.33333528]]\n"
165+
]
166+
}
167+
],
168+
"source": [
169+
"# Perform a forward pass of our training data through this layer\n",
170+
"dense1.forward(X)\n",
171+
"\n",
172+
"# Perform a forward pass through activation function\n",
173+
"# it takes the output of first dense layer here\n",
174+
"activation1.forward(dense1.output)\n",
175+
"\n",
176+
"\n",
177+
"# Perform a forward pass through second Dense layer\n",
178+
"# it takes outputs of activation function of first layer as inputs\n",
179+
"dense2.forward(activation1.output)\n",
180+
"\n",
181+
"# Perform a forward pass through activation function\n",
182+
"# it takes the output of second dense layer here\n",
183+
"activation2.forward(dense2.output)\n",
184+
"\n",
185+
"# Let's see output of the first few samples:\n",
186+
"print(activation2.output[:5])"
187+
]
188+
},
189+
{
190+
"cell_type": "code",
191+
"execution_count": 9,
192+
"metadata": {},
193+
"outputs": [
194+
{
195+
"name": "stdout",
196+
"output_type": "stream",
197+
"text": [
198+
"loss: 1.0986104\n"
199+
]
200+
}
201+
],
202+
"source": [
203+
"# Perform a forward pass through loss function\n",
204+
"# it takes the output of second dense layer here and returns loss\n",
205+
"loss = loss_function.calculate(activation2.output, y)\n",
206+
"\n",
207+
"# Print loss value\n",
208+
"print('loss:', loss)"
209+
]
210+
}
211+
],
212+
"metadata": {
213+
"interpreter": {
214+
"hash": "9aeb3df5fbd6abcbd1bcdbd5cd117a35ff65e92160c2808640f7d55a504d1a5d"
215+
},
216+
"kernelspec": {
217+
"display_name": "Python 3.8.12 ('python38')",
218+
"language": "python",
219+
"name": "python3"
220+
},
221+
"language_info": {
222+
"codemirror_mode": {
223+
"name": "ipython",
224+
"version": 3
225+
},
226+
"file_extension": ".py",
227+
"mimetype": "text/x-python",
228+
"name": "python",
229+
"nbconvert_exporter": "python",
230+
"pygments_lexer": "ipython3",
231+
"version": "3.8.12"
232+
},
233+
"orig_nbformat": 4
234+
},
235+
"nbformat": 4,
236+
"nbformat_minor": 2
237+
}

‎Chapter 05/README.md

+116
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
## Calculating Network Error with Loss
2+
3+
---
4+
5+
With any neural network, the aim is to reduce the error in the network. And to do this, we need to calculate the error. When we have the error, we can apply some math magic to manuplate wights and biases in such a way that the error is reduced.
6+
7+
The loss function, also known as cost function, is used to calculate the error. Since loss is the model's error, we would ideally want it to be 0.
8+
9+
### Categorical Cross Entropy
10+
11+
12+
13+
![](./assets/Categorical_Cross_Entropy.png)
14+
15+
This is the equation for categorical cross entropy.
16+
17+
Imagine we have outputs of `[0.1, 0.7, 0.2]` and targets of `[0, 1, 0]`, then the calculation would be:
18+
19+
```python
20+
LOSS = -(0 * log(0.1)) + (1 * log(0.7)) + (0 * log(0.2))
21+
#(I am lazy to calculate this)
22+
```
23+
24+
Now we will look at a sample, in python
25+
26+
```python
27+
import math
28+
29+
#Output of the softmax layer
30+
softmax_output = [0.7, 0.1, 0.2]
31+
32+
#The actual output we need
33+
target_output = [1,0,0]
34+
35+
#Calculate the loss
36+
loss = -1 * (
37+
math.log(softmax_output[0]) * target_output[0] +
38+
math.log(softmax_output[1]) * target_output[1] +
39+
math.log(softmax_output[2]) * target_output[2]
40+
)
41+
42+
print(loss)
43+
44+
>>>
45+
0.3566749439387324
46+
```
47+
48+
Since we are mainly working with a classification problem, the target values will be 0, except for that 1 class which is the target.
49+
50+
So, what we can do is, ignore all other outputs and calculate the loss only for that class of which the ouput was supposed to be 1. Because all other outputs are multipled by 0.\
51+
52+
Now we will try to dynamically calculate the the entropy
53+
54+
```python
55+
softmax_outputs = [ [0.7, 0.1, 0.2],
56+
[0.1, 0.5, 0.4],
57+
[0.02, 0.9, 0.08]]
58+
59+
class_targets = [0, 1, 1]
60+
61+
for targ_idx, distribution in zip(class_targets, softmax_outputs):
62+
print(distribution[targ_idx])
63+
>>>
64+
0.7
65+
0.5
66+
0.9
67+
```
68+
69+
70+
You can see in this sample that by using zip(), we iterated over the produced outputs and desiered outputs. Now that we are sure that it selected the right output, we can use it to calculate the loss.
71+
72+
Now we make a class to implement this in python
73+
74+
```python
75+
# Cross-entropy loss
76+
class Loss_CategoricalCrossentropy(Loss):
77+
78+
# Forward pass
79+
def forward(self, y_pred, y_true):
80+
81+
# Number of samples in a batch
82+
samples = len(y_pred)
83+
84+
# Clip data to prevent division by 0
85+
# Clip both sides to not drag mean towards any value
86+
y_pred_clipped = np.clip(y_pred, 1e-7, 1 - 1e-7)
87+
88+
89+
# Probabilities for target values -
90+
# only if categorical labels
91+
if len(y_true.shape) == 1:
92+
correct_confidences = y_pred_clipped[
93+
range(samples),
94+
y_true
95+
]
96+
97+
# Mask values - only for one-hot encoded labels
98+
elif len(y_true.shape) == 2:
99+
correct_confidences = np.sum(
100+
y_pred_clipped*y_true,
101+
axis=1
102+
)
103+
104+
# Losses
105+
negative_log_likelihoods = -np.log(correct_confidences)
106+
return negative_log_likelihoods
107+
```
108+
109+
110+
Now we will update the complete code upto now
111+
112+
[Complete code upto now](./1.Complete_Code.ipynb)
113+
114+
---
115+
116+
Chapter 5 of nnfs book

‎Chapter 05/assets/Categorical_Cross_Entropy.png

368 KB
Loading

0 commit comments

Comments
 (0)