Goal: Develop a classifier that can classify handwritten
digits.
1. Generate the training and testing sets using the MNIST data
folder that contains
60,000 training and 10,000 test images. Your training sets should
include 300
images of each handwritten digits. Your testing sets should include
100 images of
each digit.
2. Generate an input data matrix (784 by 3,000) and an output
matrix (10 by 3,000)
for training.
3. Generate an input data matrix (784 by 1,000) and an output
matrix (10 by 1,000)
for testing.
Write a Matlab code to solve the problem. Tell me how we solved the problem.
/////////////////////////////////////////////
What information you need..........................
The basic format is
magic number
size in dimension 0
size in dimension 1
size in dimension 2
.....
size in dimension N
data
The magic number is an integer (MSB first). The first 2 bytes are always 0.
The third byte codes the type of the data:
0x08: unsigned byte
0x09: signed byte
0x0B: short (2 bytes)
0x0C: int (4 bytes)
0x0D: float (4 bytes)
0x0E: double (8 bytes)
The 4-th byte codes the number of dimensions of the vector/matrix: 1 for vectors, 2 for matrices....
The sizes in each dimension are 4-byte integers (MSB first, high endian, like in most non-Intel processors).
Code:
%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
image_size = 28 # width and length
no_of_different_labels = 10 # i.e. 0, 1, 2, 3, ..., 9
image_pixels = image_size * image_size
data_path = "data/mnist/"
train_data = np.loadtxt(data_path + "mnist_train.csv",
delimiter=",")
test_data = np.loadtxt(data_path + "mnist_test.csv",
delimiter=",")
test_data[:10]
Output::
array([[7., 0., 0., ..., 0., 0., 0.],
[2., 0., 0., ..., 0., 0., 0.],
[1., 0., 0., ..., 0., 0., 0.],
...,
[9., 0., 0., ..., 0., 0., 0.],
[5., 0., 0., ..., 0., 0., 0.],
[9., 0., 0., ..., 0., 0., 0.]])
test_data[test_data==255] test_data.shape
Output::
(10000, 785)
import numpy as np
lr = np.arange(10)
for label in range(10):
one_hot = (lr==label).astype(np.int)
print("label: ", label, " in one-hot representation: ", one_hot)
label: 0 in one-hot representation: [1 0 0 0 0 0 0 0 0 0] label: 1 in one-hot representation: [0 1 0 0 0 0 0 0 0 0] label: 2 in one-hot representation: [0 0 1 0 0 0 0 0 0 0] label: 3 in one-hot representation: [0 0 0 1 0 0 0 0 0 0] label: 4 in one-hot representation: [0 0 0 0 1 0 0 0 0 0] label: 5 in one-hot representation: [0 0 0 0 0 1 0 0 0 0] label: 6 in one-hot representation: [0 0 0 0 0 0 1 0 0 0] label: 7 in one-hot representation: [0 0 0 0 0 0 0 1 0 0] label: 8 in one-hot representation: [0 0 0 0 0 0 0 0 1 0] label: 9 in one-hot representation: [0 0 0 0 0 0 0 0 0 1]
The data is stored like in a C array, i.e. the index in the last dimension changes the fastest.
lr = np.arange(no_of_different_labels) # transform labels into one hot representation train_labels_one_hot = (lr==train_labels).astype(np.float) test_labels_one_hot = (lr==test_labels).astype(np.float) # we don't want zeroes and ones in the labels neither: train_labels_one_hot[train_labels_one_hot==0] = 0.01 train_labels_one_hot[train_labels_one_hot==1] = 0.99 test_labels_one_hot[test_labels_one_hot==0] = 0.01 test_labels_one_hot[test_labels_one_hot==1] = 0.99
mport numpy as np
@np.vectorize
def sigmoid(x):
return 1 / (1 + np.e ** -x)
activation_function = sigmoid
from scipy.stats import truncnorm
def truncated_normal(mean=0, sd=1, low=0, upp=10):
return truncnorm((low - mean) / sd,
(upp - mean) / sd,
loc=mean,
scale=sd)
class NeuralNetwork:
def __init__(self,
no_of_in_nodes,
no_of_out_nodes,
no_of_hidden_nodes,
learning_rate):
self.no_of_in_nodes = no_of_in_nodes
self.no_of_out_nodes = no_of_out_nodes
self.no_of_hidden_nodes = no_of_hidden_nodes
self.learning_rate = learning_rate
self.create_weight_matrices()
def create_weight_matrices(self):
"""
A method to initialize the weight
matrices of the neural network
"""
rad = 1 / np.sqrt(self.no_of_in_nodes)
X = truncated_normal(mean=0,
sd=1,
low=-rad,
upp=rad)
self.wih = X.rvs((self.no_of_hidden_nodes,
self.no_of_in_nodes))
rad = 1 / np.sqrt(self.no_of_hidden_nodes)
X = truncated_normal(mean=0, sd=1, low=-rad, upp=rad)
self.who = X.rvs((self.no_of_out_nodes,
self.no_of_hidden_nodes))
def train(self, input_vector, target_vector):
"""
input_vector and target_vector can
be tuple, list or ndarray
"""
input_vector = np.array(input_vector, ndmin=2).T
target_vector = np.array(target_vector, ndmin=2).T
output_vector1 = np.dot(self.wih,
input_vector)
output_hidden = activation_function(output_vector1)
output_vector2 = np.dot(self.who,
output_hidden)
output_network = activation_function(output_vector2)
output_errors = target_vector - output_network
# update the weights:
tmp = output_errors * output_network \
* (1.0 - output_network)
tmp = self.learning_rate * np.dot(tmp,
output_hidden.T)
self.who += tmp
# calculate hidden errors:
hidden_errors = np.dot(self.who.T,
output_errors)
# update the weights:
tmp = hidden_errors * output_hidden * \
(1.0 - output_hidden)
self.wih += self.learning_rate \
* np.dot(tmp, input_vector.T)
def run(self, input_vector):
# input_vector can be tuple, list or ndarray
input_vector = np.array(input_vector, ndmin=2).T
output_vector = np.dot(self.wih,
input_vector)
output_vector = activation_function(output_vector)
output_vector = np.dot(self.who,
output_vector)
output_vector = activation_function(output_vector)
return output_vector
def confusion_matrix(self, data_array, labels):
cm = np.zeros((10, 10), int)
for i in range(len(data_array)):
res = self.run(data_array[i])
res_max = res.argmax()
target = labels[i][0]
cm[res_max, int(target)] += 1
return cm
def precision(self, label, confusion_matrix):
col = confusion_matrix[:, label]
return confusion_matrix[label, label] / col.sum()
def recall(self, label, confusion_matrix):
row = confusion_matrix[label, :]
return confusion_matrix[label, label] / row.sum()
def evaluate(self, data, labels):
corrects, wrongs = 0, 0
for i in range(len(data)):
res = self.run(data[i])
res_max = res.argmax()
if res_max == labels[i]:
corrects += 1
else:
wrongs += 1
return corrects, wrongs
ANN = NeuralNetwork(no_of_in_nodes = image_pixels,
no_of_out_nodes = 10,
no_of_hidden_nodes = 100,
learning_rate = 0.1)
for i in range(len(train_imgs)):
ANN.train(train_imgs[i], train_labels_one_hot[i])
for i in range(20):
res = ANN.run(test_imgs[i])
print(test_labels[i], np.argmax(res), np.max(res))
[7.] 7 0.9829245583409039 [2.] 2 0.7372766887508578 [1.] 1 0.9881823673106839 [0.] 0 0.9873289971465894 [4.] 4 0.9456335245615916 [1.] 1 0.9880120617106172 [4.] 4 0.976550583573903 [9.] 9 0.964909168118122 [5.] 6 0.36615932726182665 [9.] 9 0.9848677489827125 [0.] 0 0.9204097234781773 [6.] 6 0.8897871402453337 [9.] 9 0.9936811621891628 [0.] 0 0.9832119513084644 [1.] 1 0.988750833073612 [5.] 5 0.9156741221523511 [9.] 9 0.9812577974620423 [7.] 7 0.9888560485875889 [3.] 3 0.8772868556722897 [4.] 4 0.9900030761222965
corrects, wrongs = ANN.evaluate(train_imgs, train_labels)
print("accuracy train: ", corrects / ( corrects + wrongs))
corrects, wrongs = ANN.evaluate(test_imgs, test_labels)
print("accuracy: test", corrects / ( corrects + wrongs))
cm = ANN.confusion_matrix(train_imgs, train_labels)
print(cm)
for i in range(10):
print("digit: ", i, "precision: ", ANN.precision(i, cm), "recall: ", ANN.recall(i, cm))
accuracy train: 0.9469166666666666 accuracy: test 0.9459 [[5802 0 53 21 9 42 35 8 14 20] [ 1 6620 45 22 6 29 14 50 75 7] [ 5 22 5486 51 10 11 5 53 11 3] [ 6 36 114 5788 2 114 1 35 76 72] [ 8 16 54 8 5439 41 10 52 25 90] [ 5 2 3 44 0 4922 20 3 5 11] [ 37 4 54 19 71 72 5789 3 41 4] [ 0 5 31 38 7 4 0 5762 1 32] [ 52 20 103 83 9 102 43 21 5535 38] [ 7 17 15 57 289 84 1 278 68 5672]] digit: 0 precision: 0.9795711632618606 recall: 0.9663557628247835 digit: 1 precision: 0.9819044793829724 recall: 0.9637501819769981 digit: 2 precision: 0.9207787848271232 recall: 0.9697719639384833 digit: 3 precision: 0.9440548034578372 recall: 0.9269698910954516 digit: 4 precision: 0.9310167750770284 recall: 0.9470659933832491 digit: 5 precision: 0.9079505626268216 recall: 0.9814556331006979 digit: 6 precision: 0.978202095302467 recall: 0.9499507712504103 digit: 7 precision: 0.9197126895450918 recall: 0.9799319727891157 digit: 8 precision: 0.945992138096052 recall: 0.9215784215784216 digit: 9 precision: 0.953437552529837 recall: 0.87422934648582
Multiple Runs
We can repeat the training multiple times. Each run is called an "epoch".
epochs = 3
NN = NeuralNetwork(no_of_in_nodes = image_pixels,
no_of_out_nodes = 10,
no_of_hidden_nodes = 100,
learning_rate = 0.1)
for epoch in range(epochs):
print("epoch: ", epoch)
for i in range(len(train_imgs)):
NN.train(train_imgs[i],
train_labels_one_hot[i])
corrects, wrongs = NN.evaluate(train_imgs, train_labels)
print("accuracy train: ", corrects / ( corrects + wrongs))
corrects, wrongs = NN.evaluate(test_imgs, test_labels)
print("accuracy: test", corrects / ( corrects + wrongs))
epoch: 0 accruracy train: 0.94515 accruracy: test 0.9459 epoch: 1 accruracy train: 0.9626833333333333 accruracy: test 0.9582 epoch: 2 accruracy train: 0.96995 accruracy: test 0.9626
We want to do the multiple training of the training set inside
of our network. To this purpose we rewrite the method train and add
a method train_single. train_single is more or less what we called
'train' before. Whereas the new 'train' method is doing the epoch
counting. For testing purposes, we save the weight matrices after
each epoch in
the list intermediate_weights. This list is returned as the output
of train:
import numpy as np
@np.vectorize
def sigmoid(x):
return 1 / (1 + np.e ** -x)
activation_function = sigmoid
from scipy.stats import truncnorm
def truncated_normal(mean=0, sd=1, low=0, upp=10):
return truncnorm((low - mean) / sd,
(upp - mean) / sd,
loc=mean,
scale=sd)
class NeuralNetwork:
def __init__(self,
no_of_in_nodes,
no_of_out_nodes,
no_of_hidden_nodes,
learning_rate):
self.no_of_in_nodes = no_of_in_nodes
self.no_of_out_nodes = no_of_out_nodes
self.no_of_hidden_nodes = no_of_hidden_nodes
self.learning_rate = learning_rate
self.create_weight_matrices()
def create_weight_matrices(self):
""" A method to initialize the weight matrices of the neural network"""
rad = 1 / np.sqrt(self.no_of_in_nodes)
X = truncated_normal(mean=0,
sd=1,
low=-rad,
upp=rad)
self.wih = X.rvs((self.no_of_hidden_nodes,
self.no_of_in_nodes))
rad = 1 / np.sqrt(self.no_of_hidden_nodes)
X = truncated_normal(mean=0,
sd=1,
low=-rad,
upp=rad)
self.who = X.rvs((self.no_of_out_nodes,
self.no_of_hidden_nodes))
def train_single(self, input_vector, target_vector):
"""
input_vector and target_vector can be tuple,
list or ndarray
"""
output_vectors = []
input_vector = np.array(input_vector, ndmin=2).T
target_vector = np.array(target_vector, ndmin=2).T
output_vector1 = np.dot(self.wih,
input_vector)
output_hidden = activation_function(output_vector1)
output_vector2 = np.dot(self.who,
output_hidden)
output_network = activation_function(output_vector2)
output_errors = target_vector - output_network
# update the weights:
tmp = output_errors * output_network * \
(1.0 - output_network)
tmp = self.learning_rate * np.dot(tmp,
output_hidden.T)
self.who += tmp
# calculate hidden errors:
hidden_errors = np.dot(self.who.T,
output_errors)
# update the weights:
tmp = hidden_errors * output_hidden * (1.0 - output_hidden)
self.wih += self.learning_rate * np.dot(tmp, input_vector.T)
def train(self, data_array,
labels_one_hot_array,
epochs=1,
intermediate_results=False):
intermediate_weights = []
for epoch in range(epochs):
print("*", end="")
for i in range(len(data_array)):
self.train_single(data_array[i],
labels_one_hot_array[i])
if intermediate_results:
intermediate_weights.append((self.wih.copy(),
self.who.copy()))
return intermediate_weights
def confusion_matrix(self, data_array, labels):
cm = {}
for i in range(len(data_array)):
res = self.run(data_array[i])
res_max = res.argmax()
target = labels[i][0]
if (target, res_max) in cm:
cm[(target, res_max)] += 1
else:
cm[(target, res_max)] = 1
return cm
def run(self, input_vector):
""" input_vector can be tuple, list or ndarray """
input_vector = np.array(input_vector, ndmin=2).T
output_vector = np.dot(self.wih,
input_vector)
output_vector = activation_function(output_vector)
output_vector = np.dot(self.who,
output_vector)
output_vector = activation_function(output_vector)
return output_vector
def evaluate(self, data, labels):
corrects, wrongs = 0, 0
for i in range(len(data)):
res = self.run(data[i])
res_max = res.argmax()
if res_max == labels[i]:
corrects += 1
else:
wrongs += 1
return corrects, wrongs
epochs = 10
ANN = NeuralNetwork(no_of_in_nodes = image_pixels,
no_of_out_nodes = 10,
no_of_hidden_nodes = 100,
learning_rate = 0.15)
weights = ANN.train(train_imgs,
train_labels_one_hot,
epochs=epochs,
intermediate_results=True)
**********
cm = ANN.confusion_matrix(train_imgs, train_labels)
print(ANN.run(train_imgs[i]))
[[2.60149245e-03] [2.52542556e-03] [6.57990628e-03] [1.32663729e-03] [1.34985384e-03] [2.63840265e-04] [2.18329159e-04] [1.32693720e-04] [9.84326084e-01] [4.34559417e-02]]
Get Answers For Free
Most questions answered within 1 hours.