{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Digit Recognizer - MNIST\n##### https://www.kaggle.com/competitions/digit-recognizer","metadata":{}},{"cell_type":"markdown","source":"### Imports\n","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom matplotlib import pyplot as plt","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-03-25T19:35:12.489738Z","iopub.execute_input":"2023-03-25T19:35:12.490173Z","iopub.status.idle":"2023-03-25T19:35:12.496057Z","shell.execute_reply.started":"2023-03-25T19:35:12.490140Z","shell.execute_reply":"2023-03-25T19:35:12.494427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load Data","metadata":{}},{"cell_type":"code","source":"# In each row, first column contains a label (digit). The remainder 784 columns (28px x 28px image) contain pixel values of 0-255.\nmnist_data = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\nmnist_data.head()","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-03-25T19:35:12.498182Z","iopub.execute_input":"2023-03-25T19:35:12.498584Z","iopub.status.idle":"2023-03-25T19:35:16.364275Z","shell.execute_reply.started":"2023-03-25T19:35:12.498546Z","shell.execute_reply":"2023-03-25T19:35:16.363007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Transform Data","metadata":{}},{"cell_type":"code","source":"# Shuffle data by rows\nmnist_data = mnist_data.sample(frac=1, axis=0)\nmnist_data","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-03-25T19:35:16.366217Z","iopub.execute_input":"2023-03-25T19:35:16.367399Z","iopub.status.idle":"2023-03-25T19:35:16.622901Z","shell.execute_reply.started":"2023-03-25T19:35:16.367345Z","shell.execute_reply":"2023-03-25T19:35:16.621289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Transpose the data so that the first row will be equal to all the labels and the remainder of rows in each column will be the corresponding pixel value\ntransposed_mnist_data = mnist_data.T\ntransposed_mnist_data.head()","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-03-25T19:35:16.625525Z","iopub.execute_input":"2023-03-25T19:35:16.625932Z","iopub.status.idle":"2023-03-25T19:35:16.652733Z","shell.execute_reply.started":"2023-03-25T19:35:16.625892Z","shell.execute_reply":"2023-03-25T19:35:16.651368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create NumPy array from Pandas dataframe\nmnist_data_arr = transposed_mnist_data.values","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-03-25T19:35:16.654484Z","iopub.execute_input":"2023-03-25T19:35:16.654850Z","iopub.status.idle":"2023-03-25T19:35:16.659017Z","shell.execute_reply.started":"2023-03-25T19:35:16.654814Z","shell.execute_reply":"2023-03-25T19:35:16.658012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mnist_data_arr","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-03-25T19:35:16.660358Z","iopub.execute_input":"2023-03-25T19:35:16.660951Z","iopub.status.idle":"2023-03-25T19:35:16.678058Z","shell.execute_reply.started":"2023-03-25T19:35:16.660914Z","shell.execute_reply":"2023-03-25T19:35:16.676288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Divide data into test and train sets","metadata":{}},{"cell_type":"code","source":"# Y will contain 41000 labels that will be used for training the network.\n# Label = a number from 0 to 9.\nY = mnist_data_arr[0, 1000:]\nY","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-03-25T19:35:16.679913Z","iopub.execute_input":"2023-03-25T19:35:16.680433Z","iopub.status.idle":"2023-03-25T19:35:16.694500Z","shell.execute_reply.started":"2023-03-25T19:35:16.680394Z","shell.execute_reply":"2023-03-25T19:35:16.693008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Y_test contains 1000 labels that will be used to evaluate the network after each training epoch\nY_test = mnist_data_arr[0, :1000]\nY_test","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-03-25T19:35:16.695940Z","iopub.execute_input":"2023-03-25T19:35:16.696425Z","iopub.status.idle":"2023-03-25T19:35:16.711237Z","shell.execute_reply.started":"2023-03-25T19:35:16.696379Z","shell.execute_reply":"2023-03-25T19:35:16.709949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X will contain 41000 columns, each representing one training examples, each column/example contains 784 pixel values.\n# Because the pixelvalues are between 0 and 255, and we want them to be between 0 and 1, we will divide by 255.0\n# Each column corresponds to one label from Y\nX = (mnist_data_arr[1:, 1000:] / 255.0)\nX","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-03-25T19:35:16.712733Z","iopub.execute_input":"2023-03-25T19:35:16.713323Z","iopub.status.idle":"2023-03-25T19:35:16.879394Z","shell.execute_reply.started":"2023-03-25T19:35:16.713275Z","shell.execute_reply":"2023-03-25T19:35:16.878233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X_test contains examples just like X but in X_test's case there is only 1000 examples that correspond to the labels from Y_test\nX_test = (mnist_data_arr[1:, :1000] / 255.0)\nX_test","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-03-25T19:35:16.884402Z","iopub.execute_input":"2023-03-25T19:35:16.885751Z","iopub.status.idle":"2023-03-25T19:35:16.895559Z","shell.execute_reply.started":"2023-03-25T19:35:16.885701Z","shell.execute_reply":"2023-03-25T19:35:16.893899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Transform a label so that 3 is represented by a column vector of [0, 0, 0, 1.0, 0, 0, 0, 0, 0, 0] instead of just 3\n# This is needed because the output layer of our network contains 10 neurons, each representing a number from 0 to 9\ndef one_hot(y):\n    desired_output = np.zeros((10, 1))\n    desired_output[y] = 1.0\n    return desired_output","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-03-25T19:35:16.897139Z","iopub.execute_input":"2023-03-25T19:35:16.897628Z","iopub.status.idle":"2023-03-25T19:35:16.905423Z","shell.execute_reply.started":"2023-03-25T19:35:16.897576Z","shell.execute_reply":"2023-03-25T19:35:16.903924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Transform all our labels from Y using one_hot function defined above\n# Each column is one example\none_hot_Y = np.array([one_hot(y) for y in Y]).T.reshape((10, 41000))\none_hot_Y","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-03-25T19:35:16.907014Z","iopub.execute_input":"2023-03-25T19:35:16.907939Z","iopub.status.idle":"2023-03-25T19:35:16.993351Z","shell.execute_reply.started":"2023-03-25T19:35:16.907896Z","shell.execute_reply":"2023-03-25T19:35:16.992364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Setup of weights and biases\n##### This will depend on the # of layers and the # of neurons in each of them.\n##### We will have 1 hidden layer with 30 neurons and 1 output layer with 10 neurons\n##### Our input layer will be the MNIST date which is 784 rows (==neurons)","metadata":{}},{"cell_type":"code","source":"W1 = np.random.rand(30, 784) - 0.5 # Weights matrix between input layer and first layer\nW2 = np.random.rand(10, 30)  - 0.5 # Weights matrix between first layer and output layer\nb1 = np.random.rand(30, 1)   - 0.5 # Biases vector in first layer\nb2 = np.random.rand(10, 1)   - 0.5 # biases vector in output layer","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-03-25T19:35:16.994676Z","iopub.execute_input":"2023-03-25T19:35:16.995271Z","iopub.status.idle":"2023-03-25T19:35:17.001416Z","shell.execute_reply.started":"2023-03-25T19:35:16.995235Z","shell.execute_reply":"2023-03-25T19:35:17.000427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Helper functions\n##### Below functions are need to calculate the cost, activations, and errors","metadata":{}},{"cell_type":"code","source":"def sigmoid(Z):\n    # Sigmoid function, applied elementwise if z is a matrix\n    return 1.0 / (1.0 + np.exp(-Z))\n\ndef sigmoid_derivative(Z):\n    # sigmoid(z) * (1 - sigmoid(z))\n    # sigmoig(z) = activation so we can do it like below\n    return sigmoid(Z) * (1.0 - sigmoid(Z))\n","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-03-25T19:35:17.002851Z","iopub.execute_input":"2023-03-25T19:35:17.003409Z","iopub.status.idle":"2023-03-25T19:35:17.016252Z","shell.execute_reply.started":"2023-03-25T19:35:17.003373Z","shell.execute_reply":"2023-03-25T19:35:17.014810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cost(A, Y):\n    return np.power((A - Y) / 2, 2)\n\ndef cost_derivative(A, Y):\n    return A - Y","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-03-25T19:35:17.018715Z","iopub.execute_input":"2023-03-25T19:35:17.019334Z","iopub.status.idle":"2023-03-25T19:35:17.028936Z","shell.execute_reply.started":"2023-03-25T19:35:17.019271Z","shell.execute_reply":"2023-03-25T19:35:17.027535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Below function will output % of predictions that match the expected output","metadata":{}},{"cell_type":"code","source":"def calculate_accuracy(predictions, expected):\n    max_indices = np.argmax(predictions, axis=0) # Transform each prediction (10 neurons from output layer) into one number between 0 and 9\n    print(np.sum(max_indices == expected) / len(expected) * 100, \"%\") # Calculate % of predictions that are correct","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-03-25T19:35:17.033560Z","iopub.execute_input":"2023-03-25T19:35:17.034124Z","iopub.status.idle":"2023-03-25T19:35:17.043567Z","shell.execute_reply.started":"2023-03-25T19:35:17.034077Z","shell.execute_reply":"2023-03-25T19:35:17.042306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Stochastic Gradient Descent\n#### Instead of going through all examples each time we take a random sampling of sample_size\n#### We will train our model n number of times, where n = epochs","metadata":{}},{"cell_type":"code","source":"# Setup hyperparameters\nepochs = 1000\nsample_size = 100\nlearning_rate = 0.5\n\nfor epoch in range(epochs):\n    # Create a random number between 1000 + sample_size and 41000, this is because the first 1000 examples from MNIST data are used for checking network's accuracy\n    random_int = np.random.randint(1000 + sample_size, 41001)\n\n    # Take a random sample of 100 examples from X\n    X_sample = X[:, random_int - sample_size : random_int]\n    # Take corresponding labels in one_hot form\n    one_hot_Y_sample = one_hot_Y[:, random_int - sample_size : random_int]\n\n    # Feedforward\n    #   Calculate activations in the first and second layer.\n    Z1 = np.dot(W1, X_sample) + b1\n    A1 = sigmoid(Z1)\n    Z2 = np.dot(W2, A1) + b2\n    A2 = sigmoid(Z2) # this will be our output\n\n    # Backpropagation\n    #   Calculate error in the output layer and backpropagate it to the first layer\n    error_A2 = cost_derivative(A2, one_hot_Y_sample) * sigmoid_derivative(Z2) # Fundamental equation of backpropagation #1\n    error_A1 = np.dot(W2.T, error_A2) # Fundamental equation of backpropagation #2\n\n    #   Gradient descent\n    #       Calculate how much does the cost function depend on each weight and bias\n    delta_W2 = np.dot(error_A2, A1.T)        / sample_size # Fundamental equation of backpropagation #4\n    delta_W1 = np.dot(error_A1, X_sample.T)  / sample_size # Fundamental equation of backpropagation #4\n    delta_b2 = np.sum(error_A2)              / sample_size # Fundamental equation of backpropagation #3\n    delta_b1 = np.sum(error_A1)              / sample_size # Fundamental equation of backpropagation #3\n    #       Use above calculations to adjust weights and biases accordingly\n    W1 = W1 - delta_W1 * learning_rate\n    W2 = W2 - delta_W2 * learning_rate\n    b1 = b1 - delta_b1 * learning_rate\n    b2 = b2 - delta_b2 * learning_rate\n\n    # Feedforward => checking accuracy using X_test data\n    # A2 will be the output for X_test\n    Z1 = np.dot(W1, X_test) + b1\n    A1 = sigmoid(Z1)\n    Z2 = np.dot(W2, A1) + b2\n    A2 = sigmoid(Z2)\n\n    # Every 100 iterations we will print the current accuracy of the network\n    if epoch % 100 == 0:\n        calculate_accuracy(A2, Y_test)","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-03-25T19:35:17.045070Z","iopub.execute_input":"2023-03-25T19:35:17.045471Z","iopub.status.idle":"2023-03-25T19:35:22.985886Z","shell.execute_reply.started":"2023-03-25T19:35:17.045432Z","shell.execute_reply":"2023-03-25T19:35:22.984058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Verifying\n### After all of the above cells are run our network will have weights and biases set\n### We can then use the below function to test those weights and biases to see how well the network performs","metadata":{}},{"cell_type":"code","source":"def test_weights_and_biases(idx):\n    current_image = X[:, idx, None]\n\n    Z1 = np.dot(W1, current_image) + b1\n    A1 = sigmoid(Z1)\n    Z2 = np.dot(W2, A1) + b2\n    A2 = sigmoid(Z2)\n\n    label = Y[idx]\n\n    print(\"Prediction: \", np.argmax(A2))\n    print(\"Label:      \", label)\n\n    current_image = current_image.reshape((28, 28)) * 255\n    plt.gray()\n    plt.imshow(current_image, interpolation='nearest')\n    plt.show()","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-03-25T19:35:22.987758Z","iopub.execute_input":"2023-03-25T19:35:22.989061Z","iopub.status.idle":"2023-03-25T19:35:23.002668Z","shell.execute_reply.started":"2023-03-25T19:35:22.988964Z","shell.execute_reply":"2023-03-25T19:35:23.001047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We feed an index between 0 and 41000\n# The function outputs the following:\n# Prediction = what the network thinks the number is\n# Label      = the actual number\n# A 28x28 pixel photo of the number\ntest_weights_and_biases(40121)","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-03-25T19:35:23.004780Z","iopub.execute_input":"2023-03-25T19:35:23.005631Z","iopub.status.idle":"2023-03-25T19:35:23.242356Z","shell.execute_reply.started":"2023-03-25T19:35:23.005568Z","shell.execute_reply":"2023-03-25T19:35:23.241050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}}]}