{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30646,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"First neural network for digit recognition.\nSpecified below is the project algorithm including model algorithm then model's features\n\n1. Load and prepare the dataset\n\n2. Define the neural network architecture\n\n- input layer (784 features from (28x28 pixels))\n- 1st hidden layer (10 nodes) + relu activation\n- 2nd hidden layer (10 nodes) + relu activation\n- output layer (linear)\n\n3. Initialize model parameters (weights and biases)\n\n\n4. Implement forward propagation\n\n\n5. Define the loss funciton\n\n\n6. Implement back propagation\n\n\n7. Train the neural network\n\n\n8. Evaluate the model\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot as plt\nfrom sklearn.model_selection import train_test_split\n\n","metadata":{"execution":{"iopub.status.busy":"2024-02-16T16:51:44.540390Z","iopub.execute_input":"2024-02-16T16:51:44.540977Z","iopub.status.idle":"2024-02-16T16:51:44.552408Z","shell.execute_reply.started":"2024-02-16T16:51:44.540932Z","shell.execute_reply":"2024-02-16T16:51:44.550356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntest_data = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\n\n# Shuffling data\ntrain_data = np.array(train_data)\ntest_data = np.array(test_data)\n\nnp.random.shuffle(train_data)\n\n# Split training data into features and targets\ntrain_features = train_data[:, 1:]  # All rows, all columns except the first\ntrain_targets = train_data[:, 0]  # All rows, only the first column\n\n# Split test data into features\ntest_features = test_data[:, :]  # All rows, all columns\n\n# splitting dataset\ntrain_features = train_features.reshape(42000, 784)\ntest_features = test_features.reshape(28000, 784)\n\n# normalize inputs from 0-255 to 0-1\ntrain_features = train_features / 255.0 \ntest_features = test_features / 255.0\n\n# Split the training data into training and validation sets\nx_train, X_val, y_train, y_val = train_test_split(train_features, train_targets, test_size=0.2, random_state=42)\n\nx_test = test_features  # Features from test.csv\n\n############################################################################################################################\n# train_data = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\n# test_data = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\n\n# # Shuffling data\n# train_data = np.array(train_data)\n# test_data = np.array(test_data)\n\n# #m, n = data.shape\n# np.random.shuffle(train_data)\n# np.random.shuffle(test_data)\n\n# # Split training data into features and targets\n# train_features = train_data[:, 1:]  # All rows, all columns except the first\n# train_targets = train_data[:, 0]  # All rows, only the first column\n\n# # Split test data into features and targets\n# test_features = test_data[:, :]  # All rows, all columns except the first\n# test_targets = test_data[:, 0]  # All rows, only the first column\n\n# # Split the training data into training and validation sets\n# x_train, X_val, y_train, y_val = train_test_split(train_features, train_targets, test_size=0.2, random_state=42)\n\n# x_test = test_features  # Features from test.csv\n#################################################################################################################\n \n# # Split into features and targets\n# features = train_data.drop('label', axis=1).values\n# targets = data['label'].values\n\n# # Define the split size\n# split_size = int(0.8 * len(data))  # 80% for training\n\n# Split the data\n# train_features, test_features = features[:split_size], features[split_size:]\n# train_targets, test_targets = targets[:split_size], targets[split_size:]","metadata":{"execution":{"iopub.status.busy":"2024-02-16T16:51:44.555494Z","iopub.execute_input":"2024-02-16T16:51:44.556430Z","iopub.status.idle":"2024-02-16T16:51:51.147049Z","shell.execute_reply.started":"2024-02-16T16:51:44.556379Z","shell.execute_reply":"2024-02-16T16:51:51.145606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#that is compitable with pandas and i am manipulating the dataset using numpy LOL\n# train_data.head()\n# test_data.head()\n\nprint(train_features.shape)\nprint(test_features.shape)\n\n# print(\"Features shape:\", features.shape)\n# print(\"Targets shape:\", targets.shape)\n\n\n","metadata":{"execution":{"iopub.status.busy":"2024-02-16T16:51:51.149738Z","iopub.execute_input":"2024-02-16T16:51:51.150124Z","iopub.status.idle":"2024-02-16T16:51:51.156751Z","shell.execute_reply.started":"2024-02-16T16:51:51.150093Z","shell.execute_reply":"2024-02-16T16:51:51.155387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n\n\n\n\n\n# # splitting dataset\n# train_features = train_features.reshape(42000, 784)\n# print(train_features.shape)\n# test_features = test_features.reshape(28000, 783)\n# print(test_features.shape)\n\n# # normalize inputs from 0-255 to 0-1\n# train_features = train_features / 255.0 \n# test_features = test_features / 255.0\n\n# print(train_targets.shape)\n# print(test_targets.shape)\n\n# x_train = train_features\n# y_train = train_targets\n\n# x_test = test_features\n# y_test = test_targets\n\n# X_val = test_features\n#y_val = test_targets\n\n####################################################################################33\n# data_train = data[0:40000]                   # not transpose applied\n# y_train = data_train[0]\n# x_train = data_train[1:n]\n\n# data_dev = data[40000:]                      # not transpose applied\n# y_dev = data_dev[0]\n# x_dev = data_dev[1:n]\n\n# print(x_train[0], x_dev[0])\n# print(y_train[0], y_dev[0])\n\n\n# data_dev = data[0:1000].T\n# y_dev = data_dev[0]\n# x_dev = data_dev[1:n]\n\n# data_train = data[1000:m].T\n# y_train = data_train[0]\n# x_train = data_train[1:n]\n\n","metadata":{"execution":{"iopub.status.busy":"2024-02-16T16:51:51.158325Z","iopub.execute_input":"2024-02-16T16:51:51.158725Z","iopub.status.idle":"2024-02-16T16:51:51.168450Z","shell.execute_reply.started":"2024-02-16T16:51:51.158688Z","shell.execute_reply":"2024-02-16T16:51:51.167327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualizing the data\n\n# Numpy\n\n# print(x_dev.shape, y_dev.shape, x_train.shape, y_train.shape)\n\n# #Matplotlib\n# image = x_train[:, 0]\n\n# image = image.reshape((28, 28))\n\n# plt.imshow(image, cmap='gray')\n# plt.show()\n\n# visualizing the first 10 images in the dataset and their labels\n%matplotlib inline\nimport matplotlib.pyplot as plt\nplt.figure(figsize=(10, 1))\nfor i in range(10):\n    plt.subplot(1, 10, i+1)\n    plt.imshow(x_train[i].reshape(28, 28), cmap=\"gray\")\n    plt.axis('off')\nplt.show()\nprint('label for each of the above image: %s' % (y_train[0:10]))\n","metadata":{"execution":{"iopub.status.busy":"2024-02-16T16:51:51.171187Z","iopub.execute_input":"2024-02-16T16:51:51.172385Z","iopub.status.idle":"2024-02-16T16:51:51.510807Z","shell.execute_reply.started":"2024-02-16T16:51:51.172319Z","shell.execute_reply":"2024-02-16T16:51:51.509550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# linear_layer class to create a Connected Linear layer of neurons \n\nclass Linear():\n    def __init__(self, in_size, out_size):\n        self.w = np.random.randn(in_size, out_size) * 0.01\n        self.b = np.zeros((1, out_size))\n        self.params = [self.w, self.b]\n        self.grad_w = None\n        self.grad_b = None\n        self.grad_input = None\n    \n    def forward(self, x):\n        self.x = x\n        self.output = np.dot(x, self.w) + self.b\n        return self.output\n    \n    def backward(self, next_grad):\n        self.grad_w = np.dot(self.x.T, next_grad)\n        self.grad_b = np.sum(next_grad, axis=0)\n        self.grad_input = np.dot(next_grad, self.w.T)\n        return self.grad_input, [self.grad_w, self.grad_b]\n\nnp.random.seed(3)\n\n\n","metadata":{"execution":{"iopub.status.busy":"2024-02-16T16:51:51.512542Z","iopub.execute_input":"2024-02-16T16:51:51.512940Z","iopub.status.idle":"2024-02-16T16:51:51.524403Z","shell.execute_reply.started":"2024-02-16T16:51:51.512908Z","shell.execute_reply":"2024-02-16T16:51:51.522994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# creating functions -> relu, softmax, fwd prop\n\n# Relu activation\nclass Relu():\n    def __init__(self):\n        self.params = []\n        self.grad_input = None\n    \n    def forward(self, x):\n        self.output = np.maximum(x, 0)\n        return self.output\n        \n    def backward(self, next_grad):\n        self.grad_input = next_grad.copy()\n        self.grad_input[self.output <= 0] = 0\n        return self.grad_input, []\n    \ndef softmax(x):\n    exp_x = np.exp(x - np.max(x, axis=1, keepdims=True))\n    out = exp_x / np.sum(exp_x, axis=1, keepdims=True)\n    return out\n    \nclass CrossEntropy:\n    \n    def forward(self, x, y):\n        self.m = y.shape[0]\n        self.p = softmax(x)\n        cross_entropy = -np.log(self.p[range(self.m), y])\n        loss = cross_entropy[0] / self.m\n        return loss\n        \n    def backward(self, x, y):\n        y_idx = y.argmax()\n        grad = softmax(x)\n        grad[range(self.m), y] -= 1\n        grad /= self.m\n        return grad\n        \n\nclass NN():\n    def __init__(self, lossfunc=CrossEntropy()):\n        self.params = []\n        self.layers = []\n        self.loss_func = lossfunc\n        self.grads = []\n        \n    def add_layer(self, layer):\n        self.layers.append(layer)\n        self.params.append(layer.params)\n\n    def forward(self, x):\n        for layer in self.layers:\n            x = layer.forward(x)\n        return x\n    \n    def backward(self, next_grad):\n        self.clear_grad_param()\n        for layer in reversed(self.layers):\n            next_grad, grad = layer.backward(next_grad)\n            self.grads.append(grad)\n        return self.grads\n    \n    def train_step(self, x, y):\n        out = self.forward(x)\n        loss = self.loss_func.forward(out,y)\n        next_grad = self.loss_func.backward(out,y)\n        l2 = self.backward(next_grad)\n        return loss, l2\n    \n    def predict(self, x):\n        x = self.forward(x)\n        return np.argmax(x, axis=1)\n    \n    def predict_scores(self, x):\n        x = self.forward(x)\n        return x\n    \n    def clear_grad_param(self):\n        self.grads = []\n    \ndef update_params(velocity, params, grads, learning_rate=0.01, mu=0.9):\n    for v, p, g, in zip(velocity, params, reversed(grads)):\n        for i in range(len(g)):\n            v[i] = mu * v[i] + learning_rate * g[i]\n            p[i] -= v[i]\n            print('Max gradient value:',np.amax(v[i]))\n            print('Gradient shape:',v[i].shape)\n    \ndef minibatch(x, y, minibatch_size):\n    n = x.shape[0]\n    minibatches = []\n    permutation = np.random.permutation(x.shape[0])\n    x = x[permutation]\n    y = y[permutation]\n    \n    for i in range(0, n , minibatch_size):\n        x_batch = x[i:i + minibatch_size, :]\n        y_batch = y[i:i + minibatch_size, ]\n        minibatches.append((x_batch, y_batch))\n        \n    return minibatches    \n    \n# # relu\n# def relu(Z):\n#     return np.maximum(0, Z)\n\n# # softamx\n# def softmax(Z):\n#     exp_z_j = np.exp(Z)                           # element exponent\n#     sum_exp_z_n = exp_z_j / np.sum(exp_z_j)\n#     return sum_exp_z_n\n\n# # forward propagation\n# def fwd_prop(W1, B1, W2, B2, X):\n#     Z1 = np.dot(W1, X) + B1\n#     A1 = relu(Z1)\n#     Z2 = np.dot(W2, A1) + B2\n#     A2 = softmax(Z2)\n#     return Z1, A1, Z2, A2\n\n# # one hot encoding\n# def one_hot(Y):\n#     one_hot_Y = np.zeros((Y.size, Y.max() + 1))\n#     one_hot_Y[np.arange(Y.size), Y] = 1\n#     one_hot_Y = one_hot_Y.T\n#     return one_hot_Y\n\n# def derive_relu(Z):\n#     return Z > 0\n\n# # backward propagation\n# def back_prop(Z1, A1, Z2, A2, W2, X, Y):\n#     m = Y.size\n#     one_hot_Y = one_hot(Y)\n#     dZ2 = A2 - one_hot_Y\n#     dW2 = 1 / (m * np.dot(dZ2, A1.T))\n#     dB2 = 1 / (m * np.sum(dZ2, axis=0))\n#     dZ1 = np.dot(W2.T, dZ2) * derive_relu(Z1)\n#     dW1 = 1 / (m * np.dot(dZ1, X.T))\n#     dB1 = 1 / (m * np.sum(dZ1, axis=0))\n#     return dW1, dB1, dW2, dB2\n\n# # updating the parameters simultaneously\n# def update_params(W1, B1, W2, B2, dW1, dB1, dW2, dB2, alpha):\n#     W1 = W1 - alpha * dW1\n#     B1 = B1 - alpha * dB1\n#     W2 = W2 - alpha * dW2\n#     B2 = B2 - alpha * dB2\n#     return W1, B1, W2, B2\n\n\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2024-02-16T16:51:51.526856Z","iopub.execute_input":"2024-02-16T16:51:51.527443Z","iopub.status.idle":"2024-02-16T16:51:51.555461Z","shell.execute_reply.started":"2024-02-16T16:51:51.527406Z","shell.execute_reply":"2024-02-16T16:51:51.554221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training Loop\n\ndef train(net, x_train, y_train, minibatch_size, epoch, learning_rate, mu=0.9, X_val=None, y_val=None):\n    val_loss_epoch = []\n    minibatches = minibatch(x_train, y_train, minibatch_size)\n    minibatches_val = minibatch(X_val, y_val, minibatch_size)\n\n    \n    for i in range(epoch):\n        loss_batch = []\n        val_loss_batch = []\n        velocity = []\n        for param_layer in net.params:\n            p = [np.zeros_like(param) for param in list(param_layer)]\n            velocity.append(p)\n            \n        # iterate over mini batches\n        for x_mini, y_mini in minibatches:\n            loss, grads = net.train_step(x_mini, y_mini)\n            loss_batch.append(loss)\n            update_params(velocity, net.params, grads, learning_rate=learning_rate, mu=mu)\n\n        for X_mini_val, y_mini_val in minibatches_val:\n            val_loss, _ = net.train_step(x_mini, y_mini)\n            val_loss_batch.append(val_loss)\n            \n        # accuracy of model at end of epoch after all mini batch updates\n        m_train = x_train.shape[0]\n        m_val = X_val.shape[0]\n        y_train_pred = np.array([], dtype=\"int64\")\n        y_val_pred = np.array([], dtype=\"int64\")\n        y_train1 = []\n        y_vall = []\n        for i in range(0, m_train, minibatch_size):\n            x_tr = x_train[i:i + minibatch_size, : ]\n            y_tr = y_train[i:i + minibatch_size,]\n            y_train1 = np.append(y_train1, y_tr)\n            y_train_pred = np.append(y_train_pred, net.predict(x_tr))\n\n        for i in range(0, m_val, minibatch_size):\n            x_va = X_val[i:i + minibatch_size, : ]\n            y_va = y_val[i:i + minibatch_size,]\n            y_vall = np.append(y_vall, y_va)\n            y_val_pred = np.append(y_val_pred, net.predict(x_va))\n            \n        train_acc = check_accuracy(y_train1, y_train_pred)\n        val_acc = check_accuracy(y_vall, y_val_pred)\n\n        mean_train_loss = sum(loss_batch) / float(len(loss_batch))\n        mean_val_loss = sum(val_loss_batch) / float(len(val_loss_batch))\n        \n        val_loss_epoch.append(mean_val_loss)\n        print(\"Loss = {0} | Training Accuracy = {1} | Val Loss = {2} | Val Accuracy = {3}\".format(mean_train_loss, train_acc, mean_val_loss, val_acc))\n    return net\n\ndef check_accuracy(y_true, y_pred):\n    return np.mean(y_pred == y_true)\n\nfrom random import shuffle\n\n## input size\ninput_dim = x_train.shape[1]\n\n## hyperparameters\niterations = 10\nlearning_rate = 0.11\nhidden_nodes = 32\noutput_nodes = 10\n\n## define neural net\nnn = NN()\nnn.add_layer(Linear(input_dim, hidden_nodes))\nnn.add_layer(Relu())\nnn.add_layer(Linear(hidden_nodes, output_nodes))\n\nnn = train(nn, x_train , y_train, minibatch_size=200, epoch=10, \\\n           learning_rate=learning_rate, X_val=X_val, y_val=y_val)\n\n\n# # implementing gradient descent\n\n\n# def get_accuracy(predictions, Y):\n#     print(predictions, Y)\n#     return np.sum(predictions == Y)/Y.size\n\n# def get_prediciton(A2):\n#     return np.argmax(A2, axis=0)\n\n# def gradient_descent(X, Y, iterations, alpha):\n#     W1, B1, W2, B2 = initial_params()\n#     for i in range(iterations):\n#         Z1, A1, Z2, A2 = fwd_prop(W1, B1, W2, B2, X)\n#         dW1, dB1, dW2, dB2 = back_prop(Z1, A1, Z2, A2, W2, X, Y)\n#         W1, B1, W2, B2 = update_params(W1, B1, W2, B2, dW1, dB1, dW2, dB2, alpha)\n#         if (i % 10 == 0):\n#             print('Iteration: ', i)\n#             print('Accuracy: ', get_accuracy(get_prediciton(A2), Y))\n#     return W1, B1, W2, B2\n","metadata":{"execution":{"iopub.status.busy":"2024-02-16T16:57:26.237327Z","iopub.execute_input":"2024-02-16T16:57:26.237832Z","iopub.status.idle":"2024-02-16T16:57:35.706338Z","shell.execute_reply.started":"2024-02-16T16:57:26.237790Z","shell.execute_reply":"2024-02-16T16:57:35.704849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(X_val[190].reshape(28,28), cmap='gray')\n\n# Predict Scores for each class\nprediction = nn.predict_scores(X_val[190])[0]","metadata":{"execution":{"iopub.status.busy":"2024-02-16T16:52:00.196469Z","iopub.execute_input":"2024-02-16T16:52:00.203264Z","iopub.status.idle":"2024-02-16T16:52:00.513042Z","shell.execute_reply.started":"2024-02-16T16:52:00.203179Z","shell.execute_reply":"2024-02-16T16:52:00.511895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print (\"Scores\")\nprint (prediction)","metadata":{"execution":{"iopub.status.busy":"2024-02-16T16:52:00.514691Z","iopub.execute_input":"2024-02-16T16:52:00.515174Z","iopub.status.idle":"2024-02-16T16:52:00.522146Z","shell.execute_reply.started":"2024-02-16T16:52:00.515122Z","shell.execute_reply":"2024-02-16T16:52:00.521101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.argmax(prediction)\n\npredict_class = nn.predict(X_val[190])[0]\npredict_class\n\n# Original class\ny_val[190]","metadata":{"execution":{"iopub.status.busy":"2024-02-16T16:52:00.523476Z","iopub.execute_input":"2024-02-16T16:52:00.523846Z","iopub.status.idle":"2024-02-16T16:52:00.540790Z","shell.execute_reply.started":"2024-02-16T16:52:00.523809Z","shell.execute_reply":"2024-02-16T16:52:00.539376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ny_test_pred = nn.predict(x_test)\n# Create a pandas DataFrame for your submission\nsubmission = pd.DataFrame({\n    \"ImageId\": range(1, len(y_test_pred) + 1),  # ImageId starts from 1\n    \"Label\": y_test_pred\n})\n# Save the DataFrame to a CSV file\nsubmission.to_csv('submission.csv', index=False)\n\n# # Assuming 'model' is your trained model and 'test_images' is your test dataset\n# predictions = nn.predict(test_data)\n\n# # Select the index with the maximum probability\n# predictions = np.argmax(predictions, axis=1)\n\n# # Create a DataFrame for submission\n# submission_df = pd.DataFrame({'ImageId': range(1, 28001), 'Label': predictions})\n\n# # Save the DataFrame to a csv file\n# submission_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-02-16T17:03:25.637420Z","iopub.execute_input":"2024-02-16T17:03:25.637904Z","iopub.status.idle":"2024-02-16T17:03:25.814078Z","shell.execute_reply.started":"2024-02-16T17:03:25.637863Z","shell.execute_reply":"2024-02-16T17:03:25.812407Z"},"trusted":true},"execution_count":null,"outputs":[]}]}