{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot as plt\n\ntrainData = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntestData = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-24T20:49:56.075687Z","iopub.execute_input":"2024-10-24T20:49:56.076121Z","iopub.status.idle":"2024-10-24T20:50:03.147229Z","shell.execute_reply.started":"2024-10-24T20:49:56.076079Z","shell.execute_reply":"2024-10-24T20:50:03.145903Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"trainData.head()","metadata":{"execution":{"iopub.status.busy":"2024-10-24T20:50:03.149017Z","iopub.execute_input":"2024-10-24T20:50:03.149400Z","iopub.status.idle":"2024-10-24T20:50:03.182203Z","shell.execute_reply.started":"2024-10-24T20:50:03.149363Z","shell.execute_reply":"2024-10-24T20:50:03.180552Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"testData.head()","metadata":{"execution":{"iopub.status.busy":"2024-10-24T20:50:03.184199Z","iopub.execute_input":"2024-10-24T20:50:03.184710Z","iopub.status.idle":"2024-10-24T20:50:03.205268Z","shell.execute_reply.started":"2024-10-24T20:50:03.184657Z","shell.execute_reply":"2024-10-24T20:50:03.203971Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Here, we see that the test dataset is only useful for the competition organizers, but we cannot use it due to the absence of a \"label\" column. To address this, I will split the training dataset into training and testing portions.","metadata":{}},{"cell_type":"markdown","source":"Firstly some function to import the datasets. getTestDataset can be used for a competition submission.","metadata":{}},{"cell_type":"code","source":"def getDataset(filePath):\n    trainData = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\n    trainData = np.array(trainData)\n    rows, columns = trainData.shape\n    np.random.shuffle(trainData)\n\n    testData = trainData[0:2000].T\n    testLabels = testData[0]\n    testPixels = testData[1:columns + 1]\n    testPixels = testPixels / 255\n\n    trainData = trainData[2000:rows].T\n    trainLabels = trainData[0]\n    trainPixels = trainData[1:columns + 1]\n    trainPixels = trainPixels / 255\n\n    return trainLabels, trainPixels, testLabels, testPixels\n\ndef getTestDataset(filePath):\n    testData = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\n    testPixels = np.array(testData).T\n    testPixels = testPixels / 255\n\n    return testPixels","metadata":{"execution":{"iopub.status.busy":"2024-10-24T20:50:03.207831Z","iopub.execute_input":"2024-10-24T20:50:03.208216Z","iopub.status.idle":"2024-10-24T20:50:03.218271Z","shell.execute_reply.started":"2024-10-24T20:50:03.208178Z","shell.execute_reply":"2024-10-24T20:50:03.216831Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Now, a function to initialize the network's parameters in a random state","metadata":{}},{"cell_type":"code","source":"def initParams(layerSizes):\n    params = {}\n    numberOfLayers = len(layerSizes) - 1\n    for l in range(1, numberOfLayers + 1):\n        params[f\"W{l}\"] = np.random.rand(layerSizes[l], layerSizes[l - 1]) - 0.5\n        params[f\"b{l}\"] = np.random.rand(layerSizes[l], 1) - 0.5\n    return params","metadata":{"execution":{"iopub.status.busy":"2024-10-24T20:50:03.219608Z","iopub.execute_input":"2024-10-24T20:50:03.220134Z","iopub.status.idle":"2024-10-24T20:50:03.229989Z","shell.execute_reply.started":"2024-10-24T20:50:03.220078Z","shell.execute_reply":"2024-10-24T20:50:03.228834Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Now, some basic functions that will be used in backward and forward propagation","metadata":{}},{"cell_type":"code","source":"def ReLU(Z):\n    return np.maximum(Z, 0)\n\ndef softmax(Z):\n    A = np.exp(Z) / np.sum(np.exp(Z), axis=0)\n    return A\n\ndef ReluDeriv(Z):\n    return Z > 0\n\ndef desiredOutputs(Y):\n    desiredOut = np.zeros((Y.size, Y.max() + 1))\n    desiredOut[np.arange(Y.size), Y] = 1\n    desiredOut = desiredOut.T\n    return desiredOut","metadata":{"execution":{"iopub.status.busy":"2024-10-24T20:50:03.231402Z","iopub.execute_input":"2024-10-24T20:50:03.231812Z","iopub.status.idle":"2024-10-24T20:50:03.245423Z","shell.execute_reply.started":"2024-10-24T20:50:03.231775Z","shell.execute_reply":"2024-10-24T20:50:03.243846Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Now, the forward and backward propagation algorithms","metadata":{}},{"cell_type":"code","source":"def forwardProp(pixels, params, numberOfLayers):\n    cache = {\"A0\": pixels}\n    A_prev = pixels\n\n    for l in range(1, numberOfLayers):\n        Z = params[f\"W{l}\"].dot(A_prev) + params[f\"b{l}\"]\n        A = ReLU(Z)\n        cache[f\"Z{l}\"] = Z\n        cache[f\"A{l}\"] = A\n        A_prev = A\n\n    ZL = params[f\"W{numberOfLayers}\"].dot(A_prev) + params[f\"b{numberOfLayers}\"]\n    AL = softmax(ZL)\n    cache[f\"Z{numberOfLayers}\"] = ZL\n    cache[f\"A{numberOfLayers}\"] = AL\n    return cache\n\ndef backwardProp(params, cache, pixels, labels, numberOfLayers):\n    grads = {}\n    desiredOut = desiredOutputs(labels)\n    AL = cache[f\"A{numberOfLayers}\"]\n\n    dZL = AL - desiredOut\n    grads[f\"dW{numberOfLayers}\"] = 1 / pixels.shape[1] * dZL.dot(cache[f\"A{numberOfLayers - 1}\"].T)\n    grads[f\"db{numberOfLayers}\"] = 1 / pixels.shape[1] * np.sum(dZL, axis=1, keepdims=True)\n\n    dA_prev = params[f\"W{numberOfLayers}\"].T.dot(dZL)\n\n    for l in range(numberOfLayers - 1, 0, -1):\n        dZ = dA_prev * ReluDeriv(cache[f\"Z{l}\"])\n        grads[f\"dW{l}\"] = 1 / pixels.shape[1] * dZ.dot(cache[f\"A{l - 1}\"].T)\n        grads[f\"db{l}\"] = 1 / pixels.shape[1] * np.sum(dZ, axis=1, keepdims=True)\n        dA_prev = params[f\"W{l}\"].T.dot(dZ)\n\n    return grads\n\ndef updateParams(params, grads, alpha, numberOfLayers):\n    for l in range(1, numberOfLayers + 1):\n        params[f\"W{l}\"] -= alpha * grads[f\"dW{l}\"]\n        params[f\"b{l}\"] -= alpha * grads[f\"db{l}\"]\n    return params","metadata":{"execution":{"iopub.status.busy":"2024-10-24T20:50:03.246808Z","iopub.execute_input":"2024-10-24T20:50:03.247197Z","iopub.status.idle":"2024-10-24T20:50:03.261748Z","shell.execute_reply.started":"2024-10-24T20:50:03.247159Z","shell.execute_reply":"2024-10-24T20:50:03.260046Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Now, some functions to evaluate results","metadata":{}},{"cell_type":"code","source":"def getPredictions(AL):\n    return np.argmax(AL, 0)\n\ndef getAccuracy(predictions, Y):\n    return np.sum(predictions == Y) / Y.size\n\ndef makePredictions(pixels, params, layerSizes):\n    numberOfLayers = len(layerSizes) - 1\n    cache = forwardProp(pixels, params, numberOfLayers)\n    predictions = getPredictions(cache[f\"A{numberOfLayers}\"])\n    return predictions\n\ndef testPrediction(trainLabels, trainPixels, index, params, layerSizes):\n    current_image = trainPixels[:, index, None]\n    prediction = makePredictions(trainPixels[:, index, None], params, layerSizes)\n    label = trainLabels[index]\n    print(\"Prediction: \", prediction)\n    print(\"Label: \", label)\n\n    current_image = current_image.reshape((28, 28)) * 255\n    plt.gray()\n    plt.imshow(current_image, interpolation='nearest')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-24T20:50:03.263280Z","iopub.execute_input":"2024-10-24T20:50:03.263828Z","iopub.status.idle":"2024-10-24T20:50:03.281674Z","shell.execute_reply.started":"2024-10-24T20:50:03.263647Z","shell.execute_reply":"2024-10-24T20:50:03.280114Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Finally, the gradient descent function to train the network.","metadata":{}},{"cell_type":"code","source":"def gradientDescent(X,Y, alpha, iterations, layerSizes):\n\n    numberOfLayers = len(layerSizes) - 1\n    params = initParams(layerSizes)\n\n    iteration = []\n    accuracy = []\n\n    for i in range(iterations):\n        cache = forwardProp(X, params, numberOfLayers)\n        grads = backwardProp(params, cache,X, Y, numberOfLayers)\n        params = updateParams(params, grads, alpha, numberOfLayers)\n\n        if i % 10 == 0:\n            AL = cache[f\"A{numberOfLayers}\"]\n            predictions = getPredictions(AL)\n            print(f\"Iteration: {i}, Accuracy: {getAccuracy(predictions, Y)}\")\n            iteration.append(i)\n            accuracy.append(getAccuracy(predictions, Y))\n\n    return params, iteration, accuracy","metadata":{"execution":{"iopub.status.busy":"2024-10-24T20:50:03.283204Z","iopub.execute_input":"2024-10-24T20:50:03.283695Z","iopub.status.idle":"2024-10-24T20:50:03.301573Z","shell.execute_reply.started":"2024-10-24T20:50:03.283639Z","shell.execute_reply":"2024-10-24T20:50:03.300075Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The neural network can now be created, trained and tested","metadata":{}},{"cell_type":"code","source":"layerSizes = [784, 10, 10]\n\ntrainLabels, trainPixels, testLabels, testPixels = getDataset(\"train.csv\")\nparams, iteration, accuracy = gradientDescent(trainPixels,trainLabels, 0.10, 500, layerSizes)\nparams['layerSizes'] = layerSizes\n\nplt.plot(iteration, accuracy, marker='o', color='b')\nplt.xlabel('Iterations')\nplt.ylabel('Accuracy')\nplt.title('Accuracy vs Iterations')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-24T20:50:03.305315Z","iopub.execute_input":"2024-10-24T20:50:03.306409Z","iopub.status.idle":"2024-10-24T20:51:18.909415Z","shell.execute_reply.started":"2024-10-24T20:50:03.306361Z","shell.execute_reply":"2024-10-24T20:51:18.907816Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"testPrediction(trainLabels, trainPixels, 100, params, params['layerSizes'])\n\ntestResults = makePredictions(testPixels, params, params['layerSizes'])\nprint(f\"model accuracy : {getAccuracy(testResults, testLabels)}\")","metadata":{"execution":{"iopub.status.busy":"2024-10-24T20:51:18.911011Z","iopub.execute_input":"2024-10-24T20:51:18.911465Z","iopub.status.idle":"2024-10-24T20:51:19.116956Z","shell.execute_reply.started":"2024-10-24T20:51:18.911414Z","shell.execute_reply":"2024-10-24T20:51:19.112947Z"},"trusted":true},"outputs":[],"execution_count":null}]}