{"metadata":{"colab":{"provenance":[],"toc_visible":true,"collapsed_sections":["fXTBSwm9VjdR","EKGnwMJZTMa3","9XemUWJ0mo5l","AYy-p_8Kmyu9","fyraGzVemr9t","it6ziYKWZSdH","uV5JfoitZZRu","hT1wFpxeagXC","RApWtAqa29Av","9xOBwXU5bHQA","aBsTMt1FbOTc","Xm6NxBeza5nS"]},"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30673,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Digit Recognizer\nSOFE 4620 Machine Learning and Data Mining Group Project<br>\n<br>\nMatthew Gardiner<br>\nEric Hua","metadata":{"id":"nQoUwNAEVKm0"}},{"cell_type":"markdown","source":"## Import Statements","metadata":{"id":"fXTBSwm9VjdR"}},{"cell_type":"code","source":"import tensorflow as tf\nimport numpy as np\nfrom matplotlib import pyplot as plt\nfrom sklearn.metrics import ConfusionMatrixDisplay\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split","metadata":{"id":"oOO-Mr4vTrtw","executionInfo":{"status":"ok","timestamp":1712331525262,"user_tz":240,"elapsed":11687,"user":{"displayName":"Matthew Gardiner","userId":"17566100832524217443"}},"execution":{"iopub.status.busy":"2024-04-05T18:28:46.785054Z","iopub.execute_input":"2024-04-05T18:28:46.785747Z","iopub.status.idle":"2024-04-05T18:29:03.334939Z","shell.execute_reply.started":"2024-04-05T18:28:46.785712Z","shell.execute_reply":"2024-04-05T18:29:03.333480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Functions","metadata":{"id":"DRRcyyLjVn9p"}},{"cell_type":"markdown","source":"## load_and_preprocess_data\nLoads in the MNIST dataset directly from tensorflow.<br>\nNormalizes the pixel values.<br>\nReturns the processed data.","metadata":{"id":"EKGnwMJZTMa3"}},{"cell_type":"code","source":"def load_and_preprocess_data():\n    \n    #Using the actual MNIST dataset\n    #mnist = tf.keras.datasets.mnist\n    #(train_images, train_labels), (test_images, test_labels) = mnist.load_data()\n\n    # Normalize the pixel values\n    #train_images = normalize_images(train_images)\n    #test_images = normalize_images(test_images)\n\n    df_train = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\n    df_test = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")\n\n    labels = df_train['label'].to_numpy()\n    pixels = np.reshape(df_train.drop(columns=['label']).to_numpy(),(42000,28,28))\n\n    competition_test_images = np.reshape(df_test.to_numpy(),(28000,28,28))\n    \n    train_images, test_images, train_labels, test_labels = train_test_split(pixels, labels, test_size=0.2, random_state=123, stratify=labels)\n    \n    train_images = normalize_images(train_images)\n    test_images = normalize_images(test_images)\n    \n    return (train_images, train_labels), (test_images, test_labels), competition_test_images","metadata":{"id":"bb9xwxTjSgVY","executionInfo":{"status":"ok","timestamp":1712331531167,"user_tz":240,"elapsed":145,"user":{"displayName":"Matthew Gardiner","userId":"17566100832524217443"}},"execution":{"iopub.status.busy":"2024-04-05T18:29:19.217892Z","iopub.execute_input":"2024-04-05T18:29:19.218278Z","iopub.status.idle":"2024-04-05T18:29:19.228218Z","shell.execute_reply.started":"2024-04-05T18:29:19.218249Z","shell.execute_reply":"2024-04-05T18:29:19.226572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def normalize_images(images):\n    # Normalize pixel values to range [0, 1]\n    return images / 255.0","metadata":{"id":"13cysprXUUmr","executionInfo":{"status":"ok","timestamp":1712331531306,"user_tz":240,"elapsed":1,"user":{"displayName":"Matthew Gardiner","userId":"17566100832524217443"}},"execution":{"iopub.status.busy":"2024-04-05T18:29:21.431802Z","iopub.execute_input":"2024-04-05T18:29:21.432252Z","iopub.status.idle":"2024-04-05T18:29:21.439264Z","shell.execute_reply.started":"2024-04-05T18:29:21.432220Z","shell.execute_reply":"2024-04-05T18:29:21.437538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## build_model\nDefines the models architecture","metadata":{"id":"uXmHBmd7T8P3"}},{"cell_type":"markdown","source":"## Simple ANN","metadata":{"id":"9XemUWJ0mo5l"}},{"cell_type":"markdown","source":"Here is a simple artificial neural network with only fully connected layers plus some dropout regularitzation","metadata":{"id":"Jweja_AMZiLU"}},{"cell_type":"code","source":"def build_modelANN():\n    # Define the model architecture\n    model = tf.keras.models.Sequential([\n        tf.keras.layers.Flatten(input_shape=(28, 28)),\n        tf.keras.layers.Dense(128, activation='relu'),\n        tf.keras.layers.Dropout(0.2),\n        tf.keras.layers.Dense(10, activation='softmax')\n    ])\n    return model","metadata":{"id":"uKOYYIL7TS6R","executionInfo":{"status":"ok","timestamp":1712331533854,"user_tz":240,"elapsed":180,"user":{"displayName":"Matthew Gardiner","userId":"17566100832524217443"}},"execution":{"iopub.status.busy":"2024-04-05T18:29:23.436263Z","iopub.execute_input":"2024-04-05T18:29:23.436638Z","iopub.status.idle":"2024-04-05T18:29:23.443730Z","shell.execute_reply.started":"2024-04-05T18:29:23.436601Z","shell.execute_reply":"2024-04-05T18:29:23.442209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Simple CNN","metadata":{"id":"AYy-p_8Kmyu9"}},{"cell_type":"markdown","source":"Here is a simpler convolutional artificial neural network where instead of two convolutional layers per pooling layer there is only one. This is ultimately to save time when training the model as the complex CNN below can take upwards of 15 minutes to train.","metadata":{"id":"uFcDafhXaD_y"}},{"cell_type":"code","source":"def build_modelCNNSimple():\n    # Define the model architecture\n    model = tf.keras.models.Sequential([\n        tf.keras.layers.Input(shape=(28,28,1)),\n        tf.keras.layers.Conv2D(32, 3, activation='relu'),\n        tf.keras.layers.MaxPooling2D(pool_size=2),\n\n        tf.keras.layers.Flatten(),\n        tf.keras.layers.Dense(128, activation='relu'),\n        tf.keras.layers.Dropout(0.2),\n        tf.keras.layers.Dense(10, activation='softmax')\n    ])\n    return model","metadata":{"id":"BBqQipTwwDpJ","executionInfo":{"status":"ok","timestamp":1712331536433,"user_tz":240,"elapsed":176,"user":{"displayName":"Matthew Gardiner","userId":"17566100832524217443"}},"execution":{"iopub.status.busy":"2024-04-05T18:29:25.231314Z","iopub.execute_input":"2024-04-05T18:29:25.231763Z","iopub.status.idle":"2024-04-05T18:29:25.241050Z","shell.execute_reply.started":"2024-04-05T18:29:25.231733Z","shell.execute_reply":"2024-04-05T18:29:25.239653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Main CNN","metadata":{"id":"fyraGzVemr9t"}},{"cell_type":"markdown","source":"Here is a convolutional artificial neural network with two convolution plus pooling sub-layers, followed by 3 fully connected layers to classify. Dropout regularization was also used throughout","metadata":{"id":"Y1fGqFj3ZrBW"}},{"cell_type":"code","source":"def build_modelCNN():\n    # Define the model architecture\n    # Based off AlexNet\n    model = tf.keras.models.Sequential([\n\n        tf.keras.layers.Input(shape=(28,28,1)),\n\n        tf.keras.layers.Conv2D(32, 3, activation='relu', strides=1, padding='same'),\n        tf.keras.layers.MaxPooling2D(pool_size=2),\n        tf.keras.layers.Dropout(0.2),\n\n        tf.keras.layers.Conv2D(64, 3, activation='relu', strides=1, padding='same'),\n        tf.keras.layers.Conv2D(64, 3, activation='relu', strides=1, padding='same'),\n        tf.keras.layers.MaxPooling2D(pool_size=2),\n        tf.keras.layers.Dropout(0.2),\n\n        tf.keras.layers.Conv2D(128, 3, activation='relu', strides=1, padding='same'),\n        tf.keras.layers.Conv2D(128, 3, activation='relu', strides=1, padding='same'),\n        tf.keras.layers.MaxPooling2D(pool_size=2),\n        tf.keras.layers.Dropout(0.2),\n\n        #fully connected layers to start prediction\n        #begin by flattening the output\n        tf.keras.layers.Flatten(),\n        tf.keras.layers.Dense(256, activation='relu'),\n        tf.keras.layers.Dense(128, activation='relu'),\n        tf.keras.layers.Dense(10, activation='softmax')\n    ])\n    return model","metadata":{"id":"dmKbs4rUZatx","executionInfo":{"status":"ok","timestamp":1712331537255,"user_tz":240,"elapsed":2,"user":{"displayName":"Matthew Gardiner","userId":"17566100832524217443"}},"execution":{"iopub.status.busy":"2024-04-05T20:54:42.981816Z","iopub.execute_input":"2024-04-05T20:54:42.982271Z","iopub.status.idle":"2024-04-05T20:54:43.002992Z","shell.execute_reply.started":"2024-04-05T20:54:42.982238Z","shell.execute_reply":"2024-04-05T20:54:43.001408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## train_model\nCompiles and trains the model using Adam optimization.\n\nAlso prints a graph showing training accuracy over epochs","metadata":{"id":"it6ziYKWZSdH"}},{"cell_type":"code","source":"def train_model(model, train_images, train_labels, epochs=20):\n    # Compile the model\n    print(\"Training:\")\n    model.compile(optimizer='adam',\n                  loss='sparse_categorical_crossentropy',\n                  metrics=['accuracy'])\n\n    # Train the model\n    model_history = model.fit(train_images, train_labels, epochs=epochs)\n\n    plt.plot(model_history.history['accuracy'])\n    plt.legend(['accuracy'],loc='lower right')\n    plt.title(\"model train accuracy\")\n    plt.xlabel(\"epoch\")\n    plt.ylabel(\"accuracy\")\n    plt.show()","metadata":{"id":"JLLYjGljUB1j","executionInfo":{"status":"ok","timestamp":1712331544672,"user_tz":240,"elapsed":151,"user":{"displayName":"Matthew Gardiner","userId":"17566100832524217443"}},"execution":{"iopub.status.busy":"2024-04-05T18:31:36.120124Z","iopub.execute_input":"2024-04-05T18:31:36.120552Z","iopub.status.idle":"2024-04-05T18:31:36.129111Z","shell.execute_reply.started":"2024-04-05T18:31:36.120523Z","shell.execute_reply":"2024-04-05T18:31:36.127753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## evaluate_model\nEvaluates the models performance.","metadata":{"id":"uV5JfoitZZRu"}},{"cell_type":"code","source":"def evaluate_model(model, test_images, test_labels):\n    # Evaluate the model\n    print(\"Evaluation:\")\n    test_loss, test_acc = model.evaluate(test_images, test_labels)\n    print(f\"Test accuracy: {test_acc}\")","metadata":{"id":"15ZSrdjAUcRa","executionInfo":{"status":"ok","timestamp":1712331545429,"user_tz":240,"elapsed":2,"user":{"displayName":"Matthew Gardiner","userId":"17566100832524217443"}},"execution":{"iopub.status.busy":"2024-04-05T18:29:34.583222Z","iopub.execute_input":"2024-04-05T18:29:34.583585Z","iopub.status.idle":"2024-04-05T18:29:34.589269Z","shell.execute_reply.started":"2024-04-05T18:29:34.583550Z","shell.execute_reply":"2024-04-05T18:29:34.588162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## make_predictions\nSimply predicts on the first few samples and prints the result; a more human approach of displaying the results","metadata":{"id":"hT1wFpxeagXC"}},{"cell_type":"code","source":"def make_predictions(model, test_images, num_samples=5):\n    # Make predictions\n    predictions = model.predict(test_images[:num_samples])\n    predicted_labels = predictions.argmax(axis=1)\n    print(\"Predictions:\")\n    print(predicted_labels)","metadata":{"id":"Dn1UIk16UduH","executionInfo":{"status":"ok","timestamp":1712331546527,"user_tz":240,"elapsed":1,"user":{"displayName":"Matthew Gardiner","userId":"17566100832524217443"}},"execution":{"iopub.status.busy":"2024-04-05T18:29:35.913686Z","iopub.execute_input":"2024-04-05T18:29:35.914173Z","iopub.status.idle":"2024-04-05T18:29:35.920290Z","shell.execute_reply.started":"2024-04-05T18:29:35.914140Z","shell.execute_reply":"2024-04-05T18:29:35.919072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## display errors\n\ndisplays 8 most important errors\n\nCode from:\n\n[1] rgarg1234, “Digit recognizer: CNN,” Kaggle, https://www.kaggle.com/code/rgarg1234/digit-recognizer-cnn (accessed Apr. 4, 2024).\n","metadata":{"id":"RApWtAqa29Av"}},{"cell_type":"code","source":"def display_important_errors(predictions, predicted_labels, test_labels, test_images): #[1]\n  errors = (predicted_labels - test_labels != 0)\n\n  Y_pred_classes_errors = predicted_labels[errors]\n  Y_pred_errors = predictions[errors]\n  Y_true_errors = test_labels[errors]\n  X_val_errors = test_images[errors]\n\n  def display_errors(errors_index,img_errors,pred_errors, obs_errors):\n    n = 0\n    nrows = 2\n    ncols = 4\n    fig, ax = plt.subplots(nrows,ncols,figsize=(15,15))\n    for row in range(nrows):\n        for col in range(ncols):\n            error = errors_index[n]\n            ax[row,col].imshow((img_errors[error]).reshape((28,28)))\n            ax[row,col].set_title(\"Predicted label :{}\\nTrue label :{}\".format(pred_errors[error],obs_errors[error]))\n            n += 1\n  Y_pred_errors_prob = np.max(Y_pred_errors,axis = 1)\n\n  true_prob_errors = np.diagonal(np.take(Y_pred_errors, Y_true_errors, axis=1))\n\n  delta_pred_true_errors = Y_pred_errors_prob - true_prob_errors\n\n  sorted_dela_errors = np.argsort(delta_pred_true_errors)\n\n\n  most_important_errors = sorted_dela_errors\n\n  display_errors(most_important_errors, X_val_errors, Y_pred_classes_errors, Y_true_errors)","metadata":{"id":"PWNSm-Jjxb_R","executionInfo":{"status":"ok","timestamp":1712333781318,"user_tz":240,"elapsed":8,"user":{"displayName":"Matthew Gardiner","userId":"17566100832524217443"}},"execution":{"iopub.status.busy":"2024-04-05T18:29:37.694758Z","iopub.execute_input":"2024-04-05T18:29:37.695247Z","iopub.status.idle":"2024-04-05T18:29:37.708065Z","shell.execute_reply.started":"2024-04-05T18:29:37.695213Z","shell.execute_reply":"2024-04-05T18:29:37.706557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Main Code","metadata":{"id":"eVmR2ROudOme"}},{"cell_type":"markdown","source":"## ANN\nThis is the running module of the simple artificial neural network","metadata":{"id":"9xOBwXU5bHQA"}},{"cell_type":"code","source":"#ANN Training\n(train_images, train_labels), (test_images, test_labels), competition_test_images = load_and_preprocess_data()\n\nmodelANN = build_modelANN()\ntrain_model(modelANN, train_images, train_labels)\nevaluate_model(modelANN, test_images, test_labels)\n\n\n#Confusion Matrix\n#make predictions\npredictions = modelANN.predict(test_images)\npredicted_labels = predictions.argmax(axis=1)\n\n#display confusion matrix on test/predicted labels\nConfusionMatrixDisplay.from_predictions(test_labels, predicted_labels)\nplt.show()\n\n#display errors\ndisplay_important_errors(predictions, predicted_labels, test_labels, test_images)\n\n#model and layers summary\nmodelANN.summary()","metadata":{"id":"HRGaf5yQwuia","executionInfo":{"status":"ok","timestamp":1712333781318,"user_tz":240,"elapsed":90336,"user":{"displayName":"Matthew Gardiner","userId":"17566100832524217443"}},"outputId":"2e372e82-a06e-42c0-9530-6d108a65f740","execution":{"iopub.status.busy":"2024-04-05T17:10:48.764203Z","iopub.execute_input":"2024-04-05T17:10:48.765002Z","iopub.status.idle":"2024-04-05T17:11:21.030330Z","shell.execute_reply.started":"2024-04-05T17:10:48.764967Z","shell.execute_reply":"2024-04-05T17:11:21.028975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As can be seen, the accuracy of this model is quite high, but for the purpose of this problem, fairly low. Ideally, you would probably want an accuracy above 99%.","metadata":{"id":"P_Io2FV4cHIg"}},{"cell_type":"markdown","source":"## Simple CNN\nThis is the running module for the simple convolutional neural network","metadata":{"id":"aBsTMt1FbOTc"}},{"cell_type":"code","source":"#CNN Simple Training\n(train_images, train_labels), (test_images, test_labels), competition_test_images = load_and_preprocess_data()\n\ntrain_images2 = np.expand_dims(train_images, axis=-1)\ntest_images2 = np.expand_dims(test_images, axis=-1)\n\nmodelCNNSimple = build_modelCNNSimple()\ntrain_model(modelCNNSimple, train_images2, train_labels)\nevaluate_model(modelCNNSimple, test_images2, test_labels)\n\n\n#Confusion Matrix\n#make predictions\npredictions = modelCNNSimple.predict(test_images)\npredicted_labels = predictions.argmax(axis=1)\n\n#display confusion matrix on test/predicted labels\nConfusionMatrixDisplay.from_predictions(test_labels, predicted_labels)\nplt.show()\n\n#display errors\ndisplay_important_errors(predictions, predicted_labels, test_labels, test_images)\n\n# summary of model and layers\nmodelCNNSimple.summary()","metadata":{"id":"_73veVPtwvny","executionInfo":{"status":"ok","timestamp":1712214023434,"user_tz":240,"elapsed":212659,"user":{"displayName":"Matthew Gardiner","userId":"17566100832524217443"}},"outputId":"6084deae-326d-4fa6-c266-341d7f061906","execution":{"iopub.status.busy":"2024-04-05T17:42:31.518967Z","iopub.execute_input":"2024-04-05T17:42:31.519380Z","iopub.status.idle":"2024-04-05T17:45:19.249181Z","shell.execute_reply.started":"2024-04-05T17:42:31.519349Z","shell.execute_reply":"2024-04-05T17:45:19.247866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Main CNN\nThis is the running module of the main convolutional neural network","metadata":{"id":"Xm6NxBeza5nS"}},{"cell_type":"code","source":"#Advanced CNN Training\n(train_images, train_labels), (test_images, test_labels), competition_test_images = load_and_preprocess_data()\n\ntrain_images2 = np.expand_dims(train_images, axis=-1)\ntest_images2 = np.expand_dims(test_images, axis=-1)\n\nmodelCNN = build_modelCNN()\ntrain_model(modelCNN, train_images2, train_labels)\nevaluate_model(modelCNN, test_images2, test_labels)\n\n\n#Confusion Matrix\n#make predictions\npredictions = modelCNN.predict(test_images)\npredicted_labels = predictions.argmax(axis=1)\n\n#display confusion matrix on test/predicted labels\nConfusionMatrixDisplay.from_predictions(test_labels, predicted_labels)\nplt.show()\n\n#display errors\ndisplay_important_errors(predictions, predicted_labels, test_labels, test_images)\n\n#model and layers summary\nmodelCNN.summary()","metadata":{"id":"1vh-OK63UfUq","outputId":"4bba1adb-1f04-4d14-e5d0-4e0891760574","executionInfo":{"status":"ok","timestamp":1712332182433,"user_tz":240,"elapsed":531043,"user":{"displayName":"Matthew Gardiner","userId":"17566100832524217443"}},"execution":{"iopub.status.busy":"2024-04-05T18:57:16.876930Z","iopub.execute_input":"2024-04-05T18:57:16.877995Z","iopub.status.idle":"2024-04-05T18:57:45.692673Z","shell.execute_reply.started":"2024-04-05T18:57:16.877940Z","shell.execute_reply":"2024-04-05T18:57:45.690872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission Prediction and Submission","metadata":{}},{"cell_type":"code","source":"(train_images, train_labels), (test_images, test_labels), competition_test_images = load_and_preprocess_data()\n\n#y_test_pred = modelCNN.predict(competition_test_images)\n\npredictions = modelCNN.predict(competition_test_images)\npredicted_labels = predictions.argmax(axis=1)\n\ndf_result = pd.DataFrame([])\ndf_result['ImageId'] = list(range(1, 28001))\ndf_result['Label'] = predicted_labels\ndf_result.to_csv('/kaggle/working/submission.csv', index=None)","metadata":{"execution":{"iopub.status.busy":"2024-04-05T18:00:04.804154Z","iopub.execute_input":"2024-04-05T18:00:04.804617Z","iopub.status.idle":"2024-04-05T18:00:19.642939Z","shell.execute_reply.started":"2024-04-05T18:00:04.804585Z","shell.execute_reply":"2024-04-05T18:00:19.640626Z"},"trusted":true},"execution_count":null,"outputs":[]}]}