{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30699,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Digit Recognizer on MNIST using CNN","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"# import all necessary libraries/modules","metadata":{}},{"cell_type":"code","source":"# Warnings handling\nimport warnings\n# \n# For preparing data for training\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.image as pimg\n# \n# For training\nfrom tensorflow import __version__, one_hot, argmax\nfrom tensorflow.keras import Sequential\nfrom tensorflow.keras.layers import Input, Conv2D, MaxPooling2D, GlobalAveragePooling2D, Flatten, Dropout, BatchNormalization, Dense\nfrom tensorflow.keras.callbacks import Callback, ReduceLROnPlateau, EarlyStopping, ModelCheckpoint\nfrom tensorflow.keras.optimizers import RMSprop\nfrom tensorflow.nn import relu, elu, softmax\nfrom tensorflow.keras.losses import sparse_categorical_crossentropy\nfrom tensorflow.keras.models import save_model, load_model\n#\n\nclass EpochWarning(ResourceWarning):\n    pass\n    \ndef issue_warning(message):\n    warnings.warn(message, EpochWarning)\n\nwarnings.simplefilter('always', EpochWarning)\nwarnings.simplefilter('ignore')\n\nprint(__version__)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T21:26:59.051738Z","iopub.execute_input":"2024-05-21T21:26:59.052312Z","iopub.status.idle":"2024-05-21T21:27:11.235987Z","shell.execute_reply.started":"2024-05-21T21:26:59.052283Z","shell.execute_reply":"2024-05-21T21:27:11.235063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare the train and test dataset","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-05-21T21:27:11.237707Z","iopub.execute_input":"2024-05-21T21:27:11.238237Z","iopub.status.idle":"2024-05-21T21:27:15.689404Z","shell.execute_reply.started":"2024-05-21T21:27:11.238212Z","shell.execute_reply":"2024-05-21T21:27:15.688369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-05-21T21:27:15.690629Z","iopub.execute_input":"2024-05-21T21:27:15.690930Z","iopub.status.idle":"2024-05-21T21:27:18.641228Z","shell.execute_reply.started":"2024-05-21T21:27:15.690905Z","shell.execute_reply":"2024-05-21T21:27:18.640200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Remove any null rows**","metadata":{}},{"cell_type":"code","source":"train_df = train_df.dropna()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T21:27:18.643927Z","iopub.execute_input":"2024-05-21T21:27:18.644581Z","iopub.status.idle":"2024-05-21T21:27:18.760099Z","shell.execute_reply.started":"2024-05-21T21:27:18.644548Z","shell.execute_reply":"2024-05-21T21:27:18.759191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**View the datasets**","metadata":{}},{"cell_type":"code","source":"train_df[0:10]","metadata":{"execution":{"iopub.status.busy":"2024-05-21T21:27:18.761177Z","iopub.execute_input":"2024-05-21T21:27:18.761436Z","iopub.status.idle":"2024-05-21T21:27:18.787393Z","shell.execute_reply.started":"2024-05-21T21:27:18.761414Z","shell.execute_reply":"2024-05-21T21:27:18.786534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df[0:10]","metadata":{"execution":{"iopub.status.busy":"2024-05-21T21:27:18.788483Z","iopub.execute_input":"2024-05-21T21:27:18.788793Z","iopub.status.idle":"2024-05-21T21:27:18.806073Z","shell.execute_reply.started":"2024-05-21T21:27:18.788763Z","shell.execute_reply":"2024-05-21T21:27:18.805142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Add the train dataset label to the label header as a numpy and the rest as a numpy array**","metadata":{}},{"cell_type":"code","source":"train_labels = train_df.iloc[:, 0].to_numpy()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T21:27:18.807273Z","iopub.execute_input":"2024-05-21T21:27:18.807590Z","iopub.status.idle":"2024-05-21T21:27:18.814156Z","shell.execute_reply.started":"2024-05-21T21:27:18.807561Z","shell.execute_reply":"2024-05-21T21:27:18.813281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels","metadata":{"execution":{"iopub.status.busy":"2024-05-21T21:27:18.815230Z","iopub.execute_input":"2024-05-21T21:27:18.815492Z","iopub.status.idle":"2024-05-21T21:27:18.824919Z","shell.execute_reply.started":"2024-05-21T21:27:18.815471Z","shell.execute_reply":"2024-05-21T21:27:18.824152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_features = train_df.iloc[:, 1:].to_numpy()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T21:27:18.826000Z","iopub.execute_input":"2024-05-21T21:27:18.826260Z","iopub.status.idle":"2024-05-21T21:27:18.835619Z","shell.execute_reply.started":"2024-05-21T21:27:18.826238Z","shell.execute_reply":"2024-05-21T21:27:18.834907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_features","metadata":{"execution":{"iopub.status.busy":"2024-05-21T21:27:18.838823Z","iopub.execute_input":"2024-05-21T21:27:18.839083Z","iopub.status.idle":"2024-05-21T21:27:18.846161Z","shell.execute_reply.started":"2024-05-21T21:27:18.839061Z","shell.execute_reply":"2024-05-21T21:27:18.845410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_features = test_df.to_numpy()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T21:27:18.847082Z","iopub.execute_input":"2024-05-21T21:27:18.847405Z","iopub.status.idle":"2024-05-21T21:27:18.854805Z","shell.execute_reply.started":"2024-05-21T21:27:18.847375Z","shell.execute_reply":"2024-05-21T21:27:18.853940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_features","metadata":{"execution":{"iopub.status.busy":"2024-05-21T21:27:18.855794Z","iopub.execute_input":"2024-05-21T21:27:18.856166Z","iopub.status.idle":"2024-05-21T21:27:18.866470Z","shell.execute_reply.started":"2024-05-21T21:27:18.856123Z","shell.execute_reply":"2024-05-21T21:27:18.865626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define constants","metadata":{}},{"cell_type":"code","source":"SHAPE = (28, 28, 1)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T21:27:18.867500Z","iopub.execute_input":"2024-05-21T21:27:18.867766Z","iopub.status.idle":"2024-05-21T21:27:18.872956Z","shell.execute_reply.started":"2024-05-21T21:27:18.867744Z","shell.execute_reply":"2024-05-21T21:27:18.872181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Reshape the train and test features numpy arrays**","metadata":{}},{"cell_type":"code","source":"train_features = np.array([\n    np.reshape(feature, (SHAPE)) for feature in train_features\n])","metadata":{"execution":{"iopub.status.busy":"2024-05-21T21:27:18.874222Z","iopub.execute_input":"2024-05-21T21:27:18.874487Z","iopub.status.idle":"2024-05-21T21:27:19.113674Z","shell.execute_reply.started":"2024-05-21T21:27:18.874466Z","shell.execute_reply":"2024-05-21T21:27:19.112832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_features = np.array([\n    np.reshape(feature, (SHAPE)) for feature in test_features\n])","metadata":{"execution":{"iopub.status.busy":"2024-05-21T21:27:19.114880Z","iopub.execute_input":"2024-05-21T21:27:19.115253Z","iopub.status.idle":"2024-05-21T21:27:19.280061Z","shell.execute_reply.started":"2024-05-21T21:27:19.115223Z","shell.execute_reply":"2024-05-21T21:27:19.279077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show the first train feature shape\ntrain_features[0].shape","metadata":{"execution":{"iopub.status.busy":"2024-05-21T21:27:19.281440Z","iopub.execute_input":"2024-05-21T21:27:19.281730Z","iopub.status.idle":"2024-05-21T21:27:19.287633Z","shell.execute_reply.started":"2024-05-21T21:27:19.281706Z","shell.execute_reply":"2024-05-21T21:27:19.286751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize the images in the training and testing to see what it looks like","metadata":{}},{"cell_type":"code","source":"%matplotlib inline\nnrows = 4\nncols = 4\npic_index = 0\nfig = plt.gcf()\nfig.set_size_inches(ncols * 4, nrows * 4)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T21:27:19.288996Z","iopub.execute_input":"2024-05-21T21:27:19.289284Z","iopub.status.idle":"2024-05-21T21:27:19.308581Z","shell.execute_reply.started":"2024-05-21T21:27:19.289257Z","shell.execute_reply":"2024-05-21T21:27:19.307800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pic_index += 8\n\nvisualized_img_arr = train_features[pic_index -8: pic_index] + test_features[pic_index -8: pic_index]\n\n# Plot the images\nfor indx, img_arr in enumerate(visualized_img_arr):\n    sp = plt.subplot(nrows, ncols, indx + 1)\n    sp.axis(\"off\")\n    plt.imshow(img_arr)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T21:27:19.309623Z","iopub.execute_input":"2024-05-21T21:27:19.309879Z","iopub.status.idle":"2024-05-21T21:27:19.543838Z","shell.execute_reply.started":"2024-05-21T21:27:19.309856Z","shell.execute_reply":"2024-05-21T21:27:19.542660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train the model","metadata":{}},{"cell_type":"markdown","source":"**Define the hyper parameters used**","metadata":{}},{"cell_type":"code","source":"cnn_neurons = [16, 64, 128]\ndnn_hidden_neurons = 1024\nn_dropout = .2\ndnn_output_neurons = 10\nlearning_rate = 1e-4\nexpected_accuracy = .99\ntraining_patience = 10\nn_epochs = 10\nbatch_size = 32\nmodel_filename = \"/kaggle/working/mnist-digit-recog.keras\"","metadata":{"execution":{"iopub.status.busy":"2024-05-21T21:27:19.545522Z","iopub.execute_input":"2024-05-21T21:27:19.546057Z","iopub.status.idle":"2024-05-21T21:27:19.554460Z","shell.execute_reply.started":"2024-05-21T21:27:19.546002Z","shell.execute_reply":"2024-05-21T21:27:19.552775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Sequential([\n    Input(shape=SHAPE),\n    # 1cnn\n    Conv2D(cnn_neurons[0], (3, 3), activation=relu),\n    MaxPooling2D(2, 2),\n    # 2cnn\n    Conv2D(cnn_neurons[1], (3, 3), activation=relu),\n    MaxPooling2D(2, 2),\n    # 3cnn\n    Conv2D(cnn_neurons[2], (3, 3), activation=elu),\n    GlobalAveragePooling2D(),\n    BatchNormalization(),\n    # dnn\n    Flatten(),\n    Dropout(n_dropout), \n    Dense(dnn_hidden_neurons, activation=relu),\n    Dense(dnn_output_neurons, activation=softmax)\n])","metadata":{"execution":{"iopub.status.busy":"2024-05-21T21:27:19.556236Z","iopub.execute_input":"2024-05-21T21:27:19.557444Z","iopub.status.idle":"2024-05-21T21:27:20.270854Z","shell.execute_reply.started":"2024-05-21T21:27:19.557399Z","shell.execute_reply":"2024-05-21T21:27:20.270068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T21:27:20.271849Z","iopub.execute_input":"2024-05-21T21:27:20.272117Z","iopub.status.idle":"2024-05-21T21:27:20.296955Z","shell.execute_reply.started":"2024-05-21T21:27:20.272095Z","shell.execute_reply":"2024-05-21T21:27:20.296116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(\n    optimizer = RMSprop(learning_rate=learning_rate),\n    loss=sparse_categorical_crossentropy,\n    metrics=[\"accuracy\"]\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T21:27:20.298176Z","iopub.execute_input":"2024-05-21T21:27:20.298981Z","iopub.status.idle":"2024-05-21T21:27:20.312769Z","shell.execute_reply.started":"2024-05-21T21:27:20.298949Z","shell.execute_reply":"2024-05-21T21:27:20.312066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Define the callbacks used**","metadata":{}},{"cell_type":"code","source":"class EpochStopping(Callback):\n    def on_epoch_end(self, epoch, logs={}):\n        if (logs.get(\"accuracy\") is not None and logs.get(\"accuracy\") >= expected_accuracy):\n            # issue_warning(f\"This model training stopped since it has reached the expected training result\")\n            self.model.stop_training = True\n\nepoch_stopping = EpochStopping()\nearly_stopping = EarlyStopping(monitor=\"loss\", patience=training_patience, restore_best_weights=True)\nmodel_checkpoint = ModelCheckpoint(filepath=model_filename)\nreduce_lr = ReduceLROnPlateau(monitor=\"loss\", patience=training_patience/2)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T21:27:20.313666Z","iopub.execute_input":"2024-05-21T21:27:20.313914Z","iopub.status.idle":"2024-05-21T21:27:20.319995Z","shell.execute_reply.started":"2024-05-21T21:27:20.313892Z","shell.execute_reply":"2024-05-21T21:27:20.319019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Start training the model**","metadata":{}},{"cell_type":"code","source":"history = model.fit(\n    train_features,\n    train_labels,\n    epochs = n_epochs,\n    batch_size = batch_size,\n    callbacks=[epoch_stopping, early_stopping, reduce_lr, model_checkpoint],\n    verbose = 1\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T21:27:20.320953Z","iopub.execute_input":"2024-05-21T21:27:20.321268Z","iopub.status.idle":"2024-05-21T21:27:52.358686Z","shell.execute_reply.started":"2024-05-21T21:27:20.321244Z","shell.execute_reply":"2024-05-21T21:27:52.357939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Save the model**","metadata":{}},{"cell_type":"code","source":"# if model checkpoint not used then use this\n# save_model(model, model_filename)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T21:27:52.359833Z","iopub.execute_input":"2024-05-21T21:27:52.360107Z","iopub.status.idle":"2024-05-21T21:27:52.364008Z","shell.execute_reply.started":"2024-05-21T21:27:52.360083Z","shell.execute_reply":"2024-05-21T21:27:52.363118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Load the model if necessary**","metadata":{}},{"cell_type":"code","source":"# mymodel = load_model(model_filename)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T21:27:52.365264Z","iopub.execute_input":"2024-05-21T21:27:52.365582Z","iopub.status.idle":"2024-05-21T21:27:52.378501Z","shell.execute_reply.started":"2024-05-21T21:27:52.365552Z","shell.execute_reply":"2024-05-21T21:27:52.377741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot a graph to see how the accuracy perfomed against the losses while training","metadata":{}},{"cell_type":"code","source":"def plot(history):\n    plt.plot(history.history[\"accuracy\"], \"g\", label=\"Training Accuracy\")\n    plt.plot(history.history[\"loss\"], \"r\", label=\"Training Loss\")\n    plt.xlabel(\"Epochs\")\n    plt.ylabel(\"Value\")\n    plt.title(\"Training Accuracy & Loss\")\n    plt.legend()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T21:27:52.379541Z","iopub.execute_input":"2024-05-21T21:27:52.379824Z","iopub.status.idle":"2024-05-21T21:27:52.389638Z","shell.execute_reply.started":"2024-05-21T21:27:52.379802Z","shell.execute_reply":"2024-05-21T21:27:52.388873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot\nplot(history)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T21:27:52.394101Z","iopub.execute_input":"2024-05-21T21:27:52.394386Z","iopub.status.idle":"2024-05-21T21:27:52.692389Z","shell.execute_reply.started":"2024-05-21T21:27:52.394362Z","shell.execute_reply":"2024-05-21T21:27:52.691442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submit the predictions of the trained model","metadata":{}},{"cell_type":"markdown","source":"**Save the predicted result**","metadata":{}},{"cell_type":"code","source":"def adjustPredictedLabel(pred_arrs):\n    predicted_labels = [np.argmax(pred_arr) for pred_arr in pred_arrs]\n    return np.array(predicted_labels)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T21:27:52.693504Z","iopub.execute_input":"2024-05-21T21:27:52.693791Z","iopub.status.idle":"2024-05-21T21:27:52.698326Z","shell.execute_reply.started":"2024-05-21T21:27:52.693767Z","shell.execute_reply":"2024-05-21T21:27:52.697421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"make_predictions = model.predict(test_features)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T21:27:52.699543Z","iopub.execute_input":"2024-05-21T21:27:52.699813Z","iopub.status.idle":"2024-05-21T21:27:55.234750Z","shell.execute_reply.started":"2024-05-21T21:27:52.699791Z","shell.execute_reply":"2024-05-21T21:27:55.233826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create image_index array\nimg_indx = np.arange(1, len(test_features) + 1)\n# Stack img_indx and predictions\nresults_array = np.column_stack((img_indx, adjustPredictedLabel(make_predictions)))","metadata":{"execution":{"iopub.status.busy":"2024-05-21T21:27:55.236062Z","iopub.execute_input":"2024-05-21T21:27:55.236430Z","iopub.status.idle":"2024-05-21T21:27:55.341125Z","shell.execute_reply.started":"2024-05-21T21:27:55.236398Z","shell.execute_reply":"2024-05-21T21:27:55.340281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The imageid corresponds the the test csv rows for a particular label\ndata = {\n    \"ImageId\": results_array[:, 0],\n    \"Label\": results_array[:, 1]\n}\nsubmission_df = pd.DataFrame(data)\nsubmission_df.to_csv(\"/kaggle/working/submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T21:27:55.342364Z","iopub.execute_input":"2024-05-21T21:27:55.342723Z","iopub.status.idle":"2024-05-21T21:27:55.392831Z","shell.execute_reply.started":"2024-05-21T21:27:55.342692Z","shell.execute_reply":"2024-05-21T21:27:55.391883Z"},"trusted":true},"execution_count":null,"outputs":[]}]}