{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30805,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\n\nimport itertools\n\nimport tensorflow as tf\nfrom tensorflow.keras.layers import Input \nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.optimizers import SGD,Adam\nfrom keras.optimizers import RMSprop\nfrom keras.callbacks import ReduceLROnPlateau\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-14T19:23:07.111333Z","iopub.execute_input":"2024-12-14T19:23:07.111672Z","iopub.status.idle":"2024-12-14T19:23:18.964014Z","shell.execute_reply.started":"2024-12-14T19:23:07.111641Z","shell.execute_reply":"2024-12-14T19:23:18.963276Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntest  = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\n\ntrain.shape, test.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T19:23:21.949518Z","iopub.execute_input":"2024-12-14T19:23:21.950451Z","iopub.status.idle":"2024-12-14T19:23:26.018338Z","shell.execute_reply.started":"2024-12-14T19:23:21.950414Z","shell.execute_reply":"2024-12-14T19:23:26.017430Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train = train.drop('label',axis=1)\ny_train = train['label']\n\nX_test = test.copy()\ny_train.value_counts().plot(kind='bar',legend=None)\nplt.xticks(fontsize=12,rotation=0)\n\nplt.title('Labels and their counts',fontsize=20)\n\nplt.tight_layout()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T19:23:28.637764Z","iopub.execute_input":"2024-12-14T19:23:28.638069Z","iopub.status.idle":"2024-12-14T19:23:29.017251Z","shell.execute_reply.started":"2024-12-14T19:23:28.638043Z","shell.execute_reply":"2024-12-14T19:23:29.016418Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Normalize the data\nX_train = X_train / 255.0\nX_test  = X_test / 255.0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T19:23:32.348013Z","iopub.execute_input":"2024-12-14T19:23:32.348720Z","iopub.status.idle":"2024-12-14T19:23:32.444104Z","shell.execute_reply.started":"2024-12-14T19:23:32.348687Z","shell.execute_reply":"2024-12-14T19:23:32.443425Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Reshape image in 3 dimensions (height = 28px, width = 28px , canal = 1)\nX_train = X_train.values.reshape(-1,28,28,1)\nX_test  = X_test.values.reshape(-1,28,28,1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T19:23:34.592729Z","iopub.execute_input":"2024-12-14T19:23:34.593616Z","iopub.status.idle":"2024-12-14T19:23:34.597668Z","shell.execute_reply.started":"2024-12-14T19:23:34.593579Z","shell.execute_reply":"2024-12-14T19:23:34.596896Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"classes = y_train.unique().astype(str)\n\ni = np.random.choice(len(X_train))\nplt.imshow(X_train[i],cmap='gray')\nplt.title(f\"Label: {y_train[i]}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T19:23:36.625767Z","iopub.execute_input":"2024-12-14T19:23:36.626634Z","iopub.status.idle":"2024-12-14T19:23:36.873122Z","shell.execute_reply.started":"2024-12-14T19:23:36.626596Z","shell.execute_reply":"2024-12-14T19:23:36.872379Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Train-validation splitting\nX_train2,X_validation,y_train2,y_validation = train_test_split(X_train,y_train,\n                                                             test_size=0.3,random_state=0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T19:23:38.108328Z","iopub.execute_input":"2024-12-14T19:23:38.108903Z","iopub.status.idle":"2024-12-14T19:23:38.490986Z","shell.execute_reply.started":"2024-12-14T19:23:38.108869Z","shell.execute_reply":"2024-12-14T19:23:38.490202Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define the optimizer\noptimizer = tf.keras.optimizers.RMSprop(learning_rate=0.001,rho=0.9,epsilon=1e-08,decay=0.0)\n\n# Defining the learning rate reduction routine\nlearning_rate_reduction = ReduceLROnPlateau(monitor='val_accuracy', \n                                            patience=5, \n                                            verbose=1, \n                                            factor=0.5, \n                                            min_lr=0.0000001)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T19:23:47.795956Z","iopub.execute_input":"2024-12-14T19:23:47.796769Z","iopub.status.idle":"2024-12-14T19:23:47.803157Z","shell.execute_reply.started":"2024-12-14T19:23:47.796734Z","shell.execute_reply":"2024-12-14T19:23:47.802452Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Defining the checkpoint callback\ncheckpoint_path = \"results.weights.h5\"\n\ncheckpoint_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path,\n                                      save_best_only=True,\n                                      monitor='val_accuracy',\n                                      mode='max',\n                                      verbose=1,\n                                      save_weights_only=True,\n                                      save_freq='epoch')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T19:23:50.576975Z","iopub.execute_input":"2024-12-14T19:23:50.577594Z","iopub.status.idle":"2024-12-14T19:23:50.582134Z","shell.execute_reply.started":"2024-12-14T19:23:50.577559Z","shell.execute_reply":"2024-12-14T19:23:50.581190Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"datagen = ImageDataGenerator(\n        rotation_range=5,         # Randomly rotate images by up to 5 degrees\n        zoom_range=0.1,            # Randomly zoom images by up to 10%\n        width_shift_range=0.1,     # Randomly shift images horizontally by up to 10% of width\n        height_shift_range=0.1,    # Randomly shift images vertically by up to 10% of height\n        shear_range= 0.1, \n        fill_mode='nearest'\n)\n\ndatagen.fit(X_train2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T19:23:54.723832Z","iopub.execute_input":"2024-12-14T19:23:54.724698Z","iopub.status.idle":"2024-12-14T19:23:54.796903Z","shell.execute_reply.started":"2024-12-14T19:23:54.724649Z","shell.execute_reply":"2024-12-14T19:23:54.796164Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tf.random.set_seed(42)\n\nDROPOUT = 0.3\n\nmodel = tf.keras.models.Sequential([\n        tf.keras.layers.Input(shape=X_train[0].shape), \n        ### First block of layers ###\n        tf.keras.layers.Conv2D(32, (3,3), activation=\"leaky_relu\", padding=\"same\"),\n        tf.keras.layers.BatchNormalization(),\n        tf.keras.layers.Conv2D(32, (3,3), activation=\"leaky_relu\", padding=\"same\"),\n        tf.keras.layers.BatchNormalization(),\n        tf.keras.layers.MaxPooling2D((2,2)),\n        tf.keras.layers.Dropout(DROPOUT),\n        ### Second block of layers ###\n        tf.keras.layers.Conv2D(64, (3,3), activation=\"leaky_relu\", padding=\"same\"),\n        tf.keras.layers.BatchNormalization(),\n        tf.keras.layers.Conv2D(64, (3,3), activation=\"leaky_relu\", padding=\"same\"),\n        tf.keras.layers.BatchNormalization(),\n        tf.keras.layers.MaxPooling2D((2,2)),\n        tf.keras.layers.Dropout(DROPOUT),\n        ### Third block of layers ###\n        tf.keras.layers.Conv2D(128, (3,3), activation=\"leaky_relu\", padding=\"same\"),\n        tf.keras.layers.BatchNormalization(),\n        tf.keras.layers.Conv2D(128, (3,3), activation=\"leaky_relu\", padding=\"same\"),\n        tf.keras.layers.BatchNormalization(),\n        tf.keras.layers.MaxPooling2D((2,2)),\n        tf.keras.layers.Dropout(DROPOUT),\n        ### Fourth block of layers ###\n        tf.keras.layers.Conv2D(256, (3,3), activation=\"leaky_relu\", padding=\"same\"),\n        tf.keras.layers.BatchNormalization(),\n        tf.keras.layers.Conv2D(256, (3,3), activation=\"leaky_relu\", padding=\"same\"),\n        tf.keras.layers.BatchNormalization(),\n        tf.keras.layers.MaxPooling2D((2,2)),\n        tf.keras.layers.Dropout(DROPOUT),\n        ### Final layers before output ###\n        tf.keras.layers.Flatten(),\n        tf.keras.layers.Dropout(DROPOUT),\n        tf.keras.layers.Dense(256, activation=\"leaky_relu\"),\n        tf.keras.layers.BatchNormalization(),\n        tf.keras.layers.Dense(10, activation=\"softmax\")\n])  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T19:23:58.007674Z","iopub.execute_input":"2024-12-14T19:23:58.008249Z","iopub.status.idle":"2024-12-14T19:23:58.212177Z","shell.execute_reply.started":"2024-12-14T19:23:58.008185Z","shell.execute_reply":"2024-12-14T19:23:58.211500Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tf.keras.utils.plot_model(model, show_shapes=True, to_file='model.png')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T19:23:59.874694Z","iopub.execute_input":"2024-12-14T19:23:59.875293Z","iopub.status.idle":"2024-12-14T19:24:01.052459Z","shell.execute_reply.started":"2024-12-14T19:23:59.875260Z","shell.execute_reply":"2024-12-14T19:24:01.051598Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Compile the model\nmodel.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n                 optimizer=optimizer,            \n                 metrics=[\"accuracy\"])\n\n# Fit the model\nhistory = model.fit(datagen.flow(X_train2,y_train2,batch_size=32),\n                      epochs=45,\n                      validation_data=(X_validation,y_validation), \n                      callbacks=[learning_rate_reduction,checkpoint_callback])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T19:24:50.767177Z","iopub.execute_input":"2024-12-14T19:24:50.768016Z","iopub.status.idle":"2024-12-14T19:33:50.779226Z","shell.execute_reply.started":"2024-12-14T19:24:50.767986Z","shell.execute_reply":"2024-12-14T19:33:50.778569Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.plot(history.history['loss'],label='train loss')\nplt.plot(history.history['val_loss'],label='validation loss')\nplt.xlabel('epochs',fontsize=12)\nplt.ylabel('loss',fontsize=12)\nplt.legend()\n\nplt.title('Train and Validation Losses',fontsize=20)\n\nplt.tight_layout()\nplt.gca().set_facecolor('gainsboro')\nplt.gcf().patch.set_facecolor('lightsteelblue')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T19:33:58.409722Z","iopub.execute_input":"2024-12-14T19:33:58.410025Z","iopub.status.idle":"2024-12-14T19:33:58.644818Z","shell.execute_reply.started":"2024-12-14T19:33:58.410001Z","shell.execute_reply":"2024-12-14T19:33:58.644026Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.plot(history.history['accuracy'],label='train accuracy')\nplt.plot(history.history['val_accuracy'],label='validation accuracy')\nplt.xlabel('epochs',fontsize=12)\nplt.ylabel('accuracy',fontsize=12)\nplt.legend()\n\nplt.title('Train and Validation Accuracies',fontsize=20)\n\nplt.tight_layout()\nplt.gca().set_facecolor('gainsboro')\nplt.gcf().patch.set_facecolor('lightsteelblue')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T19:34:00.548445Z","iopub.execute_input":"2024-12-14T19:34:00.548790Z","iopub.status.idle":"2024-12-14T19:34:00.919842Z","shell.execute_reply.started":"2024-12-14T19:34:00.548761Z","shell.execute_reply":"2024-12-14T19:34:00.918972Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"score = model.evaluate(X_validation,y_validation,verbose=0)\nprint('Test Loss: {:.4f}'.format(score[0]))\nprint('Test Accuracy: {:.4f}'.format(score[1]))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T19:34:11.073446Z","iopub.execute_input":"2024-12-14T19:34:11.073793Z","iopub.status.idle":"2024-12-14T19:34:12.219227Z","shell.execute_reply.started":"2024-12-14T19:34:11.073763Z","shell.execute_reply":"2024-12-14T19:34:12.218358Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.load_weights(checkpoint_path)\nloss,acc = model.evaluate(X_validation,y_validation,verbose=0)\n\nprint('Restored model')\nprint('Test Loss: {:.4f}'.format(loss))\nprint('Test Accuracy: {:.4f}'.format(acc))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T19:34:13.986125Z","iopub.execute_input":"2024-12-14T19:34:13.986548Z","iopub.status.idle":"2024-12-14T19:34:15.241790Z","shell.execute_reply.started":"2024-12-14T19:34:13.986518Z","shell.execute_reply":"2024-12-14T19:34:15.240965Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def make_confusion_matrix(y_true,y_pred,classes=None,figsize=(10,10),text_size=15):\n    \"\"\"Makes a labelled confusion matrix comparing predictions and ground truth labels.\n\n       If classes is passed, confusion matrix will be labelled, if not, integer class values\n       will be used.\n\n       Args:\n         y_true: Array of truth labels (must be same shape as y_pred).\n         y_pred: Array of predicted labels (must be same shape as y_true).\n         classes: Array of class labels (e.g. string form). If `None`, integer labels are used.\n         figsize: Size of output figure (default=(10,10)).\n         text_size: Size of output figure text (default=15).\n\n       Returns:\n         A labelled confusion matrix plot comparing y_true and y_pred.\n\n       Example usage:\n         make_confusion_matrix(y_true=test_labels, # ground truth test labels\n                          y_pred=y_preds, # predicted labels\n                          classes=class_names, # array of class label names\n                          figsize=(15,15),\n                          text_size=10)\n    \"\"\"\n    # Create the confustion matrix\n    cm = confusion_matrix(y_true, y_pred)\n    cm_norm = cm.astype(\"float\") / cm.sum(axis=1)[:,np.newaxis] # normalize it\n    n_classes = cm.shape[0] # find the number of classes we're dealing with\n\n    # Plot the figure and make it pretty\n    fig, ax = plt.subplots(figsize=figsize)\n    cax = ax.matshow(cm, cmap=plt.cm.Blues) # colors will represent how 'correct' a class is, darker == better\n    fig.colorbar(cax)\n\n    # Are there a list of classes?\n    if classes:\n        labels = classes\n    else:\n        labels = np.arange(cm.shape[0])\n\n    # Label the axes\n    ax.set(title=\"Confusion Matrix\",\n         xlabel=\"Predicted label\",\n         ylabel=\"True label\",\n         xticks=np.arange(n_classes), # create enough axis slots for each class\n         yticks=np.arange(n_classes),\n         xticklabels=labels, # axes will labeled with class names (if they exist) or ints\n         yticklabels=labels)\n\n  # Make x-axis labels appear on bottom\n    ax.xaxis.set_label_position(\"bottom\")\n    ax.xaxis.tick_bottom()\n\n  # Set the threshold for different colors\n    threshold = (cm.max() + cm.min()) / 2.\n\n  # Plot the text on each cell\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, f\"{cm[i,j]} ({cm_norm[i,j]*100:.1f}%)\",\n             horizontalalignment=\"center\",\n             color=\"white\" if cm[i,j] > threshold else \"black\",\n             size=text_size)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T19:34:16.895271Z","iopub.execute_input":"2024-12-14T19:34:16.895605Z","iopub.status.idle":"2024-12-14T19:34:16.904114Z","shell.execute_reply.started":"2024-12-14T19:34:16.895577Z","shell.execute_reply":"2024-12-14T19:34:16.903278Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# These are the predicted class probabilities (probs)\ny_probs = model.predict(X_validation)\n\n# I am printing the first prediction.\ny_probs[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T19:34:18.678427Z","iopub.execute_input":"2024-12-14T19:34:18.679329Z","iopub.status.idle":"2024-12-14T19:34:20.800587Z","shell.execute_reply.started":"2024-12-14T19:34:18.679271Z","shell.execute_reply":"2024-12-14T19:34:20.799711Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# The probs need to be converted in the predicted labels \n# by choosing the highest probability in the list\npred_labels = y_probs.argmax(axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T19:34:22.157740Z","iopub.execute_input":"2024-12-14T19:34:22.158557Z","iopub.status.idle":"2024-12-14T19:34:22.162721Z","shell.execute_reply.started":"2024-12-14T19:34:22.158525Z","shell.execute_reply":"2024-12-14T19:34:22.161606Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"classes = list(y_train.unique())\nclasses.sort()\nclasses = [str(x) for x in classes]\n\nclasses","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T19:34:24.286941Z","iopub.execute_input":"2024-12-14T19:34:24.287722Z","iopub.status.idle":"2024-12-14T19:34:24.293990Z","shell.execute_reply.started":"2024-12-14T19:34:24.287690Z","shell.execute_reply":"2024-12-14T19:34:24.293279Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"make_confusion_matrix(y_true=y_validation,\n                      y_pred=pred_labels,\n                      classes=classes,\n                      figsize=(15,15),\n                      text_size=10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T19:34:28.446481Z","iopub.execute_input":"2024-12-14T19:34:28.447462Z","iopub.status.idle":"2024-12-14T19:34:29.121903Z","shell.execute_reply.started":"2024-12-14T19:34:28.447421Z","shell.execute_reply":"2024-12-14T19:34:29.121051Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"misclassified_idx = np.where(y_validation != pred_labels)[0]\n\nplt.figure(figsize=(10,10))\n\nfor i in range(9):\n    idx = np.random.choice(misclassified_idx)\n    plt.subplot(3,3,i+1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.grid(False)\n    plt.imshow(X_validation[idx].reshape((28,28)),cmap='gray')\n    label_index = int(pred_labels[idx])\n    plt.title(f'Pred label: {classes[pred_labels[idx]]}',fontsize=8)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T19:34:32.359892Z","iopub.execute_input":"2024-12-14T19:34:32.360774Z","iopub.status.idle":"2024-12-14T19:34:33.088015Z","shell.execute_reply.started":"2024-12-14T19:34:32.360736Z","shell.execute_reply":"2024-12-14T19:34:33.087178Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# predict results\nresults = model.predict(X_test)\n\n# select the indix with the maximum probability\nresults = np.argmax(results,axis = 1)\n\nresults = pd.Series(results,name=\"Label\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T19:34:38.533588Z","iopub.execute_input":"2024-12-14T19:34:38.534304Z","iopub.status.idle":"2024-12-14T19:34:40.797329Z","shell.execute_reply.started":"2024-12-14T19:34:38.534269Z","shell.execute_reply":"2024-12-14T19:34:40.796530Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\n\nsubmission.to_csv(\"cnn_mnist_results.csv\",index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T19:34:43.250357Z","iopub.execute_input":"2024-12-14T19:34:43.250703Z","iopub.status.idle":"2024-12-14T19:34:43.276018Z","shell.execute_reply.started":"2024-12-14T19:34:43.250673Z","shell.execute_reply":"2024-12-14T19:34:43.275381Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}