{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30715,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-06-04T08:45:16.906859Z","iopub.execute_input":"2024-06-04T08:45:16.907813Z","iopub.status.idle":"2024-06-04T08:45:16.924859Z","shell.execute_reply.started":"2024-06-04T08:45:16.907770Z","shell.execute_reply":"2024-06-04T08:45:16.923454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the data\ndf = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")\nsample = pd.read_csv(\"/kaggle/input/digit-recognizer/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:45:16.928244Z","iopub.execute_input":"2024-06-04T08:45:16.928667Z","iopub.status.idle":"2024-06-04T08:45:22.188109Z","shell.execute_reply.started":"2024-06-04T08:45:16.928621Z","shell.execute_reply":"2024-06-04T08:45:22.186871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport seaborn as sns\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\nimport itertools\n\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D\nfrom tensorflow.keras.optimizers import RMSprop\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:45:22.189597Z","iopub.execute_input":"2024-06-04T08:45:22.189945Z","iopub.status.idle":"2024-06-04T08:45:22.198654Z","shell.execute_reply.started":"2024-06-04T08:45:22.189917Z","shell.execute_reply":"2024-06-04T08:45:22.197315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:45:22.202080Z","iopub.execute_input":"2024-06-04T08:45:22.202577Z","iopub.status.idle":"2024-06-04T08:45:22.242482Z","shell.execute_reply.started":"2024-06-04T08:45:22.202507Z","shell.execute_reply":"2024-06-04T08:45:22.240925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:45:22.244496Z","iopub.execute_input":"2024-06-04T08:45:22.245023Z","iopub.status.idle":"2024-06-04T08:45:22.274186Z","shell.execute_reply.started":"2024-06-04T08:45:22.244978Z","shell.execute_reply":"2024-06-04T08:45:22.272797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:45:22.276040Z","iopub.execute_input":"2024-06-04T08:45:22.276498Z","iopub.status.idle":"2024-06-04T08:45:22.290843Z","shell.execute_reply.started":"2024-06-04T08:45:22.276443Z","shell.execute_reply":"2024-06-04T08:45:22.289277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = df[\"label\"]\nx = df.drop(labels = [\"label\"],axis = 1) ","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:45:22.292692Z","iopub.execute_input":"2024-06-04T08:45:22.293137Z","iopub.status.idle":"2024-06-04T08:45:22.446272Z","shell.execute_reply.started":"2024-06-04T08:45:22.293098Z","shell.execute_reply":"2024-06-04T08:45:22.444904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reshape image in 3 dimensions (height = 28px, width = 28px , canal = 1)\nx = x.values.reshape(-1,28,28,1)\ntest = test.values.reshape(-1,28,28,1)","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:45:22.448104Z","iopub.execute_input":"2024-06-04T08:45:22.448557Z","iopub.status.idle":"2024-06-04T08:45:22.455151Z","shell.execute_reply.started":"2024-06-04T08:45:22.448507Z","shell.execute_reply":"2024-06-04T08:45:22.453740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Normalize the data\nx = x / 255.0\ntest = test / 255.0","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:45:22.460016Z","iopub.execute_input":"2024-06-04T08:45:22.460369Z","iopub.status.idle":"2024-06-04T08:45:22.651760Z","shell.execute_reply.started":"2024-06-04T08:45:22.460340Z","shell.execute_reply":"2024-06-04T08:45:22.650495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(x[15][:,:,0]) # the 15th image\nprint(y[15])","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:45:22.654327Z","iopub.execute_input":"2024-06-04T08:45:22.654831Z","iopub.status.idle":"2024-06-04T08:45:22.883741Z","shell.execute_reply.started":"2024-06-04T08:45:22.654788Z","shell.execute_reply":"2024-06-04T08:45:22.882178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Encode labels to one hot vectors (ex : 2 -> [0,0,1,0,0,0,0,0,0,0])\ny = to_categorical(y, num_classes = 10)","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:45:22.885294Z","iopub.execute_input":"2024-06-04T08:45:22.885687Z","iopub.status.idle":"2024-06-04T08:45:22.893523Z","shell.execute_reply.started":"2024-06-04T08:45:22.885655Z","shell.execute_reply":"2024-06-04T08:45:22.891791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split the train and the validation set for the fitting\nx_train, x_val, y_train, y_val = train_test_split(x, y, test_size = 0.1, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:45:22.895836Z","iopub.execute_input":"2024-06-04T08:45:22.896295Z","iopub.status.idle":"2024-06-04T08:45:23.352130Z","shell.execute_reply.started":"2024-06-04T08:45:22.896260Z","shell.execute_reply":"2024-06-04T08:45:23.350798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set the CNN model \n# CNN architecture is In -> [[Conv2D->relu]*2 -> MaxPooling2D -> Dropout]*2 -> Flatten -> Dense -> Dropout -> Out\n\nmodel = Sequential()\n\nmodel.add(Conv2D(filters = 32, kernel_size = (5,5), padding='Same', activation='relu', input_shape=(28, 28, 1)))\nmodel.add(Conv2D(filters = 32, kernel_size = (5,5), padding='Same', activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Conv2D(filters = 64, kernel_size = (3,3), padding='Same', activation='relu'))\nmodel.add(Conv2D(filters = 64, kernel_size = (3,3), padding='Same', activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Flatten())\nmodel.add(Dense(256, activation=\"relu\"))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(10, activation=\"softmax\"))\n\noptimizer = RMSprop(learning_rate=0.001, rho=0.9, epsilon=1e-08)\n\nmodel.compile(optimizer=optimizer, loss=\"categorical_crossentropy\", metrics=[\"accuracy\"])","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:45:23.353986Z","iopub.execute_input":"2024-06-04T08:45:23.354478Z","iopub.status.idle":"2024-06-04T08:45:23.534192Z","shell.execute_reply.started":"2024-06-04T08:45:23.354436Z","shell.execute_reply":"2024-06-04T08:45:23.532954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Its better to have a decreasing learning rate during the training to reach minimum loss.\n\n# To keep the advantage of the fast computation time with a high LR, \n# I decreased the LR dynamically every X steps (epochs) when accuracy is not improved.\n\n# With the ReduceLROnPlateau function from Keras.callbacks, \n# I choose to reduce the LR by half if the accuracy is not improved after 3 epochs.\n\n# Set a learning rate annealer\nlearning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', patience=3, verbose=1, factor=0.5, min_lr=0.00001)","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:45:23.535530Z","iopub.execute_input":"2024-06-04T08:45:23.535888Z","iopub.status.idle":"2024-06-04T08:45:23.542341Z","shell.execute_reply.started":"2024-06-04T08:45:23.535859Z","shell.execute_reply":"2024-06-04T08:45:23.541190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DATA AUGMENTATION\n# In order to avoid overfitting problem, we need to expand artificially our handwritten digit dataset. \n\n# Lets try without data augmentation\n# history = model.fit(x_train, y_train, batch_size = 64, epochs = 1, validation_data = (x_val, y_val), verbose = 1)","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:45:23.543785Z","iopub.execute_input":"2024-06-04T08:45:23.544141Z","iopub.status.idle":"2024-06-04T08:45:23.553320Z","shell.execute_reply.started":"2024-06-04T08:45:23.544112Z","shell.execute_reply":"2024-06-04T08:45:23.551816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Without data augmentation I obtained a val accuracy of 0.97.\n\ndatagen = ImageDataGenerator(\n        featurewise_center=False,\n        samplewise_center=False,\n        featurewise_std_normalization=False,\n        samplewise_std_normalization=False,\n        zca_whitening=False,\n        rotation_range=10,  # Randomly rotate some training images by 10 degrees\n        zoom_range = 0.1, # Randomly Zoom by 10% some training images\n        width_shift_range=0.1,  # Randomly shift images horizontally by 10% of the width\n        height_shift_range=0.1,  # Randomly shift images vertically by 10% of the height\n        vertical_flip=False) # (I did not apply a vertical_flip nor horizontal_flip because of symetrical numbers 6 and 9)\n\ndatagen.fit(x_train)","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:45:23.555593Z","iopub.execute_input":"2024-06-04T08:45:23.556001Z","iopub.status.idle":"2024-06-04T08:45:23.661913Z","shell.execute_reply.started":"2024-06-04T08:45:23.555970Z","shell.execute_reply":"2024-06-04T08:45:23.660605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(\n    datagen.flow(x_train, y_train, batch_size=80),  # The generator yielding batches of training data\n    epochs=10,  # Total number of epochs to train the model\n    validation_data=(x_val, y_val),  # Validation data to evaluate the model after each epoch\n    verbose=1,  # Verbosity mode (0 = silent, 1 = progress bar, 2 = one line per epoch)\n    steps_per_epoch=x_train.shape[0] // 80,  # Number of batches to process per epoch\n    callbacks=[learning_rate_reduction],  # Callbacks to apply at the end of each epoch\n)","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:45:23.663341Z","iopub.execute_input":"2024-06-04T08:45:23.663696Z","iopub.status.idle":"2024-06-04T08:57:21.591746Z","shell.execute_reply.started":"2024-06-04T08:45:23.663665Z","shell.execute_reply":"2024-06-04T08:57:21.590491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# MODEL EVALUATION","metadata":{}},{"cell_type":"code","source":"# Plot the loss and accuracy curves for training and validation \n\n\nfig, ax = plt.subplots(2, 1)  # Create a figure with 2 subplots\nax[0].plot(history.history['loss'], color='b', label=\"Training loss\")\nax[0].plot(history.history['val_loss'], color='r', label=\"Validation loss\")\nax[0].legend(loc='best', shadow=True)  # Add legend to first subplot\n\nax[1].plot(history.history['accuracy'], color='b', label=\"Training accuracy\")\nax[1].plot(history.history['val_accuracy'], color='r', label=\"Validation accuracy\")\nax[1].legend(loc='best', shadow=True)  # Add legend to second subplot\n\nplt.show()  # Display the plot\n","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:58:59.924045Z","iopub.execute_input":"2024-06-04T08:58:59.924531Z","iopub.status.idle":"2024-06-04T08:59:00.354855Z","shell.execute_reply.started":"2024-06-04T08:58:59.924498Z","shell.execute_reply":"2024-06-04T08:59:00.353576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# looks like underfitting but its ok","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:59:20.850559Z","iopub.execute_input":"2024-06-04T08:59:20.851525Z","iopub.status.idle":"2024-06-04T08:59:20.857854Z","shell.execute_reply.started":"2024-06-04T08:59:20.851485Z","shell.execute_reply":"2024-06-04T08:59:20.856474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Look at confusion matrix \n\ndef plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, cm[i, j],\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n\n# Predict the values from the validation dataset\nY_pred = model.predict(x_val)\n# Convert predictions classes to one hot vectors \nY_pred_classes = np.argmax(Y_pred,axis = 1) \n# Convert validation observations to one hot vectors\nY_true = np.argmax(y_val,axis = 1) \n# compute the confusion matrix\nconfusion_mtx = confusion_matrix(Y_true, Y_pred_classes) \n# plot the confusion matrix\nplot_confusion_matrix(confusion_mtx, classes = range(10)) ","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:59:22.914613Z","iopub.execute_input":"2024-06-04T08:59:22.915119Z","iopub.status.idle":"2024-06-04T08:59:27.726184Z","shell.execute_reply.started":"2024-06-04T08:59:22.915082Z","shell.execute_reply":"2024-06-04T08:59:27.724653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# looks like 8 was the hardest to predict ","metadata":{"execution":{"iopub.status.busy":"2024-06-04T08:59:52.408172Z","iopub.execute_input":"2024-06-04T08:59:52.408668Z","iopub.status.idle":"2024-06-04T08:59:52.414227Z","shell.execute_reply.started":"2024-06-04T08:59:52.408630Z","shell.execute_reply":"2024-06-04T08:59:52.412884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Display some error results \n\n# Errors are difference between predicted labels and true labels\nerrors = (Y_pred_classes - Y_true != 0)\n\nY_pred_classes_errors = Y_pred_classes[errors]\nY_pred_errors = Y_pred[errors]\nY_true_errors = Y_true[errors]\nX_val_errors = x_val[errors]\n\ndef display_errors(errors_index,img_errors,pred_errors, obs_errors):\n    \"\"\" This function shows 6 images with their predicted and real labels\"\"\"\n    n = 0\n    nrows = 2\n    ncols = 3\n    fig, ax = plt.subplots(nrows,ncols,sharex=True,sharey=True)\n    for row in range(nrows):\n        for col in range(ncols):\n            error = errors_index[n]\n            ax[row,col].imshow((img_errors[error]).reshape((28,28)))\n            ax[row,col].set_title(\"Predicted label :{}\\nTrue label :{}\".format(pred_errors[error],obs_errors[error]))\n            n += 1\n\n# Probabilities of the wrong predicted numbers\nY_pred_errors_prob = np.max(Y_pred_errors,axis = 1)\n\n# Predicted probabilities of the true values in the error set\ntrue_prob_errors = np.diagonal(np.take(Y_pred_errors, Y_true_errors, axis=1))\n\n# Difference between the probability of the predicted label and the true label\ndelta_pred_true_errors = Y_pred_errors_prob - true_prob_errors\n\n# Sorted list of the delta prob errors\nsorted_dela_errors = np.argsort(delta_pred_true_errors)\n\n# Top 6 errors \nmost_important_errors = sorted_dela_errors[-6:]\n\n# Show the top 6 errors\ndisplay_errors(most_important_errors, X_val_errors, Y_pred_classes_errors, Y_true_errors)","metadata":{"execution":{"iopub.status.busy":"2024-06-04T09:00:05.057878Z","iopub.execute_input":"2024-06-04T09:00:05.058366Z","iopub.status.idle":"2024-06-04T09:00:06.270313Z","shell.execute_reply.started":"2024-06-04T09:00:05.058321Z","shell.execute_reply":"2024-06-04T09:00:06.269122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# the errors aren't that big of a deal","metadata":{"execution":{"iopub.status.busy":"2024-06-04T09:00:43.433499Z","iopub.execute_input":"2024-06-04T09:00:43.434051Z","iopub.status.idle":"2024-06-04T09:00:43.440377Z","shell.execute_reply.started":"2024-06-04T09:00:43.434014Z","shell.execute_reply":"2024-06-04T09:00:43.438640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predict results\nresults = model.predict(test)\n\n# select the indix with the maximum probability\nresults = np.argmax(results,axis = 1)\n\nresults = pd.Series(results,name=\"Label\")\n\nsubmission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\n\nsubmission.to_csv(\"cnn_mnist_datagen.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2024-06-04T09:00:45.759086Z","iopub.execute_input":"2024-06-04T09:00:45.760491Z","iopub.status.idle":"2024-06-04T09:01:11.535891Z","shell.execute_reply.started":"2024-06-04T09:00:45.760441Z","shell.execute_reply":"2024-06-04T09:01:11.534428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('digit_model.h5')","metadata":{"execution":{"iopub.status.busy":"2024-06-04T09:01:29.017353Z","iopub.execute_input":"2024-06-04T09:01:29.018553Z","iopub.status.idle":"2024-06-04T09:01:29.083761Z","shell.execute_reply.started":"2024-06-04T09:01:29.018486Z","shell.execute_reply":"2024-06-04T09:01:29.082359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}