{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom skimage import io\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom keras.optimizers import RMSprop\nfrom tensorflow.keras.models import Sequential\nfrom tensorboard.plugins.hparams import api as hp\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ReduceLROnPlateau, EarlyStopping\nfrom tensorflow.keras.layers import Dense, Conv2D, Flatten, Dropout, MaxPooling2D\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.config.experimental.list_physical_devices('GPU')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preparing the data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/digit-recognizer/train.csv')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We reshape the data to have images ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"Y_train = train['label'].to_numpy()\nX_train = train.iloc[:, 1:].to_numpy()\ndel train\nX_train = X_train.reshape(-1,28,28,1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(Y_train)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We have roughly the same number of examples for each category","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test = pd.read_csv('../input/digit-recognizer/test.csv').to_numpy()\nX_test = X_test.reshape(-1,28,28,1)\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"we normalize the images by dividing by 255","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = X_train / 255.0\nX_test = X_test / 255.0","execution_count":null,"outputs":[]},{"metadata":{"tags":[],"trusted":true},"cell_type":"code","source":"from keras.utils.np_utils import to_categorical \nY_train = to_categorical(Y_train, num_classes = 10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"we split the data into a train and validation set","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.1, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"io.imshow(X_train[0][:,:,0])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Parameters tuning","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"We reduce the learning rate in our models if the validation loss is not reducing","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 32\nepochs = 10\nlearning_rate_reduction = ReduceLROnPlateau(monitor='val_loss', factor=0.2,patience=5, min_lr=0.001)\nearly_stopping = EarlyStopping(\n    monitor='val_loss', min_delta=0.001, patience=5, verbose=0, mode='auto',\n    baseline=None, restore_best_weights=True\n)\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We use data augmentation in order to avoid overfitting","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"datagen = ImageDataGenerator(\n        rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n        zoom_range = 0.1, # Randomly zoom image \n        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n        )\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can use tensorboard to keep track of the tuning","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%load_ext tensorboard","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm -rf ./logs/ ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Here I chose to optimize the number of filters, the number of units, the kernel size and the optimizer. We could try to optimize other parameters like batch size, activation, dropout. However we need to keep in mind that this takes a lot of time.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"HP_NUM_FILTERS = hp.HParam('num_filters', hp.Discrete([32, 64]))\nHP_NUM_UNITS = hp.HParam('num_units', hp.Discrete([256, 512]))\nHP_KERNEL_SIZE = hp.HParam('kernel_size', hp.Discrete([3, 5]))\nHP_OPTIMIZER = hp.HParam('optimizer', hp.Discrete(['adam', 'sgd', 'RMSprop']))\n\nMETRIC_ACCURACY = 'accuracy'\n\nwith tf.summary.create_file_writer('logs/hparam_tuning').as_default():\n  hp.hparams_config(\n    hparams=[HP_NUM_FILTERS, HP_NUM_UNITS, HP_OPTIMIZER, HP_KERNEL_SIZE],\n    metrics=[hp.Metric(METRIC_ACCURACY, display_name='Accuracy')],\n  )\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_val_model(logdir, hparams):\n    batch_size = 32\n    model = tf.keras.models.Sequential([\n        Conv2D(hparams[HP_NUM_FILTERS], hparams[HP_KERNEL_SIZE], padding='same', activation='relu', input_shape=(28, 28, 1)),\n        Conv2D(hparams[HP_NUM_FILTERS], hparams[HP_KERNEL_SIZE], padding='same', activation='relu'),\n        MaxPooling2D(),\n        Dropout(0.25),\n\n        Conv2D(hparams[HP_NUM_FILTERS]*2, hparams[HP_KERNEL_SIZE], padding='same', activation='relu'),\n        Conv2D(hparams[HP_NUM_FILTERS]*2, hparams[HP_KERNEL_SIZE], padding='same', activation='relu'),\n        MaxPooling2D(strides=(2,2)),\n        Dropout(0.25),\n\n        Flatten(),\n        Dense(hparams[HP_NUM_UNITS], activation='relu'),\n        Dropout(0.5),\n        Dense(10, activation='softmax')\n    ])\n\n    model.compile(\n        optimizer=hparams[HP_OPTIMIZER],\n        loss='categorical_crossentropy',\n        metrics=['accuracy'],\n    )\n\n    history = model.fit( datagen.flow( X_train,Y_train, batch_size=batch_size), verbose=0,\n                            epochs = 10, validation_data = (X_val,Y_val),\n                            steps_per_epoch=X_train.shape[0] // batch_size\n                            , callbacks=[learning_rate_reduction,\n                             tf.keras.callbacks.TensorBoard(logdir)\n                                        ]\n            ) \n            \n    accuracy = np.array(history.history['val_accuracy']).max()\n    print(f'best accuracy: {round(accuracy,4)}')\n    return accuracy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run(run_dir, hparams):\n  with tf.summary.create_file_writer(run_dir).as_default():\n    hp.hparams(hparams)  # record the values used in this trial\n    accuracy = train_val_model(run_dir, hparams)\n    tf.summary.scalar(METRIC_ACCURACY, accuracy, step=1)\n    return accuracy ","execution_count":null,"outputs":[]},{"metadata":{"tags":[],"trusted":true},"cell_type":"code","source":"from itertools import product\n\nsession_num = 0\nresults = []\nfor num_filters, num_units, kernel_size, optimizer in product(HP_NUM_FILTERS.domain.values, \n                                                HP_NUM_UNITS.domain.values,\n                                                HP_KERNEL_SIZE.domain.values,\n                                                HP_OPTIMIZER.domain.values): \n    hparams = {\n        HP_NUM_FILTERS: num_filters,\n        HP_NUM_UNITS: num_units,\n        HP_KERNEL_SIZE: kernel_size,\n        HP_OPTIMIZER: optimizer,\n    }\n\n    run_name = f'run{session_num}_{str({h.name: hparams[h] for h in hparams})}' \n    print(f'---{run_name}')\n    accuracy = run('logs/hparam_tuning/' + run_name, hparams)\n    session_num += 1\n    \n    results.append([num_filters, num_units, kernel_size, optimizer, accuracy])\n\nresults = pd.DataFrame(results, columns=['num_filters', 'num_units', 'kernel_size', 'optimizer', 'accuracy'])\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We obtain the following best results","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"results.sort_values(by='accuracy', ascending=False).head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axs = plt.subplots(ncols=4, figsize=(20,5), sharey=True)\nfor i, col in enumerate(results.columns[0:-1]):\n    sns.scatterplot(x=col, y='accuracy', data=results, ax=axs[i])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can see that the optimizer makes the biggest difference, with the ```adam``` optimizer being the best. However we can't see a clear difference for the other parameters.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Using tensorboard we can see more details for each run.","execution_count":null},{"metadata":{"tags":[],"trusted":true},"cell_type":"code","source":"%tensorboard --logdir logs/hparam_tuning","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Best model performance","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Now that we determined the best parameters, we can see how our model performs.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"best_params = results.iloc[results['accuracy'].argmax()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_params","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = tf.keras.models.Sequential([\n        Conv2D(best_params['num_filters'], int(best_params['kernel_size']), padding='same', activation='relu', input_shape=(28, 28, 1)),\n        Conv2D(best_params['num_filters'], int(best_params['kernel_size']), padding='same', activation='relu'),\n        MaxPooling2D(),\n        Dropout(0.25),\n\n        Conv2D(best_params['num_filters']*2, int(best_params['kernel_size']), padding='same', activation='relu'),\n        Conv2D(best_params['num_filters']*2, int(best_params['kernel_size']), padding='same', activation='relu'),\n        MaxPooling2D(strides=(2,2)),\n        Dropout(0.25),\n\n        Flatten(),\n        Dense(best_params['num_units'], activation='relu'),\n        Dropout(0.5),\n        Dense(10, activation='softmax')\n    ])\n\nmodel.compile(\n        optimizer=best_params['optimizer'],\n        loss='categorical_crossentropy',\n        metrics=['accuracy'],\n    )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"With the ```early_stopping``` callback we can avoid overfitting.","execution_count":null},{"metadata":{"tags":[],"trusted":true},"cell_type":"code","source":"\nhistory = model.fit(datagen.flow(X_train,Y_train, batch_size=batch_size),\n                              epochs = 20, validation_data = (X_val,Y_val),\n                              steps_per_epoch=X_train.shape[0] // batch_size\n                              , callbacks=[learning_rate_reduction, early_stopping])","execution_count":null,"outputs":[]},{"metadata":{"tags":[],"trusted":true},"cell_type":"code","source":"def plot_fit(history):\n    fig, axes = plt.subplots(1,2, figsize=(10,5))\n    axes[0].plot(history.history['loss'], label='Train loss')\n    axes[0].plot(history.history['val_loss'], label='Validation loss')\n    axes[1].plot(history.history['accuracy'], label='Train accuracy')\n    axes[1].plot(history.history['val_accuracy'], label='Validation accuracy')\n    axes[0].legend()\n    axes[1].legend()\n    plt.show()\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_fit(history)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"As we can see, the validation accuracy is higher than the train accuracy, and the validation loss does not increase in the end. This is a good indicator that there is no overfitting. ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import ConfusionMatrixDisplay\nval_predictions = model.predict(X_val)\ncm = confusion_matrix(np.argmax(Y_val, axis=1), np.argmax(val_predictions, axis=1), normalize='true')\n\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm.round(2), display_labels=np.arange(10))\n\ndisp = disp.plot()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"With the confusion matrix we can know the predictions for each category.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_image(predictions_array, img, true_prediction):\n  plt.grid(False)\n  plt.xticks([])\n  plt.yticks([])\n\n  plt.imshow(img, cmap=plt.cm.binary)\n\n  predicted_label = np.argmax(predictions_array)\n\n  plt.xlabel(f\"Pred: {predicted_label} ({100*np.max(predictions_array):2.0f}%) | True: {true_prediction}\")\n\ndef plot_value_array(predictions_array):\n  plt.grid(False)\n  plt.xticks(range(10))\n  plt.yticks([])\n  thisplot = plt.bar(range(10), predictions_array, color=\"#777777\")\n  plt.ylim([0, 1])\n  predicted_label = np.argmax(predictions_array)\n\n  thisplot[predicted_label].set_color('red')\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"wrong_predictions = np.argwhere(np.argmax(Y_val, axis=1) !=np.argmax(val_predictions, axis=1))\n\nnum_rows = 5\nnum_cols = 3\nnum_images = num_rows*num_cols\nplt.figure(figsize=(2*2*num_cols, 2*num_rows))\nfor i, image_idx in enumerate(wrong_predictions[0:num_images].reshape(-1)):\n  plt.subplot(num_rows, 2*num_cols, 2*i+1)\n  plot_image(val_predictions[image_idx], X_val[image_idx].reshape(28,28), np.argmax(Y_val[image_idx]))\n  plt.subplot(num_rows, 2*num_cols, 2*i+2)\n  plot_value_array(val_predictions[image_idx])\nplt.tight_layout()\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"As we can see the model didn't perform well for some confusing examples, wich are hard to predict even for humans.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Training on the whole dataset","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"In order to have the best predictions, we can use all our training examples to fit the model that will be used with test set.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model = tf.keras.models.Sequential([\n        Conv2D(best_params['num_filters'], int(best_params['kernel_size']), padding='same', activation='relu', input_shape=(28, 28, 1)),\n        Conv2D(best_params['num_filters'], int(best_params['kernel_size']), padding='same', activation='relu'),\n        MaxPooling2D(),\n        Dropout(0.25),\n\n        Conv2D(best_params['num_filters']*2, int(best_params['kernel_size']), padding='same', activation='relu'),\n        Conv2D(best_params['num_filters']*2, int(best_params['kernel_size']), padding='same', activation='relu'),\n        MaxPooling2D(strides=(2,2)),\n        Dropout(0.25),\n\n        Flatten(),\n        Dense(best_params['num_units'], activation='relu'),\n        Dropout(0.5),\n        Dense(10, activation='softmax')\n    ])\n\nmodel.compile(\n        optimizer=best_params['optimizer'],\n        loss='categorical_crossentropy',\n        metrics=['accuracy'],\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = np.concatenate((X_train,X_val))\nY_train = np.concatenate((Y_train,Y_val))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Here I set the number of epochs to 14, since that's when the fitting stopped with the early stopping.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\nmodel.fit(datagen.flow(X_train,Y_train, batch_size=batch_size),\n                              epochs = 14,\n                              steps_per_epoch=X_train.shape[0] // batch_size\n                              , callbacks=[learning_rate_reduction])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = model.predict(X_test)\npredictions = np.argmax(predictions,axis=1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submissions=pd.DataFrame({\"ImageId\": list(range(1,len(predictions)+1)),\n                         \"Label\": predictions})\n\nsubmissions.to_csv(\"submission.csv\", index=False, header=True)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}