{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30636,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nimport pandas as pd","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-01-25T08:43:45.572260Z","iopub.execute_input":"2024-01-25T08:43:45.572830Z","iopub.status.idle":"2024-01-25T08:43:47.430318Z","shell.execute_reply.started":"2024-01-25T08:43:45.572803Z","shell.execute_reply":"2024-01-25T08:43:47.429383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Import and explore data","metadata":{}},{"cell_type":"code","source":"# load train data\ndf_train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\n\nX = df_train.drop('label', axis=1)\ny = df_train['label']","metadata":{"execution":{"iopub.status.busy":"2024-01-25T08:46:36.358741Z","iopub.execute_input":"2024-01-25T08:46:36.359212Z","iopub.status.idle":"2024-01-25T08:46:38.961125Z","shell.execute_reply.started":"2024-01-25T08:46:36.359172Z","shell.execute_reply":"2024-01-25T08:46:38.960059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# prepare data for ML, by splitting them in a train and validation dataset\n\nfrom sklearn.model_selection import train_test_split\n\nX_train, X_validation, y_train, y_validation = train_test_split(X, y, \n                                                                test_size=0.1, \n                                                                random_state=42,\n                                                                stratify=y)","metadata":{"execution":{"iopub.status.busy":"2024-01-25T08:48:58.015811Z","iopub.execute_input":"2024-01-25T08:48:58.016481Z","iopub.status.idle":"2024-01-25T08:48:58.216114Z","shell.execute_reply.started":"2024-01-25T08:48:58.016449Z","shell.execute_reply":"2024-01-25T08:48:58.215053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def apply_preprocessing(X, y):\n\n    # scale the features\n    print(np.max(X))\n    X_scaled = np.array(X/255)\n    X_tensor = X_scaled.reshape(len(X_scaled), 28, 28, 1)\n    \n    # apply one-hot encoding to labels\n    y_onehot = pd.get_dummies(y)\n    \n    return X_tensor, y_onehot\n\n\nX_train_tensor, y_train_onehot = apply_preprocessing(X_train, y_train)\nX_validation_tensor, y_validation_onehot = apply_preprocessing(X_validation, y_validation)","metadata":{"execution":{"iopub.status.busy":"2024-01-25T08:48:59.149860Z","iopub.execute_input":"2024-01-25T08:48:59.150225Z","iopub.status.idle":"2024-01-25T08:48:59.334246Z","shell.execute_reply.started":"2024-01-25T08:48:59.150195Z","shell.execute_reply":"2024-01-25T08:48:59.333353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show few digits\n\nfor i in range(9):\n    plt.subplot(3, 3, i+1)\n    plt.imshow(X_train_tensor[i, :])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-25T08:49:00.499684Z","iopub.execute_input":"2024-01-25T08:49:00.500553Z","iopub.status.idle":"2024-01-25T08:49:01.441712Z","shell.execute_reply.started":"2024-01-25T08:49:00.500522Z","shell.execute_reply":"2024-01-25T08:49:01.440806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show the distribution of labels\n\nplt.figure(figsize=(15, 5))\n\nplt.subplot(1, 2, 1)\nfor i in range(10):\n    plt.hist(y_train[y_train==i], width=0.5)\nplt.xlim(-0.5, 10)\nplt.title('Training dataset')\nplt.ylabel('Frequency')\nplt.xlabel('Label')\n\nplt.subplot(1, 2, 2)\nfor i in range(10):\n    plt.hist(y_validation[y_validation==i], width=0.5)\nplt.xlim(-0.5, 10)\nplt.title('Validation dataset')\nplt.ylabel('Frequency')\nplt.xlabel('Label')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-25T08:56:45.693640Z","iopub.execute_input":"2024-01-25T08:56:45.693912Z","iopub.status.idle":"2024-01-25T08:56:46.454377Z","shell.execute_reply.started":"2024-01-25T08:56:45.693886Z","shell.execute_reply":"2024-01-25T08:56:46.453326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Create ML model\nWe create in this section a Convolutional Neural Network, inspired by LeNet5","metadata":{}},{"cell_type":"code","source":"from keras.layers import Dense, Conv2D,Flatten, AveragePooling2D, Dropout\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\nfrom keras.models import Sequential, load_model","metadata":{"execution":{"iopub.status.busy":"2024-01-25T08:51:36.816756Z","iopub.execute_input":"2024-01-25T08:51:36.817260Z","iopub.status.idle":"2024-01-25T08:51:53.461230Z","shell.execute_reply.started":"2024-01-25T08:51:36.817224Z","shell.execute_reply":"2024-01-25T08:51:53.460388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We create a CNN inspired by the old architecture called LeNet5, that consists of 2 convolutional layers and 3 fully connected layers. \n\nWe provide a scaling_factor to increase the dimension of the Neural Network and we also use Dropout and EarlyStopping to avoid overfitting.","metadata":{}},{"cell_type":"code","source":"def get_model(scaling_factor):\n    \n    assert type(scaling_factor) == int, \"The scaling factor must be an integer\"\n    \n    # creation a sequential model\n    model = Sequential()\n\n    # 1st convolutional layer\n    # the dimension of the scanner is 5x5. With padding='same' the dimension of the image does not change\n    model.add(Conv2D(filters=6*scaling_factor, \n                     kernel_size=(5, 5), \n                     padding='same', \n                     activation='relu', \n                     input_shape=(28, 28, 1)))\n    # with average pooling 2D the dimension of the image changes from 28x28 to 14x14\n    model.add(AveragePooling2D())\n\n    # 2nd convolutional layers \n    # the dimension of the scanner is 5x5. With padding='valid' and filter = 5x5,\n    # the dimension of the image goes from 14x14 to 10x10\n    model.add(Conv2D(filters=16*scaling_factor, \n                     kernel_size=(5,5), \n                     padding='valid', \n                     activation='relu'))\n    # with average pooling 2D the dimension of the image changes from 10x10 to 5x5\n    model.add(AveragePooling2D())\n    \n    # from 2D to 1D\n    model.add(Flatten())\n\n    # 3rd layer\n    model.add(Dense(units=120*scaling_factor, \n                    activation='relu'))\n    model.add(Dropout(0.55))\n\n    # 4th layer\n    model.add(Dense(units=84*scaling_factor, \n                    activation='relu'))\n    model.add(Dropout(0.5))\n\n    # 5th layer (output)\n    model.add(Dense(units=10, \n                    activation = 'softmax'))\n\n    model.compile(optimizer='adam',\n                  loss='categorical_crossentropy',\n                  metrics=['accuracy'])\n    \n    return model\n\n\ndef fit_model(model, train_data, validation_data, patience, epochs, batch_size, validation_split):\n    # stop the training if the validation accuracy does not improve after a certain number of epochs\n    early_stopping_monitor = EarlyStopping(patience=patience,\n                                           monitor='val_accuracy')\n\n    # save the best model\n    model_checkpoint_callback = ModelCheckpoint(filepath='best_model.hdf5',\n                                                save_weights_only=False,\n                                                monitor='val_accuracy',\n                                                mode='auto',\n                                                save_best_only=True,\n                                                verbose=1)\n\n    print(model.summary())\n\n    X_train, y_train = train_data\n    X_validation, y_validation = validation_data\n    \n    # save the history of the training\n    history = model.fit(X_train, y_train,\n                        validation_data=(X_validation, y_validation),\n                        batch_size=batch_size,\n                        epochs=epochs,\n                        callbacks=[early_stopping_monitor,model_checkpoint_callback])\n    \n    return model, history","metadata":{"execution":{"iopub.status.busy":"2024-01-25T08:55:45.829951Z","iopub.execute_input":"2024-01-25T08:55:45.830336Z","iopub.status.idle":"2024-01-25T08:55:45.842162Z","shell.execute_reply.started":"2024-01-25T08:55:45.830307Z","shell.execute_reply":"2024-01-25T08:55:45.841271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Train the model","metadata":{}},{"cell_type":"code","source":"model_CNN = get_model(scaling_factor=4)\n\ntrained_model, history = fit_model(model = model_CNN,\n                                   train_data = (X_train_tensor, y_train_onehot), \n                                   validation_data = (X_validation_tensor, y_validation_onehot),\n                                   patience=10, \n                                   epochs=100, \n                                   batch_size=256, \n                                   validation_split=0.1)","metadata":{"execution":{"iopub.status.busy":"2024-01-25T08:56:08.285086Z","iopub.execute_input":"2024-01-25T08:56:08.285448Z","iopub.status.idle":"2024-01-25T08:56:45.518913Z","shell.execute_reply.started":"2024-01-25T08:56:08.285403Z","shell.execute_reply":"2024-01-25T08:56:45.517933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Inspect the results and make predictions","metadata":{}},{"cell_type":"markdown","source":"### History of training","metadata":{}},{"cell_type":"code","source":"epochs_grid = np.arange(1, len(history.history['accuracy'])+1)\n\nplt.plot(epochs_grid, history.history['accuracy'], label='training')\nplt.plot(epochs_grid, history.history['val_accuracy'], label='validation')\nplt.grid()\nplt.xlabel('Epochs')\nplt.ylabel('Accuracy')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-25T08:56:54.683194Z","iopub.execute_input":"2024-01-25T08:56:54.683658Z","iopub.status.idle":"2024-01-25T08:56:54.977799Z","shell.execute_reply.started":"2024-01-25T08:56:54.683622Z","shell.execute_reply":"2024-01-25T08:56:54.976859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Compute predictions using the validation dataset","metadata":{}},{"cell_type":"code","source":"def make_predictions(trained_model, X_test):\n    X_test = np.array(X_test)/255\n    X_test_scaled = X_test.reshape(len(X_test), 28, 28)\n    X_test_tensor = X_test.reshape(len(X_test_scaled), 28, 28, 1)\n    \n    y_pred = np.argmax(trained_model.predict(X_test_tensor), axis=1)\n    \n    df_submission = pd.DataFrame(np.arange(1, len(X_test)+1))\n    df_submission['predictions'] = y_pred\n    df_submission.columns=['ImageId', 'Label']\n    \n    return df_submission","metadata":{"execution":{"iopub.status.busy":"2024-01-25T09:08:04.199755Z","iopub.execute_input":"2024-01-25T09:08:04.200482Z","iopub.status.idle":"2024-01-25T09:08:04.206952Z","shell.execute_reply.started":"2024-01-25T09:08:04.200451Z","shell.execute_reply":"2024-01-25T09:08:04.205826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load the best model\nbest_model = load_model('/kaggle/working/best_model.hdf5')\n\ny_validation_pred = make_predictions(best_model, X_validation)\ny_validation_pred = np.array(y_validation_pred['Label'])\nvalidation_score = np.mean(y_validation_pred == y_validation)\n\nprint('The validation score of the best model is', validation_score)","metadata":{"execution":{"iopub.status.busy":"2024-01-25T09:08:06.953348Z","iopub.execute_input":"2024-01-25T09:08:06.954108Z","iopub.status.idle":"2024-01-25T09:08:07.540248Z","shell.execute_reply.started":"2024-01-25T09:08:06.954074Z","shell.execute_reply":"2024-01-25T09:08:07.539375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nimport seaborn as sns\n\ndf_confusion_matrix = pd.DataFrame(confusion_matrix(y_validation, y_validation_pred))\n\nsns.heatmap(df_confusion_matrix, annot=True, cmap=\"OrRd\", fmt='g', cbar=False)\nplt.title('Confusion matrix')\nplt.yticks(rotation=0)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-25T09:08:09.147133Z","iopub.execute_input":"2024-01-25T09:08:09.147780Z","iopub.status.idle":"2024-01-25T09:08:09.586908Z","shell.execute_reply.started":"2024-01-25T09:08:09.147742Z","shell.execute_reply":"2024-01-25T09:08:09.585909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show few digits of the validation dataset\n\nfor i in range(9):\n    plt.subplot(3, 3, i+1)\n    plt.imshow(X_validation_tensor[i, :])\n    plt.title('Predicted '+str(y_validation_pred[i]))\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-25T09:08:59.199527Z","iopub.execute_input":"2024-01-25T09:08:59.199919Z","iopub.status.idle":"2024-01-25T09:09:00.615712Z","shell.execute_reply.started":"2024-01-25T09:08:59.199886Z","shell.execute_reply":"2024-01-25T09:09:00.614847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Submit predictions","metadata":{}},{"cell_type":"code","source":"df_submission_example = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')\ndf_submission_example.head()","metadata":{"execution":{"iopub.status.busy":"2024-01-25T09:07:29.704413Z","iopub.execute_input":"2024-01-25T09:07:29.704746Z","iopub.status.idle":"2024-01-25T09:07:29.722953Z","shell.execute_reply.started":"2024-01-25T09:07:29.704722Z","shell.execute_reply":"2024-01-25T09:07:29.722015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test data (there are no labels in the test dataset)\nX_test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\n\ndf_submission = make_predictions(best_model, X_test)\n\ndf_submission.to_csv('submission.csv', index=False)\ndf_submission","metadata":{"execution":{"iopub.status.busy":"2024-01-25T09:07:31.541524Z","iopub.execute_input":"2024-01-25T09:07:31.542495Z","iopub.status.idle":"2024-01-25T09:07:35.446810Z","shell.execute_reply.started":"2024-01-25T09:07:31.542458Z","shell.execute_reply":"2024-01-25T09:07:35.445823Z"},"trusted":true},"execution_count":null,"outputs":[]}]}