{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30648,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nimport pandas as pd","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-02-01T21:18:53.929197Z","iopub.execute_input":"2024-02-01T21:18:53.929592Z","iopub.status.idle":"2024-02-01T21:18:54.283171Z","shell.execute_reply.started":"2024-02-01T21:18:53.929551Z","shell.execute_reply":"2024-02-01T21:18:54.282393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Import and explore data","metadata":{}},{"cell_type":"code","source":"# load train data\ndf_train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\n\nX = df_train.drop('label', axis=1)\ny = df_train['label']","metadata":{"execution":{"iopub.status.busy":"2024-02-01T21:18:54.287404Z","iopub.execute_input":"2024-02-01T21:18:54.287662Z","iopub.status.idle":"2024-02-01T21:18:57.142792Z","shell.execute_reply.started":"2024-02-01T21:18:54.287638Z","shell.execute_reply":"2024-02-01T21:18:57.141910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# prepare data for ML, by splitting them in a train and validation dataset\n\nfrom sklearn.model_selection import train_test_split\n\nX_train, X_validation, y_train, y_validation = train_test_split(X, y, \n                                                                test_size=0.1, \n                                                                random_state=42,\n                                                                stratify=y)","metadata":{"execution":{"iopub.status.busy":"2024-02-01T21:18:57.144073Z","iopub.execute_input":"2024-02-01T21:18:57.144421Z","iopub.status.idle":"2024-02-01T21:18:57.835227Z","shell.execute_reply.started":"2024-02-01T21:18:57.144391Z","shell.execute_reply":"2024-02-01T21:18:57.834274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nfrom tqdm import tqdm\n\ndef apply_preprocessing(X, y):\n\n    # scale the features\n    X_scaled = np.array(X/255)\n    X_reshaped = X_scaled.reshape(len(X_scaled), 28, 28)\n    \n    # resize images, since MobileNet only accepts a minimum of 32x32 for images\n    new_imgs = []\n    for i in tqdm(range(len(X_reshaped))):\n        new_imgs.append(cv2.resize(X_reshaped[i, :], (32, 32)))\n    new_imgs = np.array(new_imgs)\n    \n    X_tensor3d = np.zeros([len(X_scaled), 32, 32, 3])\n    \n    for i in range(3):\n        X_tensor3d[:, :, :, i] = new_imgs\n    \n    # apply one-hot encoding to labels\n    y_onehot = pd.get_dummies(y)\n    \n    return X_tensor3d, y_onehot\n\n\nX_train_tensor, y_train_onehot = apply_preprocessing(X_train, y_train)\nX_validation_tensor, y_validation_onehot = apply_preprocessing(X_validation, y_validation)","metadata":{"execution":{"iopub.status.busy":"2024-02-01T21:18:57.838198Z","iopub.execute_input":"2024-02-01T21:18:57.838662Z","iopub.status.idle":"2024-02-01T21:18:59.559808Z","shell.execute_reply.started":"2024-02-01T21:18:57.838633Z","shell.execute_reply":"2024-02-01T21:18:59.558762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show few digits\n\nfor i in range(9):\n    plt.subplot(3, 3, i+1)\n    plt.imshow(X_train_tensor[i, :])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-01T21:18:59.560974Z","iopub.execute_input":"2024-02-01T21:18:59.561288Z","iopub.status.idle":"2024-02-01T21:19:00.480960Z","shell.execute_reply.started":"2024-02-01T21:18:59.561259Z","shell.execute_reply":"2024-02-01T21:19:00.480001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show the distribution of labels\n\nplt.figure(figsize=(15, 5))\n\nplt.subplot(1, 2, 1)\nfor i in range(10):\n    plt.hist(y_train[y_train==i], width=0.5)\nplt.xlim(-0.5, 10)\nplt.title('Training dataset')\nplt.ylabel('Frequency')\nplt.xlabel('Label')\n\nplt.subplot(1, 2, 2)\nfor i in range(10):\n    plt.hist(y_validation[y_validation==i], width=0.5)\nplt.xlim(-0.5, 10)\nplt.title('Validation dataset')\nplt.ylabel('Frequency')\nplt.xlabel('Label')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-01T21:19:00.481870Z","iopub.execute_input":"2024-02-01T21:19:00.482153Z","iopub.status.idle":"2024-02-01T21:19:01.246312Z","shell.execute_reply.started":"2024-02-01T21:19:00.482129Z","shell.execute_reply":"2024-02-01T21:19:01.245309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Create ML model\nWe import in this section the pre-trained model called MobileNet_v2 and we adapt it for the MNIST dataset.\nWe follow the Keras documentation to achieve this goal.","metadata":{}},{"cell_type":"code","source":"from keras.layers import Dense, Conv2D,Flatten, AveragePooling2D, Dropout, GlobalAveragePooling2D\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\nfrom keras.models import Sequential, load_model, Model\nfrom keras.applications import MobileNetV2","metadata":{"execution":{"iopub.status.busy":"2024-02-01T21:19:01.247560Z","iopub.execute_input":"2024-02-01T21:19:01.247904Z","iopub.status.idle":"2024-02-01T21:19:04.439187Z","shell.execute_reply.started":"2024-02-01T21:19:01.247874Z","shell.execute_reply":"2024-02-01T21:19:04.438325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create the base pre-trained model\nbase_model = MobileNetV2(weights='imagenet', include_top=False, input_shape=(32, 32, 3))\n\n# add a global spatial average pooling layer\nx = base_model.output\nx = GlobalAveragePooling2D()(x)\n\n# let's add two fully-connected layer\nx = Dense(100, activation='relu')(x)\nx = Dropout(0.8)(x)\n\nx = Dense(100, activation='relu')(x)\nx = Dropout(0.5)(x)\n\n# and a logistic layer -- let's say we have 200 classes\npredictions = Dense(10, activation='softmax')(x)\n\n# this is the model we will train\nmodel = Model(inputs=base_model.input, outputs=predictions)","metadata":{"execution":{"iopub.status.busy":"2024-02-01T21:19:04.440371Z","iopub.execute_input":"2024-02-01T21:19:04.440927Z","iopub.status.idle":"2024-02-01T21:19:06.616660Z","shell.execute_reply.started":"2024-02-01T21:19:04.440898Z","shell.execute_reply":"2024-02-01T21:19:06.615579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2024-02-01T21:19:06.617927Z","iopub.execute_input":"2024-02-01T21:19:06.618248Z","iopub.status.idle":"2024-02-01T21:19:06.992462Z","shell.execute_reply.started":"2024-02-01T21:19:06.618223Z","shell.execute_reply":"2024-02-01T21:19:06.991251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Train the model\nWe re-train entirely the MobileNet_v2, a powerful Convolutional Neural Network that consists of more than 2 million of parameters.","metadata":{}},{"cell_type":"code","source":"# let's visualize layer names and layer indices\nfor i, layer in enumerate(base_model.layers):\n    print(i, layer.name)","metadata":{"execution":{"iopub.status.busy":"2024-02-01T21:19:06.994189Z","iopub.execute_input":"2024-02-01T21:19:06.994458Z","iopub.status.idle":"2024-02-01T21:19:07.002672Z","shell.execute_reply.started":"2024-02-01T21:19:06.994433Z","shell.execute_reply":"2024-02-01T21:19:07.001714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# we need to recompile the model for these modifications to take effect\n# we use SGD with a low learning rate\nfrom keras.optimizers import SGD\nmodel.compile(optimizer=SGD(lr=0.0001, momentum=0.9), \n              loss='categorical_crossentropy', \n              metrics=['accuracy'])\n\n\n# stop the model if there is no improvement for 20 epochs\nearly_stopping_monitor = EarlyStopping(patience=50,\n                                       monitor='val_accuracy')\n\n# save the best model\nmodel_checkpoint_callback = ModelCheckpoint(filepath='best_model.hdf5',\n                                            save_weights_only=False,\n                                            monitor='val_accuracy',\n                                            mode='auto',\n                                            save_best_only=True,\n                                            verbose=1)\n\n# we train our model again (this time fine-tuning the top 2 MobileNet blocks\n# alongside the top Dense layers\nhistory = model.fit(X_train_tensor, y_train_onehot,\n                    validation_data=(X_validation_tensor, y_validation_onehot),\n                    batch_size=128,\n                    epochs=500,\n                    callbacks=[early_stopping_monitor, model_checkpoint_callback])","metadata":{"execution":{"iopub.status.busy":"2024-02-01T21:19:07.003727Z","iopub.execute_input":"2024-02-01T21:19:07.003987Z","iopub.status.idle":"2024-02-01T21:20:26.429894Z","shell.execute_reply.started":"2024-02-01T21:19:07.003965Z","shell.execute_reply":"2024-02-01T21:20:26.427995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Inspect the results and make predictions","metadata":{}},{"cell_type":"code","source":"epochs_grid = np.arange(1, len(history.history['accuracy'])+1)\n\nplt.plot(epochs_grid, history.history['accuracy'], label='training')\nplt.plot(epochs_grid, history.history['val_accuracy'], label='validation')\nplt.grid()\nplt.xlabel('Epochs')\nplt.ylabel('Accuracy')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-01T21:20:26.430761Z","iopub.status.idle":"2024-02-01T21:20:26.431157Z","shell.execute_reply.started":"2024-02-01T21:20:26.430962Z","shell.execute_reply":"2024-02-01T21:20:26.430978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Compute predictions on the validation dataset","metadata":{}},{"cell_type":"code","source":"def make_predictions(trained_model, X_test):\n    # scale the features\n    X_scaled = np.array(X_test/255)\n    X_reshaped = X_scaled.reshape(len(X_scaled), 28, 28)\n    \n    # resize images, since MobileNet only accepts a minimum of 32x32 for images\n    new_imgs = []\n    for i in tqdm(range(len(X_reshaped))):\n        new_imgs.append(cv2.resize(X_reshaped[i, :], (32, 32)))\n    new_imgs = np.array(new_imgs)\n    \n    X_tensor3d = np.zeros([len(X_scaled), 32, 32, 3])\n    \n    for i in range(3):\n        X_tensor3d[:, :, :, i] = new_imgs\n    \n    y_pred = np.argmax(trained_model.predict(X_tensor3d), axis=1)\n    \n    df_submission = pd.DataFrame(np.arange(1, len(X_test)+1))\n    df_submission['predictions'] = y_pred\n    df_submission.columns=['ImageId', 'Label']\n    \n    return df_submission","metadata":{"execution":{"iopub.status.busy":"2024-02-01T21:20:26.432170Z","iopub.status.idle":"2024-02-01T21:20:26.432532Z","shell.execute_reply.started":"2024-02-01T21:20:26.432340Z","shell.execute_reply":"2024-02-01T21:20:26.432355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load the best model\nbest_model = load_model('/kaggle/working/best_model.hdf5')\n\ny_validation_pred = make_predictions(best_model, X_validation)\ny_validation_pred = np.array(y_validation_pred['Label'])\nvalidation_score = np.mean(y_validation_pred == y_validation)\n\nprint('The validation score of the best model is', validation_score)","metadata":{"execution":{"iopub.status.busy":"2024-02-01T21:20:26.434195Z","iopub.status.idle":"2024-02-01T21:20:26.434584Z","shell.execute_reply.started":"2024-02-01T21:20:26.434411Z","shell.execute_reply":"2024-02-01T21:20:26.434431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nimport seaborn as sns\n\ndf_confusion_matrix = pd.DataFrame(confusion_matrix(y_validation, y_validation_pred))\n\nsns.heatmap(df_confusion_matrix, annot=True, cmap=\"OrRd\", fmt='g', cbar=False)\nplt.title('Confusion matrix')\nplt.yticks(rotation=0)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-01T21:20:26.435748Z","iopub.status.idle":"2024-02-01T21:20:26.436116Z","shell.execute_reply.started":"2024-02-01T21:20:26.435913Z","shell.execute_reply":"2024-02-01T21:20:26.435926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show few digits of the validation dataset\n\nfor i in range(9):\n    plt.subplot(3, 3, i+1)\n    plt.imshow(X_validation_tensor[i, :])\n    plt.title('Predicted '+str(y_validation_pred[i]))\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-01T21:20:26.437280Z","iopub.status.idle":"2024-02-01T21:20:26.437642Z","shell.execute_reply.started":"2024-02-01T21:20:26.437474Z","shell.execute_reply":"2024-02-01T21:20:26.437489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Submit predictions","metadata":{}},{"cell_type":"code","source":"df_submission_example = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')\ndf_submission_example.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-01T21:20:26.438972Z","iopub.status.idle":"2024-02-01T21:20:26.439317Z","shell.execute_reply.started":"2024-02-01T21:20:26.439158Z","shell.execute_reply":"2024-02-01T21:20:26.439172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test data (there are no labels in the test dataset)\nX_test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\n\ndf_submission = make_predictions(best_model, X_test)\n\ndf_submission.to_csv('submission.csv', index=False)\ndf_submission","metadata":{"execution":{"iopub.status.busy":"2024-02-01T21:20:26.440641Z","iopub.status.idle":"2024-02-01T21:20:26.440949Z","shell.execute_reply.started":"2024-02-01T21:20:26.440796Z","shell.execute_reply":"2024-02-01T21:20:26.440809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}