{"cells":[{"metadata":{},"cell_type":"markdown","source":"# SimpleCNN for Handwritten Digits Recognition (MNIST)\n\nbased on the [original MNIST dataset](http://yann.lecun.com/exdb/mnist/)"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport seaborn as sns\nsns.set()\n\nimport matplotlib.pyplot as plt\n\nimport os\n\npath_input='../input/digit-recognizer/'\npath_output='./'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"data =pd.read_csv(path_input+'train.csv')\ndata.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X=data.iloc[: ,1:].values \ny=data[\"label\"].values\nX=X.reshape(42000,28,28)\nX.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(9):  \n    plt.subplot(330 + 1 + i)\n    plt.imshow(X[i], cmap=plt.get_cmap('gray'))\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Preprocessing"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nx_train, x_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=np.random.RandomState(0))\nx_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import keras.utils as utils\n\n# Expand last dimension for CNN to enable channel (28,28,1)\nx_train = np.expand_dims(x_train, axis=-1)\nx_test = np.expand_dims(x_test, axis=-1)\n\ndef reduce(x_train,x_test):\n    x_train = x_train.astype('float32') / 255\n    x_test = x_test.astype('float32') / 255\n    return x_train,x_test\n\nx_train,x_test=reduce(x_train,x_test)\n\n# One hot encode the target\ny_train = utils.to_categorical(y_train, num_classes = 10)\ny_test = utils.to_categorical(y_test, num_classes = 10)\n\n# Input image dimensions.\ninput_shape = x_train.shape[1:]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Convolutional Network Architecture"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\nfrom keras.optimizers import Adam\nfrom keras.callbacks import ReduceLROnPlateau,ModelCheckpoint,LearningRateScheduler,EarlyStopping\nfrom keras.layers import DepthwiseConv2D\nfrom keras.layers import (Input, Conv2D, BatchNormalization, ZeroPadding2D,\n                          GlobalAveragePooling2D, Activation, Dense, MaxPooling2D, Dropout, Flatten)\nfrom keras.models import Model, Sequential\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**A very simple CNN**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def SimpleNet(input_shape):\n  model = Sequential()\n  model.add(Conv2D(32, kernel_size=(3, 3),\n                 activation='relu',\n                 input_shape=input_shape))\n  model.add(Conv2D(64, (3, 3), activation='relu'))\n  model.add(MaxPooling2D(pool_size=(2, 2)))\n  model.add(Dropout(0.25))\n  model.add(Flatten())\n  model.add(Dense(128, activation='relu'))\n  model.add(Dropout(0.5))\n  model.add(Dense(10, activation='softmax'))\n  return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model=SimpleNet(input_shape)\nbatch_size=256 #it depends on your computational ressources","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Learning Rate and Model Compiling\nLearning Rate Schedule\nLearning rate is scheduled to be reduced after 80, 120, 160, 180 epochs.\nCalled automatically every epoch as part of callbacks during training."},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import optimizers,losses,metrics\n\ndef lr_schedule(epoch):\n    lr = 1e-3\n    if epoch > 180:\n        lr *= 0.5e-3\n    elif epoch > 160:\n        lr *= 1e-3\n    elif epoch > 120:\n        lr *= 1e-2\n    elif epoch > 80:\n        lr *= 1e-1\n    print('Learning rate: ', lr)\n    return lr\n\n\n\nmodel.compile(optimizers.Adam(lr_schedule(0)),\n                    losses.categorical_crossentropy,\n                    [metrics.categorical_accuracy])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Augmentation\n\nTo avoid overfitting"},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_augmentation(x_train):\n    print('Using real-time data augmentation.')\n    # This will do preprocessing and realtime data augmentation:\n    datagen = ImageDataGenerator(\n        # set input mean to 0 over the dataset\n        featurewise_center=False,\n        # set each sample mean to 0\n        samplewise_center=False,\n        # divide inputs by std of dataset\n        featurewise_std_normalization=False,\n        # divide each input by its std\n        samplewise_std_normalization=False,\n        # apply ZCA whitening\n        zca_whitening=False,\n        # epsilon for ZCA whitening\n        zca_epsilon=1e-06,\n        # randomly rotate images in the range (deg 0 to 180)\n        rotation_range=0,\n        # randomly shift images horizontally\n        width_shift_range=0.1,\n        # randomly shift images vertically\n        height_shift_range=0.1,\n        # set range for random shear\n        shear_range=0.,\n        # set range for random zoom\n        zoom_range=0.,\n        # set range for random channel shifts\n        channel_shift_range=0.,\n        # set mode for filling points outside the input boundaries\n        fill_mode='nearest',\n        # value used for fill_mode = \"constant\"\n        cval=0.,\n        # randomly flip images\n        horizontal_flip=True,\n        # randomly flip images\n        vertical_flip=False,\n        # set rescaling factor (applied before any other transformation)\n        rescale=None,\n        # set function that will be applied on each input\n        preprocessing_function=None,\n        # image data format, either \"channels_first\" or \"channels_last\" if your image has colors\n        data_format=None,\n        # fraction of images reserved for validation (strictly between 0 and 1)\n        validation_split=0.0)\n    datagen.fit(x_train)\n    return datagen","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Training"},{"metadata":{},"cell_type":"markdown","source":"### Backup and Model parameters\nMake a backup for saving the model in case of problem and some parameters for a better training"},{"metadata":{"trusted":true},"cell_type":"code","source":"save_dir = os.path.join(os.getcwd(), 'saved_models')\nmodel_name = '_mnist_%s_model.{epoch:03d}.h5' % 'simplecnn'\nif not os.path.isdir(save_dir):\n    os.makedirs(save_dir)\nfilepath = os.path.join(save_dir, model_name)\n\n# Prepare callbacks for model saving and for learning rate adjustment.\ncheckpoint = ModelCheckpoint(filepath=filepath,\n                             monitor='val_acc',\n                             verbose=1,\n                             save_best_only=True)\nlr_scheduler = LearningRateScheduler(lr_schedule)\nlr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1),\n                               cooldown=0,\n                               patience=5,\n                               min_lr=0.5e-6)\ncallbacks = [checkpoint, lr_reducer, lr_scheduler]\n\n#early stopping control epochs and time (0.9 acc min wanted)\nearly_stopp=EarlyStopping(monitor=\"val_acc\",\n                          min_delta=1e-2,\n                          patience=2,\n                          verbose=2,\n                          mode=\"auto\",\n                          baseline=0.9,\n                          restore_best_weights=True)\ncallbacks.append(early_stopp)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"datagen=data_augmentation(x_train)\n# Fit the model on the batches generated by datagen.flow()\ntraining=model.fit(datagen.flow(x_train, y_train, batch_size=batch_size),\n                    steps_per_epoch = 400,\n                    epochs = 50,\n                    validation_data = (x_test,y_test),\n                    validation_steps = 200,\n                    shuffle=True,\n                    workers=4)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model Evaluation and Results with metrics"},{"metadata":{"trusted":true},"cell_type":"code","source":"scores = model.evaluate(x_test, y_test, verbose=1)\nprint('Test loss:', scores[0])\nprint('Test accuracy:', scores[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import classification_report,confusion_matrix\n\ndef plot_results(labels,preds,model_name,plot_type='all'):\n    def plot_confusion_matrix(labels, preds,model_name):\n        plt.figure(1, figsize= (8, 8))\n        plt.title(\"Confusion matrix for \"+model_name)\n        mat = confusion_matrix(labels, preds)\n        sns.heatmap(mat.T, square=True, annot=True, fmt='d', cbar=False)\n        plt.xlabel('true label')\n        plt.ylabel('predicted label')\n        plt.show()\n        plt.close()\n\n    def plot_classif_report(labels,preds,model_name):\n        clf=classification_report(labels, preds,output_dict=True)\n        plt.figure(1, figsize= (8,8))\n        ax = plt.axes()\n        sns.heatmap(pd.DataFrame(clf).iloc[:-1, :].T,annot=True)#annot=True\n        ax.set_title(\"Classification report for \"+model_name)\n        plt.show()\n        plt.close()\n    if 'confusion' in plot_type:\n        plot_confusion_matrix(labels,preds,model_name)\n    elif 'report' in plot_type:\n        plot_classif_report(labels,preds,model_name)\n    elif 'all' in plot_type:\n        plot_confusion_matrix(labels,preds,model_name)\n        plot_classif_report(labels,preds,model_name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred = model.predict(x_test, batch_size, verbose=1)\ny_pred_bool = np.argmax(y_pred, axis=1)\ny_test_bool=np.argmax(y_test, axis=1)\n\nplot_results(y_test_bool,y_pred_bool,'SimpleCNN on MNIST',plot_type='all')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 99% accuracy and F1-score, great !! "},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_accuracy_and_loss(training,net_name):\n    '''\n    Explicit function tool to plot accuracy and loss after training a network\n\n    Parameters\n    ----------\n    training : fit generator of the framework (ex : training=model.fit(...)).\n    net_name (string) : name of the network considered (with dataset to be precise).\n    '''\n    plt.figure(1, figsize= (15, 10))\n    # plot train and test accuracy\n    plt.subplot(221)\n    plt.plot(training.history['categorical_accuracy'],label='train')\n    plt.plot(training.history['val_categorical_accuracy'],label='test')\n    plt.ylabel('Accuracy')\n    plt.xlabel('Epochs')\n    plt.title(net_name+' Accuracy')\n    plt.legend(['train', 'test'], loc='upper left')\n    # plot train and test loss\n    plt.subplot(222)\n    plt.plot(training.history['loss'])\n    plt.plot(training.history['val_loss'])\n    plt.title(net_name+' Loss')\n    plt.ylabel('Loss')\n    plt.xlabel('Epochs')\n    plt.legend(['train', 'test'], loc='upper right')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_accuracy_and_loss(training,'SimpleCNN')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submission"},{"metadata":{},"cell_type":"markdown","source":"Apply preprocessing to the test data"},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\nX_test = (test / 255).values\nX_test = X_test.reshape((X_test.shape[0], 28, 28, 1))\nX_test.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Make the submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame(np.argmax(model.predict(X_test), axis=1), columns=['Label'], \n                      index=pd.read_csv('../input/digit-recognizer/sample_submission.csv')['ImageId'])\n\nsubmission.index.name = 'ImageId'\nsubmission.to_csv(path_output+'submission.csv',index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}