{"cells":[{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport seaborn as sns\n%matplotlib inline\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\nimport itertools\n\nfrom keras.utils.np_utils import to_categorical # convert to one-hot-encoding\nfrom keras.models import Sequential, Model\n#from keras.layers import, Dropout, Flatten, Conv2D, MaxPool2D, Input, ZeroPadding2D\nfrom keras.layers import Dense, Dropout, Flatten, Input, Activation, Add, AveragePooling2D\nfrom keras.layers import Conv2D, MaxPooling2D, BatchNormalization, ZeroPadding2D, MaxPool2D\nfrom keras import backend as K\nfrom keras.optimizers import RMSprop\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ReduceLROnPlateau\nfrom keras.initializers import glorot_uniform\n\nsns.set(style='white', context='notebook', palette='deep')\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"fc30e725-866c-4ca0-9026-e0f7f372ef74","collapsed":true,"_uuid":"476d5ae66f4b723ffe2793187d0ce793e7810715","trusted":false},"cell_type":"code","source":"# Load the dataset for training and test\ntrain = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")\n\n# get labels from training set to train models\nY_train = train[\"label\"] \n# drop labels from original training set, and generate a new training set\nX_train = train.drop(labels = [\"label\"],axis = 1)  \n# free some RAM space \ndel train\n\n# We check that neither training set nor test set includes null value :\n#     X_train.isnull().any().describe() \n#     test.isnull().any().describe()\n\n# Normalization, transform from [0,255] to [0,1], to speed up CNN running\nX_train = X_train / 255.0\ntest = test / 255.0\n\n# Tranfor DataFrame to ndarray and Reshape them in 28*28*1 \nX_train = X_train.values.reshape(-1, 28, 28, 1)\ntest = test.values.reshape(-1,28,28,1)\n\n# Label encoding\n# Encode labels to one hot vectors (ex : 2 -> [0,0,1,0,0,0,0,0,0,0])\n#                                        0 -> [1,0,0,0,0,0,0,0,0,0]\n#                                        5 -> [0,0,0,0,0,1,0,0,0,0]\n#                                        9 -> [0,0,0,0,0,0,0,0,0,1]\nY_train = to_categorical(Y_train, num_classes = 10)\n\n# Split the train and the validation set for the fitting\nrandom_seed = 0\nX_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.1, \n                                                  random_state=random_seed)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"40b67336-bf74-4f00-bfdd-ebec14ccaa11","collapsed":true,"_uuid":"1fa06833a85f42cbd596f47fe09b15bf796155c6","trusted":false},"cell_type":"code","source":"def Simpel_CNN(InputShape=(28,28,1), filters=[32,32,64,64]):\n    '''\n    Set the CNN model \n    Input -> [[Conv2D->relu]*2 -> MaxPool2D -> Dropout]*2 -> Flatten -> Dense -> Dropout -> Out\n    \n    Arguments:\n        InputShape -- input tensor of shape (n_H_prev, n_W_prev, n_C_prev) (in this case, it is (28,28,1))\n        filters -- python list of integers, defining the number of filters in the CONV layers of the main path\n                   [32,32,64,64] in this case\n    '''\n    F1, F2, F3, F4 = filters\n    model = Sequential()\n\n    model.add(Conv2D(filters = F1, kernel_size = (5,5),padding = 'Same', \n                     activation ='relu', input_shape = InputShape))\n    model.add(Conv2D(filters = F2, kernel_size = (5,5),padding = 'Same', \n                     activation ='relu'))\n    model.add(MaxPool2D(pool_size=(2,2)))\n    model.add(Dropout(0.25))\n\n\n    model.add(Conv2D(filters = F3, kernel_size = (3,3),padding = 'Same', \n                     activation ='relu'))\n    model.add(Conv2D(filters = F4, kernel_size = (3,3),padding = 'Same', \n                     activation ='relu'))\n    model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))\n    model.add(Dropout(0.25))\n\n\n    model.add(Flatten())\n    model.add(Dense(256, activation = \"relu\"))\n    model.add(Dropout(0.5))\n    model.add(Dense(10, activation = \"softmax\"))\n    \n    return model\n              \n\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"4ddcbd3d-f69d-4d4e-8f53-44cd5b294c78","collapsed":true,"_uuid":"38321ce3b4f7a7801f8fdb3c0fd3bdca1220e5a3","trusted":false},"cell_type":"code","source":"\ndef Compile_Fit_Simple_CNN():\n    # Define the optimizer\n    # optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)\n    # Compile the Simple_CNN model\n    model = Simpel_CNN()\n    model.compile(optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0) , \n              loss = \"categorical_crossentropy\", metrics=[\"accuracy\"])\n    # Set a learning rate annealer\n    learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', \n                                                patience=3, \n                                                verbose=1, \n                                                factor=0.5, \n                                                min_lr=0.00001)\n    # Data augmentation:\n    #     Randomly rotate some training images by 10 degrees\n    #     Randomly Zoom by 10% some training images\n    #     Randomly shift images horizontally by 10% of the width\n    #     Randomly shift images vertically by 10% of the height\n    datagen = ImageDataGenerator(\n            featurewise_center=False,  # set input mean to 0 over the dataset\n            samplewise_center=False,  # set each sample mean to 0\n            featurewise_std_normalization=False,  # divide inputs by std of the dataset\n            samplewise_std_normalization=False,  # divide each input by its std\n            zca_whitening=False,  # apply ZCA whitening\n            rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n            zoom_range = 0.1, # Randomly zoom image \n            width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n            height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n            horizontal_flip=False,  # randomly flip images\n            vertical_flip=False)  # randomly flip images\n\n    datagen.fit(X_train)\n\n    epochs = 20 # Turn epochs to 30 to get 0.9967 accuracy\n    batch_size = 86\n    history = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),\n                                  epochs = epochs, \n                                  validation_data = (X_val,Y_val),\n                                  verbose = 1, steps_per_epoch=X_train.shape[0] // batch_size, \n                                  callbacks=[learning_rate_reduction])\n    return history","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"5abae2e0-0506-4e57-b9c6-0e72e4b470ee","collapsed":true,"_uuid":"312d254242ea9fd86a6c2ec9333ea92fa4647290","trusted":false},"cell_type":"code","source":"# Construct a ResNet50 (learned from Andrew Ng's online-course)\n\n# step 1. Identity block function\n\ndef identity_block(X, f, filters, stage, block):\n    \"\"\"\n    Implementation of the identity block as defined in Figure 3\n    \n    Arguments:\n    X -- input tensor of shape (m, n_H_prev, n_W_prev, n_C_prev)\n    f -- integer, specifying the shape of the middle CONV's window for the main path\n    filters -- python list of integers, defining the number of filters in the CONV layers of the main path\n    stage -- integer, used to name the layers, depending on their position in the network\n    block -- string/character, used to name the layers, depending on their position in the network\n    \n    Returns:\n    X -- output of the identity block, tensor of shape (n_H, n_W, n_C)\n    \"\"\"\n    \n    # defining name basis\n    conv_name_base = 'res' + str(stage) + block + '_branch'\n    bn_name_base = 'bn' + str(stage) + block + '_branch'\n    \n    # Retrieve Filters\n    F1, F2, F3 = filters\n    \n    # Save the input value. You'll need this later to add back to the main path. \n    X_shortcut = X\n    \n    # First component of main path\n    X = Conv2D(filters = F1, kernel_size = (1, 1), strides = (1,1), padding = 'valid', name = conv_name_base + '2a', \n               kernel_initializer = glorot_uniform(seed=0))(X)\n    X = BatchNormalization(axis = 3, name = bn_name_base + '2a')(X)\n    X = Activation('relu')(X)\n    \n    ### START CODE HERE ###\n    \n    # Second component of main path (≈3 lines)\n    X = Conv2D(filters = F2, kernel_size = (f, f), strides = (1,1), padding = 'same', name = conv_name_base + '2b', \n              kernel_initializer = glorot_uniform(seed=0))(X)\n    X = BatchNormalization(axis = 3, name = bn_name_base + '2b')(X)\n    X = Activation('relu')(X)\n\n    # Third component of main path (≈2 lines)\n    X = Conv2D(filters = F3, kernel_size = (1, 1), strides = (1,1), padding = 'valid', name = conv_name_base + '2c',\n              kernel_initializer = glorot_uniform(seed=0))(X)\n    X = BatchNormalization(axis = 3, name = bn_name_base + '2c')(X)\n\n    # Final step: Add shortcut value to main path, and pass it through a RELU activation (≈2 lines)\n    X = Add()([X, X_shortcut])\n    X = Activation('relu')(X)\n    \n    ### END CODE HERE ###\n    \n    return X\n\n\n# step 2. Convolutional block function\ndef convolutional_block(X, f, filters, stage, block, s = 2):\n    \"\"\"\n    Implementation of the convolutional block as defined in Figure 4\n    \n    Arguments:\n    X -- input tensor of shape (m, n_H_prev, n_W_prev, n_C_prev)\n    f -- integer, specifying the shape of the middle CONV's window for the main path\n    filters -- python list of integers, defining the number of filters in the CONV layers of the main path\n    stage -- integer, used to name the layers, depending on their position in the network\n    block -- string/character, used to name the layers, depending on their position in the network\n    s -- Integer, specifying the stride to be used\n    \n    Returns:\n    X -- output of the convolutional block, tensor of shape (n_H, n_W, n_C)\n    \"\"\"\n    \n    # defining name basis\n    conv_name_base = 'res' + str(stage) + block + '_branch'\n    bn_name_base = 'bn' + str(stage) + block + '_branch'\n    \n    # Retrieve Filters\n    F1, F2, F3 = filters\n    \n    # Save the input value\n    X_shortcut = X\n\n\n    ##### MAIN PATH #####\n    # First component of main path \n    X = Conv2D(F1, (1, 1), strides = (s,s), name = conv_name_base + '2a', padding='valid',\n               kernel_initializer = glorot_uniform(seed=0))(X)\n    X = BatchNormalization(axis = 3, name = bn_name_base + '2a')(X)\n    X = Activation('relu')(X)\n    \n    ### START CODE HERE ###\n\n    # Second component of main path (≈3 lines)\n    X = Conv2D(F2, (f, f), strides = (1,1), name = conv_name_base + '2b', padding='same',\n               kernel_initializer = glorot_uniform(seed=0))(X)\n    X = BatchNormalization(axis=3, name = bn_name_base + '2b')(X)\n    X = Activation('relu')(X)\n\n    # Third component of main path (≈2 lines)\n    X = Conv2D(F3, (1, 1), strides = (1, 1), name = conv_name_base + '2c', padding='valid',\n               kernel_initializer = glorot_uniform(seed=0))(X)\n    X = BatchNormalization(axis=3, name = bn_name_base + '2c')(X)\n\n    ##### SHORTCUT PATH #### (≈2 lines)\n    X_shortcut = Conv2D(F3, (1, 1), strides = (s, s), name = conv_name_base + 'l', padding='valid',\n                        kernel_initializer = glorot_uniform(seed=0))(X_shortcut)\n    X_shortcut = BatchNormalization(axis=3, name = bn_name_base + 'l')(X_shortcut)\n\n    # Final step: Add shortcut value to main path, and pass it through a RELU activation (≈2 lines)\n    X = Add()([X, X_shortcut])\n    X = Activation('relu')(X)\n    \n    ### END CODE HERE ###\n    \n    return X\n\n# step 3. Build ResNet50\ndef ResNet50(input_shape, classes):\n    \"\"\"\n    Implementation of the popular ResNet50 the following architecture:\n    CONV2D -> BATCHNORM -> RELU -> MAXPOOL -> CONVBLOCK -> IDBLOCK*2 -> CONVBLOCK -> IDBLOCK*3\n    -> CONVBLOCK -> IDBLOCK*5 -> CONVBLOCK -> IDBLOCK*2 -> AVGPOOL -> TOPLAYER\n\n    Arguments:\n    input_shape -- shape of the images of the dataset\n    classes -- integer, number of classes\n\n    Returns:\n    model -- a Model() instance in Keras\n    \"\"\"\n    \n    # Define the input as a tensor with shape input_shape\n    X_input = Input(input_shape)\n\n    \n    # Zero-Padding\n    X = ZeroPadding2D((3, 3))(X_input)\n    \n    # Stage 1\n    X = Conv2D(64, (3, 3), strides = (1, 1), name = 'conv1', kernel_initializer = glorot_uniform(seed=0))(X)\n    X = BatchNormalization(axis = 3, name = 'bn_conv1')(X)\n    X = Activation('relu')(X)\n    X = MaxPooling2D((3, 3), strides=(1, 1))(X)\n\n    X = Dropout(rate=0.3)(X)  # Set into ResNet50\n\n    # Stage 2\n    X = convolutional_block(X, f = 3, filters = [64, 64, 256], stage = 2, block='a', s = 1)\n    X = identity_block(X, 3, [64, 64, 256], stage=2, block='b')\n    X = identity_block(X, 3, [64, 64, 256], stage=2, block='c')\n\n    ### START CODE HERE ###\n\n    # Stage 3 (≈4 lines)\n    X = convolutional_block(X, f=3, filters = [128, 128, 512], stage = 3, block='a', s = 2)\n    X = identity_block(X, f=3, filters = [128,128,512], stage=3, block='b')\n    X = identity_block(X, f=3, filters = [128,128,512], stage=3, block='c')\n    X = identity_block(X, f=3, filters = [128,128,512], stage=3, block='d')\n\n    # Stage 4 (≈6 lines)\n    X = convolutional_block(X, f=3, filters = [256, 256, 1024], stage = 4, block='a', s = 2)\n    X = identity_block(X, f=3, filters = [256,256,1024], stage=4, block='b')\n    X = identity_block(X, f=3, filters = [256,256,1024], stage=4, block='c')\n    X = identity_block(X, f=3, filters = [256,256,1024], stage=4, block='d')\n    X = identity_block(X, f=3, filters = [256,256,1024], stage=4, block='e')\n    X = identity_block(X, f=3, filters = [256,256,1024], stage=4, block='f')\n\n    # Stage 5 (≈3 lines)\n    X = convolutional_block(X, f=3, filters = [512, 512, 2048], stage = 5, block='a', s = 2)\n    X = identity_block(X, f=3, filters = [512,512,2048], stage=5, block='b')\n    X = identity_block(X, f=3, filters = [512,512,2048], stage=5, block='c')\n\n    # AVGPOOL (≈1 line). Use \"X = AveragePooling2D(...)(X)\"\n    X = AveragePooling2D(pool_size=(2,2))(X)\n    \n    X = Dropout(rate=0.3)(X) # add to ResNet50\n\n    # output layer\n    X = Flatten()(X)\n    \n    X = Dense(36, activation='relu', kernel_initializer = glorot_uniform(seed=0))(X)  ##\n   # X = Dropout(rate=0.2)(X)                                                            ###  Set into ResNet50\n    #X = Dense(36, activation='relu', kernel_initializer = glorot_uniform(seed=0))(X)  ## \n    \n    X = Dense(36, activation='sigmoid', kernel_initializer = glorot_uniform(seed=0))(X)  ##\n    X = Dropout(rate=0.3)(X)                                                            ###  Set into ResNet50\n    X = Dense(36, activation='sigmoid', kernel_initializer = glorot_uniform(seed=0))(X)  ## \n    \n    X = Dense(classes, activation='softmax', name='fc' + str(classes), kernel_initializer = glorot_uniform(seed=0))(X)\n    \n    \n    # Create model\n    model = Model(inputs = X_input, outputs = X, name='ResNet50')\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"55ef9ae2-649a-452c-b55a-d7d444eb4e8c","collapsed":true,"_uuid":"3ff3e01a4a7d30177b74edc16b63ef8ebeebfaaf","trusted":false},"cell_type":"code","source":"def Compile_Fit_ResNet50(input_shape, number_of_classes) :\n    '''\n    Compile and fit a ResNet50 model with the Digit Recognizer training images' set\n    \n    Arguments :\n    input_shape -- shape of the images of the dataset\n    number_of_classes -- integer, number of classes\n    \n    Returns:\n    A fitted ResNet50 model using data augmentation\n    '''\n    Digit_ResNet50_Model = ResNet50(input_shape=input_shape, classes=number_of_classes)\n    Digit_ResNet50_Model.compile(optimizer=\"Adam\", loss=\"categorical_crossentropy\",\n                                 metrics = [\"accuracy\"])\n    \n    datagen = ImageDataGenerator(\n            featurewise_center=False,  # set input mean to 0 over the dataset\n            samplewise_center=False,  # set each sample mean to 0\n            featurewise_std_normalization=False,  # divide inputs by std of the dataset\n            samplewise_std_normalization=False,  # divide each input by its std\n            zca_whitening=False,  # apply ZCA whitening\n            rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n            zoom_range = 0.1, # Randomly zoom image \n            width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n            height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n            horizontal_flip=False,  # randomly flip images\n            vertical_flip=False)  # randomly flip images\n\n    datagen.fit(X_train)\n    \n    learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', \n                                                patience=3, \n                                                verbose=1, \n                                                factor=0.5, \n                                                min_lr=0.00001)\n    \n    epochs = 20 # Turn epochs to 30 to get 0.9967 accuracy\n    batch_size = 86\n    \n    history = Digit_ResNet50_Model.fit_generator(\n                                  datagen.flow(X_train,Y_train, batch_size=batch_size),\n                                  epochs = epochs, \n                                  validation_data = (X_val,Y_val),\n                                  verbose = 1, \n                                  steps_per_epoch=X_train.shape[0] // batch_size, \n                                  callbacks=[learning_rate_reduction])\n    return history\n    ","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"ce6e72db-87a0-4006-938d-e8f36eac2641","collapsed":true,"_uuid":"2c32fa86d7f165ea40463e4ac9d40b7a7ef1d910","trusted":false},"cell_type":"code","source":"def Visualize_Learning_Curve(fittedModel, model_name):\n    '''\n    Visualize the Simple_CNN's learning curve\n    \n    Argument:\n        fittedModel --> a fitting result of a model, model.fit()\n        model_name  --> a string, will be same as the plot's title\n    '''\n    history = fittedModel\n    fig, ax = plt.subplots(1,2)\n    plt.suptitle(model_name)\n    ax[0].plot(history.history['loss'], color='b', label=\"Training loss\")\n    ax[0].plot(history.history['val_loss'], color='r', label=\"validation loss\",axes =ax[0])\n    legend = ax[0].legend(loc='best', shadow=True)\n\n    ax[1].plot(history.history['acc'], color='b', label=\"Training accuracy\")\n    ax[1].plot(history.history['val_acc'], color='r',label=\"Validation accuracy\")\n    legend = ax[1].legend(loc='best', shadow=True)\n    # end def","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"19586026-a3aa-48fb-8ec6-9ed94d0abc64","collapsed":true,"_uuid":"8b7a930e13a9ef82ced821b17de4465e85c0b686","trusted":false},"cell_type":"code","source":"def Get_Confusion_Matrix(fittedModel, X_val, Y_val):\n    '''\n    Compute a confusion matrix w.r.t a fitted model\n    \n    Arguments:\n        fittedModel --> a fitting of model, model.fit()\n        X_val       --> image set for validation\n        Y_val       --> labels, corresponding to X_val\n    \n    Return:\n        ConfutionMatrix, utilization of the function confusion_matrx()\n    '''\n    model = fittedModel.model\n    # Do prediction from the validation dataset, get probabilities for each label\n    Y_predicted_prob = model.predict(X_val)     # shape = (2100,10)\n    # The highest probability's index gives the predicted digit\n    Y_predicted_digits = np.argmax(Y_predicted_prob,axis = 1)  # shape = (2100,)\n    # Get the true digits from the encoded labels in the validation set \n    Y_true_digits = np.argmax(Y_val,axis = 1)          # shape = (2100,)\n\n    ConfusionMatrix = confusion_matrix(Y_true_digits, Y_predicted_digits) \n    \n    return ConfusionMatrix\n\n\n\ndef plot_confusion_matrix(cm, classes, model_name,\n                          normalize=False,\n                          title='Confusion matrix ',\n                          cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title + model_name)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, cm[i, j],\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n    # end def","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"83695d58-8756-4c94-9796-0b0280757a31","collapsed":true,"_uuid":"af693222e6b57c095b04394ac9774841f1e48683","trusted":false},"cell_type":"code","source":"def Preparation_for_display_errors(fittedModel, X_val, Y_val, topNumber):\n    '''\n    Generate parameters will be used to display errors\n    \n    Arguments :\n        fittedModel --> a fitting result of a model, model.fit()\n        X_val       --> image set for validation\n        Y_val       --> labels, corresponding to X_val\n        topNumber   --> integer, number of top errors will be displayed\n    \n    Return :\n        most_important_errors (errors_index) --> indices of errors will be displayed. \n        True_pixel (img_errors)              --> true pixel-values corresponding to the wrong prediction\n        Wrong_predicted_digits (pred_errors) --> digits (labels) are wrong prediction\n        True_digits (obs_errors)             --> true digits (labels) corresponding to the wrong prediction\n    '''\n    \n    model = fittedModel.model\n    # Do prediction from the validation dataset, get probabilities for each label\n    Y_predicted_prob = model.predict(X_val)     # shape = (2100,10)\n    # The highest probability's index gives the predicted digit\n    Y_predicted_digits = np.argmax(Y_predicted_prob,axis = 1)  # shape = (2100,)\n    # Get the true digits from the encoded labels in the validation set \n    Y_true_digits = np.argmax(Y_val,axis = 1)          # shape = (2100,)\n\n    # Visualize error prediction v.s. real digits in validation set\n    errors = (Y_true_digits - Y_predicted_digits != 0) # [a boolean list]\n    Wrong_predicted_digits = Y_predicted_digits[errors] # these digits are wrong prediction\n    Wrong_predicted_prob   = Y_predicted_prob[errors]  # these probabilities are wrong prediction\n\n    # Probabilities of the wrong predicted digits\n    Wrong_predicted_prob_digits = np.max(Wrong_predicted_prob, axis=1) \n\n    True_digits = Y_true_digits[errors] # true digits corresponding to the wrong prediction\n    True_pixel = X_val[errors] # true pixel-values corresponding to the wrong prediction\n\n    # Predicted probabilities of the true values in the error set\n    true_prob_errors = np.diagonal(np.take(Wrong_predicted_prob, True_digits, axis=1))\n\n    # Difference between the probability of the predicted label and the true label\n    delta_pred_true_errors = Wrong_predicted_prob_digits - true_prob_errors\n\n    # Sorted list of the delta prob errors\n    sorted_dela_errors = np.argsort(delta_pred_true_errors)\n    \n    # Top errors \n    most_important_errors = sorted_dela_errors[-topNumber:]\n    \n    return most_important_errors, True_pixel, Wrong_predicted_digits, True_digits\n\n\ndef display_errors(errors_index,img_errors,pred_errors, obs_errors, model_name):\n    \"\"\" \n    This function shows 6 images with their predicted and real labels\n    \n    Arguments:\n        errors_index --> indices of errors will be displayed\n        img_errors   --> true pixel-values corresponding to the wrong prediction\n        pred_errors  --> digits (labels) are wrong prediction\n        obs_errors   --> true digits (labels) corresponding to the wrong prediction\n    \"\"\"\n    n = 0\n    nrows = 2\n    ncols = 3\n    fig, ax = plt.subplots(nrows,ncols,sharex=True,sharey=True)\n    plt.suptitle(\"Top 6 errors for \" + model_name, y=1.05)\n    for row in range(nrows):\n        for col in range(ncols):\n            error = errors_index[n]\n            ax[row,col].imshow((img_errors[error]).reshape((28,28)))\n            ax[row,col].set_title(\"Predicted label :{}\\nTrue label :{}\".format(pred_errors[error],\n                                                                               obs_errors[error]))\n            n += 1\n            \n    plt.subplots_adjust(top=0.92, bottom=0.08, left=0.10, right=0.95, hspace=0.25,\n                    wspace=0.35)\n    plt.tight_layout()   # adjust the distance between subplots  \n            \n    # end ","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"31057a05-657e-42f2-a5b3-21feed1cc4d8","collapsed":true,"_uuid":"a61208e124d5034b74bc293a1b927cf16b67d56e","trusted":false},"cell_type":"code","source":"def Predict_and_Submit(fittedModel, TestSet, csv_name):\n    '''\n    Use a fitted model to generate prediction on test set and a submission csv\n    \n    Argument:\n    fittedModel --> a fitted model, expected form: history.model\n    TestSet     --> imported data set for test and scoring\n    csv_name    --> string, name of the returned csv file\n    \n    Return:\n    csv file for submission\n    '''\n    prediction =  fittedModel.predict(TestSet) # an array of probabilities \n    predicted_Digits = np.argmax(prediction, axis=1) # a highest probability in each row gives a predicted digit\n    predicted_Digits = pd.Series(predicted_Digits, name=\"Label\")\n    submission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),\n                            predicted_Digits],axis = 1)\n    submission_CSV = submission.to_csv(csv_name+\".csv\", index=False)\n    \n    return submission_CSV\n    ","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"3bbe6919-ac42-441d-b7d1-508a70a283a6","_uuid":"9e4a90bc059bf70fe8175c10ad3e11800b394f8b","trusted":false,"collapsed":true},"cell_type":"code","source":"Modelling_SimpleCNN = Compile_Fit_Simple_CNN()\nModelling_ResNet50 = Compile_Fit_ResNet50(input_shape=X_train.shape[1:], number_of_classes=10)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"6f4c016c-6acb-48ce-bd74-2c62e50bdb0f","_uuid":"29560bffa382da1049d8fa8a3f1e7d69f8fe84f9","trusted":false,"collapsed":true},"cell_type":"code","source":"Visualize_Learning_Curve(Modelling_ResNet50, \"ResNet50\")\nVisualize_Learning_Curve(Modelling_SimpleCNN, \"SimpleCNN\")","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"f0c9540c-0fd3-4c76-a07c-223770d792b5","_uuid":"1792dd1e0f69317b7801d706c5cd58e83c5e09c1","trusted":false,"collapsed":true},"cell_type":"code","source":"ConfusionMatrix_ResNet50 = Get_Confusion_Matrix(Modelling_ResNet50, X_val=X_val, Y_val=Y_val)\nplot_confusion_matrix(ConfusionMatrix_ResNet50, classes = range(10), model_name=\"ResNet50\") \n\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"ca68a093-e1aa-4588-b1b9-5574d2ed0a86","_uuid":"02d9592726f516a91949479ade66893a19b21620","trusted":false,"collapsed":true},"cell_type":"code","source":"ConfusionMatrix_SimpleCNN = Get_Confusion_Matrix(Modelling_SimpleCNN, X_val=X_val, Y_val=Y_val)\nplot_confusion_matrix(ConfusionMatrix_SimpleCNN, classes = range(10), model_name=\"SimpleCNN\") ","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"5908241a-5ae8-4e2d-8448-2115ef6812f5","_uuid":"e4dfdfad402efebba833c9ab94acc3a09909a316","trusted":false,"collapsed":true},"cell_type":"code","source":"most_important_errors, True_pixel, Wrong_predicted_digits, True_digits = Preparation_for_display_errors(Modelling_ResNet50, \n                                                                                                       X_val, Y_val, 6)\ndisplay_errors(most_important_errors, True_pixel, Wrong_predicted_digits, True_digits, \"ResNet50\")","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"3415c960-e107-496a-95c5-1dec995065e4","_uuid":"27a0b419f1ff19a1d9c36143e7b363d4a049ee22","trusted":false,"collapsed":true},"cell_type":"code","source":"most_important_errors, True_pixel, Wrong_predicted_digits, True_digits = Preparation_for_display_errors(Modelling_SimpleCNN, \n                                                                                                        X_val, Y_val, 6)\ndisplay_errors(most_important_errors, True_pixel, Wrong_predicted_digits, True_digits, \"SimpleCNN\")","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"2b792ab3-8c31-4c6c-9014-250952749c31","_uuid":"5285d16e4ef693340950755f93cccf2e58d49a16","trusted":false,"collapsed":true},"cell_type":"code","source":"submission_SimpleCNN = Predict_and_Submit(Modelling_SimpleCNN.model, test, \"submission_SimpleCNN\")\nsubmission_ResNet50 = Predict_and_Submit(Modelling_ResNet50.model, test, \"submission_ResNet50\")","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}