{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Digit Recognizer","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport seaborn as sns\nimport os","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"../input/digit-recognizer/train.csv\")\ntest  = pd.read_csv(\"../input/digit-recognizer/test.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('train.shape  :', train.shape)\nprint('test.shape   :', test.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def print_long_list(x, n_items=7, str_len=15):\n    print('len:', len(x))\n    for i in range(0, len(x), n_items):\n        for j in x[i:min(i+n_items, len(x))]:\n            print(str('{0: <'+str(str_len)+'}').format(str(j)), end=' ')\n        print()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_datatypes(df):\n    dtypes_col, dtypes_cnt = {}, {}\n    for col in df:\n        x = str(df[col].dtype)\n        if x not in dtypes_col:\n            dtypes_col[x] = [col]\n            dtypes_cnt[x] = 1\n        else:\n            dtypes_col[x] += [col]\n            dtypes_cnt[x] += 1\n    return dtypes_cnt, dtypes_col","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compare_columns(df1, df2):\n    df1_columns_set = set(df1.columns)\n    df2_columns_set = set(df2.columns)\n    print('df1_columns_set - df2_columns_set :', df1_columns_set - df2_columns_set)\n    print('df2_columns_set - df1_columns_set :', df2_columns_set - df1_columns_set)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dtypes_cnt, dtypes_col = get_datatypes(train)\ndtypes_cnt","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dtypes_cnt, dtypes_col = get_datatypes(test)\ndtypes_cnt","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"compare_columns(train, test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_train = train[\"label\"]\nX_train = train.drop(labels = [\"label\"],axis = 1)\nX_test = test\ndel train, test","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,6))\ng = sns.countplot(Y_train, palette=\"icefire\")\nplt.title(\"Number of digit classes\")\nY_train.value_counts()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.isnull().any().describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test.isnull().any().describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = X_train / 255.0\nX_test  = X_test  / 255.0","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = X_train.values.reshape(-1, 28, 28, 1)\nX_test  = X_test.values.reshape(-1, 28, 28, 1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.metrics         import confusion_matrix\n\nfrom tensorflow.keras.utils               import to_categorical\nfrom tensorflow.keras.models              import Sequential\nfrom tensorflow.keras.layers              import (Dense, Dropout, Flatten, Conv2D, MaxPool2D,\n                                                  Lambda, BatchNormalization)\nfrom tensorflow.keras.optimizers          import RMSprop\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks           import ReduceLROnPlateau\nfrom tensorflow.keras.optimizers          import RMSprop, Adam, SGD\nfrom tensorflow.keras.utils               import plot_model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('X_train.shape  :', X_train.shape)\nprint('Y_train.shape  :', Y_train.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_gen = ImageDataGenerator(\n                rotation_range=10, \n                zoom_range = 0.1,\n                width_shift_range=0.1,  \n                height_shift_range=0.1,\n            )\ndata_gen.fit(X_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('X_train.shape  :', X_train.shape)\nprint('Y_train.shape  :', Y_train.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_train = to_categorical(Y_train, num_classes = 10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print_long_list(Y_train[:5], n_items=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('X_train.shape  :', X_train.shape)\nprint('X_val.shape    :', X_val.shape)\nprint('Y_train.shape  :', Y_train.shape)\nprint('Y_val.shape    :', Y_val.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(X_train[100][:,:,0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('X_train.shape  :', X_train.shape)\nprint('X_val.shape    :', X_val.shape)\nprint('Y_train.shape  :', Y_train.shape)\nprint('Y_val.shape    :', Y_val.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_px = X_train.mean().astype(np.float32)\nstd_px  = X_train.std().astype(np.float32)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def linear_model(optimizer):\n    model= Sequential()\n    model.add(Lambda(lambda x: (x-mean_px)/std_px, input_shape=(28,28,1)))\n    model.add(Flatten())\n    model.add(Dense(10, activation='softmax'))\n    model.compile(optimizer = optimizer,\n                  loss='categorical_crossentropy', \n                  metrics=['accuracy'])\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fully_connected_nn_model(optimizer):\n    model= Sequential()\n    model.add(Lambda(lambda x: (x-mean_px)/std_px, input_shape=(28,28,1)))\n    model.add(Flatten())\n    model.add(Dense(512, activation='relu'))\n    model.add(Dense(10, activation='softmax'))\n    model.compile(optimizer = optimizer,\n                  loss='categorical_crossentropy', \n                  metrics=['accuracy'])\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def conv_nn_model(optimizer):\n    model= Sequential()\n    model.add(Lambda(lambda x: (x-mean_px)/std_px, input_shape=(28,28,1)))\n    model.add(Conv2D(32,(3,3), activation='relu')),\n    model.add(Conv2D(32,(3,3), activation='relu')),\n    model.add(MaxPool2D()),\n    model.add(Conv2D(64,(3,3), activation='relu')),\n    model.add(Conv2D(64,(3,3), activation='relu')),\n    model.add(MaxPool2D()),\n    model.add(Flatten())\n    model.add(Dense(512, activation='relu'))\n    model.add(Dense(10, activation='softmax'))\n    model.compile(optimizer = optimizer,\n                  loss='categorical_crossentropy',\n                  metrics=['accuracy'])\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def conv_nn_with_batch_normalization_model(optimizer):\n    model= Sequential()\n    model.add(Lambda(lambda x: (x-mean_px)/std_px, input_shape=(28,28,1)))\n    model.add(Conv2D(32,(3,3), activation='relu')),\n    model.add(BatchNormalization(axis=1)),\n    model.add(Conv2D(32,(3,3), activation='relu')),\n    model.add(MaxPool2D()),\n    model.add(BatchNormalization(axis=1)),\n    model.add(Conv2D(64,(3,3), activation='relu')),\n    model.add(BatchNormalization(axis=1)),\n    model.add(Conv2D(64,(3,3), activation='relu')),\n    model.add(MaxPool2D()),\n    model.add(Flatten())\n    model.add(Dense(512, activation='relu'))\n    model.add(BatchNormalization()),\n    model.add(Dense(10, activation='softmax'))\n    model.compile(optimizer = optimizer,\n                  loss='categorical_crossentropy',\n                  metrics=['accuracy'])\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def conv_nn_with_dropout_model(optimizer):\n    model = Sequential()\n    model.add(Conv2D(filters = 32, kernel_size = (5,5), padding = 'Same', activation ='relu', \n                     input_shape = (28,28,1)))\n    model.add(Conv2D(filters = 32, kernel_size = (5,5), padding = 'Same', activation ='relu'))\n    model.add(MaxPool2D(pool_size=(2,2)))\n    model.add(Dropout(0.25))\n    model.add(Conv2D(filters = 64, kernel_size = (3,3), padding = 'Same', activation ='relu'))\n    model.add(Conv2D(filters = 64, kernel_size = (3,3), padding = 'Same', activation ='relu'))\n    model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))\n    model.add(Dropout(0.25))\n    model.add(Flatten())\n    model.add(Dense(256, activation = \"relu\"))\n    model.add(Dropout(0.5))\n    model.add(Dense(10, activation = \"softmax\"))\n    model.compile(optimizer = optimizer,\n                  loss='categorical_crossentropy',\n                  metrics=['accuracy'])\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fit_model(model, lr_reduction, epochs=10, batch_size=100):\n    batches = data_gen.flow(X_train,Y_train, batch_size=batch_size)\n    history = model.fit_generator(generator = batches,\n                                  epochs = epochs,\n                                  validation_data = (X_val,Y_val),\n                                  steps_per_epoch = batch_size,\n                                  validation_steps = batch_size,\n                                  callbacks = [lr_reduction],\n                                 )\n    return model, history","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def apply_learning_algorithm(model):\n    lr_reduction = ReduceLROnPlateau(monitor='val_acc', patience=2, factor=0.5, min_lr=0.00001)\n    model, history = fit_model(model, lr_reduction, epochs=30, batch_size=75)\n    return model, history","metadata":{"_kg_hide-output":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = [\n    linear_model(RMSprop(learning_rate=0.001)),\n    linear_model(Adam(learning_rate=0.001)),\n    fully_connected_nn_model(RMSprop(learning_rate=0.001)),\n    fully_connected_nn_model(Adam(learning_rate=0.001)),\n    conv_nn_model(RMSprop(learning_rate=0.001)),\n    conv_nn_model(Adam(learning_rate=0.001)),\n    conv_nn_with_batch_normalization_model(RMSprop(learning_rate=0.001)),\n    conv_nn_with_batch_normalization_model(Adam(learning_rate=0.001)),\n    conv_nn_with_dropout_model(RMSprop(learning_rate=0.001)),\n    conv_nn_with_dropout_model(Adam(learning_rate=0.001)),\n]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_names = [\n    'linear_model(RMSprop(lr=0.001))',\n    'linear_model(Adam(lr=0.001))',\n    'fully_connected_nn_model(RMSprop(lr=0.001))',\n    'fully_connected_nn_model(Adam(lr=0.001))',\n    'conv_nn_model(RMSprop(lr=0.001))',\n    'conv_nn_model(Adam(lr=0.001))',\n    'conv_nn_with_batch_normalization_model(RMSprop(lr=0.001))',\n    'conv_nn_with_batch_normalization_model(Adam(lr=0.001))',\n    'conv_nn_with_dropout_model(RMSprop(lr=0.001))',\n    'conv_nn_with_dropout_model(Adam(lr=0.001))',\n]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_history, all_models = [], []\nfor model in models:\n    fitted_model, history = apply_learning_algorithm(model)\n    all_models += [fitted_model]\n    all_history += [history]","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_acc = pd.DataFrame(model_names, columns = ['NN Model'])\nfinal_acc['Accuracy'] = [max(history.history['accuracy']) for history in all_history]\nfinal_acc","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (12,6))\nsns.barplot(final_acc['NN Model'], final_acc['Accuracy'])\nplt.xlabel('Model',   fontsize = 12)\nplt.ylabel('Accuracy', fontsize = 12)\nplt.ylim(0.6, 1.0)\nplt.xticks(rotation=90)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_model_name = final_acc.sort_values(by=['Accuracy']).tail(1)['NN Model'].iloc[0]\nbest_model = all_models[model_names.index(best_model_name)]\nbest_history = all_history[model_names.index(best_model_name)]\nprint(best_model_name, best_model)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display_histpry_details(history):\n    for i in history.history:\n        print('===>', i)\n        vals = [round(x, 3) for x in history.history[i]]\n        print_long_list(vals, n_items=10, str_len=9)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_model_history(history, fig_title, shape=(24,6)):\n    fig, ax = plt.subplots(1,2, figsize=shape)\n    fig.suptitle(fig_title, fontsize=16)\n    ax[0].plot(history.history['loss'],     color='b', label=\"Training loss\")\n    ax[0].plot(history.history['val_loss'], color='r', label=\"validation loss\",axes =ax[0])\n    legend = ax[0].legend(loc='best', shadow=True)\n    ax[0].set_xlabel('Epochs')\n    ax[0].set_ylabel('Loss')\n    ax[1].plot(history.history['accuracy'],     color='b', label=\"Training accuracy\")\n    ax[1].plot(history.history['val_accuracy'], color='r', label=\"Validation accuracy\")\n    legend = ax[1].legend(loc='best', shadow=True)\n    ax[1].set_xlabel('Epochs')\n    ax[1].set_ylabel('Accuracy')\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_histpry_details(best_history)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_model_history(best_history, 'History of the Model')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_confusion_matrix(cm, shape=(10,10)):\n    f, ax = plt.subplots(figsize=shape)\n    _ = sns.heatmap(cm, \n                    square=True, \n                    cbar_kws={'shrink':.9 }, \n                    ax=ax,\n                    annot=True, \n                    linewidths=0.01,\n                    vmax=1.0, \n                    linecolor='white',\n                    fmt= '.0f',\n                    annot_kws={'fontsize':14},\n                    )\n    plt.title('Correlation of Prediction', y=1.05, size=15)\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_pred = best_model.predict(X_val)\nY_pred_classes = np.argmax(Y_pred,axis = 1) \nY_true = np.argmax(Y_val,axis = 1) \ncm = confusion_matrix(Y_true, Y_pred_classes)\nplot_confusion_matrix(cm, shape=(10,10))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display_errors(sorted_dela_errors, img_errors, pred_errors, obs_errors):\n    i, nrows, ncols  = 0, 4, 8\n    shape = (ncols*3, nrows*3.25)\n    errors_index = sorted_dela_errors[-nrows*ncols:]\n    fig, ax = plt.subplots(nrows, ncols, sharex=True, sharey=True, figsize=shape)\n    for row in range(nrows):\n        for col in range(ncols):\n            error = errors_index[i]\n            ax[row,col].imshow((img_errors[error]).reshape((28,28)))\n            ax[row,col].set_title(\"Predicted label :{}\\nTrue label :{}\".format(pred_errors[error],obs_errors[error]))\n            i += 1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"errors = (Y_pred_classes - Y_true != 0)\nY_pred_classes_errors = Y_pred_classes[errors]\nY_pred_errors = Y_pred[errors]\nY_true_errors = Y_true[errors]\nX_val_errors = X_val[errors]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_pred_errors_prob = np.max(Y_pred_errors, axis = 1)\ntrue_prob_errors = np.diagonal(np.take(Y_pred_errors, Y_true_errors, axis=1))\ndelta_pred_true_errors = Y_pred_errors_prob - true_prob_errors\nsorted_dela_errors = np.argsort(delta_pred_true_errors)\ndisplay_errors(sorted_dela_errors, X_val_errors, Y_pred_classes_errors, Y_true_errors)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = best_model.predict(X_test)\nresults = np.argmax(results,axis = 1)\nresults = pd.Series(results,name=\"Label\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\nsubmission.to_csv(\"cnn_mnist_datagen.csv\",index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}