{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Digit Recognizer | CNN | Augmentation | Ensemble | All I did for 99.757% Accuracy","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Please upvote if helpful","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Import Stuffs","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nnp.random.seed(0)\n\nfrom os.path import join\nimport random\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom matplotlib import rcParams\n%matplotlib inline\nimport itertools\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\n\nfrom keras.utils.np_utils import to_categorical\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, BatchNormalization\nfrom keras.optimizers import RMSprop\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ReduceLROnPlateau, EarlyStopping","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Set parameters for Seaborn and Matplotlib","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.set(style = 'white' , context = 'notebook' , palette = 'deep')\nrcParams['figure.figsize'] = 10,6","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Load Train and Test data ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"data_path = \"../input/digit-recognizer/\"\ntrain  = pd.read_csv(join(data_path,\"train.csv\"))\ntest  = pd.read_csv(join(data_path,\"test.csv\"))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Prepare Train data and show Category distribution","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"Y_train = train[\"label\"]\nX_train = train.drop(labels = [\"label\"], axis = 1)\ndel train\n\np = sns.countplot(Y_train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Reshape X_train and convert Y_train to Categorical","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def process_data(data):\n    data = data/255.0\n    data = data.values.reshape(-1,28,28,1)\n    return data\n\nX_train = process_data(X_train)\nY_train = to_categorical(Y_train,num_classes = 10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Visualise few random Images","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_image(size,images):\n    if len(images)!= size[0]*size[1]:\n        raise Exception(\"number of images doesn't match the size of plot\")\n    fig, ax = plt.subplots(size[0],size[1],figsize=(10,10))\n    for i in range(size[0]):\n        for j in range(size[1]):\n            ax[i][j].imshow(images[i*size[1]+j][:,:,0], cmap = \"gray_r\")\n\nimages = random.sample(list(X_train),9)\nplot_image((3,3),images)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Build 10 CNN Models","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"num = 5 #make it 10\nmodel = [0]*num\nfor i in  range(num):\n    model[i] = Sequential()\n    model[i].add(Conv2D(filters = 32, kernel_size = (5,5), padding = \"same\", activation = \"relu\", input_shape = (28,28,1)))\n    model[i].add(BatchNormalization())\n    model[i].add(Conv2D(filters = 32, kernel_size = (5,5), padding = \"same\", activation = \"relu\"))\n    model[i].add(BatchNormalization())\n    model[i].add(MaxPool2D(pool_size = (2,2)))\n    model[i].add(Dropout(0.25))\n    model[i].add(Conv2D(filters = 64, kernel_size = (3,3), padding = \"same\", activation = \"relu\"))\n    model[i].add(BatchNormalization())\n    model[i].add(Conv2D(filters = 64, kernel_size = (3,3), padding = \"same\", activation = \"relu\"))\n    model[i].add(BatchNormalization())\n    model[i].add(MaxPool2D(pool_size=(2,2), strides=(2,2)))\n    model[i].add(Dropout(0.25))\n    model[i].add(Conv2D(filters = 64, kernel_size = (3,3), padding = \"same\", activation = \"relu\"))\n    model[i].add(BatchNormalization())\n    model[i].add(Dropout(0.25))\n    model[i].add(Flatten())\n    model[i].add(Dense(1024, activation = \"relu\"))\n    model[i].add(BatchNormalization())\n    model[i].add(Dropout(0.25))\n    model[i].add(Dense(10, activation = \"softmax\"))\n    model[i].compile(optimizer = \"adam\", loss = \"categorical_crossentropy\", metrics = [\"accuracy\"])\nmodel[0].summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Data Augmentation using Keras ImageDataGenerator","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"datagen = ImageDataGenerator(featurewise_center = False,\n                             samplewise_center = False,\n                             featurewise_std_normalization = False,\n                             samplewise_std_normalization = False,\n                             zca_whitening = False,\n                             rotation_range = 10,\n                             zoom_range = 0.1,\n                             width_shift_range = 0.1,\n                             height_shift_range = 0.1,\n                             horizontal_flip = False,\n                             vertical_flip = False\n                            )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Define Batch & Epoch and Train all Models","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = 32\nbatch_size = 256\nhistory = [0]*num\nfor i in range(num):\n    random_seed = i\n    X_train_, X_val_, Y_train_, Y_val_ = train_test_split(X_train, Y_train, test_size = 0.2, random_state=random_seed)\n    learning_rate_reduction = ReduceLROnPlateau(monitor='val_accuracy',\n                                                patience=3,\n                                                verbose=1,\n                                                factor=0.5,\n                                                min_lr=0.00001)\n    early_stopping = EarlyStopping(monitor='val_accuracy',\n                                   min_delta=0,\n                                   patience=10, \n                                   verbose =1, \n                                   mode='auto')\n    history[i] = model[i].fit_generator(datagen.flow(X_train_, Y_train_, batch_size = batch_size), epochs = epochs, validation_data = (X_val_,Y_val_), verbose = 1, steps_per_epoch = X_train.shape[0]//batch_size, callbacks=[learning_rate_reduction,early_stopping])\n    print(\"CNN {0:d}: Epochs={1:d}, Train accuracy={2:.5f}, Validation accuracy={3:.5f}\".format(i+1,epochs,max(history[i].history['accuracy']),max(history[i].history['val_accuracy'])))    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Predict Function to Combine Results of all Models (#Ensemble)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict(X_data):\n    results = np.zeros((X_data.shape[0],10))\n    for j in range(num):\n        results = results + model[j].predict(X_data)\n    return results","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Plot Confusion Matrix for last set of Validation Data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n    \n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, cm[i, j],\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n\n    \nY_pred = predict(X_val_)\nY_pred_classes = np.argmax(Y_pred,axis = 1) \nY_true = np.argmax(Y_val_,axis = 1) \nconfusion_mtx = confusion_matrix(Y_true, Y_pred_classes) \nplot_confusion_matrix(confusion_mtx, classes = range(10)) ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Run on Test Data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test = process_data(test)\nresults = predict(test)\nresults = np.argmax(results,axis = 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Generate Table and save Results to CSV File","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"results = pd.Series(results,name=\"Label\")\nsubmission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\nsubmission.to_csv(\"keras_cnn_mnist_aug_ensemble2.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Got Accuracy of 0.99757","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### First one... consider upvoting if helpful","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}