{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom keras.utils.np_utils import to_categorical\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, BatchNormalization\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import LearningRateScheduler\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_train = train[\"label\"]\nX_train = train.drop(labels = [\"label\"],axis = 1)\n\n# Given images are not already normalized (i.e. the pixel values are in the range [0,255]).\n# So normalize them by dividing by 255\nX_train = X_train / 255.0\ntest = test / 255.0","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.values.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = X_train.values.reshape(-1,28,28,1)\nX_test = test.values.reshape(-1,28,28,1)\nY_train = to_categorical(Y_train, num_classes = 10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.figure(figsize=(15,4.5))\nfor i in range(30):  \n    plt.subplot(3, 10, i+1)\n    plt.imshow(X_train[i].reshape((28,28)),cmap=plt.cm.binary)\n    plt.axis('off')\nplt.subplots_adjust(wspace=-0.1, hspace=-0.1)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datagen = ImageDataGenerator(\n        rotation_range=10,  # randomly rotate images through any degree between 0 and 360\n        zoom_range = 0.10,  \n        width_shift_range=0.1, # a float number, that would indicate the percentage of width or height of the image to shift. \n        height_shift_range=0.1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape # (42000, 28, 28, 1)\nX_train[25,].shape # (28, 28, 1)\nX_train[25,].reshape((1,28,28,1)).shape # (1, 28, 28, 1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train[25,].shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train[41999,].shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(X_train[18,].reshape((28,28)),cmap=plt.cm.binary)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_random = X_train[25,].reshape((1,28,28,1))\nY_train_random = Y_train[25,].reshape((1,10))\n\nplt.figure(figsize=(15,4.5))\n\nfor i in range(30):  \n    plt.subplot(3, 10, i+1)\n    X_train_augmented, Y_train_augmented = datagen.flow(X_train_random,Y_train_random).next()\n    plt.imshow(X_train_augmented[0].reshape((28,28)),cmap=plt.cm.binary)\n    plt.axis('off')\n    # Now for the second and third row take 2 different digit\n    # 9 and 7 respectively\n    # And because I am printing 9 columns for the same image, I have to \n    # advance the 'i' value by 10\n    if i==9:\n        X_train_random = X_train[11,].reshape((1,28,28,1))\n    if i==19:\n        X_train_random = X_train[18,].reshape((1,28,28,1))\nplt.subplots_adjust(wspace=-0.1, hspace=-0.1)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_model(num_cnn):\n    model = [0] *num_cnn\n    for j in range(num_cnn):\n        model[j] = Sequential()\n\n        model[j].add(Conv2D(32, kernel_size = 3, activation='relu', input_shape = (28, 28, 1))) # Two stacked 3x3 filters instead of single 5x5 filters. \n        model[j].add(BatchNormalization()) # Batch normalization is added\n        model[j].add(Conv2D(32, kernel_size = 3, activation='relu')) # and ReLU activation replaces sigmoid.\n        model[j].add(BatchNormalization())\n        model[j].add(Conv2D(32, kernel_size = 5, strides=2, padding='same', activation='relu')) # Stride 2 replaces pooling layers. These become learnable pooling layers.\n        model[j].add(BatchNormalization())\n        model[j].add(Dropout(0.4)) #  Dropout is added\n\n        model[j].add(Conv2D(64, kernel_size = 3, activation='relu'))\n        model[j].add(BatchNormalization())\n        model[j].add(Conv2D(64, kernel_size = 3, activation='relu'))\n        model[j].add(BatchNormalization())\n        model[j].add(Conv2D(64, kernel_size = 5, strides=2, padding='same', activation='relu'))\n        model[j].add(BatchNormalization())\n        model[j].add(Dropout(0.4))\n\n        model[j].add(Conv2D(128, kernel_size = 4, activation='relu'))\n        model[j].add(BatchNormalization())\n        model[j].add(Flatten())\n        model[j].add(Dropout(0.4))\n        model[j].add(Dense(10, activation='softmax'))\n\n        model[j].compile(optimizer=\"adam\", loss=\"categorical_crossentropy\", metrics=[\"accuracy\"])\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_cnn=10\nmodel = build_model(num_cnn)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(num_epochs):\n    # Keep reducing the Learning Rate\n    lr_scheduler = LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x)\n    # Training starts\n    history = [0] * num_cnn\n    for cnn in range(num_cnn):\n        X_train_split, X_val_split, Y_train_split, Y_val_split = train_test_split(\n            X_train, Y_train, test_size=0.1\n        )\n        history[cnn] = model[cnn].fit(\n            datagen.flow(X_train_split, Y_train_split, batch_size=64),\n            epochs=num_epochs,\n            steps_per_epoch=X_train_split.shape[0] // 64,\n            validation_data=(X_val_split, Y_val_split),\n            callbacks=[lr_scheduler],\n            verbose=0,\n        )\n        print(\n            \"CNN {0:d}: Epochs={1:d}, Train accuracy={2:.5f}, Validation accuracy={3:.5f}\".format(\n                cnn + 1,\n                num_epochs,\n                max(history[cnn].history[\"accuracy\"]),\n                max(history[cnn].history[\"val_accuracy\"]),\n            )\n        )\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_epochs = 45\ntrain(num_epochs)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_ensemble_prediction = np.zeros( (X_test.shape[0],10) )\n\nfinal_ensemble_prediction.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for j in range(num_cnn):\n    final_ensemble_prediction = np.zeros( (X_test.shape[0],10) )\n    final_ensemble_prediction = final_ensemble_prediction + model[j].predict(X_test)\n\n    final_ensemble_prediction = np.argmax(final_ensemble_prediction,axis = 1)\n    \n#     print(\"final_ensemble_prediction.shape \", final_ensemble_prediction.shape)\n\n    final_ensemble_prediction = pd.Series(final_ensemble_prediction,name=\"Label\")\n\n    submission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),final_ensemble_prediction],axis = 1)\n\n    submission.to_csv(\"MNIST_ensemble_cnn.csv\",index=False)\n    \n    print('success')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}