{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b18c794b69073d1c7a36de2a16bc18c4f79a70a7"},"cell_type":"code","source":"\n#libraries\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport seaborn as sns\nsns.set() # setting seaborn default for plots\n\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.model_selection import train_test_split\n\nfrom keras.utils import np_utils\nfrom keras.datasets import mnist\n\n# for Convolutional Neural Network (CNN) model\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Activation, Flatten\nfrom keras.optimizers import Adam\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.utils import np_utils\nfrom keras.layers import Conv2D, MaxPooling2D, ZeroPadding2D, GlobalAveragePooling2D\nfrom keras.layers.advanced_activations import LeakyReLU \nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.optimizers import RMSprop\nfrom keras.callbacks import LearningRateScheduler\n\nfrom keras import backend as K\nK.set_image_dim_ordering('th')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f1d9e6c7930116d4f3d05d4701893f5ce217adce"},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')\nprint (train.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b9359e853475c32481b44505ccdd2a60b10fbe07"},"cell_type":"code","source":"test = pd.read_csv('../input/test.csv')\nprint(test.shape)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"04eb05afa40df8434cce01c339f7902cd48a014f"},"cell_type":"code","source":"# Separating the labels from training dataset and making it as x_label\ny_train = train['label']\nx_train = train.drop(labels=['label'],axis=1)\nx_test = test\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c996e81ae4ecc7e441e3793cec9e3b3b954899e2"},"cell_type":"code","source":"# Set values of the Data\nx_train = x_train.values.astype('float32') # pixel values of all images in train set\ny_train = y_train.values.astype('int32') # labels of all images\nx_test = test.values.astype('float32') # pixel values of all images in test set","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"277829a7e6c2bdf2c5b97a2c47d3fa14f2e9bf4d"},"cell_type":"code","source":"# fix random seed for reproducibility\nrandom_seed = 7\nnp.random.seed(random_seed)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8a5e28676897433bbfbf1295340e5981cf47130a"},"cell_type":"markdown","source":"**\nConverting Output into one hot code**\n\nA one hot encoding is a representation of categorical variables as binary vectors. This first requires that the categorical values be mapped to integer values. Then, each integer value is represented as a binary vector that is all zero values except the index of the integer, which is marked with a 1 and as this is a multi classification problem so we can convert the output class values into one-hot format which is simply a binary matrix, i.e.\n\nvalue 0 will be converted to one-hot format as [1, 0, 0, 0, 0, 0, 0, 0, 0]\n\nvalue 1 will be converted to one-hot format as [0, 1, 0, 0, 0, 0, 0, 0, 0] etc"},{"metadata":{"trusted":true,"_uuid":"e49b0b3e5cd42ae31022de8dd1b8274901f08246"},"cell_type":"code","source":"# one hot encode outputs'\ny_train = np_utils.to_categorical(y_train)\nnum_classes = y_train.shape[1]\nnum_classes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bceaf8140ee1d06c839e01e1686ef316802da51c"},"cell_type":"code","source":"# Reshaping the Image for CNN 2-dimesional input in [samples][pixels][width][height]\nx_train = x_train.reshape(x_train.shape[0], 1, 28, 28).astype('float32')\nx_test = x_test.reshape(x_test.shape[0], 1, 28, 28).astype('float32')\nnum_pixels = x_train.shape[1]\nprint (num_pixels, x_train.shape, x_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b57892a84c00763f4fbeafef84b60cd8a17abcc5"},"cell_type":"markdown","source":"# Ensemble\nUsing ensemble of cnn for training and prediction. Using 10 CNNs.\nModel idea and code from [here](https://www.kaggle.com/cdeotte/25-million-images-0-99757-mnist#)"},{"metadata":{"trusted":true,"_uuid":"410355674a6aaa607ed7474221b267a3d1693bae"},"cell_type":"code","source":"nn = 10\nmodel = [0]*nn\n\nfor j in range(nn):\n    model[j] = Sequential()\n    model[j].add(Conv2D(filters=32, kernel_size=(3, 3), input_shape=(1, 28, 28), activation='relu',data_format='channels_first'))\n    model[j].add(BatchNormalization())\n    model[j].add(Conv2D(filters=32, kernel_size=(3, 3), activation='relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Conv2D(32, kernel_size = 5, strides=2, padding='same', activation='relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Dropout(0.4))\n    \n    model[j].add(Conv2D(filters=64, kernel_size=(3, 3), activation='relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Conv2D(filters=64, kernel_size=(3, 3), activation='relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Conv2D(64, kernel_size = 5, strides=2, padding='same', activation='relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Dropout(0.4))\n    \n    model[j].add(Conv2D(128, kernel_size = 4, activation='relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Flatten())\n    model[j].add(Dropout(0.4))\n    model[j].add(Dense(10, activation='softmax'))\n    \n    # Compile model\n    model[j].compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"397289632b67068409056580832c063282b5a39d"},"cell_type":"code","source":"# With data augmentation to prevent overfitting\ndatagen = ImageDataGenerator(\n        rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n        zoom_range = 0.1, # Randomly zoom image \n        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n        vertical_flip=False)  # randomly flip images\n\n\ndatagen.fit(x_train)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f75f0f1fc6d87ab510faf578c9a9286ac27e1fcf"},"cell_type":"markdown","source":"## Train 10 CNNS\nEverytime before the training, it divides the dataset into training and validation"},{"metadata":{"trusted":true,"_uuid":"6bc642bbe349d040e3b611ccd875fa006225aa4e"},"cell_type":"code","source":"# DECREASE LEARNING RATE EACH EPOCH\nannealer = LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x)\nhistory = [0] * nn\nepochs = 50\n\n# Fit the model\nfor j in range(nn):\n    x_train2, x_val, y_train2, y_val = train_test_split(x_train, y_train, test_size = 0.10, random_state=random_seed)\n    history[j] = model[j].fit_generator(datagen.flow(x_train2,y_train2, batch_size=64),\n                              epochs = epochs, validation_data = (x_val,y_val),\n                              verbose = 0, steps_per_epoch=(len(x_train)//64),validation_steps=(len(x_val)//64),callbacks=[annealer])\n    print(\"CNN {0:d}: Epochs={1:d}, Train accuracy={2:.5f}, Validation accuracy={3:.5f}\".format(\n        j+1,epochs,max(history[j].history['acc']),max(history[j].history['val_acc']) ))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ce47140d93bd39a49107042c34495a4339771b91"},"cell_type":"markdown","source":"## Ensemble Predictions and submitting the result\nTill now I got 0.997 accuracy with just 5 models."},{"metadata":{"trusted":true,"_uuid":"5fe728af8fc75888f4712dd8371ccb6b142cd9ed","_kg_hide-output":true},"cell_type":"code","source":"results = np.zeros( (x_test.shape[0],10) )\nfor j in range(nn):\n    results = results + model[j].predict(x_test)\nresults = np.argmax(results,axis = 1)\nresults = pd.Series(results,name=\"Label\")\nsubmission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\nsubmission.to_csv(\"ENSEMBLE.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f0dd7dfa2547817c5ea8b600bd003e83f717801"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}