{"cells":[{"metadata":{"_uuid":"8ba359691acdf7dc97f036da1cc10afef539000a"},"cell_type":"markdown","source":"# Training Keras network for Digit Recognition - Acc 0.996 (top 10%)"},{"metadata":{"_uuid":"30d530ded3906db941857815085d06ebd731b447"},"cell_type":"markdown","source":"## Importing python modules"},{"metadata":{"trusted":true,"_uuid":"d5960f00de8935207dc17e5e1f9dfe428f7b4863"},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport keras\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e18cfb05483b971a974be6b0655f987b0431b849"},"cell_type":"markdown","source":"## Loading train & test data"},{"metadata":{"trusted":true,"_uuid":"39c97d9af507069a88c602dbba69d2b2dd443f9e","_kg_hide-input":false},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')\ntrain.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"405bcb4242b891f86e33840cc18c409b560cf8d2"},"cell_type":"markdown","source":"## Separating labels from image pixels from train dataset"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"3722145b59ea743a0347698121ce8a422c2c4036"},"cell_type":"code","source":"y_train = train['label']\ntrain.drop(axis=1, labels=['label'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9aecbf6850b17bcbf5c1477b01e117871d29749c"},"cell_type":"markdown","source":"## Checking if label of each image is separated from its pixels"},{"metadata":{"trusted":true,"_uuid":"9a3aa1b582abbd1dd192c7638edb2874663e91af"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cb1eef3b609c626a3c32904698aa607fc64f90fa"},"cell_type":"markdown","source":"## Getting insights from our train dataset"},{"metadata":{"trusted":true,"_uuid":"e967b3efeaa01ae498a9f06810d59caed04ff6e1"},"cell_type":"code","source":"y_train.value_counts()    ## Images of 1's is maximum & Images of 5's is minimum","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c8bc09b2f2056b3c401ef787096319f06146efc1"},"cell_type":"markdown","source":"## Normalization"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"54a71f98568e2b32041c3b4d346f663a52f0996a"},"cell_type":"code","source":"train /= 255.\ntest /= 255.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3d1df5fe078e9407f12b642f4ddbb84a0411430c"},"cell_type":"markdown","source":"## Changing data-tpe of our feature's to float32"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"c2c38d32a5733c30d28afea177b7bf129d0bf291"},"cell_type":"code","source":"test = np.array(test).astype(np.float32)\nX = np.array(train).astype(np.float32)\ndel train","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3a973846fb890bb2e157ea41650aa1c6b21e0019"},"cell_type":"markdown","source":"## Reshaping our Images - converting them into 2D images"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"597e71072fca83046f676ab083f975888c22cf34"},"cell_type":"code","source":"X = X.reshape(-1,28,28,1)\ntest = test.reshape(-1,28,28,1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"840890182963aa8371ccaf13ebea223fed6faac2"},"cell_type":"markdown","source":"## Visualising our train data"},{"metadata":{"trusted":true,"_uuid":"e8709035cbac50acec4b0451703844847596614c"},"cell_type":"code","source":"plt.imshow(X[0][:,:,0], cmap='gray')\nprint('label = '+str(y_train[0]))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3715cfbc62c04fe2504bdc7f6a7805a4d6f0c0e7"},"cell_type":"markdown","source":"## One-hot-encoding labels for training"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"04402fb4e7f946967220f0bac1fc94f66853f6d4"},"cell_type":"code","source":"yOHE = keras.utils.to_categorical(y_train, num_classes=10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a91086875f2b2f460ccc713f68a02966a123a714"},"cell_type":"markdown","source":"## Splitting data into train data & test data"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"9ba5cca642a4d1ae81bb26f98c34875fd045e96b"},"cell_type":"code","source":"X_train, X_val, Y_train, Y_val = train_test_split(X, yOHE, test_size=0.1, random_state=2)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e4a6548cf9c575eeb83c03f6fef8bf0f438419b2"},"cell_type":"markdown","source":"## Creating our model using Keras"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"b54da47ac39e62d055bd9985176404619cc3dd6f"},"cell_type":"code","source":"model = keras.models.Sequential()\n\nmodel.add(keras.layers.Conv2D(filters=32, kernel_size=(5,5), activation='relu', padding='Same', input_shape=(28,28,1)))\nmodel.add(keras.layers.Conv2D(filters=32, kernel_size=(5,5), activation='relu', padding='Same'))\nmodel.add(keras.layers.MaxPool2D(pool_size=(2,2)))\nmodel.add(keras.layers.Dropout(0.5))\n\nmodel.add(keras.layers.Conv2D(filters=32, kernel_size=(3,3), activation='relu', padding='Same'))\nmodel.add(keras.layers.Conv2D(filters=32, kernel_size=(3,3), activation='relu', padding='Same'))\nmodel.add(keras.layers.MaxPool2D(pool_size=(2,2), strides=(2,2)))\n\nmodel.add(keras.layers.Conv2D(filters=96, kernel_size=(3,3), activation='relu', padding='Same'))\nmodel.add(keras.layers.Conv2D(filters=96, kernel_size=(3,3), activation='relu', padding='Same'))\nmodel.add(keras.layers.MaxPool2D(pool_size=(2,2), strides=(2,2)))\nmodel.add(keras.layers.Dropout(0.5))\n\nmodel.add(keras.layers.Flatten())\nmodel.add(keras.layers.Dense(256, activation='relu'))\nmodel.add(keras.layers.Dropout(0.5))\nmodel.add(keras.layers.Dense(10, activation='softmax'))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b7b405339762eb5d6d6966aeee907756302dead8"},"cell_type":"markdown","source":"## Setting optimizer & annealer"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"68b771767cda25d6a2b938e36f8a3cbe53b235b9"},"cell_type":"code","source":"#optim = keras.optimizers.RMSprop(lr=0.001,rho=0.9, epsilon=1e-08, decay=0.0)\n#optim = keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0001) ---> val_acc = 99.59\noptim = keras.optimizers.Adadelta()\nlr_reducer = keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=3, verbose=1, min_lr=0.00001)\nmodel.compile(optimizer=optim, loss='categorical_crossentropy', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d632dc50bd83e1b1d5e878ea5999e811d526c102"},"cell_type":"markdown","source":"## Data Augmentation"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"5594c3162b53f55ab0626a5188d90da22bb75465"},"cell_type":"code","source":"datagen = keras.preprocessing.image.ImageDataGenerator(\n        featurewise_center=False,                         # set input mean to 0 over the dataset\n        samplewise_center=False,                          # set each sample mean to 0\n        featurewise_std_normalization=False,              # divide inputs by std of the dataset\n        samplewise_std_normalization=False,               # divide each input by its std\n        zca_whitening=False,                              # apply ZCA whitening\n        rotation_range=10,                                # randomly rotate images in the range (degrees, 0 to 180)\n        zoom_range = 0.15,                                # Randomly zoom image \n        width_shift_range=0.1,                            # randomly shift images horizontally (fraction of total width)\n        height_shift_range=0.1,                           # randomly shift images vertically (fraction of total height)\n        horizontal_flip=False,                            # randomly flip images\n        vertical_flip=False)                              # randomly flip images\n\ndatagen.fit(X_train)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7363b94b8b68c7114c8c1393da4c66fb298c7bf2"},"cell_type":"markdown","source":"## Callbacks"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"aef200dd142e40e2fa5400e9f01fd11f94c56dba"},"cell_type":"code","source":"filepath=\"BestWeights_Adadelta.h5\"\ncheckpoint = keras.callbacks.ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max')\ncallbacks_list = [checkpoint]\ncallbacks_list.append(lr_reducer)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f7543d11e8c8f3bd028eda1de7dfc45618b2dc54"},"cell_type":"markdown","source":"## Model Training"},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"792aabdb0a271257c4a351bde20b96f8427d94d9"},"cell_type":"code","source":"batch_size = 64\nhistory = model.fit_generator(  datagen.flow(X_train, Y_train, batch_size=batch_size), \n                             epochs=30, \n                             validation_data=(X_val, Y_val),\n                             steps_per_epoch=X_train.shape[0] // batch_size, \n                             callbacks=callbacks_list, \n                             verbose=1\n                          )","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5b2956e3ed5c95abbd915f196f84179c15a5c908"},"cell_type":"markdown","source":"## Visualising loss & accuracy curves"},{"metadata":{"trusted":true,"_uuid":"4710aa3b48c312f39ee104f45151e0fe2bd8ad68"},"cell_type":"code","source":"fig, ax = plt.subplots(2,1)\nax[0].plot(history.history['loss'], color='b', label=\"Training loss\")\nax[0].plot(history.history['val_loss'], color='r', label=\"validation loss\",axes =ax[0])\nlegend = ax[0].legend(loc='best', shadow=True)\n\nax[1].plot(history.history['acc'], color='b', label=\"Training accuracy\")\nax[1].plot(history.history['val_acc'], color='r',label=\"Validation accuracy\")\nlegend = ax[1].legend(loc='best', shadow=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d15aac6b4607c1002ffbeda5aef468fd9ea57012"},"cell_type":"markdown","source":"## Loading best weights saved in file while training"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"67c3cd524abd1cda37d8dfc0e6572f713138bbef"},"cell_type":"code","source":"model.load_weights('BestWeights_Adadelta.h5')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c2091f70388c956d9bd7ae272aa1f951c06896f9"},"cell_type":"markdown","source":"## Predicting test data"},{"metadata":{"trusted":true,"_uuid":"b5b45a271c0fc003ce3571c225d7ba4e29041faf"},"cell_type":"code","source":"preds = model.predict(test, batch_size=batch_size, verbose = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"1281f8f8d56a69128af6131e4befd839ed952c01"},"cell_type":"code","source":"final_preds = np.argmax(preds, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"85a1679bcbbad3df7e2263ed9b0582f522b1e3d0"},"cell_type":"code","source":"final_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b017e5f17ab34abb508c96d150e9e12ad17e289f"},"cell_type":"code","source":"plt.imshow(test[7,:,:,0], cmap='gray')\nprint(final_preds[7])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d4530bd530353c5cfa21f6a0472335abf8b5e28c"},"cell_type":"markdown","source":"## Creating file for submission"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"646efe9c0ddfbec0c3876c1bed8a51e3afa17da0"},"cell_type":"code","source":"dictionary = {'Label':final_preds}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"30e1ed7dc92f3e297700845a234ebe59024a6682"},"cell_type":"code","source":"pred_df = pd.DataFrame(data=dictionary, index=list(range(1,len(final_preds)+1)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5ed28c5cae84ba1b49c228cb65b00c1608210be8"},"cell_type":"code","source":"pred_df.to_csv('submission.csv')\n\"\"\"Accuracy on kaggle :- 99.66% ----> Top 10% \"\"\"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"75fcff7ca5b17277c63499b913f94ee6282c40cc"},"cell_type":"markdown","source":"## You can achieve upto 99.66% of accuracy using this kernel on Kaggle - Digit Recognizer"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"ce583c7b158c34c5f8b1aa72924ca44a0176580f"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}