{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom scipy import misc\nimport matplotlib.image as mpimg\nimport matplotlib.pyplot as plt\n\nnp.random.seed(3)\nimport itertools as it","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"t = pd.read_csv('../input/train.csv')\nts = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f01c978abe2b1ae325298b9272260845182a6abd"},"cell_type":"markdown","source":"## Missing value check"},{"metadata":{"trusted":true,"_uuid":"092b4eaef3e67ca81295538b4e281bc7841205c4"},"cell_type":"code","source":"t.isnull().sum().sum()\nts.isnull().sum().sum()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"82a74874a3719c428930fe1f58cc65d4b2e3280e"},"cell_type":"markdown","source":"## Split data"},{"metadata":{"trusted":true,"_uuid":"266af32adc4ea5dca9e82b5763d083eb4d8110a8"},"cell_type":"code","source":"ts = ts/255.0\nX = t.drop('label',axis=1)/255.0\nY = t.label","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"db145ff575c98c4f80a937bea05afb8b574e471a"},"cell_type":"markdown","source":"## Label encoding"},{"metadata":{"trusted":true,"_uuid":"d6c3c6c0854445b8a7175c829342d24d418b1faa"},"cell_type":"code","source":"from keras.utils import to_categorical as tc\nY = tc(Y,num_classes=10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"230f05d57a9026f3f40063d788c2a91eefb7a359"},"cell_type":"markdown","source":"## Reshape"},{"metadata":{"trusted":true,"_uuid":"13626fa65313446b0dc9d6fdd8b403e9f16b3c80"},"cell_type":"code","source":"X = X.values.reshape(-1,28,28,1)\nts = ts.values.reshape(-1,28,28,1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4b0b9fea23bcd43e8236c7ecc6ac7d9567e1ac54"},"cell_type":"markdown","source":"## train_test split"},{"metadata":{"trusted":true,"_uuid":"188ffda24e869c9d577f73b8acfd61007a5bd181"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split as tts\n\nX_train,X_test,Y_train,Y_test = tts(X,Y,test_size=0.3,random_state=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"de97e8d7b2f819d7bee4ae81f0c590131f476b25"},"cell_type":"code","source":"X_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e607e2d45bbf7a736201fbc70da879a734f65c9"},"cell_type":"code","source":"plt.imshow(X_train[0][:,:,0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d9cf20b1ee65fcec1eae0dd30aea51b020499bb7"},"cell_type":"markdown","source":"# CNN"},{"metadata":{"trusted":true,"_uuid":"01906f7ad061a03963f1dbedd044303a84bd566b"},"cell_type":"code","source":"#import packages\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Activation, Dropout, Conv2D, MaxPool2D,Flatten, LeakyReLU","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"03823ac7962008da7ced6d840053942473692736"},"cell_type":"code","source":"# Set the CNN model \n# my CNN architechture is In -> [[Conv2D->relu]*2 -> MaxPool2D -> Dropout]*2 -> Flatten -> Dense -> Dropout -> Out\n\nmodel = Sequential()\n\nmodel.add(Conv2D(filters = 32, kernel_size = (5,5),padding = 'Same', \n                 activation ='linear', input_shape = (28,28,1)))\nmodel.add(LeakyReLU(alpha=.001))\nmodel.add(Conv2D(filters = 32, kernel_size = (5,5),padding = 'Same', \n                 activation ='linear'))\nmodel.add(LeakyReLU(alpha=.001))\nmodel.add(MaxPool2D(pool_size=(2,2)))\nmodel.add(Dropout(0.25))\n\n\nmodel.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'Same', \n                 activation ='linear'))\nmodel.add(LeakyReLU(alpha=.001))\nmodel.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'Same', \n                 activation ='linear'))\nmodel.add(LeakyReLU(alpha=.001))\nmodel.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))\nmodel.add(Dropout(0.25))\n\n\nmodel.add(Flatten())\nmodel.add(Dense(256, activation = \"linear\"))\nmodel.add(LeakyReLU(alpha=.001))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(10, activation = \"softmax\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1f4676d39d24c8dbf567ff4fb6bcbf96482893a8"},"cell_type":"code","source":"from keras import optimizers as op\noptimizer = op.RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ccbb1f669f85d54cd9c30e0eb6765cd0c3cfb932"},"cell_type":"code","source":"# Compile the model\nmodel.compile(optimizer = optimizer , loss = \"categorical_crossentropy\", metrics=[\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7126993850cd0c4c98a5b35e5bb3be35f8cbef07"},"cell_type":"code","source":"X_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"77c9b6faf5b4f73891fe0be493b8a32b116c7ece"},"cell_type":"code","source":"# With data augmentation to prevent overfitting (accuracy 0.99286)\nfrom keras.preprocessing.image import ImageDataGenerator as imgen\ndatagen = imgen(\n        featurewise_center=False,  # set input mean to 0 over the dataset\n        samplewise_center=False,  # set each sample mean to 0\n        featurewise_std_normalization=False,  # divide inputs by std of the dataset\n        samplewise_std_normalization=False,  # divide each input by its std\n        zca_whitening=False,  # apply ZCA whitening\n        rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n        zoom_range = 0.1, # Randomly zoom image \n        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n        horizontal_flip=False,  # randomly flip images\n        vertical_flip=False)  # randomly flip images\n\n\ndatagen.fit(X_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"47aafc26e61541ae3ef256f16819f0b3094a6b9c"},"cell_type":"code","source":"from keras.callbacks import ReduceLROnPlateau\nlearning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', \n                                            patience=3, \n                                            verbose=1, \n                                            factor=0.5, \n                                            min_lr=0.00001)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"22d71e098690ce1be3272226779ce94ef986e4ad"},"cell_type":"code","source":"X_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4a16e3c510b65743db3d8b83eae2f5c3ada7a352"},"cell_type":"code","source":"history = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=86),\n                              epochs = 30, validation_data = (X_test,Y_test),\n                              verbose = 2, steps_per_epoch=X_train.shape[0] // 86\n                              , callbacks=[learning_rate_reduction])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e4c8192511f6bdef0ec6e49c48d6ea6b2836fe7a"},"cell_type":"code","source":"# predict results\nresults = model.predict(ts)\n\n# select the indix with the maximum probability\nresults = np.argmax(results,axis = 1)\n\nresults = pd.Series(results,name=\"Label\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"37c7ac9b2b9ef1ec224c0cbfa69038860c5a4dec"},"cell_type":"code","source":"sub = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\nsub.to_csv('Submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7ed109f2731e8dc08d744f0cdf366684aa927589"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"397899d7bfb2fed30a730186aae4e726638fa155"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f17dba7ceb86d5f3b0ea838d854bc1da8f85fff9"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}