{"cells":[{"metadata":{"_uuid":"6454be6908bb946bd4da67c74276648fc7c7067c"},"cell_type":"markdown","source":"# Digits Recognizer"},{"metadata":{"_uuid":"b0aa0e43adcbdd0671a9b247e4a00f7e09638801"},"cell_type":"markdown","source":"inspired by the following kernels:\n* https://www.kaggle.com/poonaml/deep-neural-network-keras-way\n* https://www.kaggle.com/yassineghouzam/introduction-to-cnn-keras-0-997-top-6"},{"metadata":{"_uuid":"69c4a6954ab9678ab20e3b3e23db9f771daf45f2"},"cell_type":"markdown","source":"## 1. Import necessary Python packages"},{"metadata":{"trusted":true,"_uuid":"7baabf8733442545bd1bf5dc433bb9f14d5ef6f7"},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nfrom keras import backend as K\nfrom keras.preprocessing.image import ImageDataGenerator\n\nimport matplotlib.pyplot as plt\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f43dadcf65b9741b8b2cbec0e05d10867547f28"},"cell_type":"markdown","source":"## 2. Open datasets"},{"metadata":{"trusted":true,"_uuid":"854c429bc0551867b3931d02d9ad8a2cc9efbccb"},"cell_type":"code","source":"df_train = pd.read_csv(\"../input/train.csv\")\ndf_test = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"476623e0b07b1abac8e0b3f928184ec80ccfa61c"},"cell_type":"markdown","source":"## 2.1. Check the data shape"},{"metadata":{"trusted":true,"_uuid":"dd0cb9f40fe4c49e08e04670eacab86d246c0db8"},"cell_type":"code","source":"df_train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f8f034e4310c18e6c80a776a28b24b0f3fbdd52f"},"cell_type":"code","source":"df_test.describe()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5b9d2951b29d559489d37ba80e1cf872a46fb5c1"},"cell_type":"markdown","source":"## 3. Manipulate data"},{"metadata":{"trusted":true,"_uuid":"095686ac3eba9339accd7268a9cb016fa09691b4"},"cell_type":"code","source":"y_train = df_train[\"label\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eb070bf22ae0cbfef971bda83c61c11871f69f7c"},"cell_type":"code","source":"y_train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da338ae721fd79daab016acfb2a963aed505843e"},"cell_type":"code","source":"X_train = df_train.drop(columns=\"label\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"77125e29a37047b68d1f5b128a425c3f625a7fbe"},"cell_type":"code","source":"X_train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"359260fcb703d6dcc51d38ef4231440b381b3053"},"cell_type":"code","source":"X_test = df_test","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1da9c59d6a857cd39a9928d5b9d9331f0fd30b8c"},"cell_type":"markdown","source":"## 3.1. Convert pixels to float"},{"metadata":{"trusted":true,"_uuid":"9c04aac3e9df4d1ff5a180ec8c2d43a5d654507a"},"cell_type":"code","source":"X_train = np.array((df_train.iloc[:,1:].values).astype('float32'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a989b4d1a5f44ecc86cd38a66d18310b5a2528e0"},"cell_type":"code","source":"X_train.min()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"96fa4ccdf20678cac487fc93d6dfa2ba678cdce1"},"cell_type":"code","source":"X_train.max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f274daae95ceeb98602ea5b27f9dc8a69865a607"},"cell_type":"code","source":"X_test = np.array(df_test.values.astype('float32'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"823bf80dac383e21d0f82e54c0ec436469cabf96"},"cell_type":"code","source":"X_train.min()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ebd7cf15d3b867b9804eba183e9adeebda8998b"},"cell_type":"code","source":"X_train.max()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6168ebf603d42bd774a4c395a59bc35045a137ba"},"cell_type":"markdown","source":"## 3.2. Scale Down/Normalize data between 0.00 to 1.00"},{"metadata":{"trusted":true,"_uuid":"f8ae9d1d8a8ae5d0ef71b535b74f5b10f2775013"},"cell_type":"code","source":"X_train = X_train/255.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"734e892ed97a4671e39c316d2549c3d88252097d"},"cell_type":"code","source":"X_train.max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3db77a50c2786742c53e603a35d6e25adcfc7e57"},"cell_type":"code","source":"X_test = X_test/255.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0783672fb99a2875a8384e1f031c3fca65dcb859"},"cell_type":"code","source":"X_test.max()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"84a49b25e0c4535c1e63ddb2d5dab608dbd7a996"},"cell_type":"markdown","source":"## 3.3. Convert label digits to integer"},{"metadata":{"trusted":true,"_uuid":"2b21219c8aeba3fc4b22f521b7a444ede742df94"},"cell_type":"code","source":"y_train = np.array(df_train.iloc[:,0].values.astype('int32'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a61420f076b95911cb36c56b21625222674dfb42"},"cell_type":"code","source":"y_train.min()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"efee2614744518f9ac490902722274c2f34ef9c1"},"cell_type":"code","source":"y_train.max()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e661592ca3fb1a62daf1a2fa8cb19642c79afecc"},"cell_type":"markdown","source":"## 3.4. Reshape Data"},{"metadata":{"trusted":true,"_uuid":"7eeaf67118380f5d62e434e870fa21f3bad82b40"},"cell_type":"code","source":"X_train = X_train.reshape(-1, 28, 28, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"953698f4e340cdf05f825844bdec0586ca71bc7c"},"cell_type":"code","source":"X_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"036a834d0c4f64af65683b674f4e8fd5f1742db2"},"cell_type":"code","source":"X_test = X_test.reshape(-1, 28, 28, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da9bdda4b50404a2fdff09158bd30aff6ebee565"},"cell_type":"code","source":"X_test.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c4e80153829b0bc20bbc38f10ae2c85eef6751af"},"cell_type":"markdown","source":"## 3.4.1. Plot a couple of data for sanity check"},{"metadata":{"trusted":true,"_uuid":"405b61c1ad0d578bc8242cbdb70ea3b981df9a2f"},"cell_type":"code","source":"plt.subplot(171)\nplt.title(y_train[1])\nplt.imshow(X_train[1][:,:,0])\n\nplt.subplot(172)\nplt.title(y_train[2])\nplt.imshow(X_train[2][:,:,0])\n\nplt.subplot(173)\nplt.title(y_train[3])\nplt.imshow(X_train[3][:,:,0])\n\nplt.subplot(174)\nplt.title(y_train[4])\nplt.imshow(X_train[4][:,:,0])\n\nplt.subplot(175)\nplt.title(y_train[5])\nplt.imshow(X_train[5][:,:,0])\n\nplt.subplot(176)\nplt.title(y_train[6])\nplt.imshow(X_train[6][:,:,0])\n\nplt.subplot(177)\nplt.title(y_train[7])\nplt.imshow(X_train[7][:,:,0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"74cc552f8849beb02e29c2f510c640f4a66c24b8"},"cell_type":"code","source":"plt.subplot(151)\nplt.imshow(X_test[1][:,:,0])\n\nplt.subplot(152)\nplt.imshow(X_test[2][:,:,0])\n\nplt.subplot(153)\nplt.imshow(X_test[3][:,:,0])\n\nplt.subplot(154)\nplt.imshow(X_test[4][:,:,0])\n\nplt.subplot(155)\nplt.imshow(X_test[5][:,:,0])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"40098fb09d05e707869810e7a9a88bd1b85d792f"},"cell_type":"markdown","source":"## 3.5. One-Hot encoding"},{"metadata":{"trusted":true,"_uuid":"42d9c856a4515a1177772de529f1d35df797c276"},"cell_type":"code","source":"from keras.utils.np_utils import to_categorical\n\ny_train_encoded = to_categorical(y_train, num_classes=10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"157235157d18ea7e3580ecf34ab137fc2d357eec"},"cell_type":"markdown","source":"## 3.5.1. Perform check at some data"},{"metadata":{"trusted":true,"_uuid":"8227f1afabe2d4a6038f41d901b2c3bff7670213"},"cell_type":"code","source":"for idx in range (20, 30):\n    print (y_train[idx], y_train_encoded[idx])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ca9214438f6d41647f59a77dca0ef86db03f15ef"},"cell_type":"markdown","source":"Good to go!"},{"metadata":{"_uuid":"a5faa4a92acf07414bf7ee3128d842e593abb65d"},"cell_type":"markdown","source":"## 3.6. Split data for Cross Validation"},{"metadata":{"trusted":true,"_uuid":"e2bf9ba18377fb448c06c95fff9b222cc9cfd499"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_val, Y_train, Y_val = train_test_split(X_train, y_train_encoded, test_size = 0.1, random_state=2)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3c5574a1d6e7c8a69c50391bb5b21fa2ab38ee11"},"cell_type":"markdown","source":"## 4. Design the Classifier"},{"metadata":{"_uuid":"b6e425e2b7a54f01461ae88d74995b3f06d08290"},"cell_type":"markdown","source":"## 4.1. Define the Convolution Neural Network (CNN) model"},{"metadata":{"trusted":true,"_uuid":"4c0dda8c3fed51b3feed9b0990fcd31295414d83"},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Conv2D, MaxPool2D\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.layers import Dense , Dropout , Lambda, Flatten\n\nmodel = Sequential()\n\nmodel.add(Conv2D(filters = 32, \n                 kernel_size = (5,5),\n                 padding = 'Same', \n                 activation ='relu', \n                 input_shape = (28,28,1)))\n\nmodel.add(BatchNormalization(axis=1))\n\nmodel.add(Conv2D(filters = 32, \n                 kernel_size = (5,5),\n                 padding = 'Same', \n                 activation ='relu'))\n\nmodel.add(MaxPool2D(pool_size=(2,2)))\n#model.add(Dropout(0.25))\n\nmodel.add(Conv2D(filters = 64, \n                 kernel_size = (3,3),\n                 padding = 'Same', \n                 activation ='relu'))\n\nmodel.add(BatchNormalization(axis=1))\n\nmodel.add(Conv2D(filters = 64, \n                 kernel_size = (3,3),\n                 padding = 'Same', \n                 activation ='relu'))\n\nmodel.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))\n#model.add(Dropout(0.25))\n\nmodel.add(Flatten())\n\nmodel.add(Dense(256, activation = \"relu\"))\n\nmodel.add(Dropout(0.5))\n\nmodel.add(Dense(10, activation = \"softmax\"))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e9e3c4c44945295e35f8ad2c430be9d144e64d30"},"cell_type":"markdown","source":"## 4.2. Set Optimizer"},{"metadata":{"trusted":true,"_uuid":"ff968620e9dbbc44b6888b680b34bd80185a32bb"},"cell_type":"code","source":"from keras.optimizers import Adam ,RMSprop\n\noptimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d6833c015b14b5145e1819b2885b6b548514d3fb"},"cell_type":"code","source":"from keras.callbacks import ReduceLROnPlateau\n\nlearning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', \n                                            patience=3, \n                                            verbose=1, \n                                            factor=0.5, \n                                            min_lr=0.00001)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9cfa39a8060da0405143bfaa24917a33d42f134d"},"cell_type":"markdown","source":"## 4.3. Compile the model"},{"metadata":{"trusted":true,"_uuid":"edcb1e4536d47fcbf0fc72591bab1a49a19e1d4b"},"cell_type":"code","source":"model.compile(optimizer = optimizer , \n              loss = \"categorical_crossentropy\", \n              metrics=[\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"290ba99dfcf490a5ace2b20bcc38bda669623ae0"},"cell_type":"markdown","source":"## 4.4. Data Augmentation"},{"metadata":{"trusted":true,"_uuid":"c6780b35b4c07fa262c6103eba47fbefa6dec453"},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\n\ndatagen = ImageDataGenerator(\n            featurewise_center=False,  # set input mean to 0 over the dataset\n            samplewise_center=False,  # set each sample mean to 0\n            featurewise_std_normalization=False,  # divide inputs by std of the dataset\n            samplewise_std_normalization=False,  # divide each input by its std\n            zca_whitening=False,  # apply ZCA whitening\n            rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n            zoom_range = 0.1, # Randomly zoom image \n            width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n            height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n            horizontal_flip=False,  # randomly flip images\n            vertical_flip=False)  # randomly flip images\n\ndatagen.fit(X_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3c430666a3ccc89a5d729aacb28ff4a067eb2772"},"cell_type":"code","source":"batches = datagen.flow(X_train, Y_train, batch_size=96)\nval_batches = datagen.flow(X_val, Y_val, batch_size=96)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"15305d1fb004fd2e5f74fb5cc547677b9231deee"},"cell_type":"markdown","source":"## 4.5. Fit the model"},{"metadata":{"trusted":true,"_uuid":"549bcb53efac722c26ee7d8b6af64152f40cd6f7"},"cell_type":"code","source":"history = model.fit_generator(generator=batches, \n                            epochs=1,\n                            validation_data=val_batches,\n                            validation_steps=val_batches.n,\n                            verbose=2,\n                            steps_per_epoch=batches.n,\n                            callbacks=[learning_rate_reduction])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9aadd90108278ac9a20560d5b3909847a3eae3d0"},"cell_type":"markdown","source":"## 5. Perform Prediction"},{"metadata":{"trusted":true,"_uuid":"34a977c9e518a9e865783c85797ae1b68d9f9653"},"cell_type":"code","source":"predictions = model.predict_classes(X_test, verbose=0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"020222fefed824f497142b87a36e1a24128aa4f2"},"cell_type":"markdown","source":"## 6. Submission"},{"metadata":{"trusted":true,"_uuid":"0b959ea6c87b89c12c6aa51023d901c7d6b6ec33"},"cell_type":"code","source":"submissions=pd.DataFrame({\"ImageId\": list(range(1,len(predictions)+1)),\n                          \"Label\": predictions})\n\nsubmissions.to_csv(\"submission.csv\", index=False, header=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"b1491db40813cb3fbca7a0a01e4194d2d1884cfe"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}