{"cells":[{"metadata":{"trusted":false,"_uuid":"17c9572c0ea860df6358cc97a37ce13fc33f9548"},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport keras\nfrom keras.models import Sequential\nfrom keras.layers import Conv2D, MaxPool2D, Dense, Flatten, Dropout\nfrom keras.optimizers import RMSprop\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f15dd90c9030216baaf5d16dab1f392216cc3bbb"},"cell_type":"markdown","source":"### Prepare data\n\n1. Read Data with pandas DataFrame\n2. Parse the \"label\" column(axis=1) as y_train\n3. Check digits distribution"},{"metadata":{"trusted":false,"_uuid":"acd0c14aa5287f51cf1876e4c40f9013ecae4735"},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"36721cb74f1042314c606b5656e32aef878fdf73"},"cell_type":"code","source":"train.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"200cd20588d43475bef5d3b81fa2de74f0d69379"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"0d0007e7cac3a910d8c77a288ffd0d0db5370485"},"cell_type":"code","source":"X_train = train.drop(labels=['label'], axis=1)\ny_train = train['label']","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"7d521097f697f759e690cc0bb2559fba6170b72d"},"cell_type":"code","source":"X_train.shape, y_train.shape","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":false,"_uuid":"228a27f8742740cc55805a096d5ed0b60cc85f2c"},"cell_type":"code","source":"print(y_train.value_counts(sort=False))\nsns.countplot(y_train)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"23b6c3a1749fc3ace94da6a5e0486e68b522c18b"},"cell_type":"markdown","source":"### Reshape data as an image matrix\n\nReshape data array to image matrix\n         [-1, 784] => [-1, 28, 28, 1]\n         \nShape of input data for keras.Conv2D should be [samples, heigth, width, channels]"},{"metadata":{"trusted":false,"_uuid":"9e5f000256a2b246ac4fd4c940ebdb00326bd0e4"},"cell_type":"code","source":"X_train = X_train.values.astype('float32').reshape(-1, 28, 28, 1)\nX_test = test.values.astype('float32').reshape(-1, 28, 28, 1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6bb2007c516821de250c37cdfd33dc3e9ac8ae45"},"cell_type":"markdown","source":"### Data visualization\n\nDisplay the digit image"},{"metadata":{"trusted":false,"_uuid":"d91aa02801a90cca2f545a504da3d7ebe60ee718"},"cell_type":"code","source":"for cnt, i in enumerate([10, 20, 30]):\n    plt.subplot(330 + (cnt+1))\n    plt.imshow(X_train[i][:, :, 0], cmap=plt.get_cmap('gray'))\n    plt.title(y_train[i])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"eaeed12e17edc9e66079947be27d56534146cbb8"},"cell_type":"markdown","source":"### Normalization\nNormalize data from [0..255] to [0..1]"},{"metadata":{"trusted":false,"_uuid":"25924f9555a4aa722961464a318ca60e778c57ed"},"cell_type":"code","source":"X_train = X_train / 255.0\nX_test = X_test / 255.0","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e03e285abfdc7ba967a113434df6ca3277862869"},"cell_type":"markdown","source":"### One hot encoding\nEncode digit labels to one hot vector, e.g. 6 => [0,0,0,0,0,1,0,0,0,0]"},{"metadata":{"trusted":false,"_uuid":"4501b3259538b6dfe6374a293446f70c58f8b5b0"},"cell_type":"code","source":"from keras.utils.np_utils import to_categorical\ny_train = to_categorical(y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"5b04906c81149cdbdef4ee9918ec2d20e460fa0e"},"cell_type":"code","source":"y_train[0]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"84a9f2224a9b9af436d2b080489aa6b1c477a378"},"cell_type":"markdown","source":"### Split data\nSplit data to be train set and validation set\n\nSet validation set to be 10%"},{"metadata":{"trusted":false,"_uuid":"b474153b845bf44e4736f5822aaf0bc2eb0b9e1c"},"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(\n    X_train, y_train, test_size=0.1, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"49c954a7cb9058a7ac430c7aaf53627b5eb93459"},"cell_type":"markdown","source":"### CNN\n1. Design model\n2. Set optimizer"},{"metadata":{"trusted":false,"_uuid":"86b5c0cb4cf7881e69c61821e3df002dc99e3b8b"},"cell_type":"code","source":"model = Sequential()\n\nmodel.add(Conv2D(filters=32, kernel_size=(3, 3), padding='SAME', \n                 activation='relu', input_shape=(28, 28, 1)))\nmodel.add(Conv2D(filters=32, kernel_size=(3, 3), padding='SAME', \n                 activation='relu'))\nmodel.add(MaxPool2D(pool_size=(2, 2)))\nmodel.add(Conv2D(filters=64, kernel_size=(3, 3), padding='SAME', \n                 activation='relu'))\nmodel.add(Conv2D(filters=64, kernel_size=(3, 3), padding='SAME', \n                 activation='relu'))\nmodel.add(MaxPool2D(pool_size=(2, 2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Flatten())\nmodel.add(Dense(256, activation='relu'))\nmodel.add(Dropout(0.25))\nmodel.add(Dense(10, activation='softmax'))          ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"274621fcc36a97375b5694cdf005aed106b3d830"},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"c56d16fdaf768cf522724e415a79c019e25490e3"},"cell_type":"code","source":"optimizer = RMSprop(lr=0.001, epsilon=1e-8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"9f9288f1d37fe54731695c0a40b5b6da2e850ed3"},"cell_type":"code","source":"model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0f2eb818a59bc13927c16972cb9a7ca6767b53ca"},"cell_type":"markdown","source":"### Data augmentation"},{"metadata":{"trusted":false,"_uuid":"ef16b96f2a73f4aa02a45bf62e840908de37d982"},"cell_type":"code","source":"from keras.preprocessing import image\nbatch_size = 64\n\ndatagen = image.ImageDataGenerator(\n     rotation_range=8,\n     width_shift_range=0.08,\n     shear_range=0.3,\n     height_shift_range=0.08,\n     zoom_range=0.08\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"05ed1e97301ebcab138cfe4731039fa51cc671fd"},"cell_type":"code","source":"train_batches = datagen.flow(X_train, y_train, batch_size=batch_size)\nval_batches = datagen.flow(X_val, y_val, batch_size=batch_size)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"19890fe27b2c11ac2ec0d2d3edca06ccefb3ba4f"},"cell_type":"markdown","source":"### Learning rate annealer"},{"metadata":{"trusted":false,"_uuid":"66787274c92b38a486be0c9692b1af76c11a01cb"},"cell_type":"code","source":"from keras.callbacks import ReduceLROnPlateau\nlearning_rate_reduction = ReduceLROnPlateau(monitor='val_loss',\n                                            factor=0.5,\n                                            patience=5,\n                                            min_lr=1e-5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b3531273c88c892debab1cd1f06c603468c66379"},"cell_type":"markdown","source":"### Fit model"},{"metadata":{"trusted":false,"_uuid":"726e4535977b40c9578e3c170e684b421dedb317"},"cell_type":"code","source":"epochs = 30","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"f245109e35148e16308d00bd91b74206e1c4bd10"},"cell_type":"code","source":"history = model.fit_generator(\n    generator=train_batches,\n    steps_per_epoch=X_train.shape[0]//batch_size,\n    validation_data=val_batches,\n    validation_steps=X_val.shape[0]//batch_size,\n    epochs=epochs, \n    verbose=2,\n    callbacks=[learning_rate_reduction])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"92ae798dc6c0c288c161005f8bfa50ff849f0d18"},"cell_type":"code","source":"plt.figure(figsize=[12, 10])\nplt.subplot(211)\nplt.plot(history.history['loss'], color='b', label='Train loss')\nplt.plot(history.history['val_loss'], color='r', label='Valid loss')\nplt.subplot(212)\nplt.plot(history.history['acc'], color='b', label='Train accuracy')\nplt.plot(history.history['val_acc'], color='r', label='Valid accuracy')\nlegend = plt.legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"988bcc21c504bdfa66ad43b6a21f392230f4ac2b"},"cell_type":"code","source":"y_pred = model.predict(X_val)\ny_pred_num = np.argmax(y_pred, axis=1)\ny_val_num = np.argmax(y_val, axis=1)\nconfusion_matrix(y_val_num, y_pred_num)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6f431a68f65629bf6c155a4ddc917be74e5038c2"},"cell_type":"markdown","source":"### Submit"},{"metadata":{"trusted":false,"_uuid":"74dcb4b3b9309c6653fc53709852e49c1f9da387"},"cell_type":"code","source":"# predict results\nresults = model.predict(X_test)\n\n# select the indix with the maximum probability\nresults = np.argmax(results, axis = 1)\n\nresults = pd.Series(results, name=\"Label\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"ea7287227c55dfa002c9891181b68e4e184f1111"},"cell_type":"code","source":"submission = pd.concat([pd.Series(range(1,28001), name=\"ImageId\"), results],axis = 1)\nsubmission.to_csv(\"MNIST-CNN.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}