{"cells":[{"metadata":{"_uuid":"5d2285936d25bcd7705b2c13ea50d8fc3a10e05f"},"cell_type":"markdown","source":"# Try the MNIST problem with Keras"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":false,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom IPython.display import Image\nimport tensorflow as tf\ntf.__version__","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true},"cell_type":"markdown","source":"## Data processing\nThe data files train.csv and test.csv contain gray-scale images of hand-drawn digits, from zero through nine.\n\nEach image is 28 pixels in height and 28 pixels in width, for a total of 784 pixels in total. Each pixel has a single pixel-value associated with it, indicating the lightness or darkness of that pixel, with higher numbers meaning darker. This pixel-value is an integer between 0 and 255, inclusive.\n\nThe training data set, (train.csv), has 785 columns. The first column, called \"label\", is the digit that was drawn by the user. The rest of the columns contain the pixel-values of the associated image."},{"metadata":{"_uuid":"dcf6d077db9ff1a166adf48fdb656d0d4e6b5766","trusted":true,"collapsed":true},"cell_type":"code","source":"img_rows, img_cols = 28, 28\nnum_classes = 10  #0,1,2,3,4,5,6,7,8,9\ntrain_file = \"../input/train.csv\"\ntest_file = \"../input/test.csv\"\n\ntrain_data = np.loadtxt(train_file, skiprows=1, delimiter=',')\ntest_data = np.loadtxt(test_file, skiprows=1, delimiter=',')","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"_kg_hide-output":false,"_uuid":"a894ba6e997c9666b0a16d655783ef4180c72afa","trusted":true},"cell_type":"code","source":"print(train_data.shape, test_data.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5c2052ee48a9d6d2cadb1b77b19d4aceb5fe27d6"},"cell_type":"markdown","source":"* https://www.tensorflow.org/api_docs/python/tf/keras/utils/to_categorical\n"},{"metadata":{"_uuid":"44ce5998d3a318aaba14bd613a47a91477f1eaf9","trusted":true,"collapsed":true},"cell_type":"code","source":"def prep_data(raw):\n    out_y = 0\n    out_x = 0\n    if raw.shape[1] == 785:  # training dataset\n        y = raw[:, 0]  # first column is label\n        out_y = tf.keras.utils.to_categorical(y, num_classes)  # Converts a class vector (integers) to binary class matrix.\n        x = raw[:,1:]\n    elif raw.shape[1] == 784:  # testing dataset\n        x = raw\n    num_images = raw.shape[0]\n    x /= 255  # scaling from 0-255 to 0-1\n    out_x = x.reshape(num_images, img_rows, img_cols, 1)\n    return out_x, out_y","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7ff5a7e7b1c45c8bd96c04a0cbe5eaf70f3e912e","trusted":true,"collapsed":true},"cell_type":"code","source":"X, y = prep_data(train_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4482aab2ee299c92f00675016f73a45b5bf001c7"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_cv, y_train, y_cv = train_test_split(X, y)\nprint(X_train.shape, X_cv.shape, y_train.shape, y_cv.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f0f6afb75be2626c923e920f4215327bbb9b0d9"},"cell_type":"code","source":"X_test, y_test = prep_data(test_data)\nprint(X_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"105e56ca2f2b285915f6407534897f8d943e0e7d"},"cell_type":"markdown","source":"Display a few sample picture"},{"metadata":{"_kg_hide-input":false,"_kg_hide-output":false,"_uuid":"f3d9e4ab1a3c84a04a9bf75514be550b1266e41b","trusted":true,"collapsed":true},"cell_type":"code","source":"size=6\nfig, axes = plt.subplots(1, size, figsize=(img_rows,img_cols))\nfor i,j in enumerate(np.random.randint(0,X.shape[0],size)):\n    plt.subplot(1, size, i+1)\n    plt.title(\"Label: %s\" % (y[j]))\n    plt.imshow(X[j].reshape(img_rows,img_cols),cmap=\"gray\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5dd469cb15166db4103d2146d6dab10bc42b863a"},"cell_type":"markdown","source":"## Model\n* https://www.tensorflow.org/api_docs/python/tf/keras/Sequential\n* https://www.tensorflow.org/api_docs/python/tf/keras/layers/Conv2D\n* https://www.tensorflow.org/api_docs/python/tf/keras/layers/Flatten\n* https://www.tensorflow.org/api_docs/python/tf/keras/layers/Dense"},{"metadata":{"_uuid":"dd03803fd660f9535ee7051dfa118347272f8080"},"cell_type":"markdown","source":"List of known classifiers and corresponding error rates are shown in\n* http://yann.lecun.com/exdb/mnist/index.html\n* https://www.tensorflow.org/tutorials/estimators/cnn"},{"metadata":{"_uuid":"f307192f10571d841b11ee640016d86dcfed2e5d","trusted":true,"collapsed":true},"cell_type":"code","source":"from tensorflow.python.keras import Sequential\nfrom tensorflow.python.keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPool2D\n\nmodel = Sequential()\nmodel.add(Conv2D(32,  # number of filters\n                 3,  # dimension of filters\n                 activation=\"relu\",\n                 padding=\"same\",  # pad with 0 to the edges to make output shapes = input shapes\n                 input_shape=(img_rows, img_cols, 1)\n                 ))\nmodel.add(MaxPool2D(pool_size=2))  # pool and halve the size. i.e. 28x28 -> 14x14\nmodel.add(Conv2D(64, 3, padding=\"same\", activation=\"relu\"))\nmodel.add(MaxPool2D(pool_size=2))\nmodel.add(Flatten())\nmodel.add(Dense(1024))\nmodel.add(Dropout(0.4))\nmodel.add(Dense(num_classes, activation=\"softmax\"))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"521a902090f7ca24c2f4992c03aa70aaace0615a"},"cell_type":"markdown","source":"Show our model here"},{"metadata":{"_uuid":"af4afa4c99b195c5329c638378ffe730a9acc6f8","trusted":true,"collapsed":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8dc06181cd43a9cc17943fc714d7093ab0dd1abd"},"cell_type":"markdown","source":"### Compile\n* https://www.tensorflow.org/api_docs/python/tf/keras/Model#compile"},{"metadata":{"_uuid":"5f3fdf9d6683838fa98790a391ce7ecba2e8f8e1","trusted":true,"collapsed":true},"cell_type":"code","source":"model.compile(optimizer = tf.keras.optimizers.Adam(lr=3e-3),\n              loss = tf.keras.losses.categorical_crossentropy,\n              metrics = [\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2e4de497f513c80b7ab135d78bde2ae5d2f02a99"},"cell_type":"markdown","source":"Search for the good learning rate\n*dd https://github.com/surmenok/keras_lr_finder"},{"metadata":{"_uuid":"5b7b6d679358c3c6d98948f5224df1f2b47a5030"},"cell_type":"markdown","source":"### Fit\n* https://www.tensorflow.org/api_docs/python/tf/keras/Model#fit"},{"metadata":{"trusted":true,"_uuid":"ac79409093c915a4399e07aa6c0b6b9d5f7d0cef","collapsed":true},"cell_type":"code","source":"from tensorflow.python.keras.callbacks import ReduceLROnPlateau\nlearning_rate_reduction = ReduceLROnPlateau(monitor='val_loss',\n                                            patience=3,\n                                            verbose=1, \n                                            factor=0.5, \n                                            min_lr=0.0001)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"efe62f28075d07e1d6f6fc827e5c469a52cb5e57","scrolled":true,"trusted":true,"collapsed":true},"cell_type":"code","source":"history = model.fit(X_train,\n                    y_train,\n                    batch_size = 512,\n                    epochs = 50,\n                    callbacks = [learning_rate_reduction],\n                    validation_data=(X_cv, y_cv)\n                    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d8bb82a07ce98c295d1d2c1d8431c7e9b1c1c070","collapsed":true},"cell_type":"code","source":"import gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"02a2ca6352d5366dbff1ce920141a7cb88f7e589"},"cell_type":"markdown","source":"### Learning Curve"},{"metadata":{"trusted":true,"_uuid":"c51c35deb8cd89f26c668c72b5ff7e2d19bc43d3","collapsed":true},"cell_type":"code","source":"plt.title(\"Learning rate curves\")\nplt.plot(history.history[\"lr\"])\nplt.ylabel(\"learning rate\")\nplt.xlabel(\"epochs\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"beb5c4246431d49cbd702744f3643a36434f2e45","scrolled":true,"trusted":true,"collapsed":true},"cell_type":"code","source":"plt.title(\"Loss curves\")\nplt.plot(history.history[\"loss\"])\nplt.plot(history.history[\"val_loss\"])\nplt.ylabel(\"loss\")\nplt.xlabel(\"epochs\")\nplt.legend(['train','cv'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bb76ffa29d671546661e5a730ff337ceaa04c76f"},"cell_type":"markdown","source":"### Confusion matrix\nval_acc: 0.9801"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"eab21bb33ce5ddeb2d42c8d4060073a31dc44950"},"cell_type":"code","source":"preds_cv = model.predict(X_cv)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c0ef3b574267e5c385a7e935b60723bd27feb97d","collapsed":true},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\ncm = confusion_matrix(np.argmax(y_cv,axis=1), np.argmax(preds_cv,axis=1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"47350cedee499685ac0fad74f0a2e77a2035fa75","collapsed":true},"cell_type":"code","source":"cm","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"42691c04ded2cc26bf2e177b5a71e4ab027697bd"},"cell_type":"markdown","source":"### Check incorrect predictions"},{"metadata":{"trusted":true,"_uuid":"61af00ed082cb1a5b557af2add95ac090e624ef7","collapsed":true},"cell_type":"code","source":"df = pd.DataFrame({\"expected\": np.argmax(y_cv,axis=1), \"predicted\": np.argmax(preds_cv,axis=1)})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"708ee418d2fbd4cd54446daa6765fc85bd8ab51d","collapsed":true},"cell_type":"code","source":"df_incorrect = df[df[\"expected\"] != df[\"predicted\"]]\ndf_incorrect.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b4e07480d4e280b4f29bb651c25d42ac95a56d63","collapsed":true},"cell_type":"code","source":"for expected in range(10):\n    index = df_incorrect[df_incorrect[\"expected\"] == expected].index\n    size=len(index)\n\n    fig, axes = plt.subplots(1, size, figsize=(img_rows,img_cols))\n    for i,j in enumerate(index):\n        plt.subplot(1, size, i+1)\n        plt.title(\"Predicted: %s\" % (np.argmax(preds_cv[j])))\n        plt.imshow(X_cv[j].reshape(img_rows,img_cols),cmap=\"gray\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a7ab0b09a7bf65a73a98d5a001e3c836efae7a34"},"cell_type":"markdown","source":"### Data augmentation\nSome digits are written at 45-90 degrees. See if data aug can help here"},{"metadata":{"trusted":true,"_uuid":"4e4a1ea707d97f5d1f36df4f7bef84bf8c8d0c94","collapsed":true},"cell_type":"code","source":"from tensorflow.python.keras.preprocessing.image import ImageDataGenerator\n\ntrain_datagen = ImageDataGenerator(rotation_range=10,\n                                   shear_range=0.4)\ncv_datagen = ImageDataGenerator(rotation_range=10,\n                                shear_range=0.4)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d844b1495801f8b2ed51c892e4f4af3ce4624eea"},"cell_type":"markdown","source":"### Fit again"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"6814753ca6646b9a865f36953c10996c1b6e4b6e"},"cell_type":"code","source":"from tensorflow.python.keras.callbacks import EarlyStopping\nearly_stopping = EarlyStopping(monitor='val_loss',\n                               patience=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"035e115de8c98802f10f7f569bcfe52331e6f76c"},"cell_type":"code","source":"history = model.fit_generator(train_datagen.flow(X_train, y_train),\n                              epochs=30,\n                              callbacks = [learning_rate_reduction, early_stopping],\n                              validation_data=cv_datagen.flow(X_cv, y_cv),\n                              workers=2,\n                              use_multiprocessing=True\n                             )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d8bb82a07ce98c295d1d2c1d8431c7e9b1c1c070"},"cell_type":"code","source":"import gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"02a2ca6352d5366dbff1ce920141a7cb88f7e589"},"cell_type":"markdown","source":"### Learning Curve"},{"metadata":{"trusted":true,"_uuid":"c51c35deb8cd89f26c668c72b5ff7e2d19bc43d3"},"cell_type":"code","source":"plt.title(\"Learning rate curves\")\nplt.plot(history.history[\"lr\"])\nplt.ylabel(\"learning rate\")\nplt.xlabel(\"epochs\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"beb5c4246431d49cbd702744f3643a36434f2e45","scrolled":false,"trusted":true},"cell_type":"code","source":"plt.title(\"Loss curves\")\nplt.plot(history.history[\"loss\"])\nplt.plot(history.history[\"val_loss\"])\nplt.ylabel(\"loss\")\nplt.xlabel(\"epochs\")\nplt.legend(['train','cv'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bb76ffa29d671546661e5a730ff337ceaa04c76f"},"cell_type":"markdown","source":"### Confusion matrix\nval_acc: 0.9873"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"eab21bb33ce5ddeb2d42c8d4060073a31dc44950"},"cell_type":"code","source":"preds_cv = model.predict(X_cv)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c0ef3b574267e5c385a7e935b60723bd27feb97d","collapsed":true},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\ncm = confusion_matrix(np.argmax(y_cv,axis=1), np.argmax(preds_cv,axis=1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"47350cedee499685ac0fad74f0a2e77a2035fa75"},"cell_type":"code","source":"cm","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a32b22a20674d2691b13bbbd97eeff590620ec0e"},"cell_type":"markdown","source":"## Predict\n* https://www.tensorflow.org/api_docs/python/tf/keras/Model#predict"},{"metadata":{"_uuid":"adadcd26e850e4d06db31f47e10f01e142536027","trusted":false,"collapsed":true},"cell_type":"code","source":"predictions = model.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"05de60d49f9c3427acebb7223c38f967c261379c","trusted":false,"collapsed":true},"cell_type":"code","source":"predictions.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"af441f119b247fd01b0e748a8e30d2d80d323205"},"cell_type":"markdown","source":"## Submit"},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"_uuid":"92e050d840e768199f91b59a2f0671866bc16690","trusted":false,"collapsed":true},"cell_type":"code","source":"!head ../input/sample_submission.csv","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ceeb1cdfa6c51d6e4b60a724e632394b8a8f1d0c","trusted":false,"collapsed":true},"cell_type":"code","source":"submission = pd.DataFrame({\"ImageId\": range(1,predictions.shape[0]+1), \"Label\": np.argmax(predictions, axis=1)})\nsubmission.to_csv('submission.csv', index=False)\n!head submission.csv","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}