{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"collapsed":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom keras.layers import *\nfrom keras.utils import *\nfrom keras.models import *\nfrom keras.callbacks import *\nfrom keras.layers.normalization import *\nfrom keras.preprocessing.image import ImageDataGenerator\nimport matplotlib.pyplot as plt\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n%matplotlib inline\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"collapsed":true},"cell_type":"code","source":"train_data = pd.read_csv(\"../input/train.csv\")\ntrain_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6939c7ad3876f8f4fc8c50a6731278e6f5a9bd1f","collapsed":true},"cell_type":"code","source":"test_data = pd.read_csv(\"../input/test.csv\")\ntest_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"beed6512a651e73b3a436e670d0a13b92f1bcd2d","collapsed":true},"cell_type":"code","source":"y_train_sparse = train_data.loc[:, \"label\"].values\ny_train = to_categorical(y_train_sparse)\nprint(y_train.shape)\nx_train = train_data.iloc[:, 1:].values.reshape(-1, 28, 28, 1) / 255.0\nprint(x_train.shape)\n\nx_val = x_train[-len(x_train) // 10:]\ny_val = y_train[-len(y_train) // 10:]\ny_val_sparse = y_train_sparse[-len(y_train_sparse) // 10:]\nx_train = x_train[:len(x_train) * 9 // 10]\ny_train = y_train[:len(y_train) * 9 // 10]\nprint(x_val.shape)\nprint(x_train.shape)\n\nx_test = test_data.values.reshape(-1, 28, 28, 1) / 255.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"06115add299231acfc4ebfe38e97a1c4fae09779"},"cell_type":"code","source":"x = Input(shape = (28, 28, 1))\ntrain_data_gen = ImageDataGenerator(width_shift_range = 0.1,\n                                    height_shift_range = 0.1,\n                                    shear_range = 0.2,\n                                    zoom_range = 0.2,\n                                    rotation_range = 30)\ntest_data_gen = ImageDataGenerator()\ntrain_data_gen.fit(x_train)\ntest_data_gen.fit(x_train)\ntrain_batches = train_data_gen.flow(x_train, y_train, batch_size = 64)\nval_batches = test_data_gen.flow(x_val, y_val, shuffle = False, batch_size = 64)\ntest_batches = test_data_gen.flow(x_test, shuffle = False, batch_size = 64)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8e44cfcdd84c08dea9c49de2daae3030d6c40497","collapsed":true},"cell_type":"code","source":"pred = Conv2D(32, (5, 5), padding = \"same\", activation = \"relu\")(x)\npred = BatchNormalization()(pred)\npred = MaxPooling2D(pool_size = (2, 2))(pred)\npred = Dropout(0.25)(pred)\n\npred = Conv2D(64, (5, 5), padding = \"same\", activation = \"relu\")(pred)\npred = BatchNormalization()(pred)\npred = MaxPooling2D(pool_size = (2, 2))(pred)\npred = Dropout(0.25)(pred)\n\npred = Flatten()(pred)\npred = Dense(1024, activation = \"relu\")(pred)\npred = BatchNormalization()(pred)\npred = Dropout(0.5)(pred)\npred = Dense(10, activation = \"softmax\")(pred)\n\nmodel1 = Model(inputs = x, outputs = pred)\nmodel1.compile(optimizer = \"adam\",\n               loss = \"categorical_crossentropy\",\n               metrics = [\"accuracy\"])\nmodel1.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cb9fa3034012fd3115ea39ef3eaa93baf5da2c02","collapsed":true},"cell_type":"code","source":"model1_history = model1.fit_generator(train_batches,\n                                      epochs = 30,\n                                      validation_data = val_batches,\n                                      callbacks = [EarlyStopping(patience = 5),\n                                                   ReduceLROnPlateau(factor = 0.3, patience = 3, min_lr = 1e-4)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d084506a0098c4b450324d4a457f21193c71c6b","collapsed":true},"cell_type":"code","source":"plt.plot(model1_history.history[\"acc\"], label = \"train\", color = \"blue\")\nplt.plot(model1_history.history[\"val_acc\"], label = \"validation\", color = \"orange\")\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"Accuracy\")\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"97af512e3bfee6f4715b2f0a8f782a2679c9aa9b","collapsed":true},"cell_type":"code","source":"plt.plot(model1_history.history[\"loss\"], label = \"train\", color = \"blue\")\nplt.plot(model1_history.history[\"val_loss\"], label = \"validation\", color = \"orange\")\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"Loss\")\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"47761a4f350b025d513ce87ea1c22f43792d402e","collapsed":true},"cell_type":"code","source":"pred = Conv2D(32, (5, 5), padding = \"same\", activation = \"relu\")(x)\npred = BatchNormalization()(pred)\npred = Conv2D(32, (5, 5), strides = 2, padding = \"valid\", activation = \"relu\")(pred)\npred = Dropout(0.25)(pred)\n\npred = Conv2D(64, (5, 5), padding = \"same\", activation = \"relu\")(pred)\npred = BatchNormalization()(pred)\npred = Conv2D(64, (5, 5), strides = 2, padding = \"valid\", activation = \"relu\")(pred)\npred = Dropout(0.25)(pred)\n\npred = Conv2D(128, (1, 1), padding = \"valid\", activation = \"relu\")(pred)\npred = BatchNormalization()(pred)\npred = Dropout(0.5)(pred)\npred = Conv2D(10, (1, 1), padding = \"valid\")(pred)\npred = GlobalAveragePooling2D()(pred)\npred = Activation(\"softmax\")(pred)\n\nmodel2 = Model(inputs = x, outputs = pred)\nmodel2.compile(optimizer = \"adam\",\n               loss = \"categorical_crossentropy\",\n               metrics = [\"accuracy\"])\nmodel2.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc635c1abf6467c01c1fb51671867cfcae5b74e2","collapsed":true},"cell_type":"code","source":"model2_history = model2.fit_generator(train_batches,\n                                      epochs = 30,\n                                      validation_data = val_batches,\n                                      callbacks = [EarlyStopping(patience = 5),\n                                                   ReduceLROnPlateau(factor = 0.3, patience = 3, min_lr = 1e-4)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c1057895efe551fd4f5ecbde3d46922d7d36ef73","collapsed":true},"cell_type":"code","source":"plt.plot(model2_history.history[\"acc\"], label = \"train\", color = \"blue\")\nplt.plot(model2_history.history[\"val_acc\"], label = \"validation\", color = \"orange\")\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"Accuracy\")\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"24ab5c0db3c061786b77335b9c8359696c6897e1","collapsed":true},"cell_type":"code","source":"plt.plot(model2_history.history[\"loss\"], label = \"train\", color = \"blue\")\nplt.plot(model2_history.history[\"val_loss\"], label = \"validation\", color = \"orange\")\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"Loss\")\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"907dd8f7ddc720c369f5e5aad5fd52c78bea31a4","collapsed":true},"cell_type":"code","source":"models = [model1, model2]\npred = Average()([model.outputs[0] for model in models])\nensemble = Model(inputs = x, outputs = pred)\n\nval_preds = ensemble.predict_generator(val_batches)\nprint(val_preds.shape)\nval_preds = np.argmax(val_preds, axis = 1)\nval_acc = np.sum(val_preds == y_val_sparse) / len(y_val_sparse)\nprint(\"Ensemble validation accuracy: %s\" % val_acc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cc24cbd4b7159ea5c2b0f02f036c5ad08f0489a6","collapsed":true},"cell_type":"code","source":"test_preds = np.argmax(ensemble.predict_generator(test_batches), axis = 1)\nres = pd.DataFrame(test_preds, index = np.arange(len(test_preds)) + 1, columns = [\"Label\"])\nres.to_csv(\"submission.csv\", index_label = \"ImageId\")\n\ntest = pd.read_csv(\"submission.csv\")\ntest.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}