{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\n\nnp.random.seed(2)\n\nfrom sklearn.model_selection import train_test_split\nimport itertools\nfrom keras.utils.np_utils import to_categorical\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Flatten, Dropout, Conv2D, MaxPool2D\nfrom keras.optimizers import RMSprop\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ReduceLROnPlateau","execution_count":1,"outputs":[{"output_type":"stream","text":"Using TensorFlow backend.\n","name":"stderr"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')","execution_count":2,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Y_train = train['label']\nX_train = train.drop(labels=['label'], axis=1)\ndel train","execution_count":3,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = X_train/255.0\ntest = test/255.0","execution_count":4,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = X_train.values.reshape(-1,28,28,1)\ntest = test.values.reshape(-1,28,28,1)","execution_count":5,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#encode labels to one hot vectors\nY_train = to_categorical(Y_train, num_classes=10)","execution_count":6,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"random_seed = 2\nX_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size=0.1, random_state=random_seed)","execution_count":7,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"g = plt.imshow(X_train[0][:,:,0])","execution_count":8,"outputs":[{"output_type":"display_data","data":{"text/plain":"<Figure size 432x288 with 1 Axes>","image/png":"iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADgJJREFUeJzt3X+s3XV9x/HXa93lNlZgdGVNrV1RgRpCtjqvBSNZaphCC0lxMcRmId1SW2JonMaYERYR2T9kmxIghNhCtZCKLgqhs/UHNjXEBDsurCsgVBi2s6X0YupsnbG09b0/7rfmAvd8v6fnfM/5nnvfz0dyc8/5fr7f833fA69+zznv7/l+HBECkM8fNF0AgGYQfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSf1hP3d2hodjpmb1c5dAKr/V/+m1OOZ21u0q/LavlHSHpBmS7o2I28rWn6lZusSXd7NLACV2xva21+34Zb/tGZLulrRM0kWSVtq+qNPHA9Bf3bznXyLpxYh4KSJek/R1SSvqKQtAr3UT/vmSfj7h/v5i2evYXmt71PbocR3rYncA6tTzT/sjYn1EjETEyJCGe707AG3qJvwHJC2YcP/txTIAU0A34X9C0gW232H7DEkfk7SlnrIA9FrHrb6IOGF7naTvabzVtzEinq2tMgA91VWfPyK2SdpWUy0A+ojTe4GkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSRF+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0iqq1l6be+VdFTSSUknImKkjqIwOI4te1/p+L6/Lt/+Z1dtaDn2yZfLH/s/xhaWjs+865zS8eHvPFE6nl1X4S98MCJ+UcPjAOgjXvYDSXUb/pD0fdtP2l5bR0EA+qPbl/2XRcQB238i6VHbz0fEYxNXKP5RWCtJM/WWLncHoC5dHfkj4kDxe0zSw5KWTLLO+ogYiYiRIQ13szsANeo4/LZn2T7z1G1JH5b0TF2FAeitbl72z5X0sO1Tj/O1iPhuLVUB6LmOwx8RL0n68xprQQP23fr+0vFj806Uji98qPzxr1izuGT0eOm2x68/t3T8c3d9pXR83Y7rWo5duIZzAGj1AUkRfiApwg8kRfiBpAg/kBThB5JyRPRtZ2d5dlziy/u2vyxmLDq/5djh28u3/d/R8nbawpsf76Skvij7u6Xqv73M2ctf7HzjBu2M7ToSh93Ouhz5gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiCpOq7ei4adv3lfy7F//8+yr9RKFw5wH7/KyT3lvfjZn259HsANW79duu3di67uat9TAUd+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0iKPv8UUDVN9hV/9EDLsT1ryi+PPZ0dWtr6WgVXveW3pdveOQ36+FU48gNJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUpV9ftsbJV0taSwiLi6WzZb0DUnnSdor6dqI+GXvyszth/dtKB1funpNy7FhTd+pqKvOf3jy8/d0/Njfe3lX6filuz5aOj4VrvvfzpH/q5KufMOyGyVtj4gLJG0v7gOYQirDHxGPSTr8hsUrJG0qbm+SdE3NdQHosU7f88+NiIPF7Vckza2pHgB90vUHfjE+2V/LCf9sr7U9anv0uI51uzsANek0/Idsz5Ok4vdYqxUjYn1EjETEyJCGO9wdgLp1Gv4tklYVt1dJeqSecgD0S2X4bT8o6XFJi2zvt71a0m2SPmT7BUl/VdwHMIVU9vkjYmWLoctrriWtqn61VN5zHv7O9OzlVz0vVec/dKOqjz/70+Xbn6yxll7hDD8gKcIPJEX4gaQIP5AU4QeSIvxAUly6ewAc/dPp+5+hrF33Z/9U3sK8823dtfI++XLrfe/+XPnU5WdXtE+nQiuvCkd+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0hq+jaYp5Az/+dEV9uX9dK7/brvjEXnl46fv3lf6XhZr37rb2aWbvvuez9ROv7Oza+Wjp8smWZ7Ol/SvF0c+YGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKY/PttUfZ3l2XGKu+H26frWtvNf+48XfbDm2/IPll6B+6W/OLR1//uOdT3MtSe/9Qute/ZwvP97VY+PNdsZ2HYnDbmddjvxAUoQfSIrwA0kRfiApwg8kRfiBpAg/kFTl9/ltb5R0taSxiLi4WHaLpDWSTn2h+qaI2NarIrM7e3nr76VLkl5uPbRtR+tzANrR7Xfq5+yhlz+o2jnyf1XSlZMsvz0iFhc/BB+YYirDHxGPSTrch1oA9FE37/nX2d5te6Ptc2qrCEBfdBr+eyS9S9JiSQclfbHVirbX2h61PXpcxzrcHYC6dRT+iDgUEScj4neSNkhaUrLu+ogYiYiRIQ13WieAmnUUftvzJtz9iKRn6ikHQL+00+p7UNJSSXNs75f0eUlLbS+WFJL2Srq+hzUC6IHK8EfEykkW39eDWtDCvlvfX7FG63nuy+aol6Q731Z+/fpuro2PwcYZfkBShB9IivADSRF+ICnCDyRF+IGkmKJ7ACwaHSod/9VYebtt6eo1Lceqpuheuqz1tpL02a0PlI6v23Fd6fiFa5gKe1Bx5AeSIvxAUoQfSIrwA0kRfiApwg8kRfiBpOjz12DGovIptA8tLZ8Gu6qPX3np7i5UnQdw91VXl47/bMeG0vErtPi0a0J/cOQHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaTo89fghq3fLh2/+V/+rnS8l338bnFp7umLIz+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJFXZ57e9QNL9kuZKCknrI+IO27MlfUPSeZL2Sro2In7Zu1KbVT5N9vOl28758uP1FtNHVdcqKJseHIOtnSP/CUmfiYiLJF0q6QbbF0m6UdL2iLhA0vbiPoApojL8EXEwIp4qbh+V9Jyk+ZJWSNpUrLZJ0jW9KhJA/U7rPb/t8yS9R9JOSXMj4mAx9IrG3xYAmCLaDr/tt0r6lqRPRcSRiWMRERr/PGCy7dbaHrU9elzHuioWQH3aCr/tIY0Hf3NEPFQsPmR7XjE+T9LYZNtGxPqIGImIkSEN11EzgBpUht+2Jd0n6bmI+NKEoS2SVhW3V0l6pP7yAPRKO1/p/YCk6yQ9bftUX+cmSbdJ+jfbqyXtk3Rtb0ocfFVf2Z2jwW31VbXyqr6uvPU3M+ssB31UGf6I+JEktxi+vN5yAPQLZ/gBSRF+ICnCDyRF+IGkCD+QFOEHkuLS3W16/uP3tBxbunpNHys5PceWva90/LN3PdDV41dN4S1x6e9BxZEfSIrwA0kRfiApwg8kRfiBpAg/kBThB5Kiz9+m937hEy3Hbr3rK6XbVn3fv8rQildLx3+8+Jslo+WX1n73va3/LklaeHPVtQjo409VHPmBpAg/kBThB5Ii/EBShB9IivADSRF+ICmPz7TVH2d5dlzi6Xe1759uKP/O/Nz55TOXl/fppUt3fbR0/Pgj57be9w/LzxE4uYc+/XSyM7brSBxudan91+HIDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJVfb5bS+QdL+kuZJC0vqIuMP2LZLWSDrVSL4pIraVPdZ07fMDg+J0+vztXMzjhKTPRMRTts+U9KTtR4ux2yPiXzstFEBzKsMfEQclHSxuH7X9nKT5vS4MQG+d1nt+2+dJeo+kncWidbZ3295o+5wW26y1PWp79LiOdVUsgPq0HX7bb5X0LUmfiogjku6R9C5JizX+yuCLk20XEesjYiQiRoY0XEPJAOrQVvhtD2k8+Jsj4iFJiohDEXEyIn4naYOkJb0rE0DdKsNv25Luk/RcRHxpwvJ5E1b7iKRn6i8PQK+082n/ByRdJ+lp26euA32TpJW2F2u8/bdX0vU9qRBAT7Tzaf+PJE3WNyzt6QMYbJzhByRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUoQfSKqvU3TbflXSvgmL5kj6Rd8KOD2DWtug1iVRW6fqrG1hRLSes32Cvob/TTu3RyNipLECSgxqbYNal0RtnWqqNl72A0kRfiCppsO/vuH9lxnU2ga1LonaOtVIbY2+5wfQnKaP/AAa0kj4bV9pe4/tF23f2EQNrdjea/tp27tsjzZcy0bbY7afmbBstu1Hbb9Q/J50mrSGarvF9oHiudtle3lDtS2wvcP2T2w/a/vvi+WNPncldTXyvPX9Zb/tGZJ+KulDkvZLekLSyoj4SV8LacH2XkkjEdF4T9j2X0r6taT7I+LiYtk/SzocEbcV/3CeExH/MCC13SLp103P3FxMKDNv4szSkq6R9Ldq8LkrqetaNfC8NXHkXyLpxYh4KSJek/R1SSsaqGPgRcRjkg6/YfEKSZuK25s0/j9P37WobSBExMGIeKq4fVTSqZmlG33uSupqRBPhny/p5xPu79dgTfkdkr5v+0nba5suZhJzi2nTJekVSXObLGYSlTM399MbZpYemOeukxmv68YHfm92WUT8haRlkm4oXt4OpBh/zzZI7Zq2Zm7ul0lmlv69Jp+7Tme8rlsT4T8gacGE+28vlg2EiDhQ/B6T9LAGb/bhQ6cmSS1+jzVcz+8N0szNk80srQF47gZpxusmwv+EpAtsv8P2GZI+JmlLA3W8ie1ZxQcxsj1L0oc1eLMPb5G0qri9StIjDdbyOoMyc3OrmaXV8HM3cDNeR0TffyQt1/gn/v8t6R+bqKFFXe+U9F/Fz7NN1ybpQY2/DDyu8c9GVkv6Y0nbJb0g6QeSZg9QbQ9IelrSbo0HbV5DtV2m8Zf0uyXtKn6WN/3cldTVyPPGGX5AUnzgByRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gqf8Hat9beTv8/18AAAAASUVORK5CYII=\n"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Sequential()\nmodel.add(Conv2D(filters=32, kernel_size=(5,5), padding='Same', activation='relu', input_shape=(28,28,1)))\nmodel.add(Conv2D(filters=32, kernel_size=(5,5), padding='Same', activation='relu'))\nmodel.add(MaxPool2D(pool_size=(2,2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Conv2D(filters=64, kernel_size=(3,3), padding='Same', activation='relu'))\nmodel.add(Conv2D(filters=64, kernel_size=(3,3), padding='Same', activation='relu'))\nmodel.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Flatten())\nmodel.add(Dense(256, activation='relu'))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(10, activation='softmax'))","execution_count":9,"outputs":[{"output_type":"stream","text":"WARNING:tensorflow:From /opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\nInstructions for updating:\nColocations handled automatically by placer.\nWARNING:tensorflow:From /opt/conda/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\nInstructions for updating:\nPlease use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)\nmodel.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])","execution_count":10,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', patience=3, verbose=1, factor=0.5, min_lr=0.00001)\nepochs=30\nbatch_size=86","execution_count":11,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"datagen = ImageDataGenerator(featurewise_center=False, samplewise_center=False, featurewise_std_normalization=False, samplewise_std_normalization=False, zca_whitening=False, rotation_range=10, zoom_range=0.1, width_shift_range=0.1, height_shift_range=0.1, horizontal_flip=False, vertical_flip=False)\ndatagen.fit(X_train)","execution_count":12,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# fit the model\nhist = model.fit_generator(datagen.flow(X_train, Y_train, batch_size=batch_size), epochs=epochs, validation_data=(X_val, Y_val), verbose=2, steps_per_epoch=X_train.shape[0], callbacks=[learning_rate_reduction])","execution_count":13,"outputs":[{"output_type":"stream","text":"WARNING:tensorflow:From /opt/conda/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\nInstructions for updating:\nUse tf.cast instead.\nEpoch 1/30\n - 740s - loss: 0.1043 - acc: 0.9763 - val_loss: 0.4814 - val_acc: 0.9643\nEpoch 2/30\n - 737s - loss: 0.2735 - acc: 0.9643 - val_loss: 0.1789 - val_acc: 0.9862\nEpoch 3/30\n - 738s - loss: 0.6740 - acc: 0.9575 - val_loss: 0.3185 - val_acc: 0.9802\nEpoch 4/30\n - 742s - loss: 0.7640 - acc: 0.9525 - val_loss: 0.3160 - val_acc: 0.9802\nEpoch 5/30\n - 731s - loss: 0.7308 - acc: 0.9546 - val_loss: 0.2341 - val_acc: 0.9855\n\nEpoch 00005: ReduceLROnPlateau reducing learning rate to 0.0005000000237487257.\nEpoch 6/30\n - 728s - loss: 0.4990 - acc: 0.9690 - val_loss: 0.1996 - val_acc: 0.9876\nEpoch 7/30\n - 728s - loss: 0.4804 - acc: 0.9702 - val_loss: 0.1957 - val_acc: 0.9879\nEpoch 8/30\n - 728s - loss: 0.4671 - acc: 0.9710 - val_loss: 0.1880 - val_acc: 0.9883\nEpoch 9/30\n - 733s - loss: 0.4506 - acc: 0.9720 - val_loss: 0.2303 - val_acc: 0.9857\nEpoch 10/30\n - 735s - loss: 0.4734 - acc: 0.9706 - val_loss: 0.1842 - val_acc: 0.9886\nEpoch 11/30\n - 731s - loss: 0.4590 - acc: 0.9715 - val_loss: 0.2917 - val_acc: 0.9819\nEpoch 12/30\n - 720s - loss: 0.4536 - acc: 0.9718 - val_loss: 0.2072 - val_acc: 0.9871\nEpoch 13/30\n - 719s - loss: 0.4335 - acc: 0.9731 - val_loss: 0.1650 - val_acc: 0.9898\nEpoch 14/30\n - 721s - loss: 0.4360 - acc: 0.9729 - val_loss: 0.1458 - val_acc: 0.9910\nEpoch 15/30\n - 718s - loss: 0.4399 - acc: 0.9727 - val_loss: 0.2034 - val_acc: 0.9874\nEpoch 16/30\n - 720s - loss: 0.4509 - acc: 0.9720 - val_loss: 0.1458 - val_acc: 0.9910\nEpoch 17/30\n - 719s - loss: 0.4249 - acc: 0.9736 - val_loss: 0.1535 - val_acc: 0.9905\n\nEpoch 00017: ReduceLROnPlateau reducing learning rate to 0.0002500000118743628.\nEpoch 18/30\n - 714s - loss: 0.3776 - acc: 0.9765 - val_loss: 0.1678 - val_acc: 0.9895\nEpoch 19/30\n - 719s - loss: 0.3737 - acc: 0.9768 - val_loss: 0.1420 - val_acc: 0.9912\nEpoch 20/30\n - 715s - loss: 0.3873 - acc: 0.9760 - val_loss: 0.1727 - val_acc: 0.9893\nEpoch 21/30\n - 711s - loss: 0.3535 - acc: 0.9780 - val_loss: 0.1458 - val_acc: 0.9910\nEpoch 22/30\n - 715s - loss: 0.3477 - acc: 0.9784 - val_loss: 0.1497 - val_acc: 0.9907\n\nEpoch 00022: ReduceLROnPlateau reducing learning rate to 0.0001250000059371814.\nEpoch 23/30\n - 710s - loss: 0.3333 - acc: 0.9793 - val_loss: 0.1305 - val_acc: 0.9919\nEpoch 24/30\n - 705s - loss: 0.3424 - acc: 0.9787 - val_loss: 0.1497 - val_acc: 0.9907\nEpoch 25/30\n - 710s - loss: 0.3407 - acc: 0.9788 - val_loss: 0.1382 - val_acc: 0.9914\nEpoch 26/30\n - 710s - loss: 0.3323 - acc: 0.9794 - val_loss: 0.1420 - val_acc: 0.9912\n\nEpoch 00026: ReduceLROnPlateau reducing learning rate to 6.25000029685907e-05.\nEpoch 27/30\n - 705s - loss: 0.3135 - acc: 0.9805 - val_loss: 0.1535 - val_acc: 0.9905\nEpoch 28/30\n - 706s - loss: 0.3124 - acc: 0.9806 - val_loss: 0.1420 - val_acc: 0.9912\nEpoch 29/30\n - 702s - loss: 0.3095 - acc: 0.9808 - val_loss: 0.1382 - val_acc: 0.9914\n\nEpoch 00029: ReduceLROnPlateau reducing learning rate to 3.125000148429535e-05.\nEpoch 30/30\n - 701s - loss: 0.3052 - acc: 0.9810 - val_loss: 0.1343 - val_acc: 0.9917\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"# predict results\nres = model.predict(test)\nres = np.argmax(res, axis=1)\nres = pd.Series(res, name='Label')","execution_count":14,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.concat([pd.Series(range(1,28001), name='ImageId'), res], axis=1)\nsubmission.to_csv('generated_data.csv', index=False)","execution_count":15,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}