{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Activation, Dropout\nfrom keras.utils import np_utils\nfrom keras.callbacks import EarlyStopping\nfrom keras.callbacks import ModelCheckpoint\nfrom keras.models import load_model\nfrom keras import regularizers\nfrom sklearn.model_selection import train_test_split\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.layers import Conv2D\nfrom keras.layers import BatchNormalization\nfrom keras.layers import Flatten\nfrom keras.callbacks import LearningRateScheduler","execution_count":66,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")\nY_train = train[\"label\"]\nX_train = train.drop(labels = [\"label\"],axis = 1)\nX_train = X_train / 255.0\nX_test = test / 255.0\nX_train = X_train.values.reshape(-1,28,28,1)\nX_test = X_test.values.reshape(-1,28,28,1)\nY_train = np_utils.to_categorical(Y_train, num_classes = 10)","execution_count":67,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n# PREVIEW IMAGES\nplt.figure(figsize=(15,4.5))\nfor i in range(30):  \n    plt.subplot(3, 10, i+1)\n    plt.imshow(X_train[i].reshape((28,28)),cmap=plt.cm.binary)\n    plt.axis('off')\nplt.subplots_adjust(wspace=-0.1, hspace=-0.1)\nplt.show()","execution_count":68,"outputs":[{"output_type":"display_data","data":{"text/plain":"<Figure size 1080x324 with 30 Axes>","image/png":"iVBORw0KGgoAAAANSUhEUgAAA2IAAAEWCAYAAAAEgyRsAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XncTPX///EjsmWNhGRJIbQILRLZKkvWLEkkn2SrlBafCKlkqxRRlLKFQvigRQjJ2mKJkhIttizZsvf743f7vD7Pc74z1zVzXTPnmuVx/+s55pw572uaOWdO53Ve70z//POPAwAAAADwz3kZPQAAAAAASDaciAEAAACAzzgRAwAAAACfcSIGAAAAAD7jRAwAAAAAfMaJGAAAAAD4jBMxAAAAAPAZJ2IAAAAA4LMsPm+P2aMBAAAAJJNMgf6RK2IAAAAA4DNOxAAAAADAZ5yIAQAAAIDPOBEDAAAAAJ9xIgYAAAAAPuNEDAAAAAB8xokYAAAAAPiMEzEAAAAA8BknYgAAAADgM07EAAAAAMBnnIgBAAAAgM84EQMAAAAAn2XJ6AEkizp16lhevHix5QkTJlhu3769r2NKqwMHDrgeHz161PLrr78ecJ3Vq1db7tatm+U8efJYvv32213rZMqUKV3jjGdnz561/MQTT1jOnDmz5cGDBwf8dyBc//zzj+Xdu3e7nhs9erTlP/74w/L48eNTfd2OHTtaHjBggOu5YsWKWT7vvOT7f4LBvuNffPGF5bVr11quUaOGZe9+tmLFitEYIgCk2+nTpy3rb8F58+YFXefYsWOWR40aFXCZG2+80XKbNm0s33vvvZZz5MjhWsf7OBYk39EPAAAAADIYJ2IAAAAA4LNMWpLiA183ltFq1aplecWKFZbPnDljWUsT9XJqLDhy5Ijljz76yHK7du1cy+ll51CUKVPG8s6dOy3fd999ruWeeuopyyVLlgxrG/Hu77//tpwzZ85Ul8mePXvUx+Sn0qVLWy5fvrzlmTNnWs6aNWvUx6HvseM4zmeffWb5zjvvjPr2o+nEiROWdT/UpUsXX7b/0ksvWe7Zs6flRC5T1H1lhw4dLE+dOtVyo0aNLOfLl8/y+++/b/n88893ve6MGTMs33HHHZEZLICI2LJli+WRI0daPnnypOW9e/daTqlk7/rrr7fcvHlzy/Xr17d89dVXp32wEbJr1y7Lzz77rOWxY8f6Oo6BAwe6Hvft29fX7XsEvN8mcY94AAAAABCjOBEDAAAAAJ9Rmhhhzz//fMB86tQpy61bt7b89ttvWw5WguanQ4cOWdZSyZQulUdL4cKFLc+ZM8dy2bJlLefNm9fXMfkl2UsTf/vtN8tXXHGFZe3alz9//qiP4/fff3c9btasmeU1a9ZEffuRpp2oqlWrZnnDhg0ZMRyj5To9evTIwJFE17///W/L2vW0a9eulrVLpapdu7blJUuWuJ7LlSuX5U2bNlkuUaJE2geLhLFv3z7L+l1zHHeXTu/n6r+0FLZhw4aWy5UrZ1mPy15Nmza1rJ/VLFkSq3G33s7x9NNPW544cWLAZZT+Fk9Lx2j9DdCqVSvL7777btivFQm6r3vvvfcs79+/3/Lx48ctV6lSxbW+lqhfeOGFAfO6dessb926NeA4KlSo4HpcvXp1y2PGjAn+B0QHpYkAAAAAEAs4EQMAAAAAn1GamE6zZ892Pb777rsta0cc7WKzfPlyy7lz547i6ML38ccfW9YuPLFELyf71eHNb6GUJuqkrjpJdqLRSb+1rHfcuHFR37a3NFEnIf78888t16xZM+pjiYQdO3ZYjqVOpFrW1KtXL8v333+/5XictHzWrFmux23btrWsf7OW2Hg7Iv6XdqvVLraO4zgHDhywPHz4cMv6XsYafW8+/fRTy1r+W7BgwaDrFy9e3PKff/5pWcudQrFs2TLXYz2mX3nllZa11Ey3ndG0XFtvIdBOmgsXLgy6frZs2SwXKVIk4DLnzp2zrJ2O0+Laa6+1rJ1DtSQ5XkoWdX/qOO7jQLD3qUGDBpa18296SxO/+eYby7t377bcuXNny8OGDXOt40fnYcdxvxdakq0dXrXk1XFC29/r9/7ll18OuA0v/e7+8ssvqW4jwihNBAAAAIBYwIkYAAAAAPiM0sQ0+PXXXy03btzY9dz69estFyhQwPL48eMtx9pEsFoq2a9fP8taepUWr732muWiRYta1tKZVatWhf262nVJ39eWLVuG/VqxKpTSxNtvv92ylpQmGp3oW79fq1evthytEouUShMXL15sWSdvjzV79uyxXLduXcvaXS8l+t5qaajuN5SWxeik0Wnx/fffW06pK1ss0b+5atWqruf0PV+xYoVl7WAZiu3bt7se6/p63Pn6668t+1WGFKpBgwZZ1klWtSwrpXKtYKWJ2hk0lNfy/gbS57Q0cu3atQG3ndEqVapk+dtvvw24jP5O0a5x3ueCfcf0OH3rrbda1mO8TjTspftqnbx86dKllnv37m35xRdfDPpaGU1vOfHu91euXGlZP0dt2rSxPHnyZMuRnLz+6NGjlrVLoZYAT5s2zbWOThgfj7S7sv6u1t8JXpQmAgAAAAA4EQMAAAAAv8VHa5oYoJO3PvDAA5Y3btwYdB2dODHWyhHViBEjLIdSjugtt7nhhhsCLqclDFdddZVl7ZSj3b68pYVazqD0Evz7778fdH0khlKlSlmeMGGC5b/++svyRRddFJVta0cxx4nPUg7tJhVKOaJOpO44jjN27FjLoezHtANe9+7dLW/bti3Vdb2aNGliWcvXtINgrHn11Vcte99v7QIZbL8ZipQmsv/uu+8sa2mtfo9igXbie+ONNyzfcsstloOVv6aXTmI8adKkoMvdc889lmOpHFFpZ0ydLFe70F1++eXp2sahQ4csv/XWW5ZD/R5qp0R9TytWrGh5/vz5lgcOHGg5WBfRjPLII49YTunWCn1v9DdWJMsRld6yoZ0SNScaLb1OqRwx1nFFDAAAAAB8xokYAAAAAPiM0sQUaMlC+/btLWs3HG+JSL169SxrV7tYo52iQumcqV14ChUq5HquTp06YW37ggsuCJi1ZNFx3F2qtIxFbdmyxbJOZtmoUaOwxoTYdd1112XYtr0TymopTSw7ffq05blz54a1bunSpV2Pwy2rvu222yxr2ZS3E1ooE8P+8MMPlp9//nnLOnHqpZdeGtb4okEnEU6p1E0nBU7PBNValus47k6V8WLOnDmWtdxfJ1HWHEkffvihZW83xvLly1vW/16xyo8yXe+xORRaNqadEseNG2dZP8eLFi2yHGvliGrmzJmWvb+dOnbsaPmVV16xnFIpMcKjx7Z9+/ZZ1mO1dlGNB1wRAwAAAACfcSIGAAAAAD7jRAwAAAAAfMY9Yh579uyxPGzYsFSXb9q0qevxO++8E/ExRcOGDRssa718MNWrV7ccrXsyBgwY4HqsLe/vuuuugOtoq+b//Oc/luP9HjG9f0TvudHW4MnC20I+VujnrVatWhk4kv9LW6h///33qS6v73Hv3r0jNo4uXbpYbty4seu5Zs2aWdbpQYLR+8Xq1q1rWfcBjuM4WbL4f1gbPXq0ZR2P3vvkOI5TsmRJv4YUV0L5jKbXsWPHLO/YscOy9z4f/fx77xGF45w8edKyTo2hbe0dx3F+/vlny3ovuN7zq/vQWL6PasGCBZb1vjbv/YXh3hem0wKcOXMm6OsWKFAg9MEmiMOHD1vW34Z6b6lOBXD27NmQXlffc71v+bHHHrPs928OrogBAAAAgM84EQMAAAAAn1Ga6LgvVWoZ2KZNmwIunydPHsvecpt4sX379lSX0UvrGdFOtlq1agHH4m3dnIiyZs1q+b777rOcjKWJ+n3LiLKzYD744APLWqITC5544omwlq9SpYrlaJX1Fi1a1PVYS6LDLVPcunWr5VCm34i2EydOBPz3smXLuh6np2W9evbZZ4M+ly9fPss5c+aMyPYiRacb0XJEbwlnNOj2tMy1efPmruW8j+OJfg61VFDL3lJSpEgRy7t27bKsU03oNDH6797pet58803L1157reV4KffUEsznnnvOckrvZbByRH0vx4wZEzBry/Xs2bO71u/cubNlvWVGfyckGp0SZMSIEWGtq6WcWr7oOO6W93369LH8+eefW9apUqpWrRrWttOCK2IAAAAA4DNOxAAAAADAZ7FT55OBtJvSxo0bU13+119/tZw7d+6ojCnatHwlmOuvv95y/vz5ozmcgLRMokGDBpanTp0acPlPPvnE8tGjRy3nypUrCqOLLi1/WLlyZQaOJOPdeOONlosVK2a5b9++lkeNGuVaJ1qltA0bNrQ8ePBgy0eOHLEcj/uEjh07+r5NLVWcPXu25UqVKlnWLrbBaAc8x3Gcyy+/PAKjC4928lLerrqRoqWZXrfccovliy++OCrbjwS/S9TatWtnWctZvSV1sVbOGY6FCxda1nLpUG5FSEnx4sUt//vf/7as3WK9ZbjxTvfpq1evDrjMnXfe6Xqs5aBDhgyxrOVw2g0wGC2LdBzHGTlypGX93jzzzDOpvla80t+o2n032C1D6qWXXrLsPR7r76lOnTpZ1u+Olpi+//77IY447bgiBgAAAAA+40QMAAAAAHyWtKWJ2qFGu4QF68Cl5VHx2qlGL4m3adMm1eX1Uu3evXstR2tC55S0bdvWcrDSRC1ROn36dNTHFE06fi1LSHZa+nHHHXdYfvTRR13LlStXLirb13I67d65atUqy/Xq1YvKthOZliF7O4alZuLEia7HAwcOjMiYUqNlk9u2bbNcqlQpy4ULF47KtlPqFKkl5bHmyiuvtLx27Vpft62dEr0T5iYKLZWrU6eOZT1+h2r8+PGWtUPslClTLN90001hv2680PI0Lfddvny5ZZ2Q2nEcZ+7cuZaDfcb0+3nVVVcFXGbGjBmux3qs0U6L2k0xlsuQ00KPA6NHj47Y686aNStirxUpXBEDAAAAAJ9xIgYAAAAAPkva0sQePXpYXr9+vWW9nKwTCi9atMhytmzZojy66NBOfKF0I4sl2i0PyUvLbbSTZ8+ePV3Lffzxx1HZvnZNzJEjR1S2kex0AvOUJi6OVRUrVrR8wQUXROx1dYJTLa33ipd9pR9dE5ctW2Y5WDmnlp0lEu3+WLJkybDX1xJf7VCrpXFaHu4tU5w+fbrlaHWxjSYds07wW7duXcunTp1yrZMnTx7L99xzj+XevXtb1g6UwXzxxReux4cOHbKsk0P/9NNPlhOtNDFSvB0vhw8fnkEjCY4rYgAAAADgM07EAAAAAMBnSVOa6C3l0Eu6Sjsi6uXkeC1HVDpBnk5uOXny5IwYDhAx2uEqmvQ7dM0111h+5ZVXLN98882W43ly2IyiE6mGIlodMlOjk65q2eDvv/8ele1p5zQtVfK67LLLorL9ePT9999b1tsOWrRoYVk7OcYLvZ1CuxhfeOGFUdme/i565JFHLOtk2Fqy5ziOc8MNN1jWroulS5eOxhCjSstXv/vuO8tnz551Lafl6qGUIIZKP7ta0nvJJZdEbBuJav78+a7HKe07MwpXxAAAAADAZ5yIAQAAAIDPEro0UScxvPvuu13PffXVV5Z14rg333zTsk70nAjOO+9/59066WwopYktW7a0/Nlnn7mey5UrVwRG93/pJeQOHTqkunzXrl0tawkZElPTpk0tr1u3zvWcdgjNkiXwbu6PP/6wvGHDBss6ObO3rEEn2tbyIPXiiy9afu655wIuAzedCHXUqFFhrav7Jj/p50pLt6JFO/d6S+21dF4nHU92Ovmudk1s0qRJRgwnXfT3jB6/P//8c8vRKk0MRsuCvZMQ/+tf/7Jcq1Yty/r7oUyZMlEcXXRcfvnlUXndLVu2WNZjk1flypUtlyhRIipjiRd6nD9x4oRlPYYsXbo0pNcqW7as5ZdffjkCowsdV8QAAAAAwGeciAEAAACAzzgRAwAAAACfJfQ9Yh9++KHlJUuWBF1O26zee++9UR1TrNAa+Wuvvdbyt99+G3B5nZ28du3arucGDx4c9Llw7Nu3z/X48ccft6z38ChtF/vUU09Z1navSEz6XR03bpzrOb03S+8X/Oijjyx/8cUXlvXeL21V3L9/f9frauvg2bNnWx4yZIjlatWqhfYHxJChQ4da1vs5HCd67dC3b99uWe/FO3XqVKrrjhw50nKwewCjTdvXHz16NCrb0PvCunfvHnS5xx57zHK07mGJR8Ha15cvXz4jhpMuCxYssKz3r8fK33LjjTe6Hut4tc293ss9b948y3osT0b33Xef5ZSm8GjWrJkPo4ldut99+OGHLXt/A4RCvzv6eS1WrFgaR5c2XBEDAAAAAJ9xIgYAAAAAPku40sSpU6da1lI1r5tvvtnye++9F9UxxaK8efNa1jKfLl26WNYZ5NXatWtdjwcMGGA5WPvcPHnyWNZLy5q9LeqDlSOqhg0bWk6kVq49evTI6CHEvKuvvtqytp51HMd54403Aq7ToEEDy9qitkqVKgFzSvSzrqWJsSCUcmO1detWy9728elp5btz507Lr776quu5iRMnWva2Yw9E22F369bNcqyVIR8/ftyy7t8cx91mPpivv/7aspYhabmSls86jrtEJ9np1DSatX19vIuH6VmKFy9u+dlnn7XcunVryytWrLBct25dfwYWQ3TfumbNGsvefZru+zp27Bj9gUWAlp7rtFB6+8qtt95q2TsFyO+//25ZS4z1WOudSikUenuBlsTr59VvXBEDAAAAAJ9xIgYAAAAAPkuI0sS//vrLct++fS0fPnw46Dq9evWyXKRIkegMLE5Ur17dcr9+/Sx36tTJckpdwZYvX265UqVKAZcpVKiQZS3dSW+3sZYtW6Zr/Vj122+/ZfQQYp6W12rpgl+0xCHWaJdY7YIYSpmilio7jrtz34MPPpjq+hMmTLCsJY+HDh1KdV2vq666yvILL7xg+bzzMv7/IWpnrRo1alhetmyZ5U8++cS1TuPGjQO+1v79+y3PnTvXspYj6n56/PjxrvULFy4c6rCTSqyVraaH/k4ZPXq0Zf39o/vEWNK0aVPL5cqVszxz5kzLyVKaqPsH/R2qcufO7Xqsv2vPP//86AwsAv744w/L2j14z549lrVDb82aNS1nz57d9Vp6TN+xY0dY49CSbu8tNnr7TazczpLxRzMAAAAASDKciAEAAACAzxKiNHHOnDmWtVNLSlIqW0xmrVq1sqzlccEuoYdq79696Vpfu0RpBx7tmgjg/9PvyzPPPGO5RYsWqa575swZ12PtXprSpMKRouWI2hVLy5tjgZYItW3b1rKWHj3yyCOudXTy6YULF1qePHmyZe0geckll1ju2bOnZSZtDo12Soz3ronaKfPXX3+1rOWvd911l+VYKN/9L+2Ip2W0q1atyojh+EJvwdBOtMOGDbOspbO6P9HyPcfJ2I5+4dC/WY9BWpqoli5dGpVx6C023uNGLN6KFDvfVAAAAABIEpyIAQAAAIDPEqI0US/pZs6c2fLZs2cta0mI4zjOjz/+GP2BxbkHHnjAspYIffTRR75sP1euXJanT59u+bbbbvNl+0BKtLOVTqAcanm0X7SD1KRJkyzfe++9GTEco93TgpVPhjIBciyoX7++Zf1c/PLLL67lQiml1pKyESNGWA6lrBRuWvqlnzfN8SJnzpyWtXStffv2lrVD3NNPP205o79HWo6nnVv79++fEcMJ2+rVqy1rZ0DdtzqO44wdO9aydp/1du4L5LHHHrMcSnfaWKRdCAcMGGC5d+/elkPtgKif93bt2ln2dqL9L33/KlSoYNn72z8WcUUMAAAAAHzGiRgAAAAA+CyTz52Eor6xK6+80rKWJvbp08e1XIcOHaI9lIRy4sQJy1qm6DiO8+mnn1r2Tgabmoceesiyt0xBLynH6kSV0aLdhG699dZUl9EJZeE/LZfVTnfvvPNORgwnKN3fHzx40LKWwGkXWsdxd00Mhe5btduXtxxMO7TGQ/lIqLRD2JYtW1zPTZw40fLmzZstFy1a1PKjjz5qWTvlITQ6YauWis2YMcNy8+bNfR1TNOlnqnPnzpbLli1refDgwZa9nym9BSA99PPsOI4zZswYyzoB9RNPPGFZj/k5cuSIyDiiQfeJWibnHfO+ffssB5tMvEyZMpY7depkWd+XRLBo0SLL+fPntzxw4EDLxYoVs+y95UR/02gHxqNHjwbcXqQ+x1EW8EPBFTEAAAAA8BknYgAAAADgs4QrTQQAP5w6dcpy1apVLffo0cOydh4FEH0XX3yxZZ0cW29VSFTakVDLjbXr36FDh1zraMdPnRBau9bt3LnT8ooVKyzrbQm///6763VLly5t+eGHH7bctWvXVP6K2KPva7Vq1SzrLRuO4y791k662l1RyxG1jB1JgdJEAAAAAIgFnIgBAAAAgM84EQMAAAAAn3GPGAAAAABED/eIAQAAAEAs4EQMAAAAAHzGiRgAAAAA+IwTMQAAAADwGSdiAAAAAOAzTsQAAAAAwGeciAEAAACAzzgRAwAAAACfcSIGAAAAAD7jRAwAAAAAfMaJGAAAAAD4LEtGDwAAEDvGjRvnejxo0CDLO3bsCLhOjRo1LDdp0sRykSJFLLdp0yZSQwQQY3r27Ol6/Oqrr1p+5plnLA8cONC3MSG2nTlzxvKuXbtcz/3444+W582bF3D9BQsWWN66davlypUrW163bp1rnUyZMgV8rQceeMDy8OHDLefOnTvg8pHEFTEAAAAA8BknYgAAAADgs0z//POPn9vzdWNAomndurVlvRQ/a9Ysy6VKlfJ1TOnx+eefB8zPPvus5VtvvdVy//79A/47QnP8+HHL8+fPt9yrVy/Le/bsca2j5SPB6HFESz+yZs1quUyZMq51Pvjgg6DP4f/S7/tNN93kem7y5MmW69ev79uYouGHH36wvHbt2nS91ttvv21Z9y+PPvqo5euuu861Tr169SxffPHF6dp+ItqwYYPlESNGWJ40aZJrubNnz1ouUaKE5eeee85yu3btojFExLA//vjD8oMPPmhZywxTEuxYE8ryoa6jx8OhQ4eGNK4QBdw4V8QAAAAAwGeciAEAAACAz+ia6DjOyZMnLR88eDDgMh999JHlTp06hb0NvTzasGFDy88//7zla6+9NuzXjRd//fWX5e3bt1ueMGGC5e+++861zpdffmlZO9podyYteUgGeln922+/tfzxxx9b7tq1q69jCoeWBzmOuwTR+1xq/05pYmiOHDliWb877777rmXdP1100UWu9UMpG9T19+/fb1nL6TZt2uRa54477rC8ePFiyyVLlkx1e8lI94fe45S+5/FIy2Hvuusuy95jQnrovlNL6rz0cxlquVSi0+P0U089ZXnv3r0hra/dVvX30+bNmy1rd1YkrubNm1tes2aN5VBKBkPVqFEjy1rq7DjubozBTJ8+3XKESxMD4ooYAAAAAPiMEzEAAAAA8FnSdk3cuXOn5X/961+WFy1aFHD5cDu1hLp+0aJFLWvpyaWXXhr2NmLNzJkzLeskjhs3brSclvfylltusTxnzhzLefPmDfu14sGMGTMs33PPPZZPnTplefTo0ZZjuTSxVq1arsfByg7TK5SyxZo1a6a6zIABA9I/mAy2cuVKy9WrVw+4jH6uunXr5nruxhtvDGt7v/32m2XtpNa3b9+g61xxxRWWdT944YUXhrXtRHP48GHL+nk9ceKEazktU86WLVv0BxZhWkqU0eWAekzWz3Ky0VLiq666ynKwY7a+b47j3qdUqVLFsnb+LVCggOXVq1dbvuyyy9IwYv8dPXrU9Vj3tStWrLD8ySefBFxfSzt//vlny40bN3Ytt2TJEsvbtm2zXKhQoTBHnDF0QuYWLVpYPn36tOVQfwtqeb1+5vQYorf/LF261LW+TtCsZYrffPONZd3X6r4pAuiaCAAAAACxgBMxAAAAAPBZ0nRN1O5djuM4w4YNsxysHDEt9FLpqFGjLOvlVC2L1Mnt3nrrLcvaTS7W6eXlDh06WNYJY72X8APRS9aO4zjZs2e3PGXKFMvLly+3PH78eMs6SWci0fdXyxHjkbccMFqliaG8bijLpPQ91Mml47GEsUmTJpYnTpwYsdctVqyY5fvuu8+yTuDsOI6zfv16y7p//vvvvyM2lng3d+5cy/p+6UTFjhOf5YiIPfoZC2VicP294/1+66Tj2h1PabdPLUnt0aNH6oP10RdffGF58ODBlrWbtuM4zrlz5yKyPf3ee+lvnt69e0dke9GmpZn6e0Zv2fF2wB45cqTl9JQH1q5dO+hzVatWtdy2bds0byO9uCIGAAAAAD7jRAwAAAAAfJbQpYl6qdx7qfvPP/+MyjaLFCliuW7dupYrVKhgWUsTVc6cOaMypkjQy8mrVq1yPaclhcEmFtW/7emnn7Z85513Wi5fvrxrHX0t7RyoE3AncxlTjRo1LGuHqljmLeHTx8HK+7TrUbBOh97OSNEqeUxpm/FAS0Fmz54d9e3p/vCxxx5zPde+ffuA48L/rFu3LuC/a0lNItAy2Yzumphs9LeQHo93794dcHntgKjlYzfccEMURhcbhgwZYllvuShcuLBrOe0ued1111nW41b+/PlT3Z7+xnEcdxfgffv2pT7gGKDl5jpBcrDuiPq5cpyIdyuMaVwRAwAAAACfcSIGAAAAAD5LuNJEnYTwgQcesKwTYzpO2iYSDsXmzZstv/TSS5ZDuZy8Y8eOqIwpEvRyvLe7odISxKZNm1rWjoZ6yT4l+fLls/zqq69a1gn5dBvJJkeOHJbz5MmTgSOJDL87D+rk0mkpZQxlQuhYoJOmFi9e3PKvv/5qWSdc79evX8S2/dVXX1n2TjQerX1wItGSnkSmk/y+8MILloOV8TuOu9y/Y8eOlnUS8Pvvvz9SQ0wY3kmqR48eHfS5/ypXrpxlnVz4ggsuiPDoYlO9evUsaydd78TT+pslPXQCZMdxlzM+/vjjEdlGtOmtJcFuWVG6D0g2XBEDAAAAAJ9xIgYAAAAAPuNEDAAAAAB8lhD3iGmrzzZt2ljW+8LS0h65UKFClvXeJ63f9bZcf+ONNyx369Yt4Pb13ohrr72iZ7zrAAAgAElEQVTWstYex4IxY8ZY7t+/f9DltFZfW9Prf4tQeNsWP/fcc5a3bNliWWe5z549e1jbiBf6mR4+fHgGjiT+6b1n6f2OaRthv+9pS6syZcpY1ik9tJ18pO5tcBz3fWFff/215ePHjwddR9/XvHnzRmws8WjNmjWWtYW43pOo9+wkAr3HVadV6NSpk2XvFB333XefZb0vTNfB/3fq1CnLnTt3dj338ccfB1ynVatWlqdNmxadgcWJhx9+2Nftee/Tvfjiiy3rlCBIDFwRAwAAAACfcSIGAAAAAD5LiNLEAwcOWD527JjllNojB3tOy3hWrFhhWUsf1M8//+x6rG3Wg22jRIkSlrV17EUXXRR0vBlBS0S0/aiWIjqO4yxatMiy/g1nzpyxfOLECcvaNrt27dqW9b+jd32l/40TVbZs2Sxru9q2bdtmxHDiTqTKEb0lufFSjhhM1apVLS9fvjykdbRMdu/evQGXWbp0qWVtU59SOaLS6QO0BK1v376WdeyJTEvfs2bNavmuu+6ynCVLQhy6A9Jy/cWLF1sOtWTVW+IeLu/xLV7p8VN/y3hLEfV3SsGCBS0/+eSTERvLf/7zn4D/rtvWY14yWr9+fcDsOI4zatQov4eTblqiHuzWIP33nj17up778ssvA66j0xf961//SnUcWh7vOMGnT9KST7/3r1wRAwAAAACfcSIGAAAAAD7LlJZugukQ9Y29/fbblnv06GFZy2scJ3jZoJbj3XnnnQHX1zKcPn36uNb3Xgb9ryZNmlh+/fXXLcdyBxztZKXlgKVLl3Ytpx3P1M6dOy0vXLjQcrAOkl7nn39+wG1MnDjRsna2TCRHjhyxXL16dcsbNmywrJ/PuXPn+jOwOKGlbrVq1Urz6/i8f4wJAwcOdD3+/vvvLU+fPj3gOqF+p4MJtn61atUsh1pKGe8uu+yygP/uLYNPdjt27LCsn1n9jIZSGlujRg3X46lTp1qO5eNzaj777DPLt912W9DlihUrZlmP2ZFUr149y3orgx/bjmXnzp2zrCVz3tsvdB+cOXPm6A8sAlauXGlZf8OotBw3wl3HewwPtk7Dhg0t58iRw3Lr1q0tN2/ePKQxpiDgxrkiBgAAAAA+40QMAAAAAHyWcKWJavPmzZYrVqzoei7Y5Umd2HTQoEGW9TLrpEmTgm5Ty/Z0EkAtk4wX11xzjeVNmzZF7HVDvbR80003WdZJnBPVjz/+aFk7pGk5ovr0008ta+kH3KI1obN2VAxWnhuPihcv7nr822+/pbqOfqd1suEGDRqEtE3tsLVs2bKAy+g+9LXXXgvpdeMRpYmh0ePTxo0bw1o3e/bslv/66y/Xc1oSH8+uv/56y2vXrrWsnREdx30cqVSpUsS2/8wzz1h+/vnnAy4zYcIEy+3bt4/YtuPFu+++a7ljx46W58yZ41qucePGfg0pYvT2ivvvv9/yrFmzLMdSaWKwdbQ8PgK/QylNBAAAAIBYwIkYAAAAAPgsoUsTlbc0cMyYMWGtr+/TxRdfbFkvvzuO47Rr186ydh2MRwcPHrSsXXs++OAD13JaIqITETZr1szyn3/+ablFixaW9TKxt3xUuyt5yykSkU7a+NBDD6W6/O23327ZO0knUpeW7n7BJFJ3RW8ZzPz58y1fcMEFlrXL1C233GK5TZs2li+88MKQtnn06FHLWtq4a9cuy1oyuX379pBeN15o11PtsKv71m+//dbXMcWa06dPux5fffXVln/44YdU19dyxMcff9yyt0toPKtcubJlLWnX46e3w260Jkq/4YYbLGtppNLSxHvvvTcq44g1p06dsqzvvXaA1nJRx4nssSojaPdSPT7o5PWRLE3U74H32Byss3mwbehE4yNHjrQcymTSAVCaCAAAAACxgBMxAAAAAPBZlowegF/69u3rejx69Oiw1tdLlXfccYflTp06uZbTy5jxLn/+/Ja1g6HmlOikhG3btrUcrMzTOzl2MpQjqq5du1rWrkk6MafSCSARPv0camdFnbBdJ4ZOiZZIxHuZ4jvvvON6vG3bNstZs2a1HMkOa7ly5bIcLxOWRpJOZqufJe2kluyGDBniehxKOaLSUvtEOk7rrQLa3ViPrZ07d7YcrVJEb2dEvU0B/9O9e3fLWj66YsUKy/FeiuiVM2dOy1oaq8faAgUKuNa54oorLOukyo8++qjlmjVrpmtcW7dutayltIcOHbJ88uRJy/pbLI2liQFxRQwAAAAAfMaJGAAAAAD4LKFLE/Uy/YIFC1zP6aXf3LlzWz5z5ozlv//+O+Draoc6LSlxHPfl1GSk3c9GjBhhWbvj5MiRw7JO7hdqyWOi0pKs884L/P9I9HL4c889F/UxJQstTVTe0kSdEDpY2WKtWrUsL1myJL1D8523RMT7OFK0/EP3DwcOHAi4fCJP+Prhhx8G/PcKFSr4PJLYsmXLFsvektlQ5M2b13IilXv99NNPlvv162dZO0uWL18+4DKRpOWIL7zwgus57Q6otzm0atXKcjxOVJwW+ltSy9v083nVVVf5OqZYoF11teTQcdy/pVevXm1Zu/imtzSxTJkylrXLuU7y7sd+gytiAAAAAOAzTsQAAAAAwGeciAEAAACAzxLiHrH9+/dbfuSRRyzPnDnTsragdBzHqVOnjmVti/vNN99Y7tGjR8D19+7da/mXX35xvW6y3yOmdeJDhw4NuMy7775rOdnvCwvXjBkzLI8aNcpyMrb81nu0UmozH+z+r1DceuutIW0/2L9rTum1koHeE+Y4jlOvXj3LX331lWWtyS9ZsqTldu3aRW9wMapu3boZPQTf6TG4efPmlnfs2BH2a7300kuWdeqFePfRRx9ZDtbGX6dDiSS9B3blypWW9Z4wr/r161seM2ZMVMYVy7Rlu/5m1H4D2qsgWeh9n4sXL3Y95338X5G8Z+uNN96wrL/rg9F7yiKJK2IAAAAA4DNOxAAAAADAZwlRmrh8+XLL2hpUL5VXrlzZtY62ob7uuusC5m3btlnW8kW1bt0612Mtt0kWc+bMsfzKK68EXKZgwYKWtX0twqPlXf/8808GjiTjpdRKvn///j6PBoH88ccflhs2bOh6bv369Zb1s1y2bFnLWrpTokSJaAwxwxw/ftzykSNHLN9xxx0ZMZyYsWLFCstpKUesWLGi5bS0t9YW8D/++GNY6xYuXNiytub2S/Xq1S3r7RehOnv2rGUtF/73v/9tWf/76HQ/Xk2aNLH82muvhT2WeKct6/VYlS9fPsve36VI3dixYy3rfjPYbUHe30na/v7LL7+0nFJp7X81aNAg5HGGgytiAAAAAOAzTsQAAAAAwGdxW5q4adMmy3fffbdlvbxYtWpVy4sWLXKtf8EFF6S6jQIFCqS6TJUqVVJdJtF16dLFspZ16GX3YB1wEB4tffFjxvdYE2qnxGh1KNQSk1gzbtw4y88//7xlLbf+8MMP07UNLafTEo9ly5ZZ1g5huj/2dqXSz6+WK7/44ouWE60cUf3000+WtcRdO/8ifPrb4NFHH7V8zTXXhLT+0aNHLb/66qthbVvLSmfNmuV6Lnv27GG9Vlr8/PPPltesWWP5kksusTxlyhTXOps3b7aspYbz5s2zrOVdwY47jRs3dj0eP3685fz586c69kSj+7stW7ZY1t+MofzGTGS1a9e2rCWHjuM+1ii9PUO7HgbjLU0M5XeTrnPbbbdZvvHGG1NdNy24IgYAAAAAPuNEDAAAAAB8FreliTpZsE62XKNGDctaOhNKKaLX0qVLLSd7h7o///zT8gMPPOB67uDBgwHX0cvOuXLlis7AksyIESMsZ8uWLQNHkjFCLTkMVkIYyvpa8uh9nZTKIf9ryZIlYW0vUrSk6rfffrOsZUE6AWuoRo4caVm7yH399deWg5Uu5cmTx3K1atVcr3vnnXdafvLJJ8MeV7zTUlKlJfXJqEiRIpb183P48OGwX0vL6zRHko5Ru7il1FEwvXSib+3Cp11KtaQqkvRYrpM7T5gwwbVc3rx5o7L9WHbu3DnLjz32WMBl9JYN/e3qR+lqrGnUqJHl0aNHu57r2LFjquun5faMUNa56qqrLL/11lthbyNcXBEDAAAAAJ9xIgYAAAAAPour0kTtyKedU/RSY/369S1rOaKu6zjuTkFq4sSJlrXESLeRjN3qpk6dalm7onm1b9/e8sCBA6M6pkSWllLaZKPfTy2RcZzQuytGip8liMHohMnaeXDjxo2WdcJXFUpXtJRceumllitVqmRZOwB6/xslIy1FWrhwoWWdfFdLNpNRixYtLL/00kuWV61alRHDMdqxtm/fvgH/vXnz5r6MpVy5cpa1NFF/F6WFfvdz585tWfcPM2fOtKwlknCc5cuXW9aumdpZWj/TyViOGEyzZs2CPte9e3fLx44dC+t1vd1SCxYsaFm7eWuZpHZj1FLpaOGKGAAAAAD4jBMxAAAAAPBZJp+7AaZrY1rWoeUbOllz0aJFLVeoUCHguo7jnoA0FPo+aZck70TFOnlqvNu+fbvl22+/3bJOROp1yy23WNb3YsCAAZb1/UNgf//9t+WcOXNanjZtmuXWrVv7OqZ44kf5sJYjaplkLNBOT4MGDbK8Y8eOgMvr/q1///6u50LpftazZ89wh5iUtExUS2Y+++wzy9ptNtnpMejhhx92PafHcO1WmF7ajVZL77Tct2LFihHbXnrpZ0rHu2/fvoDLFypUyPVYv7tZs2a1HKzrH/7H+/tZv7v6O0m71erk2gjNrl27LOvv/QULFliePn16wHW9v9F1Umb9jmgJYubMmdM+2JQF/GHCFTEAAAAA8BknYgAAAADgs7gqTdRuKX369LGsXXx0QkPXhj1/Z7ilSzr5ppbfXXHFFWG9TjzRzlBalpESLbd5/vnnLTdo0CByA0sClCZGjnZNDNa5T8sMa9asGfS1dLlY6JSI+NK2bVvL5cuXt9yrVy/LOXLk8HVM8UonaNbSRJ0Ytlu3bmG/rnayS6mTG7Bnzx7XY+2g+eabb1ru3Lmzb2NCTKM0EQAAAABiASdiAAAAAOAzTsQAAAAAwGdxdY8YAAAAAMQZ7hEDAAAAgFjAiRgAAAAA+IwTMQAAAADwGSdiAAAAAOAzTsQAAAAAwGdZMnoAQLjOnTtnefr06ZaHDBliuX79+pZffPFFfwYGAPDdr7/+avnWW2+1vHPnTsu7d++2XKBAAV/GBQCp4YoYAAAAAPiMEzEAAAAA8BmliYgLp0+ftrxkyRLLbdu2tVylShXL/fv392dgAIAMNXnyZMs///xzwGWGDRtmefDgwVEfEwCEgitiAAAAAOAzTsQAAAAAwGeciAEAAACAz7hHDDHr999/t3zvvfda1nvEChcubPmTTz6xnD179iiPLrqWLl1qedSoUZZnzpwZcPm77rrLcvfu3V3P1axZM8Kji316T+HXX39tOdj7py2vGzRoELVxAYi8t956K9VlSpUq5cNIACA8XBEDAAAAAJ9xIgYAAAAAPsv0zz//+Lk9XzemgrW0dRzHueyyy1Jdf+XKlZbnzJlj+Y8//nAtN2PGDMsVKlSw/PHHH1suUKBAqttLRr/99pvrca1atSxv27bN8rXXXmt53bp1ljNnzhzF0aWdfsf0c+D15ptvWl62bJnlQ4cOWc6UKVOq28uTJ4/rsZbdjR071vJFF12U6mvFC+/38Msvv7TcunXrVNcvVKiQ5V27dkVuYEgK+h3TsmItnXYcx9m9e3dUtq/HmjJlylh+/vnnLZcvXz4q284oU6dOtXz//fdbPnHihGXdXy5cuNBynTp1ojw6APg/Av6A44oYAAAAAPiMEzEAAAAA8FnSlCaGav369Za1E5OWdGlHtuLFi7vW17IItXPnTsvz58+3XL16dcuJVCqWEn2P3n//fctPPPGEa7m//vrLcr169SxrF8ESJUpEY4jpph0fJ02aZLlPnz5hv1bevHktX3zxxZZPnjxpeceOHZa932ktz1mwYIHl22+/PeyxxKpWrVq5HgfrjhhM1qxZLT/++OOW165d61pOy7vuuOMOy7fddltY28soWqKtn4VQzZs3z/KqVass16hRw3Kw9+Kee+5xPc6fP3/Y249Vn3/+ueUnn3zSsh4r0qJu3bqW9+/fbzlnzpyu5aZPnx5wuXz58lmeO3euZT3uxKsmTZpY1r9Nde7c2bKWfiM8s2fPtqyf7x9//DHoOu3atbP8+uuvW/aWziebo0ePWu7Xr59lva2lbNmyrnVmzZplOXfu3FEcHaKM0kQAAAAAiAWciAEAAACAz5K2NPHs2bOWtfuSThysJV25cuWy/Nhjj1l+5JFHXK+rl/AfffRRy23btrU8ZswYy8OHD7fcq1ev0P+AOPbwww9bHjlypOUsWdzzi+t/F52wOB7opMA60XRa6ATWWgKm3RSbN29uWcukHCexShO1tE5LX7RMznFC6y6pdD8Y7rqO4zjXX3+95SlTplgOpSNrpGzcuNGy7mO0nPDvv/+2fODAgYhtO5T3TztTOo67K+p7770XsbEkA90nOI7jtGjRwrLuF3Rie/0c1K5dO4qj80copYl6rHn11VejPqZ4d+rUKcv169e3vHjxYstp2T9q986nn346jaOLL/obU7v46m9MvaUgJd27d7est2bEMt2/628S7TBbs2ZNywMGDIjYtlN6rWeffTbgv+s+VccYYZQmAgAAAEAs4EQMAAAAAHyWtKWJOkHzzTffbFnfjy5duljWMkOdMNNLuy5WqlQp4DJFixa1/Nlnn1kuV65casOOK1999ZXlF154wbJOiF2xYkXLI0aMcK2vl7bjzZAhQyxrKcY111xj+cEHH3St432cGi1BatasmWWdUNZx4r808ZdffrGs76V2ikupU2Qo0luaqOtfccUVlrUstWTJkmG/bji0o+aff/4ZlW3o57dUqVKW9e/fvn275Q0bNgR9rauvvtryp59+ajlZuseGQrvtvvLKK5a9pZz79u2zrJ9fLXcfOnRoNIboG30vHMd9vNSSWy1x15LlypUrR3F08evYsWOWq1atavn777+3rN/vCy+80LJ2/fOWg2nXY+38++2331qO1a7HaaX73UGDBlnW725a6Hd63Lhxljt16pSu1400/QwEKwGMB1E8L6I0EQAAAABiASdiAAAAAOCzLKkvkhiOHDnieqyTi+plyA4dOljWzmOhGjZsWMDXveSSSyzr5fxEKEc8d+6cZf37tYxMl3nooYcs63tRsGDBaA3Rd9pZUzs+aolGev9eLUdatmxZ0OW0M9Ett9ySrm1mBJ1Yfdq0aQGX8ZYS6PusZR1azpnS+uHS9XWSUy1VjnZpok6IvnDhwoDL6ITU2sk1VFpK5O2C+F979uyx/MYbb1geOHCgazktW9SOj4nQ0S8UOrGrdlXTyci1lFgniffS/xb6OUikTrytW7d2PdZyRNWyZUvLlCP+X1oy6DjuLn5ajligQAHLuh/Lli1bwGW0A7R3OydPnrR8/PjxtAw7JnlLwOvUqWM5pbLscOnxRSd+joXSRO2IGKwcMVgXQm+H52TFFTEAAAAA8BknYgAAAADgs6QpTdSJYB3H3YlNS5dCKUc8ePCgZW83HO3kpqVIo0ePtqwTJcYj70Sw+h7oxI358uWzrOWIXbt2tZxI5Yjq/PPPt1y6dOmIva52uAq1E5OWSebMmTNiY4kmLTsKZTJRfb8dxz3RunaHrFevXsBt6Ppa2vXdd9+5XldL/oKVRukYtTRPJ6CNhscffzxgzkgpTWauZWPBOszGuxMnTrge6/uhZdwrVqywHOwzrl0xdcJ4x3F3K7v00kvTNNZYpCXt27ZtC2mdpk2bRms4CeHNN990PdaJvvUzpt0N9d+VliwePnw46DaLFStm+corrwx9sDFIS4pffPFF13OhlCPmzp3b8t1332159erVlvV99dLfsnp7QkZ1m01PeaGWXHpfJ9ySR70FQ3m7SAcbb//+/YMPNMq4IgYAAAAAPuNEDAAAAAB8xokYAAAAAPgsae4RC5XW9OfIkcPy/v37LTdq1MjyqlWrXOvrfWHz58+3rG2j45HeF3f55ZcHfU7vC9O/v1q1ahEbi7aM1Xbk3nElop49e1oeP358wGW8tdLx2LJea++1dj6YBx980PVY75lRV1xxhWWt57/wwgst632OXvpcKDXlX331leWhQ4dafvLJJ1NdN17pfbYp/bcrU6aM5fz580d1TH7S+x4efvhh13M6FUMwNWrUsKz3QGib8UjedxrLZs2aZdnbKlzpcadVq1ZRHVM82rlzp2XvNBJZsvzvZ6BOJxPsvrAvvvjCsr7X3imClE7hEu9Gjhxp+eWXXw66XNasWS3rcbtbt26WdToQPVZ4fy+dOnXK8tatWy2//fbblnv37p3q2DOK/iYJdmz2trjXx8HWCYV33Vhsmc8VMQAAAADwGSdiAAAAAOCzpClNvOyyy1yP69ata/mzzz6zrOUygwYNsqxlYFpuU716ddfrjhs3znK5cuXSMeKMcebMGcvaNlxLv7wlCDfffLNlLSkLpRxRSya0DMVxHOePP/6wrG3DtUxUSxO7dOliWcvA4p22sp07d65lLYFS3bt3dz3OmzdvdAYWYdoOPtzygWeeeSak5bR1spYmaknypEmTLOt+wnHc7622o7/mmmsCbk8/q0OGDLHcqVMn13IFChRIdeyxbPLkyZb171S1a9d2PR4xYkRUx5RRtOW67k8dx/1d/OuvvwKuv2zZMst79+61rGXw3uOZln5df/31lnPlyhXqsONaIpW+RYPeJnD8+HHXc7rv0WPo77//bnnatGmW+/TpY1lL5ry0NK9Dhw5hjjh2TZw4MaTlVq5cafm6666z/P7771vWfWW2bNksp/S+xotIlRaml7d9fSziihgAAAAA+IwTMQAAAADwWdKUJups5o7jONOnT7esHV02btxouXPnzgFf65JLLrG8fPnySA0xJmjp1dixYy3r3/zRRx+51vGWbwWis8Fr16HRo0db9l6OL1SokOUbb7zRcrFixSzPmzfP8rp161IdRzzSUiXtHKafaS3zitcynUWLFllesWJFqstrufBFF10U0ja0y1Tbtm0t6/d406ZNljNnzuxaX9fRTn86Fm/Z4X9pGe3Zs2dDGm8s03LO9u3bW86UKZNlLU/S77rjOE7BggWjOLqMo58Zb2dT7ZY5Y8aMgOv/9NNPlr///nvL2bNnt/zNN9+41pk9e7ZlLXF67rnnLLdr1y7VscerUL//gezevdv1WG890Kzvn3ZfPf/889O8bb9oV1ivw4cPW9bfQtodUcvg9fudkuHDh1suW7ZsSOvEg5T23bpPC9Z1UkuStcNsqLQz9z333BP2+pGmZYcZWYKo9NaGlG5z0PJJb9dGP3FFDAAAAAB8xokYAAAAAPgsU7DOa1Hi68ZCpd3acubMaTnYJfgKFSpY1rIxx4nPiUm1JKxevXqWr776asvaNSmlbpA7duywrN0on332Wct6aV630aJFC9dr3X///Zbz5Mlj+cCBA5bbtGljWcvLtPNdPJbq7du3z3KtWrUsb9myxbJ2pUyEElnt9haszFTLs/Szlp7SpEjTMj3tJqh0Uk/HcZxRo0ZFdUyRouWIWpL85ZdfWtb9pv79d999d5RHl7z0u6DHJy331tK6YJPBxxot32zZsmXQ5S6//HLLety59NJLLevtCPq63knHtVtgMNqJTSfgjlXafbN58+au57799lvLpUqVsqz7Y+2grJ81pcdyx3HvE/R3VbzT30j6WfPS2zz0GP76669bXrJkSUjbPO+8/10zmTJlimX9/YP/0RJJ/e3ppe+/T6WJAU8quCIGAAAAAD7jRAwAAAAAfJY0XRO9jh07ZvmJJ54IuIx2+jl48KDl7777zrK389UDDzwQqSFGlZYjaqlCq1atLL/77ruWtSTMS7sr3XnnnZa1BPHee++13LdvX8vafSol2i3w4YcftqwTPWtXtngsR1QvvPCC5c2bN1vWsi99HxOBliMGKwt+5513LMdSOaLSsQf7O7zdsuKlNFG/b1p6pCXuVapUsdywYUN/BpbkSpQoYfm9996zrJOO63ena9eulqtWrRrl0aWdjt/b+U9L1Ldt22ZZJ23Xrp3BJtBO77i0LLxixYoR20YkaQdiPV6HKpR92oIFC1yPE6kcUenE9CmVJmoJoua00E68lCNGjnZUpGsiAAAAACQRTsQAAAAAwGdJW5r48ssvW37jjTcs68TFq1atsqxd+LQ0bubMma7XjZfSxKFDh1rWCR2141uwckSd8NZx3N24dNLaYcOGWX788cdTHZN3osRPP/3Ucp8+fSxrl6f69etbjrdyxF9++cX1uFGjRpa1HFHLvn788UfLpUuXjt7gYlT58uUzegipuu+++yzr5Of79+/PgNGk36xZsyzrBKLBSpR69uxpWbudwh+nT5+2HOy/0fr16y3HcmmiTpZcp04d13MffPBBwHW0C7LmSNLX1eNnItEJmZUej0aOHGm5aNGiUR9TLNDur/ob0XHc+3v9HqaXt6M0UqZdTeMBV8QAAAAAwGeciAEAAACAz5KmNFE76jmO4/Tr18+yliP+9ttvAdfXkhydIG7Dhg2u5bQzU968edM22CjRLmfa7WfQoEGWtdRPaSlLx44dXc+dPHnS8po1ayxXrlw51TGtXbvWsnYKdBzHmTNnjmWdsHPw4MGWn3zyyVS3Eau8Za06WbOWFHXo0MFy8eLFoz8wpIuW5+qEukon3Y01Wv7qOO7PX7C/R/ebhQsXjs7AEJK5c+emuoxOxh0vpk6d6nqsx1otY/dDyZIlLVerVs3XbUfT9u3bLT/99NMBl9HbF+LlVoxI0v/2+hvFcdy3TehnUn+/6Hsc7JaNm266yfVYJ9dGYNoBUXNKdOLnjMQVMQAAAADwGSdiAAAAAOAzTsQAAAAAwGcJfY+Y3gf2yvqdsRgAAAj3SURBVCuvuJ7Te75CmfU8S5b/vVW5cuWyfODAAddyZ86cCXucftF7kM6dO2c5d+7clrU1rZo9e7blb775xvXc4sWLLV977bWW9+7da3nGjBmWp0yZYvmrr76y7G1fX716dcvTpk2zrPf0xQN9T8ePH29Z7zX0KlSokGW9D07bOSejhx56yPL8+fMtX3DBBRkxHHPs2DHLes/lkSNHAi4fyn08GcXbtvr48eMBl6tYsaLlaLWu1jbhH3/8sWW9J00/E8nixIkTrsd6P4/eS5UvXz7Leg9uiRIloji66MicObPrcbDpVSLpiiuusNy1a1fLXbp0ifq2/aJt1p955hnL+lumYMGClseNG2c5a9asUR5dfNHfP5pVrVq1Un2dTp06uR7nz58/fQNLAin9nlL9+/eP8kjCxxUxAAAAAPAZJ2IAAAAA4LOEK00cMmSI5eeee86y9zKxXl7PkSNHqq+rlzN37NhhuWfPnq7lChQoEPpgfdawYUPLF110kWUt7dGyQS03mjVrVtDXffHFFy0vXLgw1XHotrV9a7NmzVzLVa1aNdXXijedO3cOabk+ffpYvvLKK9O1zc2bN1vWlvlahhILtJR44MCBAZdZtmyZ5aeeesryHXfc4VquTp06lkP5fodKS+UWLVpk+c4777SsUw9ky5bNsrZ61hbIsWDTpk2WtYzY6+qrr7a8YMGCdG1Ty7q13Fn3QdoOWqfQ0PLPRKYlr/rfxfv90JbY+vmbMGGCZf2M4n+0tbjjOM5HH31kWY/niVoepr9n3nvvvYDL6HGdcsTwbd261fLKlSsDLqPHhDZt2kR7SAkn1Jb1t956a1THkRZcEQMAAAAAn3EiBgAAAAA+S4jSxIMHD1p+6aWXLGt3Pe0k5TihlSuNHTvWsna7K126tOUnnngivMFmoMKFC1vWsrT333/fsl5C146GKdGyBe1e16tXL8stWrSwrCWP2o0pkRw6dMiyllwG60rpOO5Oibpcy5YtLWtpYTDebWipktJyW11HS1jnzZuX6vYiRUsNtRPnihUrAi4/ZswYy6NHj3Y9px03S5UqFXD9KlWqWK5Ro4blffv2WZ40aZJrHS0B03Hpe6xZ/9u99tprAccRC/R7r59dLy2vLFKkSMBlgnVCHTFihGu5H374wbKWICotg9JS80Qr3dH3TMvmtfxTO0V66TFJvwv16tWL1BATVv369V2PvaWKiebw4cOux02bNg24nO4Tg3UARHCnTp2yfO+991o+efJkwOX1FpGM7gKcyChNBAAAAABwIgYAAAAAfkuI0kQtWdGyojfffNNyuXLlQnqtAQMGWB46dKjlm266yfLkyZMtByvPiXV6GVzz7t27LWuXMm9XumC05POqq66yrN3WksEjjzxiWTv9BSsTdBz3Z1e7cWrZYErrBxPKOg0aNLCsJXt+0s+OllfpRI0pde9UWjYYrLRRv8fpfY+1O6J2btWJ42OZt1wpGC2b++yzzwIuo+V0OllsSooXL25Zu4f17t3b8u233x7Sa/lFS4/OnTtnWTtAessJdV/wwQcfWNby+mDvmU4u3K1bN9dz7dq1sxzLnXsjSY9JS5YssRxsEnWdSFe70l5zzTVRGF1s0c+UTv7tOO6uujpJdixPOh8P9Lu/Zs2agMvocUNvCUBoQpkcOxZLEb24IgYAAAAAPuNEDAAAAAB8limlLm5REJWNVapUyXLZsmUtT5s2zbJOxOo4jjN79mzLOlGmlj516NDB8rBhwyzrhMRAINpdTj9TKXWkCyZv3ryWL774YsvacbJv376WQ+2aqGKt7EtpqdH8+fMtv/zyy5bXrVvnWifc8sJQSxO1O121atUsP/nkk5bLly8f1rZjwZ9//mlZy1QdJ7TuqaG8f/rZdRx3ubhOJKvdXWOBlhpqh1ntqqulhWmROXNmyxUqVLCs3Wa1nO688/h/qAiddr9t3Lix6zn97OlnTG/TQPj0N8Bbb70VcBktXddSeYQmlOO8li07ToaXKgYcMHtzAAAAAPAZJ2IAAAAA4LOEK00sU6aM5VatWlmeMmWKax3t+KWT5+kksY0aNbKcJUtCNJhEBtBOadrlR0uQHMdxmjdvblk7geqldZ1kM9kdO3bM8tGjR13PacmClpP9+uuvAV/rscceC7iul3YVy5MnT+iDjSMTJ050PdYOoMHocURLmnRC5rvvvtu1Tv78+dM4wsjzfi6GDx9uWbvvaqdELUE9cOCAZe08q6XyjuM4+fLls6yTvGq5TDyWtiL27N+/33LdunUtazdkx3GcG264wfLKlSujP7AE9tNPP1muXLmy5b/++ivg8lr2zKTZofn8888tB+uaqPtTb2liBqM0EQAAAABiASdiAAAAAOCzhKi3q127tuVXXnnFsk7k2rJlS9c62mVNyxmBSNNywrNnz4a0Dh2rUqclxZq9evTo4cdwEkb79u1TfJyIevXq5Xq8d+9eyzpxcL9+/SxXrFjR8smTJy1v377dsrc0USdwBaJJu/BpOaJ3X6kT0CN9duzYYTlYOeKVV15pmTLk8GlpYjA1a9aM/kAiiCtiAAAAAOAzTsQAAAAAwGeciAEAAACAzxKifT0AAEAy05b1eg/jnj17LLdp08a1znvvvRf9gSWJbt26WdapkNT48eMtd+zYMepjSjR6//yzzz6b6vLayt5xMrydPe3rAQAAACAWcCIGAAAAAD5LiPb1AAAAyaxAgQKWd+3alYEjSU6jR48OmIGUcEUMAAAAAHzGiRgAAAAA+IyuiQAAAAAQPXRNBAAAAIBYwIkYAAAAAPjM766JAS/LAQAAAEAy4YoYAAAAAPiMEzEAAAAA8BknYgAAAADgM07EAAAAAMBnnIgBAAAAgM84EQMAAAAAn3EiBgAAAAA+40QMAAAAAHzGiRgAAAAA+IwTMQAAAADwGSdiAAAAAOAzTsQAAAAAwGeciAEAAACAzzgRAwAAAACfcSIGAAAAAD7jRAwAAAAAfMaJGAAAAAD4jBMxAAAAAPAZJ2IAAAAA4DNOxAAAAADAZ5yIAQAAAIDPOBEDAAAAAJ9xIgYAAAAAPvt/A9ZB2RHFfhMAAAAASUVORK5CYII=\n"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"datagen = ImageDataGenerator(\n        rotation_range=10,  \n        zoom_range = 0.10,  \n        width_shift_range=0.1, \n        height_shift_range=0.1)","execution_count":69,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# PREVIEW AUGMENTED IMAGES\nX_train3 = X_train[9,].reshape((1,28,28,1))\nY_train3 = Y_train[9,].reshape((1,10))\nplt.figure(figsize=(15,4.5))\nfor i in range(10):  \n    plt.subplot(3, 10, i+1)\n    X_train2, Y_train2 = datagen.flow(X_train3,Y_train3).next()\n    plt.imshow(X_train2[0].reshape((28,28)),cmap=plt.cm.binary)\n    plt.axis('off')\n    if i==9: X_train3 = X_train[11,].reshape((1,28,28,1))\n    if i==19: X_train3 = X_train[18,].reshape((1,28,28,1))\nplt.subplots_adjust(wspace=-0.1, hspace=-0.1)\nplt.show()","execution_count":70,"outputs":[{"output_type":"display_data","data":{"text/plain":"<Figure size 1080x324 with 10 Axes>","image/png":"iVBORw0KGgoAAAANSUhEUgAAA2IAAAB5CAYAAABbVpJXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xm0XeP9x/FHq4YaIjGEIIkpQYyJoYaQBNFItKYUNbPU0tZYi65qFy1LV5tmobTVAUGMRagQUyQSYow5FIkYgsYYoVFK+/vjt/L13k/vvrmRm33P8H799ZHcc3LvuXs4x/7s77PEf//73yRJkiRJqs5XOvobkCRJkqRm4wcxSZIkSaqYH8QkSZIkqWJ+EJMkSZKkivlBTJIkSZIq5gcxSZIkSaqYH8QkSZIkqWJ+EJMkSZKkii1Z8b/n6tGSJEmSmskSLf2hV8QkSZIkqWJ+EJMkSZKkilVdTZSazn//+0Ujd+7cuZFfe+21wtd9+umnkZdaaqnIH374YeRll1028mqrrRZ55ZVXjrz00ksv4ncsSapnn3/+eeQXX3wx8rRp0wpf99lnn0XmOenRRx+NvMIKK0Tu27dv5EGDBkXu0aNH4Xl5HlpiiRYbWZKSV8QkSZIkqXJ+EJMkSZKkii3B2lQFnJqopvDvf/878vTp0yOPHz8+8tSpUwuPYW3xK1/54v+R3H333ZHnzJkTecstt4x8xBFHRD7kkEMKz/v1r389MiuPkhaM58h//vOfkV9//fXIb731VmRWsrgfp5TSu+++G3nFFVeM3LVr18irrrpq5OWWW670uaQca+y33HJL5GuvvTbyK6+8UngMK4w8v8yaNSvyMsssE3n11VePvNZaa0UePHhw4XmPOuqoyN26dWvbD9ABuH/ztXjzzTcj81z9wQcfRF5++eULz8W/+9rXvhZ5gw02iLz++utH5i0F1jebglMTJUmSJKkW+EFMkiRJkipmNVFaDF566aXIl112WeQrrrgi8ssvv1x4DOsfrCGxLvGf//wnMqcscppir169Cs975plnRt5rr73a8u1LTe2TTz6JzIlzEydOjMypcrNnz47MShKrjCmldM8997T4722++eaR995778iHHXZYZE5J5bEiJWuL+n8333xz5JEjR0Z+8MEHI+fv+Vgv5LbLSYms2vO8xSpknz59Cs/7rW99K/Jxxx0XeY011mj9h6gY65hjx46NfNVVV0XmNMkll/xi2Hi+H86cOTMyjwk9e/aMzP376KOPbvFr8ue1ttgwrCZKkiRJUi3wg5gkSZIkVcwPYpIkSZJUsYa7R6xsNfknn3yy8HW8v4ZjiO+6667I7EtzVPgee+zR4p+nVOz2skus5vL8889HvuaaayKPGzcuMrvpKRXH2vKej7JOOnv/7Kaz259SSkOGDInMTvquu+66gJ9Cak5PP/10ZI7+vummmyI/99xzkbmP8n5N3tOZ++yzzyL/61//isxzMkfcn3jiiZF/+MMfFp6LI+/VvB544IHI999/f+TbbrstMu9HTCmlnXbaKTKXXuCyJ3y/NGXKlMhl91GllFL37t0jcx/aYostFvBTVOv666+PfMkll0R+5JFHInPZic6dO0f+6le/Wngu7ru8r4730nEJmbXXXjvy8ccfH5nL0aRUXMZCdc17xCRJkiSpFvhBTJIkSZIq1hDduY8++ijyrbfeGvnqq6+OzHHiKZVfNmZNkfWwhx9+ODKrZltttVXheU866aTI22+/fdt+ADWc3r17Rz755JMjb7jhhpHzEbWDBg2KzHoh6w4ch8361K9//evIHMGbUrGi0q9fv8hWE6WWsaK1+uqrR2Z9mNV1jpZfc801I7PSlT+G1UTWvXiumjt3bmTWy/Jx1t///vdb/F6a0fvvvx+Zx8tOnTpFZn20kW4h2G677SJvsskmkfk+Zb311is8hvW4MvPmzYs8a9asyHwfxXptSim99957kT/++OMF/hsdZbfddovM8+6ll14amRVjHgPyn5k/J/dvnoN5ywwzz9vdunUrPC9H3jfLKHu+tjwOsjK7/PLLV/o9LS5eEZMkSZKkivlBTJIkSZIq1hDX5CdNmhT5j3/8Y+T77rsvMi8Tp5RSjx49IvNS59Zbbx2ZE3GmT58emSvL85J9SsU6I2sRvPxdLz755JPIvITOaUIpFS/Hs+bJS+0vvPBC5C5dukRmlWLYsGGF5+VEStZK+Ph6wLrD/vvvv9CPZxWB2yq3tVdffTVyvq1/8MEHpX9XD/g9c3oVf2ZuqykV9z1ur//4xz8icxLVBhtsEHmbbbaJzBpKSsWJV41UaeIxjRM/UypWjLgf9unTJ/Iqq6wSma8LK371pFevXpHXWmutyJw4N3v27MicBMfqV2s1Im7Xzz77bOTzzz8/Mqe4TZs2LTIrYSmldOSRR5b+O/WA9WtO35s6dWpk7vsppfTQQw9F5uvEbZHnF55PNtpoo8iDBw8uPO9KK620UN97reJ5Z8CAAYv0XLz945lnnon85ptvlj6GdT5Os641PKaVTZAsO+7xfJDj/s19d8SIEZGvvPLKyI8//njk0aNHF55rn332Kf8B6hhfo/z2Id4OxG2O7zd5Dn/iiScis549cODAwvPyd8zPAXw/VTWviEmSJElSxfwgJkmSJEkVa4huDSensALIS+OsIqSU0u677x65bAoLKzasjlx22WWRWYtMKaUJEyZE5qSiWq4msjbAqsF1110XmZfQ8zoCa2GszLDOwOrJW2+9Ffnvf/97ZE4pSql42fiYY46JfMABB0ReZ511/ufnaWScJMQqA6ufOV5yr+UpQ9xGOCWJPyenorKK0Fo1kVU7/jlrDVykkzWUjTfeuPC8hx56aOS8StvRWJNmzeOpp56KzDodX5c//elPkfl7SKk4NZALB3O74uS1gw8+OHL//v0j55PA6gXPD/x5FhXPT9zeOQ2VWMlhXTSl4hTAesFaEvflP//5z5E5TXLOnDmFx7OqyNeS9XhOmrz55psjs0rKhblTSumoo46KzFpqM+Nryt9DfqwoU8vVROJ+tKh1Tt7awprjG2+80eLXc9oncyObMWNG5FGjRhX+jtPJWZ0nnsN5bidOrEwppT333DMyF87mRGtOt62CV8QkSZIkqWJ+EJMkSZKkijVENZFTUDgZidPP8gob/7tsshWrE6z98LIxKxH517GaV8tYO/jrX/8amZeKOTUxX8Rw3XXXjcxaFzPrTXw8p1FyCl5KxQl3/F5Y3bniiisiswrQqLhNtTYpkbh981J+rWG1YOLEiZF/97vftfjnrMF27dq18FxcLLt79+6RuU3yteSkNi5YmtdIWIfiJKzTTjstVS2varEmzcXsWfvi9//OO+9EZv1uxRVXLDwv96uZM2dG5uvH6sjrr78eefz48ZE50Tal5lmYtAynmT766KORy6qJrILmEzs5Ia9ecJthNZaVeL5GrA6lVKzJclIlzzXcr5988snIrMTzVoOUUurbt2/kZqgmsl7I6vLf/va3yJxeye2ztUlzrIvffffdkXv37h2Zv6tGw/eCnADI4zHxONssi7LzVok11lij8Hd8Dfg+m1NRWXll/ZPvMVl/TCmlc889N/LkyZMjc1rt9ttv37YfoJ14RUySJEmSKuYHMUmSJEmqWENUE4kVjV122WWRnovVJ1bo8gVPiZdQ80lutYqXh7lwYL9+/SJzgs0mm2xSePzQoUMjswZWVj3iZWNesh85cmTh61iNKKtGslbZKAtxplScLMn6Jetw3CZbW1yY2ySraXztaqHaxIoMf8dcRJd/zm0tX7SRNQdW7ZhZ52Rt6aqrrorMSk1KxSlu3L+rqiZych6/l5SK1QpWLngc4wLVPXv2jMzpkPmEWe5XrGWzCjlu3LjInHL59ttvR2YdrKV/pxlwG2ftjhXYsoo1a2B5fZRV3HrBn58LqnOB5TXXXDPyYYcdVng8K26cdsfjHSeGnnDCCZFZi+Q2mlKxftsMuE3ydeHCw6wsltXg83MIjzu8tYDb+tFHHx2ZU6ZbO5/VC1bHeexjZZH4fqlZqoncv/NF6bt06RKZtx7wfT23Jd7K8sorr0QeM2ZM4Xk5eZlf99vf/jYyK8l8n7G4eEVMkiRJkirmBzFJkiRJqlj9X/9djHhpnZWu1qbz8TJ/vVQTWQPgIspcpJUTKDmdJqWFn37G15UT1jiZKcfXldUTTm6r92oify5WRC688MLI+UKu83F6UF5T4ut9++23R2a1Z+edd47cUa8jqy37779/5CFDhkTmxCnWD7mtptS2qhanNLI6kk/vJNZyWO2sCmstrKamVPx9LrXUUpF79eoVefjw4ZF32GGHyK3tw9x+uLA9Kx48hvA1+vTTTyM3SzWRx/0JEyYU/o51a25nb775ZuTWJtHNl0/r5X6x2WabLdRzdRRW3/v06ROZx77Wpqq1pcLJ/YXbLuVVO05NbQbc9zk9ldPp+DXM3NY4vTql4lRW1r2vv/76yDyvcypmLZ/Lub3ki4Gzos33Nvw6LhbMYwW3Z04EbRbc11NK6eCDD17gY1ghZGbVfu7cuYXHcEItbzO67777IvP8WYXaPUpLkiRJUoPyg5gkSZIkVcwPYpIkSZJUMe8RS8WR2DfddFPkBx54IPIzzzwTubV7xHivDnv7EydOjNy/f/82PVdHY9e+Pccj896a1sYIl+FrzHvE6h2797znZvXVV2/x63mfCEeRcyRsSsV7Hh555JHIP/vZzyL/9Kc/jcwlDGphjDBHdef3ISwKjmLn/UsvvPBC6WP4mnM75v1myy23XHt9i/9j5ZVXjnz88ccX/o5deN4/1rlz58jcNtp6byePA/w5eQ9E2Ujr1u4Ra1S87+O2224r/N3VV1/d4tcR75vhtsTjHkc1p1Qcvcwx0Fz6oRaWqCjD8wvvFfoy+PrxdeK2S3xdUyoeL3l/JO/taSQ8DqyzzjqRzznnnMh87XhO6NatW2TeX55S8f0T743ksYnjw3msqGXcJu69997C3/3hD3+IzOUZuE3yWFl2b+3TTz9deF4uw8Dlg2r5HtCOxPeSd911V+HvuM2VKXvPtbj4W5QkSZKkivlBTJIkSZIq1vG9oxrAy+6//OUvI8+cOTMyK0m8hJzXPXgJmjWw0047LfKBBx4YmWO68/G89Yg/Py/Nv/TSS5GnTZsWmXUljs5Nqbyq8PHHH0e+4447IvN3wcrEwo7X7yj8Plk1PP300yNzzDW3W44o5yjolFJ67LHHInM7ZP1h9OjRkffdd9+F/dbrBrdPVpJZN+Ho5Dlz5hQez+MAHz9jxozIHCOfL/WwOLHC2bdv33Z7Xm6X3Ke5ZAC3S74urN9Nnjy58Lw77rhj5O222y5yLdRhFwV/53ldi7U7bls8p3Df577MpStYe0+pWHlk9ebUU0+NvPvuu7fp+69HZa/NpEmTInfq1CkyzyHc91NK6f7774/MqhmX0eD5ZXFWkavGc+jAgQMX6rH50ip8HcuWBClblqaWtXY7CSu2PA4ys/bGn/mtt96KfPfdd5c+7zHHHBN5q622avFrmlHZ8j98/5NS8XdBHblkglfEJEmSJKlifhCTJEmSpIrVdweknXAaEus9XN2cl6NZcTj00EMLz8VaDScljhkzJjLrOl27do3MymK9Yp2QU5POOuusyC+++GJkTqzKLxnzUjtf/3nz5kU+77zzIj/00EOR99prr8gDBgwoPO9qq63W+g9RA7p06RJ52LBhC/XY/HXkFLB8lfn5uN3W8iTPtmLd6MEHH4x8ySWXRJ4yZUpkvmasOOS1Vv43q2Vnn3125OHDh0fu169fZNaZaq1Gwp/5+eefL/wdK4UTJkyIzDoitxlO8uLz8vVOKaWTTz45MquU3/72tyNvu+22kTkxspYttdRSkbktpJTSNttsE5lTO1kzZnWJlWNOqRw1alTheVkzZhXnlltuidxo1USeaziV71e/+lVkTk3kPr788stHzquwrNGdcsopkceNGxf5uOOOi7zHHnss9PfeiFgHS6l4Pua2Tqwm5tMraxXfL3ICdo7bEWuHgwcPjsz9m+epESNGFJ7r9ttvj8zzCKvOtXZOWVz4/vnOO++MzOPe7NmzI7NOn+O5qnv37u31LS40r4hJkiRJUsX8ICZJkiRJFbOamIpTj37+859H5uV01k1Ya9hiiy0Kz8XLoKyLsCrF6UKtXTatd5wExp+ZU8XKKp8plU+UZN3kxhtvjMz6J2sBeeWhESqgrXnuuecK/80aU9mi2dy+GwH3tyeffDLyZZdd1uLXcztk9YQL4qZUrC7z+DB27NjInJa69957Rz7iiCMib7rppq3/ABXjFDnuRymldPHFF0dmxYbVMFaM+LNxgdh8P7zvvvsic3oqp6r+6Ec/iszXspaxvprXXRa2/sLfy+OPPx65tcWxuf3mk2gbSdkEVL5m3F9ZST/ooIMi59MBOXWS+ywn2XG7brZqIs/rfK1Yn8v/jr8r1u45obcjp9YtDO7f+fkh/++WlE3g5vvFfLH3d955p8Wva8YFnbl/X3DBBZF53uXvKJ++XTZF+8MPP4w8derUyKzNL64J3M33W5QkSZKkDuYHMUmSJEmqmNXEVKwlDRo0aKEey0vLKRUXk7znnnsW+Jh8Mcl6x4rb1ltvHfmcc86JzKmHyy67bOR88VtOnONCk6xAjB8/PjLrKazgvfvuu23/AeoIKw4fffRR5HzhXE6zKls0s9EmLnFb4qS6ww8/PDIrrqwns3p80kknFZ6X9RlW637xi19E5tRBThncYYcdItdaNZFY50qpuO9x/2btjXUjTq1jHYwLvKZUnObJ2h333bKFYJsF911OBuRE2hynAPL42mi4LXISHfd9/vyst2+88caR82mcrH3yWMApqayKNQNOSL3qqqsiczJ0XpfllE8eN7kgMacbN3KNllgn5P59zTXXRM4Xxyaeqzt37ty+31wd4M/P8w5fC9YX8+2qZ8+ekVmf5TnoN7/5TeQTTzwxMmuKrIAvKq+ISZIkSVLF/CAmSZIkSRWzmriI8moh60ozZsxo8TGcIJRPdKl3nCrDhUn32WefRXpeVpS4sGa++Ox8fF3zBY7rDbcXTq1jhYsVTb4+KRUXi6U+ffpEPuGEExb5+6wlrBuzmsj8ZXBBbG57rIYSJwWyLlFrWOHigsoppbTeeutFZk2xR48ekddee+0Wn5evyxtvvFH4u7L6DSfJsjrSaLhfszLMfZyLYLP2nh/TeNxlNa+R6178mdddd90Wc1vk9VdOQOX+Ttyu+R6gPetKXwaP9Zyux9pW/vrwe2b9cubMmZGvu+66yKNHj47MenGO1dH99tsvMidRcnHixTWRrpZxEuekSZMi5+8r+TvidOlmfM14fDv99NMjc1r0a6+9FjnfJ3fdddfIL730UuQzzjgjMm9/4bmN75msJkqSJElSHfODmCRJkiRVzGpiG7E6wsvGd955Z+HrOM2K0xF5mX6zzTaLPGTIkHb9PhsVJ9o8++yzkcuqI/x9lU0KbG/8d1jDYsWjd+/ekVddddXC48tqbJzQxVrI73//+8icAJgv8sjva8MNN4x80UUXReYld5V77LHHInPf51Qx4rGilquJ3GbymmFZ7bAtODktn6qWVxXnYzWRVSvu6/lkx3rBfZELjbOCycVEH3744Ra/Jq8kcTola2CHHnroIn7HjY/baEopzZ49OzJ/X6zvciIwF9/tiGoi//0bbrghMuuErBT379+/8HhOoeM+yvMsF8tldZaVx65duxael+c33prAamQjL0jM8zmrrLfddlvku+66KzLrxvnrssoqq0Tm1L9mxGMfpw+3dRIx9+kXX3wxMs8vrOiy8sgpqu2pcfcCSZIkSapRfhCTJEmSpIr5QUySJEmSKuY9Yq1gx/fWW2+NzF4vO/wpFe8HYs938803j/zjH/848gYbbNA+32ydYl83H+XP0a4333xz5LzTPx/76kOHDo181FFHLfL3WYYjqF9++eXI5557bmTe27HjjjtGzkep87l4z9Edd9wR+ZZbbon8/vvvt/g95ffE8XXhfWH893kPYzMqGyXO/TmllO69997IvC+Pj1luueUic7/v169f+3yzNW7evHmR2cHnSOGU/ndE83xrrbVW5COPPDIy79Fpb/yeUyreK8NlSLbYYovIZfe68H4Q3u+WUvG+m4svvjgy9/H8MfPx3gi+Riml9IMf/CDyYYcdFpn3jjU73rPNbfHKK68sfB2Pvbzna8CAAZH33nvvyB29RAC3Ny5dwvMn7yPjEjsppbTkkl+8DXz33Xcjf/jhhwv8t/n+5dhjjy38HbdRHvu4tEhH4bGb97/x/n3eG8v9m9tRfi8X758777zzIvMed97zzW2Nvwcu/ZNScUkR7utaeDy283dUtlTKSiuttNi/J6+ISZIkSVLF/CAmSZIkSRVriGoiLxXz0jD/PB8VzpoLx4a+9957kadMmRL5/PPPj3z//fdHZqUpt8Yaa0T+yU9+Enn77beP3BHjbjsafy+s54waNarwdazxsH7BGhh/rwcffHBkrri+8sorL9o33ArWV6dPnx6Zo4O5TXLb6dGjR+G5WI+aNWtWZI7wLrPbbrtFZh0spZQOOeSQyN/4xjciW0f8Yt99+umnI3Os+Pjx4wuPKauYcKwtfxes67R1vG694OvH/ZOv2eWXXx75mWeeKTy+rGo3YsSIyKwKtfexkvvbjTfeWPi7v/zlL5FZFdxoo40is2bM3//bb78dOR/Rf88990Tm68FjIs9N/HNWXk899dTC8x5wwAGRebzLx9w3G75+Z599duSRI0dG5rab43If3/ve9yJzuY+OHsHO4/hWW20VmbdNsDbPc0tu6aWXbvG5eG7r0qVLZFY099prr8JzcZ9g7a4qZbcNpJTSWWedFZmVNI72HzZsWGSOj+dxIz+mXXvttZF5rueSCMRtp3v37pEPPPDAwtedcsopkXmrgb5QdntBXvXmsjOPPvpoZFbluR9UUe/2ipgkSZIkVcwPYpIkSZJUsbqtJvLS4wsvvBD56quvjsypP/mEOl5u5KX6adOmRebUIV6C5iXQTTbZpPC8rKXwUv3uu+8emZc9aw2nK/GSPadM5Su78795qZ0TmPhcnFJ1ww03tPhvpFRe+1xhhRUisxpRVR2RWP3h97XxxhtH5qVxVl+Z8+di3YR/XvaanHTSSZHz6XKcbFXL215bsGpErVVfOI2TNSTu65yexP1+7ty5pc/L39E3v/nNyPxdbLvttpEboYbM7e/555+PzOr1mDFj2vRcrNqxwjlo0KDIi3NSIivpeXWJVVXup5y49eCDD0bmJDhWVufMmbPQ3xe3JR4Td9ppp8iDBw8uPKZR64g8H+V4LOPvkpPorrnmmsicGMvjwDLLLFN4Xu7XO++8c2S+5rW0L7MCyG2ElXZO282n6vbu3Tsy389sueWWkbl/sBq33XbbRa5iutzCYNWM1cqUitMlOUGRdc7bb789Mvcp1gzzSc98z8lzR6dOnSJzyiaPG9/5znci59VjPr7e9++y6vZ6661X+Dpu12W4LU+dOjXyq6++2uKfp1SsjPI4wPcQnI47cODABX4fi8orYpIkSZJUMT+ISZIkSVLF6raa+PHHH0e+6aabInO6IS8Tc4JhSsUpOKyS8LJpWQ2KDj/88MJ/s9bAS5qLs2KzqHh5nZdtWeXgJL58AiXrDKwusfrFGhOnCLJSkuNiuJyq1rVr18hl08KqwmoBpyDuueeekVlRYMUjX9iS/83KB197Vp2++93vRuYEt7x+WI/TEfmaTZgwITJrBqwPcIpZSsVtmq8fK8Zjx46NzCoat8k111yz8LysJPGYss8++0RmHbGWX3tO/OS2lx/3WM3j63fBBRdE5oLrxIVJ8/oojxtc8HXFFVdc4PfeHlgR4nTG/PvhArg87+SLfc/Hqlu+MCsXYeexkvssq61cVJbT6ljrTKn+60rcFh977LHI3K5Y906peA7n+WXSpEmR+bsrm1w3fPjwwn+z7sRJvB29cHMZ/u65vRxxxBGReS7Npzyyur7OOutE5jbJmh//vY6YhthW/Dnz91+cfsp9klW1hx56qMXnKlvcuTV8/XmuYv2O1cRaq3kuKh4rzz333Mh8773vvvsWHsPbeXisfP311yOzHs5J1by9gNtujudnvofiMXiXXXYpfXx78YqYJEmSJFXMD2KSJEmSVLHava68ALxUyQmIrAuxRsJJSvl/ly0syqlDvLzJaUKsoKVUrDLl9ZFaxclUvITMikdZrSOl4uVh1jr4vG25hJ/XuAYMGBD5xBNPjMwKIicVdjRWV7ioL7dPLhac1+k4oYuP4eV7TmDk17DiUK81JdZCRo8eHZnTMFnNZAWMC6+mVHwNWGVg9bi1xdjn4xSylFLaddddI3MBTlZPaqGOyO2E+zF/ftaQeKzKJwhOnjw5MqtenCBY9lqypplXPFgH5OKxVWH1iq9FSsVprKw1sQLH/Z1VItYR8wmzrH7zWMlF1znhl1XYjl44eHHiMW6PPfZo8c/z8ynP1dzeOa23rJbE15WVsJSKtfCOqLsvirwKOx8ri19GLU2JbCvWJvOKOWtvvFVixowZkVlR577HKmeO7yt5CwWPJ/vtt19kbl/59M56x33vqaeeisyp2ay9T5w4sfD4/Jw+H9/X831pWVU8r6Xy++KE3jPOOCMy3+NXMWm6cY/skiRJklSj/CAmSZIkSRWr22oiawqssHHhYE5cYiUnpeLkP067YxVp3rx5kVmz47/HelJK9Xl5mZde+/btG5mX0LlQdr4QMadTlj0v6yacUsTpeHkN7KCDDorM17lWKzqsJzFvuummkTmFK8d6F6t1rEDxa2r1dfiyWNXi68daCevCrI488cQTpc/L12n99dePzG2P+zcrEXn1eOjQoZH5e6mF6WF8PThJduTIkZFZreOxihW6fBFd1m24r5dti5xmx+MJp3ymVJyO2BHVJ+5j+VTdYcOGRe7Vq1dkVpd4DuEUNr6WeU2VvyP++zxWNtp+3RZ8XXisZxWZ+35KxW2ZWMPjsWPWrFmReW5jrTil+qsjasHyWuuQIUMi8/aGBx54IDIXGGbmLQFcDDql4jbJ7ZDb4SH4AAADTUlEQVTvf7hN1uttBG3B4xjP51xA/JFHHonc2u1DPD+0NgVxPh6b+X49peItEMcee2xk1uOrPp833xFfkiRJkjqYH8QkSZIkqWJLtGVyWDtaLP8Ya4esH3Ah1+nTpxceU7aYKKcOsSrFzLpJo11a5sQpLmjIisill15aeAwvtXMRa+Zu3bpF5mvJaYp5PakeJzXpy+OxiLUE1uyuv/76yM8++2zkfHomayGslu2www6RWYfiNsmvzxf6reXaEquWXNzyzDPPjMwJkm3F/ZDTpLgINCdZcb/nNEAuwJ1S4x079eWxbjR+/PjIF154YWQu9JxSsU7av3//yKyBcRFoHiNYK/Y809x43uFtF3xvwmojtx2eN/LH8H1iM9aNiRN2x4wZE/nyyy+PnJ+bWINndZvncB4T+HpzKianIaZUrMRz2nQ+XXExafGk19xbhyRJkiR1AD+ISZIkSVLF/CAmSZIkSRVriHvE1H7YeeY9NK2ND+UY7GbvQqt9cMQs7/Xkdjhu3LjI+fhx3hvC0cO8T4THvrJR7PW0PXO/5DIeN9xwQ+SLLrooMu/77NSpU2SO708ppZ133jny4MGDI/N1ZT+f946x2y+1Be/95H2EkydPLnwd7zvm/Yc8FnBblNQxOHuA937deOONkd95553CY7j0Ce/55rl6ypQpkXl/Hs9T+fIUHXxvsveISZIkSVIt8IOYJEmSJFXMaqIkNRge1++9997IHL08ceLEyKwXsxKSUkobbbRR5M6dO0eup9qmJKl+fP7555FZJ2RmHZ815BquJFtNlCRJkqRa4AcxSZIkSaqY1URJamCffPJJZKcYSpLUIawmSpIkSVIt8IOYJEmSJFWs6mqiJEmSJDU9r4hJkiRJUsX8ICZJkiRJFfODmCRJkiRVzA9ikiRJklQxP4hJkiRJUsX8ICZJkiRJFfODmCRJkiRVzA9ikiRJklQxP4hJkiRJUsX8ICZJkiRJFfODmCRJkiRVzA9ikiRJklQxP4hJkiRJUsX8ICZJkiRJFfODmCRJkiRVzA9ikiRJklQxP4hJkiRJUsX8ICZJkiRJFfODmCRJkiRVzA9ikiRJklQxP4hJkiRJUsX8ICZJkiRJFfODmCRJkiRV7P8A/yrguxR5AJUAAAAASUVORK5CYII=\n"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"nets = 15\nmodel = [0] *nets\nfor j in range(nets):\n    model[j] = Sequential()\n\n    model[j].add(Conv2D(32, kernel_size = 3, activation='relu', input_shape = (28, 28, 1)))\n    model[j].add(BatchNormalization())\n    model[j].add(Conv2D(32, kernel_size = 3, activation='relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Conv2D(32, kernel_size = 5, strides=2, padding='same', activation='relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Dropout(0.4))\n\n    model[j].add(Conv2D(64, kernel_size = 3, activation='relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Conv2D(64, kernel_size = 3, activation='relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Conv2D(64, kernel_size = 5, strides=2, padding='same', activation='relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Dropout(0.4))\n\n    model[j].add(Conv2D(128, kernel_size = 4, activation='relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Flatten())\n    model[j].add(Dropout(0.4))\n    model[j].add(Dense(10, activation='softmax'))\n\n    # COMPILE WITH ADAM OPTIMIZER AND CROSS ENTROPY COST\n    model[j].compile(optimizer=\"adam\", loss=\"categorical_crossentropy\", metrics=[\"accuracy\"])","execution_count":71,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# DECREASE LEARNING RATE EACH EPOCH\nannealer = LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x)\n# TRAIN NETWORKS\nhistory = [0] * nets\nepochs = 40\nes = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=25)\nmc = ModelCheckpoint('best_model.h5', monitor='val_acc', mode='max', verbose=0, save_best_only=True)\nfor j in range(nets):\n    X_train2, X_val2, Y_train2, Y_val2 = train_test_split(X_train, Y_train, test_size = 0.2)\n    history[j] = model[j].fit_generator(datagen.flow(X_train2,Y_train2, batch_size=64),\n        epochs = epochs, steps_per_epoch = X_train2.shape[0]//64,  \n        validation_data = (X_val2,Y_val2), callbacks=[annealer,es,mc], verbose=0)\n    print(\"CNN {0:d}: Epochs={1:d}, Train accuracy={2:.5f}, Validation accuracy={3:.5f}\".format(\n            j+1,epochs,max(history[j].history['acc']),max(history[j].history['val_acc']) ))\n    if(j==14):\n        # Plot training & validation accuracy values\n        plt.plot(history[j].history['acc'])\n        plt.plot(history[j].history['val_acc'])\n        plt.title('Model accuracy')\n        plt.ylabel('Accuracy')\n        plt.xlabel('Epoch')\n        plt.legend(['Train', 'Validation'], loc='upper left')\n        plt.show()\n        # Plot training & validation loss values\n        plt.plot(history[j].history['loss'])\n        plt.plot(history[j].history['val_loss'])\n        plt.title('Model loss')\n        plt.ylabel('Loss')\n        plt.xlabel('Epoch')\n        plt.legend(['Train', 'Validation'], loc='upper left')\n        plt.show()","execution_count":null,"outputs":[{"output_type":"stream","text":"CNN 1: Epochs=40, Train accuracy=0.99554, Validation accuracy=0.99571\nCNN 2: Epochs=40, Train accuracy=0.99560, Validation accuracy=0.99595\nCNN 3: Epochs=40, Train accuracy=0.99598, Validation accuracy=0.99631\nCNN 4: Epochs=40, Train accuracy=0.99583, Validation accuracy=0.99571\nCNN 5: Epochs=40, Train accuracy=0.99557, Validation accuracy=0.99583\nCNN 6: Epochs=40, Train accuracy=0.99518, Validation accuracy=0.99571\nCNN 7: Epochs=40, Train accuracy=0.99613, Validation accuracy=0.99607\nCNN 8: Epochs=40, Train accuracy=0.99631, Validation accuracy=0.99548\nCNN 9: Epochs=40, Train accuracy=0.99530, Validation accuracy=0.99655\nCNN 10: Epochs=40, Train accuracy=0.99545, Validation accuracy=0.99631\nCNN 11: Epochs=40, Train accuracy=0.99586, Validation accuracy=0.99631\nCNN 12: Epochs=40, Train accuracy=0.99616, Validation accuracy=0.99571\nCNN 13: Epochs=40, Train accuracy=0.99601, Validation accuracy=0.99583\nCNN 14: Epochs=40, Train accuracy=0.99542, Validation accuracy=0.99631\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"saved_model = load_model('best_model.h5')\nresults = np.zeros( (X_test.shape[0],10) )\nresults = results + saved_model.predict(X_test)\nresults = np.argmax(results,axis = 1)\nresults = pd.Series(results,name=\"Label\")\nsubmission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\nsubmission.to_csv(\"MNIST-CNN-submission.csv\",index=False)\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}