{"cells":[{"metadata":{"_uuid":"8be988131b143dfe1e02be424acb3953a34e37ff"},"cell_type":"markdown","source":"### Import Libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f40efd8e83aaa6e907370db3e67e5597b16d24c4"},"cell_type":"markdown","source":"### Read Data"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")\nprint(train.shape, test.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c00b6095b6fc69e972f3101db90b3a06159c0720"},"cell_type":"markdown","source":"### Train Test Split"},{"metadata":{"trusted":true,"_uuid":"335fbf883b546f9d9c3aca1ad74049cb55f40184"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nx_train, x_val, y_train, y_val = train_test_split(train.iloc[:, 1:].values,\n                                                  train.iloc[:, 0].values, \n                                                 test_size = 0.2, \n                                                 random_state = 42,\n                                                 stratify = train.iloc[:, 0])\nprint(\"Training Data Shape = \", x_train.shape, y_train.shape)\nprint(\"Validation Data Shape = \", x_val.shape, y_val.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d8809a302be7e2a10089f416b5e284e134f8ce03"},"cell_type":"markdown","source":"### Class Distribution"},{"metadata":{"trusted":true,"_uuid":"b3026c758462bee505ef3e96881bc775be472994"},"cell_type":"code","source":"fig, ax = plt.subplots(2, 1, sharex = 'col', sharey = 'row')\nfig.set_size_inches(9, 10)   #width, height\nsns.countplot(y_train, ax = ax[0])\nax[0].set_title(\"Training Data Labels\")\nsns.countplot(y_val, ax = ax[1])\nax[1].set_title(\"Validation Data Labels\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6f42d7a2a4d1864ce0d3febd3979c0f66bfa2123"},"cell_type":"markdown","source":"### One Hot Encoding of Labels"},{"metadata":{"trusted":true,"_uuid":"e1dccecc23ee01e8d694947aa2cf82e1ba88c0dd"},"cell_type":"code","source":"from keras import utils\ny_train = utils.to_categorical(y_train, 10).astype(np.uint8)\ny_val = utils.to_categorical(y_val, 10).astype(np.uint8)\nprint(y_train.shape)\nprint(y_train[0:5, :])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b30add90af9ebd89556efd99822963db0b91544f"},"cell_type":"markdown","source":"### Normalize"},{"metadata":{"trusted":true,"_uuid":"d4e880c79ac29afdc4ca735094f8c8ecc625b8e8"},"cell_type":"code","source":"x_train = x_train/255.0\nx_val = x_val/255.0\nx_test = test.values/255.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"414c0e682c5f1752293852c6940471dee2342295"},"cell_type":"code","source":"from keras import *\nfrom keras.layers import *\nfrom keras.optimizers import *\nfrom keras.callbacks import *\nfrom keras.metrics import *","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9092feba9f6e80b081b278ff3ab831fed773e49b"},"cell_type":"markdown","source":"## Simple NN"},{"metadata":{"trusted":true,"_uuid":"65203741b1feda4e9f5769a7e439cf4aecd061cb"},"cell_type":"code","source":"img_size = 784 #28*28\nimg_wid = 28\ninputs = Input(shape = (img_size,))\nF1 = Dense(units = 32, activation = 'relu')(inputs)\noutputs = Dense(units=10, activation= 'softmax')(F1)\nmodel = Model(inputs = [inputs], outputs = [outputs])\nmodel.compile(optimizer = 'sgd', \n              loss = 'categorical_crossentropy', metrics = ['accuracy'])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"28d8e5608f9d5612dcd9b7bede0b41252ce5bf57"},"cell_type":"markdown","source":"##### Fit Model"},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"d73c619fad4c55f5fb5d31d26d92d8e74855985d"},"cell_type":"code","source":"epochs = 10\nbatch_size = 32\n\nmodel.fit(x_train, y_train, batch_size= batch_size, epochs = epochs)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"94249beef1d470802ba524e5dcc4d608f2ef341f"},"cell_type":"markdown","source":"##### Predict"},{"metadata":{"trusted":true,"_uuid":"d0095455f79d237781a8aea53b5f52e5d0b5b47f"},"cell_type":"code","source":"y_pred = model.predict(x_test).argmax(axis = 1)\nprint(y_pred.shape, np.unique(y_pred))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"20f3d10906802b72b544ccb0ad57a4b7bf2dc920"},"cell_type":"markdown","source":"##### Performance"},{"metadata":{"trusted":true,"_uuid":"bd0800cbac5170bcd77b03eab256e8b24974db3e"},"cell_type":"code","source":"valScore, valAcc = model.evaluate(x_val, y_val)\ntrainScore, trainAcc = model.evaluate(x_train, y_train)\nprint(\"Training Score = \", trainScore, \"Training Accuracy = \", trainAcc)\nprint(\"Validation Score = \", valScore, \"Validation Accuracy = \", valAcc)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5f57cb70cdda6945bf6f802cc265f04ea1226095"},"cell_type":"markdown","source":"##### Save Prediction"},{"metadata":{"trusted":true,"_uuid":"4a28edfd4c7e2c08a1081e10e3ff0778f5beea94"},"cell_type":"code","source":"sub = pd.DataFrame()\nsub['ImageId'] = np.arange(1, y_pred.shape[0] + 1)\nsub['Label'] = y_pred\nsub.to_csv(\"nnSimple.csv\",index = False, header = True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1259f73fb7b1e65fed7269141ac53a05d023f57e"},"cell_type":"markdown","source":"### Simple CNN"},{"metadata":{"trusted":true,"_uuid":"00803e32b22c1e025fa44d93609f9fa366344247"},"cell_type":"code","source":"img_wid = 28\n\nx_train = x_train.reshape(x_train.shape[0], 28, 28, 1)\nx_val = x_val.reshape(x_val.shape[0], 28, 28, 1)\ninputs = Input(shape = (img_wid, img_wid, 1))\n\nC1 = Conv2D(16, (3, 3), activation = 'relu', padding = 'same')(inputs)\nD1 = Flatten()(C1)\nF1 = Dense(units = 32, activation = 'relu')(D1)\noutputs = Dense(units = 10, activation = 'softmax')(F1)\nmodel = Model(inputs = [inputs], outputs = [outputs])\nmodel.compile(optimizer = 'sgd', \n              loss = 'categorical_crossentropy', metrics = ['accuracy'])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d82464a32668c7d4d38d43e9da239052b74587d0"},"cell_type":"markdown","source":"##### Fit Model"},{"metadata":{"trusted":true,"_uuid":"fde023ca652e8a8f2ee491d0ec09becf01bed363"},"cell_type":"code","source":"epochs = 10\nbatch_size = 32\n\nmodel.fit(x_train, y_train, batch_size= batch_size, epochs = epochs)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"672053ab50574d8144f96e0bf15a0c641d6811c4"},"cell_type":"markdown","source":"##### Predict"},{"metadata":{"trusted":true,"_uuid":"58f2d76a03bf216658b0508cd09d61647c88fb7c"},"cell_type":"code","source":"y_pred = model.predict(x_test.reshape(x_test.shape[0], 28, 28, 1)).argmax(axis = 1)\nprint(y_pred.shape, np.unique(y_pred))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"684a7f5c10b2ae1e387fb0798c36506e582f9e0c"},"cell_type":"markdown","source":"##### Performance"},{"metadata":{"trusted":true,"_uuid":"6d0ba6123ca0d5b14d8c8c1451f02b7826b1514d"},"cell_type":"code","source":"valScore, valAcc = model.evaluate(x_val, y_val)\ntrainScore, trainAcc = model.evaluate(x_train, y_train)\nprint(\"Training Score = \", trainScore, \"Training Accuracy = \", trainAcc)\nprint(\"Validation Score = \", valScore, \"Validation Accuracy = \", valAcc)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"40a865137671acdfcb2b38faed1d68e3103caca4"},"cell_type":"markdown","source":"##### Save Prediction"},{"metadata":{"trusted":true,"_uuid":"8ab9cce60d385eb2f2312018cc769c2bbeb2ce64"},"cell_type":"code","source":"sub = pd.DataFrame()\nsub['ImageId'] = np.arange(1, y_pred.shape[0] + 1)\nsub['Label'] = y_pred\nsub.to_csv(\"cnnSimple.csv\",index = False, header = True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c2ea0d070b9a7f5e6bfd75b5b9452b527b53be66"},"cell_type":"markdown","source":"## LeNet-5\n[LeNet-5 Paper](http://yann.lecun.com/exdb/publis/pdf/lecun-01a.pdf)"},{"metadata":{"trusted":true,"_uuid":"f4d10fd797766cdf07a3179378ce84a0ffd5b2e4","scrolled":true},"cell_type":"code","source":"img_wid = 28\n\nx_train = x_train.reshape(x_train.shape[0], 28, 28, 1)\nx_val = x_val.reshape(x_val.shape[0], 28, 28, 1)\ninputs = Input(shape = (img_wid, img_wid, 1))\n\n#IMG = (28, 28), \nC1 = Conv2D(16, (3, 3), activation = 'relu', padding = \"same\")(inputs)\nP1 = MaxPooling2D(pool_size=(2, 2))(C1)\n\n#IMG = (14, 14)\nC2 = Conv2D(32, (3, 3), activation = \"relu\", padding = \"same\")(P1)\nP2 = Dropout(0.15)(P1)\nP2 = MaxPooling2D(pool_size= (2, 2))(C2)\n\n#IMG = (7, 7)\nC3 = Conv2D(128, (5, 5), activation = 'relu', padding = \"same\")(P2)\nC3 = Dropout(0.15)(C3)\nF1 = Flatten()(C3)\n\n#Fully Connected Layer\nD1 = Dense(units = 512, activation = 'relu',\n           activity_regularizer=regularizers.l2(0.01))(F1)\nD1 = Dropout(0.25)(D1)\nD2 = Dense(units = 512, activation = 'relu',\n           activity_regularizer=regularizers.l2(0.01))(D1)\nD2 = Dropout(0.25)(D2)\noutputs = Dense(units = 10, activation = 'softmax')(D2)\n\nmodel = Model(inputs = [inputs], outputs = [outputs])\nmodel.compile(optimizer = 'sgd', loss = 'categorical_crossentropy',\n             metrics = ['accuracy'], )\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ddd00f01213538a23201dc9edf84999c140b3ef0"},"cell_type":"markdown","source":"##### Fit"},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"6e7d69567307caee829228852cad630e3e534065"},"cell_type":"code","source":"import datetime\nprint(datetime.datetime.now())\nepochs = 30\nbatch_size = 32\n\nmodel.fit(x_train, y_train, batch_size= batch_size, epochs = epochs,\n          validation_data = (x_val, y_val))\nprint(datetime.datetime.now())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4549fc9085947f58805f2afbb4d74eb5b45aaf42"},"cell_type":"markdown","source":"##### Predict"},{"metadata":{"trusted":true,"_uuid":"15f8de045861c2f1163d99f4d7c18621692a29da"},"cell_type":"code","source":"y_pred = model.predict(x_test.reshape(x_test.shape[0], 28, 28, 1)).argmax(axis = 1)\nprint(y_pred.shape, np.unique(y_pred))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c9706743536765a4ec6cb1ccf824ef8a845e8d91"},"cell_type":"markdown","source":"##### Performance"},{"metadata":{"trusted":true,"_uuid":"a6811b7fe5200556d2d5f41d972fce59861f80db"},"cell_type":"code","source":"valScore, valAcc = model.evaluate(x_val, y_val)\ntrainScore, trainAcc = model.evaluate(x_train, y_train)\nprint(\"Training Score = \", trainScore, \"Training Accuracy = \", trainAcc)\nprint(\"Validation Score = \", valScore, \"Validation Accuracy = \", valAcc)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2e372ec5bfc86873aea41f9e48c9543eed0a5607"},"cell_type":"markdown","source":"##### Save Prediction"},{"metadata":{"trusted":true,"_uuid":"777379368ed6968fd799ee45d688be6deddc8b50"},"cell_type":"code","source":"sub = pd.DataFrame()\nsub['ImageId'] = np.arange(1, y_pred.shape[0] + 1)\nsub['Label'] = y_pred\nsub.to_csv(\"cnnLeNet-5.csv\",index = False, header = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"61c410d5874d507ae645cb569dab97c16b9d915b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}