{"cells":[{"cell_type":"markdown","metadata":{"_cell_guid":"ed8cf668-628f-a82d-186e-9f2967e80c0c"},"source":"I have created simple deep neural network to identify different digits."},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"3a7aabfc-b35f-1f59-976e-8a8664933ae7"},"outputs":[],"source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n\n# Any results you write to the current directory are saved as output."},{"cell_type":"markdown","metadata":{"_cell_guid":"8353c195-c5df-3fff-8306-7956cdedd88a"},"source":"**Load Train and Test data**\n============================"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"78cd21be-9f62-8ab5-d86a-55fe1d27a85f"},"outputs":[],"source":"# create the training & test sets, skipping the header row with [1:]\ntrain = pd.read_csv(\"../input/train.csv\")\ntrain_images = (train.ix[:,1:].values).astype('float32')\ntrain_labels = train.ix[:,0].values.astype('int32')\n\ntest_images = (pd.read_csv(\"../input/test.csv\").values).astype('float32')"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"124181ca-c326-0480-7a67-f9ddd9d23c89"},"outputs":[],"source":"train_images.shape"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"2bd00add-a896-77e5-a9f6-fc06adf1d09a"},"outputs":[],"source":"test_images.shape"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"4a227f11-65e3-77ea-bd59-0834f5d96933"},"outputs":[],"source":"train_labels"},{"cell_type":"markdown","metadata":{"_cell_guid":"d7d51c7a-b3e7-7f41-9b83-8b0865a32eae"},"source":"**Preprocessing the digit images**\n=================================="},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"4e69979e-dde6-9bc7-cf43-dcf973149dbd"},"outputs":[],"source":"#Convert each digit pixel value to 28*28 matrix\ntrain_images = train_images.reshape((42000, 28 * 28))\ntrain_images[1]"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"ee10381e-2d00-a51f-1a5b-97cf8f003b8b"},"outputs":[],"source":"test_images = test_images.reshape((28000, 28 * 28))"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"070618e3-f6b0-4da7-afa5-16ce2eadd606"},"outputs":[],"source":"#Scaling\ntrain_images = (train_images -128)/ 128\ntest_images = (test_images -128)/ 128"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"4d565b66-84c2-2f7d-2ab5-22163743af6d"},"outputs":[],"source":"train_images[0]"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"fbaf0694-a1c0-84df-1e6d-2652f28042d6"},"outputs":[],"source":"#One Hot encoding of labels.\nfrom keras.utils.np_utils import to_categorical\ntrain_labels = to_categorical(train_labels)"},{"cell_type":"markdown","metadata":{"_cell_guid":"beaab03a-4e23-ceba-aeb5-91d25c8b23c8"},"source":"**Designing neural network architecture**\n========================================="},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"69e8cb4f-1244-4013-1c8e-77bf4d8f4035"},"outputs":[],"source":"from keras.models import Sequential\nfrom keras.layers import Dense , Dropout\n\nmodel=Sequential()\nmodel.add(Dense(32,activation='relu',input_dim=(28 * 28)))\nmodel.add(Dense(16,activation='relu'))\nmodel.add(Dense(10,activation='softmax'))"},{"cell_type":"markdown","metadata":{"_cell_guid":"de6e83c7-c6b2-12a2-a78c-5bd8ca4c1942"},"source":"**Compile network**\n===================\n\nBefore making network ready for training we have to make sure to add below things:\n\n 1.  A loss function: to measure how good the network is\n    \n 2.  An optimizer: to update network as it sees more data and reduce loss\n    value\n    \n 3.  Metrics: to monitor performance of network"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"cdb03fc6-e712-a3c8-ff11-1033333a08c0"},"outputs":[],"source":"from keras.optimizers import RMSprop\nmodel.compile(optimizer=RMSprop(lr=0.001),\n loss='categorical_crossentropy',\n metrics=['accuracy'])"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"2f833858-c0b3-da8e-3a15-0746f09322e1"},"outputs":[],"source":"history=model.fit(train_images, train_labels, validation_split = 0.05, \n            nb_epoch=20, batch_size=128)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"853edd9b-37f7-14f8-2247-038c590aa810"},"outputs":[],"source":"history_dict = history.history\nhistory_dict.keys()"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"dea40522-431c-0fa8-d341-b8be88256992"},"outputs":[],"source":"import matplotlib.pyplot as plt\n%matplotlib inline\nloss_values = history_dict['loss']\nval_loss_values = history_dict['val_loss']\nepochs = range(1, len(loss_values) + 1)\n\n# \"bo\" is for \"blue dot\"\nplt.plot(epochs, loss_values, 'bo')\n# b+ is for \"blue crosses\"\nplt.plot(epochs, val_loss_values, 'b+')\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\n\nplt.show()"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"457d4723-2b09-8cd7-be95-9f14eb8b64df"},"outputs":[],"source":"plt.clf()   # clear figure\nacc_values = history_dict['acc']\nval_acc_values = history_dict['val_acc']\n\nplt.plot(epochs, acc_values, 'bo')\nplt.plot(epochs, val_acc_values, 'b+')\nplt.xlabel('Epochs')\nplt.ylabel('Accuracy')\n\nplt.show()"},{"cell_type":"markdown","metadata":{"_cell_guid":"d0ad7aa6-5697-41c8-dc95-5951e6c6db7a"},"source":"From the graphs we can see that training loss is decreasing and training accuracy is increasing slowly. Thats what we intended to do using gredient descent.\nBut thats not the case with validation set\nafter 7th epoch val_loss is increasing and val_acc is decreasing.\nThat is called as **overfitting**.\n\nafter the second epoch, we are over-optimizing on the training data, and we ended up learning representations that are specific to the training data and do not generalize to data outside of the training set\n\nTo avoid this we will simply stop training after 7 epochs."},{"cell_type":"markdown","metadata":{"_cell_guid":"30eae9f3-e2b7-084c-ee85-008698db9597"},"source":"**creating model again from scratch**\n====================================="},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"aec5678b-e5b7-f387-2b69-f61c30ce2855"},"outputs":[],"source":"model = Sequential()\nmodel.add(Dense(64, activation='relu', input_dim=(28 * 28)))\nmodel.add(Dense(128, activation='relu'))\nmodel.add(Dropout(0.15))\nmodel.add(Dense(64, activation='relu'))\nmodel.add(Dropout(0.15))\nmodel.add(Dense(10, activation='softmax'))\n\n\nmodel.compile(optimizer=RMSprop(lr=0.0001), loss='categorical_crossentropy',\n metrics=['accuracy'])\n\nhistory=model.fit(train_images, train_labels, \n            nb_epoch=15, batch_size=64)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"aa57fe3c-00f1-186c-0243-7729b474175c"},"outputs":[],"source":"predictions = model.predict_classes(test_images, verbose=0)\n\nsubmissions=pd.DataFrame({\"ImageId\": list(range(1,len(predictions)+1)),\n                         \"Label\": predictions})\nsubmissions.to_csv(\"DR.csv\", index=False, header=True)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"3ff2e950-a3d4-3e9d-61b7-357666efa657"},"outputs":[],"source":""},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"07172964-05dc-774c-ce8e-960398460515"},"outputs":[],"source":""},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"384dd377-e982-9237-65d6-02531e33d901"},"outputs":[],"source":""}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.0"}},"nbformat":4,"nbformat_minor":0}