{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Handwritten Digit Recognition in Python with Keras"},{"metadata":{},"cell_type":"markdown","source":"## Preparing the Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.metrics import classification_report\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import confusion_matrix\nimport keras","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nprint(os.listdir(\"../input\"))\ntrain = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.shape)\nprint(test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['label'].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"columns=[]\nfor col in train.columns:\n    if col != 'label':\n        columns.append(col)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = train[columns]\nprint(X_train.shape)\ny_train = train['label']\nprint(y_train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test = test[columns]\nprint(X_test.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Since we have to categorize data into 0-9, so using one-hot encoding technique is preffered."},{"metadata":{"trusted":true},"cell_type":"code","source":"print('first 5 training labels : ',y_train[:5])\n\n# convert into one-hot encoded vectors using to_categorical function\nnum =10\ny_train = keras.utils.to_categorical(y_train,num)\n\nprint('first 5 training labels after one-hot encoding : ',y_train[:5])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Neural Network Architecture"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.layers import Dense # Dense -> fully conected layer\nfrom keras.models import Sequential\n\nimage_size = 784\nnum_classes = 10\n\nmodel = Sequential()\n\nmodel.add(Dense(units = 32, activation = 'sigmoid',input_shape =(image_size,)))\nmodel.add(Dense(units = num_classes,activation ='softmax'))\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" training and evaluating the model :\n    \n-  selected a common loss function called categorical cross entropy.\n-  selected one of the simplest optimization algorithms: Stochastic Gradient Descent (SGD)."},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n%matplotlib inline\nbatch_size = 128\nmodel.compile(optimizer = 'sgd',loss = 'categorical_crossentropy',metrics =['accuracy'])\nhistory = model.fit(X_train,y_train,batch_size =batch_size,epochs =3,verbose = False, validation_split =0.1)\npred_val1 = model.predict_classes(X_train)\npred_val1 = keras.utils.to_categorical(pred_val1,num)\nprint(classification_report(y_train, pred_val1))\nacc_model1 = accuracy_score(y_train, pred_val1)\nprint('accuracy score : ',acc_model1)\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(history.history['acc'])\nplt.plot(history.history['val_acc'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['training', 'validation'], loc='best')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can see the plot of training and validation accuracy over the time. Final accuracy is around 0.92"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Network depth and layer width"},{"metadata":{},"cell_type":"markdown","source":"### Network Depth "},{"metadata":{},"cell_type":"markdown","source":"The depth of multi-layer perceptron(also known as a fully connected neural network) is determined by it's number of hidden layers. The model we have used has only one hidden layer so it's a shallow network. \n\nso, for deep learning, we will experiment with layers of different lengths and see how it affects the performance."},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_dense(layer_size):\n    model = Sequential()\n    model.add(Dense(layer_size[0],activation = 'sigmoid',input_shape =(image_size,)))\n    \n    for s in layer_size[1:] :\n        model.add(Dense(units = s, activation = 'sigmoid'))\n    model.add(Dense(units = num_classes,activation ='softmax'))\n    \n    return model\n\ndef evaluate(model,batch_size =128, epochs =5) :\n    model.summary()\n    model.compile(optimizer = 'sgd',loss = 'categorical_crossentropy',metrics =['accuracy'])\n    history = model.fit(X_train,y_train, batch_size = batch_size, epochs = epochs,verbose =False,validation_split =.1)\n    pred_val = model.predict_classes(X_train)\n    pred_val = keras.utils.to_categorical(pred_val,num)\n    print(classification_report(y_train, pred_val))\n    acc_model = accuracy_score(y_train, pred_val)\n    print('accuracy score : ',acc_model)\n    \n    plt.plot(history.history['acc'])\n    plt.plot(history.history['val_acc'])\n    plt.title('model accuracy')\n    plt.ylabel('accuracy')\n    plt.xlabel('epoch')\n    plt.legend(['training', 'validation'], loc='best')\n    plt.show()\n\n    print()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for layers in range(1, 5):\n    model = create_dense([32] * layers)\n    evaluate(model)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Adding more layers have decreased the accuracy of the model.This is not intuitive. One cause of this may be overfitting. We can check overfitting when we can see that training accuracy is more than test accuracy but this is only in the case of 4 hidden layers. so, problem may be something else.\nSince, every layers as input gets output of previous layer so there may be prevalent information loss.\nAlso, neural networks take time to train, so we can try by incresing the time span.\n"},{"metadata":{},"cell_type":"markdown","source":"we try by incresing time span from 5 epochs to 40 epochs for model with 3 hidden layers and see the result."},{"metadata":{"trusted":true},"cell_type":"code","source":"model = create_dense([32]*3)\nevaluate(model,epochs =40)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now, we can see that, model is giving proper accuracy.\nso, we should always experiment on our models to find which of the factors are creating problems."},{"metadata":{},"cell_type":"markdown","source":"### Layer Width"},{"metadata":{},"cell_type":"markdown","source":"Width of a lyer is number of nodes in each layer. Making wider layers tends to scale the number of parameters faster than adding more layers. Every time we add a single node to layer i, we have to give that new node an edge to every node in layer i+1."},{"metadata":{},"cell_type":"markdown","source":"Using create_dense and evaluate functions, we try to compare neural networks with different widths with single hidden layer"},{"metadata":{"trusted":true},"cell_type":"code","source":"for nodes in [32, 64, 128, 256, 512, 1024, 2048]:\n    model = create_dense([nodes])\n    evaluate(model)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"From the plots, we observe that increasing number of nodes in a layer is increasing the performance from ~88% in case of 32 nodes to ~95% in acse of 2048 nodes. We can see that, in the last case with 2048 nodes, the training accuracy is somewhat predicted the accuracy of test data - so probabily there is no overfitting.\n\nThe cost of this improvement was increased training time."},{"metadata":{},"cell_type":"markdown","source":"### combining width and depth"},{"metadata":{"trusted":true},"cell_type":"code","source":"for nodes_per_layer in [32, 128, 512]:\n    for layers in [3, 4, 5]:\n        model = create_dense([nodes_per_layer] * layers)\n        evaluate(model, epochs=10*layers)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Highest accuracy achieved is 93.99% in neural network with 512 nodes and 3 layers.\nRegardless of number of nodes, all networks performed better upto 3 layers. As the layers are increased to 4 and above, accuracy is decreased. In case of 32 nodes and 5 layers, it is worst affected."},{"metadata":{"trusted":true},"cell_type":"code","source":"model = create_dense([1024]*3)\nmodel.compile(optimizer = 'sgd',loss = 'categorical_crossentropy',metrics =['accuracy'])\nmodel.fit(X_train,y_train, batch_size = 128, epochs = 30,verbose =False,validation_split =.1)\nprediction = model.predict_classes(X_test,verbose =0)\nsubmissions=pd.DataFrame({\"ImageId\": list(range(1,len(predictions)+1)),\"Label\": predictions})\nsubmissions.to_csv(\"out.csv\", index=False, header=True)\n\n","execution_count":1,"outputs":[{"output_type":"error","ename":"NameError","evalue":"name 'create_dense' is not defined","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)","\u001b[0;32m<ipython-input-1-4e386aa76a09>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcreate_dense\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1024\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'sgd'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'categorical_crossentropy'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmetrics\u001b[0m \u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m128\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m30\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mverbose\u001b[0m \u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mvalidation_split\u001b[0m \u001b[0;34m=\u001b[0m\u001b[0;36m.1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mprediction\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict_classes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mverbose\u001b[0m \u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0msubmissions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m\"ImageId\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpredictions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"Label\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mpredictions\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mNameError\u001b[0m: name 'create_dense' is not defined"]}]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.4"}},"nbformat":4,"nbformat_minor":1}