{"cells":[{"metadata":{"_cell_guid":"552bf20c-b0ff-c88c-e4bd-35cefe4b58a8","_uuid":"0a910095ad3e5f83328fcb2d4254a415ade22810"},"cell_type":"markdown","source":"This notebook will explore the dataset from Kaggle's competition on the digit recognition and will use a number of Neural Networks to demonstrate their usage and utility."},{"metadata":{"_cell_guid":"2cc7e216-e0bf-f92d-1c9f-94f36460db3b","_uuid":"14435e3b03cc929ce78b5a53dd61667fd72c9c06","trusted":false},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"51ac8122-d88f-ce4a-9518-c4ebe14e4c14","_uuid":"1be7d7355c62e8c57e4847902bb0d11ea84ed010","trusted":false},"cell_type":"code","source":"myTrainDf = pd.read_csv('../input/train.csv')\nmyTestDf = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"fb1cc6d7-499a-5df2-0ff7-295d16645294","_uuid":"64508cdbe7446a911aae36ddedcff31e41c3183e","trusted":false},"cell_type":"code","source":"# Display some digits to understand how the data is structured\ndef formatDigit (aLine):\n    return np.array(aLine).reshape((28,28))\n\nNUM_EXAMPLES = 3\n\nfor i in (myTrainDf.shape[0]*np.random.rand(NUM_EXAMPLES)).astype(int):\n    myImage = formatDigit(myTrainDf.iloc[i][1:])\n    plt.matshow(myImage, cmap='gray')\n    plt.colorbar()\n    plt.title('Example for digit ' + str(myTrainDf.iloc[i][0]))\n\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"1b77b6d9-ea08-83d8-f08c-76d0acd2d58f","_uuid":"9dcc0467ea24377ebb3ac7437010f295c975ede8"},"cell_type":"markdown","source":"Now that we see how the data is organized, let's use a MLP, with an architecture as the one taught in \"Machine Learning\" from Stanford in coursera.org to recognize the digits.\n\nThe MLP will have 3 layers.\n\n - The input layer will have 784 units\n - The hidden layer will have 25 units\n - The output layer will have, obviously, 10 units. It will output 1 or 0 depending on the classification.\n\nNote that the sizes of the input and output layers are given by the X and Y datasets."},{"metadata":{"_cell_guid":"617b6666-9ebd-d369-adb8-58f416c97e20","_uuid":"895e9cc4cc5c1f5646b418798ded547d5ee3d149","trusted":false},"cell_type":"code","source":"from sklearn.neural_network import MLPClassifier\n\nmyX = myTrainDf[myTrainDf.columns[1:]]\nmyY = myTrainDf[myTrainDf.columns[0]]\n\n# Use 'adam' solver for large datasets, alpha is the regularization term.\n# Display the optimization by showing the cost.\nmyClf = MLPClassifier(hidden_layer_sizes=25, activation='logistic', solver='adam', \n                      alpha=1e-5, verbose=True)\nmyClf.fit(myX, myY)\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"e60b71bb-2f77-32ee-0efa-9adbe81a625d","_uuid":"12ea7ccba8c4fcba2bcd89c00fdc13bcfabfc468","trusted":false},"cell_type":"code","source":"# Get the training error\nmyPredY = myClf.predict(myX)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"e6a633d9-5e2e-65a2-391e-61584184fed0","_uuid":"0e83271f72890aec743cdc723bbea0a44892c99a","trusted":false},"cell_type":"code","source":"from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score\n\n# Generic function to assess performance in two datasets\ndef showPerformance (aY, aYpred):\n    # Ensure np.array\n    print ('*** Performance Statistics ***')\n    print ('Accuracy: ', accuracy_score(aY, aYpred))\n    print ('Precision: ', precision_score(aY, aYpred, average='micro'))\n    print ('Recall: ', recall_score(aY, aYpred, average='micro'))\n    print ('F1: ', f1_score(aY, aYpred, average='micro'))\n\nshowPerformance(myY, myPredY)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c2110795-9d2a-72a1-cfb4-dd998d5e4cb0","_uuid":"9a85ed24bb81e7cb2561b4e23462caf02cfd5d5d"},"cell_type":"markdown","source":"The results are quite good, as expected, now let's make a prediction for the test set."},{"metadata":{"_cell_guid":"40c53cfa-7b5f-1b34-8a44-0930e58620d1","_uuid":"d3b3c424745008fa3084acbf140c1d7305f1eaa5","trusted":false},"cell_type":"code","source":"myYtestPred = myClf.predict(myTestDf)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"1ca0cb98-1539-4cf0-4211-652dedf12d9a","_uuid":"5e8c7cf54c110c16eed6e66e5f2de9ad29cf257b","trusted":false},"cell_type":"code","source":"myOutDf = pd.DataFrame(index=myTestDf.index, data=myYtestPred)\nmyOutDf.reset_index().to_csv('submission.csv', header=['Image_ID', 'Label'])","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"ee1e7e91-8ec6-6b6c-9589-c9e4c4939a8b","_uuid":"f1731643c8d21f57496a0e277b97bb548664255c"},"cell_type":"markdown","source":"<h1> Playing with the regularization parameter </h1>\nWe are going to build a loop to check which regularization parameter works better. First we need to sacrifice the test set in order to create a cross-validation set."},{"metadata":{"_cell_guid":"d8475c37-8e1d-c8e2-2aa9-f684c05e9886","_uuid":"0be489c8ebbb55af5458ff8b29914c43ca17e2d9","trusted":false},"cell_type":"code","source":"def splitDataset( aTrainDf, aFraction ):\n    aOutTrainDf = aTrainDf.sample(frac=aFraction)\n    aOutCrossValDf = aTrainDf.iloc[[x for x in aTrainDf.index if x not in aOutTrainDf.index]]\n    return aOutTrainDf, aOutCrossValDf\n    \nmySampleTrainDf, mySampleXvalDf = splitDataset(myTrainDf, 0.8)\n\nREG_ARRAY = [1e-2, 1e-3, 1e-4, 5e-5, 1e-5, 5e-6, 1e-6]\n# try for a number of regularization parameters to train the classifier and get performance metrics\nfor myRegParam in REG_ARRAY:\n    myX = myS","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"2b44a1af-c1ae-4e0a-114e-74f983d2becb","_uuid":"ff2e2123d34e6586abf8b2b96bd9c2a2a64a4161","trusted":false},"cell_type":"code","source":"mySampleTrainDf","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"d2c66eff-7375-80d0-5b20-759819d5ef4c","_uuid":"789d0edd4d0e65697e1fd540351ba913586b7d2f","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}