{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Chaithra K C**\n\n1. Introduction\n2. Data preparation\n    1. Load data\n    2. Check for null and missing values\n    3. Split training and valdiation set\n3. Gaussian Naive Bayes Classifier\n    1. Define the model\n    2. Confusion Matrix\n    2. Evaluate the model\n4. Multinominal Naive Bayes Classifier \n    1. Define the model\n    2. Confusion matrix\n    3. Evaluate the model\n5. Decision Trees \n    1. Define the model\n    2. Confusion matrix\n    3. Evaluate the model\n6. Prediction and submition to Kaggle\n    1. Predict and Submit results","metadata":{"id":"88553d66"}},{"cell_type":"markdown","source":"#### 1. INTRODUCTION\n\nThe data set comes from the Kaggle Digit Recognizer competition. The goal is to recognize digits 0 to 9 in handwriting images.\nWe construct prediction models using naïve Bayes and decision tree algorithms. Tune their parameters to get the best model (measured by cross validation) and compare which algorithms provide better model for this task.","metadata":{"id":"3881027d"}},{"cell_type":"markdown","source":"#### 2. DATA PREPARATION","metadata":{"id":"b571f7cb"}},{"cell_type":"code","source":"# importing required libraries\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns \nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix \nfrom sklearn.metrics import classification_report\nfrom sklearn.naive_bayes import GaussianNB, MultinomialNB\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn import tree\nimport timeit # to measure the execution time of models\n","metadata":{"executionInfo":{"elapsed":242,"status":"ok","timestamp":1648584192347,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"5ed560a5","execution":{"iopub.status.busy":"2022-03-29T23:21:13.139061Z","iopub.execute_input":"2022-03-29T23:21:13.139370Z","iopub.status.idle":"2022-03-29T23:21:13.145596Z","shell.execute_reply.started":"2022-03-29T23:21:13.139337Z","shell.execute_reply":"2022-03-29T23:21:13.144396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 2.1 LOAD DATA","metadata":{"id":"974d0388"}},{"cell_type":"code","source":"train = pd.read_csv('../input/digit-recognizer/train.csv')\ntest = pd.read_csv('../input/digit-recognizer/test.csv')\n\ntrain.shape, test.shape","metadata":{"executionInfo":{"elapsed":3761,"status":"ok","timestamp":1648584196240,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"044ce04e","outputId":"c98e506a-4683-435f-eb5a-96d03c6c782a","execution":{"iopub.status.busy":"2022-03-29T23:21:13.170062Z","iopub.execute_input":"2022-03-29T23:21:13.170644Z","iopub.status.idle":"2022-03-29T23:21:17.554130Z","shell.execute_reply.started":"2022-03-29T23:21:13.170596Z","shell.execute_reply":"2022-03-29T23:21:17.553007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head() # vieweing first 5 rows to understand the dataset","metadata":{"executionInfo":{"elapsed":17,"status":"ok","timestamp":1648584196241,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"06cf3b74","outputId":"4e1820e4-af0d-45d8-d253-e82541adf536","execution":{"iopub.status.busy":"2022-03-29T23:21:17.555633Z","iopub.execute_input":"2022-03-29T23:21:17.555923Z","iopub.status.idle":"2022-03-29T23:21:17.573407Z","shell.execute_reply.started":"2022-03-29T23:21:17.555891Z","shell.execute_reply":"2022-03-29T23:21:17.572473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"executionInfo":{"elapsed":15,"status":"ok","timestamp":1648584196241,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"ef9f587b","outputId":"c9ffbc27-2b19-4031-d60b-73b80334e25f","execution":{"iopub.status.busy":"2022-03-29T23:21:17.575010Z","iopub.execute_input":"2022-03-29T23:21:17.575360Z","iopub.status.idle":"2022-03-29T23:21:17.599567Z","shell.execute_reply.started":"2022-03-29T23:21:17.575316Z","shell.execute_reply":"2022-03-29T23:21:17.598653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 2.2 CHECK FOR NULL OR NAN VALUES","metadata":{"id":"9ae12d90"}},{"cell_type":"code","source":"sns.heatmap(train.isnull()) # no null or NA values in training data # resource intense ","metadata":{"executionInfo":{"elapsed":13,"status":"ok","timestamp":1648584196241,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"6a379847","execution":{"iopub.status.busy":"2022-03-29T23:21:17.602077Z","iopub.execute_input":"2022-03-29T23:21:17.602605Z","iopub.status.idle":"2022-03-29T23:22:01.405473Z","shell.execute_reply.started":"2022-03-29T23:21:17.602560Z","shell.execute_reply":"2022-03-29T23:22:01.403240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.heatmap(test.isnull()) # no null or NA values in testing data # resource intense ","metadata":{"executionInfo":{"elapsed":13,"status":"ok","timestamp":1648584196242,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"afb3733b","execution":{"iopub.status.busy":"2022-03-29T23:22:01.406688Z","iopub.execute_input":"2022-03-29T23:22:01.407111Z","iopub.status.idle":"2022-03-29T23:22:30.278989Z","shell.execute_reply.started":"2022-03-29T23:22:01.407078Z","shell.execute_reply":"2022-03-29T23:22:30.278111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are no Null or NA values present in the dataset of both train and test.","metadata":{"id":"47a1f3f7"}},{"cell_type":"markdown","source":"#### 2.3 SPLIT TRAINING AND VALIDATION SET ","metadata":{"id":"7710f84e"}},{"cell_type":"code","source":"Y_train = train[\"label\"]\n\n# dropping the 'label' column\nX_train = train.drop(labels = [\"label\"],axis = 1) \n\ng = sns.countplot(Y_train)\n\nY_train.value_counts()","metadata":{"executionInfo":{"elapsed":206,"status":"ok","timestamp":1648584196436,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"dc319f4c","outputId":"73325ee7-4092-4204-a96d-426ac0e0758b","execution":{"iopub.status.busy":"2022-03-29T23:22:30.280549Z","iopub.execute_input":"2022-03-29T23:22:30.280865Z","iopub.status.idle":"2022-03-29T23:22:30.542055Z","shell.execute_reply.started":"2022-03-29T23:22:30.280825Z","shell.execute_reply":"2022-03-29T23:22:30.541381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We have similar counts for the 10 digits.","metadata":{"id":"50c67350"}},{"cell_type":"code","source":"Y_train[0]","metadata":{"executionInfo":{"elapsed":205,"status":"ok","timestamp":1648584196635,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"fc51312d","outputId":"cd0e8930-eee1-46e3-da08-5bf880b44b91","execution":{"iopub.status.busy":"2022-03-29T23:22:30.543580Z","iopub.execute_input":"2022-03-29T23:22:30.544716Z","iopub.status.idle":"2022-03-29T23:22:30.551910Z","shell.execute_reply.started":"2022-03-29T23:22:30.544663Z","shell.execute_reply":"2022-03-29T23:22:30.550520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n**We can get a better sense for one of these examples by visualising the image and looking at the label**","metadata":{"id":"8431df94"}},{"cell_type":"code","source":"# visualizing the digits for few training example\nprint(Y_train[0])\ntraining_example = X_train.iloc[0]\ntraining_example = training_example.to_numpy() # converting series to numpy array\ntraining_example = training_example.reshape((28,28)) # converting 1d array of pixels to 2d (28X28) pixels matrix\nplt.imshow(training_example, cmap='gray')","metadata":{"executionInfo":{"elapsed":184,"status":"ok","timestamp":1648584196807,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"2d89311f","outputId":"c4e7f962-703d-4ea2-ce71-4f06be760cfb","execution":{"iopub.status.busy":"2022-03-29T23:22:30.554443Z","iopub.execute_input":"2022-03-29T23:22:30.555538Z","iopub.status.idle":"2022-03-29T23:22:30.716840Z","shell.execute_reply.started":"2022-03-29T23:22:30.555498Z","shell.execute_reply":"2022-03-29T23:22:30.716023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(Y_train[300])\ntraining_example = X_train.iloc[300]\ntraining_example = training_example.to_numpy() # converting series to numpy array\ntraining_example = training_example.reshape((28,28)) # converting 1d array of pixels to 2d (28X28) pixels matrix\nplt.imshow(training_example, cmap='gray')","metadata":{"executionInfo":{"elapsed":168,"status":"ok","timestamp":1648584196958,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"c968a03f","outputId":"1b2021d4-3366-45cd-a5ca-7885834b6cb8","execution":{"iopub.status.busy":"2022-03-29T23:22:30.718149Z","iopub.execute_input":"2022-03-29T23:22:30.718608Z","iopub.status.idle":"2022-03-29T23:22:30.924166Z","shell.execute_reply.started":"2022-03-29T23:22:30.718573Z","shell.execute_reply":"2022-03-29T23:22:30.923243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"A small fraction (40%) became the validation set which the model is evaluated and the rest (60%) is used to train the model","metadata":{"id":"6735a19d"}},{"cell_type":"code","source":"# Split the train and the validation set for the fitting\nX_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.4, random_state=30)","metadata":{"executionInfo":{"elapsed":224,"status":"ok","timestamp":1648584197178,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"d6ee4883","execution":{"iopub.status.busy":"2022-03-29T23:22:30.926993Z","iopub.execute_input":"2022-03-29T23:22:30.927346Z","iopub.status.idle":"2022-03-29T23:22:31.210918Z","shell.execute_reply.started":"2022-03-29T23:22:30.927311Z","shell.execute_reply":"2022-03-29T23:22:31.209865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Gaussian Naive Bayes","metadata":{"id":"3bb6b663"}},{"cell_type":"code","source":"# Gaussian distribution assumes all the features to be normally distributed \n\n# checking the distribution of all features with skewness\n\nskweness = train.skew()\n\n# variables with skewness > 1 are highly positively skewed\n# variables with skewness < -1 are highly negatively skewed\n# variables with -0.5 < skewness < 0.5 are symmetric i.e normally distributed\n\nskweness[skweness>1] # positively skewed data","metadata":{"executionInfo":{"elapsed":591,"status":"ok","timestamp":1648585288662,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"t7_V9YGYQzmz","outputId":"d8764ffb-ef5e-41da-c13a-3cbe8fdbc72d","execution":{"iopub.status.busy":"2022-03-29T23:22:31.212257Z","iopub.execute_input":"2022-03-29T23:22:31.212888Z","iopub.status.idle":"2022-03-29T23:22:31.678019Z","shell.execute_reply.started":"2022-03-29T23:22:31.212836Z","shell.execute_reply":"2022-03-29T23:22:31.677179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skweness[skweness<-1] # no negatively skewed","metadata":{"executionInfo":{"elapsed":146,"status":"ok","timestamp":1648585297878,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"rBXrgUZyVygE","outputId":"04716551-5856-409a-9b79-90574d21f17c","execution":{"iopub.status.busy":"2022-03-29T23:22:31.679088Z","iopub.execute_input":"2022-03-29T23:22:31.679353Z","iopub.status.idle":"2022-03-29T23:22:31.687035Z","shell.execute_reply.started":"2022-03-29T23:22:31.679324Z","shell.execute_reply":"2022-03-29T23:22:31.686229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# normally distributed features\nskweness[abs(skweness) < 0.5]","metadata":{"executionInfo":{"elapsed":611,"status":"ok","timestamp":1648585515235,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"UlpqJb_0WEXK","outputId":"e4ce1097-8954-40e9-e4b1-790ef1fb5694","execution":{"iopub.status.busy":"2022-03-29T23:22:31.688432Z","iopub.execute_input":"2022-03-29T23:22:31.688671Z","iopub.status.idle":"2022-03-29T23:22:31.702325Z","shell.execute_reply.started":"2022-03-29T23:22:31.688642Z","shell.execute_reply":"2022-03-29T23:22:31.701356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are many features in the data that are highly skewed.","metadata":{"id":"C-9j06AZX0yF"}},{"cell_type":"code","source":"# instantiate the model\ngnb = GaussianNB()\n\n# fit the model\ngnb.fit(X_train, Y_train)","metadata":{"executionInfo":{"elapsed":418,"status":"ok","timestamp":1648584197590,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"fe407b8b","outputId":"d2457282-6791-46e3-d118-9f1fef9d3381","execution":{"iopub.status.busy":"2022-03-29T23:22:31.703944Z","iopub.execute_input":"2022-03-29T23:22:31.704678Z","iopub.status.idle":"2022-03-29T23:22:32.198458Z","shell.execute_reply.started":"2022-03-29T23:22:31.704629Z","shell.execute_reply":"2022-03-29T23:22:32.197635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predict\nnb_pred = gnb.predict(X_val)\n\nnb_pred","metadata":{"executionInfo":{"elapsed":754,"status":"ok","timestamp":1648584198341,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"20249c10","outputId":"0b19181a-232f-4d59-d09a-d33ab6a4202d","execution":{"iopub.status.busy":"2022-03-29T23:22:32.199681Z","iopub.execute_input":"2022-03-29T23:22:32.200319Z","iopub.status.idle":"2022-03-29T23:22:33.226100Z","shell.execute_reply.started":"2022-03-29T23:22:32.200276Z","shell.execute_reply":"2022-03-29T23:22:33.225028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# validating testing data\ntab = pd.crosstab(Y_val, nb_pred, rownames=['Actual'],colnames=['Predicted'])\ntab","metadata":{"executionInfo":{"elapsed":17,"status":"ok","timestamp":1648584198342,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"b5cc9a4d","outputId":"20a74912-d85e-4cdd-ebaf-4384b56ed9c2","execution":{"iopub.status.busy":"2022-03-29T23:22:33.227634Z","iopub.execute_input":"2022-03-29T23:22:33.227961Z","iopub.status.idle":"2022-03-29T23:22:33.265982Z","shell.execute_reply.started":"2022-03-29T23:22:33.227919Z","shell.execute_reply":"2022-03-29T23:22:33.265150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.heatmap(tab)","metadata":{"executionInfo":{"elapsed":401,"status":"ok","timestamp":1648584198730,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"bdce2782","outputId":"7250573a-e066-48c2-c683-11bd09091515","execution":{"iopub.status.busy":"2022-03-29T23:22:33.267503Z","iopub.execute_input":"2022-03-29T23:22:33.267772Z","iopub.status.idle":"2022-03-29T23:22:33.605011Z","shell.execute_reply.started":"2022-03-29T23:22:33.267741Z","shell.execute_reply":"2022-03-29T23:22:33.604192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Look at confusion matrix \nimport itertools\n\ndef plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, cm[i, j],\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')","metadata":{"executionInfo":{"elapsed":10,"status":"ok","timestamp":1648584198731,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"cdc2dfe0","execution":{"iopub.status.busy":"2022-03-29T23:22:33.606792Z","iopub.execute_input":"2022-03-29T23:22:33.607239Z","iopub.status.idle":"2022-03-29T23:22:33.618362Z","shell.execute_reply.started":"2022-03-29T23:22:33.607195Z","shell.execute_reply":"2022-03-29T23:22:33.617448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# compute the confusion matrix\nconfusion_gnb = confusion_matrix(Y_val, nb_pred) \n# plot the confusion matrix\nplot_confusion_matrix(confusion_gnb, classes = range(10)) ","metadata":{"executionInfo":{"elapsed":781,"status":"ok","timestamp":1648584199503,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"d265f468","outputId":"931e78d3-6967-4853-875b-139b54c178b9","execution":{"iopub.status.busy":"2022-03-29T23:22:33.619957Z","iopub.execute_input":"2022-03-29T23:22:33.620589Z","iopub.status.idle":"2022-03-29T23:22:34.412022Z","shell.execute_reply.started":"2022-03-29T23:22:33.620539Z","shell.execute_reply":"2022-03-29T23:22:34.411212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(classification_report(Y_val, nb_pred))","metadata":{"executionInfo":{"elapsed":7,"status":"ok","timestamp":1648584199504,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"c3290383","outputId":"dcf07a41-5f9c-4a0a-9bcf-ece4da940b21","execution":{"iopub.status.busy":"2022-03-29T23:22:34.413587Z","iopub.execute_input":"2022-03-29T23:22:34.414082Z","iopub.status.idle":"2022-03-29T23:22:34.459799Z","shell.execute_reply.started":"2022-03-29T23:22:34.414040Z","shell.execute_reply":"2022-03-29T23:22:34.458941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Accuracy of Gaussian Naive Bayes classifier is 0.58 which is very low. Reason for low accuracy might be due to the fact that GNB assumes features to be normally distributed. ","metadata":{"id":"5cd2e17c"}},{"cell_type":"markdown","source":"#### Naive Bayes Classifiers:\n\n**(1) Bernoulli Naive Bayes** : It assumes that all our features are binary such that they take only two values. Means 0s can represent “word does not occur in the document” and 1s as \"word occurs in the document\" .\n\n**(2) Multinomial Naive Bayes** : Its is used when we have discrete data (e.g. movie ratings ranging 1 and 5 as each rating will have certain frequency to represent). In text learning we have the count of each word to predict the class or label.\n\n**(3) Gaussian Naive Bayes** : Because of the assumption of the normal distribution, Gaussian Naive Bayes is used in cases when all our features are continuous. For example in Iris dataset features are sepal width, petal width, sepal length, petal length. So its features can have different values in data set as width and length can vary. We can’t represent features in terms of their occurrences. This means data is continuous. Hence we use Gaussian Naive Bayes here.\n\n```Based on the above information about the different classifiers, MultiNominal Naive Bayes might be appropriate to predict discrete data (digits 0-9) ```\n\nExternal Reference - https://www.quora.com/What-is-the-difference-between-the-the-Gaussian-Bernoulli-Multinomial-and-the-regular-Naive-Bayes-algorithms\n","metadata":{"id":"e43ee537"}},{"cell_type":"markdown","source":"# Multinomial Naive Bayes","metadata":{"id":"d54588de"}},{"cell_type":"code","source":"\nstart = timeit.default_timer() # start time\n\nclf_mnb = MultinomialNB()\nclf_mnb.fit(X_train, Y_train)\n\nstop = timeit.default_timer() # end time\n\nexecution_time = stop-start\nprint(f'execution time of multinominal naive bayes classifier is {str(execution_time)} seconds')  ","metadata":{"executionInfo":{"elapsed":781,"status":"ok","timestamp":1648584200281,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"7647cf7e","outputId":"98436fed-41bc-43ef-871b-22756dc8fd09","execution":{"iopub.status.busy":"2022-03-29T23:22:34.461423Z","iopub.execute_input":"2022-03-29T23:22:34.461923Z","iopub.status.idle":"2022-03-29T23:22:35.057517Z","shell.execute_reply.started":"2022-03-29T23:22:34.461867Z","shell.execute_reply":"2022-03-29T23:22:35.056593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_mnb = clf_mnb.predict(X_val)\npred_mnb","metadata":{"execution":{"iopub.status.busy":"2022-03-29T23:22:35.059070Z","iopub.execute_input":"2022-03-29T23:22:35.059563Z","iopub.status.idle":"2022-03-29T23:22:35.172597Z","shell.execute_reply.started":"2022-03-29T23:22:35.059519Z","shell.execute_reply":"2022-03-29T23:22:35.171729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# validating testing data\ntab_mnb = pd.crosstab(Y_val, pred_mnb, rownames=['Actual'],colnames=['Predicted'])\ntab_mnb","metadata":{"executionInfo":{"elapsed":10,"status":"ok","timestamp":1648584200282,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"6e9adc72","outputId":"1556858c-1aef-4ee8-e55e-c56791379fc7","execution":{"iopub.status.busy":"2022-03-29T23:22:35.174117Z","iopub.execute_input":"2022-03-29T23:22:35.174629Z","iopub.status.idle":"2022-03-29T23:22:35.220038Z","shell.execute_reply.started":"2022-03-29T23:22:35.174584Z","shell.execute_reply":"2022-03-29T23:22:35.218949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# compute the confusion matrix\nconfusion_mnb = confusion_matrix(Y_val, pred_mnb) \n# plot the confusion matrix\nplot_confusion_matrix(confusion_mnb, classes = range(10)) ","metadata":{"executionInfo":{"elapsed":977,"status":"ok","timestamp":1648584201251,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"7f75fcad","outputId":"496e2556-ac12-47ea-9e4e-10902ed86842","execution":{"iopub.status.busy":"2022-03-29T23:22:35.221897Z","iopub.execute_input":"2022-03-29T23:22:35.222468Z","iopub.status.idle":"2022-03-29T23:22:36.023349Z","shell.execute_reply.started":"2022-03-29T23:22:35.222421Z","shell.execute_reply":"2022-03-29T23:22:36.022521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.heatmap(tab_mnb)","metadata":{"executionInfo":{"elapsed":764,"status":"ok","timestamp":1648584202011,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"a0ce8594","outputId":"d3878c11-c466-4777-e947-a79f55db91fa","execution":{"iopub.status.busy":"2022-03-29T23:22:36.024976Z","iopub.execute_input":"2022-03-29T23:22:36.025281Z","iopub.status.idle":"2022-03-29T23:22:36.597643Z","shell.execute_reply.started":"2022-03-29T23:22:36.025241Z","shell.execute_reply":"2022-03-29T23:22:36.596717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(classification_report(Y_val, pred_mnb))","metadata":{"executionInfo":{"elapsed":177,"status":"ok","timestamp":1648584202181,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"b96a0ea7","outputId":"92eb9595-b90f-40f6-897c-f4276aebf115","execution":{"iopub.status.busy":"2022-03-29T23:22:36.598886Z","iopub.execute_input":"2022-03-29T23:22:36.599121Z","iopub.status.idle":"2022-03-29T23:22:36.646872Z","shell.execute_reply.started":"2022-03-29T23:22:36.599094Z","shell.execute_reply":"2022-03-29T23:22:36.645964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### 0.83 accuracy for Multinominal Naive Bayes Classifier","metadata":{"id":"8de1f566"}},{"cell_type":"markdown","source":"# Decision Trees","metadata":{"id":"120c8c01"}},{"cell_type":"code","source":"# training the model\nclf = DecisionTreeClassifier(random_state=0, criterion='gini') # entropy to measure the purity of the split \nclf.fit(X_train,Y_train)","metadata":{"executionInfo":{"elapsed":22240,"status":"ok","timestamp":1648584224416,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"4bfb3d08","outputId":"b12e0722-b82c-45d4-cdda-45bc2f4f9769","execution":{"iopub.status.busy":"2022-03-29T23:22:36.648393Z","iopub.execute_input":"2022-03-29T23:22:36.648779Z","iopub.status.idle":"2022-03-29T23:22:43.707059Z","shell.execute_reply.started":"2022-03-29T23:22:36.648733Z","shell.execute_reply":"2022-03-29T23:22:43.705872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predicting on test data\ndt_pred=clf.predict(X_val)\ndt_pred","metadata":{"executionInfo":{"elapsed":14,"status":"ok","timestamp":1648584224417,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"6b538294","outputId":"f0abf713-ff5c-4b45-afdd-f3187682826f","execution":{"iopub.status.busy":"2022-03-29T23:22:43.710857Z","iopub.execute_input":"2022-03-29T23:22:43.711127Z","iopub.status.idle":"2022-03-29T23:22:43.770619Z","shell.execute_reply.started":"2022-03-29T23:22:43.711096Z","shell.execute_reply":"2022-03-29T23:22:43.769931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# validating testing data\ntab_dt = pd.crosstab(Y_val, dt_pred, rownames=['Actual'],colnames=['Predicted'])\ntab_dt","metadata":{"executionInfo":{"elapsed":206,"status":"ok","timestamp":1648584224616,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"ed531558","outputId":"90e7b66d-fc8d-4f5b-f3e5-a38dc3d325ad","execution":{"iopub.status.busy":"2022-03-29T23:22:43.771704Z","iopub.execute_input":"2022-03-29T23:22:43.772401Z","iopub.status.idle":"2022-03-29T23:22:43.797353Z","shell.execute_reply.started":"2022-03-29T23:22:43.772366Z","shell.execute_reply":"2022-03-29T23:22:43.796527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.heatmap(tab_dt)","metadata":{"executionInfo":{"elapsed":382,"status":"ok","timestamp":1648584224983,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"48e2e9b4","outputId":"42ecdb1d-13dd-4c64-ba66-8113e0db775e","execution":{"iopub.status.busy":"2022-03-29T23:22:43.798615Z","iopub.execute_input":"2022-03-29T23:22:43.798838Z","iopub.status.idle":"2022-03-29T23:22:44.078887Z","shell.execute_reply.started":"2022-03-29T23:22:43.798811Z","shell.execute_reply":"2022-03-29T23:22:44.078032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# compute the confusion matrix\nconfusion_dt = confusion_matrix(Y_val, dt_pred) \n# plot the confusion matrix\nplot_confusion_matrix(confusion_dt, classes = range(10)) ","metadata":{"executionInfo":{"elapsed":1410,"status":"ok","timestamp":1648584226385,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"589ab540","outputId":"7b6a1846-7679-4c7d-940f-789d9c979eb5","execution":{"iopub.status.busy":"2022-03-29T23:22:44.080073Z","iopub.execute_input":"2022-03-29T23:22:44.080313Z","iopub.status.idle":"2022-03-29T23:22:44.868624Z","shell.execute_reply.started":"2022-03-29T23:22:44.080285Z","shell.execute_reply":"2022-03-29T23:22:44.867784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(classification_report(Y_val, dt_pred))","metadata":{"executionInfo":{"elapsed":9,"status":"ok","timestamp":1648584226386,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"566f439a","outputId":"2d4c07f8-fa9d-4036-d43f-7bc7adc9432c","execution":{"iopub.status.busy":"2022-03-29T23:22:44.869697Z","iopub.execute_input":"2022-03-29T23:22:44.869908Z","iopub.status.idle":"2022-03-29T23:22:44.914820Z","shell.execute_reply.started":"2022-03-29T23:22:44.869882Z","shell.execute_reply":"2022-03-29T23:22:44.913880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### 0.85 accuracy for decision tree classifier","metadata":{"id":"0438db1c"}},{"cell_type":"markdown","source":"#### MULTI FOLD CROSS VALIDATION","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import KFold, cross_val_score\n\n\n# 3 folds\nsplits = 3\nkf =KFold(n_splits=splits, shuffle=True, random_state=42)\n\ntotal = 0\n# split()  method generate indices to split data into training and test set.\nfor train_index, test_index in kf.split(X_train, Y_train):\n    print(f'Fold:{total}, Train set: {len(train_index)}, Test set:{len(test_index)}')\n    total += 1","metadata":{"execution":{"iopub.status.busy":"2022-03-29T23:22:44.915978Z","iopub.execute_input":"2022-03-29T23:22:44.916233Z","iopub.status.idle":"2022-03-29T23:22:44.926808Z","shell.execute_reply.started":"2022-03-29T23:22:44.916201Z","shell.execute_reply":"2022-03-29T23:22:44.925871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#### Model Score Using KFold\nscore = cross_val_score(DecisionTreeClassifier(random_state= 42), X_train, Y_train, cv= kf, scoring=\"accuracy\")\nprint(f'Scores (accuracy) for each fold are: {score}')\nprint(f'Average score: {\"{:.2f}\".format(score.mean())}')","metadata":{"execution":{"iopub.status.busy":"2022-03-29T23:22:44.927949Z","iopub.execute_input":"2022-03-29T23:22:44.928800Z","iopub.status.idle":"2022-03-29T23:22:57.792493Z","shell.execute_reply.started":"2022-03-29T23:22:44.928755Z","shell.execute_reply":"2022-03-29T23:22:57.791585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Hyperparameter Tuning\nHere we are going to do tuning based on ‘max_depth’. We will try with max depth starting from 1 to 10 and depending on the final ‘accuracy’ score choose the value of max_depth.","metadata":{}},{"cell_type":"code","source":"max_depth = [1,2,3,4,5,6,7,8,9,10,11,15,20,25,30]\n\nfor val in max_depth:\n    score = cross_val_score(DecisionTreeClassifier(max_depth= val, random_state= 42), X_train, Y_train, cv= kf, scoring=\"accuracy\")\n    print(f'Average score({val}): {\"{:.3f}\".format(score.mean())}')","metadata":{"execution":{"iopub.status.busy":"2022-03-29T23:22:57.793865Z","iopub.execute_input":"2022-03-29T23:22:57.796395Z","iopub.status.idle":"2022-03-29T23:24:38.025004Z","shell.execute_reply.started":"2022-03-29T23:22:57.796349Z","shell.execute_reply":"2022-03-29T23:24:38.024093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"max_depth = 15 seems to be optimal from the above results","metadata":{}},{"cell_type":"markdown","source":"# Decision Tree with max_depth=15 and gini impurity","metadata":{}},{"cell_type":"code","source":"# training the model\nstart = timeit.default_timer() # start time\n\nclf = DecisionTreeClassifier(random_state=0, criterion='gini', max_depth=15) # gini to measure the purity of the split \nclf.fit(X_train,Y_train)\n\nstop = timeit.default_timer() # end time\n\nexecution_time = stop-start\nprint(f'execution time of decision tree classifier classifier is {str(execution_time)} seconds')  \n\n# predicting on test data\ndt_pred=clf.predict(X_val)\ndt_pred\n\n# validating testing data\ntab_dt = pd.crosstab(Y_val, dt_pred, rownames=['Actual'],colnames=['Predicted'])\ntab_dt\n\n# compute the confusion matrix\nconfusion_dt = confusion_matrix(Y_val, dt_pred) \n# plot the confusion matrix\nplot_confusion_matrix(confusion_dt, classes = range(10)) \n\nprint(classification_report(Y_val, dt_pred))","metadata":{"execution":{"iopub.status.busy":"2022-03-29T23:24:38.026541Z","iopub.execute_input":"2022-03-29T23:24:38.026772Z","iopub.status.idle":"2022-03-29T23:24:43.961190Z","shell.execute_reply.started":"2022-03-29T23:24:38.026745Z","shell.execute_reply":"2022-03-29T23:24:43.960334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Decision Tree classifier with gini impurity to measure the quality of the split and max_depth of 15 increased the accuracy of the model by 0.1. Accuracy after hyperparameter tuning is 0.86","metadata":{}},{"cell_type":"markdown","source":"# PREDICT TEST DATA USING DECISION TREE CLASSIFIER","metadata":{"id":"aeb6cc6b"}},{"cell_type":"code","source":"test_pred = clf.predict(test)\ntest_pred","metadata":{"executionInfo":{"elapsed":215,"status":"ok","timestamp":1648584226597,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"ef003e15","outputId":"ca5d7117-a027-4e92-ef8a-9c5ec5743004","execution":{"iopub.status.busy":"2022-03-29T23:24:43.962731Z","iopub.execute_input":"2022-03-29T23:24:43.963219Z","iopub.status.idle":"2022-03-29T23:24:44.058781Z","shell.execute_reply.started":"2022-03-29T23:24:43.963153Z","shell.execute_reply":"2022-03-29T23:24:44.057813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission  = pd.DataFrame({'ImageId':pd.Series(range(1,test.shape[0]+1)),'Label':test_pred})\nsubmission","metadata":{"executionInfo":{"elapsed":16,"status":"ok","timestamp":1648584226598,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"4aff59c8","outputId":"06331fc5-2ada-4880-8a24-03d52da3d554","execution":{"iopub.status.busy":"2022-03-29T23:24:44.060689Z","iopub.execute_input":"2022-03-29T23:24:44.061007Z","iopub.status.idle":"2022-03-29T23:24:44.074325Z","shell.execute_reply.started":"2022-03-29T23:24:44.060965Z","shell.execute_reply":"2022-03-29T23:24:44.073531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"decision_tree_mnist.csv\",index=False)","metadata":{"executionInfo":{"elapsed":15,"status":"ok","timestamp":1648584226599,"user":{"displayName":"Chaithra K.C","userId":"06590399226295405413"},"user_tz":240},"id":"e79ac9c5","execution":{"iopub.status.busy":"2022-03-29T23:24:44.076032Z","iopub.execute_input":"2022-03-29T23:24:44.076531Z","iopub.status.idle":"2022-03-29T23:24:44.136754Z","shell.execute_reply.started":"2022-03-29T23:24:44.076491Z","shell.execute_reply":"2022-03-29T23:24:44.135800Z"},"trusted":true},"execution_count":null,"outputs":[]}]}