{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"#Author - Aishwarya Khatwani (ak3518)\n# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nfrom sklearn.svm import SVC\nfrom sklearn.decomposition import PCA\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import learning_curve\nfrom sklearn.metrics import accuracy_score, confusion_matrix\nimport seaborn as sns\nfrom sklearn.neural_network import MLPClassifier\nimport random\n\nrandom.seed(10)\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntrain_data.head()\nx = train_data.drop(labels=[\"label\"], axis=1).to_numpy()\ny = train_data[\"label\"].to_numpy()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pca = PCA(n_components = 32)\nx_train_pca = pca.fit_transform(x_train)\nx_test_pca = pca.transform(x_test)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def svm_model(x_train_data, y_train_data):\n    model = SVC(C=0.01,kernel = 'rbf')\n    model.probability = True\n    model.fit(x_train_pca, y_train)\n    \n    return model\n    \nsvm_model(x_train_pca, y_train)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_sizes, train_scores, test_scores, fit_times, _ = \\\n        learning_curve(SVC(C=0.01,kernel = 'rbf'), x_train_pca, y_train,\n                       return_times=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_, axes = plt.subplots(1, 2, figsize=(20, 5))\n\naxes[0].set_title(\"\")\n# axes[0].set_ylim(*ylim)\naxes[0].set_xlabel(\"Training examples\")\naxes[0].set_ylabel(\"Score\")\n\ntrain_scores_mean = np.mean(train_scores, axis=1)\ntrain_scores_std = np.std(train_scores, axis=1)\ntest_scores_mean = np.mean(test_scores, axis=1)\ntest_scores_std = np.std(test_scores, axis=1)\nfit_times_mean = np.mean(fit_times, axis=1)\nfit_times_std = np.std(fit_times, axis=1)\n\n# Plot learning curve\naxes[0].grid()\naxes[0].fill_between(train_sizes, train_scores_mean - train_scores_std,\n                     train_scores_mean + train_scores_std, alpha=0.1,\n                     color=\"r\")\naxes[1].fill_between(train_sizes, test_scores_mean - test_scores_std,\n                     test_scores_mean + test_scores_std, alpha=0.1,\n                     color=\"g\")\naxes[0].plot(train_sizes, train_scores_mean, 'o-', color=\"r\",\n             label=\"Training score\")\naxes[1].plot(train_sizes, test_scores_mean, 'o-', color=\"g\",\n             label=\"Cross-validation score\")\naxes[1].grid()\naxes[0].legend(loc=\"best\")\naxes[1].legend(loc=\"best\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_sizes, train_scores, test_scores, fit_times, _ = \\\n        learning_curve(SVC(C=0.01,kernel = 'rbf'), x_test_pca, y_test,\n                       return_times=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_, axes = plt.subplots(1, 2, figsize=(20, 5))\n\naxes[0].set_title(\"\")\n# axes[0].set_ylim(*ylim)\naxes[0].set_xlabel(\"Training examples\")\naxes[0].set_ylabel(\"Score\")\n\ntrain_scores_mean = np.mean(train_scores, axis=1)\ntrain_scores_std = np.std(train_scores, axis=1)\ntest_scores_mean = np.mean(test_scores, axis=1)\ntest_scores_std = np.std(test_scores, axis=1)\nfit_times_mean = np.mean(fit_times, axis=1)\nfit_times_std = np.std(fit_times, axis=1)\n\n# Plot learning curve\naxes[0].grid()\naxes[0].fill_between(train_sizes, train_scores_mean - train_scores_std,\n                     train_scores_mean + train_scores_std, alpha=0.1,\n                     color=\"r\")\naxes[1].fill_between(train_sizes, test_scores_mean - test_scores_std,\n                     test_scores_mean + test_scores_std, alpha=0.1,\n                     color=\"g\")\naxes[0].plot(train_sizes, train_scores_mean, 'o-', color=\"r\",\n             label=\"Training score\")\naxes[1].plot(train_sizes, test_scores_mean, 'o-', color=\"g\",\n             label=\"Cross-validation score\")\naxes[1].grid()\naxes[0].legend(loc=\"best\")\naxes[1].legend(loc=\"best\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = svm_model(x_train_pca, y_train)\npred = model.predict(x_test_pca)\nprint(pred)\nprint(y_test)\naccuracy_score(y_test, pred)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nprob = model.predict_proba(x_test_pca)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import math\nprint(prob)\nprint(pred)\nprint(y_test)\n\ncorrect = []\nincorrect = []\n\nfor i in range(len(y_test)):\n    if pred[i] == y_test[i]:\n        correct.append(i)\n    else:\n        incorrect.append(i)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Images of each class where probability of the class is too high**"},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(len(correct)):\n    if y_test[correct[i]] == 9:\n        correct[i], prob[correct[i]], pred[correct[i]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"j=0\nfor i in [8,23,20,15,4,6,1,29,22,93]:\n    \n    reshaped_image = x_test[i].reshape(28,28)\n    plt.subplot(5, 2, j+1)\n    j = j+1\n    plt.imshow(reshaped_image, cmap=plt.cm.Greys_r)\n    plt.axis('off')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Images near the decision boundary, where the probability for the correct class is slightly higher/lower than the other classes.**"},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(len(incorrect)):\n    if y_test[incorrect[i]] == 9:\n        incorrect[i], prob[incorrect[i]], pred[incorrect[i]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"j=0\nfor i in [1876,6836,343,49,744,741,501,83,580,1058]:\n    \n    reshaped_image = x_test[i].reshape(28,28)\n    plt.subplot(5, 2, j+1)\n    j = j+1\n    plt.imshow(reshaped_image, cmap=plt.cm.Greys_r)\n    plt.axis('off')\nplt.show()\n\nfor i in [1876,6836,343,49,744,741,501,83,580,1058]:\n    print(pred[i])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Plotting Confusion Matrix for SVM**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_confusion_mtx(mtx):\n    plt.figure(figsize=(12,6)); plt.title(\"Confusion matrix\")\n    sns.heatmap(mtx, annot=True, cmap=\"coolwarm\", fmt=\"d\");\n    plt.ylabel(\"True label\"); plt.xlabel(\"Predicted label\")\n\nplot_confusion_mtx(confusion_matrix(y_test, pred))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**MLP Classifier**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def mlp_classifier(x, y):\n    model = MLPClassifier(hidden_layer_sizes=(150,100,50), max_iter=300)\n    model.fit(x, y)\n    return model\nmlp_model = mlp_classifier(x_train_pca, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_sizes_mlp, train_scores_mlp, test_scores_mlp, fit_times_mlp, _ = \\\n        learning_curve(MLPClassifier(hidden_layer_sizes=(150,100,50), max_iter=300), x_train_pca, y_train,\n                       return_times=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_, axes = plt.subplots(1, 2, figsize=(20, 5))\n\naxes[0].set_title(\"\")\n# axes[0].set_ylim(*ylim)\naxes[0].set_xlabel(\"Training examples\")\naxes[0].set_ylabel(\"Score\")\n\ntrain_scores_mean = np.mean(train_scores_mlp, axis=1)\ntrain_scores_std = np.std(train_scores_mlp, axis=1)\ntest_scores_mean = np.mean(test_scores_mlp, axis=1)\ntest_scores_std = np.std(test_scores_mlp, axis=1)\nfit_times_mean = np.mean(fit_times_mlp, axis=1)\nfit_times_std = np.std(fit_times_mlp, axis=1)\n\n# Plot learning curve\naxes[0].grid()\naxes[0].fill_between(train_sizes_mlp, train_scores_mean - train_scores_std,\n                     train_scores_mean + train_scores_std, alpha=0.1,\n                     color=\"r\")\naxes[1].fill_between(train_sizes_mlp, test_scores_mean - test_scores_std,\n                     test_scores_mean + test_scores_std, alpha=0.1,\n                     color=\"g\")\naxes[0].plot(train_sizes_mlp, train_scores_mean, 'o-', color=\"r\",\n             label=\"Training score\")\naxes[1].plot(train_sizes_mlp, test_scores_mean, 'o-', color=\"g\",\n             label=\"Cross-validation score\")\naxes[1].grid()\naxes[0].legend(loc=\"best\")\naxes[1].legend(loc=\"best\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_sizes_mlp, train_scores_mlp, test_scores_mlp, fit_times_mlp, _ = \\\n        learning_curve(MLPClassifier(hidden_layer_sizes=(150,100,50), max_iter=300), x_test_pca, y_test,\n                       return_times=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_, axes = plt.subplots(1, 2, figsize=(20, 5))\n\naxes[0].set_title(\"\")\n# axes[0].set_ylim(*ylim)\naxes[0].set_xlabel(\"Training examples\")\naxes[0].set_ylabel(\"Score\")\n\ntrain_scores_mean = np.mean(train_scores_mlp, axis=1)\ntrain_scores_std = np.std(train_scores_mlp, axis=1)\ntest_scores_mean = np.mean(test_scores_mlp, axis=1)\ntest_scores_std = np.std(test_scores_mlp, axis=1)\nfit_times_mean = np.mean(fit_times_mlp, axis=1)\nfit_times_std = np.std(fit_times_mlp, axis=1)\n\n# Plot learning curve\naxes[0].grid()\naxes[0].fill_between(train_sizes_mlp, train_scores_mean - train_scores_std,\n                     train_scores_mean + train_scores_std, alpha=0.1,\n                     color=\"r\")\naxes[1].fill_between(train_sizes_mlp, test_scores_mean - test_scores_std,\n                     test_scores_mean + test_scores_std, alpha=0.1,\n                     color=\"g\")\naxes[0].plot(train_sizes_mlp, train_scores_mean, 'o-', color=\"r\",\n             label=\"Training score\")\naxes[1].plot(train_sizes_mlp, test_scores_mean, 'o-', color=\"g\",\n             label=\"Cross-validation score\")\naxes[1].grid()\naxes[0].legend(loc=\"best\")\naxes[1].legend(loc=\"best\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mlp_model.probability = True\nmlp_predict = mlp_model.predict(x_test_pca)\nmlp_prob = mlp_model.predict_proba(x_test_pca)\nprint(y_test)\naccuracy_score(y_test, mlp_predict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"correct_mlp =[]\nincorrect_mlp = []\nfor i in range(len(y_test)):\n    if y_test[i] == pred[i]:\n        correct_mlp.append(i)\n    else:\n        incorrect_mlp.append(i)\n        \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(len(correct_mlp)):\n    if y_test[correct_mlp[i]] == 9:\n        print(correct_mlp[i], mlp_prob[correct_mlp[i]], mlp_predict[correct_mlp[i]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"j=0\nfor i in [8,11,37,15,4,5,1,29,22,45]:\n    \n    reshaped_image = x_test[i].reshape(28,28)\n    plt.subplot(5, 2, j+1)\n    j = j+1\n    plt.imshow(reshaped_image, cmap=plt.cm.Greys_r)\n    plt.axis('off')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(len(incorrect_mlp)):\n    if y_test[i] == 9 and mlp_predict[i] !=9:\n        print(i, mlp_prob[i], mlp_predict[i])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"j=0\nfor i in [3664,689,562,28,358,241,395,121,592,231]:\n    \n    reshaped_image = x_test[i].reshape(28,28)\n    plt.subplot(5, 2, j+1)\n    j = j+1\n    plt.imshow(reshaped_image, cmap=plt.cm.Greys_r)\n    plt.axis('off')\nplt.show()\nfor i in [3664,689,562,28,358,241,395,121,592,231]:\n    print(mlp_predict[i])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_confusion_mtx(confusion_matrix(y_test, mlp_predict))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}