{"cells":[{"metadata":{},"cell_type":"markdown","source":"This is a learning notebook. Just want to type along when reading Chapter 3 of the book Hands-On Machine Learning cited below.\n<br>\nCredit of codes below: Aurlien Gron. 2017. Hands-On Machine Learning with Scikit-Learn and TensorFlow: Concepts, Tools, and Techniques to Build Intelligent Systems (1st. ed.). O’Reilly Media, Inc.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntest = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.shape)\nprint(test.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Explore data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Visualize some pictures\neg_digit_image = train.iloc[10000, 1:].values.reshape(28, 28)\neg_digit_label = train.iloc[10000, 0]\nplt.imshow(eg_digit_image, cmap = mpl.cm.binary, interpolation = 'nearest')\nplt.axis('off')\nplt.show()\nprint(eg_digit_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# shuffle train data\nfrom sklearn.model_selection import train_test_split\nx_train, x_valid, y_train, y_valid = train_test_split(train.iloc[:, 1:].values, train.iloc[:, 0].values, test_size = 0.2, random_state = 777)\nshuffle_index = np.random.permutation(x_train.shape[0])\nx_train, y_train = x_train[shuffle_index], y_train[shuffle_index]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Binary classification on digit 5","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train_5 = (y_train == 5)\ny_valid_5 = (y_valid == 5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.linear_model import SGDClassifier\nsgd_clf = SGDClassifier(random_state = 42)\nsgd_clf.fit(x_train, y_train_5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"some_digits = train[train['label'] == 5].drop(columns = 'label').values\nsgd_clf.predict(some_digits)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# DIY cross validation\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.base import clone","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"skfolds = StratifiedKFold(n_splits = 3, random_state = 42)\nfor train_index, test_index in skfolds.split(x_train, y_train_5):\n    clone_clf = clone(sgd_clf)\n    x_train_folds = x_train[train_index]\n    y_train_folds = y_train_5[train_index]\n    x_test_fold = x_train[test_index]\n    y_test_fold = y_train_5[test_index]\n    \n    clone_clf.fit(x_train_folds, y_train_folds)\n    y_pred = clone_clf.predict(x_test_fold)\n    n_correct = sum(y_pred == y_test_fold)\n    print(n_correct / len(y_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# use cross_val_score\nfrom sklearn.model_selection import cross_val_score\ncross_val_score(sgd_clf, x_train, y_train_5, cv = 3, scoring = 'accuracy')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# cross_val_predict, confusion_matrix, precision and recall\nfrom sklearn.model_selection import cross_val_predict\ny_train_pred = cross_val_predict(sgd_clf, x_train, y_train_5, cv = 3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nconfusion_matrix(y_train_5, y_train_pred)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from sklearn.metrics import precision_score, recall_score, f1_score\nprint('Precision: ', precision_score(y_train_5, y_train_pred))\nprint('Recall: ', recall_score(y_train_5, y_train_pred))\nprint('F1: ', f1_score(y_train_5, y_train_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"recall_score(y_train_5, y_train_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_scores = sgd_clf.decision_function([some_digits[0]])\ny_scores\nthreshold = 200000\ny_some_digits_pred = (y_scores > threshold)\ny_some_digits_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_scores = cross_val_predict(sgd_clf, x_train, y_train_5, cv = 3, method = 'decision_function')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import precision_recall_curve\nprecisions, recalls, thresholds = precision_recall_curve(y_train_5, y_scores)\n\ndef plot_precision_recall_vs_threshold(precisions, recalls, thresholds):\n    plt.plot(thresholds, precisions[:-1], 'b--', label = 'Precision')\n    plt.plot(thresholds, recalls[:-1], 'g-', label = 'Recall')\n    plt.xlabel('Threshold')\n    plt.legend(loc = 'center left')\n    plt.ylim([0, 1])\n\nplot_precision_recall_vs_threshold(precisions, recalls, thresholds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"precisions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(recalls, precisions)\nplt.ylabel('Precision')\nplt.xlabel('Recall')\nplt.ylim([0, 1])\nplt.xlim([0, 1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# roc curve\nfrom sklearn.metrics import roc_curve\nfpr, tpr, threshold = roc_curve(y_train_5, y_scores)\n\ndef plot_roc_curve(fpr, tpr, label = None):\n    plt.plot(fpr, tpr, linewidth = 2, label = label)\n    plt.plot([0, 1], [0, 1], 'k--')\n    plt.axis([0, 1, 0, 1])\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n\nplot_roc_curve(fpr, tpr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# AUC\nfrom sklearn.metrics import roc_auc_score\nroc_auc_score(y_train_5, y_scores)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# compare RF to SDG\nfrom sklearn.ensemble import RandomForestClassifier\nforest_clf = RandomForestClassifier(random_state = 42)\ny_probas_forest = cross_val_predict(forest_clf, x_train, y_train_5, cv = 3, method = 'predict_proba')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_scores_forest = y_probas_forest[:, 1]\nfpr_forest, tpr_forest, thresholds_forest = roc_curve(y_train_5, y_scores_forest)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(fpr, tpr, 'b:', label = 'SGD')\nplot_roc_curve(fpr_forest, tpr_forest, 'Random Forest')\nplt.legend(loc = 'lower right')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_label_forest = cross_val_predict(forest_clf, x_train, y_train_5, cv = 3)\nprint('AUC: ', roc_auc_score(y_train_5, y_scores_forest))\nprint('Precision: ', precision_score(y_train_5, y_label_forest))\nprint('Recall: ', recall_score(y_train_5, y_label_forest))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Multiclass classification","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# use binary classifier for multiclass classification - sklearn automatically detects this\nsgd_clf.fit(x_train, y_train)\nsgd_clf.predict([some_digits[0]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sgd_clf.decision_function([some_digits[0]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sgd_clf.classes_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# force SGD to use OvO strategy\nfrom sklearn.multiclass import OneVsOneClassifier\novo_clf = OneVsOneClassifier(SGDClassifier(random_state = 42))\novo_clf.fit(x_train, y_train)\novo_clf.predict([some_digits[0]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(ovo_clf.estimators_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# random forest\nforest_clf.fit(x_train, y_train)\nforest_clf.predict([some_digits[0]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"forest_clf.predict_proba([some_digits[0]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# evaluate the classifiers\ncross_val_score(sgd_clf, x_train, y_train, cv = 3, scoring = 'accuracy')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nscaler = StandardScaler()\nx_train_scaled = scaler.fit_transform(x_train.astype(np.float64))\ncross_val_score(sgd_clf, x_train_scaled, y_train, cv = 3, scoring = 'accuracy')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# error analysis\ny_train_pred = cross_val_predict(sgd_clf, x_train_scaled, y_train, cv = 3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"conf_mx = confusion_matrix(y_train, y_train_pred)\nconf_mx","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.matshow(conf_mx, cmap = plt.cm.gray)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"row_sums = conf_mx.sum(axis = 1, keepdims = True)\nnorm_conf_mx = conf_mx / row_sums","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.fill_diagonal(norm_conf_mx, 0)\nplt.matshow(norm_conf_mx, cmap = plt.cm.gray)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Multilabel classification","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.neighbors import KNeighborsClassifier\n\ny_train_large = (y_train >= 7)\ny_train_odd = (y_train % 2 == 1)\ny_multilabel = np.c_[y_train_large, y_train_odd]\n\nknn_clf = KNeighborsClassifier()\nknn_clf.fit(x_train, y_multilabel)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"knn_clf.predict([some_digits[0]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train_knn_pred = cross_val_predict(knn_clf, x_train, y_multilabel, cv = 3)\nf1_score(y_multilabel, y_train_knn_pred, average = 'macro')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f1_score(y_multilabel, y_train_knn_pred, average = 'weighted')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Multioutput classification","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"noise = np.random.randint(0, 100, (len(x_train), 784))\nx_train_mod = x_train + noise\nnoise = np.random.randint(0, 100, (len(x_valid), 784))\nx_valid_mod = x_valid + noise\ny_train_mod = x_train\ny_valid_mod = x_valid","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train_mod[0].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(x_train_mod[0].reshape(28, 28), cmap = mpl.cm.binary, interpolation = 'nearest')\nplt.axis('off')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(x_train[0].reshape(28, 28), cmap = mpl.cm.binary, interpolation = 'nearest')\nplt.axis('off')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(x_valid[0].reshape(28, 28), cmap = mpl.cm.binary, interpolation = 'nearest')\nplt.axis('off')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"knn_clf.fit(x_train_mod, y_train_mod)\nclean_digit = knn_clf.predict([x_valid_mod[0]])\nplt.imshow(clean_digit.reshape(28, 28), cmap = mpl.cm.binary, interpolation = 'nearest')\nplt.axis('off')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Based on what's covered in this chapter, I just want to create a really simple submission and serve as a benchmark for improvements in later submissions.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"xTrain = train.iloc[:, 1:]\nyTrain = train.iloc[:, 0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xTrainScaled = scaler.fit_transform(xTrain)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xTestScaled = scaler.transform(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"knn = KNeighborsClassifier(weights = 'distance', n_neighbors = 4)\nknn.fit(xTrainScaled, yTrain)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"testPred = knn.predict(xTestScaled)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result = pd.concat([pd.DataFrame(np.arange(0, len(test)) + 1), pd.DataFrame(testPred)], axis = 1)\nresult.columns = ['ImageId', 'Label']\nresult.to_csv('digit_recognizer_20200705.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}