{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-14T16:56:14.525561Z","iopub.execute_input":"2022-02-14T16:56:14.525874Z","iopub.status.idle":"2022-02-14T16:56:14.535980Z","shell.execute_reply.started":"2022-02-14T16:56:14.525839Z","shell.execute_reply":"2022-02-14T16:56:14.535241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Imports","metadata":{}},{"cell_type":"code","source":"import optuna\nimport torch\nimport cv2\n\nimport torch.optim as torch_optim\nimport torch.optim.lr_scheduler as torch_lr_scheduler\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport lightgbm as lgbm\nimport pandas as pd\nimport numpy as np\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom scipy import ndimage\n\nfrom keras.preprocessing.image import ImageDataGenerator\n\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.decomposition import PCA\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.svm import SVC\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.metrics import f1_score, precision_score, recall_score, accuracy_score, mean_squared_error\n\nfrom lightgbm import LGBMClassifier\nfrom catboost import Pool, CatBoostClassifier, cv, CatBoostRegressor\n\nfrom optuna.visualization.matplotlib import plot_param_importances\n\nfrom torch.utils.data import Dataset, DataLoader\nfrom tqdm import tqdm\n\n%matplotlib inline\ndata_path = '/kaggle/input/digit-recognizer/'\n\nnp.random.seed(42)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T16:56:15.152079Z","iopub.execute_input":"2022-02-14T16:56:15.153012Z","iopub.status.idle":"2022-02-14T16:56:25.253429Z","shell.execute_reply.started":"2022-02-14T16:56:15.152968Z","shell.execute_reply":"2022-02-14T16:56:25.252785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading Data","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(os.path.join(data_path, 'train.csv'))\nX, y = df.drop('label', axis=1).values, df['label'].values\n\nX = X.reshape(-1, 28, 28) / 255","metadata":{"execution":{"iopub.status.busy":"2022-02-14T16:56:25.255265Z","iopub.execute_input":"2022-02-14T16:56:25.255494Z","iopub.status.idle":"2022-02-14T16:56:29.672330Z","shell.execute_reply.started":"2022-02-14T16:56:25.255469Z","shell.execute_reply":"2022-02-14T16:56:29.671502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Visualization ","metadata":{}},{"cell_type":"code","source":"def plot_grid(X, y):\n    fig, axm = plt.subplots(nrows=5, ncols=5, figsize=(8, 10))\n\n    for i, axs in enumerate(axm):\n        for j, ax in enumerate(axs):\n            idx = i * 5 + j\n            ax.imshow(X[idx], cmap='gray')\n            ax.set_title(f'Digit {y[idx]}')\n            ax.axes.xaxis.set_visible(False)\n            ax.axes.yaxis.set_visible(False)\n\n    plt.show()\n\n    print(f'We have {X.shape[0]} training images with {X.shape[1:]} H*W.')\n    print(f'Value ranges for each image is between {X.min()} and {X.max()}.')\n    print(f'We have {len(np.unique(y))} classes: {list(np.unique(y))}.')\n    \nplot_grid(X, y)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T16:56:29.673963Z","iopub.execute_input":"2022-02-14T16:56:29.674274Z","iopub.status.idle":"2022-02-14T16:56:30.793349Z","shell.execute_reply.started":"2022-02-14T16:56:29.674235Z","shell.execute_reply":"2022-02-14T16:56:30.792357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_grid_for(X, y, y_val=0):\n    fig, axm = plt.subplots(nrows=5, ncols=5, figsize=(8, 10))\n\n    idx = 0\n    for i, axs in enumerate(axm):\n        for j, ax in enumerate(axs):\n            while y[idx] != y_val:\n                idx += 1\n            ax.imshow(X[idx], cmap='gray')\n            ax.set_title(f'Digit {y[idx]}')\n            ax.axes.xaxis.set_visible(False)\n            ax.axes.yaxis.set_visible(False)\n            idx += 1\n\n    plt.show()\n\n    print(f'We have {X.shape[0]} training images with {X.shape[1:]} H*W.')\n    print(f'Value ranges for each image is between {X.min()} and {X.max()}.')\n    print(f'We have {len(np.unique(y))} classes: {list(np.unique(y))}.')\n    \nplot_grid_for(X, y)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T16:56:30.795456Z","iopub.execute_input":"2022-02-14T16:56:30.795706Z","iopub.status.idle":"2022-02-14T16:56:32.222570Z","shell.execute_reply.started":"2022-02-14T16:56:30.795678Z","shell.execute_reply":"2022-02-14T16:56:32.221749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keys, counts = np.unique(y, return_counts=True)\n\nplt.bar(keys, counts)\n\nplt.xticks(range(0, 10))\nplt.title(\"Number of digit classes\")\nplt.ylabel(\"Count\")\nplt.xlabel(\"Label\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-14T16:56:32.223662Z","iopub.execute_input":"2022-02-14T16:56:32.224149Z","iopub.status.idle":"2022-02-14T16:56:32.425947Z","shell.execute_reply.started":"2022-02-14T16:56:32.224115Z","shell.execute_reply":"2022-02-14T16:56:32.425069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Modeling\n\n","metadata":{}},{"cell_type":"markdown","source":"### Baseline","metadata":{}},{"cell_type":"code","source":"(X_train, X_test, y_train, y_test) = train_test_split(X.reshape(-1, 28 * 28), y, train_size=0.8, stratify=y, random_state=42)\n\nlogreg = LogisticRegression(solver='liblinear')\n\nlogreg.fit(X_train, y_train)\ny_pred = logreg.predict(X_test)\n\nprint(f\"Accuracy score {accuracy_score(y_test, y_pred)}\")\nprint(f\"F1 score {f1_score(y_test, y_pred, average='weighted')}\")\nprint(f\"Precision score {precision_score(y_test, y_pred, average='weighted', zero_division=0)}\")\nprint(f\"Recall score {recall_score(y_test, y_pred, average='weighted', zero_division=0)}\")","metadata":{"execution":{"iopub.status.busy":"2022-02-14T16:02:02.078555Z","iopub.execute_input":"2022-02-14T16:02:02.078733Z","iopub.status.idle":"2022-02-14T16:02:51.324275Z","shell.execute_reply.started":"2022-02-14T16:02:02.078709Z","shell.execute_reply":"2022-02-14T16:02:51.321336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### SVC","metadata":{}},{"cell_type":"code","source":"(X_train, X_test, y_train, y_test) = train_test_split(X.reshape(-1, 28 * 28), y, train_size=0.8, stratify=y, random_state=42)\n\nsvc = SVC(kernel='rbf', C=10, random_state=42)\n\nsvc.fit(X_train, y_train)\ny_pred = svc.predict(X_test)\n\nprint(f\"Accuracy score {accuracy_score(y_test, y_pred)}\")\nprint(f\"F1 score {f1_score(y_test, y_pred, average='weighted')}\")\nprint(f\"Precision score {precision_score(y_test, y_pred, average='weighted', zero_division=0)}\")\nprint(f\"Recall score {recall_score(y_test, y_pred, average='weighted', zero_division=0)}\")","metadata":{"execution":{"iopub.status.busy":"2022-02-14T16:02:51.325692Z","iopub.execute_input":"2022-02-14T16:02:51.326110Z","iopub.status.idle":"2022-02-14T16:07:35.070013Z","shell.execute_reply.started":"2022-02-14T16:02:51.326057Z","shell.execute_reply":"2022-02-14T16:07:35.068668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### DigitsPCA","metadata":{}},{"cell_type":"code","source":"class DigitsPCA:\n    def __init__(self, n_components=10):\n        self.n_components = n_components\n        self.pca = dict()\n\n    def fit(self, X_train, y_train):\n        for i in np.unique(y_train):\n            X_i = X_train[y_train == i]\n            self.pca[i] = PCA(n_components=self.n_components).fit(X_i)\n\n        return self\n\n    def score(self, X_test, y_test):\n        predictions = self.predict(X_test)\n\n        return accuracy_score(y_test, predictions)\n    \n    def predict(self, X):\n        predictions = []\n        for X_i in X:\n            X_i = X_i.reshape(1, -1)\n            reconstructed = {\n                i: mean_squared_error(\n                    X_i, self.pca[i].inverse_transform(self.pca[i].transform(X_i))\n                )\n                for i in self.pca\n            }\n            predictions.append(min(reconstructed, key=reconstructed.get))\n        return np.array(predictions)\n        \n    @property\n    def explained_variance_ratio_(self):\n        return [self.pca[i].explained_variance_ratio_ for i in self.pca]\n\n    def __repr__(self):\n        return f\"DigitsPCA(n_components={self.n_components})\"\n\n    def print_explained_variance_ratio(self):\n        for i, var in zip(self.pca, self.explained_variance_ratio_):\n            var_total = var.sum()\n            print(f'[Digit {i}]: Explained variation per principal component: {var} with a total of {var_total} variance captured.')\n\n    def plot_explained_variance_ratio(self):\n        nlabels = len(self.pca)\n        fig, axs = plt.subplots(nrows=nlabels, ncols=1, figsize=(16,16))\n        for i, var, ax in zip(self.pca, self.explained_variance_ratio_, axs):\n            ax.bar(np.arange(self.n_components), var)\n            ax.set_xlabel('Principal components')\n            ax.set_ylabel('Variance')\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-14T16:56:32.427239Z","iopub.execute_input":"2022-02-14T16:56:32.427480Z","iopub.status.idle":"2022-02-14T16:56:32.442490Z","shell.execute_reply.started":"2022-02-14T16:56:32.427450Z","shell.execute_reply":"2022-02-14T16:56:32.441582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(X_train, X_test, y_train, y_test) = train_test_split(X.reshape(-1, 28 * 28), y, train_size=0.8, stratify=y, random_state=42)\n\npca = DigitsPCA(n_components=28)\n\npca.fit(X_train, y_train)\ny_pred = pca.predict(X_test)\n\nprint(f\"Accuracy score {accuracy_score(y_test, y_pred)}\")\nprint(f\"F1 score {f1_score(y_test, y_pred, average='weighted')}\")\nprint(f\"Precision score {precision_score(y_test, y_pred, average='weighted', zero_division=0)}\")\nprint(f\"Recall score {recall_score(y_test, y_pred, average='weighted', zero_division=0)}\")","metadata":{"execution":{"iopub.status.busy":"2022-02-14T16:07:35.089649Z","iopub.execute_input":"2022-02-14T16:07:35.089970Z","iopub.status.idle":"2022-02-14T16:08:31.718190Z","shell.execute_reply.started":"2022-02-14T16:07:35.089929Z","shell.execute_reply":"2022-02-14T16:08:31.717335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx = 0\nwhile y_test[idx] != 0:\n    idx += 1\n    \ninv = pca.pca[0].inverse_transform(pca.pca[0].transform(X_test.reshape(-1, 28*28)[idx].reshape(1, -1)))\n\nplt.imshow(X_test[idx].reshape(28, 28), cmap='gray')\nplt.show()\n\nplt.imshow(inv.reshape(28, 28), cmap='gray')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-14T16:08:31.723103Z","iopub.execute_input":"2022-02-14T16:08:31.723757Z","iopub.status.idle":"2022-02-14T16:08:32.070099Z","shell.execute_reply.started":"2022-02-14T16:08:31.723709Z","shell.execute_reply":"2022-02-14T16:08:32.069257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Filters","metadata":{}},{"cell_type":"code","source":"SOBEL_KERNEL = np.array([[-1, -2, -1], [0, 0, 0], [1, 2, 1]])","metadata":{"execution":{"iopub.status.busy":"2022-02-14T16:56:32.443778Z","iopub.execute_input":"2022-02-14T16:56:32.444108Z","iopub.status.idle":"2022-02-14T16:56:32.456365Z","shell.execute_reply.started":"2022-02-14T16:56:32.444066Z","shell.execute_reply":"2022-02-14T16:56:32.455599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_s1 = np.concatenate([cv2.filter2D(x, -1, SOBEL_KERNEL) for x in X.reshape(-1, 1, 28, 28)])\nX_s2 = np.concatenate([cv2.filter2D(x, -1, SOBEL_KERNEL.T) for x in X.reshape(-1, 1, 28, 28)])\nX_s = np.concatenate([X_s1.reshape(-1, 28*28), X_s2.reshape(-1, 28*28)], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T16:56:32.458025Z","iopub.execute_input":"2022-02-14T16:56:32.458546Z","iopub.status.idle":"2022-02-14T16:56:34.547326Z","shell.execute_reply.started":"2022-02-14T16:56:32.458484Z","shell.execute_reply":"2022-02-14T16:56:34.546561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(X_train, X_test, y_train, y_test) = train_test_split(X_s, y, train_size=0.8, stratify=y, random_state=42)\n\npca_f = DigitsPCA(n_components=56)\n\npca_f.fit(X_train, y_train)\ny_pred = pca_f.predict(X_test)\n\nprint(f\"Accuracy score {accuracy_score(y_test, y_pred)}\")\nprint(f\"F1 score {f1_score(y_test, y_pred, average='weighted')}\")\nprint(f\"Precision score {precision_score(y_test, y_pred, average='weighted', zero_division=0)}\")\nprint(f\"Recall score {recall_score(y_test, y_pred, average='weighted', zero_division=0)}\")","metadata":{"execution":{"iopub.status.busy":"2022-02-14T17:01:09.005256Z","iopub.execute_input":"2022-02-14T17:01:09.005612Z","iopub.status.idle":"2022-02-14T17:03:06.378066Z","shell.execute_reply.started":"2022-02-14T17:01:09.005577Z","shell.execute_reply":"2022-02-14T17:03:06.376896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(X_train, X_test, y_train, y_test) = train_test_split(X_s, y, train_size=0.8, stratify=y, random_state=42)\n\nsvc_f = SVC(kernel='rbf', C=10, random_state=42)\n\nsvc_f.fit(X_train, y_train)\ny_pred = svc_f.predict(X_test)\n\nprint(f\"Accuracy score {accuracy_score(y_test, y_pred)}\")\nprint(f\"F1 score {f1_score(y_test, y_pred, average='weighted')}\")\nprint(f\"Precision score {precision_score(y_test, y_pred, average='weighted', zero_division=0)}\")\nprint(f\"Recall score {recall_score(y_test, y_pred, average='weighted', zero_division=0)}\")","metadata":{"execution":{"iopub.status.busy":"2022-02-14T16:12:43.378518Z","iopub.execute_input":"2022-02-14T16:12:43.379247Z","iopub.status.idle":"2022-02-14T16:27:31.864151Z","shell.execute_reply.started":"2022-02-14T16:12:43.379179Z","shell.execute_reply":"2022-02-14T16:27:31.862952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"X_submission = pd.read_csv(os.path.join(data_path, 'test.csv')).values / 255\npredictions = svc.predict(X_submission)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T16:27:31.865416Z","iopub.execute_input":"2022-02-14T16:27:31.865677Z","iopub.status.idle":"2022-02-14T16:32:09.593591Z","shell.execute_reply.started":"2022-02-14T16:27:31.865651Z","shell.execute_reply":"2022-02-14T16:32:09.592934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_grid(X_submission.reshape(-1, 28, 28), predictions)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T16:32:09.595066Z","iopub.execute_input":"2022-02-14T16:32:09.595519Z","iopub.status.idle":"2022-02-14T16:32:10.559275Z","shell.execute_reply.started":"2022-02-14T16:32:09.595487Z","shell.execute_reply":"2022-02-14T16:32:10.558486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(os.path.join(data_path, 'sample_submission.csv'))\nsubmission['Label'] = predictions\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-02-14T16:32:10.560651Z","iopub.execute_input":"2022-02-14T16:32:10.561030Z","iopub.status.idle":"2022-02-14T16:32:10.630330Z","shell.execute_reply.started":"2022-02-14T16:32:10.560984Z","shell.execute_reply":"2022-02-14T16:32:10.629489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}