{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"1. [Introduction](#introduction)\n1. [Data Collection](#datacollection)\n1. [Data Analysis](#dataanalysis)\n    1. [Explore Dataset](#exploredataset)\n    1. [Explore Target Variable](#exploretargetvariable)\n    1. [Data Visualization](#datavisualization)\n1. [Data Preprocessing](#datapreprocessing)\n    1. [PCA](#dppca)\n    1. [Deskew](#dpdeskew)\n    1. [Data Augmentation](#dpdataaugmentation)\n    1. [Pipeline](#dppipeline)\n1. [Model Training and Evaluation](#trainandevaluate)\n    1. [Estimate PCA params](#estimatepcaparams)\n    1. [Estimate Scalers](#estimatescalers)\n    1. [Estimate Deskew](#estimatedeskew)\n    1. [Estimate Different Models (default params)](#estimatedifferentmodels)\n    1. [Fine-tuning](#finetuning)\n         1. [SVC](#finetuningsvc)\n         1. [RandomForestClassifier](#finetuningrf)\n         1. [KNeighborsClassifier](#finetuningknn)\n         1. [XGBClassifier](#finetuningxgb)\n    1. [Estimations Results](#estimationsresults)\n    1. [Ensemble learning](#ensemblelearning)\n        1. [VotingClassifier](#votingclassifier)\n1. [Submission](#submission)\n1. [Conclusion](#conclusion)","metadata":{}},{"cell_type":"markdown","source":"<center>\n<div style=\"color:;\n           display:fill;\n           border-radius:5px;\n           background-color: lightgray;\n           font-size:110%;\n           letter-spacing:0.5px\">\n\n<h2 style=\"padding: 10px;\n              color:white;\">Kannada MNIST: Digits Recognizer\n</h2>\n</div>","metadata":{}},{"cell_type":"markdown","source":"# Introduction <a class=\"anchor\" id=\"introduction\"></a>","metadata":{}},{"cell_type":"markdown","source":"The goal of this competition is to provide a simple extension to the classic MNIST competition we're all familiar with. Instead of using Arabic numerals, it uses a recently-released dataset of Kannada digits.\n\nKannada is a language spoken predominantly by people of Karnataka in southwestern India. The language has roughly 45 million native speakers and is written using the Kannada script. Wikipedia\n\nThis notebook goal is to train a classifier (without using of neural networks) that can accurately recognize the digits in these images. Additionally I use here cuML library models which works with GPU.\n\nThis Notebook Covers:\n\n1. Exploratory Data Analysis\n1. Building preprocessing pipeline\n1. Data augmentation\n1. Comparison of different models' baseline scores\n1. Tuning best models\n1. Evaluation ensemble classifiers\n1. Final submission","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\nimport itertools\nimport functools\nimport re\nimport copy\nimport cudf\nimport cuml\nimport time\nimport cupy as cp\nimport xgboost as xgb\nimport lightgbm as lgb\nfrom mlxtend import classifier\nfrom sklearn.experimental import enable_halving_search_cv \nfrom sklearn.model_selection import train_test_split, GridSearchCV, HalvingGridSearchCV, cross_val_predict, learning_curve\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.linear_model import LogisticRegression, SGDClassifier\nfrom sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, AdaBoostClassifier, BaggingClassifier, VotingClassifier, ExtraTreesClassifier, StackingClassifier, HistGradientBoostingClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.neighbors import KNeighborsClassifier, RadiusNeighborsClassifier\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.svm import SVC\nfrom sklearn.preprocessing import MinMaxScaler, StandardScaler, Binarizer\nfrom sklearn.pipeline import Pipeline, make_pipeline\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.metrics import confusion_matrix, accuracy_score, classification_report, f1_score, precision_score, recall_score, make_scorer,roc_curve, precision_recall_curve\nfrom sklearn.decomposition import PCA\nfrom skopt import BayesSearchCV, space, plots\nfrom scipy.stats import randint\nfrom scipy.ndimage import interpolation\nfrom IPython.display import Markdown, display\nfrom matplotlib.colors import ListedColormap\nfrom sklearn.manifold import TSNE\nimport albumentations as A\nimport cv2\n\n# to make this notebook's output stable across runs\nnp.random.seed(42)\n\n%matplotlib inline\n# warnings.simplefilter(action='ignore', category=(RuntimeWarning, FutureWarning))\n\nON_KAGGLE = os.getenv(\"KAGGLE_KERNEL_RUN_TYPE\") is not None\n\nverbose_tuning = 1\n\ndef download_from_kaggle(competition):\n    command = \"kaggle competitions download -c \"\n    filepath = Path(\"data/\" + competition + \".zip\")\n\n    if not filepath.is_file():\n        subprocess.run((command + competition).split())\n        Path(\"data\").mkdir(parents=True, exist_ok=True)\n        shutil.unpack_archive(competition + \".zip\", \"data\")\n        shutil.move(competition + \".zip\", \"data\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-04-17T10:55:06.895999Z","iopub.execute_input":"2023-04-17T10:55:06.896645Z","iopub.status.idle":"2023-04-17T10:55:15.678841Z","shell.execute_reply.started":"2023-04-17T10:55:06.896613Z","shell.execute_reply":"2023-04-17T10:55:15.677813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Collection <a class=\"anchor\" id=\"datacollection\"></a>","metadata":{}},{"cell_type":"code","source":"competition = \"Kannada-MNIST\"\nif not ON_KAGGLE:\n    download_from_kaggle(competition)\n    train_path = \"data/train.csv\"\n    test_path = \"data/test.csv\"\nelse:\n    train_path = f\"/kaggle/input/{competition}/train.csv\"\n    test_path = f\"/kaggle/input/{competition}/test.csv\"\n\ntrain_df = pd.read_csv(train_path)\ntest_df = pd.read_csv(test_path, index_col='id')\ndigmnist_df = pd.read_csv(\"/kaggle/input/Kannada-MNIST/Dig-MNIST.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-04-17T10:55:16.589678Z","iopub.execute_input":"2023-04-17T10:55:16.590417Z","iopub.status.idle":"2023-04-17T10:55:21.932969Z","shell.execute_reply.started":"2023-04-17T10:55:16.590371Z","shell.execute_reply":"2023-04-17T10:55:21.931925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Analysis <a class=\"anchor\" id=\"dataanalysis\"></a>","metadata":{}},{"cell_type":"markdown","source":"## Explore Dataset <a class=\"anchor\" id=\"exploredataset\"></a>","metadata":{}},{"cell_type":"markdown","source":"Explore the Kannada MNIST dataset by analyzing its distribution, visualizing some of its samples.","metadata":{}},{"cell_type":"code","source":"print(f'Train dataset shape:\\t {train_df.shape}', \n      f'\\nTest dataset shape:\\t {test_df.shape}', \n      f'\\nDig-MNIST dataset shape: {digmnist_df.shape}')","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:21:25.967493Z","iopub.execute_input":"2023-04-17T09:21:25.967950Z","iopub.status.idle":"2023-04-17T09:21:25.974653Z","shell.execute_reply.started":"2023-04-17T09:21:25.967911Z","shell.execute_reply":"2023-04-17T09:21:25.973460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:21:25.976165Z","iopub.execute_input":"2023-04-17T09:21:25.977175Z","iopub.status.idle":"2023-04-17T09:21:26.022698Z","shell.execute_reply.started":"2023-04-17T09:21:25.977137Z","shell.execute_reply":"2023-04-17T09:21:26.021558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:21:26.025462Z","iopub.execute_input":"2023-04-17T09:21:26.026298Z","iopub.status.idle":"2023-04-17T09:21:26.044443Z","shell.execute_reply.started":"2023-04-17T09:21:26.026256Z","shell.execute_reply":"2023-04-17T09:21:26.043309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:21:26.045980Z","iopub.execute_input":"2023-04-17T09:21:26.046417Z","iopub.status.idle":"2023-04-17T09:21:26.063446Z","shell.execute_reply.started":"2023-04-17T09:21:26.046362Z","shell.execute_reply":"2023-04-17T09:21:26.062255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"digmnist_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:21:26.064734Z","iopub.execute_input":"2023-04-17T09:21:26.065752Z","iopub.status.idle":"2023-04-17T09:21:26.083902Z","shell.execute_reply.started":"2023-04-17T09:21:26.065714Z","shell.execute_reply":"2023-04-17T09:21:26.082900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Train dataset, missing values:', train_df.isna().sum().sum())\nprint('Test dataset, missing values:', test_df.isna().sum().sum())\nprint('Dig-MNIST dataset, missing values:', digmnist_df.isna().sum().sum())","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:21:26.085375Z","iopub.execute_input":"2023-04-17T09:21:26.085850Z","iopub.status.idle":"2023-04-17T09:21:26.165642Z","shell.execute_reply.started":"2023-04-17T09:21:26.085812Z","shell.execute_reply":"2023-04-17T09:21:26.164448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are no missing values in the train and test datasets. So there are no corrupted images","metadata":{}},{"cell_type":"markdown","source":"### Explore Target Variable <a class=\"anchor\" id=\"exploretargetvariable\"></a>","metadata":{}},{"cell_type":"code","source":"def countplot_target_var(df, label, title):\n    ax = sns.countplot(x=label, data=df)\n    ax.figure.suptitle(title)\n    for container in ax.containers:\n        ax.bar_label(container)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-04-17T09:21:26.167052Z","iopub.execute_input":"2023-04-17T09:21:26.168043Z","iopub.status.idle":"2023-04-17T09:21:26.174564Z","shell.execute_reply.started":"2023-04-17T09:21:26.168004Z","shell.execute_reply":"2023-04-17T09:21:26.173187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"countplot_target_var(train_df, 'label', 'Digits count')","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:21:26.176041Z","iopub.execute_input":"2023-04-17T09:21:26.177895Z","iopub.status.idle":"2023-04-17T09:21:26.451604Z","shell.execute_reply.started":"2023-04-17T09:21:26.177839Z","shell.execute_reply":"2023-04-17T09:21:26.450429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data Visualization <a class=\"anchor\" id=\"datavisualization\"></a>","metadata":{}},{"cell_type":"code","source":"def plot_digits(instances, actual_labels, predicted_labels, images_per_row=10, figsize=(8,8), title_first = False, only_label = False, **options):\n    size = 28\n    images_per_row = min(len(instances), images_per_row)\n    n_rows = (len(instances) - 1) // images_per_row + 1\n    n_empty = n_rows * images_per_row - len(instances)\n    padded_instances = np.concatenate([instances, np.zeros((n_empty, size * size))], axis=0)\n    fig = plt.figure(figsize=figsize)\n    for i in range(len(padded_instances)):\n        ax = fig.add_subplot(n_rows, images_per_row, i + 1)\n        grid_data = padded_instances[i].reshape(28,28)\n        plt.imshow(grid_data, cmap = mpl.cm.binary, interpolation=\"nearest\")\n        ax.get_xaxis().set_ticks([])\n        ax.get_yaxis().set_ticks([])\n        ax.patch.set_linewidth('1')\n        ax.spines['bottom'].set_color('lightgray')\n        ax.spines['top'].set_color('lightgray') \n        ax.spines['right'].set_color('lightgray')\n        ax.spines['left'].set_color('lightgray')\n        fig.tight_layout()\n        if title_first and i >= images_per_row:\n            continue\n        prefix_label_actual = '' if only_label else 'Actual: '\n        prefix_label_pred = '' if only_label else 'Predicted: '\n        actual_title = f'{prefix_label_actual}{actual_labels[i]}' if len(actual_labels) else ''\n        predicted_title = f'\\n{prefix_label_pred}{predicted_labels[i]}' if len(predicted_labels) else ''\n        ax.set_title(f'{actual_title}{predicted_title}')\n        ","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-04-17T10:56:48.158309Z","iopub.execute_input":"2023-04-17T10:56:48.159280Z","iopub.status.idle":"2023-04-17T10:56:48.170936Z","shell.execute_reply.started":"2023-04-17T10:56:48.159221Z","shell.execute_reply":"2023-04-17T10:56:48.169816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Plot random digits","metadata":{}},{"cell_type":"code","source":"def plot_examples(df, class_count):\n    examples = pd.DataFrame()\n    for label in range(10):\n        label_examples = df[df['label'] == label][:class_count]\n        examples = label_examples if examples.empty else pd.concat([examples, label_examples])\n    example_images = examples.iloc[:, 1:785].to_numpy().reshape(10, class_count, 784).transpose(1, 0, 2).reshape(class_count*10, 784)\n    plot_digits(example_images, range(10), [], images_per_row=10, title_first=True, only_label=True, figsize=[8,4])","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-04-17T10:55:25.677590Z","iopub.execute_input":"2023-04-17T10:55:25.678193Z","iopub.status.idle":"2023-04-17T10:55:25.690217Z","shell.execute_reply.started":"2023-04-17T10:55:25.678147Z","shell.execute_reply":"2023-04-17T10:55:25.688789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_rand_examples(df, prop_slice=slice(1,785), size=49, images_per_row=7):\n    exmaple_indexes = randint.rvs(0, len(df)-1, size=size)\n    example_images = df.iloc[exmaple_indexes, prop_slice].to_numpy()\n    plot_digits(example_images, [], [], images_per_row=images_per_row, figsize=[8,4])","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:21:26.485146Z","iopub.execute_input":"2023-04-17T09:21:26.485424Z","iopub.status.idle":"2023-04-17T09:21:26.496114Z","shell.execute_reply.started":"2023-04-17T09:21:26.485376Z","shell.execute_reply":"2023-04-17T09:21:26.494966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(Markdown('#### Train examples'))\nplot_examples(train_df, 5)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:21:26.497340Z","iopub.execute_input":"2023-04-17T09:21:26.498285Z","iopub.status.idle":"2023-04-17T09:21:33.190506Z","shell.execute_reply.started":"2023-04-17T09:21:26.498250Z","shell.execute_reply":"2023-04-17T09:21:33.189375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(Markdown('#### Test examples'))\nplot_rand_examples(test_df, slice(0,784), 50, 10)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:21:33.192047Z","iopub.execute_input":"2023-04-17T09:21:33.192417Z","iopub.status.idle":"2023-04-17T09:21:39.422595Z","shell.execute_reply.started":"2023-04-17T09:21:33.192362Z","shell.execute_reply":"2023-04-17T09:21:39.421436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(Markdown('#### Dig_MNIST examples'))\nplot_examples(digmnist_df, 5)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:21:39.424154Z","iopub.execute_input":"2023-04-17T09:21:39.424540Z","iopub.status.idle":"2023-04-17T09:21:45.890365Z","shell.execute_reply.started":"2023-04-17T09:21:39.424502Z","shell.execute_reply":"2023-04-17T09:21:45.889350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### t-SNE","metadata":{}},{"cell_type":"code","source":"def plot_tsne(df, labels, title):\n    pca_tsne_pipe = Pipeline([\n        (\"pca\", cuml.decomposition.PCA(n_components=100, random_state=42)),\n        (\"tsne\", cuml.TSNE(n_components=2, random_state=42)),\n    ])\n    data_reduced = pca_tsne_pipe.fit_transform(df)\n    plt.scatter(data_reduced[:, 0], data_reduced[:, 1], s=5, c=labels, cmap='Spectral')\n    if isinstance(labels, pd.Series):\n        plt.colorbar(boundaries=np.arange(11)-0.5).set_ticks(np.arange(10))\n    plt.title(title, fontsize=14)\n    plt.axis('off')\n    plt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-04-17T09:21:45.892302Z","iopub.execute_input":"2023-04-17T09:21:45.892708Z","iopub.status.idle":"2023-04-17T09:21:45.899907Z","shell.execute_reply.started":"2023-04-17T09:21:45.892669Z","shell.execute_reply":"2023-04-17T09:21:45.898878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_tsne(train_df.drop(\"label\", axis=1).astype('float32'), \n          train_df.loc[:, 'label'], \n          'Visualizing MNIST through t-SNE (train_data)'\n         )","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:21:45.901419Z","iopub.execute_input":"2023-04-17T09:21:45.902038Z","iopub.status.idle":"2023-04-17T09:21:57.901803Z","shell.execute_reply.started":"2023-04-17T09:21:45.901997Z","shell.execute_reply":"2023-04-17T09:21:57.900819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see that the classes of objects are well separated from each other in the visualization, which indicates the high quality of the data and the possibility of using machine learning models for classification tasks. Some objects belonging to different classes may be close to each other, which may indicate the difficulty in separating them by machine learning models.","metadata":{}},{"cell_type":"code","source":"plot_tsne(digmnist_df.drop(\"label\", axis=1).astype('float32'), \n          digmnist_df.loc[:, 'label'], \n          'Visualizing MNIST through t-SNE (digmnist_df)'\n         )","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:21:57.902973Z","iopub.execute_input":"2023-04-17T09:21:57.903318Z","iopub.status.idle":"2023-04-17T09:21:59.932631Z","shell.execute_reply.started":"2023-04-17T09:21:57.903283Z","shell.execute_reply":"2023-04-17T09:21:59.931598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_tsne(test_df.astype('float32'), \n          None, \n          'Visualizing MNIST through t-SNE (test data)'\n         )","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:21:59.934028Z","iopub.execute_input":"2023-04-17T09:21:59.939117Z","iopub.status.idle":"2023-04-17T09:22:01.368894Z","shell.execute_reply.started":"2023-04-17T09:21:59.939086Z","shell.execute_reply":"2023-04-17T09:22:01.367481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_merged_df = pd.concat([train_df, digmnist_df])","metadata":{"execution":{"iopub.status.busy":"2023-04-17T10:55:36.717738Z","iopub.execute_input":"2023-04-17T10:55:36.718456Z","iopub.status.idle":"2023-04-17T10:55:36.855910Z","shell.execute_reply.started":"2023-04-17T10:55:36.718416Z","shell.execute_reply":"2023-04-17T10:55:36.854872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_tsne(train_merged_df.drop(\"label\", axis=1).astype('float32'), \n          train_merged_df.loc[:, 'label'], \n          'Visualizing MNIST through t-SNE (train_merged_df)'\n         )","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:22:01.532348Z","iopub.execute_input":"2023-04-17T09:22:01.532780Z","iopub.status.idle":"2023-04-17T09:22:06.206605Z","shell.execute_reply.started":"2023-04-17T09:22:01.532741Z","shell.execute_reply":"2023-04-17T09:22:06.205594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_merged_df.info()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:22:06.208170Z","iopub.execute_input":"2023-04-17T09:22:06.208984Z","iopub.status.idle":"2023-04-17T09:22:06.250640Z","shell.execute_reply.started":"2023-04-17T09:22:06.208946Z","shell.execute_reply":"2023-04-17T09:22:06.249576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Preprocessing <a class=\"anchor\" id=\"datapreprocessing\"></a>","metadata":{}},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(train_merged_df.drop(\"label\", axis=1), \n                                                  train_merged_df[\"label\"], \n                                                  test_size=0.20, \n                                                  stratify = train_merged_df[\"label\"], \n                                                  shuffle=True, random_state=42)\nX_train = X_train.astype('float32')\nX_val = X_val.astype('float32')\ny_train = y_train.to_numpy()\ny_val = y_val.to_numpy()\nprint(f'Train shape. \\t\\tX: {X_train.shape}, y: {y_train.shape}',\n    f'\\nValidation shape. \\tX: {X_val.shape}, y: {y_val.shape}')","metadata":{"execution":{"iopub.status.busy":"2023-04-17T10:55:40.514944Z","iopub.execute_input":"2023-04-17T10:55:40.515817Z","iopub.status.idle":"2023-04-17T10:55:41.176693Z","shell.execute_reply.started":"2023-04-17T10:55:40.515747Z","shell.execute_reply":"2023-04-17T10:55:41.175523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## PCA <a class=\"anchor\" id=\"dppca\"></a>","metadata":{}},{"cell_type":"code","source":"def plot_pca(pca, n_comp_example, ax, title):\n    ax.plot(np.cumsum(pca.explained_variance_ratio_))\n    cev_example = np.cumsum(pca.explained_variance_ratio_[:n_comp_example+1])[n_comp_example]\n    ax.hlines(y=cev_example, xmin=0, xmax=n_comp_example, color='r', ls='--', lw=1)\n    ax.vlines(x=n_comp_example, ymin=0, ymax=cev_example, color='r', ls='--', lw=1)\n    ax.plot([n_comp_example], [cev_example], \"ro\")\n    ax.set_xticks(list(ax.get_xticks()) + [n_comp_example])\n    ax.set_yticks(list(ax.get_yticks()) + [round(cev_example, 2)])\n    ax.set_xlim(0, pca.n_components+round(pca.n_components*0.05, 0))\n    ax.set_ylim(0, 1.05)\n    ax.set_xlabel('Number of components')\n    ax.set_ylabel('Cumulative explained variance')\n    ax.grid()\n    ax.set_title(title)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-04-17T09:22:06.944858Z","iopub.execute_input":"2023-04-17T09:22:06.945291Z","iopub.status.idle":"2023-04-17T09:22:06.953644Z","shell.execute_reply.started":"2023-04-17T09:22:06.945252Z","shell.execute_reply":"2023-04-17T09:22:06.952384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_pca_cumvar(comp_examplet=100):\n    f, (ax0, ax1) = plt.subplots(1, 2, figsize=(14,4))\n    X_train_scaled1 = cuml.preprocessing.MinMaxScaler().fit_transform(X_train)\n    X_train_scaled2 = cuml.preprocessing.StandardScaler().fit_transform(X_train)\n    pca1 = cuml.decomposition.PCA(n_components=X_train.shape[1]).fit(X_train_scaled1)\n    pca2 = cuml.decomposition.PCA(n_components=X_train.shape[1]).fit(X_train_scaled2)\n    plot_pca(pca1, comp_examplet, ax0, \"PCA with MinMaxScaler\")\n    plot_pca(pca2, comp_examplet, ax1, \"PCA with StandardScaler\")\n    plt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-04-17T09:22:06.955236Z","iopub.execute_input":"2023-04-17T09:22:06.956045Z","iopub.status.idle":"2023-04-17T09:22:06.965572Z","shell.execute_reply.started":"2023-04-17T09:22:06.955951Z","shell.execute_reply":"2023-04-17T09:22:06.964375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_pca_cumvar(150)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:22:06.967165Z","iopub.execute_input":"2023-04-17T09:22:06.967685Z","iopub.status.idle":"2023-04-17T09:22:21.148235Z","shell.execute_reply.started":"2023-04-17T09:22:06.967647Z","shell.execute_reply":"2023-04-17T09:22:21.147267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see that 150 components describe about 89% of the variance with MinMaxScaler. PCA with MinMaxScaler also performs better than StandardScaler. We will check the performance with the following models.","metadata":{}},{"cell_type":"markdown","source":"Let's see how the digits look like after the reconstruction.","metadata":{}},{"cell_type":"code","source":"def plot_pca_reconstruct(df):\n    random_indexes = randint.rvs(0, len(X_train)-1, size=8)\n    example_images = df.iloc[random_indexes]\n    pca = cuml.decomposition.PCA(n_components=100)\n    X_train_pca = pca.fit_transform(X_train)\n    X_train_pca_inverted = pca.inverse_transform(X_train_pca)\n    example_images_pca_inverted = X_train_pca_inverted[random_indexes]\n    images_to_plot = np.concatenate([example_images.to_numpy(), example_images_pca_inverted])\n    plot_digits(images_to_plot, [], [], images_per_row=8, figsize=(8,2))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-04-17T09:22:21.151437Z","iopub.execute_input":"2023-04-17T09:22:21.151729Z","iopub.status.idle":"2023-04-17T09:22:21.157840Z","shell.execute_reply.started":"2023-04-17T09:22:21.151701Z","shell.execute_reply":"2023-04-17T09:22:21.156851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_pca_reconstruct(X_train)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:22:21.166128Z","iopub.execute_input":"2023-04-17T09:22:21.166891Z","iopub.status.idle":"2023-04-17T09:22:22.529629Z","shell.execute_reply.started":"2023-04-17T09:22:21.166850Z","shell.execute_reply":"2023-04-17T09:22:22.528481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The quality of the reconstructed images has decreased, but the numbers are still recognizable. Of course, it depends on the number of components.","metadata":{}},{"cell_type":"markdown","source":"## Deskew <a class=\"anchor\" id=\"dpdeskew\"></a>","metadata":{}},{"cell_type":"markdown","source":"The deskew technique is effective for the standard MNIST dataset, so try deskewing digits and check the performance in the training section.","metadata":{}},{"cell_type":"code","source":"def moments(image):\n    c0,c1 = np.mgrid[:image.shape[0],:image.shape[1]] # A trick in numPy to create a mesh grid\n    totalImage = np.sum(image) #sum of pixels\n    m0 = np.sum(c0*image)/totalImage #mu_x\n    m1 = np.sum(c1*image)/totalImage #mu_y\n    m00 = np.sum((c0-m0)**2*image)/totalImage #var(x)\n    m11 = np.sum((c1-m1)**2*image)/totalImage #var(y)\n    m01 = np.sum((c0-m0)*(c1-m1)*image)/totalImage #covariance(x,y)\n    mu_vector = np.array([m0,m1]) # Notice that these are \\mu_x, \\mu_y respectively\n    covariance_matrix = np.array([[m00,m01],[m01,m11]]) # Do you see a similarity between the covariance matrix\n    return mu_vector, covariance_matrix\n\ndef deskew(image):\n    c,v = moments(image)\n    alpha = v[0,1]/v[0,0]\n    affine = np.array([[1,0],[alpha,1]])\n    ocenter = np.array(image.shape)/2.0\n    offset = c-np.dot(affine,ocenter)\n    return interpolation.affine_transform(image,affine,offset=offset)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T10:55:46.381305Z","iopub.execute_input":"2023-04-17T10:55:46.382012Z","iopub.status.idle":"2023-04-17T10:55:46.391072Z","shell.execute_reply.started":"2023-04-17T10:55:46.381972Z","shell.execute_reply":"2023-04-17T10:55:46.389816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_deskewed = np.apply_along_axis(lambda x: deskew(x.reshape(28,28)).flatten(), \n                                         axis=1, arr=X_train.astype(np.uint8).to_numpy()).astype('float32')\nX_val_deskewed = np.apply_along_axis(lambda x: deskew(x.reshape(28,28)).flatten(), \n                                       axis=1, arr=X_val.astype(np.uint8).to_numpy()).astype('float32')\nX_full_deskewed = np.concatenate([X_train_deskewed, X_val_deskewed])\ntest_data_deskewed = np.apply_along_axis(lambda x: deskew(x.reshape(28,28)).flatten(), \n                                           axis=1, arr=test_df.astype(np.uint8).to_numpy()).astype('float32')\nprint(f\"X_train_deskewed shape: {X_train_deskewed.shape}\")\nprint(f\"X_val_deskewed shape: {X_val_deskewed.shape}\")\nprint(f\"X_full_deskewed shape: {X_full_deskewed.shape}\")\nprint(f\"test_data_deskewed shape: {test_data_deskewed.shape}\")","metadata":{"execution":{"iopub.status.busy":"2023-04-17T10:55:48.254391Z","iopub.execute_input":"2023-04-17T10:55:48.255375Z","iopub.status.idle":"2023-04-17T10:56:18.345183Z","shell.execute_reply.started":"2023-04-17T10:55:48.255335Z","shell.execute_reply":"2023-04-17T10:56:18.340956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_deskewed_images(df, df_deskewed, size=8):\n    example_indexes = randint.rvs(0, len(df)-1, size=size)\n    original_images = df.iloc[example_indexes]\n    deskewed_images = df_deskewed[example_indexes]\n    plot_digits(original_images, [], [], images_per_row=8, figsize=(10,2))\n    plt.suptitle('Original images')\n    plt.show()\n    plot_digits(deskewed_images, [], [], images_per_row=8, figsize=(10,2))\n    plt.suptitle('Deskewed images')\n    plt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-04-17T09:22:49.406572Z","iopub.execute_input":"2023-04-17T09:22:49.407581Z","iopub.status.idle":"2023-04-17T09:22:49.424238Z","shell.execute_reply.started":"2023-04-17T09:22:49.407533Z","shell.execute_reply":"2023-04-17T09:22:49.420286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_deskewed_images(X_train, X_train_deskewed, 8)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:22:49.430178Z","iopub.execute_input":"2023-04-17T09:22:49.432186Z","iopub.status.idle":"2023-04-17T09:22:50.541233Z","shell.execute_reply.started":"2023-04-17T09:22:49.432143Z","shell.execute_reply":"2023-04-17T09:22:50.539982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's visualize the deskewed data using t-SNE.","metadata":{}},{"cell_type":"code","source":"plot_tsne(X_train_deskewed, \n          y_train, \n          'Visualizing MNIST through t-SNE (deskewed data)'\n         )","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:22:50.543273Z","iopub.execute_input":"2023-04-17T09:22:50.544567Z","iopub.status.idle":"2023-04-17T09:22:54.161901Z","shell.execute_reply.started":"2023-04-17T09:22:50.544525Z","shell.execute_reply":"2023-04-17T09:22:54.160617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data augmentation <a class=\"anchor\" id=\"dpdataaugmentation\"></a>","metadata":{}},{"cell_type":"markdown","source":"Create functions to augment data.","metadata":{}},{"cell_type":"code","source":"def transform_image(input_img, shift_limit=0.05, rotate_limit=4, scale_limit=0.1):\n    transform = A.ShiftScaleRotate(p=1, shift_limit = shift_limit, rotate_limit = rotate_limit, scale_limit=scale_limit)\n    # random.seed(7) \n    augmented_image = transform(image=input_img)['image']\n    return augmented_image.flatten()\n\ndef get_augmented_data(X_data, y_data, factor=1, shift=0.05, rotate=8, scale=0.2):\n    X_augmented = [X_data]\n    y_augmented = [y_data]\n    for batch in range(factor):\n        augmented_images = np.apply_along_axis(lambda x: transform_image(x.reshape(28, 28)), \n                                               axis=1, arr=X_data)\n        X_augmented.append(augmented_images)\n        y_augmented.append(y_data)\n    X_augmented_all = np.concatenate(X_augmented)\n    y_augmented_all = np.concatenate(y_augmented)\n    return X_augmented_all, y_augmented_all","metadata":{"execution":{"iopub.status.busy":"2023-04-17T10:56:37.169012Z","iopub.execute_input":"2023-04-17T10:56:37.169473Z","iopub.status.idle":"2023-04-17T10:56:37.180237Z","shell.execute_reply.started":"2023-04-17T10:56:37.169429Z","shell.execute_reply":"2023-04-17T10:56:37.178332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_indexes = randint.rvs(0, len(X_train_deskewed)-1, size=1)\nexample_images = X_train_deskewed[example_indexes]\nexample_labels = y_train[example_indexes]\naugmented_images, _ = get_augmented_data(example_images, example_labels)\ndisplay(Markdown(f'#### <div style=\"color: black;\">Augmentation: original and augmented</div>'))\nplot_digits(augmented_images, [], [], images_per_row=9, figsize=(10,2))\nplt.suptitle(f'Digit: {y_train[example_indexes]}')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T10:56:52.131094Z","iopub.execute_input":"2023-04-17T10:56:52.131672Z","iopub.status.idle":"2023-04-17T10:56:52.328149Z","shell.execute_reply.started":"2023-04-17T10:56:52.131634Z","shell.execute_reply":"2023-04-17T10:56:52.326826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pipeline <a class=\"anchor\" id=\"dppipeline\"></a>","metadata":{}},{"cell_type":"code","source":"preprocessor = Pipeline([\n    ('scaler', cuml.preprocessing.MinMaxScaler()),\n    ('pca', cuml.decomposition.PCA(n_components=75))\n])","metadata":{"execution":{"iopub.status.busy":"2023-04-17T10:56:56.410711Z","iopub.execute_input":"2023-04-17T10:56:56.411102Z","iopub.status.idle":"2023-04-17T10:57:00.260357Z","shell.execute_reply.started":"2023-04-17T10:56:56.411068Z","shell.execute_reply":"2023-04-17T10:57:00.259058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_pipeline = Pipeline([\n    ('preprocessor', preprocessor),\n    ('model', None)\n], verbose=False)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T10:57:00.263031Z","iopub.execute_input":"2023-04-17T10:57:00.263938Z","iopub.status.idle":"2023-04-17T10:57:00.270907Z","shell.execute_reply.started":"2023-04-17T10:57:00.263889Z","shell.execute_reply":"2023-04-17T10:57:00.269827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Deskewing data is a time-consuming process, so we will not create a separate transformer for it and just use the X_train_deskewed, X_val_deskewed, X_full_deskewed, test_data_deskewed and X_train_des_aug variables.","metadata":{}},{"cell_type":"markdown","source":"# Model Training and Evaluation <a class=\"anchor\" id=\"trainandevaluate\"></a>","metadata":{}},{"cell_type":"markdown","source":"Creating classes for reporting. Instead of using separate functions, I also created EstimatorReport and GridSearchReport classes for reporting functionality.","metadata":{}},{"cell_type":"code","source":"import scikitplot as skplt\n\nclass EstimatorReport():\n    \"\"\"\n    A class used for generating a report on an estimator's performance \n\n    Attributes\n    ----------\n    X_val : np.ndarray\n        Validation dataset features.\n    y_val : np.ndarray\n        Validation dataset labels.\n    X_train : np.ndarray\n        Training dataset features.\n    y_train : np.ndarray\n        Training dataset labels.\n    estimator : estimator object\n        Scikit-learn estimator object.\n    estimator_name : str\n        Name of the estimator.\n    plot_micro : bool\n        Whether micro average ROC curve should be plotted.\n    plot_macro : bool\n        Whether macro average ROC curve should be plotted.\n    metrics_calculated : bool\n        Whether the performance metrics have been calculated.\n\n    Methods\n    -------\n    calculate_metrics()\n        Calculates accuracy, confusion matrix, classification report and predicted probabilities\n        based on the validation set and sets metrics_calculated flag to True.\n    plot_curves(figsize=(18, 4))\n        Plots ROC and PR curves and learning curve based on the validation set.\n    display_report(figsize=(10, 5))\n        Displays a report on estimator's performance based on validation set.\n    __plot_confusion_matrix(ax)\n        Plots confusion matrix.\n    __plot_classification_report(ax)\n        Plots classification report.\n    \"\"\"\n    def __init__(\n        self,\n        estimator,\n        X_val: np.ndarray,\n        y_val: np.ndarray,\n        X_train: np.ndarray,\n        y_train: np.ndarray,\n        estimator_name: str,\n    ) -> None:\n        self.X_val = X_val\n        self.y_val = y_val\n        self.X_train = X_train\n        self.y_train = y_train\n        self.estimator = estimator\n        self.estimator_name = estimator_name\n        self.plot_micro = False\n        self.plot_macro = False\n        self.metrics_calculated = False\n    \n    def calculate_metrics(self) -> None:\n        self.y_val_predicted = self.estimator.predict(self.X_val)\n        if isinstance(self.y_val_predicted, cudf.Series):\n            self.y_val_predicted = self.y_val_predicted.to_numpy()\n        self.accuracy = accuracy_score(self.y_val, self.y_val_predicted)\n        self.conf_mx = confusion_matrix(self.y_val, self.y_val_predicted)\n        self.classif_report = classification_report(\n            self.y_val, self.y_val_predicted, output_dict=True\n        )\n        try:\n            self.y_val_probas = self.estimator.predict_proba(self.X_val)\n        except:\n            self.y_val_probas = self.estimator.decision_function(self.X_val)\n        self.metrics_calculated = True\n        \n    def plot_curves(self, figsize=(18, 4)) -> None:\n        if not self.metrics_calculated:\n            self.calculate_metrics()\n        fig, axes = plt.subplots( \n            1, 3, figsize=figsize, sharex=False, \n            gridspec_kw={'width_ratios': [1, 1, 1]}, dpi=100\n        )\n        skplt.metrics.plot_roc(\n            y_true=self.y_val, y_probas=self.y_val_probas, ax=axes[0], \n            title='ROC curve', plot_micro=self.plot_micro, plot_macro=self.plot_macro\n        )\n        skplt.metrics.plot_precision_recall(\n            y_true=self.y_val, y_probas=self.y_val_probas, ax=axes[1], \n            title='PR curve', plot_micro=self.plot_micro\n        )\n        skplt.estimators.plot_learning_curve(\n            self.estimator, self.X_train, self.y_train, ax=axes[2]\n        )\n        axes[0].grid()\n        axes[1].grid()\n        \n    def display_report(self, figsize=(10, 5)) -> None:\n        if not self.metrics_calculated:\n            self.calculate_metrics()\n        display(Markdown(f'### <span style=\"color: black;\">Evaluation on the Validation Set (estimator: {self.estimator_name})</span>'))\n        display(Markdown('Accuracy: {:.4f}%'.format(self.accuracy*100)))\n        fig, (ax0, ax1) = plt.subplots(1, 2, figsize=figsize, sharex=False, \n                                   gridspec_kw={'width_ratios': [1, 2.2]}, dpi=100)\n        self.__plot_classification_report(ax0)\n        self.__plot_confusion_matrix(ax1)\n        fig.tight_layout()\n        plt.show()\n        \n    def __plot_confusion_matrix(self, ax) -> None:\n        norm_conf_mx = self.conf_mx.astype('float') / self.conf_mx.sum(axis=1, keepdims=True)\n        np.fill_diagonal(norm_conf_mx, 0)\n        sns.heatmap(\n            norm_conf_mx, ax=ax, \n            annot=False, cmap=plt.cm.gray, cbar=False\n        )\n        plt.title('Confusion matrix')\n        thresh = norm_conf_mx.max() / 2\n        for i, j in itertools.product(range(self.conf_mx.shape[0]), range(self.conf_mx.shape[1])):\n            plt.text(j+0.5, i+0.6, self.conf_mx[i, j],\n                ha=\"center\",\n                va='baseline',\n                color=\"white\" if norm_conf_mx[i, j] < thresh else \"black\")\n        plt.tight_layout()\n        plt.ylabel('True label')\n        plt.xlabel('Predicted label')\n    \n    def __plot_classification_report(self, ax) -> None:\n        clf_report_df = pd.DataFrame(self.classif_report ).iloc[:-1, :].T\n        classes_count = len(self.estimator.classes_)\n        mask_classes = np.zeros_like(clf_report_df, dtype=bool)\n        mask_classes[classes_count:, :] = True\n        mask_metrics = np.zeros_like(clf_report_df, dtype=bool)\n        mask_metrics[:classes_count, :] = True\n        cmap_white = ListedColormap(['white'])\n        sns.heatmap(\n            clf_report_df, ax=ax, mask=mask_classes, \n            annot=True, cmap='viridis', fmt='.6f', cbar=False\n        )\n        sns.heatmap(\n            clf_report_df, ax=ax, mask=mask_metrics, \n            annot=True, cmap=cmap_white, fmt='.6f', cbar=False\n        )\n        ax.set_title(\"Classification report\", fontsize=10, pad=10)\n        ax.xaxis.tick_top()\n        ax.xaxis.set_label_position('top')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-04-17T10:57:02.284193Z","iopub.execute_input":"2023-04-17T10:57:02.284569Z","iopub.status.idle":"2023-04-17T10:57:02.323129Z","shell.execute_reply.started":"2023-04-17T10:57:02.284534Z","shell.execute_reply":"2023-04-17T10:57:02.322115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas.io.formats.style\n\nclass GridSearchReport:\n    \"\"\"\n    A class for generating GridSearchCV reports with a summary of the results\n\n    Attributes:\n        grid_search (sklearn.model_selection.GridSearchCV): GridSearchCV object\n        param_model (str): Name of the model parameter in the hyperparameter grid\n\n    Methods:\n        get_summary(filter_models=[], columns=[], sortby='mean_test_f1')\n            Returns a summary of the GridSearchCV results in a styled Pandas DataFrame\n        display_report()\n            Displays the raw GridSearchCV results and the best parameters found\n    \"\"\"\n    def __init__(self, grid_search, param_model='model') -> None:\n        \"\"\"\n        Initializes a new instance of the GridSearchReport class\n\n        Args:\n            grid_search (sklearn.model_selection.GridSearchCV): GridSearchCV object\n            param_model (str, optional): Name of the model parameter in the hyperparameter grid. Default is 'model'\n        \"\"\"\n        self.grid_search = grid_search\n        self.param_model = param_model\n\n    def get_summary(self, filter_models=[], columns=[], sortby='mean_test_score') -> pd.io.formats.style.Styler:\n        \"\"\"\n        Returns a summary of the GridSearchCV results in a styled Pandas DataFrame\n\n        Args:\n            filter_models (list, optional): List of models (as strings) to filter the results by. Default is an empty list\n            columns (list, optional): List of additional columns of interest to include in the summary. Default is an empty list\n            sortby (str, optional): Column to sort the results by. Default is 'mean_test_f1'\n\n        Returns:\n            pandas.io.formats.style.Styler: A styled Pandas DataFrame with the summary of the GridSearchCV results\n        \"\"\"\n        cv_results = self.grid_search.cv_results_\n        cv_results_df = pd.DataFrame(cv_results)\n        try:\n            regex_pattern = f'(?<=\\'{self.param_model}\\'. )\\w+(?=\\()'\n            cv_results_df['model'] = cv_results_df['params'].astype('str').str.findall(regex_pattern).apply(', '.join)\n        except:\n            print(\"Error: 'model' parameter not found in cv_results\")\n            return\n        if len(filter_models) > 0:\n            if len(set(filter_models) - set(cv_results_df['model'].unique())) > 0:\n                print('Error: filter_models contains models not present in cv_results_df')\n                return\n            cv_results_df = cv_results_df[cv_results_df['model'].isin(filter_models)]\n        mean_columns = cv_results_df.filter(regex='^mean_test').columns.tolist()\n        mean_columns = mean_columns + columns if len(columns) > 0 else mean_columns\n        if len(mean_columns) == 0:\n            print('Error: mean_columns is empty')\n            return\n        filtered_columns = np.concatenate((['model'], mean_columns))\n        return (cv_results_df.sort_values(by=[sortby], ascending=False)\n            .groupby('model')\n            .first()\n            .reindex(columns=filtered_columns)\n            .loc[:, mean_columns]\n            .sort_values(by=[sortby], ascending=False)\n            .style\n            .background_gradient(cmap='viridis')\n        )\n\n    def display_report(self, sortby='rank_test_score', ascending=True) -> None:\n        \"\"\"\n        Displays the raw GridSearchCV results and the best parameters found\n        \"\"\"\n        display(pd.DataFrame(self.grid_search.cv_results_)\n            .sort_values(by=[sortby], ascending=ascending)\n            .reset_index(drop=True)\n            .head()\n        )\n        if hasattr(self.grid_search, 'best_score_'):\n            display(f\"Best score: {self.grid_search.best_score_}\")\n        if hasattr(self.grid_search, 'best_params_'):\n            display(\"Best params:\")\n            display(self.grid_search.best_params_)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-04-17T10:57:05.598881Z","iopub.execute_input":"2023-04-17T10:57:05.599304Z","iopub.status.idle":"2023-04-17T10:57:05.659418Z","shell.execute_reply.started":"2023-04-17T10:57:05.599267Z","shell.execute_reply":"2023-04-17T10:57:05.658479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_to_results(report, results_df):\n    row = {\n        'model': report.estimator_name,\n        'accuracy': report.accuracy\n    }\n    model_name = report.estimator_name\n    if model_name in results_df['model'].values:\n        row_index = results_df.loc[results_df['model'] == model_name].index[0]\n        results_df.loc[row_index, 'accuracy'] = row['accuracy']\n    else:\n        results_df = results_df.append(row, ignore_index=True)\n    return results_df","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-04-17T10:57:08.702845Z","iopub.execute_input":"2023-04-17T10:57:08.703331Z","iopub.status.idle":"2023-04-17T10:57:08.713308Z","shell.execute_reply.started":"2023-04-17T10:57:08.703295Z","shell.execute_reply":"2023-04-17T10:57:08.712230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_errors(estimator, X, y, size, per_row, figsize=(8,5)):\n    y_predicted = estimator.predict(X)\n    errors = (y_predicted - y != 0)\n    X_errors = X_val_deskewed[errors]\n    y_errors = y[errors]\n    y_errors_predicted = y_predicted[errors]\n    exmaple_indexes = randint.rvs(0, len(X_errors)-1, size=size)\n    example_images = X_errors[exmaple_indexes]\n    example_labels = y_errors[exmaple_indexes]\n    example_labels_pred = y_errors_predicted[exmaple_indexes]\n    plot_digits(example_images, example_labels, example_labels_pred, images_per_row=per_row, figsize=figsize)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-04-17T10:57:09.501489Z","iopub.execute_input":"2023-04-17T10:57:09.501879Z","iopub.status.idle":"2023-04-17T10:57:09.508664Z","shell.execute_reply.started":"2023-04-17T10:57:09.501845Z","shell.execute_reply":"2023-04-17T10:57:09.507579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Define DataFrame for estimation results.","metadata":{}},{"cell_type":"code","source":"results_df = pd.DataFrame(columns=['model', 'accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-04-17T10:57:13.070143Z","iopub.execute_input":"2023-04-17T10:57:13.070885Z","iopub.status.idle":"2023-04-17T10:57:13.077879Z","shell.execute_reply.started":"2023-04-17T10:57:13.070837Z","shell.execute_reply":"2023-04-17T10:57:13.076376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Estimate PCA params <a class=\"anchor\" id=\"estimatepcaparams\"></a>","metadata":{}},{"cell_type":"markdown","source":"In the following, we will estimate PCA, scalers, and deskewing (using the SVC model).","metadata":{}},{"cell_type":"code","source":"pipeline_pca = Pipeline([\n    ('pca', None),\n    ('model', cuml.svm.SVC()),\n])\nparams_pca = [\n    {\n        'pca': [cuml.decomposition.PCA()],\n        'pca__n_components': np.arange(25, 200, 50)\n    },\n    {\n        'pca': [None],\n    }\n]\ngrid_search_pca = GridSearchCV(pipeline_pca, params_pca, cv=3, \n                                      scoring='accuracy', verbose=verbose_tuning, \n#                                       n_jobs=-1\n                                     )\ngrid_search_pca.fit(X_train[:20000], y_train[:20000])\nGridSearchReport(grid_search_pca).display_report()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:22:54.568273Z","iopub.execute_input":"2023-04-17T09:22:54.568789Z","iopub.status.idle":"2023-04-17T09:23:41.155358Z","shell.execute_reply.started":"2023-04-17T09:22:54.568755Z","shell.execute_reply":"2023-04-17T09:23:41.154331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grid_search_pca_df = pd.DataFrame(grid_search_pca.cv_results_)\nmean_test_score_no_pca = grid_search_pca_df[grid_search_pca_df[\"param_pca\"].isna()]['mean_test_score'].values[0]\ndisplay(f\"Best score without pca: {mean_test_score_no_pca}\")","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:23:41.156738Z","iopub.execute_input":"2023-04-17T09:23:41.157371Z","iopub.status.idle":"2023-04-17T09:23:41.167925Z","shell.execute_reply.started":"2023-04-17T09:23:41.157333Z","shell.execute_reply":"2023-04-17T09:23:41.166849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The accuracy increased after applying PCA.  So we see that not all the information of the digits is important for accurate classifications.","metadata":{}},{"cell_type":"markdown","source":"### Estimate scalers <a class=\"anchor\" id=\"estimatescalers\"></a>","metadata":{}},{"cell_type":"markdown","source":"Determine which scaler to use.","metadata":{}},{"cell_type":"code","source":"pipeline_scaler = Pipeline([\n    ('pca', cuml.decomposition.PCA(n_components=75)),\n    ('scaler', None),\n    ('model', cuml.svm.SVC()),\n])\nparams_scaler = [\n    {\n        'scaler': [cuml.preprocessing.MinMaxScaler(), cuml.preprocessing.StandardScaler(), cuml.preprocessing.Binarizer(threshold=50)]\n    }\n]\ngrid_search_scaler = GridSearchCV(pipeline_scaler, params_scaler, cv=3, \n                                  scoring='accuracy', n_jobs=-1, verbose=verbose_tuning)\ngrid_search_scaler.fit(X_train[:20000], y_train[:20000])\nGridSearchReport(grid_search_scaler).display_report()\nreport_scaler = EstimatorReport(grid_search_scaler, X_val, y_val, [], [], \"SVC (with PCA and scaler)\")\nreport_scaler.display_report()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:23:41.169770Z","iopub.execute_input":"2023-04-17T09:23:41.170207Z","iopub.status.idle":"2023-04-17T09:24:18.804368Z","shell.execute_reply.started":"2023-04-17T09:23:41.170170Z","shell.execute_reply":"2023-04-17T09:24:18.803354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"During tuning and testing, MinMaxScaler showed better performance and speed, so I finally chose it.","metadata":{}},{"cell_type":"markdown","source":"### Estimate Deskew <a class=\"anchor\" id=\"estimatedeskew\"></a>","metadata":{}},{"cell_type":"code","source":"pipe_no_deskewed = copy.copy(full_pipeline).set_params(model=cuml.svm.SVC())\npipe_no_deskewed.fit(X_train, y_train)\nreport_nodes = EstimatorReport(pipe_no_deskewed, X_val, y_val, [], [], \"SVC (PCA, Scaler, no deskew)\")\nreport_nodes.display_report()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:24:18.806644Z","iopub.execute_input":"2023-04-17T09:24:18.807498Z","iopub.status.idle":"2023-04-17T09:24:25.124692Z","shell.execute_reply.started":"2023-04-17T09:24:18.807456Z","shell.execute_reply":"2023-04-17T09:24:25.123365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pipe_deskewed = copy.copy(full_pipeline).set_params(model=cuml.svm.SVC())\npipe_deskewed.fit(X_train_deskewed, y_train)\nreport_des = EstimatorReport(pipe_deskewed, X_val_deskewed, y_val, [], [], \"SVC (PCA, Scaler, deskew)\")\nreport_des.display_report()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:24:25.126185Z","iopub.execute_input":"2023-04-17T09:24:25.127002Z","iopub.status.idle":"2023-04-17T09:24:32.163471Z","shell.execute_reply.started":"2023-04-17T09:24:25.126962Z","shell.execute_reply":"2023-04-17T09:24:32.162382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The model performance with deskewed data is better, so we will continue to use it.","metadata":{}},{"cell_type":"code","source":"del X_train\ndel train_df\ndel digmnist_df","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:24:32.165135Z","iopub.execute_input":"2023-04-17T09:24:32.165549Z","iopub.status.idle":"2023-04-17T09:24:32.173012Z","shell.execute_reply.started":"2023-04-17T09:24:32.165510Z","shell.execute_reply":"2023-04-17T09:24:32.172001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Estimate different models (default params) <a class=\"anchor\" id=\"estimatedifferentmodels\"></a>","metadata":{}},{"cell_type":"markdown","source":"Estimate different models using default hyperparams.","metadata":{}},{"cell_type":"code","source":"params = [\n    {'model': [\n        SGDClassifier(max_iter=4000),\n        cuml.linear_model.LogisticRegression(max_iter=8000),\n        cuml.svm.SVC(),\n        cuml.neighbors.KNeighborsClassifier(),\n        cuml.naive_bayes.GaussianNB(),\n        DecisionTreeClassifier(),\n        cuml.ensemble.RandomForestClassifier(),\n        ExtraTreesClassifier(),\n        xgb.XGBClassifier(tree_method='gpu_hist', predictor='gpu_predictor'),\n        lgb.LGBMClassifier(device=\"gpu\"),\n        HistGradientBoostingClassifier(),\n        AdaBoostClassifier(),\n        BaggingClassifier(),\n    ]}]\ngrid_search_allmodels = GridSearchCV(full_pipeline, params, cv=3, \n                                     scoring=['accuracy', 'f1_macro'], n_jobs=1, \n                                     verbose=10, refit=False)\ngrid_search_allmodels.fit(X_train_deskewed, y_train)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-04-17T09:24:32.174741Z","iopub.execute_input":"2023-04-17T09:24:32.175510Z","iopub.status.idle":"2023-04-17T09:28:34.283665Z","shell.execute_reply.started":"2023-04-17T09:24:32.175463Z","shell.execute_reply":"2023-04-17T09:28:34.279845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"report_gridsearch_all = GridSearchReport(grid_search_allmodels)\nreport_gridsearch_all.display_report(sortby='mean_test_accuracy', ascending=False)\ndisplay(report_gridsearch_all.get_summary(columns=['mean_fit_time'], sortby='mean_test_accuracy'))","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:28:34.287279Z","iopub.status.idle":"2023-04-17T09:28:34.288226Z","shell.execute_reply.started":"2023-04-17T09:28:34.287939Z","shell.execute_reply":"2023-04-17T09:28:34.287969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Fine-Tuning <a class=\"anchor\" id=\"finetuning\"></a>","metadata":{}},{"cell_type":"markdown","source":"Below I use different tools for hyperparameteres search like GridSearcCV, HalvingGridSearchCV, BayesSearchCV. It depends on fitting speed, search complexity.","metadata":{}},{"cell_type":"markdown","source":"#### SVC <a class=\"anchor\" id=\"finetuningsvc\"></a>","metadata":{}},{"cell_type":"code","source":"params_svc = [\n    {\n        'model': [SVC()],\n        'model__kernel': ['rbf'],\n        'model__C': np.arange(3, 12, 1),\n        'model__gamma': np.arange(0.01, 0.04, 0.003),\n#         'model__probability': [True],\n    }\n]\ngrid_search_svc = HalvingGridSearchCV(full_pipeline, params_svc, cv=3, \n                                      scoring='accuracy', verbose=verbose_tuning, \n                                      n_jobs=-1)\ngrid_search_svc.fit(X_train_deskewed, y_train)\nbest_svc = grid_search_svc.best_estimator_\nGridSearchReport(grid_search_svc).display_report()\nreport_svc = EstimatorReport(best_svc, X_val_deskewed, y_val, [], [], \"SVC\")\nreport_svc.display_report()\nresults_df = add_to_results(report_svc, results_df)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:28:34.294173Z","iopub.status.idle":"2023-04-17T09:28:34.295121Z","shell.execute_reply.started":"2023-04-17T09:28:34.294830Z","shell.execute_reply":"2023-04-17T09:28:34.294861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Errors visualization","metadata":{}},{"cell_type":"markdown","source":"Let's look at some errors after fine-tuning the SVC classifier.","metadata":{}},{"cell_type":"code","source":"plot_errors(best_svc, X_val_deskewed, y_val, 10, 5)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:28:34.296814Z","iopub.status.idle":"2023-04-17T09:28:34.297760Z","shell.execute_reply.started":"2023-04-17T09:28:34.297445Z","shell.execute_reply":"2023-04-17T09:28:34.297475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### RandomForestClassifier <a class=\"anchor\" id=\"finetuningrf\"></a>","metadata":{}},{"cell_type":"code","source":"params_rf = [\n    {\n        'model': [cuml.ensemble.RandomForestClassifier()],\n        'model__n_estimators': space.Integer(150, 800),\n        'model__min_samples_leaf': [2, 3],\n    }\n]\nbayes_search_rf = BayesSearchCV(full_pipeline, params_rf, cv=3, n_iter = 12, \n                                scoring='accuracy', refit=True, verbose=verbose_tuning, n_points = 1, \n                                return_train_score=True, random_state=42)\nbayes_search_rf.fit(X_train_deskewed, y_train)\nbest_rf = bayes_search_rf.best_estimator_\nGridSearchReport(bayes_search_rf).display_report()\nreport_rf = EstimatorReport(best_rf, X_val_deskewed, y_val, [], [], \"RandomForestClassifier\")\nreport_rf.display_report()\nresults_df = add_to_results(report_rf, results_df)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:28:34.302756Z","iopub.status.idle":"2023-04-17T09:28:34.303658Z","shell.execute_reply.started":"2023-04-17T09:28:34.303338Z","shell.execute_reply":"2023-04-17T09:28:34.303365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plots.plot_objective(bayes_search_rf.optimizer_results_[0],\n                     size = 2.5, levels = 25)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:28:34.305135Z","iopub.status.idle":"2023-04-17T09:28:34.305977Z","shell.execute_reply.started":"2023-04-17T09:28:34.305715Z","shell.execute_reply":"2023-04-17T09:28:34.305742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### KNeighborsClassifier <a class=\"anchor\" id=\"finetuningknn\"></a>","metadata":{}},{"cell_type":"code","source":"params_knn = [\n    {\n        'model': [cuml.neighbors.KNeighborsClassifier()],\n        'model__n_neighbors': space.Integer(2, 20),\n    }\n]\nbayes_search_knn = BayesSearchCV(full_pipeline, params_knn, cv=3, n_iter = 10, \n                                scoring='accuracy', refit=True, verbose=verbose_tuning, n_points = 1, \n                                return_train_score=True, random_state=42)\nbayes_search_knn.fit(X_train_deskewed, y_train)\nbest_knn = bayes_search_knn.best_estimator_\nGridSearchReport(bayes_search_knn).display_report()\nreport_knn = EstimatorReport(best_knn, X_val_deskewed, y_val, [], [], \"KNeighborsClassifier\")\nreport_knn.display_report()\nresults_df = add_to_results(report_knn, results_df)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:28:34.316718Z","iopub.status.idle":"2023-04-17T09:28:34.317611Z","shell.execute_reply.started":"2023-04-17T09:28:34.317300Z","shell.execute_reply":"2023-04-17T09:28:34.317331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plots.plot_objective(bayes_search_knn.optimizer_results_[0],\n                     size = 2.5, levels = 25)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:28:34.319200Z","iopub.status.idle":"2023-04-17T09:28:34.320156Z","shell.execute_reply.started":"2023-04-17T09:28:34.319857Z","shell.execute_reply":"2023-04-17T09:28:34.319889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### XGBClassifier <a class=\"anchor\" id=\"finetuningxgb\"></a>","metadata":{}},{"cell_type":"code","source":"params_xgb = [\n    {\n        'model': [xgb.XGBClassifier()],\n        'model__n_estimators': space.Integer(100, 200),\n        'model__learning_rate': space.Real(0.1, 0.8),\n        'model__max_depth': space.Integer(8, 15),\n        'model__alpha': space.Real(0.01, 1),\n        'model__lambda': space.Real(0.001, 0.1),\n        'model__min_child_weight': space.Integer(1, 5),\n        'model__colsample_bytree': space.Real(0.1, 1),\n        'model__subsample': space.Real(0.1, 1),\n        'model__num_class': [10],\n        'model__objective': ['multi:softprob'],\n        'model__tree_method': ['gpu_hist'],\n        'model__predictor': ['gpu_predictor'],\n        'model__n_jobs': [-1],\n    }\n]\nbayes_search_xgb = BayesSearchCV(full_pipeline, params_xgb, cv=3, n_iter = 15, \n                                 scoring='accuracy', refit=True, verbose=verbose_tuning, n_points = 1, \n                                 return_train_score=True, random_state=42).fit(X_train_deskewed, y_train)\nbest_xgb = bayes_search_xgb.best_estimator_\nGridSearchReport(bayes_search_xgb).display_report()\nreport_xgb = EstimatorReport(best_xgb, X_val_deskewed, y_val, [], [], \"XGBClassifier\")\nreport_xgb.display_report()\nresults_df = add_to_results(report_xgb, results_df)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:28:34.321942Z","iopub.status.idle":"2023-04-17T09:28:34.322987Z","shell.execute_reply.started":"2023-04-17T09:28:34.322680Z","shell.execute_reply":"2023-04-17T09:28:34.322712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plots.plot_objective(bayes_search_xgb.optimizer_results_[0],\n                     size = 2.5, levels = 25)\nplt.show()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-04-17T09:28:34.324721Z","iopub.status.idle":"2023-04-17T09:28:34.325652Z","shell.execute_reply.started":"2023-04-17T09:28:34.325342Z","shell.execute_reply":"2023-04-17T09:28:34.325368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Estimations results <a class=\"anchor\" id=\"estimationsresults\"></a>","metadata":{}},{"cell_type":"markdown","source":"Compare the performance of models before and after tuning.","metadata":{}},{"cell_type":"code","source":"def display_merged_results(init_df, tuned_df):\n    merged_df = pd.merge(init_df, tuned_df, on='model', how='outer')\n    merged_df = merged_df.rename(columns={'accuracy_x': 'accuracy_init', 'accuracy_y': 'accuracy_tuned'})\n    display(merged_df\n            .sort_values(by='accuracy_tuned', ascending=False)\n            .reset_index(drop=True)\n            .style\n            .background_gradient(cmap='viridis')\n            .set_properties(**{'text-align': 'left'})\n           )","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-04-17T09:28:34.327126Z","iopub.status.idle":"2023-04-17T09:28:34.328033Z","shell.execute_reply.started":"2023-04-17T09:28:34.327731Z","shell.execute_reply":"2023-04-17T09:28:34.327759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"report_alldefaults_df = (report_gridsearch_all\n                         .get_summary(sortby='mean_test_accuracy')\n                         .data\n                         .reset_index()\n                         .rename(columns={'mean_test_accuracy': 'accuracy'})\n                         .drop(['mean_test_f1_macro'], axis=1)\n                        )\ndisplay_merged_results(report_alldefaults_df, results_df)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:28:34.329687Z","iopub.status.idle":"2023-04-17T09:28:34.330554Z","shell.execute_reply.started":"2023-04-17T09:28:34.330240Z","shell.execute_reply":"2023-04-17T09:28:34.330266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Ensemble learning <a class=\"anchor\" id=\"ensemblelearning\"></a>","metadata":{}},{"cell_type":"markdown","source":"#### Data augmentation","metadata":{}},{"cell_type":"code","source":"X_train_des_aug, y_train_aug = get_augmented_data(X_train_deskewed, y_train, factor=5, rotate=3)\nprint(f'Train shape. \\tX: {X_train_deskewed.shape}, y: {y_train.shape}')\nprint(f'Train (augmented) shape. \\tX: {X_train_des_aug.shape}, y: {y_train_aug.shape}')\nprint('Proportion: \\t{:.0f}:1'.format(X_train_des_aug.shape[0]/X_train_deskewed.shape[0]))","metadata":{"execution":{"iopub.status.busy":"2023-04-17T10:57:20.271738Z","iopub.execute_input":"2023-04-17T10:57:20.272715Z","iopub.status.idle":"2023-04-17T10:57:34.084006Z","shell.execute_reply.started":"2023-04-17T10:57:20.272674Z","shell.execute_reply":"2023-04-17T10:57:34.082826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shuffle_idx = np.random.permutation(len(X_train_des_aug))\nX_train_des_aug = X_train_des_aug[shuffle_idx]\ny_train_aug = y_train_aug[shuffle_idx]","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:28:34.341101Z","iopub.status.idle":"2023-04-17T09:28:34.341990Z","shell.execute_reply.started":"2023-04-17T09:28:34.341713Z","shell.execute_reply":"2023-04-17T09:28:34.341741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Visualize some digits to make sure everything is correct after the augmentation and shuffling.","metadata":{}},{"cell_type":"code","source":"exmaple_indexes = randint.rvs(0, len(X_train_des_aug)-1, size=8)\nexample_images = X_train_des_aug[exmaple_indexes]\nexample_labels = y_train_aug[exmaple_indexes]\nplot_digits(example_images, example_labels, [], images_per_row=8, figsize=(8,2))","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:28:34.343676Z","iopub.status.idle":"2023-04-17T09:28:34.344576Z","shell.execute_reply.started":"2023-04-17T09:28:34.344282Z","shell.execute_reply":"2023-04-17T09:28:34.344310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For ensemble learning, I use cuML versions of sklearn models for faster execution (it can be 5-10 times faster than standard sklearn versions).","metadata":{}},{"cell_type":"code","source":"# Create models without fine-tuning\nif True == True:\n    svc_cl = cuml.svm.SVC(C=7, gamma=0.034, kernel='rbf', probability=True)\n    rf_cl = cuml.ensemble.RandomForestClassifier(min_samples_leaf=2, n_estimators=750)\n    knn_cl = cuml.neighbors.KNeighborsClassifier(n_neighbors=5)\n    xgb_params = {\n        'lambda': 0.020060125791021795,\n        'learning_rate': 0.11458425114095297,\n        'alpha': 0.026597487104710826,\n        'colsample_bytree': 0.4584663483147582,\n        'subsample': 0.3848037664698972,\n        'max_depth': 9,\n        'n_estimators': 195,\n        'min_child_weight': 4,\n        'num_class': 10,\n        'objective': 'multi:softprob',\n        'eval_metric': \"merror\",\n        'predictor': 'gpu_predictor',\n        'tree_method': 'gpu_hist',\n    }\n    xgb_cl = xgb.XGBClassifier(**xgb_params)","metadata":{"execution":{"iopub.status.busy":"2023-04-17T10:58:22.917615Z","iopub.execute_input":"2023-04-17T10:58:22.918230Z","iopub.status.idle":"2023-04-17T10:58:22.928841Z","shell.execute_reply.started":"2023-04-17T10:58:22.918192Z","shell.execute_reply":"2023-04-17T10:58:22.927791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### VotingClassifier <a class=\"anchor\" id=\"votingclassifier\"></a>","metadata":{}},{"cell_type":"markdown","source":"Create an Ensemble Voting Classifier with the best classifiers and fit it using the augmented data.","metadata":{}},{"cell_type":"code","source":"estimators_vt = [\n    ('best_svc', svc_cl),\n    ('best_knn', knn_cl),\n    ('best_xgb', xgb_cl),\n    ('best_rf', rf_cl),\n]\npipe_vt = copy.copy(full_pipeline)\npipe_vt.set_params(model=VotingClassifier(estimators=estimators_vt, voting='soft', verbose=1, \n                                          weights=[14, 9, 1, 1]\n                                         ))\npipe_vt.fit(X_train_des_aug, y_train_aug)\nreport_vt = EstimatorReport(pipe_vt, X_val_deskewed, y_val, [], [], \"VotingClassifier\")\nreport_vt.display_report()","metadata":{"execution":{"iopub.status.busy":"2023-04-17T11:05:34.390313Z","iopub.execute_input":"2023-04-17T11:05:34.391261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission <a class=\"anchor\" id=\"submission\"></a>","metadata":{}},{"cell_type":"markdown","source":"Let's evaluate our results on the test set and save it in submission dataframe.","metadata":{}},{"cell_type":"code","source":"def create_submissions(estimator, test_data, index_data, index_col, target_col):\n    sumbission = pd.DataFrame(\n        {\n            index_col: index_data,\n            target_col: estimator.predict(test_data),\n        }\n    ).set_index(index_col)\n    sumbission.to_csv(\"submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:28:34.360602Z","iopub.status.idle":"2023-04-17T09:28:34.361542Z","shell.execute_reply.started":"2023-04-17T09:28:34.361223Z","shell.execute_reply":"2023-04-17T09:28:34.361252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_submissions(pipe_vt, test_data_deskewed, np.arange(0, len(test_data_deskewed)), 'id', 'label')","metadata":{"execution":{"iopub.status.busy":"2023-04-17T09:28:34.363266Z","iopub.status.idle":"2023-04-17T09:28:34.364234Z","shell.execute_reply.started":"2023-04-17T09:28:34.363926Z","shell.execute_reply":"2023-04-17T09:28:34.363956Z"},"trusted":true},"execution_count":null,"outputs":[]}]}