{"metadata":{"colab":{"machine_shape":"hm","provenance":[]},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.5"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30761,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":" ! pip install -q kaggle","metadata":{"id":"D1QDJ_IGSeqA"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from google.colab import files\nfiles.upload()","metadata":{"executionInfo":{"elapsed":7336,"status":"ok","timestamp":1728979501953,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"Kie9mbamSkr_","outputId":"56d078c7-ae5f-40d6-b18c-e85f0bf9300f"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! mkdir ~/.kaggle\n! cp kaggle.json ~/.kaggle/","metadata":{"id":"2dBpMjxiSonw"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" ! chmod 600 ~/.kaggle/kaggle.json","metadata":{"id":"bBTy4rraSp_H"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! kaggle competitions download -c digit-recognizer","metadata":{"executionInfo":{"elapsed":1667,"status":"ok","timestamp":1728979503965,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"dET9g1_BSwE0","outputId":"979e2c61-b936-48d0-ccc4-ba81a52d3cfb"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! mkdir digit-recognizer\n! unzip digit-recognizer.zip -d digit-recognizer","metadata":{"executionInfo":{"elapsed":761,"status":"ok","timestamp":1728979504725,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"mMtcbz2MSyeF","outputId":"45caf3cb-40b3-46d1-e2fc-8353f3a0529a"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Digit Recognizer**","metadata":{"id":"-TAEcN6u3RaM"}},{"cell_type":"markdown","source":"## Frame the Problem","metadata":{"id":"fED5xCKMSi3v"}},{"cell_type":"markdown","source":"In this project, we aim to identify handwritten digits from the test set and evaluate the model's performance based on accuracy (as requested in the competition).\n","metadata":{"id":"GrEoii1f3VhK"}},{"cell_type":"markdown","source":"## Get  the Data and Quick Look at Data Structure\n","metadata":{"id":"5ubXKGnsowqS"}},{"cell_type":"markdown","source":"To begin, we load the data from CSV files using Pandas. The training data includes both images and labels, while the test data only contains the images.","metadata":{"id":"Oh03ik0r4VPU"}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt","metadata":{"id":"Pdj3zKReo6xc"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(\"digit-recognizer/train.csv\",)\ntest_features_df = pd.read_csv(\"digit-recognizer/test.csv\",)","metadata":{"id":"CGt9ImPho2DS"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" Next, we inspect the structure of the dataset by displaying basic information such as the number of entries and data types","metadata":{"id":"XRezBoGZ56mg"}},{"cell_type":"code","source":"train_df.info(show_counts=True)","metadata":{"executionInfo":{"elapsed":7,"status":"ok","timestamp":1728979506915,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"0i4t5FyDvGDt","outputId":"020788a1-6ec4-41a5-8909-0de117ab17b9"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"executionInfo":{"elapsed":5,"status":"ok","timestamp":1728979506915,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"rJnO-QDMV1eX","outputId":"5f304f15-c803-4379-c19f-1eece84af92d"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"After checking the shape of the training dataset, which consists of 42,000 images, each represented by 785 features, we proceed to determine the maximum value in each pixel for the images.","metadata":{"id":"8xSE1HlG3TgS"}},{"cell_type":"code","source":"train_df.describe()","metadata":{"executionInfo":{"elapsed":1701,"status":"ok","timestamp":1728979508613,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"X0Eh6JNpWhAf","outputId":"3f9927e2-3783-40ba-b7c0-e7ff48476a66"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_pixel_value = train_df.iloc[:, 1:].max().max()\nprint(f\"Maximum pixel value in the dataset: {max_pixel_value}\")\n\nmin_pixel_value = train_df.iloc[:, 1:].min().min()\nprint(f\"Minimum pixel value in the dataset: {min_pixel_value}\")","metadata":{"executionInfo":{"elapsed":3,"status":"ok","timestamp":1728979508613,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"x7AzpuKw3sb0","outputId":"f2dc90bc-3c63-4e23-a3e2-774712828a1c"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As observed, our dataset contains 42,000 images, with each image represented by 784 features, corresponding to a 28 x 28 pixel grid. We’ve also established that the maximum pixel value is 255. Now, let’s take a look at the first 20 images and their corresponding labels.\n","metadata":{"id":"6n06CAvq6Qba"}},{"cell_type":"code","source":"def plot_digit(image_data):\n    image = image_data.reshape(28, 28)\n    plt.imshow(image, cmap=\"binary\")\n    plt.axis(\"off\")\n\nplt.figure(figsize=(10, 6))\n\nfor index in range(20):\n    plt.subplot(4, 5, index + 1)\n    image_data = train_df.iloc[index, 1:].values\n    label = train_df.iloc[index, 0]\n    plot_digit(image_data)\n    plt.title(f\"Label: {label}\")\n\nplt.tight_layout()\nplt.show()","metadata":{"executionInfo":{"elapsed":1103,"status":"ok","timestamp":1728979509714,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"VBW-r59LXDTq","outputId":"c6e6c20d-91da-4021-b8ea-58a1aca95eb4"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Next, we examine the distribution of each class to determine if special handling is required.","metadata":{"id":"kdXXlDv4-lkF"}},{"cell_type":"code","source":"label_distribution = train_df['label'].value_counts()\nprint(\"Label Distribution:\")\nprint(label_distribution)\n\nlabel_distribution.plot(kind='bar', color='blue')\nplt.title(\"Label Distribution\")\nplt.xlabel(\"Class Labels\")\nplt.ylabel(\"Frequency\")\nplt.show()\n","metadata":{"executionInfo":{"elapsed":7,"status":"ok","timestamp":1728979509715,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"aYYOGxCiaRTW","outputId":"7f8d8a82-3621-46a3-909c-772a20c86ebf"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create a Validation set","metadata":{"id":"RRBqu4nuZ8kb"}},{"cell_type":"markdown","source":"Now, we will split the training set into 80% for training and 20% for validation, as no special handling is needed due to (almost) balanced class distribution as we saw earlier.","metadata":{"id":"y6ulALrHaMzs"}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX = train_df.drop(columns=['label'])\ny = train_df['label']\n\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)","metadata":{"id":"LkvMY0c1aBFR"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Checking class distribution in training set\nprint(\"Training set class distribution:\")\nprint(y_train.value_counts(normalize=True))  # Normalized to show proportion\n\n# Checking class distribution in validation set\nprint(\"\\nValidation set class distribution:\")\nprint(y_val.value_counts(normalize=True))\n","metadata":{"executionInfo":{"elapsed":6,"status":"ok","timestamp":1728979511704,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"mQ49Uf9Bb4NI","outputId":"506bc6d0-0beb-48d2-f060-43cbe5c3b590"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training and Selecting a Model","metadata":{"id":"N1NdlA5rYVAa"}},{"cell_type":"markdown","source":"## Create a Smaller Dataset","metadata":{"id":"HIpW_UHVvkyz"}},{"cell_type":"markdown","source":"To simplify the model selection process and speed up training, we first create a smaller subset of the dataset. This reduced set allows for faster model evaluation and selection.","metadata":{"id":"HiP3XGVqzNAe"}},{"cell_type":"code","source":"X_small_train, _, y_small_train, _ = train_test_split(X_train, y_train, test_size=0.8, random_state=42, stratify=y_train)\n\nprint(f\"Size of the smaller training set: {X_small_train.shape[0]} images\")\n","metadata":{"executionInfo":{"elapsed":5,"status":"ok","timestamp":1728979511704,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"omchJ8Yvu5xk","outputId":"9570b3fe-83c0-4036-c980-10337db5fd26"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Stochastic Gradient Descent - One versus Rest","metadata":{"id":"C5U4QPsfqV94"}},{"cell_type":"markdown","source":"### Training the Model","metadata":{"id":"VhVM53yg9u6W"}},{"cell_type":"markdown","source":"The first model we will use is SGD with the OvR strategy, as we are limited to linear classification models, `linear_model.LogisticRegression` and `linear_model.SGDClassifier`. For each model, we assess the performance using GridSearch instead of a baseline to find the best parameters (using the small training dataset). The parameters included in the GridSearch are penalty and alpha. This is because GridSearch allows us to systematically explore the hyperparameter space and optimize the model's performance, rather than relying on arbitrary or default settings that a baseline might use.\n\n","metadata":{"id":"MoiuoKYb7IVr"}},{"cell_type":"code","source":"from google.colab import drive\ndrive.mount('/content/drive')","metadata":{"executionInfo":{"elapsed":59547,"status":"ok","timestamp":1728979571248,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"El9tlIxqgThV","outputId":"3abd1bd4-7a30-4a0a-b91f-6066436d944a"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split, GridSearchCV, StratifiedKFold\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.metrics import make_scorer, accuracy_score, f1_score, confusion_matrix, classification_report\nimport joblib\nimport json\nfrom google.colab import files\n\novr_preprocessor = StandardScaler()\n\novr_model = SGDClassifier(max_iter=1500, random_state=42)\n\novr_pipeline = Pipeline(steps=[\n    ('scaler', ovr_preprocessor),\n    ('classifier', ovr_model)\n])\n\novr_param_grid = [\n    {\n        'classifier__penalty': ['l2', 'l1'],\n        'classifier__alpha': [0.0001, 0.001, 0.01],\n    },\n    {\n        'classifier__penalty': ['elasticnet'],\n        'classifier__alpha': [0.0001, 0.001, 0.01],\n        'classifier__l1_ratio': [0.15, 0.5, 0.85],\n    }\n]\n\novr_strat_kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n\novr_scoring = {\n    'accuracy': make_scorer(accuracy_score),\n}\n\novr_grid_search = GridSearchCV(\n    ovr_pipeline, ovr_param_grid, cv=ovr_strat_kfold, scoring=ovr_scoring, refit='accuracy', verbose=1, n_jobs=-1\n)\n\nprint(\"Training and tuning SGDClassifier OvR on the smaller training set...\")\novr_grid_search.fit(X_small_train, y_small_train)\n\novr_best_estimator = ovr_grid_search.best_estimator_\novr_best_params = ovr_grid_search.best_params_\n\nprint(f\"Best parameters for SGDClassifier OvR: {ovr_best_params}\")\nprint(f\"Best Accuracy for SGDClassifier OvR (Cross-Validation): {ovr_grid_search.best_score_:.4f}\")\n\novr_y_pred = ovr_grid_search.predict(X_val)\nprint(\"\\nClassification Report for SGDClassifier OvR on Validation Set:\\n\")\nprint(classification_report(y_val, ovr_y_pred))\nprint(f\"Confusion Matrix for SGDClassifier OvR on Validation Set:\\n\")\nprint(confusion_matrix(y_val, ovr_y_pred))\n\novr_filename = \"SGDClassifier_OvR_best_model.pkl\"\njoblib.dump(ovr_best_estimator, ovr_filename)\nprint(f\"Saved SGDClassifier OvR to {ovr_filename}\")\n\novr_params_filename = 'SGDClassifier_OvR_best_params.json'\nwith open(ovr_params_filename, 'w') as f:\n    json.dump(ovr_best_params, f)\nprint(f\"Saved best parameters to {ovr_params_filename}\")\n\nfiles.download(ovr_filename)\nfiles.download(ovr_params_filename)","metadata":{"executionInfo":{"elapsed":273450,"status":"ok","timestamp":1728979844695,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"rzcsdznpRDPK","outputId":"374bca26-1b00-4d15-e3b1-88d1329d61f3"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We've obtained the best parameters for SGD with the OvR strategy with accuracy 0.90. Now, we want to evaluate the model by displaying a more detailed confusion matrix.","metadata":{"id":"BBjkFHLa8K18"}},{"cell_type":"markdown","source":"### Error Analysis","metadata":{"id":"gZPsmrphzi5B"}},{"cell_type":"code","source":"from sklearn.metrics import ConfusionMatrixDisplay\nimport matplotlib.pyplot as plt\n\nConfusionMatrixDisplay.from_predictions(y_val, ovr_y_pred,\n                                        normalize=\"true\", values_format=\".0%\")\nplt.title('Confusion Matrix for SGDClassifier on Validation Set')\nplt.show()","metadata":{"executionInfo":{"elapsed":502,"status":"ok","timestamp":1728979845193,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"S_U9KtHzFqQi","outputId":"8009a212-8a5a-48e3-a9ad-5c01e43c188e"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see, we have poor accuracy in classes 5, 8, and 9. Now, we want to further analyze which classes other classes are being misclassified as.\n","metadata":{"id":"fhsicDLt8wNx"}},{"cell_type":"code","source":"from sklearn.metrics import ConfusionMatrixDisplay\nimport matplotlib.pyplot as plt\n\novr_y_pred = ovr_grid_search.predict(X_val)\n\nsample_weight = (ovr_y_pred != y_val).astype(int)\n\nConfusionMatrixDisplay.from_predictions(y_val, ovr_y_pred,\n                                        sample_weight=sample_weight,\n                                        normalize=\"true\", values_format=\".0%\")\nplt.title('Confusion Matrix of Misclassified Samples on Validation Data')\nplt.show()","metadata":{"executionInfo":{"elapsed":682,"status":"ok","timestamp":1728979845873,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"1L_WZnsXJdPU","outputId":"5a71822e-b57e-4499-85a8-6d85fc08c48e"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see, several classes are being highly misclassified as others:\n*   1 is misclassified as 8\n*   4 is misclassified as 9\n*   7 is misclassified as 9\nTo investigate this further, we will plot a comparison of images between these classes to understand why they are performing poorly.","metadata":{"id":"5xvdMDfe9sAN"}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\ncl_a1, cl_b1 = 7, 9  # First comparison (7 vs 9)\ncl_a2, cl_b2 = 4, 9  # Second comparison (4 vs 9)\ncl_a3, cl_b3 = 1, 8  # Third comparison (1 vs 8)\n\nX_val_images = X_val.values\nX_val_images = X_val_images.reshape(-1, 28, 28)\n\n# First comparison (7 vs 9)\nX_aa1 = X_val_images[(y_val == cl_a1) & (ovr_y_pred == cl_a1)]\nX_ab1 = X_val_images[(y_val == cl_a1) & (ovr_y_pred == cl_b1)]\nX_ba1 = X_val_images[(y_val == cl_b1) & (ovr_y_pred == cl_a1)]\nX_bb1 = X_val_images[(y_val == cl_b1) & (ovr_y_pred == cl_b1)]\n\n# Second comparison (4 vs 9)\nX_aa2 = X_val_images[(y_val == cl_a2) & (ovr_y_pred == cl_a2)]\nX_ab2 = X_val_images[(y_val == cl_a2) & (ovr_y_pred == cl_b2)]\nX_ba2 = X_val_images[(y_val == cl_b2) & (ovr_y_pred == cl_a2)]\nX_bb2 = X_val_images[(y_val == cl_b2) & (ovr_y_pred == cl_b2)]\n\n# Third comparison (1 vs 8)\nX_aa3 = X_val_images[(y_val == cl_a3) & (ovr_y_pred == cl_a3)]\nX_ab3 = X_val_images[(y_val == cl_a3) & (ovr_y_pred == cl_b3)]\nX_ba3 = X_val_images[(y_val == cl_b3) & (ovr_y_pred == cl_a3)]\nX_bb3 = X_val_images[(y_val == cl_b3) & (ovr_y_pred == cl_b3)]\n\nsize = 5\npad = 0.2\n\n# First figure (7 vs 9)\nplt.figure(figsize=(size, size))\nfor images, (label_col, label_row) in [(X_ba1, (0, 0)), (X_bb1, (1, 0)),\n                                       (X_aa1, (0, 1)), (X_ab1, (1, 1))]:\n    for idx, image_data in enumerate(images[:size*size]):\n        x = idx % size + label_col * (size + pad)\n        y = idx // size + label_row * (size + pad)\n        plt.imshow(image_data, cmap=\"binary\", extent=(x, x + 1, y, y + 1))\nplt.xticks([size / 2, size + pad + size / 2], [str(cl_a1), str(cl_b1)])\nplt.yticks([size / 2, size + pad + size / 2], [str(cl_b1), str(cl_a1)])\nplt.plot([size + pad / 2, size + pad / 2], [0, 2 * size + pad], \"k:\")\nplt.plot([0, 2 * size + pad], [size + pad / 2, size + pad / 2], \"k:\")\nplt.axis([0, 2 * size + pad, 0, 2 * size + pad])\nplt.xlabel(\"Predicted label\")\nplt.ylabel(\"True label\")\nplt.title(f'Comparison between {cl_a1} and {cl_b1}')\nplt.show()\n\n# Second figure (4 vs 9)\nplt.figure(figsize=(size, size))\nfor images, (label_col, label_row) in [(X_ba2, (0, 0)), (X_bb2, (1, 0)),\n                                       (X_aa2, (0, 1)), (X_ab2, (1, 1))]:\n    for idx, image_data in enumerate(images[:size*size]):\n        x = idx % size + label_col * (size + pad)\n        y = idx // size + label_row * (size + pad)\n        plt.imshow(image_data, cmap=\"binary\", extent=(x, x + 1, y, y + 1))\nplt.xticks([size / 2, size + pad + size / 2], [str(cl_a2), str(cl_b2)])\nplt.yticks([size / 2, size + pad + size / 2], [str(cl_b2), str(cl_a2)])\nplt.plot([size + pad / 2, size + pad / 2], [0, 2 * size + pad], \"k:\")\nplt.plot([0, 2 * size + pad], [size + pad / 2, size + pad / 2], \"k:\")\nplt.axis([0, 2 * size + pad, 0, 2 * size + pad])\nplt.xlabel(\"Predicted label\")\nplt.ylabel(\"True label\")\nplt.title(f'Comparison between {cl_a2} and {cl_b2}')\nplt.show()\n\n# Third figure (1 vs 8)\nplt.figure(figsize=(size, size))\nfor images, (label_col, label_row) in [(X_ba3, (0, 0)), (X_bb3, (1, 0)),\n                                       (X_aa3, (0, 1)), (X_ab3, (1, 1))]:\n    for idx, image_data in enumerate(images[:size*size]):\n        x = idx % size + label_col * (size + pad)\n        y = idx // size + label_row * (size + pad)\n        plt.imshow(image_data, cmap=\"binary\", extent=(x, x + 1, y, y + 1))\nplt.xticks([size / 2, size + pad + size / 2], [str(cl_a3), str(cl_b3)])\nplt.yticks([size / 2, size + pad + size / 2], [str(cl_b3), str(cl_a3)])\nplt.plot([size + pad / 2, size + pad / 2], [0, 2 * size + pad], \"k:\")\nplt.plot([0, 2 * size + pad], [size + pad / 2, size + pad / 2], \"k:\")\nplt.axis([0, 2 * size + pad, 0, 2 * size + pad])\nplt.xlabel(\"Predicted label\")\nplt.ylabel(\"True label\")\nplt.title(f'Comparison between {cl_a3} and {cl_b3}')\nplt.show()\n\n","metadata":{"executionInfo":{"elapsed":1317,"status":"ok","timestamp":1728979847187,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"4XtEYD0wR8nz","outputId":"f3fd5aa6-5478-46c3-c050-34bade0086eb"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It can be seen that this model struggles with certain misclassifications. For 7 and 9, as well as 4 and 9, one possible solution is to implement a holes counter. Meanwhile, for 1 and 8, a possible solution is augmenting the data with shifted or rotated images (as we can see, 1 that is misclassified as 8 tends to be more rotated). Before trying these solutions, we will test an SGD hybrid strategy that combines OvO and OvR, separating the confusing classes (1, 4, 7, 8, 9) from the non-confusing ones. We will use OvO for the confusing classes and OvR for the non-confusing ones.\n\n","metadata":{"id":"GyA4QRTKzoGd"}},{"cell_type":"markdown","source":"## Stochastic Gradient Descent - Hybrid","metadata":{"id":"EaSfzyw4Ngf7"}},{"cell_type":"markdown","source":"### Training the Model","metadata":{"id":"8eRGb4pPEMuY"}},{"cell_type":"markdown","source":"Now, we will implement the hybrid method. In this approach, we combine One-vs-Rest (OvR) and One-vs-One (OvO) strategies to improve prediction accuracy on confusing classes. The OvR model, previously trained, is loaded to make general predictions across all classes. During prediction, the OvR classifier is used for all samples, but for those predicted to be in the confusing classes, the OvO hybrid predictions replace the OvR predictions.","metadata":{"id":"_VEYROemDse0"}},{"cell_type":"code","source":"import json\nimport joblib\nfrom sklearn.multiclass import OneVsRestClassifier, OneVsOneClassifier\nfrom google.colab import files\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.metrics import classification_report, confusion_matrix\n\n# Load the best parameters from the saved JSON file\nwith open('SGDClassifier_OvR_best_params.json', 'r') as f:\n    best_params_ovo_hybrid = json.load(f)\n\n# Remove the \"classifier__\" prefix from the parameters\nbest_params_ovo_hybrid_cleaned = {key.split('__')[-1]: value for key, value in best_params_ovo_hybrid.items()}\n\n# Define the confusing classes\nconfusing_classes = [1, 4, 7, 8, 9]\n\n# Load the previously trained OvR model\novr_classifier = joblib.load(\"SGDClassifier_OvR_best_model.pkl\")\n\n# Train the OvO Hybrid model for the confusing classes\ndef create_pipeline(params):\n    model = SGDClassifier(random_state=42, **params)\n    pipeline = Pipeline([\n        ('scaler', StandardScaler()),\n        ('classifier', model)\n    ])\n    return pipeline\n\n# Create pipeline for OvO Hybrid using cleaned parameters\npipeline_ovo_hybrid = create_pipeline(best_params_ovo_hybrid_cleaned)\n\n# Extract training data for confusing classes\nX_train_confusing = X_small_train[np.isin(y_small_train, confusing_classes)]\ny_train_confusing = y_small_train[np.isin(y_small_train, confusing_classes)]\n\n# Train OvO Hybrid classifier for confusing classes\novo_hybrid_classifier = OneVsOneClassifier(pipeline_ovo_hybrid)\novo_hybrid_classifier.fit(X_train_confusing, y_train_confusing)\n\n# Make predictions on the validation set using OvR\ny_pred_ovr = ovr_classifier.predict(X_val)\n\n# Identify which samples in the validation set are confusing\nconfusing_indices = np.isin(y_pred_ovr, confusing_classes)\n\n# For confusing samples, replace OvR predictions with OvO Hybrid predictions\nX_val_confusing = X_val[confusing_indices]\ny_pred_confusing = ovo_hybrid_classifier.predict(X_val_confusing)\n\n# Combine OvR and OvO Hybrid predictions\ny_pred_combined = y_pred_ovr.copy()\ny_pred_combined[confusing_indices] = y_pred_confusing\n\n# Evaluate the combined predictions\nprint(\"\\nClassification Report for Hybrid OvO + OvR Classifier:\\n\")\nprint(classification_report(y_val, y_pred_combined))\nprint(f\"Confusion Matrix for Hybrid OvO + OvR Classifier:\\n\")\nprint(confusion_matrix(y_val, y_pred_combined))\n\n# Save and download the OvO Hybrid model\njoblib.dump(ovo_hybrid_classifier, \"OvO_Hybrid_classifier.pkl\")\nfiles.download(\"OvO_Hybrid_classifier.pkl\")\n","metadata":{"executionInfo":{"elapsed":583,"status":"ok","timestamp":1728979847766,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"FFhrAuDfVeoR","outputId":"fd88b968-4baf-4a09-f516-b9205c8e84d0"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We observed a decrease in accuracy from 0.9 to 0.89 compared to the previous OvR model. Next, we will further evaluate the model by displaying the confusion matrix to see whether using OvO for the confusing classes has any impact.\n","metadata":{"id":"XRef3Qw4EO8t"}},{"cell_type":"markdown","source":"### Error Analysis","metadata":{"id":"DY9Oj9jk2pi5"}},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(12, 5))\n\n# Confusion matrix for the Hybrid OvO + OvR Classifier\nConfusionMatrixDisplay.from_predictions(\n    y_val, y_pred_combined, normalize=\"true\", values_format=\".0%\", ax=axes[0], cmap='viridis'\n)\naxes[0].set_title('Confusion Matrix for Hybrid OvO + OvR Classifier')\n\n# Confusion matrix for the OvR-only Classifier\nConfusionMatrixDisplay.from_predictions(\n    y_val, ovr_y_pred, normalize=\"true\", values_format=\".0%\", ax=axes[1], cmap='viridis'\n)\naxes[1].set_title('Confusion Matrix for OvR-only Classifier')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"executionInfo":{"elapsed":628,"status":"ok","timestamp":1728979848390,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"rIko6LWZX1X-","outputId":"fd7be898-af2b-4cae-a7b7-fca4108fd860"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see, the accuracy has decreased for the previously confusing classes (1, 4, 9), while the accuracy for classes 7 and 8 has slightly increased. Now, we want to analyze the misclassified samples to understand the performance differences.","metadata":{"id":"KSmftonu0nUl"}},{"cell_type":"code","source":"\nfig, axes = plt.subplots(1, 2, figsize=(10, 4))\n\n# Misclassified samples for OvR Best Model\nsample_weight_1 = (ovr_y_pred != y_val).astype(int)\nConfusionMatrixDisplay.from_predictions(\n    y_val, ovr_y_pred, sample_weight=sample_weight_1,\n    ax=axes[0], cmap='viridis'\n)\naxes[0].set_title('Misclassified Samples (OvR Best Model)')\n\n# Misclassified samples for Hybrid OvO + OvR Model\nsample_weight_2 = (y_pred_combined != y_val).astype(int)\nConfusionMatrixDisplay.from_predictions(\n    y_val, y_pred_combined, sample_weight=sample_weight_2,\n    ax=axes[1], cmap='viridis'\n)\naxes[1].set_title('Misclassified Samples (Hybrid OvO + OvR Model)')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"executionInfo":{"elapsed":1129,"status":"ok","timestamp":1728979849516,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"CetMwtd_3gcq","outputId":"24b2661e-617f-4158-d964-d2d5de320d7a"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here, we can see that the most previous misclassifications have been increased with the hybrid strategy:\n\n*   1 misclassified as 8 (increased from 22 to 32),\n*   4 misclassified as 9 (increased from 36 to 53),\n*   7 misclassified as 9 (reduced from 38 to 23).\n\nSo, the hybrid strategy may not good enough for our case.","metadata":{"id":"JrsIfVBR1RYZ"}},{"cell_type":"markdown","source":"## Stochastic Gradient Descent - One versus One","metadata":{"id":"l3rJFaYONmje"}},{"cell_type":"markdown","source":"We observed the reduced in performance with the hybrid strategy. Now, we are curious to see whether using SGD with a pure OvO strategy can enhance the performance.","metadata":{"id":"EuUQU3pp2AK6"}},{"cell_type":"code","source":"from sklearn.model_selection import GridSearchCV, StratifiedKFold\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.multiclass import OneVsOneClassifier\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.metrics import make_scorer, accuracy_score, f1_score, confusion_matrix, classification_report\nimport joblib\nimport json\nfrom google.colab import files\n\n# Preprocessor\novo_preprocessor = StandardScaler()\n\n# Define the base classifier\novo_base_model = SGDClassifier(max_iter=1500, random_state=42)\n\n# Wrap the base classifier in OneVsOneClassifier\novo_classifier = OneVsOneClassifier(ovo_base_model)\n\n# Create pipeline with OneVsOneClassifier\novo_pipeline = Pipeline(steps=[\n    ('scaler', ovo_preprocessor),\n    ('classifier', ovo_classifier)\n])\n\n# Define parameter grid for the SGDClassifier (inside the OvO)\novo_param_grid = [\n    # For 'l2' and 'l1' penalties\n    {\n        'classifier__estimator__penalty': ['l2', 'l1'],\n        'classifier__estimator__alpha': [0.0001, 0.001, 0.01],\n    },\n    # For 'elasticnet' penalty\n    {\n        'classifier__estimator__penalty': ['elasticnet'],\n        'classifier__estimator__alpha': [0.0001, 0.001, 0.01],\n        'classifier__estimator__l1_ratio': [0.15, 0.5, 0.85],\n    }\n]\n\n# Stratified K-Fold CV\novo_strat_kfold = StratifiedKFold(n_splits=5)\n\n# Define scoring metrics\novo_scoring = {\n    'accuracy': make_scorer(accuracy_score),\n    'f1_macro': make_scorer(f1_score, average='macro')\n}\n\n# GridSearchCV with all scoring metrics for OvO classifier\novo_grid_search = GridSearchCV(\n    ovo_pipeline, ovo_param_grid, cv=ovo_strat_kfold, scoring=ovo_scoring, refit='accuracy', verbose=1, n_jobs=-1\n)\n\nprint(\"Training and tuning OvO SGDClassifier on small training set...\")\n\n# Fit the OvO classifier on small training set\novo_grid_search.fit(X_small_train, y_small_train)\n\n# Get best estimator and parameters\novo_best_estimator = ovo_grid_search.best_estimator_\novo_best_params = ovo_grid_search.best_params_\n\nprint(f\"Best parameters for OvO SGDClassifier: {ovo_best_params}\")\nprint(f\"Best Accuracy for OvO SGDClassifier: {ovo_grid_search.best_score_:.4f}\")\n\n# Evaluate on validation set\novo_y_pred = ovo_grid_search.predict(X_val)\nprint(\"\\nClassification Report for OvO SGDClassifier on Validation Set:\\n\")\nprint(classification_report(y_val, ovo_y_pred))\nprint(f\"Confusion Matrix for OvO SGDClassifier on Validation Set:\\n\")\nprint(confusion_matrix(y_val, ovo_y_pred))\n\n# Save best estimator\novo_filename = \"OvO_SGDClassifier_best_model.pkl\"\njoblib.dump(ovo_best_estimator, ovo_filename)\nprint(f\"Saved OvO SGDClassifier to {ovo_filename}\")\n\n# Save best parameters\novo_params_filename = 'OvO_SGDClassifier_best_params.json'\nwith open(ovo_params_filename, 'w') as f:\n    json.dump(ovo_best_params, f)\nprint(f\"Saved best parameters to {ovo_params_filename}\")\n\n# Download files\nfiles.download(ovo_filename)\nfiles.download(ovo_params_filename)\n","metadata":{"executionInfo":{"elapsed":43252,"status":"ok","timestamp":1728979892763,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"uK9YbLRd7rcw","outputId":"7c9ddd1d-d3a2-441b-b439-64cdc2769d44"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see, the performance improved with the pure OvO strategy, so this model will be a good candidate.\n","metadata":{"id":"glWgndGL2lA4"}},{"cell_type":"markdown","source":"## Logistic Regression","metadata":{"id":"1laX3rgYNrT2"}},{"cell_type":"markdown","source":"Lastly, we will try using logistic regression to see if it offers better performance than the previous models.","metadata":{"id":"mRQQxOcc2yzr"}},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.metrics import accuracy_score, classification_report, confusion_matrix\nimport joblib\nimport json\n\n# Preprocessor\npreprocessor = StandardScaler()\n\n# Base Model (Logistic Regression)\nbase_model = LogisticRegression(max_iter=1000, random_state=42)\n\n# Pipeline\npipeline = Pipeline(steps=[\n    ('scaler', preprocessor),\n    ('classifier', base_model)\n])\n\n# Expanded Parameter Grid\nparam_grid = [\n    # 'l2' penalty with 'lbfgs' solver\n    {\n        'classifier__penalty': ['l2'],\n        'classifier__C': [0.1, 1, 10],\n        'classifier__solver': ['lbfgs', 'newton-cg'],\n        'classifier__multi_class': ['multinomial'],\n        'classifier__tol': [1e-4, 1e-3],\n    },\n    # 'l1' penalty with 'saga' solver\n    {\n        'classifier__penalty': ['l1'],\n        'classifier__C': [0.1, 1, 10],\n        'classifier__solver': ['saga'],\n        'classifier__multi_class': ['multinomial'],\n        'classifier__tol': [1e-4, 1e-3],\n    },\n    # 'elasticnet' penalty with 'saga' solver\n    {\n        'classifier__penalty': ['elasticnet'],\n        'classifier__C': [0.1, 1, 10],\n        'classifier__l1_ratio': [0.1, 0.5, 0.9],\n        'classifier__solver': ['saga'],\n        'classifier__multi_class': ['multinomial'],\n        'classifier__tol': [1e-4, 1e-3],\n    }\n]\n\n# Cross-Validation Strategy\nstrat_kfold = StratifiedKFold(n_splits=3)\n\n# Scoring Metric: Use accuracy as the sole metric\nscoring = 'accuracy'\n\n# Grid Search with expanded parameters\ngrid_search = GridSearchCV(\n    pipeline,\n    param_grid,\n    cv=strat_kfold,\n    scoring=scoring,  # Use accuracy for evaluation\n    verbose=1,\n    n_jobs=-1\n)\nprint(\"Training and tuning LogisticRegression with expanded parameter grid on small training set...\")\n\n# Fit the model on the small training set\ngrid_search.fit(X_small_train, y_small_train)\n\n# Best Model and Parameters\nbest_estimator = grid_search.best_estimator_\nbest_params = grid_search.best_params_\nbest_score = grid_search.best_score_\nprint(f\"\\nBest parameters: {best_params}\")\nprint(f\"Best cross-validated accuracy: {best_score:.4f} ({best_score * 100:.2f}%)\")\n\n# Evaluation on Validation Set\ny_pred = best_estimator.predict(X_val)\nval_accuracy = accuracy_score(y_val, y_pred)\nprint(f\"\\nValidation Set Accuracy: {val_accuracy:.4f} ({val_accuracy * 100:.2f}%)\")\n\n# Classification Report and Confusion Matrix\nprint(\"\\nClassification Report:\\n\")\nprint(classification_report(y_val, y_pred))\nprint(f\"Confusion Matrix:\\n{confusion_matrix(y_val, y_pred)}\")\n\n# Save the Model\njoblib.dump(best_estimator, 'LogisticRegression_SmallTrain.pkl')\nwith open('best_params_small_train.json', 'w') as f:\n    json.dump(best_params, f)\n\nprint(\"\\nModel and best parameters saved.\")\n","metadata":{"executionInfo":{"elapsed":2230651,"status":"ok","timestamp":1728982123410,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"8tG6YL3OZzfA","outputId":"5d9d227e-6e7e-4bd6-f26f-11e917c2620c"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This model produce lower accuracy than the previous model. So, we will  use Stochastic Gradient Descent with OvO strategy","metadata":{"id":"4xQ5VHSFID_5"}},{"cell_type":"markdown","source":"# Fine Tuning The Model","metadata":{"id":"5DTscpNe3THH"}},{"cell_type":"markdown","source":"### Further Error Analysis","metadata":{"id":"cRWDGV24I3Om"}},{"cell_type":"markdown","source":"We observed that the best model was Stochastic Gradient Descent with a One-vs-One strategy. Now, we want to explore how we can further improve it by conducting an error analysis using the real training dataset.","metadata":{"id":"315jYcgy3syO"}},{"cell_type":"code","source":"from sklearn.metrics import ConfusionMatrixDisplay\nimport matplotlib.pyplot as plt\n\n# Step 1: Generate the confusion matrix display for the OvO classifier\nfig, ax = plt.subplots(figsize=(8, 6))\n\n# Confusion matrix for the OvO classifier\nConfusionMatrixDisplay.from_predictions(\n    y_val, ovo_y_pred, normalize=\"true\", values_format=\".0%\", ax=ax, cmap='viridis'\n)\n\n# Step 2: Set title and display the plot\nax.set_title('Confusion Matrix for OvO SGDClassifier on Validation Set')\nplt.tight_layout()\nplt.show()\n","metadata":{"executionInfo":{"elapsed":4,"status":"ok","timestamp":1728982123410,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"Reid6c3RUNoK","outputId":"23254818-f3bf-43ec-9511-9d1c717a4981"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import ConfusionMatrixDisplay\nimport matplotlib.pyplot as plt\n\n\nsample_weight = (ovo_y_pred != y_val).astype(int)\n\n# Display the confusion matrix for misclassified samples using OvO predictions\nConfusionMatrixDisplay.from_predictions(y_val, ovo_y_pred,\n                                        sample_weight=sample_weight,\n                                        normalize=\"true\", values_format=\".0%\")\nplt.title('Confusion Matrix of Misclassified Samples on Validation Data (OvO)')\nplt.show()\n","metadata":{"executionInfo":{"elapsed":727,"status":"ok","timestamp":1728982124134,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"iwD51DmyU_eZ","outputId":"bf4b7ddd-09b6-448a-8ba6-c558ebf15d92"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From this, we can see that some classes are still difficult to differentiate with the current models:\n- 1 misclassified as 8\n- 4 misclassified as 9\n- 7 misclassified as 9\n\nTo address this issue, we will further analyze the images by comparing correctly classified and misclassified samples side by side.\n","metadata":{"id":"5bSezLrJEltm"}},{"cell_type":"code","source":"class_pairs = [\n    (1, 8),\n    (4, 9),\n    (7, 9)\n]\n\nX_val_images = X_val.values\nX_val_images = X_val_images.reshape(-1, 28, 28)\n\n# Function to plot the comparison between two classes\ndef plot_comparison(cl_a, cl_b, size=5, pad=0.2):\n    # Get the images for each case\n    X_aa = X_val_images[(y_val == cl_a) & (ovr_y_pred == cl_a)]\n    X_ab = X_val_images[(y_val == cl_a) & (ovr_y_pred == cl_b)]\n    X_ba = X_val_images[(y_val == cl_b) & (ovr_y_pred == cl_a)]\n    X_bb = X_val_images[(y_val == cl_b) & (ovr_y_pred == cl_b)]\n\n    # Create the plot\n    plt.figure(figsize=(size, size))\n    for images, (label_col, label_row) in [(X_ba, (0, 0)), (X_bb, (1, 0)),\n                                           (X_aa, (0, 1)), (X_ab, (1, 1))]:\n        for idx, image_data in enumerate(images[:size*size]):\n            x = idx % size + label_col * (size + pad)\n            y = idx // size + label_row * (size + pad)\n            plt.imshow(image_data, cmap=\"binary\", extent=(x, x + 1, y, y + 1))\n\n    # Add labels, lines, and title\n    plt.xticks([size / 2, size + pad + size / 2], [str(cl_a), str(cl_b)])\n    plt.yticks([size / 2, size + pad + size / 2], [str(cl_b), str(cl_a)])\n    plt.plot([size + pad / 2, size + pad / 2], [0, 2 * size + pad], \"k:\")\n    plt.plot([0, 2 * size + pad], [size + pad / 2, size + pad / 2], \"k:\")\n    plt.axis([0, 2 * size + pad, 0, 2 * size + pad])\n    plt.xlabel(\"Predicted label\")\n    plt.ylabel(\"True label\")\n    plt.title(f'Comparison between {cl_a} and {cl_b}')\n    plt.show()\n\n# Plot for each class pair\nfor cl_a, cl_b in class_pairs:\n    plot_comparison(cl_a, cl_b)\n\n","metadata":{"executionInfo":{"elapsed":764,"status":"ok","timestamp":1728982124891,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"CzkASn4TIwCa","outputId":"8ac00cab-ff1f-4d5b-9ac7-757a56cc433d"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The previous analysis still holds: for 7 and 9, as well as 4 and 9, one possible solution is to implement a holes counter. Meanwhile, for 1 and 8, a possible solution is to augment the data with shifted or rotated images (as we can see, 1 that is misclassified as 8 tends to be more rotated).","metadata":{"id":"GYdXMf_zJPp7"}},{"cell_type":"markdown","source":"### Creating Close Loops Counter","metadata":{"id":"p0mO8FWOJzci"}},{"cell_type":"markdown","source":"Here, we aim to create and test a closed loops function that will hopefully help us better differentiate between 4, 7 and 9.\n","metadata":{"id":"9_vNLjk6KRMR"}},{"cell_type":"code","source":"from skimage.measure import label, regionprops\nfrom skimage.morphology import remove_small_objects, binary_closing\nimport numpy as np\nfrom skimage.morphology import remove_small_holes, binary_opening, disk\nfrom skimage.filters import threshold_otsu\n\ndef compute_num_holes(image, min_hole_size=5):\n    image_2d = image.reshape(28, 28)\n\n    thresh = threshold_otsu(image_2d)\n    binary_image = image_2d > thresh\n\n    structuring_element = disk(2)  # A circular element for closing\n    binary_image = binary_closing(binary_image, structuring_element)\n\n    binary_image = binary_opening(binary_image, disk(1))\n\n    inverted_image = ~binary_image\n\n    labeled_image = label(inverted_image, connectivity=2)\n\n    labeled_image = remove_small_objects(labeled_image, min_size=min_hole_size)\n\n    num_holes = len(np.unique(labeled_image)) - 2\n    num_holes = max(num_holes, 0)\n\n    return num_holes\n\nif isinstance(X_train, pd.DataFrame):\n    X_train_array = X_train.to_numpy()\nelse:\n    X_train_array = X_train\n\nnum_holes_train = [compute_num_holes(img) for img in X_train_array]\n\ndf_train = pd.DataFrame({\n    'row_number': np.arange(len(X_train_array)),\n    'true_label': y_train,\n    'num_holes': num_holes_train\n})\n\nhole_counts_train = df_train.groupby(['true_label', 'num_holes']).size().unstack(fill_value=0)\n\nprint(hole_counts_train)","metadata":{"executionInfo":{"elapsed":21169,"status":"ok","timestamp":1728982146058,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"jBnyhF2FJ24N","outputId":"ac4d3d45-9f4c-48b6-8f78-1ca14996f202"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here, we can see that our closed loops counter did a good job of counting holes in classes 1 and 7. Now, we want to plot a list of images in classes 1 and 7 that were detected to have more than 0 holes.","metadata":{"id":"-Tgi_RIf_6ad"}},{"cell_type":"code","source":"# Number of misclassified images to show\nnum_images_to_show = 28\nimages_per_row = 7\n\n# Get the misclassified rows for class 1 and 7 where holes are more than 0\nmisclassified_1_7 = df_train[\n    (df_train['true_label'].isin([1, 7])) & (df_train['num_holes'] > 0)\n].head(num_images_to_show)\n\n# Get the rows for class 8 where holes are fewer than 2\nmisclassified_8 = df_train[\n    (df_train['true_label'] == 8) & (df_train['num_holes'] == 0)\n].head(num_images_to_show)\n\n# Get the rows for class 4 where holes are more than 0\nmisclassified_4 = df_train[\n    (df_train['true_label'] == 4) & (df_train['num_holes'] > 0)\n].head(num_images_to_show)\n\n# Combine all sets of misclassified rows\nmisclassified_images = pd.concat([misclassified_1_7, misclassified_4, misclassified_8]).head(num_images_to_show)\n\n# Get the row numbers\nrow_positions = misclassified_images['row_number'].tolist()\n\n# Calculate the number of rows needed\nnum_rows = int(np.ceil(num_images_to_show / images_per_row))\n\n# Create subplots\nfig, axes = plt.subplots(num_rows, images_per_row, figsize=(12, num_rows * 2))\naxes = axes.flatten()\n\n# Loop over the misclassified images and plot each one\nfor i, pos in enumerate(row_positions):\n    image = X_train_array[pos].reshape(28, 28)\n    axes[i].imshow(image, cmap='gray')\n    label = misclassified_images.iloc[i]['true_label']\n    holes = misclassified_images.iloc[i]['num_holes']\n    axes[i].set_title(f\"Label: {label}, Holes: {holes}\")\n    axes[i].axis('off')\n\n# Hide any unused subplots\nfor i in range(len(misclassified_images), len(axes)):\n    axes[i].axis('off')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"executionInfo":{"elapsed":1522,"status":"ok","timestamp":1728982147579,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"Zdt8uhdRKDw7","outputId":"c757cac9-2806-4e93-f1be-681ce18c9ec3"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here, we can see that the mistakes made by the closed loops counter are purely due to its own inaccuracies (resulting from low accuracy). Since this model is currently our best, we will explore another proposed solution and consider how this feature can be integrated into the pipeline later.","metadata":{"id":"Ky2xiWzTCVIo"}},{"cell_type":"markdown","source":"### Augment Dataset (Shifting Images)","metadata":{"id":"I_aeLi06RK_D"}},{"cell_type":"markdown","source":"Here, we will incorporate the real dataset with shifted images—up, down, left, and right—for each image, effectively increasing the size of the training dataset by five times.","metadata":{"id":"Vy7F4eo1DQ6X"}},{"cell_type":"code","source":"from scipy.ndimage import shift\n\nif isinstance(X_train, list):\n    X_train = np.array(X_train, dtype=np.float32)\n\nX_train = np.array(X_train, dtype=np.float32)\ny_train = np.array(y_train, dtype=np.int32)\n\n\ndef shift_image(image, direction):\n    if direction == 'up':\n        return shift(image, [-1, 0], cval=0)\n    elif direction == 'down':\n        return shift(image, [1, 0], cval=0)\n    elif direction == 'left':\n        return shift(image, [0, -1], cval=0)\n    elif direction == 'right':\n        return shift(image, [0, 1], cval=0)\n    else:\n        raise ValueError(\"Direction must be one of 'up', 'down', 'left', 'right'.\")\n\ndef augment_dataset(X_train, y_train):\n    augmented_images = []\n    augmented_labels = []\n\n    for img, label in zip(X_train, y_train):\n        img_reshaped = img.reshape(28, 28)  # Reshape flat image to 28x28\n\n        # Append the original image\n        augmented_images.append(img)\n        augmented_labels.append(label)\n\n        # Append shifted images\n        for direction in ['up', 'down', 'left', 'right']:\n            shifted_img = shift_image(img_reshaped, direction)\n            augmented_images.append(shifted_img.flatten())  # Flatten back to 1D\n            augmented_labels.append(label)\n\n    return np.array(augmented_images), np.array(augmented_labels)\n\n# Example usage:\nX_augmented, y_augmented = augment_dataset(X_train, y_train)\n\nprint(f\"Original dataset size: {X_train.shape}\")\nprint(f\"Augmented dataset size: {X_augmented.shape}\")\n","metadata":{"executionInfo":{"elapsed":12204,"status":"ok","timestamp":1728982289547,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"h7MNv-k6RMjS","outputId":"2534d484-6e23-4364-ed17-bbc9996270d9"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here, before training the SGD with the OvO strategy using the augmented dataset, we will first retrain it with the original training dataset to ensure a fair comparison.","metadata":{"id":"cUrGxnAmoTs2"}},{"cell_type":"code","source":"# Preprocessor\novo_preprocessor = StandardScaler()\n\n# Define the base classifier\novo_base_model = SGDClassifier(max_iter=1500, random_state=42)\n\n# Wrap the base classifier in OneVsOneClassifier\novo_classifier = OneVsOneClassifier(ovo_base_model)\n\n# Create pipeline with OneVsOneClassifier\novo_pipeline = Pipeline(steps=[\n    ('scaler', ovo_preprocessor),\n    ('classifier', ovo_classifier)\n])\n\n# Define parameter grid for the SGDClassifier (inside the OvO)\novo_param_grid = [\n    # For 'l2' and 'l1' penalties\n    {\n        'classifier__estimator__penalty': ['l2', 'l1'],\n        'classifier__estimator__alpha': [0.0001, 0.001, 0.01],\n    },\n    # For 'elasticnet' penalty\n    {\n        'classifier__estimator__penalty': ['elasticnet'],\n        'classifier__estimator__alpha': [0.0001, 0.001, 0.01],\n        'classifier__estimator__l1_ratio': [0.15, 0.5, 0.85],\n    }\n]\n\n# Stratified K-Fold CV\novo_strat_kfold = StratifiedKFold(n_splits=5)\n\n# Define scoring metrics\novo_scoring = {\n    'accuracy': make_scorer(accuracy_score),\n    'f1_macro': make_scorer(f1_score, average='macro')\n}\n\n# GridSearchCV with all scoring metrics for OvO classifier\novo_grid_search = GridSearchCV(\n    ovo_pipeline, ovo_param_grid, cv=ovo_strat_kfold, scoring=ovo_scoring, refit='accuracy', verbose=1, n_jobs=-1\n)\n\nprint(\"Training and tuning OvO SGDClassifier on small training set...\")\n\n# Fit the OvO classifier on small training set\novo_grid_search.fit(X_train, y_train)\n\n# Get best estimator and parameters\novo_best_estimator = ovo_grid_search.best_estimator_\novo_best_params = ovo_grid_search.best_params_\n\nprint(f\"Best parameters for OvO SGDClassifier: {ovo_best_params}\")\nprint(f\"Best Accuracy for OvO SGDClassifier: {ovo_grid_search.best_score_:.4f}\")\n\n# Evaluate on validation set\novo_y_pred = ovo_grid_search.predict(X_val)\nprint(\"\\nClassification Report for OvO SGDClassifier on Validation Set:\\n\")\nprint(classification_report(y_val, ovo_y_pred))\nprint(f\"Confusion Matrix for OvO SGDClassifier on Validation Set:\\n\")\nprint(confusion_matrix(y_val, ovo_y_pred))\n\n\n# Download files\nfiles.download(ovo_filename)\nfiles.download(ovo_params_filename)","metadata":{"executionInfo":{"elapsed":729141,"status":"ok","timestamp":1728983018686,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"kVe7FiiXTMQR","outputId":"c1e1d6b7-3d68-4e30-f1a2-6c51ab5ba226"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We observed that the accuracy of the OvO model is 0.93 with the original training data. Now, we will train it using the augmented dataset.","metadata":{"id":"8eC4lUlKR95z"}},{"cell_type":"code","source":"# Preprocessor for the model with shifted data\novo_shift_preprocessor = StandardScaler()\n\n# Define the base classifier\novo_shift_base_model = SGDClassifier(max_iter=1500, random_state=42)\n\n# Wrap the base classifier in OneVsOneClassifier\novo_shift_classifier = OneVsOneClassifier(ovo_shift_base_model)\n\n# Create pipeline with OneVsOneClassifier\novo_shift_pipeline = Pipeline(steps=[\n    ('scaler', ovo_shift_preprocessor),\n    ('classifier', ovo_shift_classifier)\n])\n\n# Define parameter grid for the SGDClassifier (inside the OvO)\novo_shift_param_grid = [\n    # For 'l2' and 'l1' penalties\n    {\n        'classifier__estimator__penalty': ['l2', 'l1'],\n        'classifier__estimator__alpha': [0.0001, 0.001, 0.01],\n    },\n    # For 'elasticnet' penalty\n    {\n        'classifier__estimator__penalty': ['elasticnet'],\n        'classifier__estimator__alpha': [0.0001, 0.001, 0.01],\n        'classifier__estimator__l1_ratio': [0.15, 0.5, 0.85],\n    }\n]\n\n# Stratified K-Fold CV\novo_shift_strat_kfold = StratifiedKFold(n_splits=5)\n\n# Define scoring metrics\novo_shift_scoring = {\n    'accuracy': make_scorer(accuracy_score),\n    'f1_macro': make_scorer(f1_score, average='macro')\n}\n\n# GridSearchCV with all scoring metrics for OvO classifier\novo_shift_grid_search = GridSearchCV(\n    ovo_shift_pipeline, ovo_shift_param_grid, cv=ovo_shift_strat_kfold, scoring=ovo_shift_scoring, refit='accuracy', verbose=1, n_jobs=-1\n)\n\nprint(\"Training and tuning OvO SGDClassifier with shift augmentation...\")\n\n# Fit the OvO classifier on the shifted dataset\novo_shift_grid_search.fit(X_augmented, y_augmented)\n\n# Get best estimator and parameters\novo_shift_best_estimator = ovo_shift_grid_search.best_estimator_\novo_shift_best_params = ovo_shift_grid_search.best_params_\n\nprint(f\"Best parameters for OvO SGDClassifier (with shift): {ovo_shift_best_params}\")\nprint(f\"Best Accuracy for OvO SGDClassifier (with shift): {ovo_shift_grid_search.best_score_:.4f}\")\n\n# Evaluate on validation set\novo_shift_y_pred = ovo_shift_grid_search.predict(X_val)\nprint(\"\\nClassification Report for OvO SGDClassifier on Validation Set (with shift):\\n\")\nprint(classification_report(y_val, ovo_shift_y_pred))\nprint(f\"Confusion Matrix for OvO SGDClassifier on Validation Set (with shift):\\n\")\nprint(confusion_matrix(y_val, ovo_shift_y_pred))\n\n# Save best estimator for model with shift\novo_shift_filename = \"OvO_SGDClassifier_best_model_with_shift.pkl\"\njoblib.dump(ovo_shift_best_estimator, ovo_shift_filename)\nprint(f\"Saved OvO SGDClassifier with shift to {ovo_shift_filename}\")\n\n# Save best parameters for model with shift\novo_shift_params_filename = 'OvO_SGDClassifier_best_params_with_shift.json'\nwith open(ovo_shift_params_filename, 'w') as f:\n    json.dump(ovo_shift_best_params, f)\nprint(f\"Saved best parameters for OvO SGDClassifier with shift to {ovo_shift_params_filename}\")\n\n# Download files\nfiles.download(ovo_shift_filename)\nfiles.download(ovo_shift_params_filename)\n","metadata":{"executionInfo":{"elapsed":3101495,"status":"ok","timestamp":1728988730233,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"NbDz36cfSB82","outputId":"26bf69be-f172-41fc-a77e-b22e49742dcf"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see that by augmenting the dataset, the accuracy increased from 0.93 to 0.94. Now, we will display the confusion matrix to identify which classes are most commonly misclassified as others.","metadata":{"id":"0xTdcW_jFQcd"}},{"cell_type":"code","source":"sample_weight_pure = (ovo_y_pred != y_val).astype(int)\nsample_weight_shift = (ovo_shift_y_pred != y_val).astype(int)\n\nfig, axes = plt.subplots(1, 2, figsize=(14, 6))\n\nConfusionMatrixDisplay.from_predictions(y_val, ovo_y_pred,\n                                        sample_weight=sample_weight_pure,\n                                        normalize=\"true\", values_format=\".0%\",\n                                        ax=axes[0])\naxes[0].set_title('Confusion Matrix of Misclassified Samples on Validation Data (OvO Pure)')\n\nConfusionMatrixDisplay.from_predictions(y_val, ovo_shift_y_pred,\n                                        sample_weight=sample_weight_shift,\n                                        normalize=\"true\", values_format=\".0%\",\n                                        ax=axes[1])\naxes[1].set_title('Confusion Matrix of Misclassified Samples on Validation Data (OvO with Shift)')\n\nplt.tight_layout()\nplt.show()","metadata":{"executionInfo":{"elapsed":1136,"status":"ok","timestamp":1728988731367,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"3Wpln7CJLVu2","outputId":"b1a0d0e0-0472-44c3-e6af-38e2372f957c"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now, we want to evaluate how the closed loops hole counter performs with the augmented dataset.","metadata":{"id":"PGxPCoCJOGfw"}},{"cell_type":"code","source":"from skimage.measure import label as skimage_label  # Rename the label function to avoid conflict\n\ndef compute_num_holes(image, min_hole_size=5, binarization_threshold=0.1):\n    image_2d = image.reshape(28, 28)\n\n    binary_image = image_2d > binarization_threshold\n\n    structuring_element = disk(2)\n    binary_image = binary_closing(binary_image, structuring_element)\n\n    inverted_image = ~binary_image\n\n    labeled_image = skimage_label(inverted_image, connectivity=2)\n\n    labeled_image = remove_small_objects(labeled_image, min_size=min_hole_size)\n\n    num_holes = len(np.unique(labeled_image)) - 2\n    num_holes = max(num_holes, 0)\n\n    return num_holes\n\n\nif isinstance(X_augmented, pd.DataFrame):\n    X_augmented_array = X_augmented.to_numpy()\nelse:\n    X_augmented_array = X_augmented\n\n# Compute the number of holes for each image in the augmented training set\nnum_holes_augmented = [compute_num_holes(img) for img in X_augmented_array]\n\n# Create a DataFrame with true labels, number of holes, and row numbers\ndf_augmented = pd.DataFrame({\n    'row_number': np.arange(len(X_augmented_array)),\n    'true_label': y_augmented,\n    'num_holes': num_holes_augmented\n})\n\n# Group by true label and number of holes to see the counts for the augmented dataset\nhole_counts_augmented = df_augmented.groupby(['true_label', 'num_holes']).size().unstack(fill_value=0)\n\n# Display the counts for each digit in the augmented dataset\nprint(hole_counts_augmented)\n\n","metadata":{"executionInfo":{"elapsed":56372,"status":"ok","timestamp":1728988787739,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"s6ml92rnLvfr","outputId":"d309738a-6853-4fef-ae9a-fc31a649ae53"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see, the closed loop counter still has good accuracy in detecting 1, but performs poorly in detecting 7. We will determine later how to incorporate this feature into the pipeline.","metadata":{"id":"Gu2O2BZpRz97"}},{"cell_type":"markdown","source":"### Augment Dataset (Rotating + Shifting Images)","metadata":{"id":"8gTDaTITudc8"}},{"cell_type":"markdown","source":"Here, we will further augment the data by adding a 45-degree clockwise and a 45-degree counterclockwise rotation to each image to see if we can enhance the performance further. This will result in the new augmented dataset being 7 times larger.","metadata":{"id":"Ogim6J1mhbpI"}},{"cell_type":"code","source":"from skimage.transform import rotate\nfrom scipy.ndimage import shift\n\ndef augment_image_with_rotation(image):\n    \"\"\"\n    Augment the image by creating shifted and rotated versions.\n    \"\"\"\n    image_2d = image.reshape(28, 28)  # Reshape the flat image into 28x28\n\n    # Shift the image in all four directions\n    shifted_up = shift(image_2d, [-1, 0], cval=0)\n    shifted_down = shift(image_2d, [1, 0], cval=0)\n    shifted_left = shift(image_2d, [0, -1], cval=0)\n    shifted_right = shift(image_2d, [0, 1], cval=0)\n\n    # Rotate the image by 45 degrees and -45 degrees\n    rotated_45 = rotate(image_2d, 45, mode='edge')\n    rotated_minus_45 = rotate(image_2d, -45, mode='edge')\n\n    # Flatten the images back into 1D arrays\n    augmented_images = [\n        image,  # original\n        shifted_up.flatten(),\n        shifted_down.flatten(),\n        shifted_left.flatten(),\n        shifted_right.flatten(),\n        rotated_45.flatten(),\n        rotated_minus_45.flatten()\n    ]\n\n    return augmented_images\n\ndef augment_dataset_with_rotation(X_train, y_train):\n    \"\"\"\n    Create an augmented dataset with shifted and rotated images.\n    \"\"\"\n    X_augmented_with_rotation = []\n    y_augmented_with_rotation = []\n\n    for img, label in zip(X_train, y_train):\n        augmented_images = augment_image_with_rotation(img)\n        X_augmented_with_rotation.extend(augmented_images)\n        y_augmented_with_rotation.extend([label] * len(augmented_images))\n\n    return np.array(X_augmented_with_rotation), np.array(y_augmented_with_rotation)\n\n# Example usage:\nX_augmented_with_rotation, y_augmented_with_rotation = augment_dataset_with_rotation(X_train, y_train)\nprint(f\"Original dataset size: {X_train.shape}\")\nprint(f\"Augmented dataset size with rotation: {X_augmented_with_rotation.shape}\")\n","metadata":{"executionInfo":{"elapsed":25555,"status":"ok","timestamp":1728988938972,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"xf2BPHduuhUQ","outputId":"b319e01d-60df-4169-ef90-0e6344cd9321"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here, we will train the model using the SGDClassifier with the OvO strategy on the dataset augmented with both shifting and rotating.\n","metadata":{"id":"roiz1I4Koy6d"}},{"cell_type":"code","source":"# Preprocessor for the model with rotated and shifted data\novo_rot_preprocessor = StandardScaler()\n\n# Define the base classifier\novo_rot_base_model = SGDClassifier(max_iter=1500, random_state=42)\n\n# Wrap the base classifier in OneVsOneClassifier\novo_rot_classifier = OneVsOneClassifier(ovo_rot_base_model)\n\n# Create pipeline with OneVsOneClassifier\novo_rot_pipeline = Pipeline(steps=[\n    ('scaler', ovo_rot_preprocessor),\n    ('classifier', ovo_rot_classifier)\n])\n\n# Define parameter grid for the SGDClassifier (inside the OvO)\novo_rot_param_grid = [\n    # For 'l2' and 'l1' penalties\n    {\n        'classifier__estimator__penalty': ['l2', 'l1'],\n        'classifier__estimator__alpha': [0.0001, 0.001, 0.01],\n    },\n    # For 'elasticnet' penalty\n    {\n        'classifier__estimator__penalty': ['elasticnet'],\n        'classifier__estimator__alpha': [0.0001, 0.001, 0.01],\n        'classifier__estimator__l1_ratio': [0.15, 0.5, 0.85],\n    }\n]\n\n# Stratified K-Fold CV\novo_rot_strat_kfold = StratifiedKFold(n_splits=5)\n\n# Define scoring metrics\novo_rot_scoring = {\n    'accuracy': make_scorer(accuracy_score),\n    'f1_macro': make_scorer(f1_score, average='macro')\n}\n\n# GridSearchCV with all scoring metrics for OvO classifier\novo_rot_grid_search = GridSearchCV(\n    ovo_rot_pipeline, ovo_rot_param_grid, cv=ovo_rot_strat_kfold, scoring=ovo_rot_scoring, refit='accuracy', verbose=1, n_jobs=-1\n)\n\nprint(\"Training and tuning OvO SGDClassifier with rotation augmentation...\")\n\n# Fit the OvO classifier on the augmented dataset\novo_rot_grid_search.fit(X_augmented_with_rotation, y_augmented_with_rotation)\n\n# Get best estimator and parameters\novo_rot_best_estimator = ovo_rot_grid_search.best_estimator_\novo_rot_best_params = ovo_rot_grid_search.best_params_\n\nprint(f\"Best parameters for OvO SGDClassifier (with rotation): {ovo_rot_best_params}\")\nprint(f\"Best Accuracy for OvO SGDClassifier (with rotation): {ovo_rot_grid_search.best_score_:.4f}\")\n\n# Evaluate on validation set\novo_rot_y_pred = ovo_rot_grid_search.predict(X_val)\nprint(\"\\nClassification Report for OvO SGDClassifier on Validation Set (with rotation):\\n\")\nprint(classification_report(y_val, ovo_rot_y_pred))\nprint(f\"Confusion Matrix for OvO SGDClassifier on Validation Set (with rotation):\\n\")\nprint(confusion_matrix(y_val, ovo_rot_y_pred))\n\n# Save best estimator for model with rotation\novo_rot_filename = \"OvO_SGDClassifier_best_model_with_rot.pkl\"\njoblib.dump(ovo_rot_best_estimator, ovo_rot_filename)\nprint(f\"Saved OvO SGDClassifier with rotation to {ovo_rot_filename}\")\n\n# Save best parameters for model with rotation\novo_rot_params_filename = 'OvO_SGDClassifier_best_params_with_rot.json'\nwith open(ovo_rot_params_filename, 'w') as f:\n    json.dump(ovo_rot_best_params, f)\nprint(f\"Saved best parameters for OvO SGDClassifier with rotation to {ovo_rot_params_filename}\")\n\n# Download files\nfiles.download(ovo_rot_filename)\nfiles.download(ovo_rot_params_filename)\n","metadata":{"id":"iqLfdtIDu57B","executionInfo":{"status":"ok","timestamp":1728998453878,"user_tz":-180,"elapsed":1351355,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"}},"outputId":"6f87b351-d18a-4938-8873-80890e7b8f27"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see, the performance did not improve. The best model remains the SGDClassifier using the OvO strategy on the dataset augmented with shifts only.","metadata":{"id":"zfzRaL8ep72B"}},{"cell_type":"markdown","source":"### Confidence Level","metadata":{"id":"pu7iGLT0SJ2o"}},{"cell_type":"markdown","source":"Here, we will try whether we can improve out model performance by applying some threshold on class 8 since there are so many digits that misclassified as class 8. We will try to check average confidence for correctly classified as 8 and misclassified as 8 on the augmented data (not validation data) to avoid data leakage","metadata":{"id":"KOKIAs1-qR_A"}},{"cell_type":"code","source":"def analyze_class_8_confidence(X_train_augmented, y_train_augmented, y_train_pred, model):\n    correct_confidences = []\n    incorrect_confidences = []\n\n    # Get decision function values for all samples\n    decision_values = model.decision_function(X_train_augmented)\n\n    for idx, (true_label, pred_label) in enumerate(zip(y_train_augmented, y_train_pred)):\n        # We are interested in class 8 predictions\n        if pred_label == 8:\n            confidence = decision_values[idx][8]  # Confidence for class 8\n            if true_label == 8:\n                correct_confidences.append(confidence)\n            else:\n                incorrect_confidences.append(confidence)\n\n    # Compute average confidence scores\n    avg_correct_confidence = np.mean(correct_confidences) if correct_confidences else 'N/A'\n    avg_incorrect_confidence = np.mean(incorrect_confidences) if incorrect_confidences else 'N/A'\n\n    print(f\"Average Confidence for Correctly Classified as 8: {avg_correct_confidence}\")\n    print(f\"Average Confidence for Misclassified as 8: {avg_incorrect_confidence}\")\n\n    return correct_confidences, incorrect_confidences\n\n# Predict on training set using the trained OvO model (on augmented data)\ny_pred_augmented = ovo_shift_best_estimator.predict(X_augmented)\n\n# Run the analysis for class 8 confidence levels on training augmented data\ncorrect_confidences_8_train, incorrect_confidences_8_train = analyze_class_8_confidence(\n    X_augmented, y_augmented, y_pred_augmented, ovo_shift_best_estimator\n)\n","metadata":{"executionInfo":{"elapsed":14511,"status":"ok","timestamp":1729003866370,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"fvv3F4e6SMUP","outputId":"41097888-1ecc-4b79-fcbc-08a73fa528c5"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now, we will apply a threshold of 9 as the minimum confidence value to be classified as 8. If this threshold is not met, we will classify the digit into the second-best class based on confidence.\n","metadata":{"id":"twA5YaSBrGqp"}},{"cell_type":"code","source":"# Load the previously trained model\novo_shift_best_estimator = joblib.load(\"OvO_SGDClassifier_best_model_with_shift.pkl\")\n\n# Define the confidence threshold for class 8\nconfidence_threshold_8 = 9\n\ndef apply_confidence_threshold(X, model, threshold):\n    decision_values = model.decision_function(X)\n\n    predictions = model.predict(X)\n\n    for i in range(len(predictions)):\n        if predictions[i] == 8:\n            confidence = decision_values[i][8]\n            if confidence < threshold:\n                second_best_class = np.argsort(decision_values[i])[::-1][1]\n                predictions[i] = second_best_class\n\n    return predictions\n\ny_pred_with_threshold = apply_confidence_threshold(X_val, ovo_shift_best_estimator, confidence_threshold_8)\n\nprint(\"\\nClassification Report after applying confidence threshold for class 8:\\n\")\nprint(classification_report(y_val, y_pred_with_threshold))\nprint(f\"Confusion Matrix after applying confidence threshold for class 8:\\n\")\nprint(confusion_matrix(y_val, y_pred_with_threshold))\n\n","metadata":{"executionInfo":{"elapsed":1841,"status":"ok","timestamp":1729004268535,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"YY3ts5KKSfrg","outputId":"448b2b54-c48b-47ed-d009-47b62c305970"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom sklearn.metrics import ConfusionMatrixDisplay\n\n# First, plot the confusion matrix for the original OvO predictions with shift\nfig, axes = plt.subplots(1, 2, figsize=(12, 6))\n\n# Plot for OvO with shifted data\nsample_weight_shift = (ovo_shift_y_pred != y_val).astype(int)\nConfusionMatrixDisplay.from_predictions(\n    y_val, ovo_shift_y_pred,\n    sample_weight=sample_weight_shift,\n    ax=axes[0], cmap=\"viridis\"\n)\naxes[0].set_title('OvO with Shift Augmentation')\n\n# Apply the confidence threshold for class 8\ny_pred_with_threshold = apply_confidence_threshold(X_val, ovo_shift_best_estimator, confidence_threshold_8)\n\n# Plot for OvO with shifted data and confidence threshold for class 8\nsample_weight_threshold = (y_pred_with_threshold != y_val).astype(int)\nConfusionMatrixDisplay.from_predictions(\n    y_val, y_pred_with_threshold,\n    sample_weight=sample_weight_threshold,\n    ax=axes[1], cmap=\"viridis\"\n)\naxes[1].set_title('OvO with Shift and Confidence Threshold for 8')\n\n# Adjust layout and show the plots\nplt.tight_layout()\nplt.show()\n","metadata":{"executionInfo":{"elapsed":2339,"status":"ok","timestamp":1728932198836,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"gOrGrjJaYhVx","outputId":"9e678e8a-80b4-42de-8e8c-6586d35f94c2"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here, as we can see, although the overall accuracy remains the same, the number of digits misclassified as 8 has been significantly reduced by applying the confidence threshold!\n","metadata":{"id":"TDn1tcjAsLYF"}},{"cell_type":"markdown","source":"# Summarizing The Result","metadata":{"id":"2tPXG6zoKErY"}},{"cell_type":"markdown","source":"From our previous efforts, we will add two features: the closed loops counter and the confidence threshold. Both of these will be applied at the end of the pipeline. We will ensure that class 1 has 0 holes (since the closed loops counter performs perfectly on class 1) and set 9 as the confidence threshold for class 8.","metadata":{"id":"54HMvXdktgRB"}},{"cell_type":"code","source":"from sklearn.base import BaseEstimator, TransformerMixin\n\nclass ConfidenceThresholdTransformer(BaseEstimator, TransformerMixin):\n    def __init__(self, model, threshold):\n        self.model = model\n        self.threshold = threshold\n\n    def fit(self, X, y=None):\n        return self\n\n    def transform(self, X, y=None):\n        decision_values = self.model.decision_function(X)\n        predictions = self.model.predict(X)\n\n        for i in range(len(predictions)):\n            if predictions[i] == 8:\n                confidence = decision_values[i][8]\n                if confidence < self.threshold:\n                    second_best_class = np.argsort(decision_values[i])[::-1][1]\n                    predictions[i] = second_best_class\n\n        return predictions\n\nclass HoleCheckerTransformer(BaseEstimator, TransformerMixin):\n    def __init__(self, model, min_hole_size=5, binarization_threshold=0.1):\n        self.model = model\n        self.min_hole_size = min_hole_size\n        self.binarization_threshold = binarization_threshold\n\n    def compute_num_holes(self, image):\n        image_2d = image.reshape(28, 28)\n        binary_image = image_2d > self.binarization_threshold\n        structuring_element = disk(2)\n        binary_image = binary_closing(binary_image, structuring_element)\n        inverted_image = ~binary_image\n        labeled_image = skimage_label(inverted_image, connectivity=2)\n        labeled_image = remove_small_objects(labeled_image, min_size=self.min_hole_size)\n        num_holes = len(np.unique(labeled_image)) - 2\n        return max(num_holes, 0)\n\n    def fit(self, X, y=None):\n        return self\n\n    def transform(self, X, predictions):\n        decision_values = self.model.decision_function(X)\n        corrected_predictions = predictions.copy()\n\n        for idx, image in enumerate(X):\n            if predictions[idx] == 1:\n                num_holes = self.compute_num_holes(image)\n                if num_holes > 0:  # Class 1 should not have any holes\n                    second_best_class = np.argsort(decision_values[idx])[::-1][1]\n                    corrected_predictions[idx] = second_best_class\n\n        return corrected_predictions\n\n\n# Load the pre-trained OvO shift model\novo_shift_model = joblib.load('OvO_SGDClassifier_best_model_with_shift.pkl')\n\n# Set the confidence threshold for class 8\nconfidence_threshold_8 = 9\n\n# Initial transformer pipeline\npipeline_with_transformers = Pipeline(steps=[\n    ('confidence_threshold', ConfidenceThresholdTransformer(model=ovo_shift_model, threshold=confidence_threshold_8)),\n    ('hole_checker', HoleCheckerTransformer(model=ovo_shift_model))\n])\n\npredictions_after_threshold = pipeline_with_transformers.named_steps['confidence_threshold'].transform(X_val)\nfinal_predictions = pipeline_with_transformers.named_steps['hole_checker'].transform(X_val, predictions_after_threshold)\n\n# Evaluate the performance with the transformers applied\nprint(\"\\nClassification Report with Confidence Threshold and Hole Checker:\\n\")\nprint(classification_report(y_val, final_predictions))\nprint(f\"Confusion Matrix with Confidence Threshold and Hole Checker:\\n\")\nprint(confusion_matrix(y_val, final_predictions))\n\n","metadata":{"executionInfo":{"elapsed":1967,"status":"ok","timestamp":1729007411759,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"iBXPd6SfGNvZ","outputId":"c0c94f3a-8b3b-4417-895b-023782460884"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the saved OvO normal model (already trained)\novo_best_model = joblib.load('OvO_SGDClassifier_best_model.pkl')\novo_y_pred = ovo_best_model.predict(X_val)\n\n# Print the classification report and confusion matrix for the normal OvO model\nprint(\"\\nClassification Report for OvO SGDClassifier (Normal) on Validation Set:\\n\")\nprint(classification_report(y_val, ovo_y_pred))\nprint(f\"Confusion Matrix for OvO SGDClassifier (Normal) on Validation Set:\\n\")\nprint(confusion_matrix(y_val, ovo_y_pred))\n\n\nprint(\"\\nClassification Report with Shift + Hole Checker on Validation Set:\\n\")\nprint(classification_report(y_val, final_predictions))\nprint(f\"Confusion Matrix with Shift + Hole Checker on Validation Set:\\n\")\nprint(confusion_matrix(y_val, final_predictions))\n\n\nsample_weight_normal = (ovo_y_pred != y_val).astype(int)\nsample_weight_final = (final_predictions != y_val).astype(int)\n\nfig, axes = plt.subplots(1, 2, figsize=(14, 6))\n\nConfusionMatrixDisplay.from_predictions(\n    y_val, ovo_y_pred,\n    sample_weight=sample_weight_normal,\n    ax=axes[0]\n)\naxes[0].set_title('OvO Normal (No Augmentation)')\n\nConfusionMatrixDisplay.from_predictions(\n    y_val, final_predictions,\n    sample_weight=sample_weight_final,\n    ax=axes[1]\n)\naxes[1].set_title('OvO with Shift Augmentation + Hole Checker')\n\n\nplt.tight_layout()\nplt.show()\n\n","metadata":{"executionInfo":{"elapsed":3476,"status":"ok","timestamp":1729008037930,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"},"user_tz":-180},"id":"mHoJYj_LPfhl","outputId":"2eb46185-9177-4f50-c763-23998ad088f2"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here, as we can see from the comparison between the normal OvO and the OvO with the closed loops checker and the threshold applied to class 8, the accuracy increased from 0.91 to 0.94. We also observe a reduction in the number of digits misclassified as 8 (from 117 to 50). Unfortunately, there is a slight increase in misclassifications for class 1 (from 36 to 41).","metadata":{"id":"o2FeAaR46Vt3"}},{"cell_type":"markdown","source":"# Testing on the Test Set","metadata":{"id":"RP51Wzrg38PB"}},{"cell_type":"markdown","source":"Now, we will test the code in the test set","metadata":{"id":"Ahq0lh2k7Xg_"}},{"cell_type":"code","source":"test_features_df = pd.read_csv(\"/content/digit-recognizer/test.csv\")\nX_test = test_features_df.to_numpy()\n\ntest_predictions_after_threshold = pipeline_with_transformers.named_steps['confidence_threshold'].transform(X_test)\ntest_final_predictions = pipeline_with_transformers.named_steps['hole_checker'].transform(X_test, test_predictions_after_threshold)\n\nsubmission_df = pd.DataFrame({\n    'ImageId': np.arange(1, len(test_final_predictions) + 1),\n    'Label': test_final_predictions\n})\n\nsubmission_df.to_csv(\"submission.csv\", index=False)\n\nprint(\"Submission file 'submission.csv' created successfully.\")","metadata":{"id":"MRaOaDF87Zu_","executionInfo":{"status":"ok","timestamp":1729008933944,"user_tz":-180,"elapsed":8502,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"}},"outputId":"d452cf74-f875-4c9d-8e56-d1c2add60e57"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\n\nX_test = test_features_df.to_numpy()\n\ntest_predictions_after_threshold = pipeline_with_transformers.named_steps['confidence_threshold'].transform(X_test)\ntest_final_predictions = pipeline_with_transformers.named_steps['hole_checker'].transform(X_test, test_predictions_after_threshold)\n\nrandom_indices = random.sample(range(len(X_test)), 20)\n\nfig, axes = plt.subplots(4, 5, figsize=(10, 8))\naxes = axes.flatten()\n\nfor i, idx in enumerate(random_indices):\n    image = X_test[idx].reshape(28, 28)  # Reshape the flat image back into 28x28\n    axes[i].imshow(image, cmap='gray')\n    axes[i].set_title(f\"Predicted: {test_final_predictions[idx]}\")\n    axes[i].axis('off')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"id":"d-Ap3F7o9tTy","executionInfo":{"status":"ok","timestamp":1729009220272,"user_tz":-180,"elapsed":8857,"user":{"displayName":"Axel Dawne","userId":"13665121235676681634"}},"outputId":"e26bc516-7bcc-4e3f-d50c-10625e2e697a"},"execution_count":null,"outputs":[]}]}