{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"colab":{"provenance":[]},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":3004,"databundleVersionId":861823}],"dockerImageVersionId":31286,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd","metadata":{"id":"dALLoRwPpN4m","trusted":true,"execution":{"iopub.status.busy":"2026-02-21T14:57:42.284649Z","iopub.execute_input":"2026-02-21T14:57:42.285006Z","iopub.status.idle":"2026-02-21T14:57:42.638435Z","shell.execute_reply.started":"2026-02-21T14:57:42.284977Z","shell.execute_reply":"2026-02-21T14:57:42.637597Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/competitions/digit-recognizer/train.csv')\ntest = pd.read_csv('/kaggle/input/competitions/digit-recognizer/test.csv')","metadata":{"id":"6m18WuQAq31a","trusted":true,"execution":{"iopub.status.busy":"2026-02-21T15:00:00.973681Z","iopub.execute_input":"2026-02-21T15:00:00.973983Z","iopub.status.idle":"2026-02-21T15:00:04.740871Z","shell.execute_reply.started":"2026-02-21T15:00:00.973958Z","shell.execute_reply":"2026-02-21T15:00:04.739266Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.info()","metadata":{"id":"uHv0lG14q-OG","outputId":"1035a1be-dcea-4f08-b1d0-60de3a414872","trusted":true,"execution":{"iopub.status.busy":"2026-02-21T14:57:59.180565Z","iopub.execute_input":"2026-02-21T14:57:59.181032Z","iopub.status.idle":"2026-02-21T14:57:59.226259Z","shell.execute_reply.started":"2026-02-21T14:57:59.180999Z","shell.execute_reply":"2026-02-21T14:57:59.225201Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.duplicated().sum()","metadata":{"id":"d5Bo4SkqrL20","outputId":"61b1ee38-467c-4a9d-ec6c-a0e731a3a562","trusted":true,"execution":{"iopub.status.busy":"2026-02-21T14:58:02.459829Z","iopub.execute_input":"2026-02-21T14:58:02.460678Z","iopub.status.idle":"2026-02-21T14:58:02.964071Z","shell.execute_reply.started":"2026-02-21T14:58:02.460632Z","shell.execute_reply":"2026-02-21T14:58:02.963228Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.isna().sum().sum()","metadata":{"id":"6DdGMotKrhfa","outputId":"ea47e7a6-2b7a-419b-fb14-09556e60e990","trusted":true,"execution":{"iopub.status.busy":"2026-02-21T14:58:10.591255Z","iopub.execute_input":"2026-02-21T14:58:10.591556Z","iopub.status.idle":"2026-02-21T14:58:10.626114Z","shell.execute_reply.started":"2026-02-21T14:58:10.591532Z","shell.execute_reply":"2026-02-21T14:58:10.625231Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = df.drop('label', axis=1)\ny = df['label']\n\nprint('Shape of X:', X.shape)\nprint('Shape of y:', y.shape)","metadata":{"id":"2099f430","outputId":"1f342e75-bfd2-4214-efe2-50fd65794141","trusted":true,"execution":{"iopub.status.busy":"2026-02-21T15:00:09.214658Z","iopub.execute_input":"2026-02-21T15:00:09.215177Z","iopub.status.idle":"2026-02-21T15:00:09.304111Z","shell.execute_reply.started":"2026-02-21T15:00:09.215130Z","shell.execute_reply":"2026-02-21T15:00:09.302836Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\nprint('Shape of X_train:', X_train.shape)\nprint('Shape of X_test:', X_test.shape)\nprint('Shape of y_train:', y_train.shape)\nprint('Shape of y_test:', y_test.shape)","metadata":{"id":"48112088","outputId":"379d57e2-7c8a-499d-b883-481fd08d5977","trusted":true,"execution":{"iopub.status.busy":"2026-02-21T15:00:16.806496Z","iopub.execute_input":"2026-02-21T15:00:16.807269Z","iopub.status.idle":"2026-02-21T15:00:17.054518Z","shell.execute_reply.started":"2026-02-21T15:00:16.807225Z","shell.execute_reply":"2026-02-21T15:00:17.053293Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"id":"hkNiUjjQ--oh"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"id":"PjbgpL_9--rZ"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Detect Blank or Constant Images**","metadata":{"id":"uC15qjzG_015"}},{"cell_type":"code","source":"# blank_images = np.where(np.var(X, axis=1) == 0)[0]\n# X = np.delete(X, blank_images, axis=0)\n# y = np.delete(y, blank_images, axis=0)","metadata":{"id":"IG9NP248--uN","trusted":true,"execution":{"iopub.status.busy":"2026-02-21T14:58:29.332137Z","iopub.execute_input":"2026-02-21T14:58:29.332980Z","iopub.status.idle":"2026-02-21T14:58:29.642436Z","shell.execute_reply.started":"2026-02-21T14:58:29.332943Z","shell.execute_reply":"2026-02-21T14:58:29.641256Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Detect Images Very Different From Others**","metadata":{"id":"doo5Pnvb_sU9"}},{"cell_type":"code","source":"from sklearn.ensemble import IsolationForest\n\niso = IsolationForest(contamination=0.02)\nlabels = iso.fit_predict(X)\n\n# -1 = corrupted\ncorrupted_index = np.where(labels == -1)[0]","metadata":{"id":"u6TZTVuo--xv","trusted":true,"execution":{"iopub.status.busy":"2026-02-21T15:40:46.073768Z","iopub.execute_input":"2026-02-21T15:40:46.074240Z","iopub.status.idle":"2026-02-21T15:40:47.344112Z","shell.execute_reply.started":"2026-02-21T15:40:46.074209Z","shell.execute_reply":"2026-02-21T15:40:47.343223Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_clean = X[labels == 1]\ny_clean = y[labels == 1]","metadata":{"id":"oAbTzFie-_B6","trusted":true,"execution":{"iopub.status.busy":"2026-02-21T15:40:49.740047Z","iopub.execute_input":"2026-02-21T15:40:49.740973Z","iopub.status.idle":"2026-02-21T15:40:49.851318Z","shell.execute_reply.started":"2026-02-21T15:40:49.740943Z","shell.execute_reply":"2026-02-21T15:40:49.850348Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Detect Wrong Labels**","metadata":{"id":"n8M6CEkq_ihp"}},{"cell_type":"code","source":"# from xgboost import XGBClassifier\n\n# # Instantiate the XGBoost classifier\n# model = XGBClassifier()","metadata":{"id":"BHpzkm6iFHqr","trusted":true,"execution":{"iopub.status.busy":"2026-02-21T14:59:37.725941Z","iopub.execute_input":"2026-02-21T14:59:37.726859Z","iopub.status.idle":"2026-02-21T14:59:38.035805Z","shell.execute_reply.started":"2026-02-21T14:59:37.726821Z","shell.execute_reply":"2026-02-21T14:59:38.034711Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# proba = model.predict_proba(X)\n# pred = model.predict(X)\n\n# confidence = np.max(proba, axis=1)\n\n# suspicious = np.where((pred != y) & (confidence > 0.95))[0]\n\n# X_clean = np.delete(X_clean, suspicious, axis=0)\n# y_clean = np.delete(y_clean, suspicious, axis=0)\n","metadata":{"id":"euaIfI3E_PYT","trusted":true,"execution":{"iopub.status.busy":"2026-02-21T15:41:01.190209Z","iopub.execute_input":"2026-02-21T15:41:01.191419Z","iopub.status.idle":"2026-02-21T15:41:08.959684Z","shell.execute_reply.started":"2026-02-21T15:41:01.191366Z","shell.execute_reply":"2026-02-21T15:41:08.958918Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"id":"1K5Hb8bo_Pa8"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"id":"05VLTARq_PeW"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"id":"qpOcxXoj-_FU"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"id":"4YKrJQ6H-_bg"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from xgboost import XGBClassifier\n\n# Instantiate the XGBoost classifier\nmodel = XGBClassifier(n_estimators=389,\n                      max_depth=9,\n                      learning_rate=0.03188378512912393,\n                      subsample=0.6056720221492032,\n                      colsample_bytree=0.6088100535693801,\n                      gamma=0.007892399218429512,\n                      reg_alpha=0.0009903092101875566,\n                      reg_lambda=0.6691621857459197,\n                      random_state=42,\n                      eval_metric='mlogloss')\n\n# Train the model\nmodel.fit(X_clean, y_clean)\nprint(\"XGBoost model trained successfully.\")","metadata":{"id":"9adffa02","outputId":"857a65c0-af1a-4205-88a3-2160fa93c4ec","trusted":true,"execution":{"iopub.status.busy":"2026-02-21T15:41:41.132042Z","iopub.execute_input":"2026-02-21T15:41:41.132383Z","iopub.status.idle":"2026-02-21T15:57:22.815486Z","shell.execute_reply.started":"2026-02-21T15:41:41.132356Z","shell.execute_reply":"2026-02-21T15:57:22.814319Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# y_pred = model.predict(X_test)\n# print(\"Predictions made successfully.\")","metadata":{"id":"e5b8403e","outputId":"67436a04-8203-4889-aaec-ffe6ab7abe83","trusted":true,"execution":{"iopub.status.busy":"2026-02-21T15:14:43.316740Z","iopub.execute_input":"2026-02-21T15:14:43.317338Z","iopub.status.idle":"2026-02-21T15:14:44.109639Z","shell.execute_reply.started":"2026-02-21T15:14:43.317307Z","shell.execute_reply":"2026-02-21T15:14:44.108004Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from sklearn.metrics import accuracy_score\n\n# accuracy = accuracy_score(y_test, y_pred)\n# print(f\"Accuracy Score: {accuracy:.4f}\")","metadata":{"id":"a81ad65f","outputId":"08a33e31-21ed-4545-9688-f3a8697b3c0d","trusted":true,"execution":{"iopub.status.busy":"2026-02-21T15:15:01.809539Z","iopub.execute_input":"2026-02-21T15:15:01.810767Z","iopub.status.idle":"2026-02-21T15:15:01.818563Z","shell.execute_reply.started":"2026-02-21T15:15:01.810732Z","shell.execute_reply":"2026-02-21T15:15:01.817740Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Accuracy Score: 0.9750","metadata":{"id":"vMgcQ6zZUBlx"}},{"cell_type":"code","source":"test_predictions = model.predict(test)\n\n# Create a DataFrame for submission\nsubmission = pd.DataFrame({'ImageId': range(1, len(test_predictions) + 1), 'Label': test_predictions})\n\n# Save the submission file to CSV\nsubmission.to_csv('submission.csv', index=False)\n\nprint(\"Predictions on test data generated and saved to 'submission.csv'.\")\ndisplay(submission.head())","metadata":{"id":"a85MXcuyZOAH","trusted":true,"execution":{"iopub.status.busy":"2026-02-21T16:05:45.505937Z","iopub.execute_input":"2026-02-21T16:05:45.506908Z","iopub.status.idle":"2026-02-21T16:05:47.948838Z","shell.execute_reply.started":"2026-02-21T16:05:45.506874Z","shell.execute_reply":"2026-02-21T16:05:47.948078Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Task\nCreate a 25% stratified subset of the entire dataset (`X`, `y`) to be used for Optuna's hyperparameter tuning, naming the subsets `X_optuna` and `y_optuna`. Then, define an Optuna objective function that performs stratified K-Fold cross-validation on `X_optuna` and `y_optuna` to suggest hyperparameters for an XGBoost model, returning the average accuracy. Subsequently, run an Optuna study to find the optimal hyperparameters. Train the final XGBoost model using these optimal hyperparameters on the full training dataset (`X_train`, `y_train`), make predictions on `X_test`, and evaluate its performance with accuracy and a confusion matrix. Finally, summarize the hyperparameter tuning and model performance results.","metadata":{"id":"2169b3a5"}},{"cell_type":"markdown","source":"## Create 25% Stratified Subset for Optuna\n\n### Subtask:\nGenerate a 25% stratified sample of the entire dataset (`X`, `y`) to be used for Optuna's hyperparameter tuning, maintaining class distribution. Name this subset `X_optuna` and `y_optuna`.\n","metadata":{"id":"662e9e5f"}},{"cell_type":"markdown","source":"**Reasoning**:\nTo generate a 25% stratified subset of the dataset, I will use `train_test_split` with `test_size=0.25` and `stratify=y` to ensure the class distribution is maintained. The resulting test sets will be assigned to `X_optuna` and `y_optuna`.\n\n","metadata":{"id":"263a90eb"}},{"cell_type":"code","source":"# from sklearn.model_selection import train_test_split\n\n# # Generate a 25% stratified sample of the entire dataset (X, y)\n# _, X_optuna, _, y_optuna = train_test_split(X, y, test_size=0.05, stratify=y, random_state=42)\n\n# print('Shape of X_optuna:', X_optuna.shape)\n# print('Shape of y_optuna:', y_optuna.shape)","metadata":{"id":"e6ff9333","outputId":"066c07e0-5aa0-4184-f6b4-ed9411395726"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# pip install optuna","metadata":{"id":"Iv0-3ybxu95m","outputId":"f79af880-d929-4b65-e177-d2414738844d"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import optuna\n# from xgboost import XGBClassifier\n# from sklearn.model_selection import StratifiedKFold\n# from sklearn.metrics import accuracy_score\n\n# def objective(trial):\n#     # 3. Suggest hyperparameters for XGBoost\n#     param = {\n#         'n_estimators': trial.suggest_int('n_estimators', 100, 1000),\n#         'max_depth': trial.suggest_int('max_depth', 3, 9),\n#         'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3, log=True),\n#         'subsample': trial.suggest_float('subsample', 0.6, 1.0),\n#         'colsample_bytree': trial.suggest_float('colsample_bytree', 0.6, 1.0),\n#         'gamma': trial.suggest_float('gamma', 0, 0.5),\n#         'reg_alpha': trial.suggest_float('reg_alpha', 0, 0.5),\n#         'reg_lambda': trial.suggest_float('reg_lambda', 0.5, 1.0),\n#         'random_state': 42,\n\n#         'eval_metric': 'mlogloss' # Specify evaluation metric for multi-class classification\n#     }\n\n#     # 4. Initialize StratifiedKFold\n#     skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n\n#     # 5. Initialize an empty list to store accuracies\n#     accuracies = []\n\n#     # 6. Loop through the folds for stratified K-Fold cross-validation\n#     for train_idx, val_idx in skf.split(X_optuna, y_optuna):\n#         # 7. Create train and validation sets for the current fold\n#         X_train_fold, X_val_fold = X_optuna.iloc[train_idx], X_optuna.iloc[val_idx]\n#         y_train_fold, y_val_fold = y_optuna.iloc[train_idx], y_optuna.iloc[val_idx]\n\n#         # 8. Instantiate XGBClassifier with suggested hyperparameters\n#         model = XGBClassifier(**param)\n\n#         # 9. Fit the model\n#         model.fit(X_train_fold, y_train_fold)\n\n#         # 10. Make predictions and calculate accuracy\n#         y_pred_fold = model.predict(X_val_fold)\n#         accuracy_fold = accuracy_score(y_val_fold, y_pred_fold)\n\n#         # 11. Append accuracy to the list\n#         accuracies.append(accuracy_fold)\n\n#     # 12. Return the mean of the accuracies list\n#     print(f\"Trial finished with mean accuracy: {np.mean(accuracies):.4f}\")\n#     return np.mean(accuracies)\n\n# print(\"Optuna objective function 'objective' defined successfully.\")","metadata":{"id":"cb8029cc","outputId":"0ff06f3e-102d-4282-ea65-a48b6e375359"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Reasoning**:\nInitialize an Optuna study to find optimal hyperparameters for the XGBoost model by maximizing the average accuracy over 50 trials.\n\n","metadata":{"id":"1a41550b"}},{"cell_type":"code","source":"# study = optuna.create_study(direction='maximize')\n# study.optimize(objective, n_trials=50)\n\n# print(\"Optuna study completed.\")","metadata":{"collapsed":true,"id":"dd06f8d0","outputId":"f12d0f8f-b464-4802-80da-793929c96253","jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Trial 12 finished with value: 0.9171428571428573 and parameters: {'n_estimators': 389, 'max_depth': 9, 'learning_rate': 0.03188378512912393, 'subsample': 0.6056720221492032, 'colsample_bytree': 0.6088100535693801, 'gamma': 0.007892399218429512, 'reg_alpha': 0.0009903092101875566, 'reg_lambda': 0.6691621857459197}. Best is trial 12 with value: 0.9171428571428573.\nTrial finished with mean accuracy: 0.9171","metadata":{"id":"gN6t8jgEE5to"}}]}