{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":31153,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Upgrade Scikit-Learn","metadata":{}},{"cell_type":"code","source":"%%bash\n\npip install --upgrade 'scikit-learn<1.6.0'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T08:47:47.433560Z","iopub.execute_input":"2025-10-16T08:47:47.433847Z","iopub.status.idle":"2025-10-16T08:47:58.210650Z","shell.execute_reply.started":"2025-10-16T08:47:47.433825Z","shell.execute_reply":"2025-10-16T08:47:58.209578Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Load the Digit Recognizer dataset","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2025-10-09T09:19:16.182739Z","iopub.execute_input":"2025-10-09T09:19:16.183018Z","iopub.status.idle":"2025-10-09T09:19:17.221957Z","shell.execute_reply.started":"2025-10-09T09:19:16.182997Z","shell.execute_reply":"2025-10-09T09:19:17.221096Z"}}},{"cell_type":"code","source":"import pathlib\n\nimport pandas as pd\n\n\nDATA_DIR = pathlib.Path(\"/kaggle/input/digit-recognizer\")\n\n\ntrain_df = pd.read_csv(DATA_DIR / \"train.csv\")\ntrain_features_df = train_df.drop(\"label\", axis=1)\ntrain_labels = train_df.loc[:, \"label\"]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T08:48:44.093082Z","iopub.execute_input":"2025-10-16T08:48:44.093435Z","iopub.status.idle":"2025-10-16T08:48:48.814862Z","shell.execute_reply.started":"2025-10-16T08:48:44.093409Z","shell.execute_reply":"2025-10-16T08:48:48.814069Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## SGDClassifier Benchmark","metadata":{}},{"cell_type":"code","source":"import numpy as np\nfrom sklearn import decomposition, linear_model, preprocessing, pipeline\n\n\nRANDOM_STATE = np.random.RandomState(42)\n\n\nsgd_classifier = pipeline.make_pipeline(\n    preprocessing.StandardScaler(\n        with_mean=False,\n        with_std=True\n    ),\n    decomposition.PCA(\n        n_components=0.95,\n        random_state=RANDOM_STATE,\n        whiten=True\n    ),\n    linear_model.SGDClassifier(\n        early_stopping=True,\n        loss=\"log_loss\",\n        n_jobs=-1,\n        random_state=RANDOM_STATE\n    )\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T08:56:01.949195Z","iopub.execute_input":"2025-10-16T08:56:01.949597Z","iopub.status.idle":"2025-10-16T08:56:02.822754Z","shell.execute_reply.started":"2025-10-16T08:56:01.949566Z","shell.execute_reply":"2025-10-16T08:56:02.821619Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%timeit -n 1 -r 1\n\n_ = sgd_classifier.fit(\n    train_features_df,\n    train_labels\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T08:56:44.357500Z","iopub.execute_input":"2025-10-16T08:56:44.357847Z","iopub.status.idle":"2025-10-16T08:56:47.831180Z","shell.execute_reply.started":"2025-10-16T08:56:44.357824Z","shell.execute_reply":"2025-10-16T08:56:47.829187Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn import metrics\n\n\ntrain_predictions = sgd_classifier.predict(train_features_df)\n\ntrain_accuracy = metrics.accuracy_score(\n    train_labels,\n    train_predictions\n)\n\nprint(f\"Training accuracy: {train_accuracy}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T08:57:36.531187Z","iopub.execute_input":"2025-10-16T08:57:36.531642Z","iopub.status.idle":"2025-10-16T08:57:37.268700Z","shell.execute_reply.started":"2025-10-16T08:57:36.531607Z","shell.execute_reply":"2025-10-16T08:57:37.266679Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn import model_selection\n\n\ncv_accuracy = model_selection.cross_val_score(\n    sgd_classifier,\n    train_features_df,\n    train_labels,\n    cv=4,\n    n_jobs=-1,\n    scoring=\"accuracy\",\n    verbose=1\n)\n\nprint(f\"CV accuracy: {cv_accuracy.mean()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T08:58:32.884938Z","iopub.execute_input":"2025-10-16T08:58:32.885488Z","iopub.status.idle":"2025-10-16T08:58:44.756150Z","shell.execute_reply.started":"2025-10-16T08:58:32.885453Z","shell.execute_reply":"2025-10-16T08:58:44.754629Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Flagging Outliers using IsolationForest","metadata":{}},{"cell_type":"code","source":"from sklearn import base, ensemble\n\n\nclass IsolationForestOutlierFlagger(base.BaseEstimator, base.TransformerMixin):\n    \n    def __init__(self, contamination=\"auto\", random_state=None, dtype=np.float64):\n        self.contamination = contamination\n        self.random_state = random_state\n        self.dtype = dtype\n\n    def fit(self, X, y=None):\n        self.detector_ = ensemble.IsolationForest(\n            contamination=self.contamination,\n            random_state=self.random_state,\n            n_jobs=-1\n        )\n        self.detector_.fit(X)\n        return self\n\n    def transform(self, X, y=None):\n        # predict returns 1 for inliers, -1 for outliers\n        predicted_outliers = (\n            self.detector_\n                .predict(X)\n        )\n\n        # create an outlier flag\n        is_outlier = predicted_outliers == -1\n        is_outlier = (\n            is_outlier.astype(self.dtype)\n                      .reshape(-1, 1)\n        )    \n                \n        # append flag as an extra feature\n        X_out = np.hstack([X, is_outlier])\n        \n        return X_out","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T09:08:43.352201Z","iopub.execute_input":"2025-10-16T09:08:43.352911Z","iopub.status.idle":"2025-10-16T09:08:43.406042Z","shell.execute_reply.started":"2025-10-16T09:08:43.352882Z","shell.execute_reply":"2025-10-16T09:08:43.405031Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"outlier_flagger = pipeline.make_pipeline(\n    preprocessing.StandardScaler(),\n    IsolationForestOutlierFlagger(\n        random_state=RANDOM_STATE,\n    ),\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T09:10:37.961900Z","iopub.execute_input":"2025-10-16T09:10:37.962380Z","iopub.status.idle":"2025-10-16T09:10:37.968353Z","shell.execute_reply.started":"2025-10-16T09:10:37.962353Z","shell.execute_reply":"2025-10-16T09:10:37.967217Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"prepared_train_features = outlier_flagger.fit_transform(train_features_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T09:11:14.114404Z","iopub.execute_input":"2025-10-16T09:11:14.114765Z","iopub.status.idle":"2025-10-16T09:11:16.184982Z","shell.execute_reply.started":"2025-10-16T09:11:14.114740Z","shell.execute_reply":"2025-10-16T09:11:16.183693Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"prepared_train_features.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T09:11:17.791473Z","iopub.execute_input":"2025-10-16T09:11:17.791833Z","iopub.status.idle":"2025-10-16T09:11:17.803855Z","shell.execute_reply.started":"2025-10-16T09:11:17.791807Z","shell.execute_reply":"2025-10-16T09:11:17.801435Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"number_outliers = np.sum(prepared_train_features[:, -1])\nprint(number_outliers)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T09:14:00.009872Z","iopub.execute_input":"2025-10-16T09:14:00.010588Z","iopub.status.idle":"2025-10-16T09:14:00.015800Z","shell.execute_reply.started":"2025-10-16T09:14:00.010559Z","shell.execute_reply":"2025-10-16T09:14:00.014665Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n\nis_outlier = prepared_train_features[:, -1] == 1\noutlier_train_features = train_features_df.iloc[is_outlier, :]\noutlier_train_labels = train_labels.iloc[is_outlier]\n\nfig, axes = plt.subplots(1, 10, figsize=(15, 5), sharex=True, sharey=True)\nfor i in range(10):\n    k = RANDOM_STATE.randint(len(outlier_train_features))\n    axes[i].imshow(outlier_train_features.iloc[k, :].to_numpy().reshape(28, 28))\n    axes[i].set_title(outlier_train_labels.iloc[k])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T09:11:43.165553Z","iopub.execute_input":"2025-10-16T09:11:43.165895Z","iopub.status.idle":"2025-10-16T09:11:44.591734Z","shell.execute_reply.started":"2025-10-16T09:11:43.165870Z","shell.execute_reply":"2025-10-16T09:11:44.590414Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sgd_classifier = pipeline.make_pipeline(\n    preprocessing.StandardScaler(\n        with_mean=False,\n        with_std=True,\n    ),\n    IsolationForestOutlierFlagger(\n        random_state=RANDOM_STATE,\n    ),\n    decomposition.PCA(\n        n_components=0.95,\n        random_state=RANDOM_STATE,\n        whiten=True,\n    ),\n    linear_model.SGDClassifier(\n        early_stopping=True,\n        loss=\"log_loss\",\n        n_jobs=-1,\n        random_state=RANDOM_STATE\n    )\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T09:12:03.933092Z","iopub.execute_input":"2025-10-16T09:12:03.933479Z","iopub.status.idle":"2025-10-16T09:12:03.939578Z","shell.execute_reply.started":"2025-10-16T09:12:03.933452Z","shell.execute_reply":"2025-10-16T09:12:03.938454Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_ = sgd_classifier.fit(\n    train_features_df,\n    train_labels\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T09:12:11.478534Z","iopub.execute_input":"2025-10-16T09:12:11.478930Z","iopub.status.idle":"2025-10-16T09:12:16.140940Z","shell.execute_reply.started":"2025-10-16T09:12:11.478904Z","shell.execute_reply":"2025-10-16T09:12:16.139716Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_predictions = sgd_classifier.predict(train_features_df)\n\ntrain_accuracy = metrics.accuracy_score(\n    train_labels,\n    train_predictions\n)\n\nprint(f\"Training accuracy: {train_accuracy}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T09:12:36.347942Z","iopub.execute_input":"2025-10-16T09:12:36.348462Z","iopub.status.idle":"2025-10-16T09:12:37.695495Z","shell.execute_reply.started":"2025-10-16T09:12:36.348430Z","shell.execute_reply":"2025-10-16T09:12:37.694469Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cv_accuracy = model_selection.cross_val_score(\n    sgd_classifier,\n    train_features_df,\n    train_labels,\n    cv=4,\n    n_jobs=-1,\n    scoring=\"accuracy\",\n    verbose=1\n)\n\nprint(f\"CV accuracy: {cv_accuracy.mean()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T09:14:59.779711Z","iopub.execute_input":"2025-10-16T09:14:59.780233Z","iopub.status.idle":"2025-10-16T09:15:10.905978Z","shell.execute_reply.started":"2025-10-16T09:14:59.780205Z","shell.execute_reply":"2025-10-16T09:15:10.905109Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Flagging Outliers using LocalOutlierFactor","metadata":{}},{"cell_type":"code","source":"from sklearn import neighbors\n\n\nclass LocalOutlierFactorOutlierFlagger(base.BaseEstimator, base.TransformerMixin):\n    \n    def __init__(self, n_neighbors=20, contamination=\"auto\", dtype=np.float64):\n        self.n_neighbors = n_neighbors\n        self.contamination = contamination\n        self.dtype = dtype\n\n    def fit(self, X, y=None):\n        self.detector_ = neighbors.LocalOutlierFactor(\n            n_neighbors=self.n_neighbors,\n            contamination=self.contamination,\n            novelty=True,\n            n_jobs=-1,\n        )\n        self.detector_.fit(X)\n        return self\n\n    def transform(self, X, y=None):\n        # predict returns 1 for inliers, -1 for outliers\n        predicted_outliers = (\n            self.detector_\n                .predict(X)\n        )\n\n        # create an outlier flag\n        is_outlier = predicted_outliers == -1\n        is_outlier = (\n            is_outlier.astype(self.dtype)\n                      .reshape(-1, 1)\n        )    \n                \n        # append flag as an extra feature\n        X_out = np.hstack([X, is_outlier])\n        \n        return X_out\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T09:18:07.123539Z","iopub.execute_input":"2025-10-16T09:18:07.123872Z","iopub.status.idle":"2025-10-16T09:18:07.132669Z","shell.execute_reply.started":"2025-10-16T09:18:07.123850Z","shell.execute_reply":"2025-10-16T09:18:07.131280Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"outlier_flagger = pipeline.make_pipeline(\n    preprocessing.StandardScaler(\n        with_mean=False,\n        with_std=True,\n    ),\n    LocalOutlierFactorOutlierFlagger(),\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T09:18:09.396873Z","iopub.execute_input":"2025-10-16T09:18:09.397932Z","iopub.status.idle":"2025-10-16T09:18:09.402591Z","shell.execute_reply.started":"2025-10-16T09:18:09.397900Z","shell.execute_reply":"2025-10-16T09:18:09.401560Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"prepared_train_features = outlier_flagger.fit_transform(train_features_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T09:18:10.673661Z","iopub.execute_input":"2025-10-16T09:18:10.673980Z","iopub.status.idle":"2025-10-16T09:19:59.458005Z","shell.execute_reply.started":"2025-10-16T09:18:10.673956Z","shell.execute_reply":"2025-10-16T09:19:59.456825Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"is_outlier = prepared_train_features[:, -1] == 1\noutlier_train_features = train_features_df.iloc[is_outlier]\noutlier_train_labels = train_labels.iloc[is_outlier]\n\nfig, axes = plt.subplots(1, 10, figsize=(15, 5), sharex=True, sharey=True)\nfor i in range(10):\n    k = RANDOM_STATE.randint(len(outlier_train_features))\n    axes[i].imshow(outlier_train_features.iloc[k, :].to_numpy().reshape(28, 28))\n    axes[i].set_title(outlier_train_labels.iloc[k])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T09:19:59.459685Z","iopub.execute_input":"2025-10-16T09:19:59.459970Z","iopub.status.idle":"2025-10-16T09:20:00.540806Z","shell.execute_reply.started":"2025-10-16T09:19:59.459948Z","shell.execute_reply":"2025-10-16T09:20:00.539630Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"features_preparation_pipeline = pipeline.make_pipeline(\n    preprocessing.StandardScaler(\n        with_mean=False,\n        with_std=True,\n    ),\n    LocalOutlierFactorOutlierFlagger(),\n        decomposition.PCA(\n        n_components=0.95,\n        random_state=RANDOM_STATE,\n        whiten=True,\n    ),\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T09:23:56.195638Z","iopub.execute_input":"2025-10-16T09:23:56.195990Z","iopub.status.idle":"2025-10-16T09:23:56.201999Z","shell.execute_reply.started":"2025-10-16T09:23:56.195964Z","shell.execute_reply":"2025-10-16T09:23:56.200673Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%timeit -n 1 -r 1\n\n\nprepared_train_features = features_preparation_pipeline.fit_transform(\n    train_features_df\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T09:25:43.853102Z","iopub.execute_input":"2025-10-16T09:25:43.853458Z","iopub.status.idle":"2025-10-16T09:27:37.221910Z","shell.execute_reply.started":"2025-10-16T09:25:43.853426Z","shell.execute_reply":"2025-10-16T09:27:37.220771Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sgd_classifier = linear_model.SGDClassifier(\n    early_stopping=True,\n    loss=\"log_loss\",\n    n_jobs=-1,\n    random_state=RANDOM_STATE\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T09:27:37.223737Z","iopub.execute_input":"2025-10-16T09:27:37.224132Z","iopub.status.idle":"2025-10-16T09:27:37.230319Z","shell.execute_reply.started":"2025-10-16T09:27:37.224104Z","shell.execute_reply":"2025-10-16T09:27:37.229063Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%timeit -n 1 -r 1\n\n\n_ = sgd_classifier.fit(\n    prepared_train_features,\n    train_labels\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T09:27:37.231642Z","iopub.execute_input":"2025-10-16T09:27:37.232096Z","iopub.status.idle":"2025-10-16T09:27:40.418187Z","shell.execute_reply.started":"2025-10-16T09:27:37.232002Z","shell.execute_reply":"2025-10-16T09:27:40.417251Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_predictions = sgd_classifier.predict(prepared_train_features)\n\ntrain_accuracy = metrics.accuracy_score(\n    train_labels,\n    train_predictions\n)\n\nprint(f\"Training accuracy: {train_accuracy}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T09:27:40.419870Z","iopub.execute_input":"2025-10-16T09:27:40.420225Z","iopub.status.idle":"2025-10-16T09:27:40.547612Z","shell.execute_reply.started":"2025-10-16T09:27:40.420201Z","shell.execute_reply":"2025-10-16T09:27:40.546427Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cv_accuracy = model_selection.cross_val_score(\n    sgd_classifier,\n    prepared_train_features,\n    train_labels,\n    cv=4,\n    n_jobs=-1,\n    scoring=\"accuracy\",\n    verbose=1\n)\n\nprint(f\"CV accuracy: {cv_accuracy.mean()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T09:27:50.541789Z","iopub.execute_input":"2025-10-16T09:27:50.542147Z","iopub.status.idle":"2025-10-16T09:28:02.399632Z","shell.execute_reply.started":"2025-10-16T09:27:50.542125Z","shell.execute_reply":"2025-10-16T09:28:02.398413Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}