{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Stacking Methods for Classification","metadata":{}},{"cell_type":"markdown","source":"## Exercise 1: Load the data\n\nUse Pandas to load the training and testing data.","metadata":{}},{"cell_type":"code","source":"%%bash\n\nls /kaggle/input/digit-recognizer","metadata":{"execution":{"iopub.status.busy":"2025-10-15T12:00:10.802399Z","iopub.execute_input":"2025-10-15T12:00:10.802773Z","iopub.status.idle":"2025-10-15T12:00:10.827733Z","shell.execute_reply.started":"2025-10-15T12:00:10.802740Z","shell.execute_reply":"2025-10-15T12:00:10.826443Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Solution","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\n\nTRAIN_DF = pd.read_csv(\n    \"/kaggle/input/digit-recognizer/train.csv\"\n)\n\nTEST_FEATURES_DF = pd.read_csv(\n    \"/kaggle/input/digit-recognizer/test.csv\"\n)\n","metadata":{"execution":{"iopub.status.busy":"2025-10-15T13:44:18.927032Z","iopub.execute_input":"2025-10-15T13:44:18.927598Z","iopub.status.idle":"2025-10-15T13:44:24.488647Z","shell.execute_reply.started":"2025-10-15T13:44:18.927541Z","shell.execute_reply":"2025-10-15T13:44:24.486955Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Exercise 2: Train/Val/Test Split\n\nSplit the data into a training set, validation set, and a testing set. The size of the validation set should be half the size of the test set. Make sure to use stratified sampling so that the distribution of classes is the same in both the training, validation, and testing sets.","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Solution","metadata":{}},{"cell_type":"code","source":"import numpy as np\nfrom sklearn import model_selection\n\n\nPRNG = np.random.RandomState(42)\n_stratify = TRAIN_DF.loc[:, \"label\"]\n\ntrain_df, _test_df = model_selection.train_test_split(\n    TRAIN_DF,\n    random_state=PRNG,\n    test_size=0.2,\n    stratify=_stratify,\n)\n\n_stratify = _test_df.loc[:, \"label\"]\n\nval_df, test_df = model_selection.train_test_split(\n    _test_df,\n    random_state=PRNG,\n    test_size=2 / 3,\n    stratify=_stratify,\n)\n\ntrain_features_df = train_df.drop(\"label\", axis=1)\ntrain_label = train_df.loc[:, \"label\"]\n\nval_features_df = val_df.drop(\"label\", axis=1)\nval_label = val_df.loc[:, \"label\"]\n\ntest_features_df = test_df.drop(\"label\", axis=1)\ntest_label = test_df.loc[:, \"label\"]\n","metadata":{"execution":{"iopub.status.busy":"2025-10-15T13:46:00.650579Z","iopub.execute_input":"2025-10-15T13:46:00.650984Z","iopub.status.idle":"2025-10-15T13:46:01.677027Z","shell.execute_reply.started":"2025-10-15T13:46:00.650949Z","shell.execute_reply":"2025-10-15T13:46:01.675803Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2025-10-15T13:46:02.820084Z","iopub.execute_input":"2025-10-15T13:46:02.820651Z","iopub.status.idle":"2025-10-15T13:46:02.876723Z","shell.execute_reply.started":"2025-10-15T13:46:02.820613Z","shell.execute_reply":"2025-10-15T13:46:02.875234Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"val_df.info()","metadata":{"execution":{"iopub.status.busy":"2025-10-15T13:46:10.069751Z","iopub.execute_input":"2025-10-15T13:46:10.070168Z","iopub.status.idle":"2025-10-15T13:46:10.112750Z","shell.execute_reply.started":"2025-10-15T13:46:10.070132Z","shell.execute_reply":"2025-10-15T13:46:10.110931Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.info()","metadata":{"execution":{"iopub.status.busy":"2025-10-15T13:46:15.199809Z","iopub.execute_input":"2025-10-15T13:46:15.200226Z","iopub.status.idle":"2025-10-15T13:46:15.242482Z","shell.execute_reply.started":"2025-10-15T13:46:15.200191Z","shell.execute_reply":"2025-10-15T13:46:15.241147Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Exercise 3: Prepare the data for ML\n\nCreate a data preparation pipeline to prepare the raw training data for machine learning algorithms. Your data preparation pipeline should have three stages.\n\n1. **Data Cleaning:** use [impute.SimpleImputer](https://scikit-learn.org/stable/modules/generated/sklearn.impute.SimpleImputer.html) to make sure your pipeline handles missing values properly.\n2. **Feature Selection:** use [feature_selection.VarianceThreshold](https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.VarianceThreshold.html) to make sure that you pipeline removes any useless features from the training data.\n3. **Feature Scaling:** use [preprocessing.StandardScaler](https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.StandardScaler.html) to make sure that the features have mean zero.\n\nYour data preparation pipeline should return a Pandas DataFrame.","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Solution","metadata":{}},{"cell_type":"code","source":"from sklearn import feature_selection, impute, pipeline, preprocessing\n\n\ndata_preparation_pipeline = pipeline.make_pipeline(\n    impute.SimpleImputer(strategy=\"most_frequent\"),\n    feature_selection.VarianceThreshold(threshold=0.0),\n    preprocessing.StandardScaler(with_mean=False)\n).set_output(transform=\"pandas\")\n","metadata":{"execution":{"iopub.status.busy":"2025-10-15T13:46:59.291791Z","iopub.execute_input":"2025-10-15T13:46:59.292219Z","iopub.status.idle":"2025-10-15T13:46:59.388835Z","shell.execute_reply.started":"2025-10-15T13:46:59.292183Z","shell.execute_reply":"2025-10-15T13:46:59.387176Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_preparation_pipeline","metadata":{"execution":{"iopub.status.busy":"2025-10-15T13:47:01.730301Z","iopub.execute_input":"2025-10-15T13:47:01.730833Z","iopub.status.idle":"2025-10-15T13:47:01.747818Z","shell.execute_reply.started":"2025-10-15T13:47:01.730801Z","shell.execute_reply":"2025-10-15T13:47:01.746024Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Exercise 4: Train different classifiers\n\nTrain the following classifiers. \n\n1. [ensemble.ExtraTreesClassifier](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.ExtraTreesClassifier.html)\n2. [ensemble.HistGradientBoostingClassifier](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.HistGradientBoostingClassifier.html)\n3. [neighbors.KNeighborsClassifier](https://scikit-learn.org/stable/modules/generated/sklearn.neighbors.KNeighborsClassifier.html)\n4. [linear_model.SGDClassifier](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.SGDClassifier.html)","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Solution","metadata":{}},{"cell_type":"code","source":"import joblib\nimport pathlib\nfrom sklearn import decomposition, ensemble, linear_model, neighbors\n\n\nNAMED_CLASSIFIERS = {\n    \"extratreesclassifier\": pipeline.make_pipeline(\n        data_preparation_pipeline,\n        ensemble.ExtraTreesClassifier(\n            bootstrap=True,\n            criterion=\"log_loss\",\n            max_features=\"sqrt\",\n            max_samples=0.8,\n            n_estimators=100,\n            n_jobs=-1,\n            random_state=PRNG,\n        )\n    ),\n    \"histgradientboostingclassifier\": pipeline.make_pipeline(\n        data_preparation_pipeline,\n        ensemble.HistGradientBoostingClassifier(\n            early_stopping=True,\n            loss=\"log_loss\",\n            n_iter_no_change=10,\n            random_state=PRNG,\n            validation_fraction=0.1,\n        )\n    ),\n    \"kneighborsclassifier\": pipeline.make_pipeline(\n        data_preparation_pipeline,\n        decomposition.PCA(\n            n_components=0.95,\n            whiten=True,\n        ),\n        neighbors.KNeighborsClassifier(\n            n_jobs=-1,\n            n_neighbors=3,\n            weights=\"distance\",\n        )\n    ),\n    \"sgdclassifier\": pipeline.make_pipeline(\n        data_preparation_pipeline,\n        linear_model.SGDClassifier(\n            loss=\"hinge\",\n            n_jobs=-1,\n            random_state=PRNG,\n        )\n    ),\n}","metadata":{"execution":{"iopub.status.busy":"2025-10-15T13:47:10.197167Z","iopub.execute_input":"2025-10-15T13:47:10.197610Z","iopub.status.idle":"2025-10-15T13:47:10.245547Z","shell.execute_reply.started":"2025-10-15T13:47:10.197574Z","shell.execute_reply":"2025-10-15T13:47:10.244308Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for (name, classifier) in NAMED_CLASSIFIERS.items():\n\n    model_filepath = pathlib.Path(f\"/kaggle/working/{name}.pkl\")\n\n    if not model_filepath.exists():\n\n        _ = classifier.fit(\n            train_features_df,\n            train_label\n        )\n\n        _ = joblib.dump(\n            classifier,\n            model_filepath\n        )\n        print(f\"Done training {name}!\")\n\n    NAMED_CLASSIFIERS[name] = joblib.load(model_filepath)\n","metadata":{"execution":{"iopub.status.busy":"2025-10-15T13:47:13.278272Z","iopub.execute_input":"2025-10-15T13:47:13.278890Z","iopub.status.idle":"2025-10-15T13:51:02.421245Z","shell.execute_reply.started":"2025-10-15T13:47:13.278852Z","shell.execute_reply":"2025-10-15T13:51:02.419784Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Exercise 5: Create a blender training set\n\nMake predictions with each of your classifiers on the validation set and create a training set for your blender model with the resulting predictions. Each training instance for your blender model should be a vector containing the set of predictions from all your classifiers; the blender training label should be the corresponding class label from the validation set. ","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Solution","metadata":{}},{"cell_type":"code","source":"_val_predictions = {}\nfor (name, classifier) in NAMED_CLASSIFIERS.items():\n    _val_predictions[name] = classifier.predict(val_features_df)\n    \nblender_train_features_df = (\n    pd.DataFrame\n      .from_dict(_val_predictions)\n)\n\nblender_train_label = val_label","metadata":{"execution":{"iopub.status.busy":"2025-10-15T13:51:02.423238Z","iopub.execute_input":"2025-10-15T13:51:02.423632Z","iopub.status.idle":"2025-10-15T13:51:04.613148Z","shell.execute_reply.started":"2025-10-15T13:51:02.423597Z","shell.execute_reply":"2025-10-15T13:51:04.611920Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"blender_train_features_df.tail()","metadata":{"execution":{"iopub.status.busy":"2025-10-15T13:51:04.614346Z","iopub.execute_input":"2025-10-15T13:51:04.614749Z","iopub.status.idle":"2025-10-15T13:51:04.637690Z","shell.execute_reply.started":"2025-10-15T13:51:04.614709Z","shell.execute_reply":"2025-10-15T13:51:04.634891Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"blender_train_features_df.info()","metadata":{"execution":{"iopub.status.busy":"2025-10-15T13:51:48.400826Z","iopub.execute_input":"2025-10-15T13:51:48.401354Z","iopub.status.idle":"2025-10-15T13:51:48.412906Z","shell.execute_reply.started":"2025-10-15T13:51:48.401319Z","shell.execute_reply":"2025-10-15T13:51:48.411184Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Exercise 6: Train a final classifier\n\nTrain a classifier on your new blender training set. The final classifier together with the individual classifiers trained above form a stacking ensemble! ","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Solution","metadata":{}},{"cell_type":"code","source":"from sklearn import svm\n\n\nBLENDER_CLASSIFIER = pipeline.make_pipeline(\n    preprocessing.MinMaxScaler(feature_range=(-1, 1)),\n    svm.SVC(\n        max_iter=1000,\n        random_state=PRNG,\n    )\n)\n\n_ = BLENDER_CLASSIFIER.fit(\n    blender_train_features_df,\n    blender_train_label\n)\n","metadata":{"execution":{"iopub.status.busy":"2025-10-15T13:53:55.541587Z","iopub.execute_input":"2025-10-15T13:53:55.541998Z","iopub.status.idle":"2025-10-15T13:53:55.611492Z","shell.execute_reply.started":"2025-10-15T13:53:55.541964Z","shell.execute_reply":"2025-10-15T13:53:55.610164Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Exercise 7: Evaluate your stacking ensemble\n\nNow evaluate the ensemble on the test set. For each image in the test set, make predictions with all your classifiers, then feed the predictions to the blender to get the ensemble’s predictions. ","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Solution","metadata":{}},{"cell_type":"code","source":"def stacked_ensemble_predict(features_df):\n    ensemble_predictions = {}\n    for (name, classifier) in NAMED_CLASSIFIERS.items():\n        ensemble_predictions[name] = classifier.predict(features_df)\n\n    blender_features_df = (\n        pd.DataFrame\n          .from_dict(ensemble_predictions)\n    )\n    stacked_predictions = BLENDER_CLASSIFIER.predict(blender_features_df) \n\n    return stacked_predictions","metadata":{"execution":{"iopub.status.busy":"2025-10-15T13:55:13.173528Z","iopub.execute_input":"2025-10-15T13:55:13.173948Z","iopub.status.idle":"2025-10-15T13:55:13.181312Z","shell.execute_reply.started":"2025-10-15T13:55:13.173912Z","shell.execute_reply":"2025-10-15T13:55:13.179612Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn import metrics\n\n\n_test_predictions = stacked_ensemble_predict(test_features_df)\n_report = metrics.classification_report(\n    test_label,\n    _test_predictions,\n)\n\nprint(_report)","metadata":{"execution":{"iopub.status.busy":"2025-10-15T13:55:13.422282Z","iopub.execute_input":"2025-10-15T13:55:13.422718Z","iopub.status.idle":"2025-10-15T13:55:17.609660Z","shell.execute_reply.started":"2025-10-15T13:55:13.422680Z","shell.execute_reply":"2025-10-15T13:55:17.608246Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Exercise 8: Using the StackingClassifier\n\nNow try again using the [ensemble.StackingClassifier](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.StackingClassifier.html). Does this implementation give better performance than your \"by hand\" implementation?","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Solution","metadata":{}},{"cell_type":"code","source":"_checkpoint = \"/kaggle/working/stacking-classifier.pkl\"\nmodel_filepath = pathlib.Path(_checkpoint)\n\nif not model_filepath.exists():\n    \n    # use CV to create the set used to train the final estimator\n    _estimators = list(NAMED_CLASSIFIERS.items())\n    _classifier = ensemble.StackingClassifier(\n        estimators=_estimators,\n        final_estimator=BLENDER_CLASSIFIER,\n        cv=3,\n        n_jobs=-1,\n    )\n\n    # use the combined training and validation sets to fit!\n    _ = _classifier.fit(\n        pd.concat([train_features_df, val_features_df]),\n        pd.concat([train_label, val_label])\n    )\n    \n    _ = joblib.dump(\n        _classifier,\n        model_filepath\n    )\n\nstacking_classifier = joblib.load(model_filepath)","metadata":{"execution":{"iopub.status.busy":"2025-10-15T13:57:46.421846Z","iopub.execute_input":"2025-10-15T13:57:46.422418Z","iopub.status.idle":"2025-10-15T14:11:25.775785Z","shell.execute_reply.started":"2025-10-15T13:57:46.422342Z","shell.execute_reply":"2025-10-15T14:11:25.773933Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_test_predictions = stacking_classifier.predict(test_features_df)\n_report = metrics.classification_report(\n    test_label,\n    _test_predictions,\n)\n\nprint(_report)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-15T14:11:25.779598Z","iopub.execute_input":"2025-10-15T14:11:25.780080Z","iopub.status.idle":"2025-10-15T14:11:31.436800Z","shell.execute_reply.started":"2025-10-15T14:11:25.780013Z","shell.execute_reply":"2025-10-15T14:11:31.435220Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Exercise 9: Submit predictions\n\nLoad the sample submission file and replace the `Label` column with your best classifier predictions on the test set.","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Solution","metadata":{}},{"cell_type":"code","source":"_checkpoint = \"/kaggle/working/stacking-classifier.pkl\"\nmodel_filepath = pathlib.Path(_checkpoint)\n\nif not model_filepath.exists():\n    \n    # use CV to create the set used to train the final estimator\n    _estimators = list(NAMED_CLASSIFIERS.items())\n    _classifier = ensemble.StackingClassifier(\n        estimators=_estimators,\n        final_estimator=BLENDER_CLASSIFIER,\n        cv=3,\n        n_jobs=-1,\n    )\n\n    # use the full training set to fit!\n    _ = _classifier.fit(\n        TRAIN_DF.drop(\"label\", axis=1),\n        TRAIN_DF.loc[:, \"label\"]\n    )\n    \n    _ = joblib.dump(\n        _classifier,\n        model_filepath\n    )\n\nstacking_classifier = joblib.load(model_filepath)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-15T12:25:42.008248Z","iopub.execute_input":"2025-10-15T12:25:42.008830Z","iopub.status.idle":"2025-10-15T12:25:43.930444Z","shell.execute_reply.started":"2025-10-15T12:25:42.008787Z","shell.execute_reply":"2025-10-15T12:25:43.929148Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_test_predictions = stacking_classifier.predict(TEST_FEATURES_DF)\n\n_ = (\n    pd.read_csv(\n        \"/kaggle/input/digit-recognizer/sample_submission.csv\",\n        index_col=\"ImageId\"\n    ).assign(\n        Label=_test_predictions\n    ).to_csv(\"submission.csv\", index=True)\n)","metadata":{"execution":{"iopub.status.busy":"2025-10-15T12:25:43.933261Z","iopub.execute_input":"2025-10-15T12:25:43.933676Z","iopub.status.idle":"2025-10-15T12:26:09.453197Z","shell.execute_reply.started":"2025-10-15T12:25:43.933641Z","shell.execute_reply":"2025-10-15T12:26:09.451883Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%bash\n\ncat submission.csv | head -n 5","metadata":{"execution":{"iopub.status.busy":"2025-10-15T12:26:09.455487Z","iopub.execute_input":"2025-10-15T12:26:09.455886Z","iopub.status.idle":"2025-10-15T12:26:09.483017Z","shell.execute_reply.started":"2025-10-15T12:26:09.455849Z","shell.execute_reply":"2025-10-15T12:26:09.481805Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}