{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"colab":{"provenance":[]},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"e3ea2c52-3ead-4484-8a13-59ff459b0f78","cell_type":"code","source":"import pathlib\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport sklearn\nfrom sklearn import dummy, ensemble, feature_selection, linear_model, metrics\nfrom sklearn import model_selection, pipeline, preprocessing\n\n\nRANDOM_STATE = np.random.RandomState(42)","metadata":{"id":"e3ea2c52-3ead-4484-8a13-59ff459b0f78","trusted":true},"outputs":[],"execution_count":null},{"id":"3ece87be-31c0-40b1-b267-7ea6007ca052","cell_type":"code","source":"print(sklearn.__version__)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"9Ptcn-IIVEsP","cell_type":"markdown","source":"# Multiclass Classification","metadata":{"id":"9Ptcn-IIVEsP"}},{"id":"b42da759-b3c9-4f2b-8d69-54f89edfee5e","cell_type":"markdown","source":"## 1. Frame the Problem","metadata":{}},{"id":"8f7acc22-eab1-427b-8975-9e9bcd837715","cell_type":"code","source":"metrics.accuracy_score?","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"3455d90a-4af0-4773-9ae6-7d93a9866784","cell_type":"code","source":"COMPETITION_METRIC = \"accuracy\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"02cc24c9-260e-4840-8872-723f0adc6f41","cell_type":"markdown","source":"## 2. Load the data","metadata":{"id":"02cc24c9-260e-4840-8872-723f0adc6f41"}},{"id":"c7f59c53-7d0d-46ad-b099-b0c204346fbb","cell_type":"code","source":"%%bash\n\nls /kaggle/input/digit-recognizer","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"92c0231a-34d9-44dc-8981-e3e34545fea6","cell_type":"code","source":"%%bash\n\ncat /kaggle/input/digit-recognizer/train.csv | head -n 5","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"c1fded7d-02a7-48c3-99e4-d19d001c8ccf","cell_type":"code","source":"DATA_DIR = pathlib.Path(\"/kaggle/input/digit-recognizer\")\n\ntrain_df = pd.read_csv(\n    DATA_DIR / \"train.csv\",\n    dtype=np.uint8,\n)\n","metadata":{"id":"c1fded7d-02a7-48c3-99e4-d19d001c8ccf","trusted":true},"outputs":[],"execution_count":null},{"id":"b895f49f-cfa5-44a6-aed8-7af2b8df5af3","cell_type":"code","source":"train_df.info()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"bc9e5eda-6d8b-4db2-bd46-b982fb402599","cell_type":"code","source":"subsampled_train_df, _ = model_selection.train_test_split(\n    train_df,\n    train_size=0.10,\n    stratify=train_df.loc[:, \"label\"],\n    random_state=RANDOM_STATE\n) ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"036b655c-aad6-459c-91db-7b78c2266647","cell_type":"code","source":"subsampled_train_df.info()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"df34d6b4-6b37-4fb4-8594-6d4e9928825c","cell_type":"code","source":"train_features_df = subsampled_train_df.drop(\"label\", axis=1)\ntrain_labels = subsampled_train_df.loc[:, \"label\"]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"185515c7-a129-4e07-89c8-aba453d036ad","cell_type":"code","source":"train_features_df.info()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"a55fd839-4324-4265-87b6-7fab4b828a9a","cell_type":"code","source":"train_features_df.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"110d452e-713f-4e44-a8c8-dce3dae0d4fe","cell_type":"code","source":"train_labels.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"e9a7087d-9c75-4053-a4f3-f917dfb1541a","cell_type":"markdown","source":"## 3. Explore the Data","metadata":{}},{"id":"dc548e89-92f4-4925-95bb-54e181379f70","cell_type":"code","source":"train_features_df.describe()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"0f5eea0b-98fc-43e0-bc39-3805f1dd887b","cell_type":"code","source":"fig, axes = plt.subplots(10, 10, sharex=True, sharey=True, figsize=(15, 15))\nfor i in range(10):\n    for j in range(10):\n        m, _ = train_features_df.shape\n        k = RANDOM_STATE.randint(m)\n        img = (train_features_df.iloc[k, :]\n                                .to_numpy()\n                                .reshape((28, 28)))\n        _ = axes[i, j].imshow(img)\n        _ = axes[i, j].set_title(train_labels.iloc[k])\n\nfig.suptitle(\"Random MNIST images\", x=0.5, y=1.05, fontsize=25)\nfig.tight_layout()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"817cf22d-2fc4-4df1-a808-649935fdc96d","cell_type":"code","source":"_ = (\n    train_labels.value_counts()\n                .sort_index()\n                .plot(kind=\"bar\")\n)\n_ = plt.xticks(rotation=-45)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"5a0c0f87-de1e-4e9a-a93e-913342379778","cell_type":"markdown","source":"## 4. Prepare the Data for ML","metadata":{}},{"id":"c5097cb7-5623-4a3a-b102-9d89013a280b","cell_type":"markdown","source":"### Feature Selection","metadata":{}},{"id":"7f6125df-660e-43f8-b6fe-49211f1daba4","cell_type":"code","source":"feature_selection.VarianceThreshold?","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"11056749-479a-4faa-a13d-265f11a73f8c","cell_type":"markdown","source":"### Feature Scaling","metadata":{}},{"id":"9d80b5a7-3e21-4007-b406-68f23d3f44f2","cell_type":"code","source":"preprocessing.MinMaxScaler?","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"3c4a2abc-252e-4d3a-8ca0-debe3d50924c","cell_type":"code","source":"preprocessing.StandardScaler?","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"z7RoRlbes0m8","cell_type":"markdown","source":"## 5. Train a Few ML Models\n","metadata":{"id":"z7RoRlbes0m8"}},{"id":"a1eafa9b-9334-4fda-945a-f7419cca7c72","cell_type":"code","source":"dummy.DummyClassifier?","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"86fdbe6f-f2c4-4225-a8b4-196f18def28b","cell_type":"code","source":"dummy_classifier = dummy.DummyClassifier(\n    strategy=\"stratified\",\n    random_state=RANDOM_STATE\n)\n\n_ = dummy_classifier.fit(train_features_df, train_labels)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"c05e2775-6dd2-44d1-b774-28efa14dfc87","cell_type":"code","source":"linear_model.SGDClassifier?","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"u4wsOXL8tr3w","cell_type":"code","source":"sgd_classifier = pipeline.make_pipeline(\n    feature_selection.VarianceThreshold(threshold=0.0),\n    preprocessing.MinMaxScaler(feature_range=(-1, 1)),\n    linear_model.SGDClassifier(\n        loss=\"log_loss\",\n        random_state=RANDOM_STATE\n    ),\n)\n\n_ = sgd_classifier.fit(train_features_df, train_labels)","metadata":{"id":"u4wsOXL8tr3w","trusted":true},"outputs":[],"execution_count":null},{"id":"ad0e9587-7913-411f-8281-d534447aa1d5","cell_type":"code","source":"ensemble.RandomForestClassifier?","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"997466b5-78c9-4eb4-8739-2f64efb4c968","cell_type":"code","source":"random_forest_classifier = pipeline.make_pipeline(\n    feature_selection.VarianceThreshold(threshold=0.0),\n    ensemble.RandomForestClassifier(\n        criterion=\"log_loss\",\n        random_state=RANDOM_STATE\n    ),\n)\n\n_ = random_forest_classifier.fit(train_features_df, train_labels)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"CsYm1kJ3vPoz","cell_type":"markdown","source":"### Model Evaluation","metadata":{"id":"CsYm1kJ3vPoz"}},{"id":"069f6f69-1fe9-48b4-a25c-619b498cd035","cell_type":"code","source":"dummy_classifier_cv_predictions = model_selection.cross_val_predict(\n    dummy_classifier,\n    X=train_features_df,\n    y=train_labels,\n    cv=3,\n    n_jobs=3,\n    verbose=1\n)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"f364a82e-b5cb-412c-97e9-28f751f0607d","cell_type":"code","source":"sgd_classifier_cv_predictions = model_selection.cross_val_predict(\n    sgd_classifier,\n    X=train_features_df,\n    y=train_labels,\n    cv=3,\n    n_jobs=3,\n    verbose=1\n)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"1244eb47-37cd-4b15-8260-94b7c5d9b808","cell_type":"code","source":"random_forest_classifier_cv_predictions = model_selection.cross_val_predict(\n    random_forest_classifier,\n    X=train_features_df,\n    y=train_labels,\n    cv=3,\n    n_jobs=3,\n    verbose=1\n)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"181a27f8-dcfb-412e-83dc-27b8346619cb","cell_type":"markdown","source":"#### Classification Report","metadata":{}},{"id":"5393ab9c-d367-415f-aee6-c52894e5fcbb","cell_type":"code","source":"_report = metrics.classification_report(\n    train_labels,\n    dummy_classifier_cv_predictions,\n)\nprint(_report)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"egHzCLiuffZz","cell_type":"code","source":"_report = metrics.classification_report(\n    train_labels,\n    sgd_classifier_cv_predictions,\n)\nprint(_report)","metadata":{"id":"egHzCLiuffZz","trusted":true},"outputs":[],"execution_count":null},{"id":"583b1155-55da-4669-9775-be0b441cc73a","cell_type":"code","source":"_report = metrics.classification_report(\n    train_labels,\n    random_forest_classifier_cv_predictions,\n)\nprint(_report)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"121b5529-6e01-4a10-b80c-bf0d78e0be05","cell_type":"markdown","source":"#### Confusion Matrix","metadata":{}},{"id":"HZYcKbZd3NxY","cell_type":"code","source":"_ = (\n    metrics.ConfusionMatrixDisplay\n           .from_predictions(\n                train_labels,\n                random_forest_classifier_cv_predictions,\n            )\n)","metadata":{"id":"HZYcKbZd3NxY","trusted":true},"outputs":[],"execution_count":null},{"id":"89a0bf8f-ca7d-43fd-804f-8819fe69c005","cell_type":"code","source":"# normalizing by \"true\" (i.e., rows) gives recalls along the diagonal!\n_ = (\n    metrics.ConfusionMatrixDisplay\n           .from_predictions(\n                train_labels,\n                random_forest_classifier_cv_predictions,\n                normalize=\"true\",\n                values_format=\".0%\"\n            )\n)","metadata":{"id":"89a0bf8f-ca7d-43fd-804f-8819fe69c005","trusted":true},"outputs":[],"execution_count":null},{"id":"o8ICivivwCL7","cell_type":"code","source":"prediction_errors = (train_labels != random_forest_classifier_cv_predictions)\n\n_ = (\n    metrics.ConfusionMatrixDisplay\n           .from_predictions(\n                train_labels,\n                random_forest_classifier_cv_predictions,\n                normalize=\"true\",\n                sample_weight=prediction_errors,\n                values_format=\".0%\"\n            )\n)","metadata":{"id":"o8ICivivwCL7","trusted":true},"outputs":[],"execution_count":null},{"id":"fbVEYVaowUis","cell_type":"code","source":"# normalizing by \"pred\" (i.e., cols) gives precision along the diagonal!\n_ = (\n    metrics.ConfusionMatrixDisplay\n           .from_predictions(\n                train_labels,\n                random_forest_classifier_cv_predictions,\n                normalize=\"pred\",\n                values_format=\".0%\"\n            )\n)","metadata":{"id":"fbVEYVaowUis","trusted":true},"outputs":[],"execution_count":null},{"id":"9a764355-230e-42cf-8c06-2535996d63e3","cell_type":"code","source":"_ = (\n    metrics.ConfusionMatrixDisplay\n           .from_predictions(\n                train_labels,\n                random_forest_classifier_cv_predictions,\n                normalize=\"pred\",\n                sample_weight=prediction_errors,\n                values_format=\".0%\"\n            )\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"_In-RxVbwG4B","cell_type":"markdown","source":"#### ROC AUC","metadata":{"id":"_In-RxVbwG4B"}},{"id":"a89ebf22-b135-4875-9bdb-5afb831eb57f","cell_type":"code","source":"metrics.roc_auc_score?","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"c6522156-99c5-423d-a0fc-0aa8ceb63257","cell_type":"code","source":"random_forest_classifier_cv_probas = model_selection.cross_val_predict(\n    random_forest_classifier,\n    X=train_features_df,\n    y=train_labels,\n    cv=3,\n    method=\"predict_proba\",\n    n_jobs=3,\n    verbose=1\n)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"2799a3e3-37fb-4d73-bf6d-d457a55a46c0","cell_type":"code","source":"metrics.roc_auc_score(\n    train_labels,\n    random_forest_classifier_cv_probas,\n    average=\"weighted\",\n    multi_class=\"ovr\",\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"e1032723-dc30-42f1-834d-93c42f409b4d","cell_type":"markdown","source":"## 6. Fine-tuning an ML Model","metadata":{}},{"id":"7d9fe971-f9a1-4295-ae91-c3d56e535afa","cell_type":"code","source":"model_selection.GridSearchCV?","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"9be95152-85b8-4d6f-81ca-759aa3eee7c9","cell_type":"code","source":"random_forest_classifier","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"b64693b9-e3dc-48f4-9824-e8ffdbd88ec2","cell_type":"code","source":"random_forest_classifier.named_steps","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"53180313-a031-46da-86db-5f12f89666b7","cell_type":"code","source":"feature_selection.VarianceThreshold?","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"43a960ec-7ee5-4833-8dc1-f6aee706b336","cell_type":"code","source":"ensemble.RandomForestClassifier?","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"5058408d-9098-447e-a1b2-ff3ac8ff4f00","cell_type":"code","source":"param_grid = [\n    {\n        \"variancethreshold__threshold\": [0.0, 0.1, 1.0],\n        \"randomforestclassifier__n_estimators\": [10, 100, 1000],\n        \"randomforestclassifier__max_depth\": [1, 10, None],\n    },\n]\n\ntuned_classifier = model_selection.GridSearchCV(\n    random_forest_classifier,\n    param_grid,\n    cv=3,\n    n_jobs=3,\n    refit=True,\n    scoring=COMPETITION_METRIC,\n    verbose=2,\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"8df81d9b-ae99-48f4-a260-d7d4608d43ff","cell_type":"code","source":"tuned_classifier","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"6fe52439-e129-4d68-a1e4-4ff166a5ee56","cell_type":"code","source":"_ = tuned_classifier.fit(train_features_df, train_labels)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"ed128205-f439-4e02-8192-193b1ae08316","cell_type":"code","source":"# accessing the best hyperparameters\ntuned_classifier.best_params_","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"bf495c75-d135-49e8-839d-d63bab43734e","cell_type":"code","source":"# accessing the score for the best hyperparameters\ntuned_classifier.best_score_","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"dc610664-92f8-405e-ae0f-b11bda582d17","cell_type":"code","source":"# accessing the best estimator\ntuned_classifier.best_estimator_","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"9f0337d1-626d-4699-9c28-1bc0ccf548e1","cell_type":"code","source":"# now we refit the best estimator using the entire training set!\n_train_features_df = train_df.drop(\"label\", axis=1)\n_train_labels = train_df.loc[:, \"label\"]\n\n_ = (\n    tuned_classifier.best_estimator_\n                    .fit(\n                        _train_features_df,\n                        _train_labels\n                    )\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"086d0ada-b5fa-4605-b5a6-88ee57783114","cell_type":"markdown","source":"## 7. Submitting to the Competition","metadata":{}},{"id":"7c8e708e-5c21-4161-b3da-abf5bb0624b1","cell_type":"markdown","source":"### Generate Test Predictions","metadata":{}},{"id":"ead7ffcb-964d-4117-8796-f7e8a039927f","cell_type":"code","source":"test_features_df = pd.read_csv(\n    DATA_DIR / \"test.csv\",\n    dtype=np.uint8\n)\ntest_predictions = tuned_classifier.predict(test_features_df)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"c9b47771-90d3-4de0-b93a-adef3b61d9be","cell_type":"markdown","source":"### Create Submission File","metadata":{}},{"id":"c232b1f3-fc53-4078-aaa1-3b5af795fd29","cell_type":"code","source":"_ = (\n    pd.read_csv(\n        \"/kaggle/input/digit-recognizer/sample_submission.csv\",\n        index_col=\"ImageId\"\n    ).assign(\n        Label=test_predictions\n    ).to_csv(\"submission.csv\", index=True)\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"cfeed214-4718-4ac6-84c9-af2105515fac","cell_type":"code","source":"%%bash\n\ncat submission.csv | head -n 5","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"e1169d9f-fac9-4e81-b2eb-057f54fe1719","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}