{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"523be573-4df4-4154-8657-7e07cc4f7984","_cell_guid":"12b19307-4321-4108-aafa-03a0cec91fdc","jupyter":{"outputs_hidden":false},"collapsed":false,"execution":{"iopub.status.busy":"2023-10-22T01:02:40.392790Z","iopub.execute_input":"2023-10-22T01:02:40.393356Z","iopub.status.idle":"2023-10-22T01:02:40.441462Z","shell.execute_reply.started":"2023-10-22T01:02:40.393196Z","shell.execute_reply":"2023-10-22T01:02:40.440617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* [EDA](#chapter1)\n    * [Imports](#subsection1)\n* [Feature Engineering](#chapter2)\n    * [Section 3.1](#section_3_1)","metadata":{}},{"cell_type":"markdown","source":"# EDA <a class=\"anchor\"  id=\"chapter1\"></a>","metadata":{"_uuid":"e23d86f2-815c-4c38-bdfa-6849bb01956d","_cell_guid":"4246a80a-f5da-4f3f-a75d-bc7bab5d2f6b","trusted":true}},{"cell_type":"markdown","source":"## Imports <a class=\"anchor\"  id=\"subsection1\"></a>","metadata":{"_uuid":"11bff5c6-d7d2-4a4d-b897-f632ba963477","_cell_guid":"e5bd386d-7cd0-4499-a47e-027e282cc7dd","trusted":true}},{"cell_type":"code","source":"import seaborn as sns\nimport warnings\nimport matplotlib.pyplot as plt\nimport plotly.express as px\n\nfrom pathlib import Path\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nfrom sklearn.preprocessing import OrdinalEncoder\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.compose import make_column_transformer\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom sklearn.preprocessing import RobustScaler\nfrom IPython.display import display\nfrom pandas.api.types import CategoricalDtype\nfrom category_encoders import MEstimateEncoder\nfrom sklearn.cluster import KMeans\nfrom sklearn.decomposition import PCA\nfrom sklearn.feature_selection import mutual_info_regression\nfrom sklearn.model_selection import KFold, cross_val_score\nfrom sklearn.ensemble import RandomForestRegressor\nfrom scipy import stats\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.linear_model import SGDOneClassSVM\nfrom sklearn.ensemble import IsolationForest\nfrom sklearn.neighbors import LocalOutlierFactor\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline\n\nfrom matplotlib.ticker import MaxNLocator\nfrom functools import partial\n\nfrom sklearn.inspection import permutation_importance\nfrom sklearn.tree import DecisionTreeRegressor, plot_tree\nfrom sklearn.metrics import mean_absolute_error, mean_squared_error\n\nfrom sklearn.preprocessing import PowerTransformer\nfrom sklearn.preprocessing import QuantileTransformer\nfrom sklearn import preprocessing\nfrom sklearn.preprocessing import MaxAbsScaler\nfrom sklearn.preprocessing import MinMaxScaler\nfrom lightgbm import LGBMRegressor\nfrom lightgbm import LGBMClassifier\nfrom catboost import CatBoostRegressor\nfrom catboost import CatBoostClassifier\nfrom sklearn.model_selection import RandomizedSearchCV\nfrom xgboost import XGBRegressor\nfrom xgboost import XGBClassifier\nfrom itertools import groupby\nimport gc\nfrom time import time\n\nimport dask\nimport dask.array as da\nimport dask.dataframe as dd\nimport polars as pl","metadata":{"_uuid":"91de1041-0ddd-45e4-9dc3-0bf0c99c423f","_cell_guid":"756c764e-933d-47ef-a310-475ea2b0ab8d","execution":{"iopub.status.busy":"2023-10-22T01:02:40.443124Z","iopub.execute_input":"2023-10-22T01:02:40.443972Z","iopub.status.idle":"2023-10-22T01:02:53.645193Z","shell.execute_reply.started":"2023-10-22T01:02:40.443941Z","shell.execute_reply":"2023-10-22T01:02:53.644033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Config","metadata":{"_uuid":"50171726-3e51-43f2-9851-b6a1d3cbf3d6","_cell_guid":"33436eec-6f9b-4ab0-a22a-1ac4af49e8bc","trusted":true}},{"cell_type":"code","source":"TARGET = 'awake'\nRESULT_BASELINE = []\n\n# Set Matplotlib defaults\nplt.style.use(\"seaborn-whitegrid\")\nplt.rc(\"figure\", autolayout=True)\nplt.rc(\n    \"axes\",\n    labelweight=\"bold\",\n    labelsize=\"large\",\n    titleweight=\"bold\",\n    titlesize=14,\n    titlepad=10,\n)\n\ndef plot_maxdeph_score(rd_search, randomizedSearchCV_model):\n    max_depth = [item['model__max_depth'] for item in rd_search.named_steps[randomizedSearchCV_model].cv_results_['params']]\n    scores = list(rd_search.named_steps[randomizedSearchCV_model].cv_results_['mean_test_score'])\n    d = pd.DataFrame([max_depth, scores]).T\n    d.columns = ['Max Depth','Score']\n    d.groupby(['Max Depth']).mean()\n    return d\n\n# Mute warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"fe8e9ba5-3562-42c8-a516-e2151d848b92","_cell_guid":"f361a707-ce5c-4b81-bf72-14756a801455","execution":{"iopub.status.busy":"2023-10-22T01:02:53.646887Z","iopub.execute_input":"2023-10-22T01:02:53.648003Z","iopub.status.idle":"2023-10-22T01:02:53.659360Z","shell.execute_reply.started":"2023-10-22T01:02:53.647962Z","shell.execute_reply":"2023-10-22T01:02:53.658083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load data","metadata":{"_uuid":"4fe6b15b-9a15-49ab-bab4-a4e6d0122120","_cell_guid":"21ead8b6-2fbe-4334-8272-da44c5364fd4","trusted":true}},{"cell_type":"code","source":"# binary class data\n# train = pd.read_parquet(\"/kaggle/input/zzzs-lightweight-training-dataset-target/Zzzs_train.parquet\")\n# multiclass data\n\n# train = pd.read_parquet(\"/kaggle/input/zzzs-lightweight-training-dataset-target/Zzzs_train_multi.parquet\")\n# test  = pd.read_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet\")\n\ntrain = (pl.scan_parquet(\"/kaggle/input/zzzs-lightweight-training-dataset-target/Zzzs_train_multi.parquet\")\n          .with_columns(\n              (pl.col(\"timestamp\")\n                 .str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%Z\")\n                 .dt.hour()\n                 .alias(\"hour\")\n              )\n          )\n          .drop(\"timestamp\")\n          .collect()\n          .to_pandas()\n)\n\n# train = dd.read_parquet(\"/kaggle/input/zzzs-lightweight-training-dataset-target/Zzzs_train_multi.parquet\")\n# test  = dd.read_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet\")","metadata":{"_uuid":"1c1959d8-4e97-4c35-bad5-7914f8701295","_cell_guid":"665ef446-ba5d-4029-a5e3-1dc75876f20e","execution":{"iopub.status.busy":"2023-10-22T01:02:53.662166Z","iopub.execute_input":"2023-10-22T01:02:53.662622Z","iopub.status.idle":"2023-10-22T01:03:11.013640Z","shell.execute_reply.started":"2023-10-22T01:02:53.662584Z","shell.execute_reply":"2023-10-22T01:03:11.012534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Statistics","metadata":{"_uuid":"2d02f48d-c859-4905-bded-5ab636d0558e","_cell_guid":"2233a912-97e9-4506-9888-745b6a12ae3d","trusted":true}},{"cell_type":"code","source":"train.head()","metadata":{"_uuid":"77a3847a-842b-46d3-b763-e69f523bd020","_cell_guid":"24367821-b373-4071-a44c-e7685545a05c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:11.015127Z","iopub.execute_input":"2023-10-22T01:03:11.015503Z","iopub.status.idle":"2023-10-22T01:03:11.038186Z","shell.execute_reply.started":"2023-10-22T01:03:11.015472Z","shell.execute_reply":"2023-10-22T01:03:11.037119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe()","metadata":{"_uuid":"ed01a9b3-5364-42cc-8ae0-d81be0faa9d5","_cell_guid":"8c9400cf-b645-4b6a-b402-be8e12de964f","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:11.039765Z","iopub.execute_input":"2023-10-22T01:03:11.040124Z","iopub.status.idle":"2023-10-22T01:03:13.230985Z","shell.execute_reply.started":"2023-10-22T01:03:11.040094Z","shell.execute_reply":"2023-10-22T01:03:13.229628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"[train.shape]","metadata":{"_uuid":"ff1583da-8c35-4525-ab6a-08e1d04d716a","_cell_guid":"5859d21c-9fac-4805-bb1c-da58925de024","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:13.232530Z","iopub.execute_input":"2023-10-22T01:03:13.232843Z","iopub.status.idle":"2023-10-22T01:03:13.239814Z","shell.execute_reply.started":"2023-10-22T01:03:13.232815Z","shell.execute_reply":"2023-10-22T01:03:13.238584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"_uuid":"d8686870-855a-4845-8192-1b479f301f74","_cell_guid":"8e0ef578-d1d7-4859-97ee-67a70b495450","execution":{"iopub.status.busy":"2023-10-22T01:03:13.241726Z","iopub.execute_input":"2023-10-22T01:03:13.242130Z","iopub.status.idle":"2023-10-22T01:03:13.266616Z","shell.execute_reply.started":"2023-10-22T01:03:13.242090Z","shell.execute_reply":"2023-10-22T01:03:13.265317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_col = train.select_dtypes(exclude=['object'])#.drop([TARGET], axis=1).copy()\ncat_col = train.select_dtypes(include=['object'])\nord_col = []\nall_col = [f for f in train.columns if f != TARGET]","metadata":{"_uuid":"daec5af8-3932-4371-bd84-81b74ea14adc","_cell_guid":"0af213b3-32b7-442e-8223-4d945737032c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:13.268562Z","iopub.execute_input":"2023-10-22T01:03:13.268990Z","iopub.status.idle":"2023-10-22T01:03:13.662508Z","shell.execute_reply.started":"2023-10-22T01:03:13.268949Z","shell.execute_reply":"2023-10-22T01:03:13.661570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(num_col.columns)\nprint(cat_col.columns)","metadata":{"_uuid":"82a866a0-9718-42ca-8af6-1d423b376b69","_cell_guid":"a538bd9d-2559-4128-b299-471ee772fea3","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:13.667857Z","iopub.execute_input":"2023-10-22T01:03:13.668180Z","iopub.status.idle":"2023-10-22T01:03:13.674265Z","shell.execute_reply.started":"2023-10-22T01:03:13.668153Z","shell.execute_reply":"2023-10-22T01:03:13.673222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Understand data","metadata":{"_uuid":"8b23b790-8777-445c-8bcf-13979da15d07","_cell_guid":"208a2e9a-7a7e-4954-9b71-8fbb8906a526","trusted":true}},{"cell_type":"markdown","source":"## Univariate analysis","metadata":{"_uuid":"a476afad-d1fc-4f0d-9efc-b9d8c597a670","_cell_guid":"c20954fa-db6f-4bb4-9516-bc80281e3d28","trusted":true}},{"cell_type":"markdown","source":"### Numeric features","metadata":{"_uuid":"462d67c8-f420-42f5-a4fa-732975a27c3e","_cell_guid":"42ab8da7-6285-4265-86d1-d1373f61d2eb","trusted":true}},{"cell_type":"markdown","source":"For numerical features, we are always concerned about the distribution of these features, including the statistical characteristics of these columns e.g mean, median, mode. Hence we will usually use Distribution plot to visualize their data distribution. Boxplots are also commonly used to unearth the statistical characteristics of each feature. More often than not, we use it to look for any outliers that we might need to filter out later on during the preprocessing step.","metadata":{"_uuid":"762322bf-d10d-43df-98fa-9ac9d6686f9e","_cell_guid":"12460bc9-a6a0-4422-8fd8-65de91f54c24","trusted":true}},{"cell_type":"code","source":"# disc_num_var = ['id']\n\n# cont_num_var = []\n# for i in num_col.columns:\n#     if i not in disc_num_var:\n#         cont_num_var.append(i)\n\n# fig = plt.figure(figsize=(16,35))\n# for index,col in enumerate(cont_num_var):\n#     plt.subplot(10,4,index+1)\n#     sns.distplot(num_col.loc[:,col].dropna(), kde=False)\n# fig.tight_layout(pad=2.0)","metadata":{"_uuid":"19019f9b-816f-413a-813e-d73d3f002eba","_cell_guid":"df761b10-6fab-462f-9962-83965a07c39e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:13.675979Z","iopub.execute_input":"2023-10-22T01:03:13.676353Z","iopub.status.idle":"2023-10-22T01:03:13.686470Z","shell.execute_reply.started":"2023-10-22T01:03:13.676321Z","shell.execute_reply":"2023-10-22T01:03:13.685239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This plot help us idenytify some variables with mostly 1 value as seen from the plots. Features with mostly 0s are highly *skewed*. Having alot of 0s in the distribution doesnt really add information for predicting problems. Hence, we can remove them during our preprocessing step.","metadata":{"_uuid":"c60f7756-5a31-4116-aa20-02e6c3a7879e","_cell_guid":"eed84582-0f34-4e0e-9f66-80876a1b1e8e","trusted":true}},{"cell_type":"markdown","source":"### Boxplot to identify outliers","metadata":{"_uuid":"309476ac-dbac-495a-9493-5de2f5b3c995","_cell_guid":"b2fa3267-7b27-43ee-a243-0e856a2f579a","trusted":true}},{"cell_type":"code","source":"# fig = plt.figure(figsize=(16,35))\n# for index,col in enumerate(cont_num_var):\n#     plt.subplot(10,4,index+1)\n#     sns.boxplot(y=col, data=num_col.dropna())\n# fig.tight_layout(pad=2.0)","metadata":{"_uuid":"b5637847-30b3-4a4d-be29-a4f9704e9f2b","_cell_guid":"395c189b-bad9-4a77-8082-b3fd087d19c2","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:13.687984Z","iopub.execute_input":"2023-10-22T01:03:13.688322Z","iopub.status.idle":"2023-10-22T01:03:13.699767Z","shell.execute_reply.started":"2023-10-22T01:03:13.688292Z","shell.execute_reply":"2023-10-22T01:03:13.698658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fig = plt.figure(figsize=(20,15))\n# for index,col in enumerate(disc_num_var):\n#     plt.subplot(10,3,index+1)\n#     sns.countplot(x=col, data=num_col.dropna())\n# fig.tight_layout(pad=1.0)","metadata":{"_uuid":"c4cf3578-4fd0-418a-998f-f3d763edb547","_cell_guid":"9367e549-a0b0-4324-9cef-bf3f91b18eb9","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:13.701641Z","iopub.execute_input":"2023-10-22T01:03:13.701974Z","iopub.status.idle":"2023-10-22T01:03:13.712203Z","shell.execute_reply.started":"2023-10-22T01:03:13.701946Z","shell.execute_reply":"2023-10-22T01:03:13.711350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Categorical features","metadata":{"_uuid":"d0e432e5-f1be-48ef-b94c-5e02210cf7cd","_cell_guid":"13ea9146-ffba-4c33-9f18-18f104b515d3","trusted":true}},{"cell_type":"markdown","source":"In the case of categorical features, we will often use countplots to visualize the count of each distinct value within each features. We could find some categorical features that consist of mainly just one value, which does not add any useful information. Thus, we should also remove them later on.","metadata":{"_uuid":"198d6836-a4fa-4435-b4b1-b0cb90976aee","_cell_guid":"4b0f3c15-2260-4009-8036-181f343dff9b","trusted":true}},{"cell_type":"code","source":"# fig = plt.figure(figsize=(18,20))\n# for index in range(len(cat_col.columns)):\n#     plt.subplot(9,5,index+1)\n#     sns.countplot(x=cat_col.iloc[:,index], data=cat_col.dropna())\n#     plt.xticks(rotation=90)\n# fig.tight_layout(pad=2.0)","metadata":{"_uuid":"6c25def2-27e0-49e1-a2ec-c9a39d89c1cf","_cell_guid":"57192368-a5b4-4faa-870f-97c921c97aa8","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:13.713756Z","iopub.execute_input":"2023-10-22T01:03:13.714391Z","iopub.status.idle":"2023-10-22T01:03:13.725739Z","shell.execute_reply.started":"2023-10-22T01:03:13.714359Z","shell.execute_reply":"2023-10-22T01:03:13.724436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Univariate Analysis helps us to understand all the features better, on an individual scale. To further deepen our insights and uncover potential pattern in the data, we will also need to find out more about the relationship between all these features with one another, which brings us to our next step in our analysis - Bivariate Analysis","metadata":{"_uuid":"8287c21d-a009-4e80-bb49-51dad5829d46","_cell_guid":"9a7c9edb-57f2-403d-a091-1905b9c9637f","trusted":true}},{"cell_type":"markdown","source":"## Bi-Variate Analysis","metadata":{"_uuid":"c7b1c603-16e1-4878-87c6-9dd8717499e0","_cell_guid":"e86c80cd-c335-4387-892c-5fe3b7ccefed","trusted":true}},{"cell_type":"markdown","source":"Bi-variate analysis looks at 2 different features to identify any possible relationship or distinctive patterns between the 2 features. One of the commonly used technique is through the Correlation Matrix. Correlation matrix is an effective tool to uncover linear relationship (Correlation) between any 2 continuous features. Correlation not only allow us to determine which features are important to TARGET, but also as a mean to investigate any multicollinearity between our independent predictors.\nMulticollinearity happens when 2 or more independent variables are highly correlated with one another. In such situation, it causes precision loss in our regression coefficients, affecting our ability to identify the most important features that are most useful to our model","metadata":{"_uuid":"34879c77-5d0e-4d39-8721-3de9a8432e58","_cell_guid":"4a3a2f69-a09a-4207-be7c-c78f8aa47647","trusted":true}},{"cell_type":"markdown","source":"### Correlation matrix","metadata":{"_uuid":"1dc3632b-9d5c-442f-a40c-44a3648d8d46","_cell_guid":"2ea50fc4-3912-4ad5-bcb8-8318263e2e52","trusted":true}},{"cell_type":"code","source":"# plt.figure(figsize=(14,12))\n# correlation = num_col.corr()\n# sns.heatmap(correlation, mask = correlation <0.8, linewidth=0.5, cmap='Blues')","metadata":{"_uuid":"dcd14afa-c501-4716-918e-c5a7089d6108","_cell_guid":"db848611-d716-4b21-b120-e83097af1542","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:13.727163Z","iopub.execute_input":"2023-10-22T01:03:13.727798Z","iopub.status.idle":"2023-10-22T01:03:13.742609Z","shell.execute_reply.started":"2023-10-22T01:03:13.727765Z","shell.execute_reply":"2023-10-22T01:03:13.741300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From the correlation matrix we didnt identify any variables which are highly correlated with each other. Otherwise it would guide us in our preprocessing steps later on as we aim to remove highly correlated features to avoid performance loss in our model","metadata":{"_uuid":"9e0e0731-7f31-48ef-b6a4-8a8d093f43ab","_cell_guid":"c8bdbb52-a98b-4ce7-bd77-8fd13762a1b8","trusted":true}},{"cell_type":"markdown","source":"#### Identifying relationship between Numerical Predictor and Target","metadata":{"_uuid":"fe40cd1b-7705-4488-9cd2-6b903140a1c9","_cell_guid":"ca26caa8-13bc-434a-b038-dab81841ceb9","trusted":true}},{"cell_type":"markdown","source":"Below, we sorted the strength of linear relationship between TARGET and other features. Hence, these features will be important factor.","metadata":{"_uuid":"395cb597-55be-4921-a3fd-6bfeaf15da62","_cell_guid":"c9dbe7c9-a3fe-4ed7-9b96-053d125ee1c4","trusted":true}},{"cell_type":"code","source":"# num_col_TARGET = train.select_dtypes(exclude=['object'])\n# correlation = num_col_TARGET.corr()\n# correlation[[TARGET]].sort_values([TARGET], ascending=False)","metadata":{"_uuid":"053038f2-a065-4336-9310-3eb0ec8ac7dd","_cell_guid":"88bfd5b1-5be7-4d98-8081-430fec756054","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:13.744249Z","iopub.execute_input":"2023-10-22T01:03:13.744584Z","iopub.status.idle":"2023-10-22T01:03:13.754456Z","shell.execute_reply.started":"2023-10-22T01:03:13.744554Z","shell.execute_reply":"2023-10-22T01:03:13.753256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Scatterplot","metadata":{"_uuid":"0f9de57e-5e2c-4b2e-b02e-1a2cc562c71a","_cell_guid":"bac5ee43-c6ec-4283-888c-adc78aaa9216","trusted":true}},{"cell_type":"markdown","source":"Using scatterplot can also help us to identify potential linear relationship between Numerical features. Although scatterplot does not provide quantitative evidence on the strength of linear relationship between our features, it is useful in helping us to visualize any sort of relationship that correlation matrix could not calculate. E.g Quadratic, Exponential relationships.","metadata":{"_uuid":"48645710-4a11-4608-afc8-3ab5a994f1d8","_cell_guid":"62ee95c5-d329-4183-aa31-edb6ac26702c","trusted":true}},{"cell_type":"code","source":"# fig = plt.figure(figsize=(16,35))\n# for index in range(len(train.columns)):\n#     plt.subplot(10,4,index+1)\n#     sns.scatterplot(x=train.iloc[:,index], y=TARGET, data=train.dropna())\n# fig.tight_layout(pad=2.0)","metadata":{"_uuid":"d5c03c7f-3996-4cfc-82bc-cc5bdd801d31","_cell_guid":"488c605c-72b2-4b81-b468-7e1c0c38bac1","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:13.756420Z","iopub.execute_input":"2023-10-22T01:03:13.756898Z","iopub.status.idle":"2023-10-22T01:03:13.767861Z","shell.execute_reply.started":"2023-10-22T01:03:13.756858Z","shell.execute_reply":"2023-10-22T01:03:13.766898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Mutual information","metadata":{"_uuid":"9c1fd29f-7207-4451-adfc-02f99658ddd3","_cell_guid":"2943d41b-115b-4372-abe8-5bf0670a8a2e","trusted":true}},{"cell_type":"markdown","source":"Mutual information describes relationships in terms of uncertainty. The mutual information between two quantities is a measure of the extent to which knowledge of one quantity reduces uncertainty about the other. If you knew the value of a feature, how much more confident would you be about the target?\n\nMutual information is a lot like correlation in that it measures a relationship between two quantities. The advantage of mutual information is that it can detect any kind of relationship, while correlation only detects linear relationships. Mutual information can capture any kind of association (not just linear, like correlation.)\n\nThe least possible mutual information between quantities is 0.0. When MI is zero, the quantities are independent: neither can tell you anything about the other. Conversely, in theory there's no upper bound to what MI can be. In practice though values above 2.0 or so are uncommon. (Mutual information is a logarithmic quantity, so it increases very slowly.)","metadata":{"_uuid":"45f3af43-3e52-4270-b0d4-a5b8d7c21519","_cell_guid":"65aabb28-8984-44b1-b27e-f83f439f2858","trusted":true}},{"cell_type":"markdown","source":"Here are some things to remember when applying mutual information:\n\n* MI can help you to understand the relative potential of a feature as a predictor of the target, considered by itself.\n* It's possible for a feature to be very informative when interacting with other features, but not so informative all alone. MI can't detect interactions between features. It is a univariate metric.\n* The actual usefulness of a feature depends on the model you use it with. A feature is only useful to the extent that its relationship with the target is one your model can learn. Just because a feature has a high MI score doesn't mean your model will be able to do anything with that information. You may need to transform the feature first to expose the association.","metadata":{"_uuid":"c452a4aa-9a84-4528-abd0-6f53c92d5440","_cell_guid":"3fa5be11-a036-4810-b2cb-23dd60d10130","trusted":true}},{"cell_type":"code","source":"def make_mi_scores(X, y):\n    X = X.copy()\n    for colname in X.select_dtypes([\"object\", \"category\"]):\n        X[colname], _ = X[colname].factorize()\n    # All discrete features should now have integer dtypes\n    discrete_features = [pd.api.types.is_float_dtype(t) for t in X.dtypes]\n    mi_scores = mutual_info_regression(X, y, discrete_features=discrete_features, random_state=0)\n    mi_scores = pd.Series(mi_scores, name=\"MI Scores\", index=X.columns)\n    mi_scores = mi_scores.sort_values(ascending=False)\n    return mi_scores\n\ndef plot_mi_scores(scores):\n    scores = scores.sort_values(ascending=True)\n    width = np.arange(len(scores))\n    ticks = list(scores.index)\n    plt.barh(width, scores)\n    plt.yticks(width, ticks)\n    plt.title(\"Mutual Information Scores\")","metadata":{"_uuid":"9097395b-9d89-4a1d-82d3-06826c407344","_cell_guid":"0f2b00a9-f60e-4d11-bbe7-e85a398b0afb","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:13.769367Z","iopub.execute_input":"2023-10-22T01:03:13.769649Z","iopub.status.idle":"2023-10-22T01:03:13.781197Z","shell.execute_reply.started":"2023-10-22T01:03:13.769623Z","shell.execute_reply":"2023-10-22T01:03:13.780306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"_uuid":"f3b25abc-62d8-465d-947e-baba9b409754","_cell_guid":"ff6be023-b3be-473b-bb19-ea54d3251a1f","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:13.782755Z","iopub.execute_input":"2023-10-22T01:03:13.783475Z","iopub.status.idle":"2023-10-22T01:03:13.795494Z","shell.execute_reply.started":"2023-10-22T01:03:13.783436Z","shell.execute_reply":"2023-10-22T01:03:13.794412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X_raw = train.fillna(method='ffill')\n# features = ['', '', '']\n# sns.relplot(\n#     x=\"value\", y=TARGET, col=\"variable\", data=X_raw.melt(id_vars=TARGET, value_vars=features), facet_kws=dict(sharex=False),\n# );","metadata":{"_uuid":"8c6440ec-ebba-4ad5-8e5d-c323971f659b","_cell_guid":"70174446-f886-40aa-ac8b-8b774854fadf","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:13.796854Z","iopub.execute_input":"2023-10-22T01:03:13.797167Z","iopub.status.idle":"2023-10-22T01:03:13.806121Z","shell.execute_reply.started":"2023-10-22T01:03:13.797139Z","shell.execute_reply":"2023-10-22T01:03:13.805102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Based on the plots, which feature do you think would have the highest mutual information with *TARGET*?\nThe highest mutual information with *TARGET* tends to constrain it to a smaller range of possible values.\n\nThe dataset may have a lot of features to work with all at once! Fortunately, you can identify the features with the most potential.\nWe can compute mutual information scores for the dataset features.","metadata":{"_uuid":"510a871c-81b2-47cd-a8d9-550fa5c3a5eb","_cell_guid":"ba9aa69f-55c1-4974-88ea-7e65e5cc30af","trusted":true}},{"cell_type":"code","source":"# X_mi = X_raw.copy()\n# y_mi = X_mi.pop(TARGET)","metadata":{"_uuid":"8a61b4c4-e635-4405-905d-116118edcde2","_cell_guid":"4188f1c9-728d-469d-bd47-57663b8d19a7","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:13.807512Z","iopub.execute_input":"2023-10-22T01:03:13.808703Z","iopub.status.idle":"2023-10-22T01:03:13.817950Z","shell.execute_reply.started":"2023-10-22T01:03:13.808663Z","shell.execute_reply":"2023-10-22T01:03:13.816710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# [X_mi.shape, y_mi.shape]","metadata":{"_uuid":"f8a6e6ce-0c8b-40ba-bfa9-8480e3be395a","_cell_guid":"491e2387-2fda-4211-8b08-c6d396a39c71","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:13.819551Z","iopub.execute_input":"2023-10-22T01:03:13.820601Z","iopub.status.idle":"2023-10-22T01:03:13.828971Z","shell.execute_reply.started":"2023-10-22T01:03:13.820559Z","shell.execute_reply":"2023-10-22T01:03:13.828110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# mi_scores = make_mi_scores(X_mi, y_mi)","metadata":{"_uuid":"8249ebbe-31f7-42e8-8159-2fcaaa507bd5","_cell_guid":"5192aae0-543a-40b2-b661-d7c9f362eb6d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:13.830439Z","iopub.execute_input":"2023-10-22T01:03:13.831365Z","iopub.status.idle":"2023-10-22T01:03:13.841517Z","shell.execute_reply.started":"2023-10-22T01:03:13.831326Z","shell.execute_reply":"2023-10-22T01:03:13.840224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(mi_scores.head(20))\n# #print(mi_scores.tail(20))  # uncomment to see bottom 20\n\n# plt.figure(dpi=100, figsize=(8, 5))\n# plot_mi_scores(mi_scores.head(20))\n# #plot_mi_scores(mi_scores.tail(20))  # uncomment to see bottom 20","metadata":{"_uuid":"ce643e15-4b9e-492f-9831-2708e89adb5e","_cell_guid":"99de4490-ee27-4530-bf5f-303218b0cc90","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:13.845575Z","iopub.execute_input":"2023-10-22T01:03:13.846590Z","iopub.status.idle":"2023-10-22T01:03:13.854077Z","shell.execute_reply.started":"2023-10-22T01:03:13.846545Z","shell.execute_reply":"2023-10-22T01:03:13.852552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Do the scores seem reasonable? Do the high scoring features represent things you'd think most people would value in a home? Do you notice any themes in what they describe?\n\nIt's good that our mutual information metric scored features highly. On the other hand, the lowest ranked features seem to mostly represent things that are rare or exceptional in some way, and so wouldn't be relevant.","metadata":{"_uuid":"cb189914-cf03-4908-8bb0-1245f37e180a","_cell_guid":"d96e5b31-51d2-45f4-91da-0ecdbb173797","trusted":true}},{"cell_type":"code","source":"# sns.catplot(x=\" \", y=TARGET, data=train, kind=\"boxen\");","metadata":{"_uuid":"26a45524-d734-49c5-8eda-a378e0070f03","_cell_guid":"37d3a69f-9dc2-4416-82e0-21db69360083","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:13.855522Z","iopub.execute_input":"2023-10-22T01:03:13.856334Z","iopub.status.idle":"2023-10-22T01:03:13.869251Z","shell.execute_reply.started":"2023-10-22T01:03:13.856292Z","shell.execute_reply":"2023-10-22T01:03:13.868094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# feature = \" \"\n\n# sns.lmplot(\n#     x=feature, y=TARGET, hue=\"O2_1\", col=\"O2_1\",\n#     data=train, scatter_kws={\"edgecolor\": 'w'}, col_wrap=3, height=4,\n# );","metadata":{"_uuid":"5e2526e2-5e98-479c-a6ec-450c7f31c2cc","_cell_guid":"fe0185c5-9578-4c24-97ac-6da88b637f42","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:13.870728Z","iopub.execute_input":"2023-10-22T01:03:13.871381Z","iopub.status.idle":"2023-10-22T01:03:13.880644Z","shell.execute_reply.started":"2023-10-22T01:03:13.871348Z","shell.execute_reply":"2023-10-22T01:03:13.879642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Check the distribution of target variable","metadata":{}},{"cell_type":"code","source":"# check the distribution of the target variable\ntrain[TARGET].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-10-22T01:03:13.882076Z","iopub.execute_input":"2023-10-22T01:03:13.882496Z","iopub.status.idle":"2023-10-22T01:03:14.055448Z","shell.execute_reply.started":"2023-10-22T01:03:13.882465Z","shell.execute_reply":"2023-10-22T01:03:14.054317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train","metadata":{"execution":{"iopub.status.busy":"2023-10-22T01:03:14.067522Z","iopub.execute_input":"2023-10-22T01:03:14.068206Z","iopub.status.idle":"2023-10-22T01:03:14.073169Z","shell.execute_reply.started":"2023-10-22T01:03:14.068162Z","shell.execute_reply":"2023-10-22T01:03:14.071981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train['timestamp'] = pd.to_datetime(train['timestamp']).apply(lambda t: t.tz_localize(None))\n#train['timestamp'] = pd.to_datetime(train[\"timestamp\"],utc=True)","metadata":{"execution":{"iopub.status.busy":"2023-10-22T01:03:14.074144Z","iopub.execute_input":"2023-10-22T01:03:14.074598Z","iopub.status.idle":"2023-10-22T01:03:14.087540Z","shell.execute_reply.started":"2023-10-22T01:03:14.074566Z","shell.execute_reply":"2023-10-22T01:03:14.086341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Processing","metadata":{"_uuid":"7b6386b0-7f53-43cb-9c20-9fb38fb0cba6","_cell_guid":"eed7e06c-6cde-480c-85b0-48065de8661f","trusted":true}},{"cell_type":"markdown","source":"## Fine-Tune model","metadata":{"_uuid":"6bdd099e-be92-4df7-9be5-52d2164b32b9","_cell_guid":"5cd6bc9b-1445-4d64-9fcd-8275e290c332","trusted":true}},{"cell_type":"code","source":"# X = train\n# y = train[TARGET]","metadata":{"execution":{"iopub.status.busy":"2023-10-22T01:03:14.088539Z","iopub.execute_input":"2023-10-22T01:03:14.088851Z","iopub.status.idle":"2023-10-22T01:03:14.100057Z","shell.execute_reply.started":"2023-10-22T01:03:14.088823Z","shell.execute_reply":"2023-10-22T01:03:14.098840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X.shape","metadata":{"_uuid":"d1c2cd82-754b-4ac4-b487-81af8e5ef336","_cell_guid":"4f850632-3973-4c50-be10-2b42e3c8471d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.101224Z","iopub.execute_input":"2023-10-22T01:03:14.101558Z","iopub.status.idle":"2023-10-22T01:03:14.114671Z","shell.execute_reply.started":"2023-10-22T01:03:14.101528Z","shell.execute_reply":"2023-10-22T01:03:14.113365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # just for baseline\n# features_num = X.select_dtypes(exclude=['object']).copy().columns\n# features_cat = X.select_dtypes(include=['object']).copy().columns\n\n# transformer_num = make_pipeline(\n#     SimpleImputer(strategy=\"constant\"), # there are a few missing values\n#     #StandardScaler(),\n#     RobustScaler()\n# )\n# transformer_cat = make_pipeline(\n#     SimpleImputer(strategy=\"constant\", fill_value=\"NA\"),\n#     OneHotEncoder(handle_unknown='ignore'),\n# )\n\n# preprocessor = make_column_transformer(\n#     (transformer_num, features_num),\n#     (transformer_cat, features_cat),\n# )","metadata":{"execution":{"iopub.status.busy":"2023-10-22T01:03:14.115959Z","iopub.execute_input":"2023-10-22T01:03:14.116320Z","iopub.status.idle":"2023-10-22T01:03:14.131064Z","shell.execute_reply.started":"2023-10-22T01:03:14.116284Z","shell.execute_reply":"2023-10-22T01:03:14.129732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=0.2)","metadata":{"_uuid":"efb4f0f1-84b2-485b-9639-3cc6cd20f317","_cell_guid":"43953788-a421-4cc0-892e-eef92d8f17a0","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.132749Z","iopub.execute_input":"2023-10-22T01:03:14.133086Z","iopub.status.idle":"2023-10-22T01:03:14.145699Z","shell.execute_reply.started":"2023-10-22T01:03:14.133056Z","shell.execute_reply":"2023-10-22T01:03:14.144505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### XGBoost","metadata":{}},{"cell_type":"code","source":"def XGBModel(X_train, y_train, preprocessor):    \n    \n#     param_lst_xgb = {\n#         'learning_rate' : [0.01, 0.03, 0.1, 0.15, 0.3, 0.5],\n#         'n_estimators' : [2, 10, 100],#500, 1000, 2000, 3000, 4000, 7000, 10000, 15000],\n#         'max_depth' : [3, 6, 9],\n#         'min_child_weight' : [1, 5, 10, 20],\n#         'reg_alpha' : [0.001, 0.01, 0.1],\n#         'reg_lambda' : [0.001, 0.01, 0.1]\n#     }\n    \n#     xgb_dflt = XGBClassifier()\n\n#     xgb_reg = RandomizedSearchCV(\n#                                     estimator = xgb_dflt, \n#                                     param_distributions = param_lst_xgb,\n#                                     n_iter = 2, \n#                                     scoring = 'accuracy',\n#                                     n_jobs = 100,\n#                                     cv = 2\n#                                 )\n\n    # Bundle preprocessing and modeling code in a pipeline\n#     xgb_pipeline = Pipeline(steps=[\n#                                     ('preprocessor', preprocessor),\n#                                     ('model_xgb_RandomizedSearchCV', xgb_reg)\n#                                  ])\n\n#     xgb_search = xgb_pipeline.fit(X_train, y_train)\n\n    # XGB with tune hyperparameters\n#     xgb = XGBClassifier(**xgb_search.named_steps['model_xgb_RandomizedSearchCV'].best_params_)\n\n    xgb = XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n       colsample_bynode=1, colsample_bytree=1, gamma=0, learning_rate=0.01,\n       max_delta_step=0, max_depth=9, min_child_weight=1, missing=1,\n       n_estimators=100, n_jobs=100, nthread=None,\n       objective='multi:softprob', random_state=0, reg_alpha=0.01,\n       reg_lambda=0.01, scale_pos_weight=1, seed=None, silent=None,\n       subsample=1, verbosity=1)\n    \n    # Bundle preprocessing and modeling code in a pipeline\n    xgb_pipeline_best_params = Pipeline(steps=[\n#                                     ('preprocessor', preprocessor),\n                                    ('model', xgb)\n                                 ])\n\n    xgb_pipeline_best_params.fit(X_train, y_train)\n\n#     print(\"best_params_ \", xgb_search.named_steps['model_xgb_RandomizedSearchCV'].best_params_)\n#     print(\"best_score_ \", xgb_search.named_steps['model_xgb_RandomizedSearchCV'].best_score_)\n#     print(\"best_estimator_ \", xgb_search.named_steps['model_xgb_RandomizedSearchCV'].best_estimator_)\n#     print(\"cv_results_ \", xgb_search.named_steps['model_xgb_RandomizedSearchCV'].cv_results_)\n#     print(\"mean_test_score \", xgb_search.named_steps['model_xgb_RandomizedSearchCV'].cv_results_['mean_test_score'])\n\n    return xgb_pipeline_best_params\n\n#xgb_pipeline = XGBModel(X_train, y_train, preprocessor)","metadata":{"execution":{"iopub.status.busy":"2023-10-22T01:03:14.147466Z","iopub.execute_input":"2023-10-22T01:03:14.147903Z","iopub.status.idle":"2023-10-22T01:03:14.160793Z","shell.execute_reply.started":"2023-10-22T01:03:14.147859Z","shell.execute_reply":"2023-10-22T01:03:14.159661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### LightGBM","metadata":{}},{"cell_type":"code","source":"def LGBMModel(X_train, y_train, preprocessor):\n    \n    lgbm_dflt = LGBMClassifier()\n\n    param_lst_lgbm = {\n        'max_depth' : [2, 5, 8, 10],\n        'learning_rate' : [0.001, 0.01, 0.03, 0.1, 0.2],\n        'n_estimators' : [10],#, 300, 500, 1000, 1500, 4000, 7000, 10000, 15000],\n        'lambda_l1' : [0.0001, 0.001, 0.01],\n        'lambda_l2' : [0, 0.0001, 0.001, 0.01],\n        'feature_fraction' : [0.4, 0.6, 0.8],\n        'min_child_samples' : [5, 10, 20, 25]\n    }\n\n    lightgbm = RandomizedSearchCV(estimator = lgbm_dflt, \n                                  param_distributions = param_lst_lgbm,\n                                  n_iter = 3, \n                                  scoring = 'accuracy',\n                                  n_jobs = 20,\n                                  cv = 5)\n\n    lgbm_pipeline = Pipeline(steps=[\n                                    ('preprocessor', preprocessor),\n                                    ('model_lightgbm_RandomizedSearchCV', lightgbm)\n                                 ])\n\n    lightgbm_search = lgbm_pipeline.fit(X_train, y_train)\n\n    #LightBGM with tuned hyperparameters\n    lgbm = LGBMClassifier(**lightgbm_search.named_steps['model_lightgbm_RandomizedSearchCV'].best_params_, verbose_eval=False)\n\n    lgbm_pipeline_best_params = Pipeline(steps=[\n                                    ('preprocessor', preprocessor),\n                                    ('model', lgbm)\n                                 ])\n\n    lgbm_pipeline_best_params.fit(X_train, y_train)\n    print(lightgbm_search.named_steps['model_lightgbm_RandomizedSearchCV'].best_score_)\n    return lgbm_pipeline_best_params\n\n#lgbm_pipeline = LGBMModel(X_train, y_train, preprocessor)","metadata":{"execution":{"iopub.status.busy":"2023-10-22T01:03:14.162585Z","iopub.execute_input":"2023-10-22T01:03:14.162894Z","iopub.status.idle":"2023-10-22T01:03:14.179007Z","shell.execute_reply.started":"2023-10-22T01:03:14.162867Z","shell.execute_reply":"2023-10-22T01:03:14.177923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### CatBoost","metadata":{}},{"cell_type":"code","source":"def CatBoostModel(X_train, y_train, preprocessor):\n    \n    cb_dflt = CatBoostClassifier(logging_level='Silent')\n\n    param_lst_cb = {\n        'n_estimators' : [10],#0, 300, 500, 1000, 1300, 1600, 4000, 7000, 10000, 15000],\n        'learning_rate' : [0.0001, 0.001, 0.01, 0.03, 0.1],\n        'l2_leaf_reg' : [0.001, 0.01, 0.1],\n        'random_strength' : [0.25, 0.5 ,1],\n        'max_depth' : [3, 6, 9],\n        'min_child_samples' : [2, 5, 10, 15, 20],\n        'rsm' : [0.5, 0.7, 0.9]\n    }\n\n    catboost = RandomizedSearchCV(estimator = cb_dflt, \n                                  param_distributions = param_lst_cb,\n                                  n_iter = 3, \n                                  scoring = 'accuracy', \n                                  n_jobs = 20,\n                                  cv = 5)\n\n    cb_pipeline = Pipeline(steps=[\n                                    ('preprocessor', preprocessor),\n                                    ('model_catboost_RandomizedSearchCV', catboost)\n                                 ])\n\n    catboost_search = cb_pipeline.fit(X_train, y_train)\n\n    # CatBoost with tuned hyperparams\n    cb = CatBoostClassifier(**catboost_search.named_steps['model_catboost_RandomizedSearchCV'].best_params_, logging_level='Silent')\n\n    # # Bundle preprocessing and modeling code in a pipeline\n    cb_pipeline_best_params = Pipeline(steps=[\n                                    ('preprocessor', preprocessor),\n                                    ('model', cb)\n                                 ])\n\n    cb_pipeline_best_params.fit(X_train, y_train)\n\n    print(catboost_search.named_steps['model_catboost_RandomizedSearchCV'].best_score_)\n    return cb_pipeline_best_params\n\n#cb_pipeline = CatBoostModel(X_train, y_train, preprocessor)","metadata":{"execution":{"iopub.status.busy":"2023-10-22T01:03:14.180794Z","iopub.execute_input":"2023-10-22T01:03:14.181552Z","iopub.status.idle":"2023-10-22T01:03:14.200245Z","shell.execute_reply.started":"2023-10-22T01:03:14.181520Z","shell.execute_reply":"2023-10-22T01:03:14.199361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Validate best pipe with valid data","metadata":{}},{"cell_type":"code","source":"def validate_best_pipe(X_valid, y_valid, xgb_pipeline, lgbm_pipeline, cb_pipeline):    \n    \n    #xgb_pipeline\n    if xgb_pipeline is not None:\n        preds = xgb_pipeline.predict(X_valid)\n\n    #     score_xgb_mae = mean_absolute_error(y_valid, preds)\n    #     rmse_xgb = np.sqrt(mean_squared_error(y_valid, preds))\n        score_xgb = xgb_pipeline.score(X_valid, y_valid)\n        #cv_xgb = mean_cross_val(xgb_pipeline, x, y)\n    \n        print('---xgb---')\n    #     print('MAE:', score_xgb_mae)\n        print('Score:', score_xgb)\n    #     print('RMSE:', rmse_xgb)\n        #print('CV:', cv_xgb)\n    \n    #lgbm_pipeline\n    if lgbm_pipeline is not None:\n        preds = lgbm_pipeline.predict(X_valid)\n\n    #     score_lgbm_mae = mean_absolute_error(y_valid, preds)\n    #     rmse_lgbm = np.sqrt(mean_squared_error(y_valid, preds))\n        score_lgbm = lgbm_pipeline.score(X_valid, y_valid)\n        #cv_lgbm = mean_cross_val(lgbm_pipeline, x, y)\n\n        print('---lgbm---')\n    #     print('MAE:', score_lgbm_mae)\n        print('Score:', score_lgbm)\n    #     print('RMSE:', rmse_xgb)\n        #print('CV:', cv_lgbm)\n    \n    #cb_pipeline\n    if cb_pipeline is not None:\n        preds = cb_pipeline.predict(X_valid)\n\n        score_cb_mae = mean_absolute_error(y_valid, preds)\n        rmse_cb = np.sqrt(mean_squared_error(y_valid, preds))\n        score_cb = cb_pipeline.score(X_valid, y_valid)\n        #cv_cb = mean_cross_val(cb_pipeline, x, y)\n\n        print('---cb_pipeline---')\n    #     print('MAE:', score_cb_mae)\n        print('Score:', score_cb)\n    #     print('RMSE:', rmse_cb)\n        #print('CV:', cv_cb)","metadata":{"execution":{"iopub.status.busy":"2023-10-22T01:03:14.202059Z","iopub.execute_input":"2023-10-22T01:03:14.202645Z","iopub.status.idle":"2023-10-22T01:03:14.220883Z","shell.execute_reply.started":"2023-10-22T01:03:14.202602Z","shell.execute_reply":"2023-10-22T01:03:14.219739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## BaseLine ","metadata":{}},{"cell_type":"code","source":"# X_train,X_valid,y_train,y_valid = train_test_split(X,y,test_size=0.2)\n\n# def score_competition(X_valid, y_valid, xgb_pipeline, lgbm_pipeline, cb_pipeline):\n#     validate_best_pipe(X_valid, y_valid, xgb_pipeline, lgbm_pipeline, cb_pipeline)\n\n# score_competition(X_valid, y_valid, xgb_pipeline, lgbm_pipeline, cb_pipeline)","metadata":{"execution":{"iopub.status.busy":"2023-10-22T01:03:14.222183Z","iopub.execute_input":"2023-10-22T01:03:14.222538Z","iopub.status.idle":"2023-10-22T01:03:14.241880Z","shell.execute_reply.started":"2023-10-22T01:03:14.222508Z","shell.execute_reply":"2023-10-22T01:03:14.240446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X.shape","metadata":{"execution":{"iopub.status.busy":"2023-10-22T01:03:14.243467Z","iopub.execute_input":"2023-10-22T01:03:14.244571Z","iopub.status.idle":"2023-10-22T01:03:14.255390Z","shell.execute_reply.started":"2023-10-22T01:03:14.244536Z","shell.execute_reply":"2023-10-22T01:03:14.254412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Redundant features","metadata":{"_uuid":"972f1745-19c5-4964-a4bd-98b1bf0b533a","_cell_guid":"7424f886-3ccf-4de5-a7de-ed7c2f338498","trusted":true}},{"cell_type":"markdown","source":"Now that we have more or less finished analysing our data and gaining insights through the various analysis and visualization, we will have to leverage on these insights to guide our preprocessing decision, so as to provide a clean and error-free data for our model to train on later on.","metadata":{"_uuid":"d11c4809-2727-4108-a8ef-3aa89a1ffdc4","_cell_guid":"32e508b3-4722-4ebd-b0c0-0ee206b409cc","trusted":true}},{"cell_type":"markdown","source":"### Multicollinearity - highly correlated features","metadata":{"_uuid":"6b8c6a8b-0b6b-4958-9e33-d1482637dda4","_cell_guid":"e0d3537b-7463-4167-a393-f00973d76a1e","trusted":true}},{"cell_type":"markdown","source":"From the above correlation matrix, we can pinpointed certain features that are highly correlated that we will remove to avoid the problem of multicollinearity.","metadata":{"_uuid":"0e54af01-9955-4660-9e53-5fe618ed0784","_cell_guid":"3114e6d0-3071-437a-ad77-bdd94fc74943","trusted":true}},{"cell_type":"code","source":"# X.drop(['Features 1','Features 2','Features 3','Features 4'], axis=1, inplace=True)","metadata":{"_uuid":"cb923d2b-b45d-47d4-aef6-c72932245d34","_cell_guid":"74985145-eebd-4b97-9e62-9ab31aaa4cd0","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.258089Z","iopub.execute_input":"2023-10-22T01:03:14.259981Z","iopub.status.idle":"2023-10-22T01:03:14.270295Z","shell.execute_reply.started":"2023-10-22T01:03:14.259943Z","shell.execute_reply":"2023-10-22T01:03:14.269378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Features with alot of missing values","metadata":{"_uuid":"703e2fa9-04e6-40fa-bbe0-9d678e9c54bf","_cell_guid":"e0b43f1a-136f-42b1-9194-42838d3f2389","trusted":true}},{"cell_type":"markdown","source":"Apart from these highly correlated features, we also can remove features that is not very useful in prediction due to many missing values that can provide any useful information.","metadata":{"_uuid":"b1822e44-23a1-4eea-bbef-ce2e82925f26","_cell_guid":"51a5ce3a-21ee-41b5-92b0-595d215c3501","trusted":true}},{"cell_type":"code","source":"# plt.figure(figsize=(25,8))\n# plt.title('Number of missing rows')\n# missing_count = pd.DataFrame(X.isnull().sum(), columns=['sum']).sort_values(by=['sum'],ascending=False).head(20).reset_index()\n# missing_count.columns = ['features','sum']\n# sns.barplot(x='features',y='sum', data = missing_count)","metadata":{"_uuid":"b4512f72-4165-4726-98e1-ca2146e3e200","_cell_guid":"6067f93f-3a83-424e-a51d-60227cae610f","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.271744Z","iopub.execute_input":"2023-10-22T01:03:14.272692Z","iopub.status.idle":"2023-10-22T01:03:14.289421Z","shell.execute_reply.started":"2023-10-22T01:03:14.272657Z","shell.execute_reply":"2023-10-22T01:03:14.288123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X.drop(['','',''], axis=1, inplace=True)","metadata":{"_uuid":"21998514-b4d9-4c8a-9291-e87acb418e4c","_cell_guid":"97642a1c-5f10-41ef-b759-a4679422821d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.290820Z","iopub.execute_input":"2023-10-22T01:03:14.291802Z","iopub.status.idle":"2023-10-22T01:03:14.302674Z","shell.execute_reply.started":"2023-10-22T01:03:14.291764Z","shell.execute_reply":"2023-10-22T01:03:14.301551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# [X.shape, y.shape]","metadata":{"_uuid":"d6e74e00-e59a-4491-8f0a-e4e64c650c85","_cell_guid":"e817a0c3-8458-4c6e-8615-c3cbeca222e3","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.304285Z","iopub.execute_input":"2023-10-22T01:03:14.304630Z","iopub.status.idle":"2023-10-22T01:03:14.318111Z","shell.execute_reply.started":"2023-10-22T01:03:14.304588Z","shell.execute_reply":"2023-10-22T01:03:14.317307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Useless features","metadata":{"_uuid":"f4aeed2d-9d57-4c67-816f-c6af73824656","_cell_guid":"bc93f774-2772-4b9e-a8bd-980df035c9de","trusted":true}},{"cell_type":"markdown","source":"We can also remove features that does not have any linear relationship with target identified on the Correlation analysis.","metadata":{"_uuid":"c39338cc-89b0-4dcb-9565-6d9ede9004ab","_cell_guid":"055d0b0f-0901-4b18-8daf-76850e686704","trusted":true}},{"cell_type":"code","source":"# fig,axes = plt.subplots(1,2, figsize=(15,5))\n# sns.regplot(x=num_col_TARGET['NH4_5'], y=TARGET, data=num_col_TARGET, ax = axes[0], line_kws={'color':'black'})\n# sns.regplot(x=num_col_TARGET['NO2_1'], y=TARGET, data=num_col_TARGET, ax = axes[1], line_kws={'color':'black'})\n# fig.tight_layout(pad=2.0)","metadata":{"_uuid":"599cc0ac-1894-48e8-b3f4-9c3e7fa95646","_cell_guid":"30308da1-dd2e-4f7e-8a22-84504b4b3059","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.319699Z","iopub.execute_input":"2023-10-22T01:03:14.320114Z","iopub.status.idle":"2023-10-22T01:03:14.329441Z","shell.execute_reply.started":"2023-10-22T01:03:14.320075Z","shell.execute_reply":"2023-10-22T01:03:14.328599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# correlation[[TARGET]].sort_values([TARGET], ascending=False).tail(10)","metadata":{"_uuid":"dc211013-4ce2-42ad-8647-35f7aa0e7284","_cell_guid":"19bb60d9-c1ce-446a-8a78-fdc32ce89c3a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.330473Z","iopub.execute_input":"2023-10-22T01:03:14.330873Z","iopub.status.idle":"2023-10-22T01:03:14.341095Z","shell.execute_reply.started":"2023-10-22T01:03:14.330835Z","shell.execute_reply":"2023-10-22T01:03:14.340054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X.drop(['',''], axis=1, inplace=True)","metadata":{"_uuid":"3aa2f2dc-9255-4c44-9e88-22003d289556","_cell_guid":"7a9d8e40-8c35-4378-a46e-f60cbd7488a0","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.342712Z","iopub.execute_input":"2023-10-22T01:03:14.343243Z","iopub.status.idle":"2023-10-22T01:03:14.355621Z","shell.execute_reply.started":"2023-10-22T01:03:14.343199Z","shell.execute_reply":"2023-10-22T01:03:14.354176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X.shape","metadata":{"_uuid":"525ff29e-a48d-476e-99d4-37983f8d8e29","_cell_guid":"3f474a37-95d3-44c9-85a2-bf3159173ce7","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.357365Z","iopub.execute_input":"2023-10-22T01:03:14.357877Z","iopub.status.idle":"2023-10-22T01:03:14.368482Z","shell.execute_reply.started":"2023-10-22T01:03:14.357845Z","shell.execute_reply":"2023-10-22T01:03:14.367146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Removing features that have mostly just 1 value","metadata":{"_uuid":"d97dc2cb-357d-4dc1-a2ac-7a868391d765","_cell_guid":"38093105-ea67-4b3c-9696-6f2d8dc9bdfc","trusted":true}},{"cell_type":"markdown","source":"Earlier during our Univariate analysis, if we found that some features mostly consist of just a single value or 0s, which is not useful to us, we set a user defined threshold at 96% where if a column has more than 96% of the same value, we will render the features to be useless and remove it, since there isnt much information we can extract from","metadata":{"_uuid":"03ae218d-34df-4e92-8c0f-afa02be2fee1","_cell_guid":"927f71a5-818e-4695-8a9e-4f92bfa2f907","trusted":true}},{"cell_type":"code","source":"# cat_col = X.select_dtypes(include=['object']).columns\n# overfit_cat = []\n# for i in cat_col:\n#     counts = X[i].value_counts()\n#     zeros = counts.iloc[0]\n#     if zeros / len(X) * 100 > 96:\n#         overfit_cat.append(i)\n\n# overfit_cat = list(overfit_cat)\n# print(overfit_cat)","metadata":{"_uuid":"ff1e8846-ff4f-469d-8fa3-84ca3c75224a","_cell_guid":"a901fd3d-724e-488c-94c9-f074042a4e67","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.370402Z","iopub.execute_input":"2023-10-22T01:03:14.370748Z","iopub.status.idle":"2023-10-22T01:03:14.380844Z","shell.execute_reply.started":"2023-10-22T01:03:14.370717Z","shell.execute_reply":"2023-10-22T01:03:14.379752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X = X.drop(overfit_cat, axis=1)","metadata":{"_uuid":"5def79d6-dc03-44fa-89b1-110382611500","_cell_guid":"9291f4d7-0198-4fc3-a33b-6ec42cf9f689","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.382295Z","iopub.execute_input":"2023-10-22T01:03:14.383178Z","iopub.status.idle":"2023-10-22T01:03:14.394135Z","shell.execute_reply.started":"2023-10-22T01:03:14.383147Z","shell.execute_reply":"2023-10-22T01:03:14.393307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# num_col = X.select_dtypes(exclude=['object']).copy()\n# overfit_num = []\n# for i in num_col:\n#     counts = X[i].value_counts()\n#     zeros = counts.iloc[0]\n#     if zeros / len(X) * 100 > 96:\n#         overfit_num.append(i)\n\n# overfit_num = list(overfit_num)\n# print(overfit_num)","metadata":{"_uuid":"89c8b499-5ffa-4892-81bc-6997eaa7251a","_cell_guid":"939c5f45-a2d6-4986-acb2-10b613be773f","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.395718Z","iopub.execute_input":"2023-10-22T01:03:14.396019Z","iopub.status.idle":"2023-10-22T01:03:14.406430Z","shell.execute_reply.started":"2023-10-22T01:03:14.395993Z","shell.execute_reply":"2023-10-22T01:03:14.405363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X = X.drop(overfit_num, axis=1)","metadata":{"_uuid":"fb0a38e3-0fb0-4351-bd34-ff435efcb95f","_cell_guid":"758de55d-eed2-4081-8a00-e9c5488886ce","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.408148Z","iopub.execute_input":"2023-10-22T01:03:14.408478Z","iopub.status.idle":"2023-10-22T01:03:14.418140Z","shell.execute_reply.started":"2023-10-22T01:03:14.408449Z","shell.execute_reply":"2023-10-22T01:03:14.417350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(\"Categorical Features with >96% of the same value: \",overfit_cat)\n# print(\"Numerical Features with >96% of the same value: \",overfit_num)","metadata":{"_uuid":"afba3d12-59f1-41a6-91c0-b6a5374a5318","_cell_guid":"0eb5a55c-9e04-4814-8805-c25a8aa27322","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.419150Z","iopub.execute_input":"2023-10-22T01:03:14.419481Z","iopub.status.idle":"2023-10-22T01:03:14.430818Z","shell.execute_reply.started":"2023-10-22T01:03:14.419454Z","shell.execute_reply":"2023-10-22T01:03:14.429686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Dealing with duplicated values","metadata":{"_uuid":"d480d41f-d58c-4873-a6f7-e36ddd0dbee9","_cell_guid":"96dd4df0-f614-4003-a72b-e75123838b19","trusted":true}},{"cell_type":"code","source":"# print('Number of duplicated values in dataset: ', X.duplicated().sum())","metadata":{"_uuid":"627f60e7-c8af-41a4-9f12-1f1e95fb491d","_cell_guid":"1e60129e-c10e-4361-9c63-3ac0b970ef7c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.432224Z","iopub.execute_input":"2023-10-22T01:03:14.432588Z","iopub.status.idle":"2023-10-22T01:03:14.443737Z","shell.execute_reply.started":"2023-10-22T01:03:14.432558Z","shell.execute_reply":"2023-10-22T01:03:14.442814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#X.drop_duplicates(inplace=True)","metadata":{"_uuid":"aba0e05a-e2e4-451b-86e0-9389363a1c31","_cell_guid":"c2a7c2af-3cb8-4c42-998d-d259d20adf2d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.444739Z","iopub.execute_input":"2023-10-22T01:03:14.445068Z","iopub.status.idle":"2023-10-22T01:03:14.455743Z","shell.execute_reply.started":"2023-10-22T01:03:14.445038Z","shell.execute_reply":"2023-10-22T01:03:14.454535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dealing with outliers","metadata":{"_uuid":"5d990aae-96b5-41f0-8b53-46939843097e","_cell_guid":"744e76d0-abca-457b-a266-ef200b5c2089","trusted":true}},{"cell_type":"markdown","source":"Removing outliers will prevent our models performance from being affected by extreme values. From our boxplot earlier, we can pinpointed features with extreme outliers and remove the them based on certain threshold value.","metadata":{"_uuid":"7b8edcda-470a-4f1f-ac98-08749ba16395","_cell_guid":"0909cbe5-19f4-4209-a4c8-ef19e67b5728","trusted":true}},{"cell_type":"code","source":"# def preprocess_outliers(data):\n#     svm = SGDOneClassSVM(nu=0.55)\n#     yhat = svm.fit_predict(data.drop('id', axis=1))\n#     mask = yhat != -1\n#     data_SVM = data.loc[mask, :].reset_index(drop=True)\n   \n\n#     iso = IsolationForest(random_state=0)\n#     yhat = iso.fit_predict(data_SVM.drop('id', axis=1))\n#     mask = yhat != -1\n#     data_ISO = data_SVM.loc[mask, :].reset_index(drop=True)\n\n\n#     lof = LocalOutlierFactor(n_neighbors=3, contamination=0.1)\n#     yhat = lof.fit_predict(data_ISO.drop(['id', 'target'], axis=1))\n#     mask = yhat != -1\n#     data_final = data_ISO.loc[mask, :].reset_index(drop=True)\n  \n#     dp = np.random.choice(data_final[data_final[TARGET]==7].index.to_numpy(), 120, replace=False)\n#     return data_final.drop(dp)\n\n# X = preprocess_outliers(X)","metadata":{"_uuid":"43f15f25-75be-4db5-8589-2b51e131c15e","_cell_guid":"b4cfba86-2958-4d5f-8e57-3fa1a6fdb1fa","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.457338Z","iopub.execute_input":"2023-10-22T01:03:14.457666Z","iopub.status.idle":"2023-10-22T01:03:14.467404Z","shell.execute_reply.started":"2023-10-22T01:03:14.457639Z","shell.execute_reply":"2023-10-22T01:03:14.466293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X.shape","metadata":{"execution":{"iopub.status.busy":"2023-10-22T01:03:14.469353Z","iopub.execute_input":"2023-10-22T01:03:14.469795Z","iopub.status.idle":"2023-10-22T01:03:14.483209Z","shell.execute_reply.started":"2023-10-22T01:03:14.469756Z","shell.execute_reply":"2023-10-22T01:03:14.482359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# out_col = ['','','','','']\n# fig = plt.figure(figsize=(20,5))\n# for index,col in enumerate(out_col):\n#     plt.subplot(1,5,index+1)\n#     sns.boxplot(y=col, data=X)\n# fig.tight_layout(pad=1.5)","metadata":{"_uuid":"f8fa18cf-e0b1-477d-afb7-5c10b23e7d4c","_cell_guid":"e05b2f36-2365-44cf-ad77-3498abb84f03","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.484165Z","iopub.execute_input":"2023-10-22T01:03:14.484520Z","iopub.status.idle":"2023-10-22T01:03:14.496512Z","shell.execute_reply.started":"2023-10-22T01:03:14.484492Z","shell.execute_reply":"2023-10-22T01:03:14.495422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Filling Missing Values","metadata":{"_uuid":"a804a60e-601d-4532-ad2a-132aa31e290b","_cell_guid":"f3b10a3f-8827-4967-b8a8-54f612dcccb9","trusted":true}},{"cell_type":"markdown","source":"Machine learning model is unable to deal with missing values, thus we need to deal with them based on our understanding of the features. These missing values are denoted NAN as we have seen earlier during our data exploration.","metadata":{"_uuid":"b17b9750-c97d-4053-82a7-c3c3bee2d2e3","_cell_guid":"ffe81114-6365-47ac-b9f3-00adf9d630de","trusted":true}},{"cell_type":"code","source":"# pd.DataFrame(X.isnull().sum(), columns=['sum']).sort_values(by=['sum'],ascending=False)","metadata":{"_uuid":"9fd055c5-19b0-4155-89e7-b2e985a6bbb6","_cell_guid":"2c836361-f375-41a4-aba3-4cf125085841","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.498164Z","iopub.execute_input":"2023-10-22T01:03:14.498904Z","iopub.status.idle":"2023-10-22T01:03:14.509271Z","shell.execute_reply.started":"2023-10-22T01:03:14.498857Z","shell.execute_reply":"2023-10-22T01:03:14.508368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Ordinal features","metadata":{"_uuid":"14910e62-daa7-4814-b508-66b9a9732dc2","_cell_guid":"2f9dfeab-b511-42e6-9ea5-828452849e9a","trusted":true}},{"cell_type":"markdown","source":"We will replace the ordinal missing values with NA, which will be mapped later on when we encode them into an ordered arrangement","metadata":{"_uuid":"9d2daf7c-20f0-47d6-8694-d7714b640020","_cell_guid":"a219bc21-dcd5-44f9-84d6-9c791af81830","trusted":true}},{"cell_type":"code","source":"# X[ord_col] = X[ord_col].fillna(\"NA\")","metadata":{"_uuid":"eb3f296e-45cd-4652-a4e7-d8699c263e1a","_cell_guid":"98945a1f-203f-401c-8022-181cd30192a4","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.510294Z","iopub.execute_input":"2023-10-22T01:03:14.510609Z","iopub.status.idle":"2023-10-22T01:03:14.522011Z","shell.execute_reply.started":"2023-10-22T01:03:14.510578Z","shell.execute_reply":"2023-10-22T01:03:14.520935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Categorical features","metadata":{"_uuid":"b77aeb8c-394a-4a3c-9473-560609da61aa","_cell_guid":"76bb2147-82a9-45df-a4f7-00c034f84fde","trusted":true}},{"cell_type":"markdown","source":"We will replace the missing value of our categorical features with the most frequent occurrence (mode) of the individual features.","metadata":{"_uuid":"f48d8c9c-f90b-470c-9afd-5704bdf4d0c0","_cell_guid":"d7e046f7-ead7-4419-bd79-b1f11d3af4b6","trusted":true}},{"cell_type":"code","source":"# X[cat_col] = X.groupby(\"CategoricalFeature\")[cat_col].transform(lambda x: x.fillna(x.mode()[0]))","metadata":{"_uuid":"1e701762-2930-45e2-9aa4-0f5af5b2f392","_cell_guid":"c902270c-6f5e-4d2a-b62f-fd55c2f9d356","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.523400Z","iopub.execute_input":"2023-10-22T01:03:14.524430Z","iopub.status.idle":"2023-10-22T01:03:14.533880Z","shell.execute_reply.started":"2023-10-22T01:03:14.524397Z","shell.execute_reply":"2023-10-22T01:03:14.532675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Numerical features","metadata":{"_uuid":"8a380fcf-f64d-4c93-aab9-016f9e0fca0c","_cell_guid":"577754fb-4658-4af3-bd28-1db2072a40d3","trusted":true}},{"cell_type":"markdown","source":"For Numerical Features, the common approach will be to replace the missing value with the mean of the feature distribution. However, certain features have wide variance in their distribution.","metadata":{"_uuid":"c204c358-c2be-4ade-a6d3-926c6907553e","_cell_guid":"81fbc0d9-1ce2-4990-9aef-4d625a4345e5","trusted":true}},{"cell_type":"code","source":"# print(\"Mean of : \", X[''].mean())\n# print(\"Mean of : \", X[''].mean())","metadata":{"_uuid":"ed1c2e3c-d8d5-47ae-b0df-68d6ce9e1df3","_cell_guid":"487acc51-063c-4ebb-8883-dbd8282f4a69","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.535490Z","iopub.execute_input":"2023-10-22T01:03:14.535789Z","iopub.status.idle":"2023-10-22T01:03:14.548455Z","shell.execute_reply.started":"2023-10-22T01:03:14.535763Z","shell.execute_reply":"2023-10-22T01:03:14.547077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# neigh_lot = X.groupby('')[''].mean().reset_index(name='')\n# neigh_garage = X.groupby('')[''].mean().reset_index(name='')\n\n# fig, axes = plt.subplots(1,2,figsize=(22,8))\n# axes[0].tick_params(axis='x', rotation=90)\n# sns.barplot(x='', y='', data=neigh_lot, ax=axes[0])\n# axes[1].tick_params(axis='x', rotation=90)\n# sns.barplot(x='', y='', data=neigh_garage, ax=axes[1])","metadata":{"_uuid":"c4f9d25b-9103-4d44-8794-d848b505c344","_cell_guid":"f6bf52b7-7855-42b1-a847-594ddfde3a55","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.549986Z","iopub.execute_input":"2023-10-22T01:03:14.550328Z","iopub.status.idle":"2023-10-22T01:03:14.560888Z","shell.execute_reply.started":"2023-10-22T01:03:14.550296Z","shell.execute_reply":"2023-10-22T01:03:14.560031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# #for correlated relationship\n# X[''] = X.groupby('')[''].transform(lambda x: x.fillna(x.mean()))\n\n# #numerical\n# X[num_col] = X[num_col] = X[num_col].fillna(X[num_col].mean())","metadata":{"_uuid":"de198346-ac18-4ffe-8e87-07ef2a0d631b","_cell_guid":"fb2361da-07a2-4edb-8438-7be46cbaed27","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.562612Z","iopub.execute_input":"2023-10-22T01:03:14.563175Z","iopub.status.idle":"2023-10-22T01:03:14.573826Z","shell.execute_reply.started":"2023-10-22T01:03:14.563143Z","shell.execute_reply":"2023-10-22T01:03:14.572740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Mapping ordinal features","metadata":{"_uuid":"0939cfb2-b5b3-4d03-85f5-b781395a80a9","_cell_guid":"2db7860c-3e80-4b51-9f70-50a9ab97b8fb","trusted":true}},{"cell_type":"markdown","source":"May there are some columns which are ordinal by nature, which represents the quality or condition of certain features. We can map the respective strings to a value. The better the quality, the higher the value.","metadata":{"_uuid":"53131e81-44a0-48d7-a3e1-af2b27f5eb76","_cell_guid":"1c377836-df6b-4f6a-a67b-2d22ed2bbe4f","trusted":true}},{"cell_type":"code","source":"# X.head","metadata":{"_uuid":"9f967ba5-973e-4b7f-8734-5d63e342ef26","_cell_guid":"1bda9dd0-f442-4622-9233-486095235182","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.575113Z","iopub.execute_input":"2023-10-22T01:03:14.575996Z","iopub.status.idle":"2023-10-22T01:03:14.588823Z","shell.execute_reply.started":"2023-10-22T01:03:14.575964Z","shell.execute_reply":"2023-10-22T01:03:14.587609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ordinal_map = {'Ex': 5,'Gd': 4, 'TA': 3, 'Fa': 2, 'Po': 1, 'NA':0}\n# fintype_map = {'GLQ': 6,'ALQ': 5,'BLQ': 4,'Rec': 3,'LwQ': 2,'Unf': 1, 'NA': 0}\n# expose_map = {'Gd': 4, 'Av': 3, 'Mn': 2, 'No': 1, 'NA': 0}\n# fence_map = {'GdPrv': 4,'MnPrv': 3,'GdWo': 2, 'MnWw': 1,'NA': 0}","metadata":{"_uuid":"088b4778-3ab8-4f5b-908e-06f64cc208d5","_cell_guid":"ffb2d593-21e4-400d-9bd1-e1f9f6fc92f3","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.590667Z","iopub.execute_input":"2023-10-22T01:03:14.591572Z","iopub.status.idle":"2023-10-22T01:03:14.601654Z","shell.execute_reply.started":"2023-10-22T01:03:14.591531Z","shell.execute_reply":"2023-10-22T01:03:14.600508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for col in ord_col:\n#     X[col] = X[col].map(ordinal_map)\n    \n# fin_col = ['','']\n# for col in fin_col:\n#     X[col] = X[col].map(fintype_map)\n\n# X[''] = X[''].map(expose_map)\n# X[''] = X[''].map(fence_map)","metadata":{"_uuid":"70703f9f-0c6f-412a-b427-5d501d5a75b3","_cell_guid":"dca2e4ef-e173-4db3-b11e-75fc37b6f7ef","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.603346Z","iopub.execute_input":"2023-10-22T01:03:14.604344Z","iopub.status.idle":"2023-10-22T01:03:14.619048Z","shell.execute_reply.started":"2023-10-22T01:03:14.604191Z","shell.execute_reply":"2023-10-22T01:03:14.617712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cols = X.select_dtypes(np.number).drop(['id', TARGET], axis=1).columns\n\n# for i in cols:\n#     X[i]=RobustScaler().fit_transform(X[i].values.reshape(-1,1))\n\n# transformer = RobustScaler().fit(X[cols])\n# X[cols] = transformer.transform(X[cols])","metadata":{"_uuid":"72c2e588-7ea2-4f04-b464-9a6e3b097a11","_cell_guid":"f19163a4-0fa1-4dc7-8cb8-c59b3ed107c4","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.620890Z","iopub.execute_input":"2023-10-22T01:03:14.621791Z","iopub.status.idle":"2023-10-22T01:03:14.631248Z","shell.execute_reply.started":"2023-10-22T01:03:14.621748Z","shell.execute_reply":"2023-10-22T01:03:14.630379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Remove bad labels","metadata":{"_uuid":"436a56a4-a9af-4f08-991c-5d450e1bdbe2","_cell_guid":"5162dc2b-c972-4322-aae9-07682986e342","trusted":true}},{"cell_type":"code","source":"# def remove_bad_lbs(X, bad_lbs):\n#     return X[~X['id'].isin(bad_lbs)]","metadata":{"_uuid":"adbfd166-db19-498e-b696-a2a30d549327","_cell_guid":"2f1fb488-bc4c-4784-b6e8-1934d6e343db","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.632542Z","iopub.execute_input":"2023-10-22T01:03:14.633633Z","iopub.status.idle":"2023-10-22T01:03:14.645525Z","shell.execute_reply.started":"2023-10-22T01:03:14.633598Z","shell.execute_reply":"2023-10-22T01:03:14.644334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X = remove_bad_lbs(X,[])","metadata":{"_uuid":"295d540c-11ac-44ea-9838-7351953652dd","_cell_guid":"32de03ff-b593-43f6-9f10-f807a2a3e917","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.646901Z","iopub.execute_input":"2023-10-22T01:03:14.647811Z","iopub.status.idle":"2023-10-22T01:03:14.657550Z","shell.execute_reply.started":"2023-10-22T01:03:14.647777Z","shell.execute_reply":"2023-10-22T01:03:14.656351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X.shape","metadata":{"execution":{"iopub.status.busy":"2023-10-22T01:03:14.658969Z","iopub.execute_input":"2023-10-22T01:03:14.659489Z","iopub.status.idle":"2023-10-22T01:03:14.669124Z","shell.execute_reply.started":"2023-10-22T01:03:14.659457Z","shell.execute_reply":"2023-10-22T01:03:14.667952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Set useless features to zero","metadata":{}},{"cell_type":"markdown","source":"We can also disable features that does not have any linear relationship with target identified on the Correlation analysis.","metadata":{}},{"cell_type":"code","source":"# selected_features = []\n# for i in X.columns:\n#     if i not in selected_features:\n#         X[i] = 0","metadata":{"execution":{"iopub.status.busy":"2023-10-22T01:03:14.670712Z","iopub.execute_input":"2023-10-22T01:03:14.671613Z","iopub.status.idle":"2023-10-22T01:03:14.682379Z","shell.execute_reply.started":"2023-10-22T01:03:14.671571Z","shell.execute_reply":"2023-10-22T01:03:14.681569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#X.shape","metadata":{"execution":{"iopub.status.busy":"2023-10-22T01:03:14.683588Z","iopub.execute_input":"2023-10-22T01:03:14.684206Z","iopub.status.idle":"2023-10-22T01:03:14.695993Z","shell.execute_reply.started":"2023-10-22T01:03:14.684175Z","shell.execute_reply":"2023-10-22T01:03:14.694807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering <a class=\"anchor\"  id=\"chapter2\"></a>","metadata":{"_uuid":"29cc00c1-bba1-4cc4-bccc-bed034ac4704","_cell_guid":"ad551464-3c19-42bf-a610-3ec2d53a01d8","trusted":true}},{"cell_type":"markdown","source":"After removing the outliers, highly correlated features and imputing missing values, we can now proceed with adding additional information for our model to train on. This is done by the means of - Feature Engineering.","metadata":{"_uuid":"18520f9c-194a-4c1a-a640-ece76e2ec945","_cell_guid":"1b65da40-4d63-4af5-a524-5f74f32b437f","trusted":true}},{"cell_type":"markdown","source":"## Creating Features","metadata":{"_uuid":"bb54d53b-a777-4e72-a05b-33077e4058e1","_cell_guid":"5b0542e5-3636-4667-961b-e76f3af1a366","trusted":true}},{"cell_type":"code","source":"def make_features(df):\n    \n    periods = 20\n    df[\"anglez\"] = abs(df[\"anglez\"])\n    df[\"anglez_diff\"] = df.groupby('series_id')['anglez'].diff(periods=periods).fillna(method=\"bfill\").astype('float16')\n    df[\"enmo_diff\"] = df.groupby('series_id')['enmo'].diff(periods=periods).fillna(method=\"bfill\").astype('float16')\n    df[\"anglez_rolling_mean\"] = df[\"anglez\"].rolling(periods,center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float16')\n    df[\"enmo_rolling_mean\"] = df[\"enmo\"].rolling(periods,center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float16')\n    df[\"anglez_rolling_max\"] = df[\"anglez\"].rolling(periods,center=True).max().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float16')\n    df[\"enmo_rolling_max\"] = df[\"enmo\"].rolling(periods,center=True).max().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float16')\n    df[\"anglez_rolling_std\"] = df[\"anglez\"].rolling(periods,center=True).std().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float16')\n    df[\"enmo_rolling_std\"] = df[\"enmo\"].rolling(periods,center=True).std().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float16')\n    df[\"anglez_diff_rolling_mean\"] = df[\"anglez_diff\"].rolling(periods,center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float16')\n    df[\"enmo_diff_rolling_mean\"] = df[\"enmo_diff\"].rolling(periods,center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float16')\n    df[\"anglez_diff_rolling_max\"] = df[\"anglez_diff\"].rolling(periods,center=True).max().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float16')\n    df[\"enmo_diff_rolling_max\"] = df[\"enmo_diff\"].rolling(periods,center=True).max().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float16')\n    \n    return df\n\nfeatures = [\n            #\"series_id\",\n            \"step\",\n            \"hour\",\n            \"anglez\",\n            \"anglez_rolling_mean\",\n            \"anglez_rolling_max\",\n            \"anglez_rolling_std\",\n            \"anglez_diff\",\n            \"anglez_diff_rolling_mean\",\n            \"anglez_diff_rolling_max\",\n            \"enmo\",\n            \"enmo_rolling_mean\",\n            \"enmo_rolling_max\",\n            \"enmo_rolling_std\",\n            \"enmo_diff\",\n            \"enmo_diff_rolling_mean\",\n            \"enmo_diff_rolling_max\",\n           ]","metadata":{"execution":{"iopub.status.busy":"2023-10-22T01:03:14.697923Z","iopub.execute_input":"2023-10-22T01:03:14.698614Z","iopub.status.idle":"2023-10-22T01:03:14.716357Z","shell.execute_reply.started":"2023-10-22T01:03:14.698581Z","shell.execute_reply":"2023-10-22T01:03:14.715278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def mathematical_transforms(df):\n#     X = pd.DataFrame()\n#     #...\n#     return X\n\n\n# def interactions(df):\n#     X = pd.get_dummies(df., prefix=\"\")\n#     X = X.mul(df., axis=0)\n#     return X\n\n\n# def counts(df):\n#     X = pd.DataFrame()\n#     X[\"\"] = df[[\n#         \"\",\n#         \"\",\n#     ]].gt(0.0).sum(axis=1)\n#     return X\n\n\n# def break_down(df):\n#     X = pd.DataFrame()\n#     X[\"\"] = df. .str.split(\"_\", n=1, expand=True)[0]\n#     return X\n\n\n# def group_transforms(df):\n#     X = pd.DataFrame()\n#     X[\"\"] = df.groupby(\"\")[\"\"].transform(\"median\")\n#     return X\n\n# def label_encode(df):\n#     X = df.copy()\n#     for colname in X.select_dtypes([\"category\"]):\n#         X[colname] = X[colname].cat.codes\n#     return X","metadata":{"_uuid":"d886dd5c-fd1c-47c2-af2d-475771f20d3d","_cell_guid":"04bad6eb-4f45-4378-9cf7-d07384366fc8","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.717722Z","iopub.execute_input":"2023-10-22T01:03:14.718978Z","iopub.status.idle":"2023-10-22T01:03:14.734103Z","shell.execute_reply.started":"2023-10-22T01:03:14.718934Z","shell.execute_reply":"2023-10-22T01:03:14.733247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Skewed target","metadata":{"_uuid":"11e719bd-8ec4-49aa-8ada-bd373dcb93db","_cell_guid":"397d691b-40f2-46d7-aa31-a2ae5f491129","trusted":true}},{"cell_type":"code","source":"# # before transforation\n# plt.figure(figsize=(10,6))\n# plt.title(\"Before transformation of Class\")\n# dist = sns.distplot(X[TARGET],norm_hist=False)","metadata":{"_uuid":"53efa278-32ec-4547-905f-eefd56c2c9a4","_cell_guid":"3818676b-e2b5-443a-a7ae-3ba2a07e2d77","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.735552Z","iopub.execute_input":"2023-10-22T01:03:14.736554Z","iopub.status.idle":"2023-10-22T01:03:14.746481Z","shell.execute_reply.started":"2023-10-22T01:03:14.736514Z","shell.execute_reply":"2023-10-22T01:03:14.745537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Distribution is skewed to the right, where the tail on the curve’s right-hand side is longer than the tail on the left-hand side, and the mean is greater than the mode. This situation is also called positive skewness.\nHaving a skewed target will affect the overall performance of our machine learning model, thus, one way to alleviate will be to using log transformation on skewed target reducing the skewness of the distribution.","metadata":{"_uuid":"d632fde2-ac28-4ce4-afa6-f2ac171643c3","_cell_guid":"831068d3-cb1c-4ca0-9063-b6760ca42386","trusted":true}},{"cell_type":"code","source":"# X[TARGET] = np.log(X[TARGET])","metadata":{"_uuid":"dd309c4c-3ab0-4a9f-ba5e-bfe07a56494b","_cell_guid":"38cffba7-4a4c-4651-84a7-3cb27272b54d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.747717Z","iopub.execute_input":"2023-10-22T01:03:14.748042Z","iopub.status.idle":"2023-10-22T01:03:14.762018Z","shell.execute_reply.started":"2023-10-22T01:03:14.748013Z","shell.execute_reply":"2023-10-22T01:03:14.760807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # after transforation\n# plt.figure(figsize=(10,6))\n# plt.title(\"After transformation of Class\")\n# dist = sns.distplot(np.log(X[TARGET]),norm_hist=False)","metadata":{"_uuid":"03f71cf2-f211-418f-b016-c2d54e29c7d0","_cell_guid":"eef163b3-16eb-4b3e-a683-2eef22896c48","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.763558Z","iopub.execute_input":"2023-10-22T01:03:14.764003Z","iopub.status.idle":"2023-10-22T01:03:14.775891Z","shell.execute_reply.started":"2023-10-22T01:03:14.763969Z","shell.execute_reply":"2023-10-22T01:03:14.774761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Clustering K-Means or Cluster-distance","metadata":{"_uuid":"c0d7e2a1-34e1-4c20-9f8f-3391fe8f296c","_cell_guid":"c8586b74-b7b5-465d-8ee0-6668b185cbf0","trusted":true}},{"cell_type":"markdown","source":"Clustering simply means the assigning of data points to groups based upon how similar the points are to each other. A clustering algorithm makes \"birds of a feather flock together,\" so to speak.\n\nWhen used for feature engineering, we could attempt to discover groups of customers representing a market segment, for instance, or geographic areas that share similar weather patterns. Adding a feature of cluster labels can help machine learning models untangle complicated relationships of space or proximity.\n\nK-means clustering measures similarity using ordinary straight-line distance (Euclidean distance, in other words). It creates clusters by placing a number of points, called centroids, inside the feature-space. Each point in the dataset is assigned to the cluster of whichever centroid it's closest to. The \"k\" in \"k-means\" is how many centroids (that is, clusters) it creates. You define the k yourself.","metadata":{"_uuid":"355868b4-fb38-4e89-9d79-ff3c124c9203","_cell_guid":"e5899210-047a-4fd8-8a35-c949394743c6","trusted":true}},{"cell_type":"code","source":"# cluster_features = [\n#     \"\",\n#     \"\"\n# ]\n\n\n# def cluster_labels(df, features, n_clusters=20):\n#     X = df.copy()\n#     X_scaled = X.loc[:, features]\n#     X_scaled = (X_scaled - X_scaled.mean(axis=0)) / X_scaled.std(axis=0)\n#     kmeans = KMeans(n_clusters=n_clusters, n_init=50, random_state=0)\n#     X_new = pd.DataFrame()\n#     X_new[\"Cluster\"] = kmeans.fit_predict(X_scaled)\n#     return X_new\n\n\n# def cluster_distance(df, features, n_clusters=20):\n#     X = df.copy()\n#     X_scaled = X.loc[:, features]\n#     X_scaled = (X_scaled - X_scaled.mean(axis=0)) / X_scaled.std(axis=0)\n#     kmeans = KMeans(n_clusters=20, n_init=50, random_state=0)\n#     X_cd = kmeans.fit_transform(X_scaled)\n#     # Label features and join to dataset\n#     X_cd = pd.DataFrame(\n#         X_cd, columns=[f\"Centroid_{i}\" for i in range(X_cd.shape[1])]\n#     )\n#     return X_cd\n\n# def plot_cluester(X):\n#     Xy = X.copy()\n#     Xy[\"Cluster\"] = Xy.Cluster.astype(\"category\")\n#     Xy[TARGET] = y\n#     sns.relplot(\n#         x=\"value\", y=TARGET, hue=\"Cluster\", col=\"variable\",\n#         height=4, aspect=1, facet_kws={'sharex': False}, col_wrap=3,\n#         data=Xy.melt(\n#             value_vars=features, id_vars=[TARGET, \"Cluster\"],\n#         ),\n#     );","metadata":{"_uuid":"7d58dc82-64a8-4777-866f-ddd5dd48ded0","_cell_guid":"45648e55-a858-462e-86df-bc6ec2ff449f","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.777146Z","iopub.execute_input":"2023-10-22T01:03:14.777901Z","iopub.status.idle":"2023-10-22T01:03:14.791649Z","shell.execute_reply.started":"2023-10-22T01:03:14.777869Z","shell.execute_reply":"2023-10-22T01:03:14.790637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot_cluester(X)","metadata":{"_uuid":"2ccdb776-d4f2-4ad8-80c8-f15a12e2a1f4","_cell_guid":"d1ab954b-af7f-4970-8ee7-39a7f10c7c1e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.792909Z","iopub.execute_input":"2023-10-22T01:03:14.793521Z","iopub.status.idle":"2023-10-22T01:03:14.806568Z","shell.execute_reply.started":"2023-10-22T01:03:14.793488Z","shell.execute_reply":"2023-10-22T01:03:14.805567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## PCA","metadata":{"_uuid":"36331ebc-5b0b-4cab-aff1-8d74d84b052e","_cell_guid":"bdc2bd44-da9f-466c-aac3-19acdc66c228","trusted":true}},{"cell_type":"markdown","source":"There are two ways you could use PCA for feature engineering.\n\nThe first way is to use it as a descriptive technique. Since the components tell you about the variation, you could compute the MI scores for the components and see what kind of variation is most predictive of your target. That could give you ideas for kinds of features to create -- a product of 'Height' and 'Diameter' if 'Size' is important, say, or a ratio of 'Height' and 'Diameter' if Shape is important. You could even try clustering on one or more of the high-scoring components.\n\nThe second way is to use the components themselves as features. Because the components expose the variational structure of the data directly, they can often be more informative than the original features.","metadata":{"_uuid":"6fb48e00-a1a8-49f5-9b73-4bd407ac2012","_cell_guid":"806ab8e9-d7c8-4638-93ba-f104805733de","trusted":true}},{"cell_type":"markdown","source":"PCA Best Practices\n* There are a few things to keep in mind when applying PCA:\n* PCA only works with numeric features, like continuous quantities or counts.\n* PCA is sensitive to scale. It's good practice to standardize your data before applying PCA, unless you know you have good reason not to.\n* Consider removing or constraining outliers, since they can have an undue influence on the results.","metadata":{"_uuid":"b012b33e-0ac7-4ae2-9642-100665e5dc60","_cell_guid":"4acdae92-68d3-49da-9093-fc4413ae3a36","trusted":true}},{"cell_type":"markdown","source":"Let's choose a few features that are highly correlated with our target.","metadata":{"_uuid":"7dfa7a20-2958-4cef-9051-8b38d95d12b1","_cell_guid":"b179c3ff-4609-47b0-ac63-95e2a25e5d46","trusted":true}},{"cell_type":"code","source":"# def apply_pca(X, standardize=True):\n#     # Standardize\n#     if standardize:\n#         X = (X - X.mean(axis=0)) / X.std(axis=0)\n#     # Create principal components\n#     pca = PCA()\n#     X_pca = pca.fit_transform(X)\n#     # Convert to dataframe\n#     component_names = [f\"PC{i+1}\" for i in range(X_pca.shape[1])]\n#     X_pca = pd.DataFrame(X_pca, columns=component_names)\n#     # Create loadings\n#     loadings = pd.DataFrame(\n#         pca.components_.T,  # transpose the matrix of loadings\n#         columns=component_names,  # so the columns are the principal components\n#         index=X.columns,  # and the rows are the original features\n#     )\n#     return pca, X_pca, loadings\n\n\n# def plot_variance(pca, width=8, dpi=100):\n#     # Create figure\n#     fig, axs = plt.subplots(1, 2)\n#     n = pca.n_components_\n#     grid = np.arange(1, n + 1)\n#     # Explained variance\n#     evr = pca.explained_variance_ratio_\n#     axs[0].bar(grid, evr)\n#     axs[0].set(\n#         xlabel=\"Component\", title=\"% Explained Variance\", ylim=(0.0, 1.0)\n#     )\n#     # Cumulative Variance\n#     cv = np.cumsum(evr)\n#     axs[1].plot(np.r_[0, grid], np.r_[0, cv], \"o-\")\n#     axs[1].set(\n#         xlabel=\"Component\", title=\"% Cumulative Variance\", ylim=(0.0, 1.0)\n#     )\n#     # Set up figure\n#     fig.set(figwidth=8, dpi=100)\n#     return axs","metadata":{"_uuid":"9b3c384e-a94a-44a0-b4e7-09abfd1e5c44","_cell_guid":"a4296ca3-4d66-49b5-b781-f584b2b517fe","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.807748Z","iopub.execute_input":"2023-10-22T01:03:14.808368Z","iopub.status.idle":"2023-10-22T01:03:14.820122Z","shell.execute_reply.started":"2023-10-22T01:03:14.808335Z","shell.execute_reply":"2023-10-22T01:03:14.818870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# #Let's choose a few features that are highly correlated with our target, SalePrice.\n# pca_features = [\n#     \"Amount\"\n# ]\n\n# def pca_inspired(df):\n#     X = pd.DataFrame()\n#     X[\"Amount\"] = df.Amount + df.\n#     X[\"Amount\"] = df. * df.\n#     return X\n\n\n# def pca_components(df, pca_features):\n#     X = df.loc[:, pca_features]\n#     _, X_pca, _ = apply_pca(X)\n#     return X_pca","metadata":{"_uuid":"fb8f8aa6-a75f-4b87-8115-284055fead78","_cell_guid":"a07037f3-1126-40be-9b86-b422768a574a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.821317Z","iopub.execute_input":"2023-10-22T01:03:14.822027Z","iopub.status.idle":"2023-10-22T01:03:14.836725Z","shell.execute_reply.started":"2023-10-22T01:03:14.821996Z","shell.execute_reply":"2023-10-22T01:03:14.835467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def corrplot(df, method=\"pearson\", annot=True, **kwargs):\n#     sns.clustermap(\n#         df.corr(method),\n#         vmin=-1.0,\n#         vmax=1.0,\n#         cmap=\"icefire\",\n#         method=\"complete\",\n#         annot=annot,\n#         **kwargs,\n#     )\n\n# corrplot(df_train, annot=None)","metadata":{"_uuid":"9fd25a01-1ea2-4143-99e1-967d6bd615b8","_cell_guid":"89bca279-17c8-490d-a272-ce2ff367d691","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.851492Z","iopub.execute_input":"2023-10-22T01:03:14.852081Z","iopub.status.idle":"2023-10-22T01:03:14.856457Z","shell.execute_reply.started":"2023-10-22T01:03:14.852049Z","shell.execute_reply":"2023-10-22T01:03:14.855506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def indicate_outliers(df):\n#     X_new = pd.DataFrame()\n#     X_new[\"Outlier\"] = (df. == \"\") & (df. == \"\")\n#     return X_new","metadata":{"_uuid":"741dd06c-c40d-4622-8782-16194e57d339","_cell_guid":"87bfefd6-e2a4-488f-9a1e-ff95b2217bba","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.857651Z","iopub.execute_input":"2023-10-22T01:03:14.858135Z","iopub.status.idle":"2023-10-22T01:03:14.869327Z","shell.execute_reply.started":"2023-10-22T01:03:14.858106Z","shell.execute_reply":"2023-10-22T01:03:14.868482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Target Encoding","metadata":{"_uuid":"df05d8bf-22ec-421e-844c-158bf54b4ff5","_cell_guid":"0b7c5af9-d666-4f28-826b-1af8ebe82e20","trusted":true}},{"cell_type":"markdown","source":"Target encoding, is instead meant for categorical features. It's a method of encoding categories as numbers, like one-hot or label encoding, with the difference that it also uses the target to create the encoding. This makes it what we call a supervised feature engineering technique.","metadata":{"_uuid":"cecd0f0e-8a1b-40ab-9d40-33487858caf6","_cell_guid":"278f878a-c4a7-446b-a7d5-b3e4b6c8db2c","trusted":true}},{"cell_type":"markdown","source":"A target encoding is any kind of encoding that replaces a feature's categories with some number derived from the target.","metadata":{"_uuid":"b301ea20-82ef-434a-9ec7-e04d852743a9","_cell_guid":"e62a6112-2ca4-41e2-9ac7-c73cb5ecf1cd","trusted":true}},{"cell_type":"markdown","source":"There is, however, a way you can use target encoding without having to use held-out encoding data. It's basically the same trick used in cross-validation:\n\n1. Split the data into folds, each fold having two splits of the dataset.\n1. Train the encoder on one split but transform the values of the other.\n1. Repeat for all the splits.\n\nThis way, training and transformation always take place on independent sets of data, just like when you use a holdout set but without any data going to waste.","metadata":{"_uuid":"d34e03bd-ea4d-4338-b410-da7aacb95677","_cell_guid":"7ce8d957-7487-48f4-a25b-863f6da52c0e","trusted":true}},{"cell_type":"code","source":"# class CrossFoldEncoder:\n#     def __init__(self, encoder, **kwargs):\n#         self.encoder_ = encoder\n#         self.kwargs_ = kwargs  # keyword arguments for the encoder\n#         self.cv_ = KFold(n_splits=5)\n\n#     # Fit an encoder on one split and transform the feature on the\n#     # other. Iterating over the splits in all folds gives a complete\n#     # transformation. We also now have one trained encoder on each\n#     # fold.\n#     def fit_transform(self, X, y, cols):\n#         self.fitted_encoders_ = []\n#         self.cols_ = cols\n#         X_encoded = []\n#         for idx_encode, idx_train in self.cv_.split(X):\n#             fitted_encoder = self.encoder_(cols=cols, **self.kwargs_)\n#             fitted_encoder.fit(\n#                 X.iloc[idx_encode, :], y.iloc[idx_encode],\n#             )\n#             X_encoded.append(fitted_encoder.transform(X.iloc[idx_train, :])[cols])\n#             self.fitted_encoders_.append(fitted_encoder)\n#         X_encoded = pd.concat(X_encoded)\n#         X_encoded.columns = [name + \"_encoded\" for name in X_encoded.columns]\n#         return X_encoded\n\n#     # To transform the test data, average the encodings learned from\n#     # each fold.\n#     def transform(self, X):\n#         from functools import reduce\n\n#         X_encoded_list = []\n#         for fitted_encoder in self.fitted_encoders_:\n#             X_encoded = fitted_encoder.transform(X)\n#             X_encoded_list.append(X_encoded[self.cols_])\n#         X_encoded = reduce(\n#             lambda x, y: x.add(y, fill_value=0), X_encoded_list\n#         ) / len(X_encoded_list)\n#         X_encoded.columns = [name + \"_encoded\" for name in X_encoded.columns]\n#         return X_encoded","metadata":{"_uuid":"c4386236-bc3e-4661-be18-5aa999801f2c","_cell_guid":"7de6a890-761a-4ab4-89e6-c25365750a12","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.870504Z","iopub.execute_input":"2023-10-22T01:03:14.871338Z","iopub.status.idle":"2023-10-22T01:03:14.890255Z","shell.execute_reply.started":"2023-10-22T01:03:14.871298Z","shell.execute_reply":"2023-10-22T01:03:14.888928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Usage:","metadata":{"_uuid":"c3f003fa-d77c-404b-af29-416053c66081","_cell_guid":"65895e08-2358-414d-8e14-dd72294788b5","trusted":true}},{"cell_type":"code","source":"# encoder = CrossFoldEncoder(MEstimateEncoder, m=1)\n# X_encoded = encoder.fit_transform(X, y, cols=[\"\", \"\"])","metadata":{"_uuid":"95d206d9-245d-4f74-b3b1-5975af6721bf","_cell_guid":"30bcd6c5-c862-4ef9-86c8-3d3ed1864664","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.891526Z","iopub.execute_input":"2023-10-22T01:03:14.892300Z","iopub.status.idle":"2023-10-22T01:03:14.903414Z","shell.execute_reply.started":"2023-10-22T01:03:14.892225Z","shell.execute_reply":"2023-10-22T01:03:14.902304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X_encoded.head()","metadata":{"_uuid":"da55e84b-870a-4a60-9891-8b2114f2f651","_cell_guid":"ab61caa3-1042-405c-8f5d-7eb00126b00d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.905007Z","iopub.execute_input":"2023-10-22T01:03:14.905380Z","iopub.status.idle":"2023-10-22T01:03:14.915915Z","shell.execute_reply.started":"2023-10-22T01:03:14.905349Z","shell.execute_reply":"2023-10-22T01:03:14.914783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X_encoded_t = encoder.transform(test)","metadata":{"_uuid":"1dc7d407-8bbc-4858-b5cd-a0efbefe05e0","_cell_guid":"273a0228-0b4a-43e9-99f0-5484f7f13a52","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.917563Z","iopub.execute_input":"2023-10-22T01:03:14.918064Z","iopub.status.idle":"2023-10-22T01:03:14.928444Z","shell.execute_reply.started":"2023-10-22T01:03:14.918023Z","shell.execute_reply":"2023-10-22T01:03:14.927366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X_encoded_t","metadata":{"_uuid":"603f73be-7136-48d3-8003-1d2f698f59ed","_cell_guid":"4086e8d1-4fd8-4ef2-a5f6-176efdb80be9","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.929950Z","iopub.execute_input":"2023-10-22T01:03:14.930291Z","iopub.status.idle":"2023-10-22T01:03:14.942393Z","shell.execute_reply.started":"2023-10-22T01:03:14.930260Z","shell.execute_reply":"2023-10-22T01:03:14.941221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X.head()","metadata":{"_uuid":"87220c8e-eda2-41e3-b69a-e7d90142a88b","_cell_guid":"8ec773ff-7683-44bc-8a13-1d44aa629ae0","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.943712Z","iopub.execute_input":"2023-10-22T01:03:14.944160Z","iopub.status.idle":"2023-10-22T01:03:14.955833Z","shell.execute_reply.started":"2023-10-22T01:03:14.944117Z","shell.execute_reply":"2023-10-22T01:03:14.954788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X['Amount'] = X_encoded['Amount_encoded']\n# X['Time'] = X_encoded['Time_encoded']","metadata":{"_uuid":"c3d097f8-fab5-4e44-8de4-da2a18b413be","_cell_guid":"5bb493e2-fbe8-4dc9-ab66-5306d36c238e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.957537Z","iopub.execute_input":"2023-10-22T01:03:14.957862Z","iopub.status.idle":"2023-10-22T01:03:14.971893Z","shell.execute_reply.started":"2023-10-22T01:03:14.957833Z","shell.execute_reply":"2023-10-22T01:03:14.970966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X = X.join(X_encoded)\n# test = test.join(X_encoded_t)","metadata":{"_uuid":"7e6985ae-b122-4efc-b104-41bde6af6e3f","_cell_guid":"2940914e-eb20-4e08-843b-6d94648e7530","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.972848Z","iopub.execute_input":"2023-10-22T01:03:14.973146Z","iopub.status.idle":"2023-10-22T01:03:14.984856Z","shell.execute_reply.started":"2023-10-22T01:03:14.973120Z","shell.execute_reply":"2023-10-22T01:03:14.984010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X.head()","metadata":{"_uuid":"0dbf0450-734e-4abb-ab4f-1de943c5e5e7","_cell_guid":"1bb6cf83-fd0c-4417-93b2-cfda25ef89d5","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:14.986367Z","iopub.execute_input":"2023-10-22T01:03:14.986826Z","iopub.status.idle":"2023-10-22T01:03:14.999385Z","shell.execute_reply.started":"2023-10-22T01:03:14.986772Z","shell.execute_reply":"2023-10-22T01:03:14.998327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test","metadata":{"_uuid":"8929c826-ac93-4414-aa5d-f28e1206c3ab","_cell_guid":"088ab973-664e-40b7-adf9-f5be63d2e4de","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:15.000689Z","iopub.execute_input":"2023-10-22T01:03:15.001001Z","iopub.status.idle":"2023-10-22T01:03:15.012455Z","shell.execute_reply.started":"2023-10-22T01:03:15.000975Z","shell.execute_reply":"2023-10-22T01:03:15.010832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Scaling Manual or by Pipeline","metadata":{"_uuid":"94df9013-4fda-4040-a7d6-94974f30d49f","_cell_guid":"30dfac1e-d96e-45d6-a57a-4b48e06bd152","trusted":true}},{"cell_type":"markdown","source":"RobustScaler is a transformation technique that removes the median and scales the data according to the quantile range (defaults to IQR: Interquartile Range). The IQR is the range between the 1st quartile (25th quantile) and the 3rd quartile (75th quantile). It is also robust to outliers, which makes it ideal for data where there are too many outliers that will drastically reduce the number of training data.\n\nFit RobustScaler on both Train and Test set is a mistake. By fitting the scaler on both train and test set, we exposed ourselves to the problem of Data Leakage. Data Leakage is a problem when information from outside the training dataset is used to create the model. If we fit the scaler on both training and test data, our training data characteristics will contain the distribution of our testset. As such, we are unknowningly passing in information about our test data into the final training data for training, which will not give us the opportunity to truly test our model on data it has never seen before.\nFit the scaler just on training data, and then transforming it on both training and test data","metadata":{"_uuid":"01b328e2-6029-4ade-ad05-bf1e3e1a32f9","_cell_guid":"e8206741-239b-4fa3-82ba-bb0c4f543175","trusted":true}},{"cell_type":"markdown","source":"If a distributions are very different for train and test set. This could cause serious problem, because the algorithm will be able to distinct between those sets more easily.\n\nWe can convert the features by breaking it down into an hour of the dayfeature or drop it.","metadata":{"_uuid":"28e66896-1a7c-4b94-9666-429a27d03a0a","_cell_guid":"ca41b9e8-a735-45f1-9eb6-24b3ab661f5a","trusted":true}},{"cell_type":"code","source":"# X = X.drop('', axis=1)\n# test = test.drop('', axis=1)","metadata":{"_uuid":"65051040-221c-47d0-a14f-7431bfdf2ce2","_cell_guid":"0b42664b-8789-4f76-ad44-24a3ffb68b38","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:15.014156Z","iopub.execute_input":"2023-10-22T01:03:15.015597Z","iopub.status.idle":"2023-10-22T01:03:15.027256Z","shell.execute_reply.started":"2023-10-22T01:03:15.015560Z","shell.execute_reply":"2023-10-22T01:03:15.026352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X.head(10)","metadata":{"_uuid":"7af034d0-52c6-4c85-93b1-140f68683252","_cell_guid":"3b9bce37-a649-457c-b316-539fbf8edfd8","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:15.028379Z","iopub.execute_input":"2023-10-22T01:03:15.028691Z","iopub.status.idle":"2023-10-22T01:03:15.043203Z","shell.execute_reply.started":"2023-10-22T01:03:15.028664Z","shell.execute_reply":"2023-10-22T01:03:15.042414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cols = X.select_dtypes(np.number).drop(['id', TARGET], axis=1).columns\n\n# transformer = RobustScaler().fit(X[cols])\n# X[cols] = transformer.transform(X[cols])\n\n# # #test[cols] = transformer.transform(test[cols])","metadata":{"_uuid":"f541aa11-dbcd-4aa5-884c-2d2039e42b94","_cell_guid":"c0fc9f9f-f6d2-469f-9c99-5aba60c0fee8","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:15.044433Z","iopub.execute_input":"2023-10-22T01:03:15.044800Z","iopub.status.idle":"2023-10-22T01:03:15.055798Z","shell.execute_reply.started":"2023-10-22T01:03:15.044768Z","shell.execute_reply":"2023-10-22T01:03:15.054712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#cols","metadata":{"_uuid":"7c846f22-22a2-4691-b884-1fe472f9c570","_cell_guid":"30039d5f-b6a7-412b-a20f-ae44857cdcfa","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:15.056740Z","iopub.execute_input":"2023-10-22T01:03:15.057040Z","iopub.status.idle":"2023-10-22T01:03:15.066011Z","shell.execute_reply.started":"2023-10-22T01:03:15.057014Z","shell.execute_reply":"2023-10-22T01:03:15.065008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X.head(10)","metadata":{"_uuid":"87ea540f-e188-4e06-a85b-e14b648b7f14","_cell_guid":"69f03457-60c5-4e77-9d53-d18e66fe8f62","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:15.067221Z","iopub.execute_input":"2023-10-22T01:03:15.068142Z","iopub.status.idle":"2023-10-22T01:03:15.076292Z","shell.execute_reply.started":"2023-10-22T01:03:15.068103Z","shell.execute_reply":"2023-10-22T01:03:15.075451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = make_features(train)\n\nX = train[features]\ny = train['awake']","metadata":{"execution":{"iopub.status.busy":"2023-10-22T01:03:15.077729Z","iopub.execute_input":"2023-10-22T01:03:15.078397Z","iopub.status.idle":"2023-10-22T01:03:30.908709Z","shell.execute_reply.started":"2023-10-22T01:03:15.078356Z","shell.execute_reply":"2023-10-22T01:03:30.907410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train","metadata":{"execution":{"iopub.status.busy":"2023-10-22T01:03:30.910166Z","iopub.execute_input":"2023-10-22T01:03:30.910642Z","iopub.status.idle":"2023-10-22T01:03:30.915074Z","shell.execute_reply.started":"2023-10-22T01:03:30.910602Z","shell.execute_reply":"2023-10-22T01:03:30.914000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"or pipeline","metadata":{"_uuid":"f02b099c-3059-4279-8a3d-f90469f93338","_cell_guid":"8478ea94-1fd8-4e89-ac18-4dd6a46e77c6","trusted":true}},{"cell_type":"code","source":"features_num = train.select_dtypes(exclude=['object']).drop([TARGET], axis=1).copy().columns\nfeatures_cat = train.select_dtypes(include=['object']).drop(['series_id'], axis=1).copy().columns \n\nprint(features_num) \nprint(features_cat)\n\ntransformer_num = make_pipeline(\n    SimpleImputer(strategy=\"constant\"), # there are a few missing values\n    #StandardScaler(),\n    RobustScaler()\n)\ntransformer_cat = make_pipeline(\n    SimpleImputer(strategy=\"constant\", fill_value=\"NA\"),\n    OneHotEncoder(handle_unknown='ignore'),\n)\n\npreprocessor = make_column_transformer(\n    (transformer_num, features_num),\n    (transformer_cat, features_cat)\n)","metadata":{"_uuid":"84cd80bf-b305-4c00-92c4-e9a9eccb845c","_cell_guid":"cc8637d8-71c8-45e0-b104-336e87e173f5","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-10-22T01:03:30.917065Z","iopub.execute_input":"2023-10-22T01:03:30.917570Z","iopub.status.idle":"2023-10-22T01:03:32.936866Z","shell.execute_reply.started":"2023-10-22T01:03:30.917470Z","shell.execute_reply":"2023-10-22T01:03:32.935757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X.shape","metadata":{"execution":{"iopub.status.busy":"2023-10-22T01:03:32.938347Z","iopub.execute_input":"2023-10-22T01:03:32.938776Z","iopub.status.idle":"2023-10-22T01:03:32.944442Z","shell.execute_reply.started":"2023-10-22T01:03:32.938735Z","shell.execute_reply":"2023-10-22T01:03:32.943245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#X","metadata":{"execution":{"iopub.status.busy":"2023-10-22T01:03:32.945707Z","iopub.execute_input":"2023-10-22T01:03:32.946013Z","iopub.status.idle":"2023-10-22T01:03:32.956135Z","shell.execute_reply.started":"2023-10-22T01:03:32.945987Z","shell.execute_reply":"2023-10-22T01:03:32.955377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save some memory\ndel train\ngc.collect();","metadata":{"execution":{"iopub.status.busy":"2023-10-22T01:03:32.957484Z","iopub.execute_input":"2023-10-22T01:03:32.958399Z","iopub.status.idle":"2023-10-22T01:03:33.236398Z","shell.execute_reply.started":"2023-10-22T01:03:32.958347Z","shell.execute_reply":"2023-10-22T01:03:33.235385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predict","metadata":{}},{"cell_type":"code","source":"X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, random_state=2020)","metadata":{"execution":{"iopub.status.busy":"2023-10-22T01:03:33.237653Z","iopub.execute_input":"2023-10-22T01:03:33.238610Z","iopub.status.idle":"2023-10-22T01:03:43.188472Z","shell.execute_reply.started":"2023-10-22T01:03:33.238577Z","shell.execute_reply":"2023-10-22T01:03:43.187462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Randomized search","metadata":{}},{"cell_type":"code","source":"# save some memory\ndel X, y\ngc.collect();","metadata":{"execution":{"iopub.status.busy":"2023-10-22T01:03:43.189841Z","iopub.execute_input":"2023-10-22T01:03:43.190260Z","iopub.status.idle":"2023-10-22T01:03:43.501119Z","shell.execute_reply.started":"2023-10-22T01:03:43.190202Z","shell.execute_reply":"2023-10-22T01:03:43.500005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xgb_pipeline = XGBModel(X_train, y_train, preprocessor)\n#lgbm_pipeline = LGBMModel(X_train, y_train, preprocessor)\n#cb_pipeline = CatBoostModel(X_train, y_train, preprocessor)\n\ndef score_competition(X_valid, y_valid, xgb_pipeline, lgbm_pipeline, cb_pipeline):\n    validate_best_pipe(X_valid, y_valid, xgb_pipeline, lgbm_pipeline, cb_pipeline)\n\nscore_competition(X_valid, y_valid, xgb_pipeline, None, None)","metadata":{"execution":{"iopub.status.busy":"2023-10-22T01:03:43.502643Z","iopub.execute_input":"2023-10-22T01:03:43.503487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# xgb_pipeline","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"or","metadata":{}},{"cell_type":"markdown","source":"### Cross validation","metadata":{}},{"cell_type":"code","source":"# def mean_cross_val(model, X, y):\n#     score = cross_val_score(model, X, y,scoring=\"accuracy\", cv=2)\n#     mean = score.mean()\n#     return mean","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Basic model","metadata":{}},{"cell_type":"code","source":"# xgb_pipeline = XGBClassifier(n_estimators=10)\n# xgb_pipeline.fit(X_train,y_train)\n# # xgm.score(X_valid,y_valid)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X.head()","metadata":{"_uuid":"7a9a76b1-7995-4433-9cb3-167caa36e171","_cell_guid":"c2154e1c-60b5-4004-a614-ba9f67af6e0c","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X.shape","metadata":{"_uuid":"6ee0d2e9-2fa5-4bd9-bd92-4267c1ecd79e","_cell_guid":"4342389d-f67d-4319-b538-5095117b775b","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.ensemble import RandomForestClassifier\n# xgb_pipeline = RandomForestClassifier(n_estimators=1,\n#                                     min_samples_leaf=300,\n#                                     random_state=42,n_jobs=-1)\n\n# xgb_pipeline.fit(X, y)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save some memory\ndel X_train, X_valid, y_train, y_valid\n#del X, y\ngc.collect();","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Multiclass classification ","metadata":{}},{"cell_type":"code","source":"test = (pl.scan_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet\")\n          .with_columns(\n              (pl.col(\"timestamp\")\n                 .str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%Z\")\n                 .dt.hour()\n                 .alias(\"hour\")\n              )\n          )\n          .drop(\"timestamp\")\n          .collect()\n          .to_pandas()\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = make_features(test)\n\nX_test = test[features]\n\ntest[\"not_awake\"] = xgb_pipeline.predict_proba(X_test)[:,0]\ntest[\"awake\"]     = xgb_pipeline.predict_proba(X_test)[:,1]\n\n# smoothing the predictions\nsmoothing_length = 2 * 231\ntest[\"score\"]  = test[\"awake\"].rolling(smoothing_length,center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\")\ntest[\"smooth\"] = test[\"not_awake\"].rolling(smoothing_length,center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\")\n\n# re-binarize\ntest[\"smooth\"] = test[\"smooth\"].round()\n\ndef get_event(df):\n    lstCV = zip(df.series_id, df.smooth)\n    lstPOI = []\n    for (c, v), g in groupby(lstCV, lambda cv: (cv[0], cv[1] != 0 and not pd.isnull(cv[1]))):\n        llg = sum(1 for item in g)\n        if v is False:\n            lstPOI.extend([0] * llg)\n        else:\n            lstPOI.extend(['onset'] + (llg - 2) * [0] + ['wakeup'] if llg > 1 else [0])\n    return lstPOI\n\ntest[\"event\"] = get_event(test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Binary classification ","metadata":{}},{"cell_type":"code","source":"# test = make_features(test)\n\n# X_test = test[features]\n\n# test[\"score\"] = xgm.predict_proba(X_test)[:,1]\n\n# test[\"not_awake\"] = 1-test[\"score\"]\n# # exponential smoothing of the predictions\n# test[\"smooth\"] = test[\"not_awake\"].ewm(span = 100).mean()\n# # re-binarize\n# test[\"smooth\"] = test[\"smooth\"].round()\n\n# test[\"event\"] = get_event(test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save test predictions to file","metadata":{"_uuid":"97f71ac7-134a-4ee9-9031-f0a7bbf0eeff","_cell_guid":"461dd8bc-ada9-4e89-bb1b-e154e9bfd23b","trusted":true}},{"cell_type":"code","source":"sample_submission = test.loc[test[\"event\"] != 0][[\"series_id\",\"step\",\"event\",\"score\"]].copy().reset_index(drop=True).reset_index(names=\"row_id\")\nsample_submission.to_csv('submission.csv', index=False)","metadata":{"_uuid":"92b2d46d-289f-4b74-96c9-0e8a77e64f34","_cell_guid":"9146f2c1-1577-4a93-89a9-a7aa5f743541","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#sample_submission","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}