{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-03T22:28:05.370565Z","iopub.execute_input":"2022-10-03T22:28:05.371428Z","iopub.status.idle":"2022-10-03T22:28:05.408192Z","shell.execute_reply.started":"2022-10-03T22:28:05.371286Z","shell.execute_reply":"2022-10-03T22:28:05.406776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2022-10-03T22:28:10.468841Z","iopub.execute_input":"2022-10-03T22:28:10.469232Z","iopub.status.idle":"2022-10-03T22:28:11.112976Z","shell.execute_reply.started":"2022-10-03T22:28:10.469200Z","shell.execute_reply":"2022-10-03T22:28:11.111518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1. Load train and test datasets","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/train.csv')\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-03T22:28:11.115121Z","iopub.execute_input":"2022-10-03T22:28:11.115504Z","iopub.status.idle":"2022-10-03T22:28:14.958059Z","shell.execute_reply.started":"2022-10-03T22:28:11.115469Z","shell.execute_reply":"2022-10-03T22:28:14.956814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = train_df.pop('target')\n\n# check that we are dealing with binary classification\ny.unique()","metadata":{"execution":{"iopub.status.busy":"2022-10-03T22:28:14.959699Z","iopub.execute_input":"2022-10-03T22:28:14.960968Z","iopub.status.idle":"2022-10-03T22:28:14.981995Z","shell.execute_reply.started":"2022-10-03T22:28:14.960920Z","shell.execute_reply":"2022-10-03T22:28:14.980809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# the target value looks skewed!\n# so we need to be careful evaluating model results\n# i'm gonna use AUC_ROC and f1 scores here to evaluate model performance\n#y.value_counts().plot.bar()\nsns.countplot(x = y)","metadata":{"execution":{"iopub.status.busy":"2022-10-03T22:29:41.346966Z","iopub.execute_input":"2022-10-03T22:29:41.347365Z","iopub.status.idle":"2022-10-03T22:29:41.569749Z","shell.execute_reply.started":"2022-10-03T22:29:41.347334Z","shell.execute_reply":"2022-10-03T22:29:41.568444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/test.csv')\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-03T22:28:15.223049Z","iopub.execute_input":"2022-10-03T22:28:15.223391Z","iopub.status.idle":"2022-10-03T22:28:17.652877Z","shell.execute_reply.started":"2022-10-03T22:28:15.223360Z","shell.execute_reply":"2022-10-03T22:28:17.651742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Train df shape: {train_df.shape}')\nprint(f'Test df shape: {test_df.shape}')","metadata":{"execution":{"iopub.status.busy":"2022-10-03T22:28:27.202237Z","iopub.execute_input":"2022-10-03T22:28:27.203041Z","iopub.status.idle":"2022-10-03T22:28:27.210188Z","shell.execute_reply.started":"2022-10-03T22:28:27.202997Z","shell.execute_reply":"2022-10-03T22:28:27.208686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Explore the data","metadata":{}},{"cell_type":"code","source":"# check duplicated rows\ntrain_df.duplicated().sum(), test_df.duplicated().sum()","metadata":{"execution":{"iopub.status.busy":"2022-10-03T22:28:30.799293Z","iopub.execute_input":"2022-10-03T22:28:30.799840Z","iopub.status.idle":"2022-10-03T22:28:32.573199Z","shell.execute_reply.started":"2022-10-03T22:28:30.799787Z","shell.execute_reply":"2022-10-03T22:28:32.572053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from competition:\n# The data contains binary features (bin_*), \n# nominal features (nom_*), \n# ordinal features (ord_*) as well as (potentially cyclical) day (of the week) and month features. \n# The string ordinal features ord_{3-5} are lexically ordered according to string.ascii_letters.\n\ntrain_df.columns, test_df.columns","metadata":{"execution":{"iopub.status.busy":"2022-10-03T22:28:32.575216Z","iopub.execute_input":"2022-10-03T22:28:32.575588Z","iopub.status.idle":"2022-10-03T22:28:32.583006Z","shell.execute_reply.started":"2022-10-03T22:28:32.575558Z","shell.execute_reply":"2022-10-03T22:28:32.581861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# maybe some of the train samples are useless because they have too many nan values\nrow_nan_values = train_df.isna().sum(axis = 1).sort_values()\nrow_nan_values","metadata":{"execution":{"iopub.status.busy":"2022-10-03T22:28:32.584425Z","iopub.execute_input":"2022-10-03T22:28:32.584894Z","iopub.status.idle":"2022-10-03T22:28:33.079807Z","shell.execute_reply.started":"2022-10-03T22:28:32.584851Z","shell.execute_reply":"2022-10-03T22:28:33.078727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# how many samples have more than 5 nan values\nrow_nan_values_more_than_5 = row_nan_values[row_nan_values > 5]\nlen(row_nan_values_more_than_5), row_nan_values_more_than_5","metadata":{"execution":{"iopub.status.busy":"2022-10-03T22:28:33.081718Z","iopub.execute_input":"2022-10-03T22:28:33.082044Z","iopub.status.idle":"2022-10-03T22:28:33.091073Z","shell.execute_reply.started":"2022-10-03T22:28:33.082015Z","shell.execute_reply":"2022-10-03T22:28:33.089787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# not too much! \n# just remove these samples from train dataset\n\ntrain_df.drop(row_nan_values_more_than_5.index.tolist(), axis=0, inplace=True)\ny.drop(row_nan_values_more_than_5.index.tolist(), inplace=True)\ntrain_df.shape, y.shape","metadata":{"execution":{"iopub.status.busy":"2022-10-03T22:28:38.507742Z","iopub.execute_input":"2022-10-03T22:28:38.508173Z","iopub.status.idle":"2022-10-03T22:28:39.100634Z","shell.execute_reply.started":"2022-10-03T22:28:38.508140Z","shell.execute_reply":"2022-10-03T22:28:39.099436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# concatenate both dataframes to preprocess both train and test dataframes\n\ndf = pd.concat([train_df.drop(columns = ['id']), test_df.drop(columns = ['id'])]).reset_index(drop=True)\ndf.shape","metadata":{"execution":{"iopub.status.busy":"2022-10-03T22:28:39.102586Z","iopub.execute_input":"2022-10-03T22:28:39.102890Z","iopub.status.idle":"2022-10-03T22:28:41.059959Z","shell.execute_reply.started":"2022-10-03T22:28:39.102863Z","shell.execute_reply":"2022-10-03T22:28:41.058846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-03T22:28:41.061294Z","iopub.execute_input":"2022-10-03T22:28:41.061686Z","iopub.status.idle":"2022-10-03T22:28:41.091642Z","shell.execute_reply.started":"2022-10-03T22:28:41.061655Z","shell.execute_reply":"2022-10-03T22:28:41.090476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_features_by_prefix(df: pd.DataFrame, prefix: str) -> pd.DataFrame:\n    return df.loc[:, [col for col in df.columns if col.startswith(prefix)]]","metadata":{"execution":{"iopub.status.busy":"2022-10-03T22:28:41.093880Z","iopub.execute_input":"2022-10-03T22:28:41.094236Z","iopub.status.idle":"2022-10-03T22:28:41.101995Z","shell.execute_reply.started":"2022-10-03T22:28:41.094204Z","shell.execute_reply":"2022-10-03T22:28:41.100989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.1 Binary data","metadata":{}},{"cell_type":"code","source":"# Lets explore binary features first\n\nbin_features = get_features_by_prefix(df, 'bin_')\nbin_features.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-03T22:28:51.208554Z","iopub.execute_input":"2022-10-03T22:28:51.208977Z","iopub.status.idle":"2022-10-03T22:28:51.249573Z","shell.execute_reply.started":"2022-10-03T22:28:51.208941Z","shell.execute_reply":"2022-10-03T22:28:51.248379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bin_features.isna().sum() / len(bin_features) # in %","metadata":{"execution":{"iopub.status.busy":"2022-10-03T22:28:51.251489Z","iopub.execute_input":"2022-10-03T22:28:51.251873Z","iopub.status.idle":"2022-10-03T22:28:51.356569Z","shell.execute_reply.started":"2022-10-03T22:28:51.251840Z","shell.execute_reply":"2022-10-03T22:28:51.355315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check bin_3/bin_4 values \nbin_features['bin_3'].unique(), bin_features['bin_4'].unique()","metadata":{"execution":{"iopub.status.busy":"2022-10-03T22:28:51.358269Z","iopub.execute_input":"2022-10-03T22:28:51.358876Z","iopub.status.idle":"2022-10-03T22:28:51.441058Z","shell.execute_reply.started":"2022-10-03T22:28:51.358827Z","shell.execute_reply":"2022-10-03T22:28:51.440161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bin_features['bin_3'] = bin_features['bin_3'].map({'F': 0, 'T' : 1})\nbin_features['bin_4'] = bin_features['bin_4'].map({'N': 0, 'Y' : 1})\nbin_features.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-03T22:28:51.443396Z","iopub.execute_input":"2022-10-03T22:28:51.444023Z","iopub.status.idle":"2022-10-03T22:28:51.570115Z","shell.execute_reply.started":"2022-10-03T22:28:51.443985Z","shell.execute_reply":"2022-10-03T22:28:51.568830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1, len(bin_features.columns), figsize=(22, 6))\nfor i, col in enumerate(bin_features.columns):\n    plt.subplot(1, 5, i + 1)\n    sns.countplot(x = col, hue = y, data = train_df)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-03T22:28:51.571393Z","iopub.execute_input":"2022-10-03T22:28:51.571789Z","iopub.status.idle":"2022-10-03T22:28:53.815856Z","shell.execute_reply.started":"2022-10-03T22:28:51.571756Z","shell.execute_reply":"2022-10-03T22:28:53.814597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\n\nsns.heatmap(pd.concat([bin_features, y], axis = 1).corr())","metadata":{"execution":{"iopub.status.busy":"2022-10-03T22:28:53.817309Z","iopub.execute_input":"2022-10-03T22:28:53.817707Z","iopub.status.idle":"2022-10-03T22:28:54.327563Z","shell.execute_reply.started":"2022-10-03T22:28:53.817670Z","shell.execute_reply":"2022-10-03T22:28:54.325976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fill NaN values with most occurring class for each column\n#bin_features.fillna(bin_features.mode().iloc[0], inplace = True)\n#bin_features.head()\n\n# Filling NA values now is in the end of preprocessing!","metadata":{"execution":{"iopub.status.busy":"2022-10-03T22:28:54.329936Z","iopub.execute_input":"2022-10-03T22:28:54.330417Z","iopub.status.idle":"2022-10-03T22:28:54.338291Z","shell.execute_reply.started":"2022-10-03T22:28:54.330370Z","shell.execute_reply":"2022-10-03T22:28:54.336798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.2 Nominal Features","metadata":{}},{"cell_type":"code","source":"# Nominal features - categorical features without any order of precedence\n\nnom_features = get_features_by_prefix(df, 'nom_')\nnom_features.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-08-28T20:28:13.716297Z","iopub.execute_input":"2022-08-28T20:28:13.716793Z","iopub.status.idle":"2022-08-28T20:28:13.859462Z","shell.execute_reply.started":"2022-08-28T20:28:13.716754Z","shell.execute_reply":"2022-08-28T20:28:13.858207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nom_features.isna().sum() / len(nom_features) # in %","metadata":{"execution":{"iopub.status.busy":"2022-08-28T20:28:13.862062Z","iopub.execute_input":"2022-08-28T20:28:13.862510Z","iopub.status.idle":"2022-08-28T20:28:14.300075Z","shell.execute_reply.started":"2022-08-28T20:28:13.862454Z","shell.execute_reply":"2022-08-28T20:28:14.298858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in nom_features.columns:\n    print(f'Column: \"{col}\", num of values: {len(nom_features[col].unique())}')","metadata":{"execution":{"iopub.status.busy":"2022-08-28T20:28:14.302034Z","iopub.execute_input":"2022-08-28T20:28:14.302775Z","iopub.status.idle":"2022-08-28T20:28:14.874802Z","shell.execute_reply.started":"2022-08-28T20:28:14.302730Z","shell.execute_reply":"2022-08-28T20:28:14.873649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nom_columns = ['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4']\nfor col in nom_columns:\n    print(nom_features[col].dropna().unique())","metadata":{"execution":{"iopub.status.busy":"2022-08-28T20:28:18.830017Z","iopub.execute_input":"2022-08-28T20:28:18.830393Z","iopub.status.idle":"2022-08-28T20:28:19.546084Z","shell.execute_reply.started":"2022-08-28T20:28:18.830360Z","shell.execute_reply":"2022-08-28T20:28:19.544875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1, len(nom_columns), figsize=(22, 6))\nfor i, col in enumerate(nom_columns):\n    plt.subplot(1, 5, i + 1)\n    sns.countplot(x = col, hue = y, data = train_df)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-28T20:28:21.601767Z","iopub.execute_input":"2022-08-28T20:28:21.602230Z","iopub.status.idle":"2022-08-28T20:28:25.192242Z","shell.execute_reply.started":"2022-08-28T20:28:21.602195Z","shell.execute_reply":"2022-08-28T20:28:25.189608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.heatmap(pd.get_dummies(pd.concat([nom_features.loc[:, nom_features.columns.isin(nom_columns)], y], axis = 1)).corr())","metadata":{"execution":{"iopub.status.busy":"2022-08-28T20:28:31.991357Z","iopub.execute_input":"2022-08-28T20:28:31.991831Z","iopub.status.idle":"2022-08-28T20:28:35.221494Z","shell.execute_reply.started":"2022-08-28T20:28:31.991793Z","shell.execute_reply":"2022-08-28T20:28:35.220554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# lets check values distribution in train dataset\nother_columns = [c for c in nom_features.columns if c not in nom_columns]\nl = len(other_columns)\n \nfig, ax = plt.subplots(l, 1, figsize=(22, 20))\nfor i, col in enumerate(other_columns):\n    plt.subplot(l, 1, i + 1)\n    sns.countplot(x = col, data = train_df, order = train_df[col].value_counts().index)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-28T20:28:47.071367Z","iopub.execute_input":"2022-08-28T20:28:47.071885Z","iopub.status.idle":"2022-08-28T20:29:52.839328Z","shell.execute_reply.started":"2022-08-28T20:28:47.071844Z","shell.execute_reply":"2022-08-28T20:29:52.838143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Nominal features should be transformed into 'dummy' variables. But taking into consideration the number of unique values we will probably get a huge dataframe!","metadata":{}},{"cell_type":"code","source":"#nom_features.fillna('NA', inplace = True)\n#nom_features.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-08-28T20:30:17.274866Z","iopub.execute_input":"2022-08-28T20:30:17.275262Z","iopub.status.idle":"2022-08-28T20:30:17.280209Z","shell.execute_reply.started":"2022-08-28T20:30:17.275228Z","shell.execute_reply":"2022-08-28T20:30:17.279029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.3 Ordinal features","metadata":{}},{"cell_type":"code","source":"# Ordinal features - categorical features with order of precedence\n\nord_features = get_features_by_prefix(df, 'ord_')\nord_features.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-28T20:30:17.295847Z","iopub.execute_input":"2022-08-28T20:30:17.296853Z","iopub.status.idle":"2022-08-28T20:30:17.373032Z","shell.execute_reply.started":"2022-08-28T20:30:17.296815Z","shell.execute_reply":"2022-08-28T20:30:17.371664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ord_features.isna().sum() / len(ord_features) # in %","metadata":{"execution":{"iopub.status.busy":"2022-08-28T20:30:17.375231Z","iopub.execute_input":"2022-08-28T20:30:17.375835Z","iopub.status.idle":"2022-08-28T20:30:17.640745Z","shell.execute_reply.started":"2022-08-28T20:30:17.375801Z","shell.execute_reply":"2022-08-28T20:30:17.639641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in ord_features.columns:\n    print(f'Column: \"{col}\", num of values: {len(ord_features[col].unique())}')","metadata":{"execution":{"iopub.status.busy":"2022-08-28T20:30:17.643423Z","iopub.execute_input":"2022-08-28T20:30:17.643910Z","iopub.status.idle":"2022-08-28T20:30:17.922820Z","shell.execute_reply.started":"2022-08-28T20:30:17.643867Z","shell.execute_reply":"2022-08-28T20:30:17.921108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ord_features.ord_0.unique(), ord_features.ord_0.hist()","metadata":{"execution":{"iopub.status.busy":"2022-08-28T20:30:17.926136Z","iopub.execute_input":"2022-08-28T20:30:17.927351Z","iopub.status.idle":"2022-08-28T20:30:18.213665Z","shell.execute_reply.started":"2022-08-28T20:30:17.927299Z","shell.execute_reply":"2022-08-28T20:30:18.212262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# see gradation here: https://www.kaggle.com/progression\n# Novice, Contributor, Expert, Master, and Grandmaster.\nord_features.ord_1.unique()","metadata":{"execution":{"iopub.status.busy":"2022-08-28T20:30:18.215244Z","iopub.execute_input":"2022-08-28T20:30:18.215850Z","iopub.status.idle":"2022-08-28T20:30:18.285813Z","shell.execute_reply.started":"2022-08-28T20:30:18.215800Z","shell.execute_reply":"2022-08-28T20:30:18.284262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ord_features['ord_1'] = ord_features['ord_1'].map({'Novice' : 1, 'Contributor' : 2, 'Expert' : 3, 'Master' : 4, 'Grandmaster' : 5})\nord_features.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-28T20:30:18.287835Z","iopub.execute_input":"2022-08-28T20:30:18.288315Z","iopub.status.idle":"2022-08-28T20:30:18.413582Z","shell.execute_reply.started":"2022-08-28T20:30:18.288249Z","shell.execute_reply":"2022-08-28T20:30:18.412350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ord_features.ord_2.unique()","metadata":{"execution":{"iopub.status.busy":"2022-08-28T20:30:18.415409Z","iopub.execute_input":"2022-08-28T20:30:18.415953Z","iopub.status.idle":"2022-08-28T20:30:18.488370Z","shell.execute_reply.started":"2022-08-28T20:30:18.415911Z","shell.execute_reply":"2022-08-28T20:30:18.486778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ord_features['ord_2'] = ord_features['ord_2'].map({'Freezing' : 1, 'Cold' : 2, 'Warm' : 3, 'Hot' : 4, 'Boiling Hot' : 5, 'Lava Hot' : 6})\nord_features.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-28T20:30:18.489731Z","iopub.execute_input":"2022-08-28T20:30:18.490180Z","iopub.status.idle":"2022-08-28T20:30:18.613828Z","shell.execute_reply.started":"2022-08-28T20:30:18.490117Z","shell.execute_reply":"2022-08-28T20:30:18.613054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(sorted(ord_features.ord_3.dropna().unique()))\nprint(sorted(ord_features.ord_4.dropna().unique()))\nprint(sorted(ord_features.ord_5.dropna().unique()))","metadata":{"execution":{"iopub.status.busy":"2022-08-28T20:30:18.615119Z","iopub.execute_input":"2022-08-28T20:30:18.615624Z","iopub.status.idle":"2022-08-28T20:30:18.978024Z","shell.execute_reply.started":"2022-08-28T20:30:18.615593Z","shell.execute_reply":"2022-08-28T20:30:18.976790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The string ordinal features ord_{3-5} are lexically ordered according to string.ascii_letters.\n\nfor col in ['ord_3', 'ord_4', 'ord_5']:\n    vals = sorted(ord_features[col].dropna().unique())\n    d = dict(zip(vals, list(range(1, len(vals) + 1))))\n    \n    ord_features[col] = ord_features[col].map(d) \n\nord_features.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-28T20:30:18.981621Z","iopub.execute_input":"2022-08-28T20:30:18.982291Z","iopub.status.idle":"2022-08-28T20:30:19.599631Z","shell.execute_reply.started":"2022-08-28T20:30:18.982246Z","shell.execute_reply":"2022-08-28T20:30:19.598286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#ord_features.fillna(0, inplace = True)\n#ord_features.head() ","metadata":{"execution":{"iopub.status.busy":"2022-08-28T20:30:29.106567Z","iopub.execute_input":"2022-08-28T20:30:29.107092Z","iopub.status.idle":"2022-08-28T20:30:29.112610Z","shell.execute_reply.started":"2022-08-28T20:30:29.107043Z","shell.execute_reply":"2022-08-28T20:30:29.111507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.4 Day and month features","metadata":{}},{"cell_type":"code","source":"day_feature = get_features_by_prefix(df, 'day')\nmonth_feature = get_features_by_prefix(df, 'month')\ndm_features = pd.concat([day_feature, month_feature], axis = 1)\ndm_features.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-08-28T20:30:29.115644Z","iopub.execute_input":"2022-08-28T20:30:29.116455Z","iopub.status.idle":"2022-08-28T20:30:29.146220Z","shell.execute_reply.started":"2022-08-28T20:30:29.116388Z","shell.execute_reply":"2022-08-28T20:30:29.145135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dm_features.isna().sum() / len(dm_features) # in %","metadata":{"execution":{"iopub.status.busy":"2022-08-28T20:30:29.148182Z","iopub.execute_input":"2022-08-28T20:30:29.148588Z","iopub.status.idle":"2022-08-28T20:30:29.162055Z","shell.execute_reply.started":"2022-08-28T20:30:29.148553Z","shell.execute_reply":"2022-08-28T20:30:29.160924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# there is no explicit correlation between target and work days/weekends and seasons\n\nl = len(dm_features.columns)\n\nfig, ax = plt.subplots(l, 1, figsize = (10, 8))\nfor i, col in enumerate(dm_features.columns):\n    plt.subplot(l, 1, i + 1)\n    sns.countplot(x = col, hue = y, data = train_df)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-28T20:30:29.163243Z","iopub.execute_input":"2022-08-28T20:30:29.163836Z","iopub.status.idle":"2022-08-28T20:30:30.017852Z","shell.execute_reply.started":"2022-08-28T20:30:29.163789Z","shell.execute_reply":"2022-08-28T20:30:30.016545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.5 Final dataframe","metadata":{}},{"cell_type":"code","source":"import scipy.sparse as sparse\n\n# final dataframe is too big to be used as it is (trere are like A LOT of dummy features))\n# so here we transform this big dataframe into sparse matrix to save memory\n\nbin_features.fillna('NA', inplace = True)\nbin_sparse_mat_dummies = pd.get_dummies(bin_features, sparse = True, drop_first = True).sparse.to_coo()\n\ncat_df = pd.concat([nom_features, ord_features, dm_features], axis = 1).fillna('NA')\ncat_sparse_mat_dummies = pd.get_dummies(cat_df, sparse = True).sparse.to_coo()\n\nsparse_mat_dummies = sparse.hstack([bin_sparse_mat_dummies, cat_sparse_mat_dummies]).tocsr()","metadata":{"execution":{"iopub.status.busy":"2022-08-28T20:35:29.975993Z","iopub.execute_input":"2022-08-28T20:35:29.976493Z","iopub.status.idle":"2022-08-28T20:35:56.495123Z","shell.execute_reply.started":"2022-08-28T20:35:29.976455Z","shell.execute_reply":"2022-08-28T20:35:56.493753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = sparse_mat_dummies[:len(train_df), :]\nX_test = sparse_mat_dummies[len(train_df):, :]\nX.shape, X_test.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-28T20:36:15.382074Z","iopub.execute_input":"2022-08-28T20:36:15.382548Z","iopub.status.idle":"2022-08-28T20:36:15.675384Z","shell.execute_reply.started":"2022-08-28T20:36:15.382501Z","shell.execute_reply":"2022-08-28T20:36:15.673812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data split","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size = 0.1, random_state = 1)\nX_train.shape, X_val.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-28T20:36:15.677996Z","iopub.execute_input":"2022-08-28T20:36:15.678489Z","iopub.status.idle":"2022-08-28T20:36:15.967224Z","shell.execute_reply.started":"2022-08-28T20:36:15.678425Z","shell.execute_reply":"2022-08-28T20:36:15.965937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Models","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import RocCurveDisplay","metadata":{"execution":{"iopub.status.busy":"2022-08-28T20:36:15.968777Z","iopub.execute_input":"2022-08-28T20:36:15.970015Z","iopub.status.idle":"2022-08-28T20:36:15.976350Z","shell.execute_reply.started":"2022-08-28T20:36:15.969955Z","shell.execute_reply":"2022-08-28T20:36:15.975316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_sub(model, file_name):\n    submission = test_df.id.to_frame()\n    submission['target'] = model.predict_proba(X_test)[:, 1]\n    submission.to_csv(file_name + '.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2022-08-28T21:35:18.686106Z","iopub.execute_input":"2022-08-28T21:35:18.686666Z","iopub.status.idle":"2022-08-28T21:35:18.693741Z","shell.execute_reply.started":"2022-08-28T21:35:18.686622Z","shell.execute_reply":"2022-08-28T21:35:18.692809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Logistic regression","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\n\nlr = LogisticRegression(max_iter = 1000, random_state = 42, C = 0.3).fit(X_train, y_train)\nRocCurveDisplay.from_estimator(lr, X_val, y_val)\nsave_sub(lr, 'lr_sub')","metadata":{"execution":{"iopub.status.busy":"2022-08-28T21:57:09.294972Z","iopub.execute_input":"2022-08-28T21:57:09.295390Z","iopub.status.idle":"2022-08-28T21:57:58.143772Z","shell.execute_reply.started":"2022-08-28T21:57:09.295358Z","shell.execute_reply":"2022-08-28T21:57:58.142616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Random Forest","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\n\nrfc = RandomForestClassifier(n_estimators = 1500, max_depth = 25, max_features = 'log2', random_state = 42)\nrfc.fit(X_train, y_train)\nRocCurveDisplay.from_estimator(rfc, X_val, y_val)\nsave_sub(rfc, 'rfc_sub')","metadata":{"execution":{"iopub.status.busy":"2022-08-28T21:57:58.145657Z","iopub.execute_input":"2022-08-28T21:57:58.146069Z","iopub.status.idle":"2022-08-28T22:19:37.140651Z","shell.execute_reply.started":"2022-08-28T21:57:58.146039Z","shell.execute_reply":"2022-08-28T22:19:37.139490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### XGBoost","metadata":{}},{"cell_type":"code","source":"import xgboost as xgb\n\nxgb_model = xgb.XGBClassifier(n_estimators = 1500, max_depth = 25, random_state = 42).fit(X_train, y_train)\nRocCurveDisplay.from_estimator(xgb_model, X_val, y_val)\nsave_sub(xgb_model, 'xgb_sub')","metadata":{"execution":{"iopub.status.busy":"2022-08-28T22:19:37.142194Z","iopub.execute_input":"2022-08-28T22:19:37.142544Z","iopub.status.idle":"2022-08-28T23:01:10.383062Z","shell.execute_reply.started":"2022-08-28T22:19:37.142506Z","shell.execute_reply":"2022-08-28T23:01:10.381811Z"},"trusted":true},"execution_count":null,"outputs":[]}]}