{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"[Same competition but using logistic regression + sparse data](https://www.kaggle.com/code/maryiaznak/cat-in-the-dat-logistic-reg-rand-forest)","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-31T22:33:31.494122Z","iopub.execute_input":"2022-08-31T22:33:31.494466Z","iopub.status.idle":"2022-08-31T22:33:31.503794Z","shell.execute_reply.started":"2022-08-31T22:33:31.494438Z","shell.execute_reply":"2022-08-31T22:33:31.502610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:33:37.227965Z","iopub.execute_input":"2022-08-31T22:33:37.228287Z","iopub.status.idle":"2022-08-31T22:33:37.232834Z","shell.execute_reply.started":"2022-08-31T22:33:37.228263Z","shell.execute_reply":"2022-08-31T22:33:37.231809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1. Load train and test datasets","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/train.csv')\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:33:37.238830Z","iopub.execute_input":"2022-08-31T22:33:37.239386Z","iopub.status.idle":"2022-08-31T22:33:39.592720Z","shell.execute_reply.started":"2022-08-31T22:33:37.239352Z","shell.execute_reply":"2022-08-31T22:33:39.591697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = train_df.pop('target')\n\n# check that we are dealing with binary classification\ny.unique()","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:33:39.594405Z","iopub.execute_input":"2022-08-31T22:33:39.594659Z","iopub.status.idle":"2022-08-31T22:33:39.613221Z","shell.execute_reply.started":"2022-08-31T22:33:39.594636Z","shell.execute_reply":"2022-08-31T22:33:39.611825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# the target value looks skewed!\n# so we need to be careful evaluating model results\n# i'm gonna use AUC_ROC and f1 scores here to evaluate model performance\nsns.countplot(x = y)","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:33:39.615122Z","iopub.execute_input":"2022-08-31T22:33:39.615824Z","iopub.status.idle":"2022-08-31T22:33:39.849349Z","shell.execute_reply.started":"2022-08-31T22:33:39.615746Z","shell.execute_reply":"2022-08-31T22:33:39.847879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/test.csv')\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:33:39.851823Z","iopub.execute_input":"2022-08-31T22:33:39.852542Z","iopub.status.idle":"2022-08-31T22:33:41.363645Z","shell.execute_reply.started":"2022-08-31T22:33:39.852515Z","shell.execute_reply":"2022-08-31T22:33:41.361626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Train df shape: {train_df.shape}')\nprint(f'Test df shape: {test_df.shape}')","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:33:41.365190Z","iopub.execute_input":"2022-08-31T22:33:41.365499Z","iopub.status.idle":"2022-08-31T22:33:41.371398Z","shell.execute_reply.started":"2022-08-31T22:33:41.365472Z","shell.execute_reply":"2022-08-31T22:33:41.370350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Explore the data","metadata":{}},{"cell_type":"code","source":"# check duplicated rows\ntrain_df.duplicated().sum(), test_df.duplicated().sum()","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:33:41.373036Z","iopub.execute_input":"2022-08-31T22:33:41.373377Z","iopub.status.idle":"2022-08-31T22:33:43.120145Z","shell.execute_reply.started":"2022-08-31T22:33:41.373344Z","shell.execute_reply":"2022-08-31T22:33:43.118726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from competition:\n# The data contains binary features (bin_*), \n# nominal features (nom_*), \n# ordinal features (ord_*) as well as (potentially cyclical) day (of the week) and month features. \n# The string ordinal features ord_{3-5} are lexically ordered according to string.ascii_letters.\n\ntrain_df.columns, test_df.columns","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:33:43.121492Z","iopub.execute_input":"2022-08-31T22:33:43.121808Z","iopub.status.idle":"2022-08-31T22:33:43.128466Z","shell.execute_reply.started":"2022-08-31T22:33:43.121748Z","shell.execute_reply":"2022-08-31T22:33:43.127490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# concatenate both dataframes to preprocess both train and test dataframes\n\ndf = pd.concat([train_df.drop(columns = ['id']), test_df.drop(columns = ['id'])]).reset_index(drop=True)\ndf.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:33:43.129716Z","iopub.execute_input":"2022-08-31T22:33:43.130604Z","iopub.status.idle":"2022-08-31T22:33:44.317012Z","shell.execute_reply.started":"2022-08-31T22:33:43.130569Z","shell.execute_reply":"2022-08-31T22:33:44.316151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:33:44.318077Z","iopub.execute_input":"2022-08-31T22:33:44.318928Z","iopub.status.idle":"2022-08-31T22:33:44.346191Z","shell.execute_reply.started":"2022-08-31T22:33:44.318884Z","shell.execute_reply":"2022-08-31T22:33:44.344817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_features_by_prefix(df: pd.DataFrame, prefix: str) -> pd.DataFrame:\n    return df.loc[:, [col for col in df.columns if col.startswith(prefix)]]","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:33:44.351854Z","iopub.execute_input":"2022-08-31T22:33:44.352250Z","iopub.status.idle":"2022-08-31T22:33:44.357656Z","shell.execute_reply.started":"2022-08-31T22:33:44.352217Z","shell.execute_reply":"2022-08-31T22:33:44.356943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.1 Binary data","metadata":{}},{"cell_type":"code","source":"# Lets explore binary features first\n\nbin_features = get_features_by_prefix(df, 'bin_')\nbin_features.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:33:44.358856Z","iopub.execute_input":"2022-08-31T22:33:44.359260Z","iopub.status.idle":"2022-08-31T22:33:44.399968Z","shell.execute_reply.started":"2022-08-31T22:33:44.359234Z","shell.execute_reply":"2022-08-31T22:33:44.398925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bin_features.isna().sum() / len(bin_features) # in %","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:33:44.401306Z","iopub.execute_input":"2022-08-31T22:33:44.404962Z","iopub.status.idle":"2022-08-31T22:33:44.496168Z","shell.execute_reply.started":"2022-08-31T22:33:44.404908Z","shell.execute_reply":"2022-08-31T22:33:44.495463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check bin_3/bin_4 values \nbin_features['bin_3'].unique(), bin_features['bin_4'].unique()","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:33:44.497162Z","iopub.execute_input":"2022-08-31T22:33:44.498043Z","iopub.status.idle":"2022-08-31T22:33:44.567479Z","shell.execute_reply.started":"2022-08-31T22:33:44.498017Z","shell.execute_reply":"2022-08-31T22:33:44.566546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bin_features['bin_3'] = bin_features['bin_3'].map({'F': 0, 'T' : 1})\nbin_features['bin_4'] = bin_features['bin_4'].map({'N': 0, 'Y' : 1})\nbin_features.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:33:44.568473Z","iopub.execute_input":"2022-08-31T22:33:44.569242Z","iopub.status.idle":"2022-08-31T22:33:44.671561Z","shell.execute_reply.started":"2022-08-31T22:33:44.569215Z","shell.execute_reply":"2022-08-31T22:33:44.670354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1, len(bin_features.columns), figsize=(22, 6))\nfor i, col in enumerate(bin_features.columns):\n    plt.subplot(1, 5, i + 1)\n    sns.countplot(x = col, hue = y, data = train_df)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:33:44.673004Z","iopub.execute_input":"2022-08-31T22:33:44.673346Z","iopub.status.idle":"2022-08-31T22:33:46.835327Z","shell.execute_reply.started":"2022-08-31T22:33:44.673313Z","shell.execute_reply":"2022-08-31T22:33:46.834352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\n\nsns.heatmap(pd.concat([bin_features, y], axis = 1).corr())","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:33:46.836938Z","iopub.execute_input":"2022-08-31T22:33:46.837304Z","iopub.status.idle":"2022-08-31T22:33:47.171158Z","shell.execute_reply.started":"2022-08-31T22:33:46.837272Z","shell.execute_reply":"2022-08-31T22:33:47.169934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fill NaN values with most occurring class for each column\n#bin_features.fillna(bin_features.mode().iloc[0], inplace = True)\n#bin_features.head()\n\n# Filling NA values now is in the end of preprocessing!","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:33:47.172678Z","iopub.execute_input":"2022-08-31T22:33:47.173527Z","iopub.status.idle":"2022-08-31T22:33:47.177542Z","shell.execute_reply.started":"2022-08-31T22:33:47.173501Z","shell.execute_reply":"2022-08-31T22:33:47.176405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.2 Nominal Features","metadata":{}},{"cell_type":"code","source":"# Nominal features - categorical features without any order of precedence\n\nnom_features = get_features_by_prefix(df, 'nom_')\nnom_features.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:33:47.179010Z","iopub.execute_input":"2022-08-31T22:33:47.179245Z","iopub.status.idle":"2022-08-31T22:33:47.278770Z","shell.execute_reply.started":"2022-08-31T22:33:47.179223Z","shell.execute_reply":"2022-08-31T22:33:47.277695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nom_features.isna().sum() / len(nom_features) # in %","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:33:47.280387Z","iopub.execute_input":"2022-08-31T22:33:47.280825Z","iopub.status.idle":"2022-08-31T22:33:47.666668Z","shell.execute_reply.started":"2022-08-31T22:33:47.280791Z","shell.execute_reply":"2022-08-31T22:33:47.665666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in nom_features.columns:\n    print(f'Column: \"{col}\", num of values: {len(nom_features[col].unique())}')","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:33:47.668435Z","iopub.execute_input":"2022-08-31T22:33:47.668821Z","iopub.status.idle":"2022-08-31T22:33:48.211556Z","shell.execute_reply.started":"2022-08-31T22:33:47.668788Z","shell.execute_reply":"2022-08-31T22:33:48.210089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nom_columns = ['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4']\nfor col in nom_columns:\n    print(nom_features[col].dropna().unique())","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:33:48.212536Z","iopub.execute_input":"2022-08-31T22:33:48.212849Z","iopub.status.idle":"2022-08-31T22:33:48.915475Z","shell.execute_reply.started":"2022-08-31T22:33:48.212825Z","shell.execute_reply":"2022-08-31T22:33:48.913201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1, len(nom_columns), figsize=(22, 6))\nfor i, col in enumerate(nom_columns):\n    plt.subplot(1, len(nom_columns), i + 1)\n    sns.countplot(x = col, hue = y, data = train_df)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:33:48.918300Z","iopub.execute_input":"2022-08-31T22:33:48.918858Z","iopub.status.idle":"2022-08-31T22:33:52.226864Z","shell.execute_reply.started":"2022-08-31T22:33:48.918788Z","shell.execute_reply":"2022-08-31T22:33:52.225839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.heatmap(pd.get_dummies(pd.concat([nom_features.loc[:, nom_features.columns.isin(nom_columns)], y], axis = 1)).corr())","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:33:52.228019Z","iopub.execute_input":"2022-08-31T22:33:52.228630Z","iopub.status.idle":"2022-08-31T22:33:55.551776Z","shell.execute_reply.started":"2022-08-31T22:33:52.228598Z","shell.execute_reply":"2022-08-31T22:33:55.550268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# lets check values distribution in train dataset\nother_columns = [c for c in nom_features.columns if c not in nom_columns]\nl = len(other_columns)\n \nfig, ax = plt.subplots(l, 1, figsize=(22, 20))\nfor i, col in enumerate(other_columns):\n    plt.subplot(l, 1, i + 1)\n    sns.countplot(x = col, data = train_df, order = train_df[col].value_counts().index)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:33:55.553678Z","iopub.execute_input":"2022-08-31T22:33:55.554584Z","iopub.status.idle":"2022-08-31T22:34:45.120854Z","shell.execute_reply.started":"2022-08-31T22:33:55.554541Z","shell.execute_reply":"2022-08-31T22:34:45.119705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Nominal features should be transformed into 'dummy' variables. But taking into consideration the number of unique values we will probably get a huge dataframe!","metadata":{}},{"cell_type":"code","source":"#nom_features.fillna('NA', inplace = True)\n#nom_features.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:34:45.122169Z","iopub.execute_input":"2022-08-31T22:34:45.122428Z","iopub.status.idle":"2022-08-31T22:34:45.126542Z","shell.execute_reply.started":"2022-08-31T22:34:45.122396Z","shell.execute_reply":"2022-08-31T22:34:45.125779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.3 Ordinal features","metadata":{}},{"cell_type":"code","source":"# Ordinal features - categorical features with order of precedence\n\nord_features = get_features_by_prefix(df, 'ord_')\nord_features.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:34:45.127505Z","iopub.execute_input":"2022-08-31T22:34:45.127711Z","iopub.status.idle":"2022-08-31T22:34:45.201656Z","shell.execute_reply.started":"2022-08-31T22:34:45.127688Z","shell.execute_reply":"2022-08-31T22:34:45.200733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ord_features.isna().sum() / len(ord_features) # in %","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:34:45.202878Z","iopub.execute_input":"2022-08-31T22:34:45.203144Z","iopub.status.idle":"2022-08-31T22:34:45.383781Z","shell.execute_reply.started":"2022-08-31T22:34:45.203120Z","shell.execute_reply":"2022-08-31T22:34:45.383067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in ord_features.columns:\n    print(f'Column: \"{col}\", num of values: {len(ord_features[col].unique())}')","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:34:45.389378Z","iopub.execute_input":"2022-08-31T22:34:45.390049Z","iopub.status.idle":"2022-08-31T22:34:45.609747Z","shell.execute_reply.started":"2022-08-31T22:34:45.390020Z","shell.execute_reply":"2022-08-31T22:34:45.608737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(len(ord_features.columns), 1, figsize=(20, 20))\nfor i, col in enumerate(ord_features.columns):\n    plt.subplot(len(ord_features.columns), 1, i + 1)\n    sns.countplot(x = col, hue = y, data = train_df)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:34:45.611290Z","iopub.execute_input":"2022-08-31T22:34:45.611714Z","iopub.status.idle":"2022-08-31T22:34:51.467192Z","shell.execute_reply.started":"2022-08-31T22:34:45.611673Z","shell.execute_reply":"2022-08-31T22:34:51.466099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(sorted(ord_features.ord_3.dropna().unique()))\nprint(sorted(ord_features.ord_4.dropna().unique()))\nprint(sorted(ord_features.ord_5.dropna().unique()))","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:34:51.469627Z","iopub.execute_input":"2022-08-31T22:34:51.470560Z","iopub.status.idle":"2022-08-31T22:34:51.772400Z","shell.execute_reply.started":"2022-08-31T22:34:51.470533Z","shell.execute_reply":"2022-08-31T22:34:51.771232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.4 Day and month features","metadata":{}},{"cell_type":"code","source":"day_feature = get_features_by_prefix(df, 'day')\nmonth_feature = get_features_by_prefix(df, 'month')\ndm_features = pd.concat([day_feature, month_feature], axis = 1)\ndm_features.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:34:51.774188Z","iopub.execute_input":"2022-08-31T22:34:51.774554Z","iopub.status.idle":"2022-08-31T22:34:51.800624Z","shell.execute_reply.started":"2022-08-31T22:34:51.774518Z","shell.execute_reply":"2022-08-31T22:34:51.799226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dm_features.isna().sum() / len(dm_features) # in %","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:34:51.801928Z","iopub.execute_input":"2022-08-31T22:34:51.802204Z","iopub.status.idle":"2022-08-31T22:34:51.813010Z","shell.execute_reply.started":"2022-08-31T22:34:51.802179Z","shell.execute_reply":"2022-08-31T22:34:51.812149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# there is no explicit correlation between target and work days/weekends and seasons\n\nl = len(dm_features.columns)\n\nfig, ax = plt.subplots(l, 1, figsize = (10, 8))\nfor i, col in enumerate(dm_features.columns):\n    plt.subplot(l, 1, i + 1)\n    sns.countplot(x = col, hue = y, data = train_df)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:34:51.814801Z","iopub.execute_input":"2022-08-31T22:34:51.815216Z","iopub.status.idle":"2022-08-31T22:34:52.665865Z","shell.execute_reply.started":"2022-08-31T22:34:51.815178Z","shell.execute_reply":"2022-08-31T22:34:52.664920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.5 Final dataframe","metadata":{}},{"cell_type":"code","source":"from sklearn import preprocessing","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:34:52.667362Z","iopub.execute_input":"2022-08-31T22:34:52.667977Z","iopub.status.idle":"2022-08-31T22:34:52.786391Z","shell.execute_reply.started":"2022-08-31T22:34:52.667941Z","shell.execute_reply":"2022-08-31T22:34:52.785543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_df = pd.concat([bin_features, nom_features, ord_features, dm_features], axis = 1).fillna('NA')\n\nfor col in final_df.columns:\n    lbl_enc = preprocessing.LabelEncoder()\n    final_df[col] = lbl_enc.fit_transform(final_df.loc[:, col].astype(str).values)\n    \nfinal_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:34:52.787356Z","iopub.execute_input":"2022-08-31T22:34:52.787604Z","iopub.status.idle":"2022-08-31T22:35:02.240349Z","shell.execute_reply.started":"2022-08-31T22:34:52.787582Z","shell.execute_reply":"2022-08-31T22:35:02.238842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = final_df.iloc[:len(train_df), :]\nX_test = final_df.iloc[len(train_df):, :]\nX.shape, X_test.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:35:02.241527Z","iopub.execute_input":"2022-08-31T22:35:02.241845Z","iopub.status.idle":"2022-08-31T22:35:02.249009Z","shell.execute_reply.started":"2022-08-31T22:35:02.241814Z","shell.execute_reply":"2022-08-31T22:35:02.247806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size = 0.1, random_state = 1)\nX_train.shape, X_val.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:35:02.250458Z","iopub.execute_input":"2022-08-31T22:35:02.250716Z","iopub.status.idle":"2022-08-31T22:35:02.529644Z","shell.execute_reply.started":"2022-08-31T22:35:02.250692Z","shell.execute_reply":"2022-08-31T22:35:02.528371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_arr = [X_train[c].values for c in X_train.columns]\nX_val_arr = [X_val[c] for c in X_val.columns]\nX_test_arr = [X_test[c] for c in X_test.columns]","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:35:02.530712Z","iopub.execute_input":"2022-08-31T22:35:02.531011Z","iopub.status.idle":"2022-08-31T22:35:02.541009Z","shell.execute_reply.started":"2022-08-31T22:35:02.530986Z","shell.execute_reply":"2022-08-31T22:35:02.539328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Model","metadata":{}},{"cell_type":"markdown","source":"### Embedding layers","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras import Input\nfrom tensorflow.keras.layers import Embedding, Dense, Dropout, Concatenate, Flatten\nfrom tensorflow.keras import callbacks","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:35:02.542441Z","iopub.execute_input":"2022-08-31T22:35:02.543359Z","iopub.status.idle":"2022-08-31T22:35:11.864886Z","shell.execute_reply.started":"2022-08-31T22:35:02.543307Z","shell.execute_reply":"2022-08-31T22:35:11.863693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from math import log2","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:42:09.732366Z","iopub.execute_input":"2022-08-31T22:42:09.733087Z","iopub.status.idle":"2022-08-31T22:42:09.737556Z","shell.execute_reply.started":"2022-08-31T22:42:09.733060Z","shell.execute_reply":"2022-08-31T22:42:09.736836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# prepare each input head\ninput_layers = []\noutput_layers = []\nfor col in final_df.columns:\n    \n    n_labels = len(final_df.loc[:, col].unique()) + 1\n    n_outputs = n_labels // 2 #int(log2(n_labels))\n    \n    input_layer = Input(shape=(1,))\n    emb_layer = Embedding(n_labels, n_outputs, name = col)(input_layer)\n    \n    input_layers.append(input_layer)\n    output_layers.append(Flatten()(emb_layer))","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:42:09.741404Z","iopub.execute_input":"2022-08-31T22:42:09.742340Z","iopub.status.idle":"2022-08-31T22:42:10.209279Z","shell.execute_reply.started":"2022-08-31T22:42:09.742311Z","shell.execute_reply":"2022-08-31T22:42:10.208343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Final Model","metadata":{}},{"cell_type":"code","source":"x = Concatenate()(output_layers)\nx = Dropout(0.2)(x)\nx = Dense(300, activation = 'relu')(x)\nx = Dropout(0.2)(x)\nx = Dense(200, activation = 'relu')(x)\nx = Dropout(0.1)(x)\nx = Dense(8, activation = 'relu')(x)\noutput = Dense(1, activation = 'sigmoid')(x)\nmodel = Model(inputs = input_layers, outputs = output)\n\nopt = tf.keras.optimizers.Adam(learning_rate = 1e-5)\nmodel.compile(optimizer = opt, loss = 'binary_crossentropy', metrics = ['acc', 'AUC'])\n#model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:42:10.210594Z","iopub.execute_input":"2022-08-31T22:42:10.210859Z","iopub.status.idle":"2022-08-31T22:42:10.287277Z","shell.execute_reply.started":"2022-08-31T22:42:10.210835Z","shell.execute_reply":"2022-08-31T22:42:10.286284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"es = callbacks.EarlyStopping(monitor='val_auc', mode = 'max', patience = 5, verbose = 1, restore_best_weights = True)\nrlr = callbacks.ReduceLROnPlateau(monitor='val_auc', mode = 'max', factor = 0.1, patience = 3, verbose = 1)","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:42:10.288625Z","iopub.execute_input":"2022-08-31T22:42:10.288994Z","iopub.status.idle":"2022-08-31T22:42:10.294401Z","shell.execute_reply.started":"2022-08-31T22:42:10.288961Z","shell.execute_reply":"2022-08-31T22:42:10.293485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(X_train_arr, y_train.values, epochs = 100, batch_size = 1000,\n                   validation_data = (X_val_arr, y_val.values), callbacks = [es, rlr])","metadata":{"execution":{"iopub.status.busy":"2022-08-31T22:42:10.296260Z","iopub.execute_input":"2022-08-31T22:42:10.297318Z","iopub.status.idle":"2022-08-31T23:09:06.336529Z","shell.execute_reply.started":"2022-08-31T22:42:10.297279Z","shell.execute_reply":"2022-08-31T23:09:06.335601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss = history.history['loss']\nval_loss = history.history['val_loss']\n\nacc = history.history['acc']\nval_acc = history.history['val_acc']\n\nauc = history.history['auc']\nval_auc = history.history['val_auc']\n\nepochs = range(1, len(loss) + 1)\n\nplt.figure(figsize=(20, 5))\n\n#loss\nplt.subplot(1,3,1)\nplt.plot(epochs, loss, 'bo', label = 'Trainig loss')\nplt.plot(epochs, val_loss, 'r', label = 'Validation loss')\nplt.legend()\n\n#accuracy\nplt.subplot(1,3,2)\nplt.plot(epochs, acc, 'bo', label = 'Training accuracy')\nplt.plot(epochs, val_acc, 'r', label = 'Validation accuracy')\nplt.legend()\n\n#auc\nplt.subplot(1,3,3)\nplt.plot(epochs, auc, 'bo', label = 'Trainig auc')\nplt.plot(epochs, val_auc, 'r', label = 'Validation auc')\nplt.legend()\n \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-31T23:10:16.228710Z","iopub.execute_input":"2022-08-31T23:10:16.229040Z","iopub.status.idle":"2022-08-31T23:10:36.021481Z","shell.execute_reply.started":"2022-08-31T23:10:16.229016Z","shell.execute_reply":"2022-08-31T23:10:36.020117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"submission = test_df.id.to_frame()\nsubmission['target'] = [pred[0] for pred in model.predict(X_test_arr)]\nsubmission.to_csv('sub.csv', index = False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}