{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-12T10:04:11.258112Z","iopub.execute_input":"2022-12-12T10:04:11.258675Z","iopub.status.idle":"2022-12-12T10:04:11.289841Z","shell.execute_reply.started":"2022-12-12T10:04:11.258573Z","shell.execute_reply":"2022-12-12T10:04:11.288564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Thank you Abhishek Thakur for your book Approaching (Almost) Any Machine Learning Problem!","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"../input/cat-in-the-dat-ii/train.csv\")\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T10:04:11.292090Z","iopub.execute_input":"2022-12-12T10:04:11.293114Z","iopub.status.idle":"2022-12-12T10:04:14.933292Z","shell.execute_reply.started":"2022-12-12T10:04:11.293072Z","shell.execute_reply":"2022-12-12T10:04:14.932425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T10:04:14.934532Z","iopub.execute_input":"2022-12-12T10:04:14.935151Z","iopub.status.idle":"2022-12-12T10:04:15.397521Z","shell.execute_reply.started":"2022-12-12T10:04:14.935112Z","shell.execute_reply":"2022-12-12T10:04:15.396151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-12T10:04:15.400624Z","iopub.execute_input":"2022-12-12T10:04:15.401069Z","iopub.status.idle":"2022-12-12T10:04:15.408597Z","shell.execute_reply.started":"2022-12-12T10:04:15.401028Z","shell.execute_reply":"2022-12-12T10:04:15.407447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2022-12-12T10:04:15.410187Z","iopub.execute_input":"2022-12-12T10:04:15.410658Z","iopub.status.idle":"2022-12-12T10:04:16.021209Z","shell.execute_reply.started":"2022-12-12T10:04:15.410617Z","shell.execute_reply":"2022-12-12T10:04:16.020357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"matplotlib.rc('xtick', labelsize=20)\nmatplotlib.rc('ytick', labelsize=20)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T10:04:16.022329Z","iopub.execute_input":"2022-12-12T10:04:16.023304Z","iopub.status.idle":"2022-12-12T10:04:16.028270Z","shell.execute_reply.started":"2022-12-12T10:04:16.023272Z","shell.execute_reply":"2022-12-12T10:04:16.027025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"b = sns.countplot(x='target', data=train)\nb.set_xlabel(\"target\", fontsize=20)\nb.set_ylabel(\"count\", fontsize=20)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T10:04:16.029628Z","iopub.execute_input":"2022-12-12T10:04:16.030427Z","iopub.status.idle":"2022-12-12T10:04:16.240842Z","shell.execute_reply.started":"2022-12-12T10:04:16.030387Z","shell.execute_reply":"2022-12-12T10:04:16.239656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see, the target is skewed, that why for evaluation we will use AUC.","metadata":{}},{"cell_type":"markdown","source":"Let's see our categories. We have the folloving variables:\n* 5 binary\n* 10 nominal\n* 6 ordinal\n* 2 cyclic\n* 1 target\n","metadata":{}},{"cell_type":"markdown","source":"Let's have a look at ord_2 feature in the train dataset. It consist of sic different categories: \n* freezing\n* cold\n* warm\n* hot\n* lava hot\n* boiling hot","metadata":{}},{"cell_type":"code","source":"mapping = {\"Freezing\":0, \"Warm\":1, \"Cold\":2, \"Boiling Hot\":3, \"Hot\":4, \"Lava Hot\":5}","metadata":{"execution":{"iopub.status.busy":"2022-12-12T10:04:16.242446Z","iopub.execute_input":"2022-12-12T10:04:16.244363Z","iopub.status.idle":"2022-12-12T10:04:16.249767Z","shell.execute_reply.started":"2022-12-12T10:04:16.244300Z","shell.execute_reply":"2022-12-12T10:04:16.248992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.loc[:, \"ord_2\"] = train.ord_2.map(mapping)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T10:04:16.251427Z","iopub.execute_input":"2022-12-12T10:04:16.252162Z","iopub.status.idle":"2022-12-12T10:04:16.460040Z","shell.execute_reply.started":"2022-12-12T10:04:16.252118Z","shell.execute_reply":"2022-12-12T10:04:16.459103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.ord_2.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T10:04:16.465081Z","iopub.execute_input":"2022-12-12T10:04:16.465451Z","iopub.status.idle":"2022-12-12T10:04:16.487905Z","shell.execute_reply.started":"2022-12-12T10:04:16.465420Z","shell.execute_reply":"2022-12-12T10:04:16.486609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Label encoding from scikit-learn\nfrom sklearn import preprocessing","metadata":{"execution":{"iopub.status.busy":"2022-12-12T10:04:16.489341Z","iopub.execute_input":"2022-12-12T10:04:16.490073Z","iopub.status.idle":"2022-12-12T10:04:16.556935Z","shell.execute_reply.started":"2022-12-12T10:04:16.490022Z","shell.execute_reply":"2022-12-12T10:04:16.555730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.groupby([\"ord_2\"])[\"id\"].count()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T10:04:16.558416Z","iopub.execute_input":"2022-12-12T10:04:16.558779Z","iopub.status.idle":"2022-12-12T10:04:16.589460Z","shell.execute_reply.started":"2022-12-12T10:04:16.558747Z","shell.execute_reply":"2022-12-12T10:04:16.588242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read the data\ntrain = pd.read_csv(\"../input/cat-in-the-dat-ii/train.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-12-12T10:04:16.591263Z","iopub.execute_input":"2022-12-12T10:04:16.592047Z","iopub.status.idle":"2022-12-12T10:04:19.203716Z","shell.execute_reply.started":"2022-12-12T10:04:16.592003Z","shell.execute_reply":"2022-12-12T10:04:19.202295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T10:04:19.205341Z","iopub.execute_input":"2022-12-12T10:04:19.205799Z","iopub.status.idle":"2022-12-12T10:04:19.235230Z","shell.execute_reply.started":"2022-12-12T10:04:19.205755Z","shell.execute_reply":"2022-12-12T10:04:19.233973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# find number of NaN values in ord_2 column\ntrain[\"ord_1\"].isna().count()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T10:04:19.236810Z","iopub.execute_input":"2022-12-12T10:04:19.237154Z","iopub.status.idle":"2022-12-12T10:04:19.272954Z","shell.execute_reply.started":"2022-12-12T10:04:19.237123Z","shell.execute_reply":"2022-12-12T10:04:19.271817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fill NaN values in ord_2 column\ntrain.loc[:, \"ord_2\"] = train.ord_2.fillna(\"NONE\")","metadata":{"execution":{"iopub.status.busy":"2022-12-12T10:04:19.274571Z","iopub.execute_input":"2022-12-12T10:04:19.275376Z","iopub.status.idle":"2022-12-12T10:04:19.326742Z","shell.execute_reply.started":"2022-12-12T10:04:19.275332Z","shell.execute_reply":"2022-12-12T10:04:19.325904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# initialize LabelEncoder\nfrom sklearn import preprocessing\nlbl_enc = preprocessing.LabelEncoder()\n# fit label encoder and transform values on ord_2 column\ntrain.loc[:, \"ord_2\"] = lbl_enc.fit_transform(train.ord_2.values)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T10:04:19.328214Z","iopub.execute_input":"2022-12-12T10:04:19.329407Z","iopub.status.idle":"2022-12-12T10:04:19.628617Z","shell.execute_reply.started":"2022-12-12T10:04:19.329361Z","shell.execute_reply":"2022-12-12T10:04:19.627711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.ord_2.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T10:04:19.630054Z","iopub.execute_input":"2022-12-12T10:04:19.631303Z","iopub.status.idle":"2022-12-12T10:04:19.644911Z","shell.execute_reply.started":"2022-12-12T10:04:19.631257Z","shell.execute_reply":"2022-12-12T10:04:19.643935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Label Encoding","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn import preprocessing\n\n# read training data\ntrain = pd.read_csv(\"../input/cat-in-the-dat-ii/train.csv\")\n\n# read test data\ntest = pd. read_csv(\"../input/cat-in-the-dat-ii/test.csv\")\n\n# create a dummy target column for test data\n# since this column does not exists\ntest.loc[:, \"target\"] = -1\n\n# concatenate both training and test data\ndata = pd.concat([train, test]).reset_index(drop=True)\n\n# make a list of features we are interested in\n# id and target is something we should not encode\nfeatures = [x for x in train.columns if x not in [\"id\", \"target\"]]\n\n# loop over the features list\nfor feat in features:\n    # create a new instance of LabelEncoder for each feature\n    lbl_enc = preprocessing.LabelEncoder()\n    \n    temp_col = data[feat].fillna(\"NONE\").astype(str).values\n    \n    data.loc[:, feat] = lbl_enc.fit_transform(temp_col)\n\n# split the training and test data again\ntrain = data[data.target != -1].reset_index(drop=True)\ntest = data[data.target == -1].reset_index(drop=True)\n                                    \n","metadata":{"execution":{"iopub.status.busy":"2022-12-12T10:04:19.646341Z","iopub.execute_input":"2022-12-12T10:04:19.647148Z","iopub.status.idle":"2022-12-12T10:04:38.262154Z","shell.execute_reply.started":"2022-12-12T10:04:19.647114Z","shell.execute_reply":"2022-12-12T10:04:38.260752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Cross validation","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn import model_selection\n\nif __name__ == \"__main__\":\n    \n    # Read training data\n    df = pd.read_csv(\"../input/cat-in-the-dat-ii/train.csv\") \n    \n    # we create a new column called kfold and fill it with -1\n    df[\"kfold\"] = -1\n    \n    # the next step is to randomize the rows of the data\n    df = df.sample(frac=1).reset_index(drop=True)\n    \n    # fetch labels\n    y = df.target.values\n    \n    # initiate the kfold class from model_selection module\n    kf = model_selection.StratifiedKFold(n_splits=5)\n    \n    # fill the new kfold column\n    for f, (t_, v_) in enumerate(kf.split(X=df, y=y)):\n        df.loc[v_, 'kfold'] = f\n        \n    # save the new csv with kfold column\n    df.to_csv(\"folds.csv\", index=False)\n    ","metadata":{"execution":{"iopub.status.busy":"2022-12-12T10:04:38.263692Z","iopub.execute_input":"2022-12-12T10:04:38.264665Z","iopub.status.idle":"2022-12-12T10:04:47.201970Z","shell.execute_reply.started":"2022-12-12T10:04:38.264624Z","shell.execute_reply":"2022-12-12T10:04:47.200828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can now check our new folds csv to see the number of samples per fold:","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/working/folds.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-12-12T10:04:47.203802Z","iopub.execute_input":"2022-12-12T10:04:47.204220Z","iopub.status.idle":"2022-12-12T10:04:49.584981Z","shell.execute_reply.started":"2022-12-12T10:04:47.204186Z","shell.execute_reply":"2022-12-12T10:04:49.583861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.kfold.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T10:04:49.586923Z","iopub.execute_input":"2022-12-12T10:04:49.587610Z","iopub.status.idle":"2022-12-12T10:04:49.602066Z","shell.execute_reply.started":"2022-12-12T10:04:49.587573Z","shell.execute_reply":"2022-12-12T10:04:49.600666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"All folds have 120000 samples. This is ok as training data has 600000 samples, and we made 5 folds, so 1 fold = 120000 samples.","metadata":{}},{"cell_type":"markdown","source":"Now, we can also check the target distribution per fold.","metadata":{}},{"cell_type":"code","source":"df[df.kfold==0].target.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T10:04:49.603774Z","iopub.execute_input":"2022-12-12T10:04:49.604125Z","iopub.status.idle":"2022-12-12T10:04:49.654917Z","shell.execute_reply.started":"2022-12-12T10:04:49.604093Z","shell.execute_reply":"2022-12-12T10:04:49.653580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[df.kfold==1].target.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T10:04:49.656912Z","iopub.execute_input":"2022-12-12T10:04:49.657809Z","iopub.status.idle":"2022-12-12T10:04:49.703373Z","shell.execute_reply.started":"2022-12-12T10:04:49.657766Z","shell.execute_reply":"2022-12-12T10:04:49.702281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[df.kfold==2].target.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T10:04:49.705041Z","iopub.execute_input":"2022-12-12T10:04:49.705805Z","iopub.status.idle":"2022-12-12T10:04:49.750822Z","shell.execute_reply.started":"2022-12-12T10:04:49.705761Z","shell.execute_reply":"2022-12-12T10:04:49.749770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[df.kfold==3].target.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T10:04:49.752190Z","iopub.execute_input":"2022-12-12T10:04:49.752563Z","iopub.status.idle":"2022-12-12T10:04:49.800438Z","shell.execute_reply.started":"2022-12-12T10:04:49.752510Z","shell.execute_reply":"2022-12-12T10:04:49.799316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[df.kfold==4].target.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T10:04:49.801948Z","iopub.execute_input":"2022-12-12T10:04:49.802344Z","iopub.status.idle":"2022-12-12T10:04:49.847616Z","shell.execute_reply.started":"2022-12-12T10:04:49.802313Z","shell.execute_reply":"2022-12-12T10:04:49.846568Z"},"trusted":true},"execution_count":null,"outputs":[]}]}