{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/train.csv', index_col='id')\ndf_test = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/test.csv', index_col='id')\nprint(df_train.shape, df_test.shape)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def description(df):\n    print(f\"Dataset Shape: {df.shape}\")\n    summary = pd.DataFrame(df.dtypes,columns=['dtypes'])\n    summary = summary.reset_index()\n    summary['Name'] = summary['index']\n    summary = summary[['Name','dtypes']]\n    summary['Missing'] = df.isnull().sum().values    \n    summary['Uniques'] = df.nunique().values\n    summary['First Value'] = df.iloc[0].values\n    summary['Second Value'] = df.iloc[1].values\n    summary['Third Value'] = df.iloc[2].values\n    return summary\ndescription(df_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"description(df_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Replacing the missing values with the most occuring value in each column."},{"metadata":{"trusted":true},"cell_type":"code","source":"def replace_nan(data):\n    for column in data.columns:\n        if data[column].isna().sum() > 0:\n            data[column] = data[column].fillna(data[column].mode()[0])\n\n\nreplace_nan(df_train)\nreplace_nan(df_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Applying trigonometric functions to encode cyclic features."},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_CE=df_train.copy()\ncolumns=['day','month']\nfor col in columns:\n    df_train_CE[col+'_sin']=np.sin((2*np.pi*df_train_CE[col])/max(df_train_CE[col]))\n    df_train_CE[col+'_cos']=np.cos((2*np.pi*df_train_CE[col])/max(df_train_CE[col]))\ndf_train_CE=df_train_CE.drop(columns,axis=1)\ndf_train=df_train_CE\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test_CE=df_test.copy()\ncolumns=['day','month']\nfor col in columns:\n    df_test_CE[col+'_sin']=np.sin((2*np.pi*df_test_CE[col])/max(df_test_CE[col]))\n    df_test_CE[col+'_cos']=np.cos((2*np.pi*df_test_CE[col])/max(df_test_CE[col]))\ndf_test_CE=df_test_CE.drop(columns,axis=1)\ndf_test=df_test_CE\ndf_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Applying One Hot Encoding for these low cardinality features."},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train=pd.get_dummies(df_train, prefix=['Color', 'Shape', 'Animal', 'Country', 'Musical Instrument', ], columns=['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4'])\ndf_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test=pd.get_dummies(df_test, prefix=['Color', 'Shape', 'Animal', 'Country', 'Musical Instrument', ], columns=['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4'])\ndf_test","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Applying Ordinal Encoding here."},{"metadata":{"trusted":true},"cell_type":"code","source":"map_ord1 = {'Novice':1,\n           'Contributor':2,\n           'Expert':3,\n           'Master':4,\n           'Grandmaster':5}\n\nmap_ord2 = {'Freezing':1, \n            'Cold':2, \n            'Warm':3, \n            'Hot':4, \n            'Boiling Hot':5, \n            'Lava Hot':6}\n\nmap_bin3 = {'T':1,\n           'F':0}\n\nmap_bin4 = {'Y':1,\n           'N':0}\n\ndf_train.ord_1 = df_train.ord_1.map(map_ord1)\ndf_test.ord_1 = df_test.ord_1.map(map_ord1)\n\ndf_train.ord_2 = df_train.ord_2.map(map_ord2)\ndf_test.ord_2 = df_test.ord_2.map(map_ord2)\n\ndf_train.bin_3 = df_train.bin_3.map(map_bin3)\ndf_test.bin_3 = df_test.bin_3.map(map_bin3)\n\ndf_train.bin_4 = df_train.bin_4.map(map_bin4)\ndf_test.bin_4 = df_test.bin_4.map(map_bin4)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's deal with the high cardinality features now which are 'nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9'. We will be using Hashing here since it is a well known encoding technique for features that have high cardinality."},{"metadata":{"trusted":true},"cell_type":"code","source":"nom_hc = ['nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for col in nom_hc:\n    df_train[f'hash_{col}'] = df_train[col].apply( lambda x: hash(str(x)) % 5000)\n    df_test[f'hash_{col}'] = df_test[col].apply( lambda x: hash(str(x)) % 5000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nom_hashed = ['hash_nom_5', 'hash_nom_6', 'hash_nom_7', 'hash_nom_8',\n            'hash_nom_9']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = df_train.drop(['nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9'], axis=1)\ndf_test = df_test.drop(['nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9'], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"description(df_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"description(df_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Finally we will use Label Encoder to encode 'ord_3', 'ord_4' & 'ord_5'."},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['ord_3_LE'] = LabelEncoder().fit_transform(df_train.ord_3)\ndf_train['ord_4_LE'] = LabelEncoder().fit_transform(df_train.ord_4)\ndf_train['ord_5_LE'] = LabelEncoder().fit_transform(df_train.ord_5)\n\ndf_train = df_train.drop(['ord_3', 'ord_4', 'ord_5'], axis=1)\n\ndescription(df_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test['ord_3_LE'] = LabelEncoder().fit_transform(df_test.ord_3)\ndf_test['ord_4_LE'] = LabelEncoder().fit_transform(df_test.ord_4)\ndf_test['ord_5_LE'] = LabelEncoder().fit_transform(df_test.ord_5)\n\ndf_test = df_test.drop(['ord_3', 'ord_4', 'ord_5'], axis=1)\n\ndescription(df_test)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}