{"cells":[{"metadata":{},"cell_type":"markdown","source":"This notebook uses Embedding to make more features.\nI am not a pro programmer as you can see but I am still learning. "},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# importing libraries\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn.preprocessing import LabelEncoder\n\nimport random\nfrom catboost import CatBoostClassifier, Pool\nfrom sklearn.metrics import roc_auc_score as rauc\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/cat-in-the-dat-ii/train.csv') \ntest = pd.read_csv('../input/cat-in-the-dat-ii/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get test id for later in submit file\ntest_id = test.id\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train['id']\ndel test['id']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target = train['target']\ndel train['target']\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_data = pd.concat([train, test], axis = 0).copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(5,3))\nplt.hist(all_data.bin_0);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"0 has highest incidence we will replace then nans with 0"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(5,3))\nplt.hist(all_data.bin_1);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"idemn for bin_1"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(5,3))\nplt.hist(all_data.bin_2);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"also for bin_2"},{"metadata":{"trusted":true},"cell_type":"code","source":"# replacing the nans\nfill_bin_cols_nan = ['bin_0', 'bin_1', 'bin_2']\nall_data[fill_bin_cols_nan] = all_data[fill_bin_cols_nan].fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# these string types\nfill_nom_cols1_nan = ['nom_0','nom_1', 'nom_2', 'nom_3', 'nom_4', 'bin_3', 'bin_4']\nall_data[fill_nom_cols1_nan] = all_data[fill_nom_cols1_nan].fillna('NAN')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(5,2))\nplt.hist(all_data.ord_0);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"1 has highes frequency"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(5,2))\nplt.hist(all_data.day, bins=20);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"day 3 has highest frequency"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(5,2))\nplt.hist(all_data.month, bins=40);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Month 8 has the majority"},{"metadata":{"trusted":true},"cell_type":"code","source":"all_data['ord_0'] = all_data.ord_0.fillna(1)\nall_data['day'] = all_data.day.fillna(3)\nall_data['month'] = all_data.month.fillna(8)\n# fill all other nans with 'ffffffffff'\ncolumns_to_encode = [ 'ord_1', 'ord_2','ord_3' , 'ord_4', 'ord_5','bin_3', 'bin_4', \n                     'nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4',\n                    'nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9']\nall_data[columns_to_encode] = all_data[columns_to_encode].fillna('fffffffff')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_data.bin_3.unique()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Label encoding\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"# label encoding\ncolumns_to_encode = [ 'ord_1', 'ord_2','ord_3' , 'ord_4', 'ord_5','bin_3', 'bin_4', \n                     'nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4']\n\nencoder = LabelEncoder()\nfor col in columns_to_encode:\n    print(col)\n    all_data[col][:len(train)] = encoder.fit_transform(all_data[col][:len(train)])\n    all_data[col][len(train):] = all_data[col][len(train):].map(lambda s: '<unknown>' if s not in encoder.classes_ else s)\n    encoder.classes_ = np.append(encoder.classes_, '<unknown>')\n    all_data[col][len(train):] = encoder.transform(all_data[col][len(train):])\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The nominals 5 to 9 have some kind of hexadecimal code. We kan make extra features from each hex digit"},{"metadata":{"trusted":true},"cell_type":"code","source":"nom_label_array = np.zeros([len(all_data), len(all_data.nom_5.values[0])])\n\nfor step in range(nom_label_array.shape[1]):\n    row=0\n    for item in all_data.nom_5.values :\n        #print(item)\n        #print(int(item[0]  ,16))\n        nom_label_array[row][step]= int(item[step]  ,16)\n        row +=1\n\nfor step in range(nom_label_array.shape[1]):        \n    col_name = 'nom_5_' + str(step)\n    all_data[col_name] = nom_label_array[:,step]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# nom_6\nfor step in range(nom_label_array.shape[1]):\n    row=0\n    for item in all_data.nom_6.values :\n        nom_label_array[row][step]= int(item[step]  ,16)\n        row +=1\n\nfor step in range(nom_label_array.shape[1]):        \n    col_name = 'nom_6_' + str(step)\n    all_data[col_name] = nom_label_array[:,step]\n\n# nom_7\nfor step in range(nom_label_array.shape[1]):\n    row=0\n    for item in all_data.nom_7.values :\n        nom_label_array[row][step]= int(item[step]  ,16)\n        row +=1\n\nfor step in range(nom_label_array.shape[1]):        \n    col_name = 'nom_7_' + str(step)\n    all_data[col_name] = nom_label_array[:,step]\n\n# nom_8\nfor step in range(nom_label_array.shape[1]):\n    row=0\n    for item in all_data.nom_8.values :\n        nom_label_array[row][step]= int(item[step]  ,16)\n        row +=1\n\nfor step in range(nom_label_array.shape[1]):        \n    col_name = 'nom_8_' + str(step)\n    all_data[col_name] = nom_label_array[:,step]\n\n# nom_9\nfor step in range(nom_label_array.shape[1]):\n    row=0\n    for item in all_data.nom_9.values :\n        nom_label_array[row][step]= int(item[step]  ,16)\n        row +=1\n\nfor step in range(nom_label_array.shape[1]):        \n    col_name = 'nom_9_' + str(step)\n    all_data[col_name] = nom_label_array[:,step]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_data.head(10).T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# drop the nominal features 5 to 9","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_data = all_data.drop(['nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9'], axis='columns')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_data.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_data.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Using Keras Embedding to do more label encoding and create more features"},{"metadata":{"trusted":true},"cell_type":"code","source":"from numpy.random import seed\nseed(1) # for reproducability\nimport tensorflow\nfrom tensorflow.keras.layers import Embedding\nfrom tensorflow.keras.models import Sequential\nall_data_input = np.asarray(all_data)\nmodel_embed = Sequential()\nmodel_embed.add(Embedding(500, 14, ))\nmodel_embed.compile('rmsprop', 'mse')\noutput_data = model_embed.predict(all_data_input)\noutput_data.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embedded_all_data = output_data.reshape(output_data.shape[0], output_data.shape[1]*output_data.shape[2])\nprint(embedded_all_data.shape)\ntrain = embedded_all_data[:len(train)]\ntest = embedded_all_data[len(train):]\nX_train, X_valid, y_train, y_valid = train_test_split(train, target, test_size=0.2, random_state=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\neval_dataset = Pool(X_valid, y_valid)\nmodel2 = CatBoostClassifier(iterations=1000 ,random_state=2, verbose=0,eval_metric='Accuracy', \n                            learning_rate=0.1, task_type='GPU'  )\n                           \nhistory = model2.fit(X_train, y_train, eval_set=eval_dataset,\n                     use_best_model=True, verbose=False, plot=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions_proba_for_valid = model2.predict_proba(X_valid)\n#print(predictions_proba.shape)\npred_proba = predictions_proba_for_valid[:,1]\n#print(pred_proba.shape)\nR_AUC_valid = rauc(y_valid, pred_proba)\nprint('Area under the ROC curve between target and predictions_proba %.5f' % R_AUC_valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(y_valid, bins=4, density=True, label='y_valid', align='left')\nplt.hist(model2.predict(X_valid), bins=4, density=True, label='predicted', align='right')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"It shows that predicted values have not the best balance"},{"metadata":{"trusted":true},"cell_type":"code","source":"# training on all train data\n\nmodel2 = CatBoostClassifier(iterations=800 ,random_state=2, verbose=0,eval_metric='Accuracy', \n                            learning_rate=0.1, task_type='GPU')\n                           \nhistory = model2.fit(X_train, y_train, verbose=False, plot=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions_proba_test = model2.predict_proba(test)[:,1]\n\n\nfilename = 'submission.csv'\npd.DataFrame({'id': test_id, \n              'target': predictions_proba_test}).to_csv(filename , index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.3"}},"nbformat":4,"nbformat_minor":1}