{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Categorical Feature Encoding Challenge II\n\n## xLearn version\n\nI created this kernel by referring to the following   \n[libffm_model](https://www.kaggle.com/ogrellier/libffm-model)\n\nThanks!\n"},{"metadata":{},"cell_type":"markdown","source":"---------------------"},{"metadata":{"_uuid":"21411361-9e0a-4244-92f9-b3b116b018e7","_cell_guid":"2f2f5d72-8be0-413d-ad41-b4adff81e458","trusted":true},"cell_type":"markdown","source":"## install xlearn\n\nref.  \nhttps://www.kaggle.com/nadare/xlearn-model-cv-42-lb\n"},{"metadata":{"_uuid":"188348d2-c585-4ebf-8604-46adc5466953","_cell_guid":"8fca38ab-affb-4fee-b20a-f13372a59b16","trusted":true},"cell_type":"code","source":"import os\nos.mkdir(\"./working\")\nos.environ['USER'] = 'root'\nos.system('pip install ../input/xlearn/xlearn/xlearn-0.40a1/')\n\nimport xlearn as xl","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"-------------------"},{"metadata":{"_uuid":"d69113ca-0a23-437e-be3f-fe1f5b95b459","_cell_guid":"48de5e77-2856-4eb2-899b-3ab32aecb46d","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport scipy\nimport gc,os\nfrom collections import Counter\nfrom sklearn.model_selection import KFold,StratifiedKFold,RepeatedKFold,RepeatedStratifiedKFold\nfrom sklearn.metrics import roc_auc_score as auc\nfrom sklearn.linear_model import LogisticRegression\nimport category_encoders as ce\n\nimport warnings\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = 50\nBIN_COL  = [f'bin_{i}' for i in range(5)]\nNOM_COL  = [f'nom_{i}' for i in range(10)]\nORD_COL  = [f'ord_{i}' for i in range(6)]\nNOM_5_9  = ['nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9']\nNOM_0_4  = ['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4']\nDATE_COL = ['day','month']\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\nfor dirname, _, filenames in os.walk('/kaggle/working/libffm-binaries'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0a348230-69c7-41cd-b77b-4ecfc6c86033","_cell_guid":"ef6e1419-a913-41b0-80eb-06983aca5893","trusted":true},"cell_type":"markdown","source":"## Read the data"},{"metadata":{"_uuid":"424f54f8-e4bd-4229-8157-8c7fedf60920","_cell_guid":"c137aebb-2692-44a8-8a5b-79c5432ed230","trusted":true},"cell_type":"code","source":"%%time\n\ndef read_csv():\n    train = pd.read_csv('../input/cat-in-the-dat-ii/train.csv')\n    test  = pd.read_csv('../input/cat-in-the-dat-ii/test.csv')\n\n    train_id = train['id']\n    test_id  = test['id']\n    train.drop('id', axis=1, inplace=True)\n    test.drop('id',  axis=1, inplace=True)\n    return train, test, train_id, test_id\n\ntrain, test, train_id, test_id = read_csv()\ndef preprocessing(df):\n    \n    df['bin_missing']  = (df[['bin_0', 'bin_1', 'bin_2', 'bin_4']].isnull().sum(axis=1)>0).replace({True:1, False:0})\n    df['nom_missing']  = (df[NOM_COL].isnull().sum(axis=1)>0).replace({True:1, False:0})\n    df['ord_missing']  = (df[ORD_COL].isnull().sum(axis=1)>0).replace({True:1, False:0})\n    df['date_missing'] = (df[DATE_COL].isnull().sum(axis=1)>0).replace({True:1, False:0})\n    \n    df.day = df.day.replace({3:5,2:6,1:7})\n    df.loc[df.ord_5.notnull(), 'ord_5_1'] = df.loc[df.ord_5.notnull(), 'ord_5'].apply(lambda x: x[0])\n    df.loc[df.ord_5.notnull(), 'ord_5_2'] = df.loc[df.ord_5.notnull(), 'ord_5'].apply(lambda x: x[1])\n    \n    return df\n\ntrain, test, train_id, test_id = read_csv()\ntrain = preprocessing(train)\ntest  = preprocessing(test)\nprint(f'train day unique value:{train.day.unique()}')\nprint(f'test  day unique value:{test.day.unique()}')\n\nfor col in test.columns:\n    if len(set(train[col].dropna().unique().tolist())^ set(test[col].dropna().unique().tolist()))>0:\n        train_only = list(set(train[col].dropna().unique().tolist()) - set(test[col].dropna().unique().tolist()))\n        test_only  = list(set(test[col].dropna().unique().tolist()) - set(train[col].dropna().unique().tolist()))\n        print(col, '(train only)', train_only, '(test only)', test_only) \n        train.loc[train[col].isin(train_only), col] = np.NaN\n        test.loc[test[col].isin(test_only), col]    = np.NaN  \n\ntest.insert(1, 'target', 0)\n\ndrop_cols = ['bin_3', 'ord_5']\ntrain.drop(columns=drop_cols, inplace=True)\ntest.drop(columns=drop_cols,  inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# Label Encode to ease creation of libffm format\n\nfeatures = [_f for _f in train if _f not in ['id', 'target']]\n\ndef factor_encoding(train, test):\n    \n    assert sorted(train.columns) == sorted(test.columns)\n    \n    full = pd.concat([train, test], axis=0, sort=False)\n    # Factorize everything\n    for f in full:\n        full[f], _ = pd.factorize(full[f])\n        full[f] += 1  # make sure no negative\n        \n    return full.iloc[:len(train)], full.iloc[len(train):]\n\ntrain_f, test_f = factor_encoding(train[features], test[features])\n\nprint(train_f,'-'*20)\nprint(train_f.head(10))\n\nprint(test_f,'-'*20)\nprint(test_f.head(10))\n\nclass LibFFMEncoder(object):\n    def __init__(self):\n        self.encoder = 1\n        self.encoding = {}\n\n    def encode_for_libffm(self, row):\n        txt = f\"{row[0]}\"\n        for i, r in enumerate(row[1:]):\n            \n#             try:\n#                 txt += f' {i+1}:{self.encoding[r]}:1'\n#             except KeyError:\n#                 self.encoding[r] = self.encoder\n#                 self.encoder += 1\n#                 txt += f' {i+1}:{self.encoding[r]}:1'\n            try:\n#                 print(f'key {i} {r}')\n                txt += f' {i+1}:{self.encoding[(i, r)]}:1'\n            except KeyError:\n#                 print(f'key error {i} {r}')\n                self.encoding[(i, r)] = self.encoder\n                self.encoder += 1\n                txt += f' {i+1}:{self.encoding[(i, r)]}:1'\n                \n        return txt\n\n\nSPLITS = 5\n    \n# Create files for testing and OOF\nfrom sklearn.model_selection import KFold\nfold_ids = [\n    [trn_, val_] for (trn_, val_) in KFold(SPLITS,True,1).split(train)\n]\nfor fold_, (trn_, val_) in enumerate(fold_ids):\n    encoder = LibFFMEncoder()\n    libffm_format_trn = pd.concat([train['target'].iloc[trn_], train_f.iloc[trn_]], axis=1).apply(\n        lambda row: encoder.encode_for_libffm(row), raw=True, axis=1\n    )\n    libffm_format_val = pd.concat([train['target'].iloc[val_], train_f.iloc[val_]], axis=1).apply(\n        lambda row: encoder.encode_for_libffm(row), raw=True, axis=1\n    )\n    print(train['target'].iloc[trn_].shape, train['target'].iloc[val_].shape, libffm_format_val.shape)\n    \n    libffm_format_trn.to_csv(f'libffm_trn_fold_{fold_+1}.txt', index=False, header=False)\n    libffm_format_val.to_csv(f'libffm_val_fold_{fold_+1}.txt', index=False, header=False)\n    \n# Create files for final model\nencoder = LibFFMEncoder()\nlibffm_format_trn = pd.concat([train['target'], train_f], axis=1).apply(\n        lambda row: encoder.encode_for_libffm(row), raw=True, axis=1\n)\nlibffm_format_tst = pd.concat([test['target'], test_f], axis=1).apply(\n    lambda row: encoder.encode_for_libffm(row), raw=True, axis=1\n)\n\nlibffm_format_trn.to_csv(f'libffm_trn.txt', index=False, header=False)\nlibffm_format_tst.to_csv(f'libffm_tst.txt', index=False, header=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Run OOF"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# create ffm model\nffm_model = xl.create_ffm() \n\n# import optuna\nfrom sklearn.metrics import log_loss, roc_auc_score\nfrom sklearn.model_selection import train_test_split\n\noutputs = []\n\n# define params\nparam = {'task':'binary', \n         'lr':0.2, \n         'k':4,\n         'lambda':0.0002, \n         'metric':'auc',\n         'epoch': 15\n        }\n        \nfor fold_ in range(1, SPLITS+1):\n    print(f'fold: {fold_}')\n    model = f\"libffm_fold_{fold_}_model\"\n    trn_fold_txt = f\"libffm_trn_fold_{fold_}.txt\"\n    val_fold_txt = f\"libffm_val_fold_{fold_}.txt\"\n    val_preds_fold_txt = f\"val_preds_fold_{fold_}.txt\"    \n\n    # set training and validation data\n    ffm_model.setTrain(trn_fold_txt)\n    \n    #xLearn will perform early-stopping by default.     \n    ffm_model.setValidate(val_fold_txt)   \n    \n    print(' fitting...')\n    ffm_model.fit(param, 'model.output')\n\n    print(' make predictions...')\n    ffm_model.setTest(val_fold_txt)\n    ffm_model.setSigmoid()\n    ffm_model.predict(\"model.output\", val_preds_fold_txt)\n    gc.collect()\n    \n    print(' auc score:',\n        roc_auc_score(\n            train['target'].iloc[fold_ids[fold_-1][1]], \n            pd.read_csv(val_preds_fold_txt, header=None).values[:,0]))   \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"85022c53-2177-48cf-928b-44f278b61c6e","_cell_guid":"493e604d-26d0-4470-89da-26679228af15","trusted":true},"cell_type":"markdown","source":"## Compute OOF score"},{"metadata":{"_uuid":"a9960c80-cb79-4342-8578-caa64ad1ca23","_cell_guid":"cdf9762d-d360-4178-905b-551275ec5c20","trusted":true},"cell_type":"code","source":"oof_preds = np.zeros(train.shape[0])\nfor fold_, (_, val_) in enumerate(fold_ids):\n    oof_preds[val_] = pd.read_csv(f'val_preds_fold_{fold_+1}.txt', header=None).values[:, 0]\nprint(roc_auc_score(train['target'], oof_preds))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.distplot(pd.Series(oof_preds))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"514bd324-5a0f-4f3b-beaf-2b2a5239bb77","_cell_guid":"c9c02a1b-9bb5-4e23-bd82-932f53e8e820","trusted":true},"cell_type":"markdown","source":"## Train a xlearn model"},{"metadata":{"_uuid":"fde7c258-cc9f-453a-8414-f96e988f9fba","_cell_guid":"7bc7203a-2f17-4f88-ae80-e49bfbc52264","trusted":true},"cell_type":"code","source":"%%time\n\n# # define params\n# param = {'task':'binary', \n#          'lr':0.2, \n#          'k':4,\n#          'lambda':0.0002, \n#          'metric':'auc',\n#          'epoch': 15,\n#         }\n\nffm_model = xl.create_ffm() \nffm_model.setTrain(\"libffm_trn.txt\")\nffm_model.fit(param, 'model.output')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nffm_model.setTest('libffm_tst.txt')\nffm_model.setSigmoid()\nffm_model.predict('model.output', 'tst_preds.txt')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b7eabc0a-084a-4613-9672-9f07c1ec1438","_cell_guid":"2f45111d-fafb-425b-a553-a5a54698fd5c","trusted":true},"cell_type":"markdown","source":"## Predict for test set\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Prepare submission\n\nsubmission = pd.DataFrame()\nsubmission['id'] = test_id#test[['id']].copy()\n# submission = test[['id']].copy()\nsubmission['target'] = pd.read_csv('tst_preds.txt', header=None).values[:,0]\nsubmission.to_csv('xlearn_prediction.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2a0bc9fa-539a-480b-a7b2-592f39728a5a","_cell_guid":"a2e6bdf3-0ae5-49d6-8cb1-1affe14365e9","trusted":true},"cell_type":"code","source":"test.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"351a50e8-c338-4f73-b67c-d8ec47b3c602","_cell_guid":"7d091101-a711-4f7d-804d-b3e75f40df47","trusted":true},"cell_type":"code","source":"submission[:50]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f751ebc9-221a-4a9a-a5df-2097ab2c3cf6","_cell_guid":"7ffa4364-c168-41a6-aae2-a2c7a661166d","trusted":true},"cell_type":"code","source":"submission.target.describe()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bf0a7399-92b4-4663-97a1-0458736dc9f3","_cell_guid":"953b1b5e-766a-4cb3-a460-c849dc079c34","trusted":true},"cell_type":"code","source":"sns.distplot(submission.target)","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}