{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Random forest with Entity Embeddings: Training the model","metadata":{}},{"cell_type":"markdown","source":"We compare the validation set performance of a random forest on two version of the ASHRAE dataset (preprocessed [in Part 1](https://www.kaggle.com/michelezoccali/random-forest-with-embeddings-tutorial-part-1)), differing in the treatment of categorical variables. These are treated:\n\n1. with **standard ordinal encoding** (discrete levels), and\n2. with **Entity Embeddings**, i.e. vectors of continuous values previously learned by a neural net.","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O\nimport os\nimport datetime\nimport warnings\nimport gc\n\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold\nfrom sklearn.ensemble import RandomForestRegressor\nimport lightgbm as lgb\n\nfrom tqdm.notebook import tqdm\n\n# plotting\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2021-07-08T14:21:43.747348Z","iopub.execute_input":"2021-07-08T14:21:43.747771Z","iopub.status.idle":"2021-07-08T14:21:46.044186Z","shell.execute_reply.started":"2021-07-08T14:21:43.747688Z","shell.execute_reply":"2021-07-08T14:21:46.04321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_path = '../input/random-forest-with-embeddings-tutorial-part-1'\n\nfor dirname, _, filenames in os.walk(data_path):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2021-07-08T14:22:22.111354Z","iopub.execute_input":"2021-07-08T14:22:22.111701Z","iopub.status.idle":"2021-07-08T14:22:22.148791Z","shell.execute_reply.started":"2021-07-08T14:22:22.111672Z","shell.execute_reply":"2021-07-08T14:22:22.147869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"X_train = pd.read_feather(f'{data_path}/X_train.feather')\ny_train = pd.read_feather(f'{data_path}/y_train.feather').meter_reading","metadata":{"execution":{"iopub.status.busy":"2021-07-08T14:23:48.279252Z","iopub.execute_input":"2021-07-08T14:23:48.279901Z","iopub.status.idle":"2021-07-08T14:23:51.769479Z","shell.execute_reply.started":"2021-07-08T14:23:48.279859Z","shell.execute_reply":"2021-07-08T14:23:51.76844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train","metadata":{"execution":{"iopub.status.busy":"2021-07-08T14:23:53.920466Z","iopub.execute_input":"2021-07-08T14:23:53.920837Z","iopub.status.idle":"2021-07-08T14:23:53.929614Z","shell.execute_reply.started":"2021-07-08T14:23:53.920797Z","shell.execute_reply":"2021-07-08T14:23:53.928541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modeling","metadata":{}},{"cell_type":"markdown","source":"Let us write a small wrapper function for the Random Forest, to be passed to a CV routine.","metadata":{}},{"cell_type":"code","source":"def RF_wrapper(Xt, yt, Xv, yv, fold=-1):\n    \n    model = RandomForestRegressor(n_jobs=-1, n_estimators=40,\n                              max_samples=200000, max_features=0.5,\n                              min_samples_leaf=5, oob_score=False).fit(Xt, yt)\n    print(f'Training fold {fold}...')\n    \n    score_train = np.sqrt(mean_squared_error(model.predict(Xt), yt))\n    oof = model.predict(Xv)\n    score = np.sqrt(mean_squared_error(oof, yv))\n    print(f'Fold {fold}: training RMSLE: {score_train},   validation RMSLE: {score}\\n')\n    return model, oof, score","metadata":{"execution":{"iopub.status.busy":"2021-07-08T14:23:56.817487Z","iopub.execute_input":"2021-07-08T14:23:56.817824Z","iopub.status.idle":"2021-07-08T14:23:56.824228Z","shell.execute_reply.started":"2021-07-08T14:23:56.817796Z","shell.execute_reply":"2021-07-08T14:23:56.823423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let us perform k-fold CV, without shuffling as this is a time series. An alternative would be to do a single train/validation split, possibly with a gap to mimic training/private split. Otherwise, one could try something like Time-series split CV.","metadata":{}},{"cell_type":"code","source":"def perform_CV(model_wrap, xs, ys, n_splits=3):\n    \n    kf = KFold(n_splits=n_splits, shuffle=False)\n\n    models = []\n    scores = []\n    oof_total = np.zeros(xs.shape[0])\n\n\n    for fold, (train_idx, val_idx) in enumerate(kf.split(xs), start=1):\n        Xt, yt = xs.iloc[train_idx], ys[train_idx]\n        Xv, yv = xs.iloc[val_idx], ys[val_idx]\n        model, oof, score = model_wrap(Xt, yt, Xv, yv, fold)\n\n        models.append(model)\n        scores.append(score)\n        oof_total[val_idx] = oof\n\n    print('Training completed.')\n    print(f'> Mean RMSLE across folds: {np.mean(scores)}, std: {np.std(scores)}')\n    print(f'> OOF RMSLE: {np.sqrt(mean_squared_error(ys, oof_total))}')\n    return models, scores, oof_total","metadata":{"execution":{"iopub.status.busy":"2021-07-08T14:24:03.134937Z","iopub.execute_input":"2021-07-08T14:24:03.135472Z","iopub.status.idle":"2021-07-08T14:24:03.144141Z","shell.execute_reply.started":"2021-07-08T14:24:03.135422Z","shell.execute_reply":"2021-07-08T14:24:03.143327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's train the random forest **without** embeddings.","metadata":{}},{"cell_type":"code","source":"%%time\nn_splits = 3\nmodels, _, _ = perform_CV(RF_wrapper, X_train, y_train, n_splits=n_splits)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T14:24:09.136823Z","iopub.execute_input":"2021-07-08T14:24:09.137375Z","iopub.status.idle":"2021-07-08T14:28:16.135051Z","shell.execute_reply.started":"2021-07-08T14:24:09.137327Z","shell.execute_reply":"2021-07-08T14:28:16.133929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's see the average feature importance across models. We can use this to retroactively drop further superfluous features during preprocessing.","metadata":{}},{"cell_type":"code","source":"importance = pd.DataFrame([model.feature_importances_ for model in models],\n                          columns=X_train.columns,\n                          index=[f'Fold {i}' for i in range(1, n_splits + 1)])\nimportance = importance.T\nimportance['Average importance'] = importance.mean(axis=1)\nimportance = importance.sort_values(by='Average importance', ascending=False)\n\nplt.figure(figsize=(10,7))\nsns.barplot(x='Average importance', y=importance.index, data=importance);","metadata":{"execution":{"iopub.status.busy":"2021-07-08T14:29:32.492548Z","iopub.execute_input":"2021-07-08T14:29:32.492919Z","iopub.status.idle":"2021-07-08T14:29:33.082084Z","shell.execute_reply.started":"2021-07-08T14:29:32.492885Z","shell.execute_reply":"2021-07-08T14:29:33.080887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X_train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T14:30:26.063923Z","iopub.execute_input":"2021-07-08T14:30:26.0643Z","iopub.status.idle":"2021-07-08T14:30:26.189777Z","shell.execute_reply.started":"2021-07-08T14:30:26.06427Z","shell.execute_reply":"2021-07-08T14:30:26.188814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now let's repeat **with** embeddings.","metadata":{}},{"cell_type":"code","source":"X_embeds = pd.read_feather(f'{data_path}/X_embeds.feather')","metadata":{"execution":{"iopub.status.busy":"2021-07-08T14:30:30.313816Z","iopub.execute_input":"2021-07-08T14:30:30.314185Z","iopub.status.idle":"2021-07-08T14:30:33.334487Z","shell.execute_reply.started":"2021-07-08T14:30:30.314149Z","shell.execute_reply":"2021-07-08T14:30:33.33362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmodels_emb, _, _ = perform_CV(RF_wrapper, X_embeds, y_train, n_splits=n_splits)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T14:30:37.036186Z","iopub.execute_input":"2021-07-08T14:30:37.036521Z","iopub.status.idle":"2021-07-08T14:37:33.513014Z","shell.execute_reply.started":"2021-07-08T14:30:37.036493Z","shell.execute_reply":"2021-07-08T14:37:33.511767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"importance = pd.DataFrame([model.feature_importances_ for model in models_emb],\n                          columns=X_embeds.columns,\n                          index=[f'Fold {i}' for i in range(1, n_splits + 1)])\nimportance = importance.T\nimportance['Average importance'] = importance.mean(axis=1)\nimportance = importance.sort_values(by='Average importance', ascending=False)\n\nplt.figure(figsize=(10,7))\nsns.barplot(x='Average importance', y=importance.index, data=importance);","metadata":{"execution":{"iopub.status.busy":"2021-07-08T14:42:32.355839Z","iopub.execute_input":"2021-07-08T14:42:32.356304Z","iopub.status.idle":"2021-07-08T14:42:33.298818Z","shell.execute_reply.started":"2021-07-08T14:42:32.356267Z","shell.execute_reply":"2021-07-08T14:42:33.297774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X_embeds, y_train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T14:43:36.81038Z","iopub.execute_input":"2021-07-08T14:43:36.810749Z","iopub.status.idle":"2021-07-08T14:43:37.081793Z","shell.execute_reply.started":"2021-07-08T14:43:36.81072Z","shell.execute_reply":"2021-07-08T14:43:37.08067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So, using these mysterious embeddings actually worked! Even if, in this case, it only helped performance a little (and was quite a bit slower too!). However, it is well worth knowing that this method exists, as in general it leads to much better performance. For more information, check out the paper [Entity Embeddings of Categorical Variables](https://arxiv.org/pdf/1604.06737.pdf).\n\nThat's it. Do upvote this kernel if you found it of any use! 🖖","metadata":{}}]}