{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Random forest with Entity Embeddings: Training set","metadata":{}},{"cell_type":"markdown","source":"This kernel prepares the ASHRAE Energy Prediction dataset for training a Random Forest. It performs the same preprocessing as in [this kernel](https://www.kaggle.com/michelezoccali/ashrae-energy-prediction-single-lgbm), while substituting categorical features with the corresponding embedding vectors previously learned by a NN in https://www.kaggle.com/michelezoccali/ashrae-with-fast-ai-part-2.","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O\nimport os\nimport datetime\nimport gc","metadata":{"execution":{"iopub.status.busy":"2021-07-08T13:17:21.994059Z","iopub.execute_input":"2021-07-08T13:17:21.994567Z","iopub.status.idle":"2021-07-08T13:17:22.007579Z","shell.execute_reply.started":"2021-07-08T13:17:21.994470Z","shell.execute_reply":"2021-07-08T13:17:22.006439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '../input/ashrae-energy-prediction'\n\nfor dirname, _, filenames in os.walk(path):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2021-07-08T13:17:22.009080Z","iopub.execute_input":"2021-07-08T13:17:22.009397Z","iopub.status.idle":"2021-07-08T13:17:22.033566Z","shell.execute_reply.started":"2021-07-08T13:17:22.009367Z","shell.execute_reply":"2021-07-08T13:17:22.032364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utilities","metadata":{}},{"cell_type":"markdown","source":"Memory reduction adapted from [this kernel.](https://www.kaggle.com/purist1024/ashrae-simple-data-cleanup-lb-1-08-no-leaks/notebook)","metadata":{}},{"cell_type":"code","source":"def reduce_mem(df):\n    result = df.copy()\n    for col in result.columns:\n        col_data = result[col]\n        dn = col_data.dtype.name\n        if not dn.startswith(\"datetime\"):\n            if dn == \"object\":  # only object feature has low cardinality\n                result[col] = pd.to_numeric(col_data.astype(\"category\").cat.codes, downcast=\"unsigned\")\n            elif dn.startswith(\"int\") | dn.startswith(\"uint\"):\n                if col_data.min() >= 0:\n                    result[col] = pd.to_numeric(col_data, downcast=\"unsigned\")\n                else:\n                    result[col] = pd.to_numeric(col_data, downcast='integer')\n            else:\n                result[col] = pd.to_numeric(col_data, downcast='float')\n    return result","metadata":{"execution":{"iopub.status.busy":"2021-07-08T13:17:22.035978Z","iopub.execute_input":"2021-07-08T13:17:22.036630Z","iopub.status.idle":"2021-07-08T13:17:22.046106Z","shell.execute_reply.started":"2021-07-08T13:17:22.036582Z","shell.execute_reply":"2021-07-08T13:17:22.045030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Routine to add lag features to weather dataset, adapted from [this kernel](https://www.kaggle.com/corochann/ashrae-training-lgbm-by-meter-type/notebook).","metadata":{}},{"cell_type":"code","source":"def add_lag_features(weather_df, window=3):\n    group_df = weather_df.groupby('site_id')\n    cols = ['air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr']\n    rolled = group_df[cols].rolling(window=window, min_periods=0)\n    lag_mean = rolled.mean().reset_index().astype(np.float16)\n    lag_max = rolled.max().reset_index().astype(np.float16)\n    lag_min = rolled.min().reset_index().astype(np.float16)\n    for col in cols:\n        weather_df[f'{col}_min_lag{window}'] = lag_min[col]\n        weather_df[f'{col}_mean_lag{window}'] = lag_mean[col]\n        weather_df[f'{col}_max_lag{window}'] = lag_max[col]\n    return weather_df","metadata":{"execution":{"iopub.status.busy":"2021-07-08T13:17:22.048307Z","iopub.execute_input":"2021-07-08T13:17:22.048913Z","iopub.status.idle":"2021-07-08T13:17:22.064900Z","shell.execute_reply.started":"2021-07-08T13:17:22.048868Z","shell.execute_reply":"2021-07-08T13:17:22.063462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"def load_data(source='train'):\n    assert source in ['train','test']\n    df = pd.read_csv(f'{path}/{source}.csv', parse_dates=['timestamp'])\n    return reduce_mem(df)\n\ndef load_building():\n    df = pd.read_csv(f'{path}/building_metadata.csv').fillna(-1)\n    return reduce_mem(df)\n\ndef load_weather(source='train', fix_timezone=True, impute=True, add_lag=True):\n    assert source in ['train','test']\n    df = pd.read_csv(f'{path}/weather_{source}.csv', parse_dates=['timestamp'])\n    if fix_timezone:\n        offsets = [5,0,9,6,8,0,6,6,5,7,8,6,0,7,6,6]\n        offset_map = {site: offset for site, offset in enumerate(offsets)}\n        df.timestamp = df.timestamp - pd.to_timedelta(df.site_id.map(offset_map), unit='h')\n    if impute:\n        site_dfs = []\n        for site in df.site_id.unique():\n            if source == 'train':\n                new_idx = pd.date_range(start='2016-1-1', end='2016-12-31-23', freq='H')\n            else:\n                new_idx = pd.date_range(start='2017-1-1', end='2018-12-31-23', freq='H')\n            site_df = df[df.site_id == site].set_index('timestamp').reindex(new_idx)\n            site_df.site_id = site\n            for col in [c for c in site_df.columns if c != 'site_id']:\n                site_df[col] = site_df[col].interpolate(limit_direction='both', method='linear')\n                site_df[col] = site_df[col].fillna(df[col].median())\n            site_dfs.append(site_df)\n        df = pd.concat(site_dfs)\n        df['timestamp'] = df.index\n        df = df.reset_index(drop=True)\n        \n    if add_lag:\n        df = add_lag_features(df, window=3)\n    \n    return reduce_mem(df)\n\ndef merged_dfs(source='train', fix_timezone=True, impute=True, add_lag=True):\n    df = load_data(source=source).merge(load_building(), on='building_id', how='left')\n    df = df.merge(load_weather(source=source, fix_timezone=fix_timezone, impute=impute, add_lag=add_lag),\n                 on=['site_id','timestamp'], how='left')\n    if source == 'train':\n        X = df.drop('meter_reading', axis=1)  \n        y = np.log1p(df.meter_reading)  # log-transform of target\n        return X, y\n    elif source == 'test':\n        return df","metadata":{"execution":{"iopub.status.busy":"2021-07-08T13:17:22.066674Z","iopub.execute_input":"2021-07-08T13:17:22.067097Z","iopub.status.idle":"2021-07-08T13:17:22.088016Z","shell.execute_reply.started":"2021-07-08T13:17:22.067048Z","shell.execute_reply":"2021-07-08T13:17:22.086737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nX_train, y_train = merged_dfs(add_lag=False)\nX_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T13:17:22.089556Z","iopub.execute_input":"2021-07-08T13:17:22.089898Z","iopub.status.idle":"2021-07-08T13:17:58.386153Z","shell.execute_reply.started":"2021-07-08T13:17:22.089869Z","shell.execute_reply":"2021-07-08T13:17:58.385114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Outlier removal and basic FE","metadata":{}},{"cell_type":"markdown","source":"Let us remove the first 141 days of electrical meter readings at site 0, which are mostly zero or contain anomalous spikes. This is the type of outlier which causes the most trouble and is comparatively easier to remove. We also extract some basic temporal features.","metadata":{}},{"cell_type":"code","source":"def _delete_bad_sitezero(X, y):\n    cond = (X.timestamp > '2016-05-20') | (X.site_id != 0) | (X.meter != 0)\n    X = X[cond]\n    y = y.reindex_like(X)\n    return X.reset_index(drop=True), y.reset_index(drop=True)\n\ndef _extract_temporal(X):\n    X['hour'] = X.timestamp.dt.hour\n    X['weekday'] = X.timestamp.dt.weekday\n    # month and year cause overfit, could try other (holiday, business, etc.)\n    return reduce_mem(X)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T13:17:58.387591Z","iopub.execute_input":"2021-07-08T13:17:58.388011Z","iopub.status.idle":"2021-07-08T13:17:58.395274Z","shell.execute_reply.started":"2021-07-08T13:17:58.387975Z","shell.execute_reply":"2021-07-08T13:17:58.394217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preprocessing\nX_train, y_train = _delete_bad_sitezero(X_train, y_train)\nX_train = _extract_temporal(X_train)\n\n# remove timestamp and other unimportant features\nto_drop = ['timestamp','sea_level_pressure','wind_direction','wind_speed']\nX_train.drop(to_drop, axis=1, inplace=True)\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T13:17:58.398764Z","iopub.execute_input":"2021-07-08T13:17:58.399147Z","iopub.status.idle":"2021-07-08T13:18:12.230605Z","shell.execute_reply.started":"2021-07-08T13:17:58.399113Z","shell.execute_reply":"2021-07-08T13:18:12.229517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.info()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T13:18:12.232530Z","iopub.execute_input":"2021-07-08T13:18:12.232885Z","iopub.status.idle":"2021-07-08T13:18:12.251748Z","shell.execute_reply.started":"2021-07-08T13:18:12.232852Z","shell.execute_reply":"2021-07-08T13:18:12.250923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Save target and training set without embeddings.","metadata":{}},{"cell_type":"code","source":"y_train = y_train.to_frame()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T13:18:12.253029Z","iopub.execute_input":"2021-07-08T13:18:12.253566Z","iopub.status.idle":"2021-07-08T13:18:12.289687Z","shell.execute_reply.started":"2021-07-08T13:18:12.253524Z","shell.execute_reply":"2021-07-08T13:18:12.288841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.to_feather('X_train.feather')\ny_train.to_feather('y_train.feather')","metadata":{"execution":{"iopub.status.busy":"2021-07-08T13:18:12.290993Z","iopub.execute_input":"2021-07-08T13:18:12.291522Z","iopub.status.idle":"2021-07-08T13:18:13.218505Z","shell.execute_reply.started":"2021-07-08T13:18:12.291489Z","shell.execute_reply":"2021-07-08T13:18:13.217349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Concatenate embedding vectors","metadata":{}},{"cell_type":"markdown","source":"Now let us load the categorical embeddings learned with a NN [here](https://www.kaggle.com/michelezoccali/ashrae-with-fast-ai-part-2). \n\nThis step is more easily performed on the GPU, where the original model was trained. It is sometimes necessary to perform this step on the CPU, however, so let us see here the simples changes required.","metadata":{}},{"cell_type":"code","source":"import pickle\nfrom fastai.tabular.all import *\n\n# Subclass the Unpickler to load cuda model on CPU\nclass CPU_Unpickler(pickle.Unpickler):\n    def find_class(self, module, name):\n        if module == 'torch.storage' and name == '_load_from_bytes':\n            return lambda b: torch.load(io.BytesIO(b), map_location='cpu')\n        else: return super().find_class(module, name)\n\nwith open('../input/ashrae-with-fast-ai-part-2/tabular_nn.pickle', mode='rb') as f:\n    #learn = pickle.load(f) becomes...\n    learn = CPU_Unpickler(f).load()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T13:18:13.219900Z","iopub.execute_input":"2021-07-08T13:18:13.220328Z","iopub.status.idle":"2021-07-08T13:18:38.702574Z","shell.execute_reply.started":"2021-07-08T13:18:13.220286Z","shell.execute_reply":"2021-07-08T13:18:38.701424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_features = ['meter','site_id','primary_use','hour','weekday']\n\ndef add_embeds(learn, x):\n    x = x.copy()\n    for i, cat in enumerate(cat_features):\n        emb = learn.embeds[i]\n        vec = tensor(x[cat], dtype=torch.int64) # this is on cpu\n        emb_data = emb(vec)\n        emb_names = [f'{cat}_{j}' for j in range(emb_data.shape[1])]\n        \n        emb_df = pd.DataFrame(emb_data, index=x.index, columns=emb_names)\n        x = x.drop(columns=cat)\n        x = x.join(emb_df)\n    return x","metadata":{"execution":{"iopub.status.busy":"2021-07-08T13:18:38.703954Z","iopub.execute_input":"2021-07-08T13:18:38.704279Z","iopub.status.idle":"2021-07-08T13:18:38.712507Z","shell.execute_reply.started":"2021-07-08T13:18:38.704246Z","shell.execute_reply":"2021-07-08T13:18:38.711404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = add_embeds(learn, X_train)\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T13:18:38.714203Z","iopub.execute_input":"2021-07-08T13:18:38.714676Z","iopub.status.idle":"2021-07-08T13:18:59.435039Z","shell.execute_reply.started":"2021-07-08T13:18:38.714633Z","shell.execute_reply":"2021-07-08T13:18:59.434098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.info()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T13:18:59.436317Z","iopub.execute_input":"2021-07-08T13:18:59.436590Z","iopub.status.idle":"2021-07-08T13:18:59.471562Z","shell.execute_reply.started":"2021-07-08T13:18:59.436564Z","shell.execute_reply":"2021-07-08T13:18:59.470402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now save the DataFrame with the embeddings.","metadata":{}},{"cell_type":"code","source":"X_train.to_feather('X_embeds.feather')","metadata":{"execution":{"iopub.status.busy":"2021-07-08T13:18:59.832696Z","iopub.execute_input":"2021-07-08T13:18:59.833113Z","iopub.status.idle":"2021-07-08T13:19:02.440023Z","shell.execute_reply.started":"2021-07-08T13:18:59.833070Z","shell.execute_reply":"2021-07-08T13:19:02.439227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now on to modeling. Take a look at [Part 2](https://www.kaggle.com/michelezoccali/lgbm-with-entity-embeddings-part-2).","metadata":{}}]}