{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os, gc, warnings\nimport random\nimport datetime\n\nfrom tqdm.notebook import tqdm\n# matplotlib and seaborn for plotting\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# import plotly.offline as py\n# py.init_notebook_mode(connected=True)\n# from plotly.offline import init_notebook_mode, iplot\n# init_notebook_mode(connected=True)\n# import plotly.graph_objs as go\n# import plotly.offline as offline\n# offline.init_notebook_mode()\n\nfrom pandas.plotting import register_matplotlib_converters\nregister_matplotlib_converters()\n\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import KFold, StratifiedKFold\nimport sklearn\n\nimport lightgbm as lgb\n\nimport pickle\n\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2021-08-17T16:24:09.473487Z","iopub.execute_input":"2021-08-17T16:24:09.473926Z","iopub.status.idle":"2021-08-17T16:24:12.548981Z","shell.execute_reply.started":"2021-08-17T16:24:09.473801Z","shell.execute_reply":"2021-08-17T16:24:12.548046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '../input/work-df/'\n# Input data files are available in the \"../input/\" directory.\nfor dirname, _, filenames in os.walk(path):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2021-08-17T16:24:12.550471Z","iopub.execute_input":"2021-08-17T16:24:12.550783Z","iopub.status.idle":"2021-08-17T16:24:12.562741Z","shell.execute_reply.started":"2021-08-17T16:24:12.550746Z","shell.execute_reply":"2021-08-17T16:24:12.561881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data and display samples","metadata":{}},{"cell_type":"code","source":"# unimportant features (see importance below)\nunimportant_cols = ['wind_direction', 'wind_speed', 'sea_level_pressure']\ntarget = 'meter_reading'\n\ndef load_data(source='train', path=path):\n    ''' load and merge all tables '''\n    assert source in ['train', 'test']\n    \n    building = pd.read_csv(f'{path}/building_metadata.csv', dtype={'building_id':np.uint16, 'site_id':np.uint8})\n    weather  = pd.read_csv(f'{path}/weather_{source}_filled.csv', parse_dates=['timestamp'],\n                                                           dtype={'site_id':np.uint8, 'air_temperature':np.float16,\n                                                                  'cloud_coverage':np.float16, 'dew_temperature':np.float16,\n                                                                  'precip_depth_1_hr':np.float16},\n                                                           usecols=lambda c: c not in unimportant_cols)\n    df = pd.read_csv(f'{path}/{source}.csv', dtype={'building_id':np.uint16, 'meter':np.uint8}, parse_dates=['timestamp'])\n    df = df.merge(building, on='building_id', how='left')\n    df = df.merge(weather, on=['site_id', 'timestamp'], how='left')\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-08-17T16:24:14.513444Z","iopub.execute_input":"2021-08-17T16:24:14.513757Z","iopub.status.idle":"2021-08-17T16:24:14.522861Z","shell.execute_reply.started":"2021-08-17T16:24:14.513729Z","shell.execute_reply":"2021-08-17T16:24:14.521909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain = load_data('train')\ntrain.sample(7)","metadata":{"tags":[],"execution":{"iopub.status.busy":"2021-08-17T16:24:14.668806Z","iopub.execute_input":"2021-08-17T16:24:14.669195Z","iopub.status.idle":"2021-08-17T16:24:40.161711Z","shell.execute_reply.started":"2021-08-17T16:24:14.669146Z","shell.execute_reply":"2021-08-17T16:24:40.160951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest = load_data('test')\ntest.sample(7)","metadata":{"tags":[],"execution":{"iopub.status.busy":"2021-08-17T16:24:40.163024Z","iopub.execute_input":"2021-08-17T16:24:40.163286Z","iopub.status.idle":"2021-08-17T16:25:31.773965Z","shell.execute_reply.started":"2021-08-17T16:24:40.163260Z","shell.execute_reply":"2021-08-17T16:25:31.773033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Training from {train.timestamp.min()} to {train.timestamp.max()}')","metadata":{"execution":{"iopub.status.busy":"2021-08-17T16:25:31.775812Z","iopub.execute_input":"2021-08-17T16:25:31.776178Z","iopub.status.idle":"2021-08-17T16:25:31.895245Z","shell.execute_reply.started":"2021-08-17T16:25:31.776131Z","shell.execute_reply":"2021-08-17T16:25:31.894135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Remove visual outlayers.","metadata":{}},{"cell_type":"code","source":"meter_arr = train[\"meter\"].unique()\nfor meter in meter_arr:\n    mask = train[\"meter\"] == meter\n    plt.figure(figsize=(20, 5))\n    sns.scatterplot(data = train[mask], x = \"meter_reading\", y = \"air_temperature\")\n    plt.xlabel(\"Meter: {}\".format(meter))\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-17T16:25:31.897038Z","iopub.execute_input":"2021-08-17T16:25:31.897392Z","iopub.status.idle":"2021-08-17T16:26:06.850700Z","shell.execute_reply.started":"2021-08-17T16:25:31.897353Z","shell.execute_reply":"2021-08-17T16:26:06.849905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Removing meter_reading out_layers for the different meters","metadata":{}},{"cell_type":"markdown","source":"* meter:0, electricity: bigger than 8000 is outlayer.\n* meter:3, hotwater: bigger than 140000 is outlayer.","metadata":{}},{"cell_type":"code","source":"mask1 = train[\"meter\"] == 0\nmask2 = train[\"meter_reading\"] > 40000\nmask = np.logical_and(mask1, mask2)\nprint(train.shape)\ntrain[mask][\"meter_reading\"] = np.mean(train[mask1][\"meter_reading\"])\nprint(train.shape)","metadata":{"tags":[],"execution":{"iopub.status.busy":"2021-08-17T16:26:06.851890Z","iopub.execute_input":"2021-08-17T16:26:06.852333Z","iopub.status.idle":"2021-08-17T16:26:07.974398Z","shell.execute_reply.started":"2021-08-17T16:26:06.852297Z","shell.execute_reply":"2021-08-17T16:26:07.973524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mask1 = train[\"meter\"] == 3\nmask2 = train[\"meter_reading\"] > 140000\nmask = np.logical_and(mask1, mask2)\nprint(train.shape)\ntrain[mask][\"meter_reading\"] = np.mean(train[mask1][\"meter_reading\"])\nprint(train.shape)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T16:26:07.975598Z","iopub.execute_input":"2021-08-17T16:26:07.976106Z","iopub.status.idle":"2021-08-17T16:26:08.172398Z","shell.execute_reply.started":"2021-08-17T16:26:07.976068Z","shell.execute_reply":"2021-08-17T16:26:08.171385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Remove building_id = [1099, 1088]. It's outlayer.","metadata":{}},{"cell_type":"code","source":"mask = train[\"building_id\"] == 778\ntrain[mask][\"meter_reading\"] = 1","metadata":{"execution":{"iopub.status.busy":"2021-08-17T16:26:08.173695Z","iopub.execute_input":"2021-08-17T16:26:08.174207Z","iopub.status.idle":"2021-08-17T16:26:08.198725Z","shell.execute_reply.started":"2021-08-17T16:26:08.174168Z","shell.execute_reply":"2021-08-17T16:26:08.197974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mask = train[\"building_id\"] == 1088\ntrain[mask][\"meter_reading\"] = 1","metadata":{"tags":[],"execution":{"iopub.status.busy":"2021-08-17T16:26:08.201304Z","iopub.execute_input":"2021-08-17T16:26:08.201634Z","iopub.status.idle":"2021-08-17T16:26:08.225275Z","shell.execute_reply.started":"2021-08-17T16:26:08.201598Z","shell.execute_reply":"2021-08-17T16:26:08.224536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for our train and valudation dataset    return df\ndef correct_error_meter0_model_use(df):\n    out = df\n    new_values = out[\"meter_reading\"] * 0.2931\n    out.loc[out.meter == 0, \"meter_reading\"] = new_values\n    return out","metadata":{"execution":{"iopub.status.busy":"2021-08-17T16:26:08.227059Z","iopub.execute_input":"2021-08-17T16:26:08.227413Z","iopub.status.idle":"2021-08-17T16:26:08.231860Z","shell.execute_reply.started":"2021-08-17T16:26:08.227378Z","shell.execute_reply":"2021-08-17T16:26:08.230921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train = correct_error_meter0_model_use(train)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T16:26:08.233331Z","iopub.execute_input":"2021-08-17T16:26:08.233748Z","iopub.status.idle":"2021-08-17T16:26:08.240732Z","shell.execute_reply.started":"2021-08-17T16:26:08.233714Z","shell.execute_reply":"2021-08-17T16:26:08.239984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# target's log-log histogram:\nax = train.meter_reading.hist()\nax.set_yscale('log')\n\n# describe raw values first\ntrain.meter_reading.describe()","metadata":{"execution":{"iopub.status.busy":"2021-08-17T16:26:08.241959Z","iopub.execute_input":"2021-08-17T16:26:08.242382Z","iopub.status.idle":"2021-08-17T16:26:10.049271Z","shell.execute_reply.started":"2021-08-17T16:26:08.242344Z","shell.execute_reply":"2021-08-17T16:26:10.048327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check the distribution in the types of meters\nmeters = train.groupby('building_id').meter.nunique()\nplt.title('Distribution of types of meters\\n{0:electricity, 1:water, 2:steam, 3:hotwater}') # from the official starter kernel\n_ = meters.hist()\n# from the graphs it looks like steam and hotwater are reversed (e.g.: 3:steam, 2:hotwater) but that shouldn't make any difference to the model","metadata":{"execution":{"iopub.status.busy":"2021-08-17T16:26:10.050616Z","iopub.execute_input":"2021-08-17T16:26:10.050968Z","iopub.status.idle":"2021-08-17T16:26:13.227557Z","shell.execute_reply.started":"2021-08-17T16:26:10.050930Z","shell.execute_reply":"2021-08-17T16:26:13.226801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## display a single time series (notice measurement errors and discontinuities)","metadata":{}},{"cell_type":"code","source":"building_id = 1258  # a building with all 4 meters\nmeters = train[train['building_id'] == building_id].meter.nunique()\n\nfor meter in range(meters):\n    fig, ax = plt.subplots()\n    plt.title(f'Building {building_id} Meter {meter}')\n    ax2 = ax.twinx()\n    # plot meter_reading\n    idx = (train['building_id'] == building_id) & (train['meter'] == meter)\n    dates = matplotlib.dates.date2num(train.loc[idx, 'timestamp'])\n    ax2.plot_date(dates, train.loc[idx, 'meter_reading'], '-', label='meter_reading', alpha=0.8)\n    # plot air_temperature\n    dates = matplotlib.dates.date2num(train.loc[train['building_id'] == building_id, 'timestamp'])\n    ax.plot_date(dates, train.loc[train['building_id'] == building_id, 'air_temperature'], '.', color='tab:cyan', label='air_temperature')\n    ax.set_ylabel('air_temperature'); ax2.set_ylabel('meter_reading')\n    ax.legend(loc='upper left'); ax2.legend(loc='upper right')","metadata":{"execution":{"iopub.status.busy":"2021-08-17T16:26:13.228740Z","iopub.execute_input":"2021-08-17T16:26:13.229085Z","iopub.status.idle":"2021-08-17T16:26:14.970590Z","shell.execute_reply.started":"2021-08-17T16:26:13.229050Z","shell.execute_reply":"2021-08-17T16:26:14.969624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## now let's see what's the expected prediction in the test set for the same building","metadata":{}},{"cell_type":"code","source":"meter = 1 # pick a meter\n\ntrain_sample = train[(train['building_id'] == building_id) & (train['meter'] == meter)]  # same train sample as above\n\ntest['meter_reading'] = 0.0\ntest_sample = test[(test['building_id'] == building_id) & (test['meter'] == meter)]  # and the same meter in the test set\n\nfig, ax = plt.subplots(figsize=(16,4))\nplt.title(f'Meter {meter}')\nax.xaxis.set_tick_params(rotation=30, labelsize=10)\nax2 = ax.twinx()\n\n# plot training sample\ndates = matplotlib.dates.date2num(train_sample['timestamp'])\nax2.plot_date(dates, train_sample['meter_reading'], '-', label='train', alpha=0.8)\nax.plot_date(dates, train_sample['air_temperature'], '.', color='tab:cyan', label='air_temperature_train')\n\n# plot test sample\ndates = matplotlib.dates.date2num(test_sample['timestamp'])\nax2.plot_date(dates, test_sample['meter_reading'], '*', label='test', alpha=0.8)\nax.plot_date(dates, test_sample['air_temperature'], '.', color='tab:cyan', label='air_temperature_test')\n\nax.set_ylabel('air_temperature'); ax2.set_ylabel('meter_reading')\nax.legend(loc='upper left'); ax2.legend(loc='upper right')\n\ndel train_sample; del test_sample; del dates","metadata":{"execution":{"iopub.status.busy":"2021-08-17T16:26:14.972008Z","iopub.execute_input":"2021-08-17T16:26:14.972356Z","iopub.status.idle":"2021-08-17T16:26:16.334166Z","shell.execute_reply.started":"2021-08-17T16:26:14.972318Z","shell.execute_reply":"2021-08-17T16:26:16.333191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# the counts above expose the missing data (Should we drop or refill the missing data?)\nprint(\"Ratio of available data (not NAN's):\")\ndata_ratios = train.count()/len(train)\ndata_ratios","metadata":{"execution":{"iopub.status.busy":"2021-08-17T16:26:16.335600Z","iopub.execute_input":"2021-08-17T16:26:16.335955Z","iopub.status.idle":"2021-08-17T16:26:18.799736Z","shell.execute_reply.started":"2021-08-17T16:26:16.335918Z","shell.execute_reply":"2021-08-17T16:26:18.798962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Is the same happening in the test set? Yes\nprint(\"Ratio of available data (not NAN's):\")\ntest.count()/len(test)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T16:26:18.800958Z","iopub.execute_input":"2021-08-17T16:26:18.801293Z","iopub.status.idle":"2021-08-17T16:26:23.939504Z","shell.execute_reply.started":"2021-08-17T16:26:18.801258Z","shell.execute_reply":"2021-08-17T16:26:23.938766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preproces data\n* The weather data sets are correctly filled in other scripts.","metadata":{}},{"cell_type":"code","source":"def ASHRAE3Preprocessor(df, data_ratios):\n    avgs = df.loc[:,data_ratios < 1.0].mean()\n    pu_le = LabelEncoder() # Asign to a categorical variable numerical values.\n    pu_le.fit(df[\"primary_use\"])\n    \n    df = df.fillna(avgs) # refill NAN with averages\n    df['primary_use'] = np.uint8(pu_le.transform(df['primary_use']))  # encode labels\n\n    # expand datetime into its components\n    df['hour'] = np.uint8(df['timestamp'].dt.hour)\n    df['day'] = np.uint8(df['timestamp'].dt.day)\n    df['weekday'] = np.uint8(df['timestamp'].dt.weekday)\n    df['month'] = np.uint8(df['timestamp'].dt.month)\n    df['year'] = np.uint8(df['timestamp'].dt.year-2000)\n\n    # parse and cast columns to a smaller type\n#     df.rename(columns={\"square_feet\": \"log_square_feet\"}, inplace=True)\n#     df['log_square_feet'] = np.float16(np.log(df['log_square_feet']))\n    df['year_built'] = np.uint8(df['year_built']-1900)\n    df['floor_count'] = np.uint8(df['floor_count'])\n\n    # remove redundant columns\n    for col in df.columns:\n        if col in ['timestamp', 'row_id']:\n            del df[col]\n\n    # extract target column\n    if 'meter_reading' in df.columns:\n        df['meter_reading'] = np.log1p(df['meter_reading']/ df['square_feet']).astype(np.float32) # comp metric uses log errors \n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2021-08-17T16:26:23.940649Z","iopub.execute_input":"2021-08-17T16:26:23.940963Z","iopub.status.idle":"2021-08-17T16:26:23.949798Z","shell.execute_reply.started":"2021-08-17T16:26:23.940937Z","shell.execute_reply":"2021-08-17T16:26:23.948866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_transform = ASHRAE3Preprocessor(train, data_ratios)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T16:26:23.951394Z","iopub.execute_input":"2021-08-17T16:26:23.952136Z","iopub.status.idle":"2021-08-17T16:26:38.953979Z","shell.execute_reply.started":"2021-08-17T16:26:23.952066Z","shell.execute_reply":"2021-08-17T16:26:38.953107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_transform.sample(7)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T16:26:38.955351Z","iopub.execute_input":"2021-08-17T16:26:38.955897Z","iopub.status.idle":"2021-08-17T16:26:39.964482Z","shell.execute_reply.started":"2021-08-17T16:26:38.955860Z","shell.execute_reply":"2021-08-17T16:26:39.963706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfig, ax = plt.subplots(figsize=(16,8))\n# use a ranked correlation to catch nonlinearities\n# plot train in all columns except year, taking 100100 random samples, checking correlation using the method 'spearman'\ncorr = train_transform[[col for col in train_transform.columns if col != 'year']].sample(100100).corr(method='spearman')\n_ = sns.heatmap(corr, annot=True,\n                xticklabels=corr.columns.values,\n                yticklabels=corr.columns.values)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T16:26:39.965594Z","iopub.execute_input":"2021-08-17T16:26:39.965938Z","iopub.status.idle":"2021-08-17T16:26:43.309704Z","shell.execute_reply.started":"2021-08-17T16:26:39.965908Z","shell.execute_reply":"2021-08-17T16:26:43.308693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train k folds","metadata":{}},{"cell_type":"code","source":"# target = meter_reading\n# force the model to use the weather data instead of dates, to avoid overfitting to the past history\nfeatures = [col for col in train_transform.columns if col not in [target, 'year', 'month', 'day']]","metadata":{"execution":{"iopub.status.busy":"2021-08-17T16:26:43.311263Z","iopub.execute_input":"2021-08-17T16:26:43.311625Z","iopub.status.idle":"2021-08-17T16:26:43.316160Z","shell.execute_reply.started":"2021-08-17T16:26:43.311591Z","shell.execute_reply":"2021-08-17T16:26:43.315226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fit_regressor(tr_idx, val_idx, features_arr, target_str):\n    # train\n    tr_x, tr_y = train_transform[features_arr].iloc[tr_idx], train_transform[target_str][tr_idx]\n    # evaluating (\"test\")\n    vl_x, vl_y = train_transform[features_arr].iloc[val_idx], train_transform[target_str][val_idx]\n    print({'train_transform size':len(tr_x), 'eval size':len(vl_x)})\n\n    tr_data = lgb.Dataset(tr_x, label=tr_y)\n    vl_data = lgb.Dataset(vl_x, label=vl_y)  \n    clf = lgb.LGBMRegressor(n_estimators=6000,\n                            learning_rate=0.28,\n                            feature_fraction=0.9,\n                            subsample=0.2,  # batches of 20% of the data\n                            subsample_freq=1,\n                            num_leaves=20,\n                            metric='rmse')\n    # Metric: Root Mean Square Error (RMSE), it tells you how concentrated the data is around the line of best fit.\n    clf.fit(tr_x, tr_y,\n            eval_set=[(vl_x, vl_y)],\n            early_stopping_rounds=50,\n            verbose=200)\n    return clf","metadata":{"execution":{"iopub.status.busy":"2021-08-17T16:26:43.317611Z","iopub.execute_input":"2021-08-17T16:26:43.318302Z","iopub.status.idle":"2021-08-17T16:26:43.330577Z","shell.execute_reply.started":"2021-08-17T16:26:43.318263Z","shell.execute_reply":"2021-08-17T16:26:43.329768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"folds = 4\nseed = 42\nkf = StratifiedKFold(n_splits=folds, shuffle=True, random_state=seed) # Provides train/test indices to split data in train/test sets.\n# oof_pred = np.zeros(train_transform.shape[0])  # out of fold predictions\nmodels = []\n\n## generating 4 train/test pair of index_arrays, and analizing wich give the better results.\nfor tr_idx, val_idx in tqdm(kf.split(train_transform, train_transform['building_id']), total=folds): # train/test indices\n    clf = fit_regressor(tr_idx, val_idx, features, target)\n    models.append(clf)\n\ngc.collect() # trigger a manual garbage collection process, cleans up a huge amount of objects.","metadata":{"execution":{"iopub.status.busy":"2021-08-17T16:26:43.335004Z","iopub.execute_input":"2021-08-17T16:26:43.335262Z","iopub.status.idle":"2021-08-17T16:47:17.838599Z","shell.execute_reply.started":"2021-08-17T16:26:43.335238Z","shell.execute_reply":"2021-08-17T16:47:17.837786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Algorithm ID3 regresion\n1. Calculate the initial system entropy based on the **objective** variable to predict.\n    * Entropy: Determine wich parameters are more important than others to have a better sort in the tree.","metadata":{}},{"cell_type":"markdown","source":"## Feature importance","metadata":{}},{"cell_type":"code","source":"_ = lgb.plot_importance(models[1], importance_type='gain', figsize=(16,8))","metadata":{"tags":[],"execution":{"iopub.status.busy":"2021-08-17T16:47:17.840692Z","iopub.execute_input":"2021-08-17T16:47:17.841058Z","iopub.status.idle":"2021-08-17T16:47:18.100553Z","shell.execute_reply.started":"2021-08-17T16:47:17.841020Z","shell.execute_reply":"2021-08-17T16:47:18.099636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# calculate the mean feature importance, so that we can update 'unimportant_cols' above\nfeature_importance = np.mean([m._Booster.feature_importance(importance_type='gain') for m in models], axis=0)\nsorted(zip(feature_importance, train_transform.columns), reverse=True)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T16:47:18.101872Z","iopub.execute_input":"2021-08-17T16:47:18.102218Z","iopub.status.idle":"2021-08-17T16:47:18.111543Z","shell.execute_reply.started":"2021-08-17T16:47:18.102182Z","shell.execute_reply":"2021-08-17T16:47:18.110544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Checking in train data RMSE error.","metadata":{}},{"cell_type":"code","source":"features = [col for col in train_transform.columns if col not in [target, 'year', 'month', 'day']]\ntr_idx = np.random.randint(0, 1552000, 1000000)\ntr_x, tr_y = train_transform[features].iloc[tr_idx], train_transform[target][tr_idx]","metadata":{"execution":{"iopub.status.busy":"2021-08-17T16:47:18.112947Z","iopub.execute_input":"2021-08-17T16:47:18.113320Z","iopub.status.idle":"2021-08-17T16:47:18.810128Z","shell.execute_reply.started":"2021-08-17T16:47:18.113252Z","shell.execute_reply":"2021-08-17T16:47:18.809225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dic = {\"real\": tr_y, \"prediction\": models[1].predict(tr_x)}\np_df = pd.DataFrame(data = dic)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T16:47:18.811476Z","iopub.execute_input":"2021-08-17T16:47:18.811911Z","iopub.status.idle":"2021-08-17T16:47:26.507385Z","shell.execute_reply.started":"2021-08-17T16:47:18.811871Z","shell.execute_reply":"2021-08-17T16:47:26.506452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"actual = p_df[\"real\"].values\npredicted = p_df[\"prediction\"].values\nmse = sklearn.metrics.mean_squared_error(actual, predicted)\nrmse = np.sqrt(mse)\nprint(rmse)","metadata":{"tags":[],"execution":{"iopub.status.busy":"2021-08-17T16:47:26.508623Z","iopub.execute_input":"2021-08-17T16:47:26.508998Z","iopub.status.idle":"2021-08-17T16:47:26.521614Z","shell.execute_reply.started":"2021-08-17T16:47:26.508962Z","shell.execute_reply":"2021-08-17T16:47:26.520739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Check prediction in the same test sample","metadata":{}},{"cell_type":"code","source":"# the counts above expose the missing data (Should we drop or refill the missing data?)\nprint(\"Ratio of available data (not NAN's):\")\ndata_ratios = test.count()/len(test)\ndata_ratios","metadata":{"execution":{"iopub.status.busy":"2021-08-17T16:47:26.523146Z","iopub.execute_input":"2021-08-17T16:47:26.523541Z","iopub.status.idle":"2021-08-17T16:47:31.383516Z","shell.execute_reply.started":"2021-08-17T16:47:26.523505Z","shell.execute_reply":"2021-08-17T16:47:31.382661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load and pre-process test data\ntest_transform = ASHRAE3Preprocessor(test, data_ratios)\ntest_transform.sample(7)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T16:47:31.384767Z","iopub.execute_input":"2021-08-17T16:47:31.385116Z","iopub.status.idle":"2021-08-17T16:48:04.525099Z","shell.execute_reply.started":"2021-08-17T16:47:31.385087Z","shell.execute_reply":"2021-08-17T16:48:04.524238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"folds = 4\nmeter = 1 # pick a meter\nbuilding_id = 1258  # a building with all 4 meters\nfeatures = [col for col in train_transform.columns if col not in [target, 'year', 'month', 'day']]\n\ndef recover_timestamp(x):\n    ''' reassemble timestamp using date components '''\n    return datetime.datetime.strptime(f'{x.year}-{x.month}-{x.day} {x.hour}', '%y-%m-%d %H')\n\nfig, ax = plt.subplots(figsize=(16,4))\nplt.title(f'Building {building_id} Meter {meter} on all {folds} prediction folds')\nax.xaxis.set_tick_params(rotation=30, labelsize=10)\nax2 = ax.twinx()\n\ntrain_sample = train_transform[(train_transform['building_id'] == building_id) & (train_transform['meter'] == meter)]  # same training sample as before\ntest_sample = test_transform[(test_transform['building_id'] == building_id) & (test_transform['meter'] == meter)]   # and the same meter in the test set\n\n# plot training sample\ndates = matplotlib.dates.date2num(train_sample[['year', 'month', 'day', 'hour']].apply(recover_timestamp, axis=1))\nax2.plot_date(dates, train_sample['meter_reading'], '-', label='train', alpha=0.8)\nax.plot_date(dates, train_sample['air_temperature'], '.', color='tab:cyan', label='air_temperature_train')\n\n\n# plot prediction sample\ndates = matplotlib.dates.date2num(test_sample[['year', 'month', 'day', 'hour']].apply(recover_timestamp, axis=1))\nax.plot_date(dates, test_sample['air_temperature'], '.', color='tab:cyan', label='air_temperature')\nfor i,model in enumerate(models):\n    ax2.plot_date(dates, np.expm1(model.predict(test_sample[features])), '-', label=f'prediction{i}', alpha=0.4)\n\nax.set_ylabel('air_temperature'); ax2.set_ylabel('meter_reading (+prediction)')\nax.legend(loc='upper left'); ax2.legend(loc='upper right')\n_ = plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-17T16:48:04.526305Z","iopub.execute_input":"2021-08-17T16:48:04.526610Z","iopub.status.idle":"2021-08-17T16:48:07.033493Z","shell.execute_reply.started":"2021-08-17T16:48:04.526583Z","shell.execute_reply":"2021-08-17T16:48:07.032727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save/ load the model to a file","metadata":{}},{"cell_type":"code","source":"# # save the model to disk\n# filename = './models_saved/ASHRAE3_ID3_16082021_3.sav'\n# pickle.dump(models, open(filename, 'wb'))","metadata":{"execution":{"iopub.status.busy":"2021-08-17T16:48:07.034752Z","iopub.execute_input":"2021-08-17T16:48:07.035113Z","iopub.status.idle":"2021-08-17T16:48:07.038872Z","shell.execute_reply.started":"2021-08-17T16:48:07.035077Z","shell.execute_reply":"2021-08-17T16:48:07.037884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # load the model from disk\n# filename = './models_saved/ASHRAE3_ID3_16082021_3.sav'\n# models = pickle.load(open(filename, 'rb'))\n# models","metadata":{"execution":{"iopub.status.busy":"2021-08-17T16:48:07.040250Z","iopub.execute_input":"2021-08-17T16:48:07.040603Z","iopub.status.idle":"2021-08-17T16:48:07.048907Z","shell.execute_reply.started":"2021-08-17T16:48:07.040547Z","shell.execute_reply":"2021-08-17T16:48:07.047989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test inferance and submission","metadata":{}},{"cell_type":"code","source":"# split test data into batches\nset_size = len(test_transform)\niterations = 50\nbatch_size = set_size // iterations\n\nprint(set_size, iterations, batch_size)\nassert set_size == iterations * batch_size","metadata":{"execution":{"iopub.status.busy":"2021-08-17T16:48:07.050164Z","iopub.execute_input":"2021-08-17T16:48:07.050549Z","iopub.status.idle":"2021-08-17T16:48:07.059390Z","shell.execute_reply.started":"2021-08-17T16:48:07.050515Z","shell.execute_reply":"2021-08-17T16:48:07.058413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meter_reading = []\nfor i in tqdm(range(iterations)):\n    pos = i*batch_size\n    fold_preds = [model.predict(test_transform[features].iloc[pos : pos+batch_size]) for model in models]\n    meter_reading.extend(np.mean(fold_preds, axis=0)) # Using all the models and making the mean between each other.\n\nprint(len(meter_reading))\nassert len(meter_reading) == set_size","metadata":{"tags":[],"execution":{"iopub.status.busy":"2021-08-17T16:48:07.060736Z","iopub.execute_input":"2021-08-17T16:48:07.061084Z","iopub.status.idle":"2021-08-17T17:08:00.155166Z","shell.execute_reply.started":"2021-08-17T16:48:07.061051Z","shell.execute_reply":"2021-08-17T17:08:00.154327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Save object data","metadata":{}},{"cell_type":"code","source":"# filename = './models_saved/meter_reading_1.csv'\n# out_df = pd.DataFrame(meter_reading)\n# out_df.to_csv(filename, index=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T17:08:00.156523Z","iopub.execute_input":"2021-08-17T17:08:00.157100Z","iopub.status.idle":"2021-08-17T17:08:00.160809Z","shell.execute_reply.started":"2021-08-17T17:08:00.157061Z","shell.execute_reply":"2021-08-17T17:08:00.159994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare data for submision.","metadata":{}},{"cell_type":"code","source":"# for our outcome in validation\ndef correct_error_meter0_scoring_use(df):\n    out = df\n    new_values = out[\"meter_reading\"] * 3.4118\n    out.loc[out.meter == 0, \"meter_reading\"] = new_values\n    return out","metadata":{"execution":{"iopub.status.busy":"2021-08-17T17:08:00.162192Z","iopub.execute_input":"2021-08-17T17:08:00.162761Z","iopub.status.idle":"2021-08-17T17:08:00.179170Z","shell.execute_reply.started":"2021-08-17T17:08:00.162722Z","shell.execute_reply":"2021-08-17T17:08:00.178375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train = correct_error_meter0_scoring_use(train)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T17:08:00.180492Z","iopub.execute_input":"2021-08-17T17:08:00.180855Z","iopub.status.idle":"2021-08-17T17:08:00.188137Z","shell.execute_reply.started":"2021-08-17T17:08:00.180805Z","shell.execute_reply":"2021-08-17T17:08:00.187215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meter_reading_exp = np.expm1(meter_reading) # Calculate exp(x) - 1 for all elements in the array.","metadata":{"execution":{"iopub.status.busy":"2021-08-17T17:08:00.189484Z","iopub.execute_input":"2021-08-17T17:08:00.189866Z","iopub.status.idle":"2021-08-17T17:08:07.454930Z","shell.execute_reply.started":"2021-08-17T17:08:00.189804Z","shell.execute_reply":"2021-08-17T17:08:07.454073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meter_reading_exp = meter_reading_exp * test_transform['square_feet']","metadata":{"execution":{"iopub.status.busy":"2021-08-17T17:08:07.456153Z","iopub.execute_input":"2021-08-17T17:08:07.456491Z","iopub.status.idle":"2021-08-17T17:08:07.597698Z","shell.execute_reply.started":"2021-08-17T17:08:07.456459Z","shell.execute_reply":"2021-08-17T17:08:07.596883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dic = {\"pred\": meter_reading_exp}\ndf_pred_out = pd.DataFrame(data= dic)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T17:08:07.599095Z","iopub.execute_input":"2021-08-17T17:08:07.599444Z","iopub.status.idle":"2021-08-17T17:08:07.722678Z","shell.execute_reply.started":"2021-08-17T17:08:07.599409Z","shell.execute_reply":"2021-08-17T17:08:07.721878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# target's log-log histogram:\nax = df_pred_out.pred.hist()\nax.set_yscale('log')\n\n# describe raw values first\ndf_pred_out.pred.describe()","metadata":{"execution":{"iopub.status.busy":"2021-08-17T17:08:07.723901Z","iopub.execute_input":"2021-08-17T17:08:07.724270Z","iopub.status.idle":"2021-08-17T17:08:10.512505Z","shell.execute_reply.started":"2021-08-17T17:08:07.724229Z","shell.execute_reply":"2021-08-17T17:08:10.511548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save submision","metadata":{}},{"cell_type":"code","source":"np.clip([-1, 1,2,3,4], a_min=0, a_max=None)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T17:08:10.513928Z","iopub.execute_input":"2021-08-17T17:08:10.514275Z","iopub.status.idle":"2021-08-17T17:08:10.520737Z","shell.execute_reply.started":"2021-08-17T17:08:10.514241Z","shell.execute_reply":"2021-08-17T17:08:10.519707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(f'{path}/sample_submission.csv')\nsubmission['meter_reading'] = np.clip(meter_reading_exp, a_min=0, a_max=None) # clip min at zero","metadata":{"execution":{"iopub.status.busy":"2021-08-17T17:08:10.525118Z","iopub.execute_input":"2021-08-17T17:08:10.525425Z","iopub.status.idle":"2021-08-17T17:08:20.722929Z","shell.execute_reply.started":"2021-08-17T17:08:10.525388Z","shell.execute_reply":"2021-08-17T17:08:20.722093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)\nsubmission.head(9)","metadata":{"execution":{"iopub.status.busy":"2021-08-17T17:09:31.739896Z","iopub.execute_input":"2021-08-17T17:09:31.740226Z","iopub.status.idle":"2021-08-17T17:11:42.695377Z","shell.execute_reply.started":"2021-08-17T17:09:31.740196Z","shell.execute_reply":"2021-08-17T17:11:42.694578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}