{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# General imports\nimport numpy as np\nimport pandas as pd\nimport os, gc, warnings, random, math\n\nfrom sklearn import metrics\nfrom sklearn.model_selection import train_test_split\n\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"########################### Helpers\n#################################################################################\n## Seeder\n# :seed to make all processes deterministic     # type: int\ndef seed_everything(seed=0):\n    random.seed(seed)\n    np.random.seed(seed)\n    \ndef rmse(predictions, targets):\n    return np.sqrt(((predictions - targets) ** 2).mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"########################### Vars\n#################################################################################\nSEED = 42\nseed_everything(SEED)\nTARGET = 'meter_reading'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"########################### DATA LOAD\n#################################################################################\nprint('Load Data')\ntrain_df = pd.read_pickle('../input/ashrae-data-minification/train.pkl')\nbuilding_df = pd.read_pickle('../input/ashrae-data-minification/building_metadata.pkl')\ntrain_weather_df = pd.read_pickle('../input/ashrae-data-minification/weather_train.pkl')\n\ntest_df = pd.read_pickle('../input/ashrae-data-minification/test.pkl')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"########################### Check building_id \n#################################################################################\ntemp_df = test_df[~test_df['building_id'].isin(train_df['building_id'])]\nprint('No intersection:', len(temp_df))\ndel test_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"########################### Merge additional data\n#################################################################################\ntemp_df = train_df[['building_id']]\ntemp_df = temp_df.merge(building_df, on=['building_id'], how='left')\ndel temp_df['building_id']\ntrain_df = pd.concat([train_df, temp_df], axis=1)\n\ndel building_df, temp_df\n\ntemp_df = train_df[['site_id','timestamp']]\ntemp_df = temp_df.merge(train_weather_df, on=['site_id','timestamp'], how='left')\ndel temp_df['site_id'], temp_df['timestamp']\ntrain_df = pd.concat([train_df, temp_df], axis=1)\n\ndel train_weather_df, temp_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"########################### Model params\nimport lightgbm as lgb\nlgb_params = {\n                    'objective':'regression',\n                    'boosting_type':'gbdt',\n                    'metric':'rmse',\n                    'n_jobs':-1,\n                    'learning_rate':0.3, #for faster training\n                    'num_leaves': 2**8,\n                    'max_depth':-1,\n                    'tree_learner':'serial',\n                    'colsample_bytree': 0.7,\n                    'subsample_freq':1,\n                    'subsample':0.7,\n                    'n_estimators':1000,\n                    'max_bin':255,\n                    'verbose':-1,\n                    'seed': SEED,\n                    'early_stopping_rounds':100, \n                } ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"----"},{"metadata":{},"cell_type":"markdown","source":"## CV concept\n\n### Basics\n\n> Cross-validation is a technique for evaluating ML models \n> by training several ML models on subsets of the available \n> input data and evaluating them on the complementary \n> subset of the data. \n\n> In k-fold cross-validation, you split the input data \n> into k subsets of data (also known as folds).\n\n\n### Main strategy\n1. Divide Train set in subsets (Training set itself + Validation set)\n2. Define Validation Metric (in our case it is RMSE/RMSLE)\n3. Stop training when Validation metric stops improving\n4. Make predictions for Test set\n\nSeems simple but he devil's always in the details."},{"metadata":{"trusted":true},"cell_type":"code","source":"########################### Create Holdout sets\n#################################################################################\n# Holdout set 1\n# Split train set by building_id -> 20% to houldout\ntrain_buildings, test_buildings = train_test_split(train_df['site_id'].unique(), test_size=0.20, random_state=SEED)\n\nholdout_subset_1 = train_df[train_df['site_id'].isin(test_buildings)].reset_index(drop=True)\ntrain_df = train_df[train_df['site_id'].isin(train_buildings)].reset_index(drop=True)\n\n# Holdout set 2\n# Split train set by site_id -> 20% to houldout                   \ntrain_buildings, test_buildings = train_test_split(train_df['building_id'].unique(), test_size=0.20, random_state=SEED)\n\nholdout_subset_2 = train_df[train_df['building_id'].isin(test_buildings)].reset_index(drop=True)\ntrain_df = train_df[train_df['building_id'].isin(train_buildings)].reset_index(drop=True)\n                    \n# Holdout set 3\n# Split train set by month -> first and last months to holdout\nholdout_subset_3 = train_df[(train_df['DT_M']==1)|(train_df['DT_M']==12)].reset_index(drop=True)\ntrain_df = train_df[(train_df['DT_M']!=1)&(train_df['DT_M']!=12)].reset_index(drop=True)\n\n# Transform target and check shape\nfor df in [train_df, holdout_subset_1, holdout_subset_2, holdout_subset_3]:\n    df[TARGET] = np.log1p(df[TARGET])\n    print(df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"########################### Features to use and eval sets\n# for validation \"purity\" we will also remove site_id, building_id, DT_M\nremove_columns = ['timestamp','site_id','building_id','DT_M',TARGET]\nfeatures_columns = [col for col in list(train_df) if col not in remove_columns]\n\nX = train_df[features_columns]\ny = train_df[TARGET]\n\nsplit_by_building = train_df['building_id']\nsplit_by_site = train_df['site_id']\nsplit_by_month = train_df['DT_M']\n\ndel train_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Let's creat dataframes to compare results\n## We will join prepdictions\nRESULTS_1 = holdout_subset_1[[TARGET]]\nRESULTS_2 = holdout_subset_2[[TARGET]]\nRESULTS_3 = holdout_subset_3[[TARGET]]\n\nall_results = {\n        1: [RESULTS_1, holdout_subset_1, '    site_id holdout'],\n        2: [RESULTS_2, holdout_subset_2, 'building_id holdout'],\n        3: [RESULTS_3, holdout_subset_3, '      month holdout']\n    }\n\nfor _,df in all_results.items():\n    df[0]['test'] = 0    \n    print('Ground RMSE for', df[2], '|',\n          rmse(df[0][TARGET], df[0]['test']))\n    del df[0]['test']\n    print('#'*20)    \n    \n# We will always use same number of splits\n# for training model\n# Number of splits depends on data structure\n# something in range 5-10\n# 5 - is a common number of splits\n# 10+ is too much (we will not have enough diversity in data)\n# Here we will use 3 for faster training\n# but you can change it by yourself\nN_SPLITS = 3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# We don't know where to stop\n# so we will try to guess \n# number of boosting rounds\nfor n_rounds in [25,50,100,200]:\n    print('#'*20)\n    print('No Validation training...', n_rounds, 'boosting rounds')\n    corrected_lgb_params = lgb_params.copy()\n    corrected_lgb_params['n_estimators'] = n_rounds\n    corrected_lgb_params['early_stopping_rounds'] = None\n\n    train_data = lgb.Dataset(X, label=y)\n    \n    estimator = lgb.train(\n                corrected_lgb_params,\n                train_data\n            )\n\n    for _,df in all_results.items():\n        df[0]['no_validation_'+str(n_rounds)] = estimator.predict(df[1][features_columns])\n        print('RMSE for',\n              df[2], '|',\n              rmse(df[0][TARGET], df[0]['no_validation_'+str(n_rounds)]))\n        print('#'*20)\n\n# Be careful. We are printing rmse results\n# for our simulated test set\n# but in real Data set we do not have True labels (obviously)\n# and can't be sure that we stopped in right round\n# lb probing can give you some idea how good our training is\n# but this leads to nowhere -> overfits or completely bad results\n# bad practice for real life problems!","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Findings\n\nThe main finding here is that we have \"data leakage\" in our dataset. And not single one.\n* Leakage by site_id -> our model doesn't generalize well for unkown site_id\n* Leakage by building_id -> our model doesn't generalize well for unkown building_id\n\nWhat we can do here and do we have to do anything?\n\nGood thing is all our test buildings and test sites present in train set.\n\nProbably we don't need to smooth differences between them and can even make differences more explicit.\n\n---"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('#'*20)\nprint('KFold (with shuffle) training...')\n\nfrom sklearn.model_selection import KFold\nfolds = KFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED)\n\nfor _,df in all_results.items():\n    df[0]['shuffle_kfold'] = 0\n        \nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(X, y)):\n    print('Fold:',fold_+1)\n    tr_x, tr_y = X.iloc[trn_idx,:], y[trn_idx]    \n    vl_x, v_y = X.iloc[val_idx,:], y[val_idx]    \n    train_data = lgb.Dataset(tr_x, label=tr_y)\n    valid_data = lgb.Dataset(vl_x, label=v_y)  \n\n    estimator = lgb.train(\n            lgb_params,\n            train_data,\n            valid_sets = [train_data, valid_data],\n            verbose_eval = 100,\n        )\n    \n    for _,df in all_results.items():\n        df[0]['shuffle_kfold'] += estimator.predict(df[1][features_columns])/N_SPLITS\n\nfor _,df in all_results.items():\n    print('RMSE for', df[2], '|',\n          rmse(df[0][TARGET], df[0]['shuffle_kfold']))\n    print('#'*20)    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('#'*20)\nprint('KFold (no shuffle) training...')\n\nfrom sklearn.model_selection import KFold\nfolds = KFold(n_splits=N_SPLITS, shuffle=False, random_state=SEED)\n\nfor _,df in all_results.items():\n    df[0]['no_shuffle_kfold'] = 0\n        \nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(X, y)):\n    print('Fold:',fold_+1)\n    tr_x, tr_y = X.iloc[trn_idx,:], y[trn_idx]    \n    vl_x, v_y = X.iloc[val_idx,:], y[val_idx]    \n    train_data = lgb.Dataset(tr_x, label=tr_y)\n    valid_data = lgb.Dataset(vl_x, label=v_y)  \n\n    estimator = lgb.train(\n            lgb_params,\n            train_data,\n            valid_sets = [train_data, valid_data],\n            verbose_eval = 100,\n        )\n    \n    for _,df in all_results.items():\n        df[0]['no_shuffle_kfold'] += estimator.predict(df[1][features_columns])/N_SPLITS\n\nfor _,df in all_results.items():\n    print('RMSE for', df[2], '|',\n          rmse(df[0][TARGET], df[0]['no_shuffle_kfold']))\n    print('#'*20)    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Findings\n\nThe main finding here is that we have one more \"data leakage\".\n* Leakage by date/month\n\nConsumptions differ a lot month by month. \n\nWe can't exclude any data by month as we need to predict consumptions for the whole year.\n\nOur task becoming more and more interesting as we have to validate our features somehow.\n\nWe can't use normal kfold for validation because if the model knows how much energy was spent at 8 am it can make a good prediction for 9 am, but we don't have such data in our test set. \n\n---"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('#'*20)\nprint('GroupKFold building_id split training...') \n\nfrom sklearn.model_selection import GroupKFold\nfolds = GroupKFold(n_splits=N_SPLITS)\n\nfor _,df in all_results.items():\n    df[0]['Groupkfold_by_building'] = 0\n      \nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(X, y, groups=split_by_building)):\n    print('Fold:',fold_+1)\n    tr_x, tr_y = X.iloc[trn_idx,:], y[trn_idx]    \n    vl_x, v_y = X.iloc[val_idx,:], y[val_idx]    \n    train_data = lgb.Dataset(tr_x, label=tr_y)\n    valid_data = lgb.Dataset(vl_x, label=v_y)  \n\n    estimator = lgb.train(\n            lgb_params,\n            train_data,\n            valid_sets = [train_data, valid_data],\n            verbose_eval = 100,\n        )\n\n    for _,df in all_results.items():\n        df[0]['Groupkfold_by_building'] += estimator.predict(df[1][features_columns])/N_SPLITS\n\nfor _,df in all_results.items():\n    print('RMSE for', df[2], '|',\n          rmse(df[0][TARGET], df[0]['Groupkfold_by_building']))\n    print('#'*20)  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('#'*20)\nprint('GroupKFold site_id split training...') \n\nfrom sklearn.model_selection import GroupKFold\nfolds = GroupKFold(n_splits=N_SPLITS)\n\nfor _,df in all_results.items():\n    df[0]['Groupkfold_by_site'] = 0\n      \nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(X, y, groups=split_by_site)):\n    print('Fold:',fold_+1)\n    tr_x, tr_y = X.iloc[trn_idx,:], y[trn_idx]    \n    vl_x, v_y = X.iloc[val_idx,:], y[val_idx]    \n    train_data = lgb.Dataset(tr_x, label=tr_y)\n    valid_data = lgb.Dataset(vl_x, label=v_y)  \n\n    estimator = lgb.train(\n            lgb_params,\n            train_data,\n            valid_sets = [train_data, valid_data],\n            verbose_eval = 100,\n        )\n\n    for _,df in all_results.items():\n        df[0]['Groupkfold_by_site'] += estimator.predict(df[1][features_columns])/N_SPLITS\n\nfor _,df in all_results.items():\n    print('RMSE for', df[2], '|',\n          rmse(df[0][TARGET], df[0]['Groupkfold_by_site']))\n    print('#'*20)  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('#'*20)\nprint('GroupKFold month split training...') \n\nfrom sklearn.model_selection import GroupKFold\nfolds = GroupKFold(n_splits=N_SPLITS)\n\nfor _,df in all_results.items():\n    df[0]['Groupkfold_by_month'] = 0\n      \nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(X, y, groups=split_by_month)):\n    print('Fold:',fold_+1)\n    tr_x, tr_y = X.iloc[trn_idx,:], y[trn_idx]    \n    vl_x, v_y = X.iloc[val_idx,:], y[val_idx]    \n    train_data = lgb.Dataset(tr_x, label=tr_y)\n    valid_data = lgb.Dataset(vl_x, label=v_y)  \n\n    estimator = lgb.train(\n            lgb_params,\n            train_data,\n            valid_sets = [train_data, valid_data],\n            verbose_eval = 100,\n        )\n\n    for _,df in all_results.items():\n        df[0]['Groupkfold_by_month'] += estimator.predict(df[1][features_columns])/N_SPLITS\n\nfor _,df in all_results.items():\n    print('RMSE for', df[2], '|',\n          rmse(df[0][TARGET], df[0]['Groupkfold_by_month']))\n    print('#'*20)  ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Findings\n\nSame as before. \"Leakage\" prevents our model to generalize well.\n\n---"},{"metadata":{},"cell_type":"markdown","source":"### Summary\n\nFor test set predictions our training set MUST have all building_ids and all months to make more accurate predictions.\n\n\nI would recommend trying train/skip/validate for feature validation:\n\n* Train set - first 4 month\n* Skip - next 4 month\n* Valid set - last 4 month\n\n\nFor test set predictions use slightly more boosting rounds than validation scheme early stopping will show.\n\nTrain several seed models (not kfold, just different seed).\n\nAverage results.\n\n---"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}