{"cells":[{"metadata":{},"cell_type":"markdown","source":"### ASHRAE - Great Energy Predictor III"},{"metadata":{"_kg_hide-input":true,"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd \nimport os, gc\nimport random\nimport datetime\n\nfrom tqdm import tqdm_notebook as tqdm #progress tool bar\n\n# matplotlib and seaborn for plotting\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom pandas.plotting import register_matplotlib_converters\nregister_matplotlib_converters()\n\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import KFold, StratifiedKFold\nfrom sklearn import preprocessing\nfrom sklearn.metrics import mean_squared_error, mean_squared_log_error\nimport lightgbm as lgb\nimport shap","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for dirname,blank,filenames in os.walk('../input/ashrae-energy-prediction'):\n    print(dirname,blank,filenames)\n    for file in filenames:\n        print(os.path.join(dirname,file))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" **Loading data**"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\npath='../input/ashrae-energy-prediction'\nunimportant_cols = []\ntarget = 'meter_reading'\n#function to load data\ndef load_data(source='train', path=path):\n    assert source in ['train', 'test']\n    df_building = pd.read_csv(f'{path}/building_metadata.csv', \n                              dtype={'building_id':np.uint16, 'site_id':np.uint8})\n    df_weather  = pd.read_csv(f'{path}/weather_{source}.csv', parse_dates=['timestamp'],\n                                                           dtype={'site_id':np.uint8, 'air_temperature':np.float16,\n                                                                  'cloud_coverage':np.float16, 'dew_temperature':np.float16,\n                                                                  'precip_depth_1_hr':np.float16},\n                                                           usecols=lambda c: c not in unimportant_cols)\n    df = pd.read_csv(f'{path}/{source}.csv', \n                     dtype={'building_id':np.uint16, 'meter':np.uint8}, \n                     parse_dates=['timestamp'])\n\n    return df_building,df_weather,df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## a very simple Function to reduce the DF size \ndef reduce_mem_usage(df, verbose=True):\n    numerics = ['int8','int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':# comparing string\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# load and display some samples\ndf_building,df_weather,df_train = load_data('train')\ndf_building_train=reduce_mem_usage(df_building)\ndf_weather_train=reduce_mem_usage(df_weather)\ndf_train=reduce_mem_usage(df_train)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Test data"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# load and display some samples\ndf_building_test,df_weather_test,df_test = load_data('test')\ndf_building_test=reduce_mem_usage(df_building_test)\ndf_weather_test=reduce_mem_usage(df_weather_test)\ndf_test=reduce_mem_usage(df_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Aligning timestamp \n#### Ref: https://www.kaggle.com/frednavruzov/aligning-temperature-timestamp<br>\n###### Align timestamps\nTimestap data is not in their local time. As energy consumptions are related to the local time, an alighment is nescessary before using timestamp. \n\nThe credit goes to [this kernel](https://www.kaggle.com/nz0722/aligned-timestamp-lgbm-by-meter-type) for the idea. Refer it for more details and explanation about below code."},{"metadata":{},"cell_type":"markdown","source":"Aligning timestamp process:\n1. concating the weather data"},{"metadata":{"trusted":true},"cell_type":"code","source":"weather = pd.concat([df_weather_train,df_weather_test],ignore_index=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"1. The hottest time of the day is around 2 p.m. Heat continues building up after noon, when the sun is highest in the sky, as long as more heat is arriving at the earth than leaving. By 2 p.m. or so, the sun is low enough in the sky for outgoing heat to be greater than incoming."},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_key = ['site_id', 'timestamp']\n#small data requirement for timestamp alignment (alginment is on the basis of air temprature which is highest at 3:00PM or 15:00 hrs)\ntemp_skeleton = weather[weather_key + ['air_temperature']].drop_duplicates(subset=weather_key).sort_values(by=weather_key).copy()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"ranking the temprature of particular date w.r.t air temprature for each site_id"},{"metadata":{"trusted":true},"cell_type":"code","source":"# calculate ranks of hourly temperatures within date/site_id chunks (extra feature is added on temprory dataset)\ntemp_skeleton['temp_rank'] = temp_skeleton.groupby(['site_id', temp_skeleton.timestamp.dt.date])['air_temperature'].rank('average')\n#calculate avg ranking of temprature including other searches","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create a dataframe of site_ids (0-16) x mean hour rank of temperature within day (0-23)(columns)\ndf_2d = temp_skeleton.groupby(['site_id', temp_skeleton.timestamp.dt.hour])['temp_rank'].mean().unstack(level=1)\n# Subtract the columnID of temperature peak by 14, getting the timestamp alignment gap.\nsite_ids_offsets = pd.Series(df_2d.values.argmax(axis=1) - 14)\nsite_ids_offsets.index.name = 'site_id'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#aligning timestamp using above result\ndef timestamp_align(df):\n    df['offset'] = df.site_id.map(site_ids_offsets)\n    df['timestamp_aligned'] = (df.timestamp - pd.to_timedelta(df.offset, unit='H'))\n    df['timestamp'] = df['timestamp_aligned']\n    del df['timestamp_aligned']\n    del df['offset']\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_weather_train_aligned=timestamp_align(df_weather_train)\ndf_weather_test_aligned=timestamp_align(df_weather_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def merging_data(df,df_building,df_weather):    \n    df = df.merge(df_building, on='building_id', how='left')\n    df = df.merge(df_weather, on=['site_id', 'timestamp'], how='left')\n    del df_building\n    del df_weather\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_aligned=merging_data(df_train,df_building,df_weather_train_aligned)\ndf_test_aligned=merging_data(df_test,df_building_test,df_weather_test_aligned)\nprint(f'shape of traindata before alignment: {df_train.shape} and shape of test data before alignment: {df_test.shape}')\nprint(f'shape of traindata after alignment: {df_train_aligned.shape} and shape of test data after alignment: {df_test_aligned.shape}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#removing unwanted columns\ndel df_test_aligned['row_id']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_aligned=reduce_mem_usage(df_train_aligned)\ndf_test_aligned=reduce_mem_usage(df_test_aligned)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'memory used in merged train data {df_train_aligned.info(verbose=False)} and memory test data:{df_test_aligned.info(verbose=False)} ')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Removing weired data on site_id==0\nthere is already so much discussion on this issue so there is no need to explain \nhttps://www.kaggle.com/corochann/ashrae-training-lgbm-by-meter-type"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_date_usage(train_df,site_id,meter,building_id):\n    train_temp_df=train_df[train_df['site_id']==site_id]\n    train_temp_df=train_temp_df[train_df['meter']==meter]\n    train_temp_df = train_temp_df[train_temp_df['building_id'] == building_id]   \n    train_temp_df['date']=train_temp_df['timestamp'].dt.date\n    train_temp_df_meter = train_temp_df.groupby('date')['meter_reading'].sum()\n    train_temp_df_meter = train_temp_df_meter.to_frame().reset_index()\n    plt.plot(train_temp_df_meter['date'],train_temp_df_meter['meter_reading'])\n    plt.xlabel('date')\n    plt.ylabel('meter_reading_transform')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_date_usage(df_train_aligned,0,0,0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#removing weirder data from site_id=0; All electricity meter is 0 until May 20 for site_id == 0 and meter=0\n#building 0 to 104 lies on site_id 0 only\ndf_train_aligned=df_train_aligned.query('not(building_id <= 104 & meter == 0 & timestamp <= \"2016-05-20\")')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'shape of traindata before alignment: {df_train.shape} and shape of test data before alignment: {df_test.shape}')\nprint(f'shape of traindata after alignment: {df_train_aligned.shape} and shape of test data after alignment: {df_test_aligned.shape}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### feature extraction\n\n* Hour of day.\n* Business hours or not.(not applicable due to different type of building)\n* Weekend or not.\n* Season of the year.\n* Public holiday or not.+weekend"},{"metadata":{"trusted":true},"cell_type":"code","source":"#PREPROCESSING TIMESTAMP IN TRAIN AND TEST\nimport holidays\ndef timestamp_preprocess(df):\n    df['date']=df['timestamp'].dt.date\n    df['hour']=df['timestamp'].dt.hour#hour of day\n    df['day']=df['timestamp'].dt.day\n    df['weekday']=df['timestamp'].dt.weekday\n    df['month']=df['timestamp'].dt.month\n    import holidays\n    us_holidays =holidays.US()\n    df['holiday']=df['date'].apply(lambda x: us_holidays.get(x))\n    df['holiday']=df['holiday'].apply(lambda x:0 if x==None else 1)\n    df['holiday'][df.weekday == 6]=1#sun\n    df['holiday'][df.weekday == 5]=1 #sat  \n    df['square_feet']=np.float16(np.log(df['square_feet']))#normalising floorspace\n    del df['floor_count']\n    del df['year_built']\n    del df['cloud_coverage']\n    del df['weekday']\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#preprocessing train data\ndf_train_preprocess=reduce_mem_usage(timestamp_preprocess(df_train_aligned))\ndf_train_preprocess['meter_reading_transform'] = np.log1p(df_train_preprocess['meter_reading']).astype(np.float32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ngc.collect()\n#preprocessing test data\ndf_test_preprocess=reduce_mem_usage(timestamp_preprocess(df_test_aligned))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#removing redundant columns\ngc.collect()\ndel df_train_preprocess['meter_reading']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nfor feature in ['air_temperature','dew_temperature','wind_speed','precip_depth_1_hr']:\n    sns.distplot(df_train_preprocess[feature], hist=False)\n    plt.show(sns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"#### filling missing value"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Import label encoder \nfrom sklearn import preprocessing \n# label_encoder object knows how to understand word labels. \nlabel_encoder = preprocessing.LabelEncoder() \n# Encode labels in column 'species'. \ndf_train_preprocess['primary_use']= label_encoder.fit_transform(df_train_preprocess['primary_use'])\ndf_test_preprocess['primary_use']=label_encoder.fit_transform(df_test_preprocess['primary_use'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_preprocess=reduce_mem_usage(df_train_preprocess)\ndf_test_preprocess=reduce_mem_usage(df_test_preprocess)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#removing unwanted columns\ndef remove_redundant_cols(df):\n    unwanted_columns=['wind_direction','wind_speed','sea_level_pressure']\n    for col in unwanted_columns:\n        del df[col]\n    return df  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ndf_test_preprocess=remove_redundant_cols(df_test_preprocess)\ndf_train_preprocess=remove_redundant_cols(df_train_preprocess)\nprint(f'memory usage df_train_preprocess:{df_train_preprocess.info(verbose=False)} and memory usage of  df_test_preprocess: {df_test_preprocess.info(verbose=False)}')\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"list_missing_columns=['air_temperature','dew_temperature','precip_depth_1_hr']\ndef fill_na(df):\n    for value in list_missing_columns:\n        df[value] = df[value].fillna(df.groupby('primary_use')[value].transform('mean'))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_preprocess=fill_na(df_train_preprocess)\ndf_test_preprocess=fill_na(df_test_preprocess)\ndf_train_preprocess=df_train_preprocess.reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_preprocess.tail()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### timeseries feature (mean,median,lag,deviation)"},{"metadata":{"trusted":true},"cell_type":"code","source":"#generating rolling mean,std deviation, max,min,actual_value\ndef rolling_feature(df):\n    df['air_temperature_mean'] = df['air_temperature'].rolling(window=7,center=False).mean()\n    df['dew_temperature_mean'] = df['dew_temperature'].rolling(window=7,center=False).mean()\n    df['precip_depth_1_hr_mean'] = df['precip_depth_1_hr'].rolling(window=7,center=False).mean()\n    df['air_temperature_std'] = df['air_temperature'].rolling(window=7,center=False).std()\n    df['dew_temperature_std'] = df['dew_temperature'].rolling(window=7,center=False).std()\n    df['precip_depth_1_hr_std'] = df['precip_depth_1_hr'].rolling(window=7,center=False).std()\n    df['air_temperature_max'] = df['air_temperature'].rolling(window=7,center=False).max()\n    df['dew_temperature_max'] = df['dew_temperature'].rolling(window=7,center=False).max()\n    df['precip_depth_1_hr_max'] = df['precip_depth_1_hr'].rolling(window=7,center=False).max()\n    df['air_temperature_min'] = df['air_temperature'].rolling(window=7,center=False).min()\n    df['dew_temperature_min'] = df['dew_temperature'].rolling(window=7,center=False).min()\n    df['precip_depth_1_hr_min'] = df['precip_depth_1_hr'].rolling(window=7,center=False).min()\n    df[\"air_temperature_mean\"].fillna( method ='bfill', inplace = True) \n    df[\"dew_temperature_mean\"].fillna( method ='bfill', inplace = True) \n    df[\"precip_depth_1_hr_mean\"].fillna( method ='bfill', inplace = True) \n    df[\"air_temperature_std\"].fillna( method ='bfill', inplace = True) \n    df[\"dew_temperature_std\"].fillna( method ='bfill', inplace = True) \n    df[\"precip_depth_1_hr_std\"].fillna( method ='bfill', inplace = True)\n    df[\"air_temperature_min\"].fillna( method ='bfill', inplace = True) \n    df[\"dew_temperature_min\"].fillna( method ='bfill', inplace = True) \n    df[\"precip_depth_1_hr_min\"].fillna( method ='bfill', inplace = True) \n    df[\"air_temperature_max\"].fillna( method ='bfill', inplace = True) \n    df[\"dew_temperature_max\"].fillna( method ='bfill', inplace = True) \n    df[\"precip_depth_1_hr_max\"].fillna( method ='bfill', inplace = True)\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_preprocess=reduce_mem_usage(rolling_feature(df_train_preprocess))\ndf_test_preprocess=reduce_mem_usage(rolling_feature(df_test_preprocess))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_preprocess.tail(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### baseline model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# # force the model to use the weather data instead of dates, to avoid overfitting to the past history\nfeatures = [col for col in df_train_preprocess.columns if col not in ['timestamp','date','meter_reading_transform', 'year', 'month', 'day']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"folds = 4\nseed = 42\ntarget='meter_reading_transform'\nkf = StratifiedKFold(n_splits=folds, shuffle=True, random_state=seed)\nmodels = []\noof_pred = np.zeros(df_train_preprocess.shape[0])  # out of fold predictions\n\n## stratify data by building_id\nfor i, (tr_idx, val_idx) in tqdm(enumerate(kf.split(df_train_preprocess, df_train_preprocess['meter'])), total=folds):\n    def fit_regressor(tr_idx, val_idx): # memory closure\n        tr_x, tr_y = df_train_preprocess[features].iloc[tr_idx],  df_train_preprocess[target].iloc[tr_idx]\n        vl_x, vl_y = df_train_preprocess[features].iloc[val_idx], df_train_preprocess[target].iloc[val_idx]\n        print({'fold':i, 'train size':len(tr_x), 'eval size':len(vl_x)})\n\n        tr_data = lgb.Dataset(tr_x, label=tr_y)\n        vl_data = lgb.Dataset(vl_x, label=vl_y)  \n        clf = lgb.LGBMRegressor(n_estimators=1000,\n                                learning_rate=0.4,\n                                feature_fraction=0.9,\n                                subsample=0.25,  # batches of 25% of the data\n                                subsample_freq=1,\n                                num_leaves=20,\n                                lambda_l1=1,  # regularisation\n                                lambda_l2=1,\n                                metric='rmse')\n        clf.fit(tr_x, tr_y,\n                eval_set=[(vl_x, vl_y)],\n#                 early_stopping_rounds=50,\n                verbose=200)\n        # out of fold predictions\n        valid_prediticion = clf.predict(vl_x, num_iteration=clf.best_iteration_)\n        oof_loss = np.sqrt(mean_squared_error(vl_y, valid_prediticion)) # target is already in log scale\n        print(f'Fold:{i} RMSLE: {oof_loss:.4f}')\n        return clf, valid_prediticion\n\n    clf, oof_pred[val_idx] = fit_regressor(tr_idx, val_idx)\n    models.append(clf)\n    \ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## inference base line model"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"oof_loss = np.sqrt(mean_squared_error(df_train_preprocess[target], oof_pred)) # target is already in log scale\nprint(f'OOF RMSLE: {oof_loss:.4f}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Feature importance"},{"metadata":{"trusted":true},"cell_type":"code","source":" _ = lgb.plot_importance(models[0], importance_type='gain')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"# split test data into batches\nset_size = len(df_test_preprocess)\niterations = 100\nbatch_size = set_size // iterations\n\nprint(set_size, iterations, batch_size)\nassert set_size == iterations * batch_size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len (models)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"meter_reading = []\nfor i in tqdm(range(iterations)):\n    pos = i*batch_size\n    fold_preds = [np.expm1(model.predict(df_test_preprocess[features].iloc[pos : pos+batch_size])) for model in models]\n    meter_reading.extend(np.mean(fold_preds, axis=0))\n\nprint(len(meter_reading))\nassert len(meter_reading) == set_size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv(f'{path}/sample_submission.csv')\nsubmission['meter_reading'] = np.clip(meter_reading, a_min=0, a_max=None) # clip min at zero","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)\n# submission.head(9)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.5"}},"nbformat":4,"nbformat_minor":1}