{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"black_day = 10\noutlier = False\nrescale = False\n\ndebug=False\nnum_rounds = 200\n\nclip0=False \n\nfolds = 3 \n\nuse_ucf=False\nucf_clip=False\n\nucf_year = [2017, 2018] \n\npredmode='all' ","metadata":{"execution":{"iopub.status.busy":"2023-10-05T17:36:05.872302Z","iopub.execute_input":"2023-10-05T17:36:05.872799Z","iopub.status.idle":"2023-10-05T17:36:05.881407Z","shell.execute_reply.started":"2023-10-05T17:36:05.872753Z","shell.execute_reply":"2023-10-05T17:36:05.880480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport os\nfrom pathlib import Path\nimport random\nimport sys\n\nfrom tqdm import tqdm_notebook as tqdm\nimport numpy as np \nimport pandas as pd \n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom IPython.core.display import display, HTML\n\nfrom sklearn import preprocessing\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import LabelEncoder\nimport lightgbm as lgb\n\nfrom sklearn.metrics import mean_squared_error","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2023-10-05T17:36:07.785872Z","iopub.execute_input":"2023-10-05T17:36:07.786547Z","iopub.status.idle":"2023-10-05T17:36:09.416676Z","shell.execute_reply.started":"2023-10-05T17:36:07.786494Z","shell.execute_reply":"2023-10-05T17:36:09.415636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir('../input')","metadata":{"execution":{"iopub.status.busy":"2023-10-05T17:36:09.418358Z","iopub.execute_input":"2023-10-05T17:36:09.418760Z","iopub.status.idle":"2023-10-05T17:36:09.426753Z","shell.execute_reply.started":"2023-10-05T17:36:09.418709Z","shell.execute_reply":"2023-10-05T17:36:09.426000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfrom pandas.api.types import is_datetime64_any_dtype as is_datetime\nfrom pandas.api.types import is_categorical_dtype\n\ndef reduce_mem_usage(df, use_float16=False):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        if is_datetime(df[col]) or is_categorical_dtype(df[col]):\n            # skip datetime type or categorical type\n            continue\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if use_float16 and c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-10-05T17:36:09.427869Z","iopub.execute_input":"2023-10-05T17:36:09.428332Z","iopub.status.idle":"2023-10-05T17:36:09.446361Z","shell.execute_reply.started":"2023-10-05T17:36:09.428264Z","shell.execute_reply":"2023-10-05T17:36:09.445379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"zone_dict={0:4,1:0,2:7,3:4,4:7,5:0,6:4,7:4,8:4,9:5,10:7,11:4,12:0,13:5,14:4,15:4} \n\ndef set_local(df):\n    for sid, zone in zone_dict.items():\n        sids = df.site_id == sid\n        df.loc[sids, 'timestamp'] = df[sids].timestamp - pd.offsets.Hour(zone)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T17:36:09.448354Z","iopub.execute_input":"2023-10-05T17:36:09.448766Z","iopub.status.idle":"2023-10-05T17:36:09.462637Z","shell.execute_reply.started":"2023-10-05T17:36:09.448695Z","shell.execute_reply":"2023-10-05T17:36:09.461566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls ../input","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-10-05T17:36:09.465083Z","iopub.execute_input":"2023-10-05T17:36:09.465667Z","iopub.status.idle":"2023-10-05T17:36:10.578978Z","shell.execute_reply.started":"2023-10-05T17:36:09.465604Z","shell.execute_reply":"2023-10-05T17:36:10.577954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fast data loading\n\nThis kernel uses the preprocessed data from my previous kernel, [\nASHRAE: feather format for fast loading](https://www.kaggle.com/corochann/ashrae-feather-format-for-fast-loading), to accelerate data loading!","metadata":{}},{"cell_type":"code","source":"%%time\nroot = Path('../input/ashrae-feather-format-for-fast-loading')\n\ntrain_df = pd.read_feather(root/'train.feather')\n\nweather_train_df = pd.read_feather(root/'weather_train.feather')\n\nbuilding_meta_df = pd.read_feather(root/'building_metadata.feather')\n\nroot_median = Path('../input/site-meter-median')\ndf_groupNum_median = pd.read_pickle(root_median/'df_groupNum_median.pickle')\n","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-10-05T17:36:10.580936Z","iopub.execute_input":"2023-10-05T17:36:10.581228Z","iopub.status.idle":"2023-10-05T17:36:16.931957Z","shell.execute_reply.started":"2023-10-05T17:36:10.581181Z","shell.execute_reply":"2023-10-05T17:36:16.930834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"building_meta_df = building_meta_df.merge(train_df[['building_id','meter']].drop_duplicates(), on='building_id')","metadata":{"execution":{"iopub.status.busy":"2023-10-05T17:36:16.933670Z","iopub.execute_input":"2023-10-05T17:36:16.933979Z","iopub.status.idle":"2023-10-05T17:36:18.840002Z","shell.execute_reply.started":"2023-10-05T17:36:16.933916Z","shell.execute_reply":"2023-10-05T17:36:18.839200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Set group # (site-meter) for training models","metadata":{}},{"cell_type":"code","source":"building_meta_df['groupNum_train'] = building_meta_df['site_id'].astype('int')*10 + building_meta_df['meter'].astype('int')","metadata":{"execution":{"iopub.status.busy":"2023-10-05T17:36:18.841225Z","iopub.execute_input":"2023-10-05T17:36:18.841489Z","iopub.status.idle":"2023-10-05T17:36:18.874621Z","shell.execute_reply.started":"2023-10-05T17:36:18.841444Z","shell.execute_reply":"2023-10-05T17:36:18.873695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"building_meta_df","metadata":{"execution":{"iopub.status.busy":"2023-10-05T17:36:18.875932Z","iopub.execute_input":"2023-10-05T17:36:18.876216Z","iopub.status.idle":"2023-10-05T17:36:18.908821Z","shell.execute_reply.started":"2023-10-05T17:36:18.876175Z","shell.execute_reply":"2023-10-05T17:36:18.907547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# remove buildings","metadata":{}},{"cell_type":"code","source":"train_df = train_df [ train_df['building_id'] != 1099 ]","metadata":{"execution":{"iopub.status.busy":"2023-10-05T17:36:18.910322Z","iopub.execute_input":"2023-10-05T17:36:18.910572Z","iopub.status.idle":"2023-10-05T17:36:20.071374Z","shell.execute_reply.started":"2023-10-05T17:36:18.910531Z","shell.execute_reply":"2023-10-05T17:36:20.070391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"building_meta_df['floor_area'] = building_meta_df.square_feet / building_meta_df.floor_count","metadata":{"execution":{"iopub.status.busy":"2023-10-05T17:36:20.074685Z","iopub.execute_input":"2023-10-05T17:36:20.074974Z","iopub.status.idle":"2023-10-05T17:36:20.080922Z","shell.execute_reply.started":"2023-10-05T17:36:20.074928Z","shell.execute_reply":"2023-10-05T17:36:20.079871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Leak Data loading and concat","metadata":{}},{"cell_type":"markdown","source":"# Site specific holiday","metadata":{}},{"cell_type":"code","source":"import holidays\n\nen_holidays = holidays.England()\nir_holidays = holidays.Ireland()\nca_holidays = holidays.Canada()\nus_holidays = holidays.UnitedStates()\n\ndef add_holiyday(df_weather):\n    en_idx = df_weather.query('site_id == 1 or site_id == 5').index\n    ir_idx = df_weather.query('site_id == 12').index\n    ca_idx = df_weather.query('site_id == 7 or site_id == 11').index\n    us_idx = df_weather.query('site_id == 0 or site_id == 2 or site_id == 3 or site_id == 4 or site_id == 6 or site_id == 8 or site_id == 9 or site_id == 10 or site_id == 13 or site_id == 14 or site_id == 15').index\n\n    df_weather['IsHoliday'] = 0\n    df_weather.loc[en_idx, 'IsHoliday'] = df_weather.loc[en_idx, 'timestamp'].apply(lambda x: en_holidays.get(x, default=0))\n    df_weather.loc[ir_idx, 'IsHoliday'] = df_weather.loc[ir_idx, 'timestamp'].apply(lambda x: ir_holidays.get(x, default=0))\n    df_weather.loc[ca_idx, 'IsHoliday'] = df_weather.loc[ca_idx, 'timestamp'].apply(lambda x: ca_holidays.get(x, default=0))\n    df_weather.loc[us_idx, 'IsHoliday'] = df_weather.loc[us_idx, 'timestamp'].apply(lambda x: us_holidays.get(x, default=0))\n\n    holiday_idx = df_weather['IsHoliday'] != 0\n    df_weather.loc[holiday_idx, 'IsHoliday'] = 1\n    df_weather['IsHoliday'] = df_weather['IsHoliday'].astype(np.uint8)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T17:36:20.084937Z","iopub.execute_input":"2023-10-05T17:36:20.085318Z","iopub.status.idle":"2023-10-05T17:36:20.104785Z","shell.execute_reply.started":"2023-10-05T17:36:20.085248Z","shell.execute_reply":"2023-10-05T17:36:20.103447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"set_local(weather_train_df)\nadd_holiyday(weather_train_df)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T17:36:20.106173Z","iopub.execute_input":"2023-10-05T17:36:20.106485Z","iopub.status.idle":"2023-10-05T17:36:21.092202Z","shell.execute_reply.started":"2023-10-05T17:36:20.106445Z","shell.execute_reply":"2023-10-05T17:36:21.091217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weather_train_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-05T17:36:21.093959Z","iopub.execute_input":"2023-10-05T17:36:21.094362Z","iopub.status.idle":"2023-10-05T17:36:21.113587Z","shell.execute_reply.started":"2023-10-05T17:36:21.094291Z","shell.execute_reply":"2023-10-05T17:36:21.112773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Threshold By Black day","metadata":{}},{"cell_type":"code","source":"\ntrain_df_black = train_df.copy()\ntrain_df_black = train_df_black.merge(building_meta_df, on=['building_id', 'meter'], how='left')\ntrain_df_black = train_df_black.merge(weather_train_df, on=['site_id', 'timestamp'], how='left')\n\ntrain_df_black['black_count']=0\n\nfor bid in train_df_black.building_id.unique():\n    df = train_df_black[train_df_black.building_id==bid]\n    for meter in df.meter.unique():\n        dfm = df[df.meter == meter]\n        b = (dfm.meter_reading == 0).astype(int)\n        train_df_black.loc[(train_df_black.building_id==bid) & (train_df_black.meter == meter), 'black_count'] = b.groupby((~b.astype(bool)).cumsum()).cumsum()","metadata":{"execution":{"iopub.status.busy":"2023-10-05T17:36:21.115002Z","iopub.execute_input":"2023-10-05T17:36:21.115415Z","iopub.status.idle":"2023-10-05T17:59:43.341527Z","shell.execute_reply.started":"2023-10-05T17:36:21.115350Z","shell.execute_reply":"2023-10-05T17:59:43.340516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df_black[train_df_black.building_id == 954].black_count.plot()","metadata":{"execution":{"iopub.status.busy":"2023-10-05T17:59:43.343262Z","iopub.execute_input":"2023-10-05T17:59:43.343522Z","iopub.status.idle":"2023-10-05T17:59:43.575823Z","shell.execute_reply.started":"2023-10-05T17:59:43.343478Z","shell.execute_reply":"2023-10-05T17:59:43.574811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"execution":{"iopub.status.busy":"2023-10-05T17:59:43.577360Z","iopub.execute_input":"2023-10-05T17:59:43.577625Z","iopub.status.idle":"2023-10-05T17:59:43.583613Z","shell.execute_reply.started":"2023-10-05T17:59:43.577579Z","shell.execute_reply":"2023-10-05T17:59:43.582754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.merge(train_df_black[['timestamp','building_id','meter','black_count']], on=['timestamp','building_id','meter'])\ntrain_df = train_df[train_df['black_count'] < 24*black_day].drop('black_count', axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T17:59:43.585189Z","iopub.execute_input":"2023-10-05T17:59:43.585525Z","iopub.status.idle":"2023-10-05T18:00:01.821874Z","shell.execute_reply.started":"2023-10-05T17:59:43.585467Z","shell.execute_reply":"2023-10-05T18:00:01.820949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:01.823507Z","iopub.execute_input":"2023-10-05T18:00:01.823890Z","iopub.status.idle":"2023-10-05T18:00:01.829461Z","shell.execute_reply.started":"2023-10-05T18:00:01.823815Z","shell.execute_reply":"2023-10-05T18:00:01.828818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_df_black\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:01.830743Z","iopub.execute_input":"2023-10-05T18:00:01.831216Z","iopub.status.idle":"2023-10-05T18:00:02.193408Z","shell.execute_reply.started":"2023-10-05T18:00:01.831147Z","shell.execute_reply":"2023-10-05T18:00:02.192387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Removing weired data on site_id 0\n\n\n","metadata":{}},{"cell_type":"code","source":"","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.query('not (building_id <= 104 & meter == 0 & timestamp <= \"2016-05-20\")')","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:02.195169Z","iopub.execute_input":"2023-10-05T18:00:02.195870Z","iopub.status.idle":"2023-10-05T18:00:03.475577Z","shell.execute_reply.started":"2023-10-05T18:00:02.195795Z","shell.execute_reply":"2023-10-05T18:00:03.474755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[train_df.building_id == 954].meter_reading.plot()","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:03.476830Z","iopub.execute_input":"2023-10-05T18:00:03.477303Z","iopub.status.idle":"2023-10-05T18:00:03.669424Z","shell.execute_reply.started":"2023-10-05T18:00:03.477222Z","shell.execute_reply":"2023-10-05T18:00:03.668677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(train_df[train_df.building_id == 954].meter_reading==0).astype(int).plot()","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:03.670684Z","iopub.execute_input":"2023-10-05T18:00:03.671236Z","iopub.status.idle":"2023-10-05T18:00:03.859179Z","shell.execute_reply.started":"2023-10-05T18:00:03.671191Z","shell.execute_reply":"2023-10-05T18:00:03.858139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[train_df.building_id == 1221].meter_reading.plot()","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:03.860942Z","iopub.execute_input":"2023-10-05T18:00:03.861385Z","iopub.status.idle":"2023-10-05T18:00:04.048012Z","shell.execute_reply.started":"2023-10-05T18:00:03.861310Z","shell.execute_reply":"2023-10-05T18:00:04.046932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.query('not (building_id == 954 & meter_reading == 0)')\ntrain_df = train_df.query('not (building_id == 1221 & meter_reading == 0)')","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:04.049738Z","iopub.execute_input":"2023-10-05T18:00:04.050134Z","iopub.status.idle":"2023-10-05T18:00:06.646862Z","shell.execute_reply.started":"2023-10-05T18:00:04.050046Z","shell.execute_reply":"2023-10-05T18:00:06.645860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[train_df.building_id == 954].meter_reading.plot()","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:06.648904Z","iopub.execute_input":"2023-10-05T18:00:06.649645Z","iopub.status.idle":"2023-10-05T18:00:06.867434Z","shell.execute_reply.started":"2023-10-05T18:00:06.649583Z","shell.execute_reply":"2023-10-05T18:00:06.866602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[train_df.building_id == 1221].meter_reading.plot()","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:06.868686Z","iopub.execute_input":"2023-10-05T18:00:06.868918Z","iopub.status.idle":"2023-10-05T18:00:07.051278Z","shell.execute_reply.started":"2023-10-05T18:00:06.868877Z","shell.execute_reply":"2023-10-05T18:00:07.050233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Delete Outliear","metadata":{}},{"cell_type":"code","source":"funny_bids = [993, 1168,  904,  954,  778, 1021]\n\nprint ('before', len(train_df))\n\nif outlier:\n    #993\n   \n    train_df.loc[(train_df.building_id == 993) & (train_df.meter == 0) & (train_df.meter_reading > 30000), 'meter_reading'] = 31921\n    train_df.loc[(train_df.building_id == 993) & (train_df.meter == 1) & (train_df.meter_reading > 90000), 'meter_reading'] =  96545.5\n\n    #1168\n    train_df = train_df[((train_df.building_id == 1168) & (train_df.meter == 0) & (train_df.meter_reading >10000)) == False]\n\n    #904\n    train_df.loc[(train_df.building_id == 904) & (train_df.meter == 0)& (train_df.meter_reading >10000), 'meter_reading'] = 11306\n\n    #954\n    train_df = train_df[((train_df.building_id == 954) & (train_df.meter_reading >10000))==False]\n\nif rescale:\n    #778 \n    train_df.loc[(train_df.building_id == 778) & (train_df.meter == 1), 'meter_reading'] = train_df.loc[(train_df.building_id == 778) & (train_df.meter == 1), 'meter_reading']/ 100\n    #\n    #1021 \n    train_df.loc[(train_df.building_id == 1021) & (train_df.meter == 3), 'meter_reading'] = train_df.loc[(train_df.building_id == 1021) & (train_df.meter == 3), 'meter_reading']/ 1000\n    \n\ntrain_df = train_df.reset_index()\n\n\nprint ('after', len(train_df))\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:07.053028Z","iopub.execute_input":"2023-10-05T18:00:07.053441Z","iopub.status.idle":"2023-10-05T18:00:07.422919Z","shell.execute_reply.started":"2023-10-05T18:00:07.053343Z","shell.execute_reply":"2023-10-05T18:00:07.422174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Site-0 Correction","metadata":{}},{"cell_type":"code","source":"\nsite_0_bids = building_meta_df[building_meta_df.site_id == 0].building_id.unique()\nprint (len(site_0_bids), len(train_df[train_df.building_id.isin(site_0_bids)].building_id.unique()))\ntrain_df[train_df.building_id.isin(site_0_bids)].head()","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:07.424528Z","iopub.execute_input":"2023-10-05T18:00:07.424822Z","iopub.status.idle":"2023-10-05T18:00:12.109711Z","shell.execute_reply.started":"2023-10-05T18:00:07.424765Z","shell.execute_reply":"2023-10-05T18:00:12.108765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.loc[(train_df.building_id.isin(site_0_bids)) & (train_df.meter==0), 'meter_reading'] = train_df[(train_df.building_id.isin(site_0_bids)) & (train_df.meter==0) ]['meter_reading'] * 0.2931","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:12.111024Z","iopub.execute_input":"2023-10-05T18:00:12.111301Z","iopub.status.idle":"2023-10-05T18:00:17.494575Z","shell.execute_reply.started":"2023-10-05T18:00:12.111258Z","shell.execute_reply":"2023-10-05T18:00:17.493708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[train_df.building_id.isin(site_0_bids)].head()","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:17.495932Z","iopub.execute_input":"2023-10-05T18:00:17.496183Z","iopub.status.idle":"2023-10-05T18:00:19.766542Z","shell.execute_reply.started":"2023-10-05T18:00:17.496147Z","shell.execute_reply":"2023-10-05T18:00:19.765358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data preprocessing\n\nNow, Let's try building GBDT (Gradient Boost Decision Tree) model to predict `meter_reading_log1p`. I will try using LightGBM in this notebook.","metadata":{}},{"cell_type":"code","source":"train_df['date'] = train_df['timestamp'].dt.date\ntrain_df['meter_reading_log1p'] = np.log1p(train_df['meter_reading'])","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:19.767998Z","iopub.execute_input":"2023-10-05T18:00:19.768397Z","iopub.status.idle":"2023-10-05T18:00:27.716549Z","shell.execute_reply.started":"2023-10-05T18:00:19.768325Z","shell.execute_reply":"2023-10-05T18:00:27.715393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Add time feature","metadata":{}},{"cell_type":"markdown","source":"Some features introduced in https://www.kaggle.com/ryches/simple-lgbm-solution by @ryches\n\nFeatures that are likely predictive:\n\n#### Weather\n\n- time of day\n- holiday\n- weekend\n- cloud_coverage + lags\n- dew_temperature + lags\n- precip_depth + lags\n- sea_level_pressure + lags\n- wind_direction + lags\n- wind_speed + lags\n\n#### Train\n\n- max, mean, min, std of the specific building historically\n\n\n\nHowever we should be careful of putting time feature, since we have only 1 year data in training,\nincluding `date` makes overfiting to training data.\n\nHow about `month`? It may be better to check performance by cross validation.\nI go not using this data in this kernel for robust modeling.","metadata":{}},{"cell_type":"code","source":"def preprocess(df):\n    df[\"hour\"] = df[\"timestamp\"].dt.hour\n    df[\"day\"] = df[\"timestamp\"].dt.day\n    df[\"weekend\"] = df[\"timestamp\"].dt.weekday\n    df[\"month\"] = df[\"timestamp\"].dt.month\n    df[\"dayofweek\"] = df[\"timestamp\"].dt.dayofweek\n\n","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:27.718244Z","iopub.execute_input":"2023-10-05T18:00:27.718545Z","iopub.status.idle":"2023-10-05T18:00:27.723666Z","shell.execute_reply.started":"2023-10-05T18:00:27.718488Z","shell.execute_reply":"2023-10-05T18:00:27.722970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocess(train_df)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:27.725038Z","iopub.execute_input":"2023-10-05T18:00:27.725602Z","iopub.status.idle":"2023-10-05T18:00:38.157770Z","shell.execute_reply.started":"2023-10-05T18:00:27.725533Z","shell.execute_reply":"2023-10-05T18:00:38.156629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# sort train. i dont know it is best","metadata":{}},{"cell_type":"code","source":"if use_ucf:\n    train_df = train_df.sort_values('month')\n    train_df = train_df.reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:38.161306Z","iopub.execute_input":"2023-10-05T18:00:38.161713Z","iopub.status.idle":"2023-10-05T18:00:38.167031Z","shell.execute_reply.started":"2023-10-05T18:00:38.161641Z","shell.execute_reply":"2023-10-05T18:00:38.166174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fill Nan value in weather dataframe by interpolation\n\n","metadata":{}},{"cell_type":"code","source":"weather_train_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:38.168901Z","iopub.execute_input":"2023-10-05T18:00:38.169291Z","iopub.status.idle":"2023-10-05T18:00:38.194360Z","shell.execute_reply.started":"2023-10-05T18:00:38.169227Z","shell.execute_reply":"2023-10-05T18:00:38.193419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weather_train_df.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:38.195397Z","iopub.execute_input":"2023-10-05T18:00:38.195745Z","iopub.status.idle":"2023-10-05T18:00:38.214700Z","shell.execute_reply.started":"2023-10-05T18:00:38.195707Z","shell.execute_reply":"2023-10-05T18:00:38.213856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weather_train_df.shape","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:38.215641Z","iopub.execute_input":"2023-10-05T18:00:38.215973Z","iopub.status.idle":"2023-10-05T18:00:38.221240Z","shell.execute_reply.started":"2023-10-05T18:00:38.215937Z","shell.execute_reply":"2023-10-05T18:00:38.220434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weather_train_df.groupby('site_id').apply(lambda group: group.isna().sum())","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:38.222409Z","iopub.execute_input":"2023-10-05T18:00:38.222832Z","iopub.status.idle":"2023-10-05T18:00:38.282124Z","shell.execute_reply.started":"2023-10-05T18:00:38.222764Z","shell.execute_reply":"2023-10-05T18:00:38.281195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weather_train_df = weather_train_df.groupby('site_id').apply(lambda group: group.interpolate(limit_direction='both'))","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:38.283426Z","iopub.execute_input":"2023-10-05T18:00:38.283663Z","iopub.status.idle":"2023-10-05T18:00:48.204527Z","shell.execute_reply.started":"2023-10-05T18:00:38.283621Z","shell.execute_reply":"2023-10-05T18:00:48.203663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weather_train_df.groupby('site_id').apply(lambda group: group.isna().sum())","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:48.205801Z","iopub.execute_input":"2023-10-05T18:00:48.206246Z","iopub.status.idle":"2023-10-05T18:00:48.255981Z","shell.execute_reply.started":"2023-10-05T18:00:48.206110Z","shell.execute_reply":"2023-10-05T18:00:48.255116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Seems number of nan has reduced by `interpolate` but some property has never appear in specific `site_id`, and nan remains for these features.","metadata":{}},{"cell_type":"markdown","source":"## lags\n\nAdding some lag feature","metadata":{}},{"cell_type":"code","source":"def add_lag_feature(weather_df, window=3):\n    group_df = weather_df.groupby('site_id')\n    cols = ['air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure', 'wind_direction', 'wind_speed']\n    rolled = group_df[cols].rolling(window=window, min_periods=0)\n    lag_mean = rolled.mean().reset_index().astype(np.float16)\n    lag_max = rolled.max().reset_index().astype(np.float16)\n    lag_min = rolled.min().reset_index().astype(np.float16)\n    lag_std = rolled.std().reset_index().astype(np.float16)\n    for col in cols:\n        weather_df[f'{col}_mean_lag{window}'] = lag_mean[col]\n        weather_df[f'{col}_max_lag{window}'] = lag_max[col]\n        weather_df[f'{col}_min_lag{window}'] = lag_min[col]\n        weather_df[f'{col}_std_lag{window}'] = lag_std[col]","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:48.257535Z","iopub.execute_input":"2023-10-05T18:00:48.257914Z","iopub.status.idle":"2023-10-05T18:00:48.268262Z","shell.execute_reply.started":"2023-10-05T18:00:48.257848Z","shell.execute_reply":"2023-10-05T18:00:48.267400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_lag_feature(weather_train_df, window=3)\nadd_lag_feature(weather_train_df, window=72)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:48.269636Z","iopub.execute_input":"2023-10-05T18:00:48.269907Z","iopub.status.idle":"2023-10-05T18:00:48.990994Z","shell.execute_reply.started":"2023-10-05T18:00:48.269853Z","shell.execute_reply":"2023-10-05T18:00:48.989929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weather_train_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:48.992441Z","iopub.execute_input":"2023-10-05T18:00:48.992733Z","iopub.status.idle":"2023-10-05T18:00:49.023742Z","shell.execute_reply.started":"2023-10-05T18:00:48.992678Z","shell.execute_reply":"2023-10-05T18:00:49.023022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weather_train_df.columns","metadata":{"_kg_hide-input":false,"_kg_hide-output":false,"execution":{"iopub.status.busy":"2023-10-05T18:00:49.025356Z","iopub.execute_input":"2023-10-05T18:00:49.025968Z","iopub.status.idle":"2023-10-05T18:00:49.039388Z","shell.execute_reply.started":"2023-10-05T18:00:49.025894Z","shell.execute_reply":"2023-10-05T18:00:49.038227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# count encoding","metadata":{}},{"cell_type":"code","source":"year_map = building_meta_df.year_built.value_counts()\nbuilding_meta_df['year_cnt'] = building_meta_df.year_built.map(year_map)\n\nbid_map = train_df.building_id.value_counts()\ntrain_df['bid_cnt'] = train_df.building_id.map(bid_map)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:49.041096Z","iopub.execute_input":"2023-10-05T18:00:49.041452Z","iopub.status.idle":"2023-10-05T18:00:50.198611Z","shell.execute_reply.started":"2023-10-05T18:00:49.041386Z","shell.execute_reply":"2023-10-05T18:00:50.197880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# categorize primary_use column to reduce memory on merge...\n\nprimary_use_list = building_meta_df['primary_use'].unique()\nprimary_use_dict = {key: value for value, key in enumerate(primary_use_list)} \nprint('primary_use_dict: ', primary_use_dict)\nbuilding_meta_df['primary_use'] = building_meta_df['primary_use'].map(primary_use_dict)\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:50.199987Z","iopub.execute_input":"2023-10-05T18:00:50.200271Z","iopub.status.idle":"2023-10-05T18:00:50.318749Z","shell.execute_reply.started":"2023-10-05T18:00:50.200226Z","shell.execute_reply":"2023-10-05T18:00:50.317848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = reduce_mem_usage(train_df, use_float16=True)\nbuilding_meta_df = reduce_mem_usage(building_meta_df, use_float16=True)\nweather_train_df = reduce_mem_usage(weather_train_df, use_float16=True)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-10-05T18:00:50.320215Z","iopub.execute_input":"2023-10-05T18:00:50.320653Z","iopub.status.idle":"2023-10-05T18:00:54.909531Z","shell.execute_reply.started":"2023-10-05T18:00:50.320588Z","shell.execute_reply":"2023-10-05T18:00:54.908643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"building_meta_df.head()","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-10-05T18:00:54.911034Z","iopub.execute_input":"2023-10-05T18:00:54.911409Z","iopub.status.idle":"2023-10-05T18:00:54.929402Z","shell.execute_reply.started":"2023-10-05T18:00:54.911339Z","shell.execute_reply":"2023-10-05T18:00:54.928298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# SG Filter for Weather","metadata":{}},{"cell_type":"code","source":"from scipy.signal import savgol_filter as sg\n\ndef add_sg(df):\n    w = 11\n    p = 2\n    for si in df.site_id.unique():\n        index = df.site_id == si\n        df.loc[index, 'air_smooth'] = sg(df[index].air_temperature, w, p)\n        df.loc[index, 'dew_smooth'] = sg(df[index].dew_temperature, w, p)\n        \n        df.loc[index, 'air_diff'] = sg(df[index].air_temperature, w, p, 1)\n        df.loc[index, 'dew_diff'] = sg(df[index].dew_temperature, w, p, 1)\n        \n        df.loc[index, 'air_diff2'] = sg(df[index].air_temperature, w, p, 2)\n        df.loc[index, 'dew_diff2'] = sg(df[index].dew_temperature, w, p, 2)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:54.938941Z","iopub.execute_input":"2023-10-05T18:00:54.939364Z","iopub.status.idle":"2023-10-05T18:00:55.014130Z","shell.execute_reply.started":"2023-10-05T18:00:54.939293Z","shell.execute_reply":"2023-10-05T18:00:55.013258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_sg(weather_train_df)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:55.015545Z","iopub.execute_input":"2023-10-05T18:00:55.015830Z","iopub.status.idle":"2023-10-05T18:00:55.585151Z","shell.execute_reply.started":"2023-10-05T18:00:55.015792Z","shell.execute_reply":"2023-10-05T18:00:55.584029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weather_train_df[weather_train_df.site_id==1].air_temperature[:100].plot()\nweather_train_df[weather_train_df.site_id==1].air_smooth[:100].plot()","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:55.586582Z","iopub.execute_input":"2023-10-05T18:00:55.586878Z","iopub.status.idle":"2023-10-05T18:00:55.749923Z","shell.execute_reply.started":"2023-10-05T18:00:55.586823Z","shell.execute_reply":"2023-10-05T18:00:55.748965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weather_train_df[weather_train_df.site_id==2].dew_temperature[:100].plot()\nweather_train_df[weather_train_df.site_id==2].dew_smooth[:100].plot()","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:55.751871Z","iopub.execute_input":"2023-10-05T18:00:55.752267Z","iopub.status.idle":"2023-10-05T18:00:55.932394Z","shell.execute_reply.started":"2023-10-05T18:00:55.752190Z","shell.execute_reply":"2023-10-05T18:00:55.931337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weather_train_df[weather_train_df.site_id==0].dew_diff[:100].plot()\nweather_train_df[weather_train_df.site_id==0].air_diff[:100].plot()","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:55.934357Z","iopub.execute_input":"2023-10-05T18:00:55.934846Z","iopub.status.idle":"2023-10-05T18:00:56.102791Z","shell.execute_reply.started":"2023-10-05T18:00:55.934767Z","shell.execute_reply":"2023-10-05T18:00:56.101775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train model\n\nTo win in kaggle competition, how to evaluate your model is important.\nWhat kind of cross validation strategy is suitable for this competition? This is time series data, so it is better to consider time-splitting.\n\nHowever this notebook is for simple tutorial, so I will proceed with KFold splitting without shuffling, so that at least near-term data is not included in validation.","metadata":{}},{"cell_type":"code","source":"category_cols = ['building_id', 'site_id', 'primary_use', 'IsHoliday', 'groupNum_train']  # , 'meter'\nfeature_cols = ['square_feet', 'year_built'] + [\n    'hour', 'weekend',\n#    'day', # 'month' ,\n#    'dayofweek',\n#    'building_median'\n    ] + [\n    'air_temperature', 'cloud_coverage',\n    'dew_temperature', 'precip_depth_1_hr',\n    'sea_level_pressure',\n#'wind_direction', 'wind_speed',\n    'air_temperature_mean_lag72',\n    'air_temperature_max_lag72', 'air_temperature_min_lag72',\n    'air_temperature_std_lag72', 'cloud_coverage_mean_lag72',\n    'dew_temperature_mean_lag72', 'precip_depth_1_hr_mean_lag72',\n    'sea_level_pressure_mean_lag72',\n#'wind_direction_mean_lag72',\n    'wind_speed_mean_lag72', \n    'air_temperature_mean_lag3',\n    'air_temperature_max_lag3',\n    'air_temperature_min_lag3', 'cloud_coverage_mean_lag3',\n    'dew_temperature_mean_lag3',\n    'precip_depth_1_hr_mean_lag3',\n    'sea_level_pressure_mean_lag3',\n#    'wind_direction_mean_lag3', 'wind_speed_mean_lag3',\n#    'floor_area',\n    'year_cnt', 'bid_cnt',\n    'dew_smooth', 'air_smooth',\n    'dew_diff', 'air_diff',\n    'dew_diff2', 'air_diff2'\n] + list(df_groupNum_median.drop('timestamp',axis=1).columns)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:56.104531Z","iopub.execute_input":"2023-10-05T18:00:56.104909Z","iopub.status.idle":"2023-10-05T18:00:56.117115Z","shell.execute_reply.started":"2023-10-05T18:00:56.104844Z","shell.execute_reply":"2023-10-05T18:00:56.115913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.merge(building_meta_df, on=['building_id','meter'], how='left')\ntrain_df = train_df.merge(weather_train_df, on=['site_id', 'timestamp'], how='left')\n\ntrain_df = reduce_mem_usage(train_df, use_float16=True)\n\ndel weather_train_df\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:00:56.118768Z","iopub.execute_input":"2023-10-05T18:00:56.119154Z","iopub.status.idle":"2023-10-05T18:01:48.972086Z","shell.execute_reply.started":"2023-10-05T18:00:56.119083Z","shell.execute_reply":"2023-10-05T18:01:48.971141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_X_y(train_df, groupNum_train):\n    \n    target_train_df = train_df[train_df['groupNum_train'] == groupNum_train].copy()        \n    target_train_df = target_train_df.merge(df_groupNum_median, on=['timestamp'], how='left')\n    target_train_df['group_median_'+str(groupNum_train)] = np.nan\n    \n    X_train = target_train_df[feature_cols + category_cols]\n    y_train = target_train_df['meter_reading_log1p'].values\n    \n    del target_train_df\n    return X_train, y_train","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-10-05T18:01:48.973647Z","iopub.execute_input":"2023-10-05T18:01:48.974232Z","iopub.status.idle":"2023-10-05T18:01:48.982208Z","shell.execute_reply.started":"2023-10-05T18:01:48.974160Z","shell.execute_reply":"2023-10-05T18:01:48.981301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fit_lgbm(train, val, devices=(-1,), seed=None, cat_features=None, num_rounds=1500, lr=0.1, bf=0.1):\n    \"\"\"Train Light GBM model\"\"\"\n    X_train, y_train = train\n    X_valid, y_valid = val\n    metric = 'l2'\n    params = {'num_leaves': 31,\n              'objective': 'regression',\n#               'max_depth': -1,\n              'learning_rate': lr,\n              \"boosting\": \"gbdt\",\n              \"bagging_freq\": 5,\n              \"bagging_fraction\": bf,\n              \"feature_fraction\": 0.9,\n              \"metric\": metric,\n#               \"verbosity\": -1,\n#               'reg_alpha': 0.1,\n#               'reg_lambda': 0.3\n              }\n    device = devices[0]\n    if device == -1:\n        # use cpu\n        pass\n    else:\n        # use gpu\n        print(f'using gpu device_id {device}...')\n        params.update({'device': 'gpu', 'gpu_device_id': device})\n\n    params['seed'] = seed\n\n    early_stop = 20\n    verbose_eval = 50\n\n    d_train = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_features)\n    d_valid = lgb.Dataset(X_valid, label=y_valid, categorical_feature=cat_features)\n    watchlist = [d_train, d_valid]\n\n    print('training LGB:')\n    model = lgb.train(params,\n                      train_set=d_train,\n                      num_boost_round=num_rounds,\n                      valid_sets=watchlist,\n                      verbose_eval=verbose_eval,\n                      early_stopping_rounds=early_stop)\n\n    # predictions\n    y_pred_valid = model.predict(X_valid, num_iteration=model.best_iteration)\n    \n    print('best_score', model.best_score)\n    log = {'train/mae': model.best_score['training']['l2'],\n           'valid/mae': model.best_score['valid_1']['l2']}\n    return model, y_pred_valid, log","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-10-05T18:01:48.983725Z","iopub.execute_input":"2023-10-05T18:01:48.984095Z","iopub.status.idle":"2023-10-05T18:01:49.001089Z","shell.execute_reply.started":"2023-10-05T18:01:48.984014Z","shell.execute_reply":"2023-10-05T18:01:49.000016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import GroupKFold, StratifiedKFold\n\nseed = 666\nshuffle = False\n#kf = KFold(n_splits=folds, shuffle=shuffle, random_state=seed)\n#kf = GroupKFold(n_splits=folds)\nkf = StratifiedKFold(n_splits=folds)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:01:49.002635Z","iopub.execute_input":"2023-10-05T18:01:49.003021Z","iopub.status.idle":"2023-10-05T18:01:49.016084Z","shell.execute_reply.started":"2023-10-05T18:01:49.002951Z","shell.execute_reply":"2023-10-05T18:01:49.015210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train model by each group # (site-meter)","metadata":{}},{"cell_type":"code","source":"def plot_feature_importance(model):\n    importance_df = pd.DataFrame(model[1].feature_importance(),\n                                 index=feature_cols + category_cols,\n                                 columns=['importance']).sort_values('importance')\n    fig, ax = plt.subplots(figsize=(8, 8))\n    importance_df.plot.barh(ax=ax)\n    fig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-10-05T18:01:49.017494Z","iopub.execute_input":"2023-10-05T18:01:49.017746Z","iopub.status.idle":"2023-10-05T18:01:49.027373Z","shell.execute_reply.started":"2023-10-05T18:01:49.017704Z","shell.execute_reply":"2023-10-05T18:01:49.026378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for groupNum_train in building_meta_df['groupNum_train'].unique():\n    X_train, y_train = create_X_y(train_df, groupNum_train=groupNum_train)\n    y_valid_pred_total = np.zeros(X_train.shape[0])\n    gc.collect()\n    print('groupNum_train', groupNum_train, X_train.shape)\n\n    cat_features = [X_train.columns.get_loc(cat_col) for cat_col in category_cols]\n    print('cat_features', cat_features)\n\n    exec('models' +str(groupNum_train)+ '=[]')\n\n    train_df_site = train_df[train_df['groupNum_train']==groupNum_train].copy()\n    \n    #for train_idx, valid_idx in kf.split(X_train, y_train):\n    #for train_idx, valid_idx in kf.split(X_train, y_train, groups=get_groups(train_df, groupNum_train)):    \n    for train_idx, valid_idx in kf.split(train_df_site, train_df_site['building_id']):\n        train_data = X_train.iloc[train_idx,:], y_train[train_idx]\n        valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]\n\n        mindex = train_df_site.iloc[valid_idx,:].month.unique()\n        print (mindex)\n\n        print('train', len(train_idx), 'valid', len(valid_idx))\n    #     model, y_pred_valid, log = fit_cb(train_data, valid_data, cat_features=cat_features, devices=[0,])\n        model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,\n                                            num_rounds=num_rounds, lr=0.05, bf=0.7)\n        y_valid_pred_total[valid_idx] = y_pred_valid\n        exec('models' +str(groupNum_train)+ '.append([mindex, model])')        \n        gc.collect()\n        if debug:\n            break\n\n    try:\n        sns.distplot(y_train)\n        sns.distplot(y_valid_pred_total)\n        plt.show()\n    except:\n        pass\n\n    del X_train, y_train\n    gc.collect()\n    \n    print('-------------------------------------------------------------')","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:01:49.028838Z","iopub.execute_input":"2023-10-05T18:01:49.029271Z","iopub.status.idle":"2023-10-05T18:30:19.035044Z","shell.execute_reply.started":"2023-10-05T18:01:49.029030Z","shell.execute_reply":"2023-10-05T18:30:19.034265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_df\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:30:19.036797Z","iopub.execute_input":"2023-10-05T18:30:19.037132Z","iopub.status.idle":"2023-10-05T18:30:19.666510Z","shell.execute_reply.started":"2023-10-05T18:30:19.037045Z","shell.execute_reply":"2023-10-05T18:30:19.665371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction on test data","metadata":{}},{"cell_type":"code","source":"print('loading...')\ntest_df = pd.read_feather(root/'test.feather')\nweather_test_df = pd.read_feather(root/'weather_test.feather')\n\nweather_test_df = weather_test_df.drop_duplicates(['timestamp', 'site_id'])\nset_local(weather_test_df)\nadd_holiyday(weather_test_df)\n\nprint('preprocessing building...')\ntest_df['date'] = test_df['timestamp'].dt.date\npreprocess(test_df)\n#test_df['building_median'] = test_df['building_id'].map(building_median)\n\nprint('preprocessing weather...')\nweather_test_df = weather_test_df.groupby('site_id').apply(lambda group: group.interpolate(limit_direction='both'))\nweather_test_df.groupby('site_id').apply(lambda group: group.isna().sum())\n\nadd_sg(weather_test_df)\n\nadd_lag_feature(weather_test_df, window=3)\nadd_lag_feature(weather_test_df, window=72)\n\n#bid_map = train_df.building_id.value_counts()\ntest_df['bid_cnt'] = test_df.building_id.map(bid_map)\n\ntest_df = test_df.merge(building_meta_df[['building_id','meter','groupNum_train']], on=['building_id','meter'], how='left')\n              \nprint('reduce mem usage...')\ntest_df = reduce_mem_usage(test_df, use_float16=True)\nweather_test_df = reduce_mem_usage(weather_test_df, use_float16=True)\n\ngc.collect()\nprint (test_df.shape)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:30:19.668707Z","iopub.execute_input":"2023-10-05T18:30:19.669331Z","iopub.status.idle":"2023-10-05T18:32:12.454650Z","shell.execute_reply.started":"2023-10-05T18:30:19.669028Z","shell.execute_reply":"2023-10-05T18:32:12.453744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = pd.read_feather(os.path.join(root, 'sample_submission.feather'))\nreduce_mem_usage(sample_submission)\n\nprint(sample_submission.shape)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:32:12.456162Z","iopub.execute_input":"2023-10-05T18:32:12.456477Z","iopub.status.idle":"2023-10-05T18:32:19.093443Z","shell.execute_reply.started":"2023-10-05T18:32:12.456418Z","shell.execute_reply":"2023-10-05T18:32:19.092331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_X(test_df, groupNum_train):\n    \n    target_test_df = test_df[test_df['groupNum_train'] == groupNum_train].copy()        \n    target_test_df = target_test_df.merge(df_groupNum_median, on=['timestamp'], how='left')\n    target_test_df = target_test_df.merge(building_meta_df, on=['building_id','meter','groupNum_train'], how='left')\n    target_test_df = target_test_df.merge(weather_test_df, on=['site_id', 'timestamp'], how='left')\n    target_test_df['group_median_'+str(groupNum_train)] = np.nan\n\n    X_test = target_test_df[feature_cols + category_cols]\n    \n    return X_test","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-10-05T18:32:19.094732Z","iopub.execute_input":"2023-10-05T18:32:19.095014Z","iopub.status.idle":"2023-10-05T18:32:19.101514Z","shell.execute_reply.started":"2023-10-05T18:32:19.094962Z","shell.execute_reply":"2023-10-05T18:32:19.100580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pred_all(X_test, models, batch_size=1000000):\n    iterations = (X_test.shape[0] + batch_size -1) // batch_size\n    print('iterations', iterations)\n\n    y_test_pred_total = np.zeros(X_test.shape[0])\n    for i, (mindex, model) in enumerate(models):\n        print(f'predicting {i}-th model')\n        for k in tqdm(range(iterations)):\n            y_pred_test = model.predict(X_test[k*batch_size:(k+1)*batch_size], num_iteration=model.best_iteration)\n            y_test_pred_total[k*batch_size:(k+1)*batch_size] += y_pred_test\n\n    y_test_pred_total /= len(models)\n    return y_test_pred_total\n\n\ndef pred(X_test, models, batch_size=1000000):\n    if predmode == 'valid':\n        print ('valid pred')\n        return pred_valid(X_test, models, batch_size=1000000)\n    elif predmode == 'train':\n        print ('train pred')\n        return pred_train(X_test, models, batch_size=1000000)\n    else:\n        print ('all pred')\n        return pred_all(X_test, models, batch_size=1000000)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-10-05T18:32:19.102969Z","iopub.execute_input":"2023-10-05T18:32:19.103510Z","iopub.status.idle":"2023-10-05T18:32:19.117321Z","shell.execute_reply.started":"2023-10-05T18:32:19.103452Z","shell.execute_reply":"2023-10-05T18:32:19.116270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for groupNum_train in building_meta_df['groupNum_train'].unique():\n    print('groupNum_train: ', groupNum_train)\n    X_test = create_X(test_df, groupNum_train=groupNum_train)\n    gc.collect()\n\n    exec('y_test= pred(X_test, models' +str(groupNum_train)+ ')')\n\n    sns.distplot(y_test)\n    plt.show()\n\n    print(X_test.shape, y_test.shape)\n    sample_submission.loc[test_df[\"groupNum_train\"] == groupNum_train,\"meter_reading\"] = np.expm1(y_test)\n    \n    del X_test, y_test\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:32:19.118859Z","iopub.execute_input":"2023-10-05T18:32:19.119423Z","iopub.status.idle":"2023-10-05T18:49:19.626817Z","shell.execute_reply.started":"2023-10-05T18:32:19.119360Z","shell.execute_reply":"2023-10-05T18:49:19.625907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# site-0 correction ","metadata":{}},{"cell_type":"code","source":"# https://www.kaggle.com/c/ashrae-energy-prediction/discussion/119261#latest-684102\nsample_submission.loc[(test_df.building_id.isin(site_0_bids)) & (test_df.meter==0), 'meter_reading'] = sample_submission[(test_df.building_id.isin(site_0_bids)) & (test_df.meter==0)]['meter_reading'] * 3.4118","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:49:19.628451Z","iopub.execute_input":"2023-10-05T18:49:19.628760Z","iopub.status.idle":"2023-10-05T18:49:31.391821Z","shell.execute_reply.started":"2023-10-05T18:49:19.628700Z","shell.execute_reply":"2023-10-05T18:49:31.390669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if rescale:   \n    sample_submission.loc[(test_df.building_id == 778) & (test_df.meter == 1), 'meter_reading'] = 100 * sample_submission.loc[(test_df.building_id == 778) & (test_df.meter == 1), 'meter_reading']\n    sample_submission.loc[(test_df.building_id == 1021) & (test_df.meter == 3), 'meter_reading'] = 1000 * sample_submission.loc[(test_df.building_id == 1021) & (test_df.meter == 3), 'meter_reading']\n    \n    plt.figure()\n    plt.subplot(211)\n    sample_submission.loc[(test_df.building_id == 778) & (test_df.meter == 1), 'meter_reading'].plot()\n    plt.subplot(212)    \n    sample_submission.loc[(test_df.building_id == 1021) & (test_df.meter == 3), 'meter_reading'].plot()","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:49:31.393268Z","iopub.execute_input":"2023-10-05T18:49:31.393605Z","iopub.status.idle":"2023-10-05T18:49:31.401995Z","shell.execute_reply.started":"2023-10-05T18:49:31.393548Z","shell.execute_reply":"2023-10-05T18:49:31.400666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if clip0:\n    sample_submission.loc[sample_submission.meter_reading < 0, 'meter_reading'] = 0","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:49:31.403309Z","iopub.execute_input":"2023-10-05T18:49:31.403554Z","iopub.status.idle":"2023-10-05T18:49:31.428608Z","shell.execute_reply.started":"2023-10-05T18:49:31.403512Z","shell.execute_reply":"2023-10-05T18:49:31.427578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not debug:\n    sample_submission.to_csv('submission_no_leak.csv', index=False, float_format='%.4f')","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:49:31.430122Z","iopub.execute_input":"2023-10-05T18:49:31.430390Z","iopub.status.idle":"2023-10-05T18:52:09.150872Z","shell.execute_reply.started":"2023-10-05T18:49:31.430347Z","shell.execute_reply":"2023-10-05T18:52:09.149986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.log1p(sample_submission['meter_reading']).hist(bins=100)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:52:09.152264Z","iopub.execute_input":"2023-10-05T18:52:09.152837Z","iopub.status.idle":"2023-10-05T18:52:11.122489Z","shell.execute_reply.started":"2023-10-05T18:52:09.152773Z","shell.execute_reply":"2023-10-05T18:52:11.121494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# replace LEAK data","metadata":{}},{"cell_type":"code","source":"# i'm now using my leak data station kernel to shortcut.\nleak_df = pd.read_feather('../input/ashrae-leak-data-station/leak.feather')\n\nprint(leak_df.duplicated().sum())\nprint(leak_df.meter.value_counts())\n\nleak_df.fillna(0, inplace=True)\nleak_df = leak_df[(leak_df.timestamp.dt.year > 2016) & (leak_df.timestamp.dt.year < 2019)]\nleak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0 # remove large negative values\nleak_df = leak_df[leak_df.building_id!=245]\n\nsample_submission.loc[sample_submission.meter_reading < 0, 'meter_reading'] = 0\n\ntest_df['pred'] = sample_submission.meter_reading\n\nleak_df = leak_df.merge(test_df[['building_id', 'meter', 'timestamp', 'pred', 'row_id']], left_on = ['building_id', 'meter', 'timestamp'], right_on = ['building_id', 'meter', 'timestamp'], how = \"left\")\nleak_df = leak_df.merge(building_meta_df[['building_id', 'site_id']].drop_duplicates(), on='building_id', how='left')","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:52:11.123829Z","iopub.execute_input":"2023-10-05T18:52:11.124093Z","iopub.status.idle":"2023-10-05T18:53:08.566101Z","shell.execute_reply.started":"2023-10-05T18:52:11.124034Z","shell.execute_reply":"2023-10-05T18:53:08.565025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"leak_df.site_id.unique()","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:53:08.567514Z","iopub.execute_input":"2023-10-05T18:53:08.567892Z","iopub.status.idle":"2023-10-05T18:53:08.738956Z","shell.execute_reply.started":"2023-10-05T18:53:08.567824Z","shell.execute_reply":"2023-10-05T18:53:08.737939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"leak_df['pred_l1p'] = np.log1p(leak_df.pred)\nleak_df['meter_reading_l1p'] = np.log1p(leak_df.meter_reading)\n\nsns.distplot(leak_df.pred_l1p)\nsns.distplot(leak_df.meter_reading_l1p)\n\nleak_score = np.sqrt(mean_squared_error(leak_df.pred_l1p, leak_df.meter_reading_l1p))","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:53:08.740423Z","iopub.execute_input":"2023-10-05T18:53:08.740798Z","iopub.status.idle":"2023-10-05T18:53:21.776185Z","shell.execute_reply.started":"2023-10-05T18:53:08.740731Z","shell.execute_reply":"2023-10-05T18:53:21.775290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"leak_df = leak_df[['meter_reading', 'row_id']].set_index('row_id').dropna()\nsample_submission.loc[leak_df.index, 'meter_reading'] = leak_df['meter_reading']","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:53:21.777425Z","iopub.execute_input":"2023-10-05T18:53:21.777679Z","iopub.status.idle":"2023-10-05T18:53:25.320489Z","shell.execute_reply.started":"2023-10-05T18:53:21.777637Z","shell.execute_reply":"2023-10-05T18:53:25.319489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:53:25.321785Z","iopub.execute_input":"2023-10-05T18:53:25.322204Z","iopub.status.idle":"2023-10-05T18:53:25.332930Z","shell.execute_reply.started":"2023-10-05T18:53:25.321996Z","shell.execute_reply":"2023-10-05T18:53:25.331875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not debug:\n    sample_submission.to_csv('submission.csv', index=False, float_format='%.4f')","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:53:25.334518Z","iopub.execute_input":"2023-10-05T18:53:25.334766Z","iopub.status.idle":"2023-10-05T18:56:03.664581Z","shell.execute_reply.started":"2023-10-05T18:53:25.334725Z","shell.execute_reply":"2023-10-05T18:56:03.663451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.tail()","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:56:03.666200Z","iopub.execute_input":"2023-10-05T18:56:03.666534Z","iopub.status.idle":"2023-10-05T18:56:03.678145Z","shell.execute_reply.started":"2023-10-05T18:56:03.666478Z","shell.execute_reply":"2023-10-05T18:56:03.677117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.log1p(sample_submission['meter_reading']).hist(bins=100)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:56:03.679634Z","iopub.execute_input":"2023-10-05T18:56:03.679887Z","iopub.status.idle":"2023-10-05T18:56:05.648521Z","shell.execute_reply.started":"2023-10-05T18:56:03.679831Z","shell.execute_reply":"2023-10-05T18:56:05.647377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Scores","metadata":{}},{"cell_type":"code","source":"print('LV score=', leak_score)","metadata":{"execution":{"iopub.status.busy":"2023-10-05T18:56:05.649785Z","iopub.execute_input":"2023-10-05T18:56:05.650033Z","iopub.status.idle":"2023-10-05T18:56:05.655253Z","shell.execute_reply.started":"2023-10-05T18:56:05.649992Z","shell.execute_reply":"2023-10-05T18:56:05.654131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}