{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport matplotlib.pyplot as plt\nfrom matplotlib import dates as md\nimport seaborn as sns\nimport plotly.graph_objs as go\nimport plotly\nfrom plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\nimport cufflinks as cf\ncf.set_config_file(offline=True)\n\nfrom sklearn.metrics import r2_score\n\nimport lightgbm as lgb\n\nfrom tqdm import tqdm\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-19T02:17:55.243026Z","iopub.execute_input":"2022-01-19T02:17:55.243803Z","iopub.status.idle":"2022-01-19T02:17:59.087892Z","shell.execute_reply.started":"2022-01-19T02:17:55.243685Z","shell.execute_reply":"2022-01-19T02:17:59.086958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load dataset","metadata":{}},{"cell_type":"code","source":"path_GEPIII = '/kaggle/input/ashrae-energy-prediction'","metadata":{"execution":{"iopub.status.busy":"2022-01-19T02:18:19.358787Z","iopub.execute_input":"2022-01-19T02:18:19.359073Z","iopub.status.idle":"2022-01-19T02:18:19.362941Z","shell.execute_reply.started":"2022-01-19T02:18:19.359041Z","shell.execute_reply":"2022-01-19T02:18:19.362099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_power_meter = pd.read_csv(os.path.join(path_GEPIII,'train.csv'))\ndf_meta = pd.read_csv(os.path.join(path_GEPIII,'building_metadata.csv'))","metadata":{"execution":{"iopub.status.busy":"2022-01-19T02:18:43.466666Z","iopub.execute_input":"2022-01-19T02:18:43.466966Z","iopub.status.idle":"2022-01-19T02:18:59.583145Z","shell.execute_reply.started":"2022-01-19T02:18:43.466920Z","shell.execute_reply":"2022-01-19T02:18:59.582346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_meta = df_meta.merge(df_power_meter[['building_id','meter']].drop_duplicates(), on='building_id')\ndf_meta['merged_id'] = df_meta['building_id'].astype('str') + '_' + df_meta['meter'].astype('str')\ndf_meta","metadata":{"execution":{"iopub.status.busy":"2022-01-19T02:18:59.584886Z","iopub.execute_input":"2022-01-19T02:18:59.585294Z","iopub.status.idle":"2022-01-19T02:19:00.689584Z","shell.execute_reply.started":"2022-01-19T02:18:59.585252Z","shell.execute_reply":"2022-01-19T02:19:00.688646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_power_meter = df_power_meter.pivot_table(index='timestamp', columns=['building_id','meter'], values='meter_reading')\ndf_power_meter.index = pd.to_datetime(df_power_meter.index)\ndf_power_meter.columns = df_power_meter.columns.get_level_values(0).astype('str')+'_'+df_power_meter.columns.get_level_values(1).astype('str')\ndf_power_meter","metadata":{"execution":{"iopub.status.busy":"2022-01-19T02:19:00.691110Z","iopub.execute_input":"2022-01-19T02:19:00.691493Z","iopub.status.idle":"2022-01-19T02:19:22.788974Z","shell.execute_reply.started":"2022-01-19T02:19:00.691452Z","shell.execute_reply":"2022-01-19T02:19:22.788193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_count = pd.DataFrame(df_power_meter.count()).reset_index().rename(columns={'index':'merged_id',0:'count'})\ndf_meta = df_meta.merge(df_count, on='merged_id')\ndf_meta","metadata":{"execution":{"iopub.status.busy":"2022-01-19T02:20:00.268850Z","iopub.execute_input":"2022-01-19T02:20:00.269259Z","iopub.status.idle":"2022-01-19T02:20:00.353145Z","shell.execute_reply.started":"2022-01-19T02:20:00.269220Z","shell.execute_reply":"2022-01-19T02:20:00.352348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train data: 64% of power meters (site0~9); validation data (site10~12); test data: 36% of power meters ((site13~15))\ndf_power_meter = df_power_meter.loc['2016']\ntrain_data = df_power_meter.loc[:, df_meta.loc[(df_meta['site_id']<10)&(df_meta['count']>8784*0.9), 'merged_id']].copy()\nvalid_data = df_power_meter.loc[:, df_meta.loc[(df_meta['site_id']<13)&(df_meta['site_id']>=10)&(df_meta['count']>8784*0.9), 'merged_id']].copy()\ntest_data = df_power_meter.loc[:, df_meta.loc[df_meta['count']>8784*0.9, 'merged_id']].drop(train_data.columns, axis=1).copy()","metadata":{"execution":{"iopub.status.busy":"2022-01-19T02:22:48.579071Z","iopub.execute_input":"2022-01-19T02:22:48.579344Z","iopub.status.idle":"2022-01-19T02:22:48.758209Z","shell.execute_reply.started":"2022-01-19T02:22:48.579316Z","shell.execute_reply":"2022-01-19T02:22:48.757315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Normalize meter readings for each meter\ndef normalize(df):\n    mean = df.mean()\n    df -= mean\n    std = df.std()\n    df /= std\n    return df, mean, std\n\ntrain_value, train_mean, train_std = normalize(train_data)\nvalid_value, valid_mean, valid_std = normalize(valid_data)\ntest_value, test_mean, test_std = normalize(test_data)","metadata":{"execution":{"iopub.status.busy":"2022-01-19T02:22:50.717810Z","iopub.execute_input":"2022-01-19T02:22:50.718345Z","iopub.status.idle":"2022-01-19T02:22:50.927672Z","shell.execute_reply.started":"2022-01-19T02:22:50.718304Z","shell.execute_reply":"2022-01-19T02:22:50.926913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Add noises\n+- 1,2 and 3 std, 0.1% respectively","metadata":{}},{"cell_type":"code","source":"# Add noises to train data\ntrain_value_noisy = train_value.copy()\n\nfor meter_name in tqdm(train_value.columns):\n    df_noisy_data = train_value_noisy[[meter_name]].copy()\n    df_noisy_data['noise'] = 0\n    \n    #Add noises (+-1,2,3 std)\n    std = df_noisy_data[meter_name].std()\n    for multiplier_std in [-3,-2,-1,1,2,3]:\n        random_hours = df_noisy_data[df_noisy_data['noise']==0].sample(frac=0.001).index\n        df_noisy_data.loc[random_hours, meter_name] = df_noisy_data.loc[random_hours, meter_name] + multiplier_std*std\n        df_noisy_data.loc[random_hours, 'noise'] = 1\n    \n    df_noisy_data[meter_name] = df_noisy_data[meter_name].fillna(method='ffill').fillna(method='bfill') \n    \n    train_value_noisy[meter_name] = df_noisy_data[meter_name].copy()    ","metadata":{"execution":{"iopub.status.busy":"2022-01-19T02:23:50.259389Z","iopub.execute_input":"2022-01-19T02:23:50.259743Z","iopub.status.idle":"2022-01-19T02:24:22.637843Z","shell.execute_reply.started":"2022-01-19T02:23:50.259714Z","shell.execute_reply":"2022-01-19T02:24:22.636938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Add noises to valid data\nvalid_value_noisy = valid_value.copy()\n\nfor meter_name in tqdm(valid_value.columns):\n    df_noisy_data = valid_value_noisy[[meter_name]].copy()\n    df_noisy_data['noise'] = 0\n    \n    #Add noises (+-1,2,3 std)\n    std = df_noisy_data[meter_name].std()\n    for multiplier_std in [-3,-2,-1,1,2,3]:\n        random_hours = df_noisy_data[df_noisy_data['noise']==0].sample(frac=0.001).index\n        df_noisy_data.loc[random_hours, meter_name] = df_noisy_data.loc[random_hours, meter_name] + multiplier_std*std\n        df_noisy_data.loc[random_hours, 'noise'] = 1\n    \n    df_noisy_data[meter_name] = df_noisy_data[meter_name].fillna(method='ffill').fillna(method='bfill') \n    \n    valid_value_noisy[meter_name] = df_noisy_data[meter_name].copy()     ","metadata":{"execution":{"iopub.status.busy":"2022-01-19T02:24:22.639803Z","iopub.execute_input":"2022-01-19T02:24:22.640089Z","iopub.status.idle":"2022-01-19T02:24:24.867855Z","shell.execute_reply.started":"2022-01-19T02:24:22.640048Z","shell.execute_reply":"2022-01-19T02:24:24.867292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Add noises to test data\ntest_value_noisy = test_value.copy()\n\nfor meter_name in tqdm(test_value.columns):\n    df_noisy_data = test_value_noisy[[meter_name]].copy()\n    df_noisy_data['noise'] = 0\n    \n    #Add noises (+-1,2,3 std)\n    std = df_noisy_data[meter_name].std()\n    for multiplier_std in [-3,-2,-1,1,2,3]:\n        random_hours = df_noisy_data[df_noisy_data['noise']==0].sample(frac=0.001).index\n        df_noisy_data.loc[random_hours, meter_name] = df_noisy_data.loc[random_hours, meter_name] + multiplier_std*std\n        df_noisy_data.loc[random_hours, 'noise'] = 1\n    \n    df_noisy_data[meter_name] = df_noisy_data[meter_name].fillna(method='ffill').fillna(method='bfill') \n    \n    test_value_noisy[meter_name] = df_noisy_data[meter_name].copy()      ","metadata":{"execution":{"iopub.status.busy":"2022-01-19T02:24:24.869270Z","iopub.execute_input":"2022-01-19T02:24:24.869581Z","iopub.status.idle":"2022-01-19T02:24:41.243105Z","shell.execute_reply.started":"2022-01-19T02:24:24.869540Z","shell.execute_reply":"2022-01-19T02:24:41.242393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot of before and after adding noises\nfor meter_name in train_value.sample(n=10, axis=1, random_state=42).columns:\n    fig, axes = plt.subplots(1,2,figsize=(15,3))\n    \n    ymin = train_value_noisy[meter_name].min()*1.05\n    ymax = train_value_noisy[meter_name].max()*1.05\n    \n    train_value[meter_name].fillna(method='ffill').fillna(method='bfill').plot(title=meter_name+' (raw data)', ylim=(ymin, ymax),\n                                                                               ax=axes[0],color='blue')\n    train_value_noisy[meter_name].fillna(method='ffill').fillna(method='bfill').plot(title=meter_name+' (add noise)', ylim=(ymin, ymax),\n                                                                                     ax=axes[1],color='orange')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-19T02:25:22.625960Z","iopub.execute_input":"2022-01-19T02:25:22.626767Z","iopub.status.idle":"2022-01-19T02:25:30.649300Z","shell.execute_reply.started":"2022-01-19T02:25:22.626730Z","shell.execute_reply":"2022-01-19T02:25:30.648470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}