{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport time\nimport tqdm as tqdm\n\nimport datetime as datetime\n\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.cluster import KMeans\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.metrics import r2_score\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport plotly.graph_objs as go\nimport plotly\nfrom plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\nimport cufflinks as cf\n\ncf.set_config_file(offline=True)\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-31T09:01:40.621711Z","iopub.execute_input":"2022-08-31T09:01:40.622104Z","iopub.status.idle":"2022-08-31T09:01:44.742700Z","shell.execute_reply.started":"2022-08-31T09:01:40.621991Z","shell.execute_reply":"2022-08-31T09:01:44.741677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_dataset = r'/kaggle/input/energy-anomaly-detection/'","metadata":{"execution":{"iopub.status.busy":"2022-08-31T09:01:44.744425Z","iopub.execute_input":"2022-08-31T09:01:44.744675Z","iopub.status.idle":"2022-08-31T09:01:44.752709Z","shell.execute_reply.started":"2022-08-31T09:01:44.744644Z","shell.execute_reply":"2022-08-31T09:01:44.751546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_features = pd.read_csv(os.path.join(path_dataset,'train_features.csv'))\ntrain_features","metadata":{"execution":{"iopub.status.busy":"2022-08-31T09:01:44.754156Z","iopub.execute_input":"2022-08-31T09:01:44.754457Z","iopub.status.idle":"2022-08-31T09:02:03.324752Z","shell.execute_reply.started":"2022-08-31T09:01:44.754426Z","shell.execute_reply":"2022-08-31T09:02:03.324027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_features.loc[train_features['meter_reading']==1, 'anomaly'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-31T09:19:26.332987Z","iopub.execute_input":"2022-08-31T09:19:26.333317Z","iopub.status.idle":"2022-08-31T09:19:26.345119Z","shell.execute_reply.started":"2022-08-31T09:19:26.333281Z","shell.execute_reply":"2022-08-31T09:19:26.344259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_features.groupby('anomaly')['meter_reading'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-31T09:19:55.418178Z","iopub.execute_input":"2022-08-31T09:19:55.418593Z","iopub.status.idle":"2022-08-31T09:19:55.618361Z","shell.execute_reply.started":"2022-08-31T09:19:55.418544Z","shell.execute_reply":"2022-08-31T09:19:55.617764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"neg = train_features[train_features['anomaly'] == 0]\npos = train_features[train_features['anomaly'] == 1]\n\nprint(neg.shape, pos.shape)\nnegs = neg.sample(n = pos.shape[0], random_state=10)\ndf_eq = pd.concat([negs, pos], axis=0)\nprint(df_eq.shape)","metadata":{"execution":{"iopub.status.busy":"2022-08-31T09:23:58.292346Z","iopub.execute_input":"2022-08-31T09:23:58.292713Z","iopub.status.idle":"2022-08-31T09:23:59.155756Z","shell.execute_reply.started":"2022-08-31T09:23:58.292674Z","shell.execute_reply":"2022-08-31T09:23:59.154778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_eq['log1p(meter_reading)'] = np.log1p(df_eq['meter_reading'])\nsns.displot(data=df_eq, \n            x='log1p(meter_reading)', hue='anomaly', kind='kde')","metadata":{"execution":{"iopub.status.busy":"2022-08-31T09:29:20.348316Z","iopub.execute_input":"2022-08-31T09:29:20.348629Z","iopub.status.idle":"2022-08-31T09:29:21.111459Z","shell.execute_reply.started":"2022-08-31T09:29:20.348593Z","shell.execute_reply":"2022-08-31T09:29:21.110274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(os.path.join(path_dataset,'train.csv'))\ndf_train['meter'] = train_features['building_meter'].str.split('-',expand=True)[1]\ndf_train['fault label'] = df_train['anomaly']\ndf_train = df_train.sort_values(['building_id','timestamp']).reset_index(drop=True)\ndf_train","metadata":{"execution":{"iopub.status.busy":"2022-08-03T07:03:24.694811Z","iopub.execute_input":"2022-08-03T07:03:24.695054Z","iopub.status.idle":"2022-08-03T07:03:31.564663Z","shell.execute_reply.started":"2022-08-03T07:03:24.695024Z","shell.execute_reply":"2022-08-03T07:03:31.563635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.loc[(df_train['fault label']!=0)&(df_train['meter_reading']==df_train['meter_reading'].shift(1)), 'fault label'] = 2\ndf_train.loc[(df_train['fault label']!=0)&(df_train['meter_reading']==df_train['meter_reading'].shift(-1)), 'fault label'] = 2","metadata":{"execution":{"iopub.status.busy":"2022-08-03T07:03:31.566546Z","iopub.execute_input":"2022-08-03T07:03:31.566840Z","iopub.status.idle":"2022-08-03T07:03:31.610925Z","shell.execute_reply.started":"2022-08-03T07:03:31.566801Z","shell.execute_reply":"2022-08-03T07:03:31.610056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fault labels:\n- 0: Normal\n- 1: Outliers\n- 2: Continuous constants","metadata":{}},{"cell_type":"code","source":"df_train['fault label'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T07:03:31.612249Z","iopub.execute_input":"2022-08-03T07:03:31.612507Z","iopub.status.idle":"2022-08-03T07:03:31.636529Z","shell.execute_reply.started":"2022-08-03T07:03:31.612478Z","shell.execute_reply":"2022-08-03T07:03:31.635559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['fault label'].value_counts()/df_train['fault label'].value_counts().sum()*100","metadata":{"execution":{"iopub.status.busy":"2022-08-03T07:03:31.637844Z","iopub.execute_input":"2022-08-03T07:03:31.638250Z","iopub.status.idle":"2022-08-03T07:03:31.680767Z","shell.execute_reply.started":"2022-08-03T07:03:31.638196Z","shell.execute_reply":"2022-08-03T07:03:31.679932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pivot_train = df_train.pivot_table(columns=['building_id','meter'], index='timestamp', values='meter_reading')\n\nconsecutives = pivot_train.copy()\nconsecutives.loc[:, :] = 0\n\nfor idx, idx_next in tqdm.tqdm(zip(pivot_train.index[:], pivot_train.index[1:])):\n    consecutives.loc[idx_next, pivot_train.loc[idx] == pivot_train.loc[idx_next]] = \\\n    consecutives.loc[idx, pivot_train.loc[idx] == pivot_train.loc[idx_next]] + 1","metadata":{"execution":{"iopub.status.busy":"2022-08-03T07:03:31.681978Z","iopub.execute_input":"2022-08-03T07:03:31.682317Z","iopub.status.idle":"2022-08-03T07:04:32.503852Z","shell.execute_reply.started":"2022-08-03T07:03:31.682268Z","shell.execute_reply":"2022-08-03T07:04:32.502839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"consecutives = consecutives.reset_index().melt(id_vars='timestamp', value_name='consecutive hours')\ndf_train = df_train.merge(consecutives, on=['building_id','meter','timestamp'])\n\ndf_train['peak_consecutive hours'] = -(df_train['consecutive hours'] - df_train['consecutive hours'].shift(1)) + 1\ndf_train.loc[df_train['peak_consecutive hours']<=1, 'peak_consecutive hours'] = 0\ndf_train.loc[~((df_train['building_id']==df_train['building_id'].shift(1))&(df_train['meter']==df_train['meter'].shift(1))), 'peak_consecutive hours'] = 0\n\ndf_train","metadata":{"execution":{"iopub.status.busy":"2022-08-03T07:04:32.505653Z","iopub.execute_input":"2022-08-03T07:04:32.505932Z","iopub.status.idle":"2022-08-03T07:04:34.324125Z","shell.execute_reply.started":"2022-08-03T07:04:32.505902Z","shell.execute_reply":"2022-08-03T07:04:34.323264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.loc[(df_train['peak_consecutive hours']>=24)&(df_train['fault label']!=0)]","metadata":{"execution":{"iopub.status.busy":"2022-08-03T07:04:34.326746Z","iopub.execute_input":"2022-08-03T07:04:34.326984Z","iopub.status.idle":"2022-08-03T07:04:34.530197Z","shell.execute_reply.started":"2022-08-03T07:04:34.326954Z","shell.execute_reply":"2022-08-03T07:04:34.529382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.loc[(df_train['peak_consecutive hours']>=24)&(df_train['fault label']!=0), 'peak_consecutive hours'].plot(title='Consecutive hours of faults', kind='hist', logy=True, bins=100, figsize=(15,5))","metadata":{"execution":{"iopub.status.busy":"2022-08-03T07:04:34.531468Z","iopub.execute_input":"2022-08-03T07:04:34.531785Z","iopub.status.idle":"2022-08-03T07:04:35.286969Z","shell.execute_reply.started":"2022-08-03T07:04:34.531752Z","shell.execute_reply":"2022-08-03T07:04:35.286109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_peak_consecutive = df_train.groupby(['building_id','meter'])['peak_consecutive hours'].max().reset_index()\ndf_peak_consecutive","metadata":{"execution":{"iopub.status.busy":"2022-08-03T07:04:35.288660Z","iopub.execute_input":"2022-08-03T07:04:35.288965Z","iopub.status.idle":"2022-08-03T07:04:35.572887Z","shell.execute_reply.started":"2022-08-03T07:04:35.288922Z","shell.execute_reply":"2022-08-03T07:04:35.572077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_peak_consecutive.loc[df_peak_consecutive['peak_consecutive hours']<24, 'length_abnorm'] = '<24 hours'\ndf_peak_consecutive.loc[df_peak_consecutive['peak_consecutive hours']>=24, 'length_abnorm'] = '1-7 days'\ndf_peak_consecutive.loc[df_peak_consecutive['peak_consecutive hours']>24*7, 'length_abnorm'] = '1-4 weeks'\ndf_peak_consecutive.loc[df_peak_consecutive['peak_consecutive hours']>24*7*4, 'length_abnorm'] = 'months'","metadata":{"execution":{"iopub.status.busy":"2022-08-03T07:04:35.574436Z","iopub.execute_input":"2022-08-03T07:04:35.574755Z","iopub.status.idle":"2022-08-03T07:04:35.584237Z","shell.execute_reply.started":"2022-08-03T07:04:35.574715Z","shell.execute_reply":"2022-08-03T07:04:35.583242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_peak_consecutive.sort_values('peak_consecutive hours',ascending=False)['length_abnorm'].value_counts(sort=False).plot(kind='barh')","metadata":{"execution":{"iopub.status.busy":"2022-08-03T07:04:35.585854Z","iopub.execute_input":"2022-08-03T07:04:35.586514Z","iopub.status.idle":"2022-08-03T07:04:35.763025Z","shell.execute_reply.started":"2022-08-03T07:04:35.586472Z","shell.execute_reply":"2022-08-03T07:04:35.762486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_peak_consecutive.sort_values('peak_consecutive hours',ascending=False)['length_abnorm'].value_counts(sort=False)/len(df_peak_consecutive)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T07:04:35.764007Z","iopub.execute_input":"2022-08-03T07:04:35.764607Z","iopub.status.idle":"2022-08-03T07:04:35.774056Z","shell.execute_reply.started":"2022-08-03T07:04:35.764556Z","shell.execute_reply":"2022-08-03T07:04:35.773260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pivot_train = df_train.pivot_table(index=['building_id','meter'], columns='fault label', values='meter_reading', aggfunc='count')\npivot_train = pivot_train.reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T07:04:35.774951Z","iopub.execute_input":"2022-08-03T07:04:35.775663Z","iopub.status.idle":"2022-08-03T07:04:36.139003Z","shell.execute_reply.started":"2022-08-03T07:04:35.775631Z","shell.execute_reply":"2022-08-03T07:04:36.138020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pivot_train[pivot_train.loc[:, 1:].sum(axis=1)!=0]","metadata":{"execution":{"iopub.status.busy":"2022-08-03T07:04:36.140297Z","iopub.execute_input":"2022-08-03T07:04:36.140600Z","iopub.status.idle":"2022-08-03T07:04:36.159111Z","shell.execute_reply.started":"2022-08-03T07:04:36.140571Z","shell.execute_reply":"2022-08-03T07:04:36.158449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for index, row in pivot_train[pivot_train.loc[:, 1:].sum(axis=1)!=0].sort_values(['building_id','meter']).sample(n=10,axis=0,random_state=42).iterrows():\n    building_id, meter, label_0, label_1, label_2 = row\n    print('building_id: '+str(int(building_id)))\n    print('meter: '+str(int(meter)))\n    \n    timeSeries = df_train[(df_train['building_id']==building_id)&(df_train['meter']==meter)].copy()\n    \n    timeSeries['fault label'] = timeSeries['fault label'].replace({0: 'Normal', 1: 'Outliers', 2: 'Continuous constants'})\n    print(timeSeries['fault label'].value_counts().to_dict())\n    \n    timeSeries = timeSeries.pivot_table(index='timestamp',columns='fault label',values='meter_reading').dropna(how='all')\n    \n    for col in ['Normal', 'Outliers', 'Continuous constants']:\n        if col not in list(timeSeries.columns):\n            timeSeries[col] = pd.Series(dtype='float')\n    \n    timeSeries = timeSeries[['Normal', 'Outliers', 'Continuous constants']]\n    timeSeries.iplot(title='building_id: '+str(int(building_id))+' / meter: '+str(int(meter)))\n\n    \n    print('--------------------------------------------------------------------------------------')","metadata":{"execution":{"iopub.status.busy":"2022-08-03T07:04:36.160251Z","iopub.execute_input":"2022-08-03T07:04:36.160454Z","iopub.status.idle":"2022-08-03T07:04:42.968468Z","shell.execute_reply.started":"2022-08-03T07:04:36.160428Z","shell.execute_reply":"2022-08-03T07:04:42.967708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.columns","metadata":{"execution":{"iopub.status.busy":"2022-08-03T07:07:27.369527Z","iopub.execute_input":"2022-08-03T07:07:27.370509Z","iopub.status.idle":"2022-08-03T07:07:27.375868Z","shell.execute_reply.started":"2022-08-03T07:07:27.370470Z","shell.execute_reply":"2022-08-03T07:07:27.375255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"    df_temp = df_train[df_train['building_id']==building_id].pivot_table(index='timestamp',values=['meter_reading', 'anomaly', 'fault label']).copy()\n    df_temp.index = pd.to_datetime(df_temp.index)\n    df_temp = df_temp.asfreq('H')    \n    df_temp['tow'] = df_temp.index.weekday*24 + df_temp.index.hour\n    df_temp['week'] = (df_temp.index.year-2016)*52 + df_temp.index.isocalendar().week\n    df_temp","metadata":{"execution":{"iopub.status.busy":"2022-08-03T07:07:51.401798Z","iopub.execute_input":"2022-08-03T07:07:51.402084Z","iopub.status.idle":"2022-08-03T07:07:51.561159Z","shell.execute_reply.started":"2022-08-03T07:07:51.402054Z","shell.execute_reply":"2022-08-03T07:07:51.560303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for index, row in pivot_train[pivot_train.loc[:, 1:].sum(axis=1)!=0].sort_values(['building_id','meter']).iterrows():\n    building_id, meter, label_0, label_1, label_2 = row\n    print('building_id: '+str(int(building_id)))\n    \n    fig, axes = plt.subplots(nrows=1, ncols=3, figsize=(18, 4))  \n\n    #my_dpi = 96\n    df_temp = df_train[df_train['building_id']==building_id].pivot_table(index='timestamp',values=['meter_reading', 'anomaly', 'fault label']).copy()\n    df_temp.index = pd.to_datetime(df_temp.index)\n    df_temp = df_temp.asfreq('H')    \n    df_temp['tow'] = df_temp.index.weekday*24 + df_temp.index.hour\n    df_temp['week'] = (df_temp.index.year-2016)*52 + df_temp.index.isocalendar().week\n    heatmap_anomaly_type = df_temp.pivot_table(index='week',columns='tow',values='fault label',dropna=False)\n    heatmap_anomaly = df_temp.pivot_table(index='week',columns='tow',values='anomaly',dropna=False)\n    heatmap_meter_reading = df_temp.pivot_table(index='week',columns='tow',values='meter_reading',dropna=False)\n\n    #fig = plt.figure(figsize=(125/my_dpi, 128/my_dpi), dpi=my_dpi)\n    ax1 = sns.heatmap(heatmap_anomaly_type, cmap=\"RdYlGn_r\", cbar=False, ax=axes[0])\n    ax1.set(xlabel=None,ylabel=None,xticklabels=[],yticklabels=[])\n    ax1.tick_params(bottom=False,left=False)\n\n    #fig = plt.figure(figsize=(125/my_dpi, 128/my_dpi), dpi=my_dpi)\n    ax2 = sns.heatmap(heatmap_anomaly, cmap=\"RdYlGn_r\", cbar=False, ax=axes[1])\n    ax2.set(xlabel=None,ylabel=None,xticklabels=[],yticklabels=[])\n    ax2.tick_params(bottom=False,left=False)    \n\n    #fig = plt.figure(figsize=(125/my_dpi, 128/my_dpi), dpi=my_dpi)\n    ax3 = sns.heatmap(heatmap_meter_reading, cmap=\"RdYlGn_r\", cbar=False, ax=axes[2])\n    ax3.set(xlabel=None,ylabel=None,xticklabels=[],yticklabels=[])\n    ax3.tick_params(bottom=False,left=False)\n\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T07:12:48.765703Z","iopub.execute_input":"2022-08-03T07:12:48.765999Z","iopub.status.idle":"2022-08-03T07:16:27.061118Z","shell.execute_reply.started":"2022-08-03T07:12:48.765969Z","shell.execute_reply":"2022-08-03T07:16:27.060089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for index, row in pivot_train[pivot_train.loc[:, 1:].sum(axis=1)!=0].sort_values(['building_id','meter']).iterrows():\n    building_id, meter, label_0, label_1, label_2 = row\n    print('building_id: '+str(int(building_id)))\n    print('meter: '+str(int(meter)))\n    \n    timeSeries = df_train[(df_train['building_id']==building_id)&(df_train['meter']==meter)].copy()\n    \n    timeSeries['fault label'] = timeSeries['fault label'].replace({0: 'Normal', 1: 'Outliers', 2: 'Continuous constants'})\n    print(timeSeries['fault label'].value_counts().to_dict())\n    \n    timeSeries = timeSeries.pivot_table(index='timestamp',columns='fault label',values='meter_reading').dropna(how='all')\n    \n    for col in ['Normal', 'Outliers', 'Continuous constants']:\n        if col not in list(timeSeries.columns):\n            timeSeries[col] = pd.Series(dtype='float')\n    \n    timeSeries = timeSeries[['Normal', 'Outliers', 'Continuous constants']]\n    timeSeries.plot(figsize=(15,5))\n    plt.legend(loc='center left', bbox_to_anchor=(1.0, 0.5))\n    plt.show()\n    \n    print('--------------------------------------------------------------------------------------')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}