{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport datetime as dt\n\n%matplotlib inline","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-10T07:43:37.378133Z","iopub.execute_input":"2022-06-10T07:43:37.380329Z","iopub.status.idle":"2022-06-10T07:43:38.400026Z","shell.execute_reply.started":"2022-06-10T07:43:37.380212Z","shell.execute_reply":"2022-06-10T07:43:38.398808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/energy-anomaly-detection/train.csv',parse_dates=['timestamp'])\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:43:38.402205Z","iopub.execute_input":"2022-06-10T07:43:38.402633Z","iopub.status.idle":"2022-06-10T07:43:40.292645Z","shell.execute_reply.started":"2022-06-10T07:43:38.402596Z","shell.execute_reply":"2022-06-10T07:43:40.291668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:43:40.293878Z","iopub.execute_input":"2022-06-10T07:43:40.294297Z","iopub.status.idle":"2022-06-10T07:43:40.302520Z","shell.execute_reply.started":"2022-06-10T07:43:40.294263Z","shell.execute_reply":"2022-06-10T07:43:40.301598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:43:40.305175Z","iopub.execute_input":"2022-06-10T07:43:40.305580Z","iopub.status.idle":"2022-06-10T07:43:40.316504Z","shell.execute_reply.started":"2022-06-10T07:43:40.305544Z","shell.execute_reply":"2022-06-10T07:43:40.315386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"{0:.2f}%\".format((train['meter_reading'].isnull().sum() / len(train)) * 100))","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:43:40.318035Z","iopub.execute_input":"2022-06-10T07:43:40.318368Z","iopub.status.idle":"2022-06-10T07:43:40.335464Z","shell.execute_reply.started":"2022-06-10T07:43:40.318339Z","shell.execute_reply":"2022-06-10T07:43:40.334646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.subplots(figsize=(15,5))\nplt.subplot(1,2,1)\nsns.countplot(x=train['anomaly'])\nplt.subplot(1,2,2)\ndata = train['anomaly'].value_counts()\nplt.pie(data, labels=['Normal', 'Anomalous'], colors=sns.color_palette('bright'), autopct='%.0f%%')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:43:40.336662Z","iopub.execute_input":"2022-06-10T07:43:40.337754Z","iopub.status.idle":"2022-06-10T07:43:40.775039Z","shell.execute_reply.started":"2022-06-10T07:43:40.337686Z","shell.execute_reply":"2022-06-10T07:43:40.773846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,5))\ntrain.groupby(train.timestamp.dt.hour).mean()['meter_reading'].plot.bar()\nplt.title('Media oraria', fontsize=20)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:43:40.776733Z","iopub.execute_input":"2022-06-10T07:43:40.777453Z","iopub.status.idle":"2022-06-10T07:43:41.305868Z","shell.execute_reply.started":"2022-06-10T07:43:40.777406Z","shell.execute_reply":"2022-06-10T07:43:41.304939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,5))\ntrain.groupby(train.timestamp.dt.month).mean()['meter_reading'].plot.bar()\nplt.title('Media mensile', fontsize=20)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:43:41.307396Z","iopub.execute_input":"2022-06-10T07:43:41.307979Z","iopub.status.idle":"2022-06-10T07:43:41.736709Z","shell.execute_reply.started":"2022-06-10T07:43:41.307944Z","shell.execute_reply":"2022-06-10T07:43:41.735831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,5))\ntrain.groupby(train.timestamp.dt.month).mean()['meter_reading'].plot.bar()\nplt.title('Media mensile', fontsize=20)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:43:41.737891Z","iopub.execute_input":"2022-06-10T07:43:41.738248Z","iopub.status.idle":"2022-06-10T07:43:42.171710Z","shell.execute_reply.started":"2022-06-10T07:43:41.738216Z","shell.execute_reply":"2022-06-10T07:43:42.170983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,5))\ntrain.groupby(train.timestamp.dt.dayofweek).mean()['meter_reading'].plot.bar()\nplt.title('Media settimanale', fontsize=20)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:43:42.174230Z","iopub.execute_input":"2022-06-10T07:43:42.175160Z","iopub.status.idle":"2022-06-10T07:43:42.600281Z","shell.execute_reply.started":"2022-06-10T07:43:42.175118Z","shell.execute_reply":"2022-06-10T07:43:42.599283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"buildings = train.groupby('building_id')\nfor group_name, df_group in buildings:\n    ewma = df_group.meter_reading.ewm(span=10).mean()\n    train.loc[df_group.index,'ewma'] = ewma.values\n    \ntrain['meter_reading'].fillna(train['ewma'], inplace=True)\n\ntrain['meter_reading'] = train.groupby('building_id')['meter_reading'].apply(lambda x: x.fillna(x.mean()))","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:43:42.601568Z","iopub.execute_input":"2022-06-10T07:43:42.601882Z","iopub.status.idle":"2022-06-10T07:43:45.749300Z","shell.execute_reply.started":"2022-06-10T07:43:42.601855Z","shell.execute_reply":"2022-06-10T07:43:45.748328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"buildings = train.groupby('building_id')\nfor group_name, df_group in buildings:\n    differences = df_group.sort_values(\"timestamp\").diff()\n    deltaKwh = differences['meter_reading'] / (differences.timestamp.dt.total_seconds() // 3600) \n    deltaKwh = (deltaKwh - deltaKwh.mean()) / deltaKwh.std()\n    train.loc[deltaKwh.index,'delta'] = deltaKwh.values","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:43:45.750736Z","iopub.execute_input":"2022-06-10T07:43:45.751217Z","iopub.status.idle":"2022-06-10T07:43:48.664132Z","shell.execute_reply.started":"2022-06-10T07:43:45.751171Z","shell.execute_reply":"2022-06-10T07:43:48.662772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"buildings = train.groupby('building_id')\nfor group_name, df_group in buildings:\n    mobileMean = df_group.meter_reading.ewm(span=24).mean()\n    meanKwh = (df_group['meter_reading'] - mobileMean) / mobileMean.std()\n    train.loc[meanKwh.index,'mean_difference'] = meanKwh.values","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:43:48.665538Z","iopub.execute_input":"2022-06-10T07:43:48.666194Z","iopub.status.idle":"2022-06-10T07:43:51.187692Z","shell.execute_reply.started":"2022-06-10T07:43:48.666143Z","shell.execute_reply":"2022-06-10T07:43:51.186945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"anomalies = train.groupby('building_id').sum()['anomaly']\nanomalies.sort_values()","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:43:51.190042Z","iopub.execute_input":"2022-06-10T07:43:51.190553Z","iopub.status.idle":"2022-06-10T07:43:51.327949Z","shell.execute_reply.started":"2022-06-10T07:43:51.190505Z","shell.execute_reply":"2022-06-10T07:43:51.326982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"building_id = 1319\n\nfilter_build =  train['building_id'] == building_id\nbuilding = train[filter_build]\nbuilding = building.sort_values(\"timestamp\")","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:43:51.330106Z","iopub.execute_input":"2022-06-10T07:43:51.330556Z","iopub.status.idle":"2022-06-10T07:43:51.341652Z","shell.execute_reply.started":"2022-06-10T07:43:51.330512Z","shell.execute_reply":"2022-06-10T07:43:51.340888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cond = (building.anomaly != building.anomaly.shift().fillna(0).astype(int)).cumsum()\noutput = building.groupby(cond).agg(anomaly=('anomaly', 'first'),lenght=('anomaly', 'size'),min=('timestamp', 'min'),max=('timestamp', 'max'))\noutput","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:43:51.344515Z","iopub.execute_input":"2022-06-10T07:43:51.345580Z","iopub.status.idle":"2022-06-10T07:43:51.385882Z","shell.execute_reply.started":"2022-06-10T07:43:51.345544Z","shell.execute_reply":"2022-06-10T07:43:51.384713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start = '2016-12-22 07:00:00'\nend = '2016-12-29 09:00:00'\n\nanomalyFilter = building['timestamp'].between(left=start,right=end)\nbuilding[anomalyFilter]","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:43:51.387372Z","iopub.execute_input":"2022-06-10T07:43:51.387861Z","iopub.status.idle":"2022-06-10T07:43:51.421517Z","shell.execute_reply.started":"2022-06-10T07:43:51.387814Z","shell.execute_reply":"2022-06-10T07:43:51.420374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ewma = building.meter_reading.ewm(span=24).mean()","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:43:51.423112Z","iopub.execute_input":"2022-06-10T07:43:51.423991Z","iopub.status.idle":"2022-06-10T07:43:51.431297Z","shell.execute_reply.started":"2022-06-10T07:43:51.423941Z","shell.execute_reply":"2022-06-10T07:43:51.429993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,5))\nplt.scatter(building['timestamp'], building['meter_reading'], marker='o',c=building['anomaly'], cmap='viridis', alpha=0.5)\nplt.plot(building['timestamp'],ewma,label='EWMA')\nplt.legend(loc='upper right')\nplt.tick_params(axis='x',labelsize=8,rotation=90)\nplt.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:43:51.433555Z","iopub.execute_input":"2022-06-10T07:43:51.434772Z","iopub.status.idle":"2022-06-10T07:43:51.934203Z","shell.execute_reply.started":"2022-06-10T07:43:51.434639Z","shell.execute_reply":"2022-06-10T07:43:51.933218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"{0:.2f}%\".format((train['meter_reading'].isnull().sum() / len(train)) * 100))","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:43:51.935469Z","iopub.execute_input":"2022-06-10T07:43:51.935942Z","iopub.status.idle":"2022-06-10T07:43:51.948195Z","shell.execute_reply.started":"2022-06-10T07:43:51.935896Z","shell.execute_reply":"2022-06-10T07:43:51.946938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.drop([\"ewma\"], axis=1)\ntrain[\"delta\"].fillna(0, inplace=True)\ntrain","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:43:51.949757Z","iopub.execute_input":"2022-06-10T07:43:51.950650Z","iopub.status.idle":"2022-06-10T07:43:51.992462Z","shell.execute_reply.started":"2022-06-10T07:43:51.950602Z","shell.execute_reply":"2022-06-10T07:43:51.991153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"plt.figure(figsize=(15,5))\nplt.scatter(train['timestamp'], train['delta'], marker='o',c=train['anomaly'], cmap='viridis', alpha=0.5)","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:24:27.048131Z","iopub.execute_input":"2022-06-10T07:24:27.048513Z","iopub.status.idle":"2022-06-10T07:25:02.446681Z","shell.execute_reply.started":"2022-06-10T07:24:27.048483Z","shell.execute_reply":"2022-06-10T07:25:02.445869Z"}}},{"cell_type":"markdown","source":"plt.figure(figsize=(15,5))\nplt.scatter(train['timestamp'], train['mean_difference'], marker='o',c=train['anomaly'], cmap='viridis', alpha=0.5)","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:25:02.448052Z","iopub.execute_input":"2022-06-10T07:25:02.448364Z","iopub.status.idle":"2022-06-10T07:25:38.078118Z","shell.execute_reply.started":"2022-06-10T07:25:02.448336Z","shell.execute_reply":"2022-06-10T07:25:38.077280Z"}}},{"cell_type":"code","source":"train['weekend'] = (train.timestamp.dt.dayofweek >= 5).astype(int)\ntrain['hour'] = train.timestamp.dt.hour\ntrain['month'] = train.timestamp.dt.month\n\ntrain['meter_reading'] = (train['meter_reading'] - train['meter_reading'].mean()) / train['meter_reading'].std()","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:43:51.995567Z","iopub.execute_input":"2022-06-10T07:43:51.996347Z","iopub.status.idle":"2022-06-10T07:43:52.540605Z","shell.execute_reply.started":"2022-06-10T07:43:51.996309Z","shell.execute_reply":"2022-06-10T07:43:52.539531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#rebalancing\ng = train.groupby('anomaly')\nrebalanced = g.apply(lambda x: x.sample(g.size().min()).reset_index(drop=True)).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:43:52.541896Z","iopub.execute_input":"2022-06-10T07:43:52.542706Z","iopub.status.idle":"2022-06-10T07:43:52.852815Z","shell.execute_reply.started":"2022-06-10T07:43:52.542639Z","shell.execute_reply":"2022-06-10T07:43:52.851798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target = rebalanced['anomaly']\nfeatures = rebalanced.drop(['building_id', 'anomaly','timestamp'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:43:52.854651Z","iopub.execute_input":"2022-06-10T07:43:52.855106Z","iopub.status.idle":"2022-06-10T07:43:52.862964Z","shell.execute_reply.started":"2022-06-10T07:43:52.855063Z","shell.execute_reply":"2022-06-10T07:43:52.861551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import GridSearchCV,RandomizedSearchCV,cross_val_score,train_test_split\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.svm import SVC","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:43:52.864670Z","iopub.execute_input":"2022-06-10T07:43:52.865354Z","iopub.status.idle":"2022-06-10T07:43:53.243084Z","shell.execute_reply.started":"2022-06-10T07:43:52.865310Z","shell.execute_reply":"2022-06-10T07:43:53.242082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"param_grid = [\n    {\n        'penalty' : ['l1', 'l2'],\n        'C' : np.logspace(-4, 4, 20),\n        'solver': ['liblinear', 'saga']\n    }\n]\n\nclf = GridSearchCV(LogisticRegression(), param_grid)\n\n# fitting the model for grid search \nclf.fit(features,target)\nclf.best_params_\n\n# summarize\nprint('Mean Accuracy: %.3f' % clf.best_score_)\nprint('Config: %s' % clf.best_params_)","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:26:58.194743Z","iopub.execute_input":"2022-06-10T07:26:58.195125Z","iopub.status.idle":"2022-06-10T07:28:22.285069Z","shell.execute_reply.started":"2022-06-10T07:26:58.195096Z","shell.execute_reply":"2022-06-10T07:28:22.284221Z"}}},{"cell_type":"markdown","source":"clf.score(features,target)","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:28:22.286725Z","iopub.execute_input":"2022-06-10T07:28:22.287244Z","iopub.status.idle":"2022-06-10T07:28:22.303697Z","shell.execute_reply.started":"2022-06-10T07:28:22.287188Z","shell.execute_reply":"2022-06-10T07:28:22.303007Z"}}},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(features,target, test_size=0.20)","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:43:53.244264Z","iopub.execute_input":"2022-06-10T07:43:53.244611Z","iopub.status.idle":"2022-06-10T07:43:53.262978Z","shell.execute_reply.started":"2022-06-10T07:43:53.244582Z","shell.execute_reply":"2022-06-10T07:43:53.262224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"from pprint import pprint\n\nn_estimators = np.arange(200,2000,100)\nmax_features = ['auto', 'sqrt']\nmax_depth = np.arange(10,100,10)\nmax_depth = np.append(max_depth,None)\nmin_samples_split = [2, 5, 10]\nmin_samples_leaf = [1, 2, 4]\nbootstrap = [True, False]\nrandom_grid = {'n_estimators': n_estimators,\n               'max_features': max_features,\n               'max_depth': max_depth,\n               'min_samples_split': min_samples_split,\n               'min_samples_leaf': min_samples_leaf,\n               'bootstrap': bootstrap}\n\npprint(random_grid)\nrf = RandomForestClassifier()\nrf_random = RandomizedSearchCV(estimator = rf, param_distributions = random_grid, n_iter = 10,cv=3, verbose=2, n_jobs = -1)\nrf_random.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:29:06.795420Z","iopub.execute_input":"2022-06-10T07:29:06.796918Z","iopub.status.idle":"2022-06-10T07:38:26.336886Z","shell.execute_reply.started":"2022-06-10T07:29:06.796879Z","shell.execute_reply":"2022-06-10T07:38:26.334471Z"}}},{"cell_type":"markdown","source":"clf.score(X_test,y_test)","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:38:26.342269Z","iopub.status.idle":"2022-06-10T07:38:26.342623Z","shell.execute_reply.started":"2022-06-10T07:38:26.342454Z","shell.execute_reply":"2022-06-10T07:38:26.342471Z"}}},{"cell_type":"code","source":"rf = RandomForestClassifier()\nrf.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:43:53.264049Z","iopub.execute_input":"2022-06-10T07:43:53.264731Z","iopub.status.idle":"2022-06-10T07:44:02.938657Z","shell.execute_reply.started":"2022-06-10T07:43:53.264700Z","shell.execute_reply":"2022-06-10T07:44:02.937690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rf.score(X_test,y_test)","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:44:02.942587Z","iopub.execute_input":"2022-06-10T07:44:02.942973Z","iopub.status.idle":"2022-06-10T07:44:03.308807Z","shell.execute_reply.started":"2022-06-10T07:44:02.942938Z","shell.execute_reply":"2022-06-10T07:44:03.308094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\nfilename = 'finalized_model.sav'\npickle.dump(rf, open(filename, 'wb'))","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:44:03.309856Z","iopub.execute_input":"2022-06-10T07:44:03.310636Z","iopub.status.idle":"2022-06-10T07:44:03.491683Z","shell.execute_reply.started":"2022-06-10T07:44:03.310604Z","shell.execute_reply":"2022-06-10T07:44:03.490603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv('../input/energy-anomaly-detection/test.csv',parse_dates=['timestamp'])\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:44:03.493109Z","iopub.execute_input":"2022-06-10T07:44:03.493453Z","iopub.status.idle":"2022-06-10T07:44:05.568091Z","shell.execute_reply.started":"2022-06-10T07:44:03.493422Z","shell.execute_reply":"2022-06-10T07:44:05.567006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"buildings = test.groupby('building_id')\nfor group_name, df_group in buildings:\n    ewma = df_group.meter_reading.ewm(span=10).mean()\n    test.loc[df_group.index,'ewma'] = ewma.values\n    \ntest['meter_reading'].fillna(test['ewma'], inplace=True)\n\ntest['meter_reading'] = test.groupby('building_id')['meter_reading'].apply(lambda x: x.fillna(x.mean()))","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:44:05.569613Z","iopub.execute_input":"2022-06-10T07:44:05.570254Z","iopub.status.idle":"2022-06-10T07:44:08.915147Z","shell.execute_reply.started":"2022-06-10T07:44:05.570217Z","shell.execute_reply":"2022-06-10T07:44:08.914019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"buildings = test.groupby('building_id')\nfor group_name, df_group in buildings:\n    differences = df_group.sort_values(\"timestamp\").diff()\n    deltaKwh = differences['meter_reading'] / (differences.timestamp.dt.total_seconds() // 3600) \n    deltaKwh = (deltaKwh - deltaKwh.mean()) / deltaKwh.std()\n    test.loc[deltaKwh.index,'delta'] = deltaKwh.values","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:44:08.916744Z","iopub.execute_input":"2022-06-10T07:44:08.917322Z","iopub.status.idle":"2022-06-10T07:44:12.022028Z","shell.execute_reply.started":"2022-06-10T07:44:08.917287Z","shell.execute_reply":"2022-06-10T07:44:12.021031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"buildings = test.groupby('building_id')\nfor group_name, df_group in buildings:\n    mobileMean = df_group.meter_reading.ewm(span=24).mean()\n    meanKwh = (df_group['meter_reading'] - mobileMean) / mobileMean.std()\n    test.loc[meanKwh.index,'mean_difference'] = meanKwh.values","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:44:12.023214Z","iopub.execute_input":"2022-06-10T07:44:12.023549Z","iopub.status.idle":"2022-06-10T07:44:14.728367Z","shell.execute_reply.started":"2022-06-10T07:44:12.023522Z","shell.execute_reply":"2022-06-10T07:44:14.727355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = test.drop([\"ewma\"], axis=1)\ntest[\"delta\"].fillna(0, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:44:14.729606Z","iopub.execute_input":"2022-06-10T07:44:14.729968Z","iopub.status.idle":"2022-06-10T07:44:14.816742Z","shell.execute_reply.started":"2022-06-10T07:44:14.729937Z","shell.execute_reply":"2022-06-10T07:44:14.816043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['weekend'] = (test.timestamp.dt.dayofweek >= 5).astype(int)\ntest['hour'] = test.timestamp.dt.hour\ntest['month'] = test.timestamp.dt.month\n\ntest['meter_reading'] = (test['meter_reading'] - test['meter_reading'].mean()) / test['meter_reading'].std()","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:44:14.817662Z","iopub.execute_input":"2022-06-10T07:44:14.818593Z","iopub.status.idle":"2022-06-10T07:44:15.363938Z","shell.execute_reply.started":"2022-06-10T07:44:14.818558Z","shell.execute_reply":"2022-06-10T07:44:15.362929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = test.drop(['building_id','timestamp'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:44:15.365358Z","iopub.execute_input":"2022-06-10T07:44:15.365698Z","iopub.status.idle":"2022-06-10T07:44:15.477981Z","shell.execute_reply.started":"2022-06-10T07:44:15.365667Z","shell.execute_reply":"2022-06-10T07:44:15.476852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xtest = test[['meter_reading','delta','mean_difference','weekend','hour','month']]\npredictions = rf.predict(xtest)\n\noutput = pd.DataFrame({'row_id': test.row_id, 'anomaly': predictions})\noutput.to_csv('submission.csv', index=False)\nprint(\"Your submission was successfully saved!\")","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:44:15.479539Z","iopub.execute_input":"2022-06-10T07:44:15.479985Z","iopub.status.idle":"2022-06-10T07:44:43.983191Z","shell.execute_reply.started":"2022-06-10T07:44:15.479943Z","shell.execute_reply":"2022-06-10T07:44:43.982134Z"},"trusted":true},"execution_count":null,"outputs":[]}]}