{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":66052,"databundleVersionId":7284431,"sourceType":"competition"}],"dockerImageVersionId":30635,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1. Import modules","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n# adfuller library \nfrom statsmodels.tsa.stattools import adfuller\nfrom sklearn import preprocessing\nfrom sklearn.preprocessing import RobustScaler\nimport lightgbm as lgb\nfrom sklearn.ensemble import IsolationForest\nfrom sklearn.preprocessing import RobustScaler\nfrom sklearn.feature_selection import VarianceThreshold\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:13:56.145228Z","iopub.execute_input":"2024-03-17T16:13:56.145617Z","iopub.status.idle":"2024-03-17T16:13:59.352160Z","shell.execute_reply.started":"2024-03-17T16:13:56.145586Z","shell.execute_reply":"2024-03-17T16:13:59.350817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Loading the dataset","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/ai-serving-grid-stability/train.csv')\ntest= pd.read_csv('../input/ai-serving-grid-stability/test.csv')\nsample_submission = pd.read_csv('../input/ai-serving-grid-stability/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:13:59.353839Z","iopub.execute_input":"2024-03-17T16:13:59.354153Z","iopub.status.idle":"2024-03-17T16:14:22.364558Z","shell.execute_reply.started":"2024-03-17T16:13:59.354124Z","shell.execute_reply":"2024-03-17T16:14:22.362841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Data Discription","metadata":{}},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:14:22.366352Z","iopub.execute_input":"2024-03-17T16:14:22.366945Z","iopub.status.idle":"2024-03-17T16:14:22.404235Z","shell.execute_reply.started":"2024-03-17T16:14:22.366912Z","shell.execute_reply":"2024-03-17T16:14:22.403227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:14:22.405767Z","iopub.execute_input":"2024-03-17T16:14:22.406090Z","iopub.status.idle":"2024-03-17T16:14:22.755300Z","shell.execute_reply.started":"2024-03-17T16:14:22.406060Z","shell.execute_reply":"2024-03-17T16:14:22.753919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.info()","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:14:22.758134Z","iopub.execute_input":"2024-03-17T16:14:22.758460Z","iopub.status.idle":"2024-03-17T16:14:22.773981Z","shell.execute_reply.started":"2024-03-17T16:14:22.758436Z","shell.execute_reply":"2024-03-17T16:14:22.772435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.info()","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:14:22.775491Z","iopub.execute_input":"2024-03-17T16:14:22.775851Z","iopub.status.idle":"2024-03-17T16:14:22.789060Z","shell.execute_reply.started":"2024-03-17T16:14:22.775818Z","shell.execute_reply":"2024-03-17T16:14:22.787584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* It seems like there is no missing data.","metadata":{}},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:14:22.790920Z","iopub.execute_input":"2024-03-17T16:14:22.791419Z","iopub.status.idle":"2024-03-17T16:14:22.806903Z","shell.execute_reply.started":"2024-03-17T16:14:22.791382Z","shell.execute_reply":"2024-03-17T16:14:22.805353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head(20).T","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:14:22.808794Z","iopub.execute_input":"2024-03-17T16:14:22.810662Z","iopub.status.idle":"2024-03-17T16:14:22.844862Z","shell.execute_reply.started":"2024-03-17T16:14:22.810634Z","shell.execute_reply":"2024-03-17T16:14:22.843658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe().T","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:14:22.846325Z","iopub.execute_input":"2024-03-17T16:14:22.846634Z","iopub.status.idle":"2024-03-17T16:14:26.553175Z","shell.execute_reply.started":"2024-03-17T16:14:22.846608Z","shell.execute_reply":"2024-03-17T16:14:26.551984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.select_dtypes(exclude = [object]).corr()","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:14:26.554833Z","iopub.execute_input":"2024-03-17T16:14:26.555228Z","iopub.status.idle":"2024-03-17T16:14:30.390381Z","shell.execute_reply.started":"2024-03-17T16:14:26.555175Z","shell.execute_reply":"2024-03-17T16:14:30.388868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:14:30.391703Z","iopub.execute_input":"2024-03-17T16:14:30.392009Z","iopub.status.idle":"2024-03-17T16:14:30.428662Z","shell.execute_reply.started":"2024-03-17T16:14:30.391986Z","shell.execute_reply":"2024-03-17T16:14:30.427171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Data Preprocessing","metadata":{}},{"cell_type":"markdown","source":"Date time of date time values should be converted to date time data type to extract meaningful data.","metadata":{}},{"cell_type":"code","source":"train['dateTime']=pd.to_datetime(train['Datum_Uhrzeit_CET'])\ntest['dateTime']=pd.to_datetime(test['Datum_Uhrzeit_CET'])\n","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:14:30.430413Z","iopub.execute_input":"2024-03-17T16:14:30.430758Z","iopub.status.idle":"2024-03-17T16:14:38.669585Z","shell.execute_reply.started":"2024-03-17T16:14:30.430728Z","shell.execute_reply":"2024-03-17T16:14:38.667768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.drop('Datum_Uhrzeit_CET',axis=1,inplace=True)\ntest.drop('Datum_Uhrzeit_CET',axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:14:38.671513Z","iopub.execute_input":"2024-03-17T16:14:38.671854Z","iopub.status.idle":"2024-03-17T16:14:39.001153Z","shell.execute_reply.started":"2024-03-17T16:14:38.671824Z","shell.execute_reply":"2024-03-17T16:14:38.999383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.info()","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:14:39.006709Z","iopub.execute_input":"2024-03-17T16:14:39.007218Z","iopub.status.idle":"2024-03-17T16:14:39.025056Z","shell.execute_reply.started":"2024-03-17T16:14:39.007160Z","shell.execute_reply":"2024-03-17T16:14:39.022932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Feature generation**","metadata":{}},{"cell_type":"markdown","source":"New data features are created from date time values.","metadata":{}},{"cell_type":"code","source":"col='dateTime'\n\ndef create_date_features(df, col):\n       \n    df[\"year\"] = df[col].dt.year.astype(np.int8)\n    df[\"month\"] = df[col].dt.month.astype(np.int8)\n#     df[\"date\"] = df[col].dt.date\n    df[\"day\"] = df[col].dt.day.astype(np.int8)\n    df[\"hour\"] = df[col].dt.hour.astype(np.int8) \n    df[\"minutes\"] = df[col].dt.minute.astype(np.int8)\n    df[\"seconds\"] = df[col].dt.second.astype(np.int8)\n    df['week']=df[col].dt.isocalendar().week.astype(np.int8)\n    df['week_day'] = df[col].dt.dayofweek.astype(np.int8)\n    df[\"is_weekend\"] = (df[col].dt.dayofweek > 5).astype(np.int8)\n    df[\"day_name\"]=df[col].dt.day_name()\n    df[\"quarter\"]=df[col].dt.quarter.astype(np.int8)\n    df[\"season\"]=(df[col].dt.month%12 // 3 + 1).astype(np.int8)\n    df['is_month_start'] = df[col].dt.is_month_start.astype(np.int8)\n    df['is_month_end'] = df[col].dt.is_month_end.astype(np.int8)\n        \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:14:39.027170Z","iopub.execute_input":"2024-03-17T16:14:39.028213Z","iopub.status.idle":"2024-03-17T16:14:39.040610Z","shell.execute_reply.started":"2024-03-17T16:14:39.028128Z","shell.execute_reply":"2024-03-17T16:14:39.038807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_date_features(train, 'dateTime')\ncreate_date_features(test, 'dateTime')","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:14:39.042145Z","iopub.execute_input":"2024-03-17T16:14:39.042580Z","iopub.status.idle":"2024-03-17T16:14:43.551989Z","shell.execute_reply.started":"2024-03-17T16:14:39.042544Z","shell.execute_reply":"2024-03-17T16:14:43.550704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:14:43.553072Z","iopub.execute_input":"2024-03-17T16:14:43.553760Z","iopub.status.idle":"2024-03-17T16:14:43.565593Z","shell.execute_reply.started":"2024-03-17T16:14:43.553728Z","shell.execute_reply":"2024-03-17T16:14:43.564475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#creating new features with date time features\ntrain[\"day_hour\"]=train[\"day_name\"].astype(str)+\"_\"+train[\"hour\"].astype(str)\ntest[\"day_hour\"]=test[\"day_name\"].astype(str)+\"_\"+test[\"hour\"].astype(str)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:14:43.567307Z","iopub.execute_input":"2024-03-17T16:14:43.567638Z","iopub.status.idle":"2024-03-17T16:14:47.172658Z","shell.execute_reply.started":"2024-03-17T16:14:43.567608Z","shell.execute_reply":"2024-03-17T16:14:47.170819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#creating trigonometric features for some of the date time features\ndate_features=[\"month\",'day','week_day','hour','year']\nfor feature in date_features:\n    train[feature+'_cos'] = np.cos(2 * np.pi * train[feature] / train[feature].max())\n    train[feature+ '_sin'] = np.sin(2 * np.pi * train[feature] / train[feature].max())\n    test[feature+ '_cos'] = np.cos(2 * np.pi * test[feature] / test[feature].max())\n    test[feature+ '_sin'] = np.sin(2 * np.pi * test[feature] / test[feature].max())","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:14:47.174939Z","iopub.execute_input":"2024-03-17T16:14:47.175316Z","iopub.status.idle":"2024-03-17T16:14:48.516377Z","shell.execute_reply.started":"2024-03-17T16:14:47.175284Z","shell.execute_reply":"2024-03-17T16:14:48.514904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.columns","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:14:48.517819Z","iopub.execute_input":"2024-03-17T16:14:48.518263Z","iopub.status.idle":"2024-03-17T16:14:48.528723Z","shell.execute_reply.started":"2024-03-17T16:14:48.518220Z","shell.execute_reply":"2024-03-17T16:14:48.526917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Create new features","metadata":{}},{"cell_type":"code","source":"\ntrain['aFRR_change']=train['aFRRactivation']-train['aFRRrequest']\ntest['aFRR_change']=test['aFRRactivation']-test['aFRRrequest']\n\ntrain['freeze_of_signal']=train['correction']-train['correctionEcho']\ntest['freeze_of_signal']=test['correction']-test['correctionEcho']\n\ntrain['change_FRCE_LFCInput']=train['FRCE']-train['LFCInput']\ntest['change_FRCE_LFCInput']=test['FRCE']-test['LFCInput']\n\ntrain['change_Demand_LFCInput']=train['Demand']-train['LFCInput']\ntest['change_Demand_LFCInput']=test['Demand']-test['LFCInput']\n\ntrain['change_BandLimitedCorrectedDemand_LFCInput']=train['BandLimitedCorrectedDemand']-train['LFCInput']\ntest['change_BandLimitedCorrectedDemand_LFCInput']=test['BandLimitedCorrectedDemand']-test['LFCInput']\n\ntrain['change_BandLimitedCorrectedDemand']=train['correctedDemand']-train['BandLimitedCorrectedDemand']\ntest['change_BandLimitedCorrectedDemand']=test['correctedDemand']-test['BandLimitedCorrectedDemand']\n\ntrain['participationCMO_IN']=train['participationIN']*train['participationCMO']\ntest['participationCMO_IN']=test['participationIN']*test['participationCMO']\n\ntrain['change_aFRRactivation_target']=train['correctedDemand']-train['aFRRactivation']\ntest['change_aFRRactivation_target']=test['correctedDemand']-test['aFRRactivation']\n\ntrain['LFCInput_FRCE']=train['LFCInput']/train['FRCE']\ntest['LFCInput_FRCE']=test['LFCInput']/test['FRCE']\n\ntrain['LFCInput_FRCE'].replace([np.inf, -np.inf], np.nan, inplace=True)\ntest['LFCInput_FRCE'].replace([np.inf, -np.inf], np.nan, inplace=True)\n\ntrain['LFCInput_FRCE']=train['LFCInput_FRCE'].fillna(0)\ntest['LFCInput_FRCE']=test['LFCInput_FRCE'].fillna(0)\n\ntrain['change_max_actual']=train['controlBandPos']-train['aFRRactivation']\ntest['change_max_actual']=test['controlBandPos']-test['aFRRactivation']\n\ntrain['change_min_actual']=train['controlBandNeg']-train['aFRRactivation']\ntest['change_min_actual']=test['controlBandNeg']-test['aFRRactivation']","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:14:48.530245Z","iopub.execute_input":"2024-03-17T16:14:48.530637Z","iopub.status.idle":"2024-03-17T16:14:49.020438Z","shell.execute_reply.started":"2024-03-17T16:14:48.530605Z","shell.execute_reply":"2024-03-17T16:14:49.018496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#reduce memory usage\ndef downcast(df):\n    for col in df.columns:\n        data_type=df[col].dtype.name\n        if data_type=='object':\n            pass\n        elif data_type=='float64':\n            df[col] = df[col].astype('float32')\n        elif data_type=='int64':\n            \n            if (col==\"controlArea\"):\n                df[col] = df[col].astype('int8')\n            elif (col=='participationCMO'):\n                df[col] = df[col].astype('int8')\n            elif (col=='participationIN'):\n                df[col] = df[col].astype('int8')\n            elif (col=='participationCMO_IN'):\n                df[col] = df[col].astype('int8')\n            else:    \n                df[col] = df[col].astype('int16')\n        elif data_type=='bool':\n            df[col] = df[col].astype('int8')\n            \ndowncast(train)\ndowncast(test)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:14:49.022180Z","iopub.execute_input":"2024-03-17T16:14:49.022713Z","iopub.status.idle":"2024-03-17T16:14:49.551037Z","shell.execute_reply.started":"2024-03-17T16:14:49.022673Z","shell.execute_reply":"2024-03-17T16:14:49.549862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_control_area_1=train[train['controlArea']==1]\n# train_control_area_2=train[train['controlArea']==2]\n\n# downsample_train_control_area_1 = train_control_area_1[['dateTime','Demand', 'correction', 'correctedDemand', 'FRCE',\n#        'controlBandPos', 'controlBandNeg', 'LFCInput', 'aFRRactivation','aFRRrequest', 'correctionEcho','BandLimitedCorrectedDemand','controlArea']].resample('30min', on='dateTime').mean().reset_index(drop=False).ffill()\n\n# downsample_train_control_area_2 = train_control_area_2[['dateTime','Demand', 'correction', 'correctedDemand', 'FRCE',\n#        'controlBandPos', 'controlBandNeg', 'LFCInput', 'aFRRactivation','aFRRrequest', 'correctionEcho','controlArea',\n#        'BandLimitedCorrectedDemand']].resample('30min', on='dateTime').mean().reset_index(drop=False).ffill()","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:14:49.552620Z","iopub.execute_input":"2024-03-17T16:14:49.553009Z","iopub.status.idle":"2024-03-17T16:14:49.560410Z","shell.execute_reply.started":"2024-03-17T16:14:49.552977Z","shell.execute_reply":"2024-03-17T16:14:49.559303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# downsample_train_control_area_1.set_index(\"dateTime\", inplace = True)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:14:49.561921Z","iopub.execute_input":"2024-03-17T16:14:49.562241Z","iopub.status.idle":"2024-03-17T16:14:49.572155Z","shell.execute_reply.started":"2024-03-17T16:14:49.562208Z","shell.execute_reply":"2024-03-17T16:14:49.570227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot_features_2=[col for col in downsample_train_control_area_2.columns if ('dateTime' not in col) &('controlArea' not in col)]\n\n# for col in plot_features_2:\n#     fig, axes = plt.subplots(1, 1, figsize=(20, 6))  # Adjusted figsize for better spacing\n#     sns.lineplot(data=downsample_train_control_area_1, x='dateTime', y=col,label='control_area_1')\n    \n#     sns.lineplot(data=downsample_train_control_area_2, x='dateTime', y=col,label='control_area_2')\n    \n#     plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:14:49.573782Z","iopub.execute_input":"2024-03-17T16:14:49.574135Z","iopub.status.idle":"2024-03-17T16:14:49.585884Z","shell.execute_reply.started":"2024-03-17T16:14:49.574083Z","shell.execute_reply":"2024-03-17T16:14:49.584777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Both train and test data frames should be merged\ndf = pd.concat([train, test]).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:14:49.586983Z","iopub.execute_input":"2024-03-17T16:14:49.587252Z","iopub.status.idle":"2024-03-17T16:14:51.883420Z","shell.execute_reply.started":"2024-03-17T16:14:49.587229Z","shell.execute_reply":"2024-03-17T16:14:51.881675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.drop(['dateTime','test_data_segment_id','id','seconds'],axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:14:51.885335Z","iopub.execute_input":"2024-03-17T16:14:51.885674Z","iopub.status.idle":"2024-03-17T16:14:52.515943Z","shell.execute_reply.started":"2024-03-17T16:14:51.885643Z","shell.execute_reply":"2024-03-17T16:14:52.514646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#generate lag and rolling mean and standard deviation features\nlags = ['Demand','correctedDemand', 'FRCE','controlBandPos', 'controlBandNeg', 'LFCInput',\n        'aFRRactivation','aFRRrequest', 'BandLimitedCorrectedDemand','aFRR_change',\n        'freeze_of_signal','change_Demand_LFCInput',\n        'change_BandLimitedCorrectedDemand_LFCInput','change_BandLimitedCorrectedDemand',\n        'LFCInput_FRCE','change_max_actual','change_min_actual','change_FRCE_LFCInput']\n\n\ndef generate_lag():\n    for lag in lags:\n        df['_lag_1_'+lag] = df[lag].shift(1)\n        \ndef generate_rolling_mean():\n    for lag in lags:\n        df['_rolling_'+lag] = df[lag].rolling(6).mean()\n        \n\n\ngenerate_lag()\ngenerate_rolling_mean()\n","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:14:52.516977Z","iopub.execute_input":"2024-03-17T16:14:52.517234Z","iopub.status.idle":"2024-03-17T16:14:56.054828Z","shell.execute_reply.started":"2024-03-17T16:14:52.517212Z","shell.execute_reply":"2024-03-17T16:14:56.053831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fill_na(df):\n    for col in df.columns:\n        if ('_lag_' in col) & (df[col].isnull().any()):\n            df[col].fillna(0, inplace=True)\n        elif ('_rolling_' in col) & (df[col].isnull().any()):\n            df[col].fillna(0, inplace=True)\n    return df\n\ndf = fill_na(df)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:14:56.055946Z","iopub.execute_input":"2024-03-17T16:14:56.056329Z","iopub.status.idle":"2024-03-17T16:14:57.994203Z","shell.execute_reply.started":"2024-03-17T16:14:56.056295Z","shell.execute_reply":"2024-03-17T16:14:57.993503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#reduce memory usage\ndowncast(df)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:14:57.995237Z","iopub.execute_input":"2024-03-17T16:14:57.996002Z","iopub.status.idle":"2024-03-17T16:14:58.266610Z","shell.execute_reply.started":"2024-03-17T16:14:57.995977Z","shell.execute_reply":"2024-03-17T16:14:58.264918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. Model Building","metadata":{}},{"cell_type":"code","source":"random_state = np.random.RandomState(42)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:14:58.267804Z","iopub.execute_input":"2024-03-17T16:14:58.268100Z","iopub.status.idle":"2024-03-17T16:14:58.274477Z","shell.execute_reply.started":"2024-03-17T16:14:58.268075Z","shell.execute_reply":"2024-03-17T16:14:58.273145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train=df.iloc[:len(train), :].copy()\nX_test=df.iloc[len(train):, :].copy()","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:14:58.276090Z","iopub.execute_input":"2024-03-17T16:14:58.276508Z","iopub.status.idle":"2024-03-17T16:15:01.157281Z","shell.execute_reply.started":"2024-03-17T16:14:58.276475Z","shell.execute_reply":"2024-03-17T16:15:01.155223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#encoding\nlabel_encoder = preprocessing.LabelEncoder()\nobj_col = X_train.select_dtypes(include = [object])\n\n#function for encoding data in object columns  \ndef encoding():\n    for feature in obj_col:\n        \n        label_encoder.fit(X_train[feature])\n        X_train[feature]=label_encoder.transform(X_train[feature])\n        X_test[feature]=label_encoder.transform(X_test[feature])\n        \n        \nencoding()","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:15:01.159624Z","iopub.execute_input":"2024-03-17T16:15:01.160072Z","iopub.status.idle":"2024-03-17T16:15:04.165629Z","shell.execute_reply.started":"2024-03-17T16:15:01.160033Z","shell.execute_reply":"2024-03-17T16:15:04.164118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#split data set\nX_train_1=X_train[X_train[\"controlArea\"]==1].copy()\nX_train_2=X_train[X_train[\"controlArea\"]==2].copy()\nX_test_1=X_test[X_test[\"controlArea\"]==1].copy()\nX_test_2=X_test[X_test[\"controlArea\"]==2].copy()","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:15:04.166782Z","iopub.execute_input":"2024-03-17T16:15:04.167060Z","iopub.status.idle":"2024-03-17T16:15:06.182113Z","shell.execute_reply.started":"2024-03-17T16:15:04.167028Z","shell.execute_reply":"2024-03-17T16:15:06.180835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* remove quasi- constant features","metadata":{}},{"cell_type":"code","source":"\nvs_constant = VarianceThreshold(threshold=0.01)\nnum=X_train_1[X_train_1.select_dtypes([np.number]).columns]\nvs_constant.fit(num)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:15:06.183909Z","iopub.execute_input":"2024-03-17T16:15:06.184312Z","iopub.status.idle":"2024-03-17T16:15:12.406078Z","shell.execute_reply.started":"2024-03-17T16:15:06.184278Z","shell.execute_reply":"2024-03-17T16:15:12.405045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"quasi=[column for column in num.columns\n                    if column not in num.columns[vs_constant.get_support()]]\n\nX_train_1.drop(quasi,axis=1,inplace=True)\nX_train_2.drop(quasi,axis=1,inplace=True)\nX_test_1.drop(quasi,axis=1,inplace=True)\nX_test_2.drop(quasi,axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:15:12.407366Z","iopub.execute_input":"2024-03-17T16:15:12.407776Z","iopub.status.idle":"2024-03-17T16:15:12.923466Z","shell.execute_reply.started":"2024-03-17T16:15:12.407752Z","shell.execute_reply":"2024-03-17T16:15:12.922417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Multi collinearity should be removed from the dataframe.","metadata":{}},{"cell_type":"code","source":"corr_matrix = X_train_1.select_dtypes(exclude=['object']).corr().abs()\n# Select upper triangle of correlation matrix\nupper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(bool))","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:15:12.924579Z","iopub.execute_input":"2024-03-17T16:15:12.924843Z","iopub.status.idle":"2024-03-17T16:15:58.628845Z","shell.execute_reply.started":"2024-03-17T16:15:12.924819Z","shell.execute_reply":"2024-03-17T16:15:58.626485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr_matrix_2 = X_train_2.select_dtypes(exclude=['object']).corr().abs()\n# Select upper triangle of correlation matrix\nupper_2 = corr_matrix_2.where(np.triu(np.ones(corr_matrix_2.shape), k=1).astype(bool))","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:15:58.631620Z","iopub.execute_input":"2024-03-17T16:15:58.632486Z","iopub.status.idle":"2024-03-17T16:16:41.940871Z","shell.execute_reply.started":"2024-03-17T16:15:58.632447Z","shell.execute_reply":"2024-03-17T16:16:41.939322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_1.drop([column for   column in upper.columns if any(upper[column] > 0.9)],axis=1,inplace=True)\nX_test_1.drop([column for   column in upper.columns if any(upper[column] > 0.9)],axis=1,inplace=True)\nX_train_2.drop([column for   column in upper_2.columns if any(upper_2[column] > 0.9)],axis=1,inplace=True)\nX_test_2.drop([column for   column in upper_2.columns if any(upper_2[column] > 0.9)],axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:16:41.942653Z","iopub.execute_input":"2024-03-17T16:16:41.943043Z","iopub.status.idle":"2024-03-17T16:16:42.512047Z","shell.execute_reply.started":"2024-03-17T16:16:41.943012Z","shell.execute_reply":"2024-03-17T16:16:42.510269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#scaling\nrscaler_1 = RobustScaler()\nrscaler_2=RobustScaler()\nX_train_Cols = X_train_1.columns\nX_train_2_Cols = X_train_2.columns\nrscaler_1.fit(X_train_1)\nrscaler_2.fit(X_train_2)\nX_train_1_scaled=pd.DataFrame(rscaler_1.transform(X_train_1), columns=X_train_Cols)\nX_train_2_scaled=pd.DataFrame(rscaler_2.transform(X_train_2), columns=X_train_2_Cols)\nX_test_1_scaled=pd.DataFrame(rscaler_1.transform(X_test_1), columns=X_train_Cols)\nX_test_2_scaled=pd.DataFrame(rscaler_2.transform(X_test_2), columns=X_train_2_Cols)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:16:42.513961Z","iopub.execute_input":"2024-03-17T16:16:42.514650Z","iopub.status.idle":"2024-03-17T16:16:54.150936Z","shell.execute_reply.started":"2024-03-17T16:16:42.514587Z","shell.execute_reply":"2024-03-17T16:16:54.150026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features_1 = list(X_train_1_scaled.columns)\nfeatures_2 = list(X_train_2_scaled.columns)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:16:54.158425Z","iopub.execute_input":"2024-03-17T16:16:54.158944Z","iopub.status.idle":"2024-03-17T16:16:54.163413Z","shell.execute_reply.started":"2024-03-17T16:16:54.158915Z","shell.execute_reply":"2024-03-17T16:16:54.162563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_1_scaled.columns","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:16:54.164617Z","iopub.execute_input":"2024-03-17T16:16:54.164859Z","iopub.status.idle":"2024-03-17T16:16:54.182346Z","shell.execute_reply.started":"2024-03-17T16:16:54.164838Z","shell.execute_reply":"2024-03-17T16:16:54.180094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_2_scaled.columns","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:16:54.184761Z","iopub.execute_input":"2024-03-17T16:16:54.185156Z","iopub.status.idle":"2024-03-17T16:16:54.199345Z","shell.execute_reply.started":"2024-03-17T16:16:54.185125Z","shell.execute_reply":"2024-03-17T16:16:54.197392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Isolation forest is used to detect anomalies in the time series based on 1 and two control areas, since its low memory requirement is compatible with a larger dataset like serving grid stability. Two isolation forest models are deployed based on the control area of the specific time step.","metadata":{}},{"cell_type":"code","source":"model_category_1=IsolationForest(n_estimators=100,max_samples='auto',contamination=0.4,random_state=random_state,n_jobs=-1)\nmodel_category_2=IsolationForest(n_estimators=100,max_samples='auto',contamination=0.4,random_state=random_state,n_jobs=-1)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:16:54.201857Z","iopub.execute_input":"2024-03-17T16:16:54.202347Z","iopub.status.idle":"2024-03-17T16:16:54.213378Z","shell.execute_reply.started":"2024-03-17T16:16:54.202311Z","shell.execute_reply":"2024-03-17T16:16:54.211284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_category_1.fit(X_train_1_scaled[features_1].values)\nmodel_category_2.fit(X_train_2_scaled[features_2].values)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:16:54.215052Z","iopub.execute_input":"2024-03-17T16:16:54.215541Z","iopub.status.idle":"2024-03-17T16:19:38.729517Z","shell.execute_reply.started":"2024-03-17T16:16:54.215513Z","shell.execute_reply":"2024-03-17T16:19:38.727829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test_1_scaled['scores'] = model_category_1.decision_function(X_test_1_scaled[features_1].values)\nX_test_2_scaled['scores'] = model_category_2.decision_function(X_test_2_scaled[features_2].values)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:19:38.732688Z","iopub.execute_input":"2024-03-17T16:19:38.733881Z","iopub.status.idle":"2024-03-17T16:19:39.944372Z","shell.execute_reply.started":"2024-03-17T16:19:38.733835Z","shell.execute_reply":"2024-03-17T16:19:39.942311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#predict anomalies\nX_test_1_scaled['anomaly_score'] = model_category_1.predict(X_test_1_scaled[features_1].values)\nX_test_2_scaled['anomaly_score'] = model_category_2.predict(X_test_2_scaled[features_2].values)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:19:39.946579Z","iopub.execute_input":"2024-03-17T16:19:39.947059Z","iopub.status.idle":"2024-03-17T16:19:41.033604Z","shell.execute_reply.started":"2024-03-17T16:19:39.947016Z","shell.execute_reply":"2024-03-17T16:19:41.031118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df= pd.concat([X_test_1_scaled, X_test_2_scaled]).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:19:41.035770Z","iopub.execute_input":"2024-03-17T16:19:41.038014Z","iopub.status.idle":"2024-03-17T16:19:41.078776Z","shell.execute_reply.started":"2024-03-17T16:19:41.037959Z","shell.execute_reply":"2024-03-17T16:19:41.075485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"extracted_col_data_segment = \"test_data_segment_id\"\ntest_df = test_df.assign(test_data_segment_id=pd.Series(test[extracted_col_data_segment]).values)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:19:41.081623Z","iopub.execute_input":"2024-03-17T16:19:41.082266Z","iopub.status.idle":"2024-03-17T16:19:41.095560Z","shell.execute_reply.started":"2024-03-17T16:19:41.082217Z","shell.execute_reply":"2024-03-17T16:19:41.092349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.groupby(['test_data_segment_id'])['anomaly_score'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:19:41.098302Z","iopub.execute_input":"2024-03-17T16:19:41.098959Z","iopub.status.idle":"2024-03-17T16:19:41.137264Z","shell.execute_reply.started":"2024-03-17T16:19:41.098905Z","shell.execute_reply":"2024-03-17T16:19:41.135946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['anomaly_score'].replace(1, 0, inplace=True)\ntest_df['anomaly_score'].replace(-1, 1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:19:41.139261Z","iopub.execute_input":"2024-03-17T16:19:41.141503Z","iopub.status.idle":"2024-03-17T16:19:41.146509Z","shell.execute_reply.started":"2024-03-17T16:19:41.141455Z","shell.execute_reply":"2024-03-17T16:19:41.145688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame(sample_submission.id, columns = ['id'])\nsubmission['anomaly'] = test_df['anomaly_score'].copy()\nsubmission\n\nsubmission.to_csv('submission.csv', index=False)\nprint(\"Your submission was successfully saved!\")","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:19:41.147994Z","iopub.execute_input":"2024-03-17T16:19:41.148431Z","iopub.status.idle":"2024-03-17T16:19:41.222496Z","shell.execute_reply.started":"2024-03-17T16:19:41.148405Z","shell.execute_reply":"2024-03-17T16:19:41.220695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission[submission['anomaly']==0]","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:19:41.224912Z","iopub.execute_input":"2024-03-17T16:19:41.225792Z","iopub.status.idle":"2024-03-17T16:19:41.243799Z","shell.execute_reply.started":"2024-03-17T16:19:41.225722Z","shell.execute_reply":"2024-03-17T16:19:41.242269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['anomaly_score']","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:19:41.245786Z","iopub.execute_input":"2024-03-17T16:19:41.246812Z","iopub.status.idle":"2024-03-17T16:19:41.259960Z","shell.execute_reply.started":"2024-03-17T16:19:41.246772Z","shell.execute_reply":"2024-03-17T16:19:41.257388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df[(test_df['test_data_segment_id']==5)&(test_df['anomaly_score']==1)]","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:19:41.262963Z","iopub.execute_input":"2024-03-17T16:19:41.263448Z","iopub.status.idle":"2024-03-17T16:19:41.299142Z","shell.execute_reply.started":"2024-03-17T16:19:41.263412Z","shell.execute_reply":"2024-03-17T16:19:41.297934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[['id','dateTime',\"controlArea\"]]","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:19:41.300107Z","iopub.execute_input":"2024-03-17T16:19:41.300417Z","iopub.status.idle":"2024-03-17T16:19:41.327416Z","shell.execute_reply.started":"2024-03-17T16:19:41.300392Z","shell.execute_reply":"2024-03-17T16:19:41.326548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(submission[submission[\"anomaly\"]==1])","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:19:41.329053Z","iopub.execute_input":"2024-03-17T16:19:41.329361Z","iopub.status.idle":"2024-03-17T16:19:41.342468Z","shell.execute_reply.started":"2024-03-17T16:19:41.329338Z","shell.execute_reply":"2024-03-17T16:19:41.340282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(submission[submission[\"anomaly\"]==0])","metadata":{"execution":{"iopub.status.busy":"2024-03-17T16:19:41.344929Z","iopub.execute_input":"2024-03-17T16:19:41.345431Z","iopub.status.idle":"2024-03-17T16:19:41.355059Z","shell.execute_reply.started":"2024-03-17T16:19:41.345394Z","shell.execute_reply":"2024-03-17T16:19:41.353498Z"},"trusted":true},"execution_count":null,"outputs":[]}]}