{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":66052,"databundleVersionId":7284431,"sourceType":"competition"}],"dockerImageVersionId":30615,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport random\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2023-12-31T14:02:17.404358Z","iopub.execute_input":"2023-12-31T14:02:17.405725Z","iopub.status.idle":"2023-12-31T14:02:17.957388Z","shell.execute_reply.started":"2023-12-31T14:02:17.405683Z","shell.execute_reply":"2023-12-31T14:02:17.956140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.random.seed(123)\n\nfrom sklearn.decomposition import PCA\nfrom sklearn.manifold import TSNE\nfrom sklearn.ensemble import IsolationForest as ISOF\nimport umap\nimport statsmodels.api as sm\n\nfrom sklearn.ensemble import GradientBoostingRegressor as GBR\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_absolute_error, r2_score, max_error\nfrom sklearn.cluster import DBSCAN\nfrom sklearn.neighbors import KernelDensity as kde\n\n# import keras as ks\n# import tensorflow as tf\n\nfrom scipy import interpolate\n\n# !pip install trimap==1.0.14\n# import trimap\n\n# !pip install pacmap==0.4\n# import pacmap\n\nimport matplotlib.pyplot as plt \nimport seaborn as sns\nimport plotly.express as px\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"7f97315e-5b74-41b9-bf9b-9d69193973c1","_cell_guid":"75b3086b-fe5e-4008-b104-a0dd893c3e4c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-31T14:02:17.960004Z","iopub.execute_input":"2023-12-31T14:02:17.960996Z","iopub.status.idle":"2023-12-31T14:03:01.499486Z","shell.execute_reply.started":"2023-12-31T14:02:17.960948Z","shell.execute_reply":"2023-12-31T14:03:01.498252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read the Data","metadata":{"_uuid":"9b88413d-d07d-4eb4-b8cb-98f3456f8cfc","_cell_guid":"2832914c-442b-406e-9cb7-26702c17c890","trusted":true}},{"cell_type":"code","source":"raw_train_data = pd.read_csv('/kaggle/input/ai-serving-grid-stability/train.csv', index_col='id', parse_dates=['Datum_Uhrzeit_CET'])\nraw_test_data = pd.read_csv('/kaggle/input/ai-serving-grid-stability/test.csv', index_col='id', parse_dates=['Datum_Uhrzeit_CET'])","metadata":{"_uuid":"4857aec2-a60a-430e-89d2-bd6fe0c31e46","_cell_guid":"0ebfde13-e1b5-4714-8337-a75ffad4a2b8","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-31T14:03:01.501249Z","iopub.execute_input":"2023-12-31T14:03:01.502059Z","iopub.status.idle":"2023-12-31T14:03:42.165752Z","shell.execute_reply.started":"2023-12-31T14:03:01.502017Z","shell.execute_reply":"2023-12-31T14:03:42.164008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_train_data.head(10)","metadata":{"_uuid":"ae9b3c42-295f-4fa9-aaed-5132ac3d919a","_cell_guid":"98fb8541-f1f7-4d73-be04-b23adaf43448","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-31T14:03:42.169586Z","iopub.execute_input":"2023-12-31T14:03:42.170140Z","iopub.status.idle":"2023-12-31T14:03:42.212067Z","shell.execute_reply.started":"2023-12-31T14:03:42.170091Z","shell.execute_reply":"2023-12-31T14:03:42.210235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_test_data.head(10)","metadata":{"_uuid":"412fab6f-6852-4fcb-85c4-c39781e09876","_cell_guid":"f0da6738-7749-49a0-854c-fff95041e2e4","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-31T14:03:42.214695Z","iopub.execute_input":"2023-12-31T14:03:42.215414Z","iopub.status.idle":"2023-12-31T14:03:42.251082Z","shell.execute_reply.started":"2023-12-31T14:03:42.215378Z","shell.execute_reply":"2023-12-31T14:03:42.249716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Explore the Data","metadata":{"_uuid":"b3b45733-69a6-4abe-adca-6202347244ed","_cell_guid":"d90af987-7d7c-432e-ab63-77c3ed4e4fb9","trusted":true}},{"cell_type":"code","source":"# The Dataset contains data from two different control areas. Make sure to separate them when handling the data ;-)\ncontrol_area_1 = raw_train_data[raw_train_data['controlArea']==1]\ncontrol_area_2 = raw_train_data[raw_train_data['controlArea']==2]\n\ncontrol_area_1[['correctedDemand', 'aFRRactivation', 'aFRRrequest']][1000:2000].plot()\ncontrol_area_2[['correctedDemand', 'aFRRactivation', 'aFRRrequest']][1000:2000].plot()","metadata":{"_uuid":"a3770f01-5aed-46aa-8b23-cc8e6aed628e","_cell_guid":"7de9bea4-020a-4743-b720-2232d0ae5201","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-31T14:03:42.252760Z","iopub.execute_input":"2023-12-31T14:03:42.253160Z","iopub.status.idle":"2023-12-31T14:03:43.844482Z","shell.execute_reply.started":"2023-12-31T14:03:42.253127Z","shell.execute_reply":"2023-12-31T14:03:43.843415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The test dataset is split into multiple data segments\nraw_test_data['test_data_segment_id'].unique()","metadata":{"_uuid":"61864e8e-35ea-4a8f-8fbd-4715e8f68639","_cell_guid":"399ddd0f-2e4b-4dc6-805a-b7da15fc6e6d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-31T14:03:43.845780Z","iopub.execute_input":"2023-12-31T14:03:43.846189Z","iopub.status.idle":"2023-12-31T14:03:43.857503Z","shell.execute_reply.started":"2023-12-31T14:03:43.846153Z","shell.execute_reply":"2023-12-31T14:03:43.855690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# All test data segments have the same length\nraw_test_data.groupby('test_data_segment_id').size()","metadata":{"_uuid":"68b159b9-de13-443f-a3d6-0518009ca053","_cell_guid":"19db0301-008d-414b-862b-38638044a489","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-31T14:03:43.858977Z","iopub.execute_input":"2023-12-31T14:03:43.859375Z","iopub.status.idle":"2023-12-31T14:03:43.879374Z","shell.execute_reply.started":"2023-12-31T14:03:43.859343Z","shell.execute_reply":"2023-12-31T14:03:43.878245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Each Data Segment belongs to one of the both control Areas\nraw_test_data.groupby('test_data_segment_id').mean()['controlArea']","metadata":{"_uuid":"838ca755-1843-479e-9ca3-ebf5668f22f4","_cell_guid":"bc6789b0-fc76-42ba-b60e-663e093cea9f","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-31T14:03:43.881648Z","iopub.execute_input":"2023-12-31T14:03:43.882115Z","iopub.status.idle":"2023-12-31T14:03:43.905996Z","shell.execute_reply.started":"2023-12-31T14:03:43.882071Z","shell.execute_reply":"2023-12-31T14:03:43.904646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Some features\n\n**Based on the work of:**\n\nhttps://www.kaggle.com/code/victorambonati/unsupervised-anomaly-detection\n\nhttps://www.kaggle.com/code/frankmollard/a-story-about-unsupervised-learning","metadata":{"_uuid":"88104ca0-d79d-4609-a839-18f145924348","_cell_guid":"91ea434d-5af7-4864-8ca2-0998a77b91be","trusted":true}},{"cell_type":"code","source":"# the hours and if it's night or day (6:00-18:00)\ndef feat_eng(df):\n    df['hour'] = df['Datum_Uhrzeit_CET'].dt.hour\n    df['daylight'] = ((df['hour'] >= 6) & (df['hour'] <= 18)).astype(int)\n    # the day of the week (Monday=0, Sunday=6) and if it's a week end day or week day.\n    df['dayofweek'] = df['Datum_Uhrzeit_CET'].dt.dayofweek\n    df['workday'] = (df['dayofweek'] < 5).astype(int)\n    return df","metadata":{"_uuid":"9ececcfd-b9f6-447e-b626-2d8aad6b1751","_cell_guid":"3c6e8a1f-d15b-4e47-8ebb-fbeb21dfe07c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-31T14:03:43.911589Z","iopub.execute_input":"2023-12-31T14:03:43.912524Z","iopub.status.idle":"2023-12-31T14:03:43.920146Z","shell.execute_reply.started":"2023-12-31T14:03:43.912482Z","shell.execute_reply":"2023-12-31T14:03:43.918367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_1 = feat_eng(control_area_1)\ntrain_2 = feat_eng(control_area_2)\ntest = feat_eng(raw_test_data)\n#\ntest_1 = test[test['controlArea']==1]\ntest_2 = test[test['controlArea']==2]","metadata":{"_uuid":"faa818d3-4c9f-4d34-9053-0fde3561c94a","_cell_guid":"0da5736d-8531-4ef8-9c32-5a6d6b615f69","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-31T14:03:43.922259Z","iopub.execute_input":"2023-12-31T14:03:43.923073Z","iopub.status.idle":"2023-12-31T14:03:45.596672Z","shell.execute_reply.started":"2023-12-31T14:03:43.923032Z","shell.execute_reply":"2023-12-31T14:03:45.595378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_1.describe()","metadata":{"_uuid":"2a1d0d0a-2516-46fc-947d-4822f334b445","_cell_guid":"a77734d1-b729-4731-ad99-a18c0de1d353","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-31T14:03:45.598403Z","iopub.execute_input":"2023-12-31T14:03:45.598802Z","iopub.status.idle":"2023-12-31T14:03:48.377496Z","shell.execute_reply.started":"2023-12-31T14:03:45.598768Z","shell.execute_reply":"2023-12-31T14:03:48.376519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_2.describe()","metadata":{"_uuid":"34705249-a5e7-43d8-b874-0ae317ed9e6f","_cell_guid":"2ddf18ae-a1e4-44c5-ba86-903dcfda46c3","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-31T14:03:48.379023Z","iopub.execute_input":"2023-12-31T14:03:48.380306Z","iopub.status.idle":"2023-12-31T14:03:50.909311Z","shell.execute_reply.started":"2023-12-31T14:03:48.380265Z","shell.execute_reply":"2023-12-31T14:03:50.908164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_1.describe()","metadata":{"_uuid":"6c5e12a2-173c-4804-83ab-d0fdbda716f3","_cell_guid":"236f3eff-f086-4152-b557-c70cf96de109","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-31T14:03:50.910706Z","iopub.execute_input":"2023-12-31T14:03:50.911086Z","iopub.status.idle":"2023-12-31T14:03:51.003115Z","shell.execute_reply.started":"2023-12-31T14:03:50.911047Z","shell.execute_reply":"2023-12-31T14:03:51.001139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PCA\n\nThe stratagy is to compute the PCA compoments and calculate the std dev of components and call outlier all observations that are higher/lower that K*std dev","metadata":{"_uuid":"42d9b217-3bb4-4628-8968-4965098a5351","_cell_guid":"84391e03-4dea-4984-a13a-381d73afb033","trusted":true}},{"cell_type":"code","source":"def barPlot(y, threshold):\n    fig, axs = plt.subplots(1,2, figsize=(20,6))\n    #plt.figure(figsize=(12,9))\n    heights1=y\n    heights2=y.cumsum()\n    colors=heights2 >= threshold\n    \n    axs[0].bar(height=heights1, x=range(1,len(y)+1), width=0.8, color=[\"black\" if x == True else \"maroon\" for x in colors])\n    axs[0].set_xticks(ticks=range(1,len(y)+1))\n    axs[0].set_yticks(ticks=np.array(list(range(0,11)))/10)\n    axs[0].set_title(\"Variance by Component\", fontsize = 16)\n    \n    axs[1].bar(height=heights2, x=range(1,len(y)+1), width=0.8, color=[\"black\" if x == True else \"maroon\" for x in colors])\n    axs[1].axhline(y=threshold, xmin=0, xmax=3, c=\"black\")\n    axs[1].set_xticks(ticks=range(1,len(y)+1))\n    axs[1].set_yticks(ticks=np.array(list(range(0,11)))/10)\n    axs[1].set_title(\n        \"Cumul. Variance of \" + str(sum(colors == False)+1) + \" Components >= \" + str(int(threshold*100)) + \" %\", \n        fontsize=14\n    )\n    plt.show()","metadata":{"_uuid":"6b45cc6d-dadf-4eae-99a0-320eaa619515","_cell_guid":"ca99abd9-933c-40bd-b645-a1428ad6abf7","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-31T14:03:51.004805Z","iopub.execute_input":"2023-12-31T14:03:51.005212Z","iopub.status.idle":"2023-12-31T14:03:51.018212Z","shell.execute_reply.started":"2023-12-31T14:03:51.005178Z","shell.execute_reply":"2023-12-31T14:03:51.016578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train_1.drop(columns=[\"Datum_Uhrzeit_CET\", \"controlArea\"])\npca = PCA()\npca.fit(X)","metadata":{"_uuid":"d8a991eb-d4aa-401e-94cd-1e5c80e3fb7d","_cell_guid":"4e95cb3a-c3c1-4afb-b227-b2d1802464a9","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-31T14:03:51.019913Z","iopub.execute_input":"2023-12-31T14:03:51.020356Z","iopub.status.idle":"2023-12-31T14:03:55.236331Z","shell.execute_reply.started":"2023-12-31T14:03:51.020316Z","shell.execute_reply":"2023-12-31T14:03:55.235156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"exp_var_cumul = np.cumsum(pca.explained_variance_ratio_)\n\npx.area(\n    x=range(1, exp_var_cumul.shape[0] + 1),\n    y=exp_var_cumul,\n    labels={\"x\": \"# Components\", \"y\": \"Explained Variance\"}\n)","metadata":{"_uuid":"a801a081-0efc-4849-87c5-0805a060407c","_cell_guid":"58ff7d77-1ff9-4638-9793-c8019a40fe08","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-31T14:03:55.238150Z","iopub.execute_input":"2023-12-31T14:03:55.238878Z","iopub.status.idle":"2023-12-31T14:03:57.289075Z","shell.execute_reply.started":"2023-12-31T14:03:55.238835Z","shell.execute_reply":"2023-12-31T14:03:57.287771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"barPlot(y=pca.explained_variance_ratio_, threshold=0.9)","metadata":{"_uuid":"028caa48-f308-4dad-a9cb-859bbf70ffe0","_cell_guid":"2b80edd8-0c40-4a35-96dd-1013d2888b5c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-31T14:03:57.290669Z","iopub.execute_input":"2023-12-31T14:03:57.291076Z","iopub.status.idle":"2023-12-31T14:03:58.059061Z","shell.execute_reply.started":"2023-12-31T14:03:57.291037Z","shell.execute_reply":"2023-12-31T14:03:58.057775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot PCA for train_1 with 2 components\n# Control Area 1\nX = train_1.drop(columns=[\"Datum_Uhrzeit_CET\", \"controlArea\"])\npca1 = PCA(n_components=2).fit(X)\ncomponents = pca1.transform(X)\ntrain_pca = pd.DataFrame(components)\n\n# fig = px.scatter(components, x=0, y=1, color=X['daylight'])\n# fig.show()","metadata":{"_uuid":"433b4736-57c2-480e-9eb8-f4e5bf588d34","_cell_guid":"5266bc6d-09bf-4fed-89d3-845700c525eb","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-31T14:03:58.060565Z","iopub.execute_input":"2023-12-31T14:03:58.061112Z","iopub.status.idle":"2023-12-31T14:04:12.780366Z","shell.execute_reply.started":"2023-12-31T14:03:58.061077Z","shell.execute_reply":"2023-12-31T14:04:12.776675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_pca.describe()","metadata":{"_uuid":"192e6bcb-bf4f-4272-8cad-0f1e457b3c2a","_cell_guid":"ab8e3fcf-7959-4107-92ab-f45c1a2f6046","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-31T14:04:12.784319Z","iopub.execute_input":"2023-12-31T14:04:12.786788Z","iopub.status.idle":"2023-12-31T14:04:13.215993Z","shell.execute_reply.started":"2023-12-31T14:04:12.786722Z","shell.execute_reply":"2023-12-31T14:04:13.214092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"std_pca_comp1_ca1 = train_pca.iloc[:,0].std()\nstd_pca_comp2_ca1 = train_pca.iloc[:,1].std()\nprint('STD comp 1 and 2: ',std_pca_comp1_ca1, '  ',std_pca_comp2_ca1)","metadata":{"_uuid":"24d47064-c3e0-435e-b7e6-124830dc01bf","_cell_guid":"da0e07a8-3486-472e-90cf-dd22559b3160","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-31T14:04:13.218211Z","iopub.execute_input":"2023-12-31T14:04:13.218765Z","iopub.status.idle":"2023-12-31T14:04:13.329552Z","shell.execute_reply.started":"2023-12-31T14:04:13.218714Z","shell.execute_reply":"2023-12-31T14:04:13.327976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Control Area 2\nX = train_2.drop(columns=[\"Datum_Uhrzeit_CET\", \"controlArea\"])\npca2 = PCA(n_components=2).fit(X)\ncomponents = pca2.transform(X)\ntrain_pca = pd.DataFrame(components)\nstd_pca_comp1_ca2 = train_pca.iloc[:,0].std()\nstd_pca_comp2_ca2 = train_pca.iloc[:,1].std()\nprint('STD comp 1 and 2: ',std_pca_comp1_ca2, '  ',std_pca_comp2_ca2)","metadata":{"_uuid":"6cc5f23a-c39f-4bdd-b620-194caee0689d","_cell_guid":"3cac5db3-a823-4239-bbf6-37dfa71b8235","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-31T14:04:13.331722Z","iopub.execute_input":"2023-12-31T14:04:13.332728Z","iopub.status.idle":"2023-12-31T14:04:26.965519Z","shell.execute_reply.started":"2023-12-31T14:04:13.332685Z","shell.execute_reply":"2023-12-31T14:04:26.964192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# PCA for test data\n# pca = PCA(n_components=train_1.shape[1]-2, random_state=1)\n# pca.fit(train_1.drop(columns=[\"Datum_Uhrzeit_CET\", \"controlArea\"]))\n# transformed_1 = pca.transform(test_1.drop(columns=[\"Datum_Uhrzeit_CET\", \"controlArea\", \"test_data_segment_id\"]))\n# reduced_pca_1 = pd.DataFrame(transformed_1)\n# transformed2 = pca.transform(test2.drop(columns=[\"timestamp\"]))\n# Reduced_PCA2 = pd.DataFrame(transformed2)","metadata":{"_uuid":"155be6a1-424b-49b9-8fc0-fc98a2e4c9ef","_cell_guid":"58fdbdf5-28e5-4134-8cc7-fd21d8989ad5","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-31T14:04:26.967354Z","iopub.execute_input":"2023-12-31T14:04:26.967773Z","iopub.status.idle":"2023-12-31T14:04:26.975866Z","shell.execute_reply.started":"2023-12-31T14:04:26.967738Z","shell.execute_reply":"2023-12-31T14:04:26.972443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create a Model to Detect Anomalies","metadata":{"_uuid":"abf6317d-b136-42bf-aaf9-7b77f779624a","_cell_guid":"06b47bdb-b0bf-4606-b2a0-1807a5cf36aa","trusted":true}},{"cell_type":"code","source":"class RandomModel():\n    \"\"\"\n    A random model just to showcase the basic handling of data\n    \"\"\"\n    def train(self, train_data):\n        print(\"training the model...\")\n    \n    def predict(self, test_data):\n        # Make random predictions just to showcase how a model could look like\n        return [random.choice([0, 1]) for _ in range(len(test_data))]\n    \n    def predict_1(self, test_data):\n        # predict all equal to 1\n        return [1 for _ in range(len(test_data))]\n    \n# model = RandomModel()\n# model.train(raw_train_data)","metadata":{"_uuid":"fad22117-da09-4674-be90-9ec6c89b9e51","_cell_guid":"06c61c4c-4b9f-4723-8f32-592e1569b801","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-31T14:04:26.978491Z","iopub.execute_input":"2023-12-31T14:04:26.979314Z","iopub.status.idle":"2023-12-31T14:04:26.991527Z","shell.execute_reply.started":"2023-12-31T14:04:26.979277Z","shell.execute_reply":"2023-12-31T14:04:26.990149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Compute PCA compoments 2 dimensions, compute STD DEV and use K * STDDEV and flag the outliers.\ndef pca_model(df_test, pca, std_pca_comp1_ca, std_pca_comp2_ca, K):\n    # PCA for train data\n    transformed = pca.transform(df_test.drop(columns=[\"Datum_Uhrzeit_CET\", \"controlArea\", \"test_data_segment_id\"]))\n    reduced_pca = pd.DataFrame(transformed)\n    reduced_pca.columns = ['comp1', 'comp2']\n    #\n    reduced_pca['outlier_comp1'] = reduced_pca['outlier_comp2'] = 0\n    reduced_pca.loc[reduced_pca[\"comp1\"] < - K*std_pca_comp1_ca, 'outlier_comp1'] = 1\n    reduced_pca.loc[reduced_pca[\"comp1\"] > K*std_pca_comp1_ca, 'outlier_comp1'] = 1\n    #\n    reduced_pca.loc[reduced_pca[\"comp2\"] < - K*std_pca_comp2_ca, 'outlier_comp2'] = 1\n    reduced_pca.loc[reduced_pca[\"comp2\"] > K*std_pca_comp2_ca, 'outlier_comp2'] = 1\n    #\n    reduced_pca['outlier'] = np.where(reduced_pca['outlier_comp1'] + reduced_pca['outlier_comp2'] == 0,0,1)\n    \n    return reduced_pca\n","metadata":{"_uuid":"7b5388ad-ee4b-4cbb-9e90-9a9b4d9952d4","_cell_guid":"d4a7b7f2-5a80-4f5f-aa18-008a8743532e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-31T14:04:26.993209Z","iopub.execute_input":"2023-12-31T14:04:26.993684Z","iopub.status.idle":"2023-12-31T14:04:27.007869Z","shell.execute_reply.started":"2023-12-31T14:04:26.993623Z","shell.execute_reply":"2023-12-31T14:04:27.006304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Detect Anomalies in Test Data Using the Radom Model","metadata":{"_uuid":"ff8aaa87-0ef5-4eb5-9b50-9b247f9898db","_cell_guid":"967792f0-573d-458c-8d14-5702cacce553","trusted":true}},{"cell_type":"code","source":"# Submission?\nSUB = True","metadata":{"_uuid":"eff137d0-fd89-46c5-88c9-6812338ca733","_cell_guid":"d19bfe2d-757c-42eb-852f-6a8b8ac35da4","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-31T14:04:27.009930Z","iopub.execute_input":"2023-12-31T14:04:27.010456Z","iopub.status.idle":"2023-12-31T14:04:27.026033Z","shell.execute_reply.started":"2023-12-31T14:04:27.010410Z","shell.execute_reply":"2023-12-31T14:04:27.024467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# K = .1 #Score 0.48\nK = .5 # Std deviations # Score 0.5235\n# K = 1 # Score  0.516\n# K = 2 # Score 0.2424\nif SUB:\n    dataset_ids = raw_test_data['test_data_segment_id'].unique()\n\n    submission = np.array([])\n\n    for dataset_id in dataset_ids:\n        sample = raw_test_data[raw_test_data['test_data_segment_id'] == dataset_id]\n        if np.any(sample.controlArea == 1):  \n            reduced_pca_1 = pca_model(sample,pca1, std_pca_comp1_ca1, std_pca_comp2_ca1, K)\n            # classifications = model.predict_1(sample)\n            submission = np.append(submission, reduced_pca_1['outlier'])\n        else: \n            reduced_pca_2 = pca_model(sample,pca2,std_pca_comp1_ca2,std_pca_comp2_ca2,K)\n            submission = np.append(submission, reduced_pca_2['outlier'])\n            \n    submission = pd.DataFrame({'anomaly': submission})\n    submission.index.name = 'id'\n    # save as submission.csv in order to submit via this kaggle notebook\n    print(submission['anomaly'].sum()/submission.shape[0])\n    submission.to_csv('submission.csv')","metadata":{"_uuid":"fe8f83ae-d8c0-4773-bff4-5c737529e0b3","_cell_guid":"6fa727fd-5acf-430c-86b8-66f676039063","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-12-31T14:04:27.028074Z","iopub.execute_input":"2023-12-31T14:04:27.028649Z","iopub.status.idle":"2023-12-31T14:04:27.343346Z","shell.execute_reply.started":"2023-12-31T14:04:27.028596Z","shell.execute_reply":"2023-12-31T14:04:27.341661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Does it make sense? Not sure","metadata":{"_uuid":"163bc521-0754-43fd-b901-4e72533fead8","_cell_guid":"d2b346e0-17dc-4fd3-8012-a942ba3cc339","trusted":true}}]}