{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.12"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":93783,"databundleVersionId":11331308,"sourceType":"competition"},{"sourceId":280335,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":240168,"modelId":261813}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":15.182031,"end_time":"2025-03-10T09:43:17.608114","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-03-10T09:43:02.426083","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"1fadda30","cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2025-03-10T09:43:05.460385Z","iopub.status.busy":"2025-03-10T09:43:05.459961Z","iopub.status.idle":"2025-03-10T09:43:06.423218Z","shell.execute_reply":"2025-03-10T09:43:06.421767Z"},"papermill":{"duration":0.970251,"end_time":"2025-03-10T09:43:06.425387","exception":false,"start_time":"2025-03-10T09:43:05.455136","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"dbc71716","cell_type":"code","source":"# Load test set and its reconstruction from Telemanom-ESA\nchannel_names = [f\"channel_{x}\" for x in range(41,46 + 1)]\ntest_set = pd.read_parquet(\"/kaggle/input/esa-adb-challenge/test.parquet\", columns=[\"id\"] + channel_names)\nreconstruction = pd.read_csv(\"/kaggle/input/telemanom-esa-baseline-channels41-46/tensorflow2/default/1/Telemanom-ESA-baseline-channels41-46/Telemanom-ESA/1b686e08c798457aa30dcfb02bd86868/ESA-Mission1/kaggle_test/1/model.h5.reconstruction.csv\")","metadata":{"execution":{"iopub.execute_input":"2025-03-10T09:43:06.433976Z","iopub.status.busy":"2025-03-10T09:43:06.433446Z","iopub.status.idle":"2025-03-10T09:43:08.767407Z","shell.execute_reply":"2025-03-10T09:43:08.766505Z"},"papermill":{"duration":2.340368,"end_time":"2025-03-10T09:43:08.769572","exception":false,"start_time":"2025-03-10T09:43:06.429204","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"56381db8","cell_type":"code","source":"# Compare test set and reconstruction for channel 41\nchannel_name = \"channel_41\"\nstart_idx, end_idx = 500, 1500\npd.DataFrame({channel_name: test_set.loc[start_idx:end_idx, channel_name], \n              \"reconstruction\": reconstruction.loc[start_idx:end_idx, channel_name]}).plot()","metadata":{"execution":{"iopub.execute_input":"2025-03-10T09:43:08.776909Z","iopub.status.busy":"2025-03-10T09:43:08.776543Z","iopub.status.idle":"2025-03-10T09:43:09.239572Z","shell.execute_reply":"2025-03-10T09:43:09.238331Z"},"papermill":{"duration":0.469332,"end_time":"2025-03-10T09:43:09.242078","exception":false,"start_time":"2025-03-10T09:43:08.772746","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"61f690bf","cell_type":"code","source":"# Load raw anomaly scores from Telemanom-ESA-channels41-46 (absolute differences between channel reconstructions and test data)\nraw_scores = pd.read_csv(\"/kaggle/input/telemanom-esa-baseline-channels41-46/tensorflow2/default/1/Telemanom-ESA-baseline-channels41-46/Telemanom-ESA/1b686e08c798457aa30dcfb02bd86868/ESA-Mission1/kaggle_test/1/docker-algorithm-scores.csv\", header=None)\nraw_scores","metadata":{"execution":{"iopub.execute_input":"2025-03-10T09:43:09.252978Z","iopub.status.busy":"2025-03-10T09:43:09.252589Z","iopub.status.idle":"2025-03-10T09:43:11.184116Z","shell.execute_reply":"2025-03-10T09:43:11.183027Z"},"papermill":{"duration":1.938975,"end_time":"2025-03-10T09:43:11.185984","exception":false,"start_time":"2025-03-10T09:43:09.247009","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"8ca1b7cc","cell_type":"code","source":"# Plot max-aggregated anomaly scores across channels\naggregated_raw_scores = raw_scores.max(axis=1)\naggregated_raw_scores.plot()","metadata":{"execution":{"iopub.execute_input":"2025-03-10T09:43:11.198191Z","iopub.status.busy":"2025-03-10T09:43:11.197773Z","iopub.status.idle":"2025-03-10T09:43:11.620909Z","shell.execute_reply":"2025-03-10T09:43:11.619721Z"},"papermill":{"duration":0.431058,"end_time":"2025-03-10T09:43:11.622553","exception":false,"start_time":"2025-03-10T09:43:11.191495","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"2d21a2c5","cell_type":"code","source":"# Load binary anomaly scores after Telemanom thresholding without pruning\nbinary_scores = pd.read_csv(\"/kaggle/input/telemanom-esa-baseline-channels41-46/tensorflow2/default/1/Telemanom-ESA-baseline-channels41-46/Telemanom-ESA/1b686e08c798457aa30dcfb02bd86868/ESA-Mission1/kaggle_test/1/anomaly_scores_nonPruned.ts\", header=None)\nbinary_scores","metadata":{"execution":{"iopub.execute_input":"2025-03-10T09:43:11.635613Z","iopub.status.busy":"2025-03-10T09:43:11.635259Z","iopub.status.idle":"2025-03-10T09:43:13.215840Z","shell.execute_reply":"2025-03-10T09:43:13.214754Z"},"papermill":{"duration":1.589112,"end_time":"2025-03-10T09:43:13.217522","exception":false,"start_time":"2025-03-10T09:43:11.628410","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"0ad21591","cell_type":"code","source":"# Plot max-aggregated anomaly scores across channels\naggregated_binary_scores = binary_scores.max(axis=1)\naggregated_binary_scores.plot()","metadata":{"execution":{"iopub.execute_input":"2025-03-10T09:43:13.230967Z","iopub.status.busy":"2025-03-10T09:43:13.230538Z","iopub.status.idle":"2025-03-10T09:43:13.570304Z","shell.execute_reply":"2025-03-10T09:43:13.568946Z"},"papermill":{"duration":0.348574,"end_time":"2025-03-10T09:43:13.572129","exception":false,"start_time":"2025-03-10T09:43:13.223555","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"3eca4a43","cell_type":"code","source":"# Save to submission file\nsubmission_df = pd.DataFrame()\nsubmission_df[\"id\"] = test_set[\"id\"]\nsubmission_df[\"is_anomaly\"] = aggregated_binary_scores\nsubmission_df.to_parquet(f\"/kaggle/working/Telemanom-ESA-nonPruned-channels41-46.parquet\")\n# Generate a file supported by the Kaggle \"Submit to competition\" option\nsubmission_df.to_csv(f\"/kaggle/working/submission.csv\", index=False)","metadata":{"execution":{"iopub.execute_input":"2025-03-10T09:43:13.586125Z","iopub.status.busy":"2025-03-10T09:43:13.585734Z","iopub.status.idle":"2025-03-10T09:43:14.252590Z","shell.execute_reply":"2025-03-10T09:43:14.251360Z"},"papermill":{"duration":0.676266,"end_time":"2025-03-10T09:43:14.254764","exception":false,"start_time":"2025-03-10T09:43:13.578498","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"id":"cdd4edaa","cell_type":"code","source":"# Load and plot binary anomaly scores after Telemanom thresholding with pruning\npruned_binary_scores = pd.read_csv(\"/kaggle/input/telemanom-esa-baseline-channels41-46/tensorflow2/default/1/Telemanom-ESA-baseline-channels41-46/Telemanom-ESA/1b686e08c798457aa30dcfb02bd86868/ESA-Mission1/kaggle_test/1/anomaly_scores_Pruned.ts\", header=None)\naggregated_pruned_binary_scores = pruned_binary_scores.max(axis=1)\nsubmission_df[\"is_anomaly\"] = aggregated_pruned_binary_scores\nsubmission_df.to_parquet(f\"/kaggle/working/Telemanom-ESA-Pruned-channels41-46.parquet\")\nsubmission_df.plot(\"id\", \"is_anomaly\")","metadata":{"execution":{"iopub.execute_input":"2025-03-10T09:43:14.269560Z","iopub.status.busy":"2025-03-10T09:43:14.269227Z","iopub.status.idle":"2025-03-10T09:43:16.976602Z","shell.execute_reply":"2025-03-10T09:43:16.975181Z"},"papermill":{"duration":2.717178,"end_time":"2025-03-10T09:43:16.978757","exception":false,"start_time":"2025-03-10T09:43:14.261579","status":"completed"},"tags":[]},"outputs":[],"execution_count":null}]}