{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":93783,"databundleVersionId":11331308,"sourceType":"competition"},{"sourceId":280335,"sourceType":"modelInstanceVersion","modelInstanceId":240168,"modelId":261813}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Import and read data","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom tensorflow.keras.models import load_model\nfrom tensorflow.keras.layers import LSTM\n\n# Read text data only of the best channels\nbest_channels = [\"channel_41\",\"channel_42\",\"channel_43\",\"channel_44\",\"channel_45\",\"channel_46\"]\ntest_df = pd.read_parquet(\"/kaggle/input/esa-adb-challenge/test.parquet\")\nX_test_input = test_df[best_channels]        # Original input\nX_test_input = np.expand_dims(X_test_input, axis=1) # New shape: (32, 1, 6)","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2025-08-06T17:59:02.750530Z","iopub.execute_input":"2025-08-06T17:59:02.751063Z","iopub.status.idle":"2025-08-06T17:59:03.036773Z","shell.execute_reply.started":"2025-08-06T17:59:02.751030Z","shell.execute_reply":"2025-08-06T17:59:03.035781Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Using the kaggle trained autoencoder LSTM to detect anomalies with the difference between original data and reconstructed.","metadata":{}},{"cell_type":"code","source":"# Load the model\nclass CompatibleLSTM(LSTM):\n    def __init__(self, *args, **kwargs):\n        kwargs.pop('time_major', None)  # Remove unsupported argument\n        super().__init__(*args, **kwargs)\n\nmodel = load_model('/kaggle/input/telemanom-esa-baseline-channels41-46/tensorflow2/default/1/Telemanom-ESA-baseline-channels41-46/Telemanom-ESA/1b686e08c798457aa30dcfb02bd86868/ESA-Mission1/kaggle_test/1/model.h5', custom_objects={'LSTM': CompatibleLSTM},compile=False)","metadata":{"execution":{"iopub.status.busy":"2025-08-06T17:59:03.038348Z","iopub.execute_input":"2025-08-06T17:59:03.038732Z","iopub.status.idle":"2025-08-06T17:59:03.147015Z","shell.execute_reply.started":"2025-08-06T17:59:03.038705Z","shell.execute_reply":"2025-08-06T17:59:03.146159Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Make the dinamic outliers and detect anomalies","metadata":{}},{"cell_type":"code","source":"# Get the reconstructed error\nreconstructed  = model.predict(X_test_input)\nreconstruction_errors = np.mean(np.square(X_test_input - reconstructed), axis=(1, 2))\n\n# Convert to pandas Series for easier slicing\nreconstruction_errors = pd.Series(reconstruction_errors)\n\n# Define adaptive IQR parameters\nwindow_size = 7000\niqr_scale_upper = 3\niqr_scale_lower = 4\n\n# Compute rolling quantiles\nq1 = reconstruction_errors.rolling(window=window_size, center=True).quantile(0.25)\nq3 = reconstruction_errors.rolling(window=window_size, center=True).quantile(0.75)\niqr = q3 - q1\nlower_bound = q1 - (iqr_scale_lower * iqr)\nupper_bound = q3 + (iqr_scale_upper * iqr)\n\n# Finally detect anomalies as outliers\nanomalies  = (reconstruction_errors > upper_bound) | (reconstruction_errors < lower_bound)\n\n# Example arrays\nids = test_df['id']\n\n# Create DataFrame\ndf = pd.DataFrame({\n    'id': ids,\n    'is_anomaly': anomalies\n})\ndf.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-06T17:59:03.147745Z","iopub.execute_input":"2025-08-06T17:59:03.147996Z","iopub.status.idle":"2025-08-06T17:59:37.485795Z","shell.execute_reply.started":"2025-08-06T17:59:03.147975Z","shell.execute_reply":"2025-08-06T17:59:37.484952Z"}},"outputs":[],"execution_count":null}]}