{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":93783,"databundleVersionId":11331308,"sourceType":"competition"},{"sourceId":8914496,"sourceType":"datasetVersion","datasetId":5360751}],"dockerImageVersionId":31259,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nfrom IPython.display import display, HTML\nfrom pathlib import Path\n\nBASE_PATH = Path(\"/kaggle/input/esa-anomaly-dataset\")\nmissions = [\"ESA-Mission1\", \"ESA-Mission2\", \"ESA-Mission3\"]\nmeta_list = []\n\nfor mission in missions:\n    file_path = BASE_PATH / mission / mission / \"labels.csv\"\n    df = pd.read_csv(file_path)\n\n    meta = {\n        \"Mission\": mission,\n        \"Rows\": df.shape[0],\n        \"Columns\": df.shape[1],\n        \"Memory (MB)\": round(df.memory_usage(deep=True).sum() / 1024**2, 2),\n        \"Unique ID\": df[\"ID\"].nunique(),\n        \"Unique Channel\": df[\"Channel\"].nunique(),\n        \"Unique StartTime\": df[\"StartTime\"].nunique(),\n        \"Unique EndTime\": df[\"EndTime\"].nunique(),\n        \"Missing % (Channel)\": round(df[\"Channel\"].isna().mean() * 100, 2),\n        \"Duplicates\": df.duplicated().sum()\n    }\n    meta_list.append(meta)\n\nmeta_df = pd.DataFrame(meta_list)\n\ndef color_missing(val):\n    return \"background-color: #ffcccc\" if val > 0 else \"background-color: #ccffcc\"\n\nstyled = (\n    meta_df.style\n    .map(color_missing, subset=[\"Missing % (Channel)\"])  # using map instead of applymap\n    .set_properties(**{\"text-align\": \"center\"})\n    .set_table_styles([\n        {\"selector\": \"th\", \"props\": [(\"background-color\", \"#1f77b4\"),\n                                     (\"color\", \"white\"),\n                                     (\"font-weight\", \"bold\")]}\n    ])\n)\n\ndisplay(HTML(\"<h3> ESA Missions Meta Review</h3>\"))\ndisplay(styled)\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-19T08:07:53.503213Z","iopub.execute_input":"2026-01-19T08:07:53.503613Z","iopub.status.idle":"2026-01-19T08:07:53.761936Z","shell.execute_reply.started":"2026-01-19T08:07:53.503581Z","shell.execute_reply":"2026-01-19T08:07:53.760907Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nfrom IPython.display import display, HTML\n\nmissions = [\"ESA-Mission1\", \"ESA-Mission2\", \"ESA-Mission3\"]\ntime_list = []\n\nfor mission in missions:\n    file_path = BASE_PATH / mission / mission / \"labels.csv\"\n    df = pd.read_csv(file_path)\n    for col in [\"StartTime\", \"EndTime\"]:\n        df[col] = pd.to_datetime(df[col], errors=\"coerce\", utc=True)\n    df[\"Duration_sec\"] = (df[\"EndTime\"] - df[\"StartTime\"]).dt.total_seconds()\n    meta = {\n        \"Mission\": mission,\n        \"Start\": df[\"StartTime\"].min(),\n        \"End\": df[\"EndTime\"].max(),\n        \"Coverage (days)\": (df[\"EndTime\"].max() - df[\"StartTime\"].min()).days,\n        \"Avg Event Duration (sec)\": round(df[\"Duration_sec\"].mean(), 2),\n        \"Median Event Duration (sec)\": round(df[\"Duration_sec\"].median(), 2),\n        \"Shortest (sec)\": df[\"Duration_sec\"].min(),\n        \"Longest (sec)\": df[\"Duration_sec\"].max()\n    }\n    time_list.append(meta)\n\ntime_df = pd.DataFrame(time_list)\n\nstyled = (\n    time_df.style\n    .set_properties(**{\"text-align\": \"center\"})\n    .set_table_styles([\n        {\"selector\": \"th\", \"props\": [(\"background-color\", \"#2ca02c\"),\n                                     (\"color\", \"white\"),\n                                     (\"font-weight\", \"bold\")]}\n    ])\n)\n\ndisplay(HTML(\"<h3>ESA Missions Time Analysis</h3>\"))\ndisplay(styled)\nplt.figure(figsize=(14, 6))\n\nfor mission in missions:\n    file_path = BASE_PATH / mission / mission / \"labels.csv\"\n    df = pd.read_csv(file_path)\n    df[\"StartTime\"] = pd.to_datetime(df[\"StartTime\"], errors=\"coerce\", utc=True)\n\n    # Using 'ME' instead of 'M' → month-end based grouping\n    events_by_month = df.resample(\"ME\", on=\"StartTime\").size()\n    plt.plot(events_by_month.index, events_by_month.values, label=mission)\n\nplt.title(\"ESA Missions – Monthly Event Count Comparison\")\nplt.xlabel(\"Time\")\nplt.ylabel(\"Number of Events\")\nplt.legend()\nplt.grid(True)\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T08:10:34.304769Z","iopub.execute_input":"2026-01-19T08:10:34.305777Z","iopub.status.idle":"2026-01-19T08:10:34.760240Z","shell.execute_reply.started":"2026-01-19T08:10:34.305732Z","shell.execute_reply":"2026-01-19T08:10:34.759059Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom IPython.display import display, HTML\n\nmissions = [\"ESA-Mission1\", \"ESA-Mission2\", \"ESA-Mission3\"]\nchannel_stats = []\n\nfor mission in missions:\n    file_path = BASE_PATH / mission / mission / \"labels.csv\"\n    df = pd.read_csv(file_path)\n\n    # Time conversion and duration calculation\n    df[\"StartTime\"] = pd.to_datetime(df[\"StartTime\"], errors=\"coerce\", utc=True)\n    df[\"EndTime\"] = pd.to_datetime(df[\"EndTime\"], errors=\"coerce\", utc=True)\n    df[\"Duration_sec\"] = (df[\"EndTime\"] - df[\"StartTime\"]).dt.total_seconds()\n\n    channel_counts = df[\"Channel\"].value_counts()\n    channel_durations = (\n        df.groupby(\"Channel\")[\"Duration_sec\"]\n        .mean()\n        .sort_values(ascending=False)\n    )\n\n    channel_stats.append({\n        \"Mission\": mission,\n        \"Number of Channels\": df[\"Channel\"].nunique(),\n        \"Most Active Channel\": channel_counts.index[0],\n        \"Event Count (Top Channel)\": int(channel_counts.iloc[0]),\n        \"Average Duration (sec)\": round(df[\"Duration_sec\"].mean(), 2)\n    })\n\n    # Visualization – Top 10 channels with most events\n    plt.figure(figsize=(12, 4))\n    channel_counts.head(10).plot(kind=\"bar\", color=\"steelblue\")\n    plt.title(f\"{mission} – Top 10 Channels by Event Count\")\n    plt.xlabel(\"Channel\")\n    plt.ylabel(\"Number of Events\")\n    plt.xticks(rotation=45)\n    plt.tight_layout()\n    plt.show()\n\n    top_channels = channel_counts.head(5).index\n    plt.figure(figsize=(10, 5))\n    df[df[\"Channel\"].isin(top_channels)].boxplot(\n        column=\"Duration_sec\", by=\"Channel\", grid=False\n    )\n    plt.title(f\"{mission} – Duration Distributions for Top 5 Channels\")\n    plt.suptitle(\"\")  # remove automatic suptitle\n    plt.xlabel(\"Channel\")\n    plt.ylabel(\"Duration (sec)\")\n    plt.yscale(\"log\")  # log scale to visualize long tails\n    plt.tight_layout()\n    plt.show()\n\nchannel_df = pd.DataFrame(channel_stats)\n\nstyled = (\n    channel_df.style\n    .set_properties(**{\"text-align\": \"center\"})\n    .set_table_styles([\n        {\"selector\": \"th\", \"props\": [(\"background-color\", \"#ff7f0e\"),\n                                     (\"color\", \"white\"),\n                                     (\"font-weight\", \"bold\")]}\n    ])\n)\n\ndisplay(HTML(\"<h3>📡 ESA Missions Channel Analysis</h3>\"))\ndisplay(styled)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T08:12:13.883470Z","iopub.execute_input":"2026-01-19T08:12:13.884758Z","iopub.status.idle":"2026-01-19T08:12:16.041249Z","shell.execute_reply.started":"2026-01-19T08:12:13.884713Z","shell.execute_reply":"2026-01-19T08:12:16.040068Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nfrom IPython.display import display, HTML\n\nmissions = [\"ESA-Mission1\", \"ESA-Mission2\", \"ESA-Mission3\"]\nid_stats = []\n\nfor mission in missions:\n    file_path = BASE_PATH / mission / mission / \"labels.csv\"\n    df = pd.read_csv(file_path)\n\n    id_counts = df[\"ID\"].value_counts()\n\n    id_stats.append({\n        \"🌍 Mission\": mission,\n        \"🆔 Total IDs\": df[\"ID\"].nunique(),\n        \"🔥 Most Active ID\": id_counts.index[0],\n        \"🔥 Event Count (Top ID)\": int(id_counts.iloc[0]),\n        \"📊 Avg Events per ID\": round(id_counts.mean(), 2),\n        \"📊 Median Events per ID\": int(id_counts.median())\n    })\n\n    # Visualization – Top 10 IDs with the most events\n    plt.figure(figsize=(12, 4))\n    id_counts.head(10).plot(kind=\"bar\", color=\"darkorange\")\n    plt.title(f\"{mission} – Top 10 IDs by Event Count\")\n    plt.xlabel(\"ID\")\n    plt.ylabel(\"Number of Events\")\n    plt.xticks(rotation=45)\n    plt.tight_layout()\n    plt.show()\n\n    # Visualization – Histogram of events per ID\n    plt.figure(figsize=(8, 4))\n    plt.hist(id_counts.values, bins=30, color=\"lightcoral\", alpha=0.7)\n    plt.title(f\"{mission} – Event Count Distribution per ID\")\n    plt.xlabel(\"Number of Events\")\n    plt.ylabel(\"Frequency\")\n    plt.tight_layout()\n    plt.show()\n\n# Comparative table\nid_df = pd.DataFrame(id_stats)\n\nstyled = (\n    id_df.style\n    .set_properties(**{\"text-align\": \"center\"})\n    .set_table_styles([\n        {\"selector\": \"th\", \"props\": [(\"background-color\", \"#9467bd\"),\n                                     (\"color\", \"white\"),\n                                     (\"font-weight\", \"bold\")]}\n    ])\n)\n\ndisplay(HTML(\"<h3>ESA Missions ID Analysis</h3>\"))\ndisplay(styled)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T08:13:06.057373Z","iopub.execute_input":"2026-01-19T08:13:06.057738Z","iopub.status.idle":"2026-01-19T08:13:07.204489Z","shell.execute_reply.started":"2026-01-19T08:13:06.057695Z","shell.execute_reply":"2026-01-19T08:13:07.203563Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nfrom IPython.display import display, HTML\n\nmissions = [\"ESA-Mission1\", \"ESA-Mission2\", \"ESA-Mission3\"]\nduration_stats = []\n\nfor mission in missions:\n    file_path = BASE_PATH / mission / mission / \"labels.csv\"\n    df = pd.read_csv(file_path)\n\n    # Time conversion\n    df[\"StartTime\"] = pd.to_datetime(df[\"StartTime\"], errors=\"coerce\", utc=True)\n    df[\"EndTime\"] = pd.to_datetime(df[\"EndTime\"], errors=\"coerce\", utc=True)\n    df[\"Duration_sec\"] = (df[\"EndTime\"] - df[\"StartTime\"]).dt.total_seconds()\n\n    # Summary statistics\n    duration_stats.append({\n        \"Mission\": mission,\n        \"Mean (sec)\": round(df[\"Duration_sec\"].mean(), 2),\n        \"Median (sec)\": round(df[\"Duration_sec\"].median(), 2),\n        \"Min (sec)\": df[\"Duration_sec\"].min(),\n        \"Max (sec)\": df[\"Duration_sec\"].max(),\n        \"% Zero Duration (End = Start)\": round((df[\"Duration_sec\"] == 0).mean() * 100, 2)\n    })\n\n    # Histogram (log scale)\n    plt.figure(figsize=(10, 4))\n    plt.hist(df[\"Duration_sec\"] / 60, bins=100, color=\"teal\", alpha=0.7)\n    plt.xscale(\"log\")\n    plt.title(f\"{mission} – Event Duration Distribution (minutes, log scale)\")\n    plt.xlabel(\"Duration (minutes, log)\")\n    plt.ylabel(\"Frequency\")\n    plt.tight_layout()\n    plt.show()\n\n    # Boxplot\n    plt.figure(figsize=(6, 4))\n    plt.boxplot(df[\"Duration_sec\"].dropna(), vert=True, showfliers=True)\n    plt.yscale(\"log\")\n    plt.title(f\"{mission} – Duration Distribution (Boxplot, log scale)\")\n    plt.ylabel(\"Duration (sec)\")\n    plt.tight_layout()\n    plt.show()\n\n# Comparative table\nduration_df = pd.DataFrame(duration_stats)\n\nstyled = (\n    duration_df.style\n    .set_properties(**{\"text-align\": \"center\"})\n    .set_table_styles([\n        {\"selector\": \"th\", \"props\": [(\"background-color\", \"#d62728\"),\n                                     (\"color\", \"white\"),\n                                     (\"font-weight\", \"bold\")]}\n    ])\n)\n\ndisplay(HTML(\"<h3>⏳ ESA Missions Duration Analysis</h3>\"))\ndisplay(styled)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T08:15:59.413627Z","iopub.execute_input":"2026-01-19T08:15:59.414566Z","iopub.status.idle":"2026-01-19T08:16:01.770842Z","shell.execute_reply.started":"2026-01-19T08:15:59.414526Z","shell.execute_reply":"2026-01-19T08:16:01.769936Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Section 6 – Advanced Visualizations (ESA Mission1–3)\n\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom IPython.display import display, HTML\n\nmissions = [\"ESA-Mission1\", \"ESA-Mission2\", \"ESA-Mission3\"]\n\nfor mission in missions:\n    file_path = BASE_PATH / mission / mission / \"labels.csv\"\n    df = pd.read_csv(file_path)\n\n    # Time conversion\n    df[\"StartTime\"] = pd.to_datetime(df[\"StartTime\"], errors=\"coerce\", utc=True)\n    df[\"Month\"] = (\n        df[\"StartTime\"]\n        .dt.tz_convert(None)\n        .dt.to_period(\"M\")\n        .astype(str)\n    )\n\n    # Channel × Month pivot table\n    pivot = pd.pivot_table(\n        df,\n        values=\"ID\",\n        index=\"Channel\",\n        columns=\"Month\",\n        aggfunc=\"count\",\n        fill_value=0\n    )\n\n    # Heatmap\n    plt.figure(figsize=(14, 6))\n    sns.heatmap(pivot, cmap=\"YlGnBu\", cbar_kws={\"label\": \"Number of Events\"})\n    plt.title(f\"{mission} – Channel × Month Heatmap\")\n    plt.xlabel(\"Month\")\n    plt.ylabel(\"Channel\")\n    plt.tight_layout()\n    plt.show()\n\n    # Timeline overlay for top 3 most active channels\n    top_channels = df[\"Channel\"].value_counts().head(3).index\n    plt.figure(figsize=(12, 5))\n    for ch in top_channels:\n        ch_events = df[df[\"Channel\"] == ch].resample(\"ME\", on=\"StartTime\").size()\n        plt.plot(ch_events.index, ch_events.values, label=ch)\n\n    plt.title(f\"{mission} – Timeline of Top 3 Active Channels\")\n    plt.xlabel(\"Time\")\n    plt.ylabel(\"Number of Events\")\n    plt.legend()\n    plt.grid(True)\n    plt.tight_layout()\n    plt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T08:16:58.023296Z","iopub.execute_input":"2026-01-19T08:16:58.024402Z","iopub.status.idle":"2026-01-19T08:17:00.776025Z","shell.execute_reply.started":"2026-01-19T08:16:58.024363Z","shell.execute_reply":"2026-01-19T08:17:00.775090Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nfrom IPython.display import display, HTML\n\nmissions = [\"ESA-Mission1\", \"ESA-Mission2\", \"ESA-Mission3\"]\nquality_stats = []\n\nfor mission in missions:\n    file_path = BASE_PATH / mission / mission / \"labels.csv\"\n    df = pd.read_csv(file_path)\n\n    # Time conversion\n    df[\"StartTime\"] = pd.to_datetime(df[\"StartTime\"], errors=\"coerce\", utc=True)\n    df[\"EndTime\"] = pd.to_datetime(df[\"EndTime\"], errors=\"coerce\", utc=True)\n\n    # Missing values\n    missing = df.isna().mean() * 100\n\n    # Number of duplicated rows\n    dupes = df.duplicated().sum()\n\n    # Inconsistent timestamps (EndTime < StartTime)\n    invalid_times = (\n        (df[\"EndTime\"] < df[\"StartTime\"]) &\n        df[\"EndTime\"].notna() &\n        df[\"StartTime\"].notna()\n    ).sum()\n\n    # Number of missing values in the Channel column\n    channel_missing = df[\"Channel\"].isna().sum()\n\n    quality_stats.append({\n        \"Mission\": mission,\n        \"Missing Values (%)\": round(missing.mean(), 2),\n        \"Duplicate Rows\": dupes,\n        \"Time Errors\": invalid_times,\n        \"Missing Channel\": channel_missing\n    })\n\n# Comparative table\nquality_df = pd.DataFrame(quality_stats)\n\nstyled = (\n    quality_df.style\n    .set_properties(**{\"text-align\": \"center\"})\n    .set_table_styles([\n        {\"selector\": \"th\", \"props\": [(\"background-color\", \"#8c564b\"),\n                                     (\"color\", \"white\"),\n                                     (\"font-weight\", \"bold\")]}\n    ])\n)\n\ndisplay(HTML(\"<h3>🛠️ ESA Missions Data Quality Checks</h3>\"))\ndisplay(styled)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T08:19:02.273098Z","iopub.execute_input":"2026-01-19T08:19:02.273561Z","iopub.status.idle":"2026-01-19T08:19:02.386458Z","shell.execute_reply.started":"2026-01-19T08:19:02.273525Z","shell.execute_reply":"2026-01-19T08:19:02.385731Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install imbalanced-learn==0.11.0 xgboost","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import train_test_split\nimport joblib","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"DATA_PATH = \"/kaggle/input/esa-adb-challenge/\"\nSAMPLE_SUBMISSION = f'{DATA_PATH}' + 'sample_submission.parquet'\nTARGET_CHANNELS = f'{DATA_PATH}' + 'target_channels.csv'\nTRAIN_DATA = f'{DATA_PATH}' + 'train.parquet'\nTEST_DATA = f'{DATA_PATH}' + 'test.parquet'\nsample_df = pd.read_parquet(SAMPLE_SUBMISSION)\ntrain_df = pd.read_parquet(TRAIN_DATA)\ntest_df = pd.read_parquet(TEST_DATA)\ntarget_df = pd.read_csv(TARGET_CHANNELS)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_df.head()\ntrain_df.describe()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"np.array(train_df.columns)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(train_df.columns)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.isnull().values.any()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dropped_cols = []","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.columns","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"threshold = 1e-3\nstds = train_df.std(numeric_only=True)\nlow_devuation_cols = stds[stds <= threshold].index\n\nprint(f'Dropping columns: {np.array(low_devuation_cols)}')\ntrain_df.drop(columns=low_devuation_cols, inplace=True)\ntest_df.drop(columns=low_devuation_cols, inplace=True)\nprint(len(train_df.columns))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def column_entropy(col: pd.Series, base: float = 2) -> float:\n    \n    counts = col.value_counts(dropna=False)\n    probs = counts / counts.sum()\n    \n    return -np.sum(probs * np.log2(probs))\nthreshold = 0.1\nentropies = train_df.apply(column_entropy)\nlow_entropy_cols = entropies[entropies < threshold].index.tolist()\nprint(f'Columns to drop: {low_entropy_cols}')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(train_df.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T07:43:12.227522Z","iopub.execute_input":"2026-01-19T07:43:12.227863Z","iopub.status.idle":"2026-01-19T07:43:12.234614Z","shell.execute_reply.started":"2026-01-19T07:43:12.227833Z","shell.execute_reply":"2026-01-19T07:43:12.233358Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"corr_matrix = train_df.corr()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T07:43:14.611930Z","iopub.execute_input":"2026-01-19T07:43:14.612685Z","iopub.status.idle":"2026-01-19T07:46:18.090089Z","shell.execute_reply.started":"2026-01-19T07:43:14.612652Z","shell.execute_reply":"2026-01-19T07:46:18.089110Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 7))\nsns.heatmap(corr_matrix, annot=False, cmap='coolwarm', fmt=\".2f\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T07:46:29.922322Z","iopub.execute_input":"2026-01-19T07:46:29.923191Z","iopub.status.idle":"2026-01-19T07:46:30.685398Z","shell.execute_reply.started":"2026-01-19T07:46:29.923153Z","shell.execute_reply":"2026-01-19T07:46:30.684265Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"corr_matrix = corr_matrix.abs()\nmask = np.triu(np.ones(corr_matrix.shape), k=1).astype(bool)\nupper = corr_matrix.where(mask)\n\nthreshold = 0.90\nhigh_corr = (\n    upper.stack()\n         .loc[lambda x: x > threshold]\n         .sort_values(ascending=False)\n)\n\nprint(\"Highly correlated columns: (|corr| > 0.90):\")\nprint(high_corr)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T07:46:34.418024Z","iopub.execute_input":"2026-01-19T07:46:34.418434Z","iopub.status.idle":"2026-01-19T07:46:34.451373Z","shell.execute_reply.started":"2026-01-19T07:46:34.418403Z","shell.execute_reply":"2026-01-19T07:46:34.450488Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"corr_matrix = corr_matrix.abs()\nmask = np.triu(np.ones(corr_matrix.shape), k=1).astype(bool)\nupper = corr_matrix.where(mask)\n\nthreshold = 0.90\nhigh_corr = (\n    upper.stack()\n         .loc[lambda x: x > threshold]\n         .sort_values(ascending=False)\n)\n\nprint(\"Highly correlated columns: (|corr| > 0.90):\")\nprint(high_corr)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T07:46:41.862372Z","iopub.execute_input":"2026-01-19T07:46:41.863101Z","iopub.status.idle":"2026-01-19T07:46:41.874604Z","shell.execute_reply.started":"2026-01-19T07:46:41.863067Z","shell.execute_reply":"2026-01-19T07:46:41.873611Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"high_corellated_cols = []\nfor (col1, col2), corr_value in high_corr.items():\n    high_corellated_cols.append(col2)\n\ntrain_df.drop(columns=high_corellated_cols, inplace=True)\ntest_df.drop(columns=high_corellated_cols, inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T07:46:46.766864Z","iopub.execute_input":"2026-01-19T07:46:46.767210Z","iopub.status.idle":"2026-01-19T07:46:47.449396Z","shell.execute_reply.started":"2026-01-19T07:46:46.767180Z","shell.execute_reply":"2026-01-19T07:46:47.448507Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dropped_cols.append(high_corellated_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T07:46:48.662142Z","iopub.execute_input":"2026-01-19T07:46:48.662908Z","iopub.status.idle":"2026-01-19T07:46:48.667213Z","shell.execute_reply.started":"2026-01-19T07:46:48.662876Z","shell.execute_reply":"2026-01-19T07:46:48.666105Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(len(train_df.columns))\ntrain_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T07:46:50.002298Z","iopub.execute_input":"2026-01-19T07:46:50.002679Z","iopub.status.idle":"2026-01-19T07:46:50.025837Z","shell.execute_reply.started":"2026-01-19T07:46:50.002649Z","shell.execute_reply":"2026-01-19T07:46:50.024803Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dataframe_no_id = train_df.drop(columns='id')\ntest_no_id = test_df.drop(columns='id')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"import pandas as pd\nimport numpy as np\n\nfrom sklearn.preprocessing import StandardScaler, RobustScaler\nfrom sklearn.decomposition import PCA\nfrom sklearn.model_selection import train_test_split\n\nfrom scipy.signal import savgol_filter\nfrom scipy.stats import zscore\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\ndef load_data(file_path):\n    df = pd.read_csv(file_path)\n    return df.copy()\ndef process_timestamp(df, time_col=\"timestamp\"):\n    df[time_col] = pd.to_datetime(df[time_col], errors=\"coerce\", utc=True)\n    df = df.dropna(subset=[time_col])\n    df = df.sort_values(time_col).reset_index(drop=True)\n    return df\ndef clean_data(df):\n    df.replace([np.inf, -np.inf], np.nan, inplace=True)\n    return df\n\ndef handle_missing_values(df):\n    df = df.interpolate(method=\"time\")\n    df = df.ffill().bfill()\n    return df\n\ndef smooth_signals(df, sensor_cols, window=11, poly=2):\n    for col in sensor_cols:\n        if df[col].isna().sum() == 0 and len(df[col]) > window:\n            df[col] = savgol_filter(df[col], window, poly)\n    return df\n\ndef remove_extreme_outliers(df, sensor_cols, threshold=5):\n    for col in sensor_cols:\n        z = np.abs(zscore(df[col], nan_policy=\"omit\"))\n        df[col] = np.where(z > threshold, np.nan, df[col])\n    return df\n\ndef resample_data(df, time_col=\"timestamp\", rule=\"1S\"):\n    df = df.set_index(time_col)\n    df = df.resample(rule).mean()\n    df = df.interpolate()\n    df = df.reset_index()\n    return df\n\ndef feature_engineering(df, sensor_cols, window=10):\n    for col in sensor_cols:\n        df[f\"{col}_roll_mean\"] = df[col].rolling(window).mean()\n        df[f\"{col}_roll_std\"] = df[col].rolling(window).std()\n        df[f\"{col}_diff\"] = df[col].diff()\n    df = df.dropna().reset_index(drop=True)\n    return df\ndef normalize_features(df, feature_cols):\n    scaler = RobustScaler()\n    df[feature_cols] = scaler.fit_transform(df[feature_cols])\n    return df, scaler\ndef create_windows(X, y=None, window_size=100, stride=10):\n    X_windows = []\n    y_windows = []\n\n    for i in range(0, len(X) - window_size, stride):\n        X_windows.append(X[i:i+window_size])\n        if y is not None:\n            y_windows.append(int(np.any(y[i:i+window_size])))\n\n    X_windows = np.array(X_windows)\n    y_windows = np.array(y_windows) if y is not None else None\n\n    return X_windows, y_windows\ndef time_based_split(X, y, train_ratio=0.7, val_ratio=0.15):\n    n = len(X)\n    train_end = int(n * train_ratio)\n    val_end = int(n * (train_ratio + val_ratio))\n\n    X_train, y_train = X[:train_end], y[:train_end]\n    X_val, y_val = X[train_end:val_end], y[train_end:val_end]\n    X_test, y_test = X[val_end:], y[val_end:]\n\n    return X_train, X_val, X_test, y_train, y_val, y_test\ndef esa_preprocessing_pipeline(\n    file_path,\n    time_col=\"timestamp\",\n    label_col=\"label\",\n    resample_rule=\"1S\",\n    window_size=100,\n    stride=10\n):\n    df = load_data(file_path)\n    df = process_timestamp(df, time_col)\n    df = clean_data(df)\n\n    sensor_cols = [c for c in df.columns if c not in [time_col, label_col]]\n\n    df = handle_missing_values(df)\n    df = smooth_signals(df, sensor_cols)\n    df = remove_extreme_outliers(df, sensor_cols)\n    df = handle_missing_values(df)\n\n    df = resample_data(df, time_col, resample_rule)\n    sensor_cols = [c for c in df.columns if c not in [time_col, label_col]]\n\n    df = feature_engineering(df, sensor_cols)\n\n    feature_cols = [c for c in df.columns if c not in [time_col, label_col]]\n    df, scaler = normalize_features(df, feature_cols)\n\n    X = df[feature_cols].values\n    y = df[label_col].values if label_col in df.columns else None\n\n    X_seq, y_seq = create_windows(X, y, window_size, stride)\n    X_train, X_val, X_test, y_train, y_val, y_test = time_based_split(X_seq, y_seq)\n\n    return {\n        \"X_train\": X_train,\n        \"X_val\": X_val,\n        \"X_test\": X_test,\n        \"y_train\": y_train,\n        \"y_val\": y_val,\n        \"y_test\": y_test,\n        \"scaler\": scaler\n    }\ndata = esa_preprocessing_pipeline(\n    file_path=\"esa_telemetry.csv\",\n    time_col=\"timestamp\",\n    label_col=\"label\",\n    resample_rule=\"1S\",\n    window_size=100,\n    stride=10\n)\n\nprint(data[\"X_train\"].shape)\nprint(data[\"y_train\"].shape)\n\"\"\"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}