{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":35297,"databundleVersionId":3589144}],"dockerImageVersionId":31287,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-03-03T13:11:40.856856Z","iopub.execute_input":"2026-03-03T13:11:40.857561Z","iopub.status.idle":"2026-03-03T13:11:41.130891Z","shell.execute_reply.started":"2026-03-03T13:11:40.857512Z","shell.execute_reply":"2026-03-03T13:11:41.130263Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================================================\n# FINAL PIGD + ISOLATION FOREST HYBRID\n# K-Fold=3 | Early Stopping\n# =========================================================\n\nimport os\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport torch\nfrom torch import nn, optim\nfrom torch.utils.data import DataLoader\n\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.ensemble import IsolationForest\nfrom sklearn.metrics import (\n    roc_auc_score,\n    average_precision_score,\n    precision_score,\n    recall_score,\n    roc_curve,\n    precision_recall_curve,\n    confusion_matrix,\n    f1_score\n)\n\nfrom sklearn.model_selection import KFold\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(\"Device:\", device)\n\n# =========================================================\n# DATA PREPARATION\n# =========================================================\ndef prepare_data(path):\n\n    df = pd.read_csv(path)\n    df[\"timestamp\"] = pd.to_datetime(df[\"timestamp\"], format=\"mixed\", errors=\"coerce\")\n    df = df.dropna(subset=[\"timestamp\"])\n    df = df.sort_values([\"building_id\",\"timestamp\"])\n\n    df[\"meter_log\"] = np.log1p(df[\"meter_reading\"])\n    df[\"shock\"] = df.groupby(\"building_id\")[\"meter_log\"].diff().fillna(0)\n    df[\"kinetic_s\"] = df.groupby(\"building_id\")[\"shock\"].diff().fillna(0)\n\n    df[\"temp_inertia\"] = df.groupby(\"building_id\")[\"air_temperature\"]\\\n        .transform(lambda x: x.rolling(6,min_periods=1).mean())\n\n    df[\"thermal_p\"] = df[\"air_temperature\"] - df[\"temp_inertia\"]\n\n    features = [\"shock\",\"thermal_p\",\"kinetic_s\",\"air_temperature\",\"square_feet\"]\n    df[features] = df[features].replace([np.inf,-np.inf],0).fillna(0)\n\n    scaler = StandardScaler()\n    X = scaler.fit_transform(df[features].values)\n    y = df[\"anomaly\"].values\n\n    return df, X, y\n\n# =========================================================\n# PIGD MODEL\n# =========================================================\nclass PIGD(nn.Module):\n    def __init__(self,input_dim):\n        super().__init__()\n        self.net = nn.Sequential(\n            nn.Linear(input_dim,256),\n            nn.ReLU(),\n            nn.Linear(256,128),\n            nn.ReLU(),\n            nn.Linear(128,input_dim)\n        )\n\n    def forward(self,x):\n        return self.net(x)\n\n# =========================================================\n# EARLY STOPPING TRAINING\n# =========================================================\ndef train_pigd(model, loader, epochs=25, patience=4):\n\n    optimizer = optim.Adam(model.parameters(), lr=1e-3)\n    loss_fn = nn.MSELoss()\n\n    best_loss = np.inf\n    counter = 0\n\n    for epoch in range(epochs):\n        total_loss = 0\n        model.train()\n\n        for batch in loader:\n            batch = batch.to(device)\n            optimizer.zero_grad()\n            recon = model(batch)\n            loss = loss_fn(recon, batch)\n            loss.backward()\n            torch.nn.utils.clip_grad_norm_(model.parameters(),1.0)\n            optimizer.step()\n            total_loss += loss.item()\n\n        if total_loss < best_loss:\n            best_loss = total_loss\n            counter = 0\n        else:\n            counter += 1\n            if counter >= patience:\n                break\n\n# =========================================================\n# MAIN PIPELINE\n# =========================================================\ndef run_pipeline(df,X,y):\n\n    kf = KFold(n_splits=3, shuffle=True, random_state=42)\n\n    final_scores = np.zeros(len(y))\n\n    for fold,(train_idx,test_idx) in enumerate(kf.split(X)):\n\n        X_train = X[train_idx][y[train_idx]==0]  # only normal\n        X_test = X[test_idx]\n\n        # ---------------- PIGD ----------------\n        pigd = PIGD(X.shape[1]).to(device)\n\n        loader = DataLoader(\n            torch.tensor(X_train,dtype=torch.float32),\n            batch_size=512,\n            shuffle=True\n        )\n\n        train_pigd(pigd, loader)\n\n        pigd.eval()\n        with torch.no_grad():\n            X_tensor = torch.tensor(X_test,dtype=torch.float32).to(device)\n            pigd_score = ((pigd(X_tensor)-X_tensor)**2).mean(dim=1).cpu().numpy()\n\n        # ---------------- ISOLATION FOREST ----------------\n        iso = IsolationForest(\n            n_estimators=200,\n            contamination=0.02,\n            random_state=42,\n            n_jobs=-1\n        )\n\n        iso.fit(X_train)\n\n        iso_score = -iso.decision_function(X_test)  # higher = more anomaly\n\n        # Weighted ensemble (boost PR)\n        final_scores[test_idx] = 0.65*pigd_score + 0.35*iso_score\n\n    # =====================================================\n    # FINAL METRICS\n    # =====================================================\n    roc_auc = roc_auc_score(y, final_scores)\n    pr_auc = average_precision_score(y, final_scores)\n\n    thresholds = np.linspace(final_scores.min(), final_scores.max(), 200)\n    best_f1 = 0\n    best_thr = thresholds[0]\n\n    for thr in thresholds:\n        pred = (final_scores>thr).astype(int)\n        f1 = f1_score(y,pred)\n        if f1>best_f1:\n            best_f1 = f1\n            best_thr = thr\n\n    y_pred = (final_scores>best_thr).astype(int)\n\n    precision = precision_score(y,y_pred)\n    recall = recall_score(y,y_pred)\n\n    tn,fp,fn,tp = confusion_matrix(y,y_pred).ravel()\n    tpr = tp/(tp+fn)\n    fpr = fp/(fp+tn)\n\n    print(\"\\n📊 FINAL RESULTS\")\n    print(\"ROC-AUC :\",roc_auc)\n    print(\"PR-AUC  :\",pr_auc)\n    print(\"Precision:\",precision)\n    print(\"Recall:\",recall)\n    print(\"TPR:\",tpr)\n    print(\"FPR:\",fpr)\n\n    # ROC Curve\n    fpr_curve,tpr_curve,_ = roc_curve(y,final_scores)\n    plt.figure()\n    plt.plot(fpr_curve,tpr_curve)\n    plt.title(\"ROC Curve\")\n    plt.show()\n\n    # PR Curve\n    prec_curve,rec_curve,_ = precision_recall_curve(y,final_scores)\n    plt.figure()\n    plt.plot(rec_curve,prec_curve)\n    plt.title(\"PR Curve\")\n    plt.show()\n\n    # Sample Building Plot\n    sample_building = df[\"building_id\"].unique()[0]\n    mask = df[\"building_id\"]==sample_building\n\n    plt.figure(figsize=(12,5))\n    plt.plot(df[mask][\"timestamp\"], final_scores[mask], label=\"Predicted Score\")\n    plt.scatter(df[mask][\"timestamp\"],\n                df[mask][\"anomaly\"]*final_scores.max(),\n                color=\"red\",label=\"Actual Anomaly\")\n    plt.legend()\n    plt.title(f\"Building {sample_building} - Predicted vs Actual\")\n    plt.xticks(rotation=45)\n    plt.show()\n\n# =========================================================\n# RUN\n# =========================================================\npath = \"/kaggle/input/competitions/energy-anomaly-detection/train_features.csv\"\n\nif os.path.exists(path):\n    df,X,y = prepare_data(path)\n    run_pipeline(df,X,y)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-03T17:50:00.673910Z","iopub.execute_input":"2026-03-03T17:50:00.674220Z","iopub.status.idle":"2026-03-03T17:58:08.468805Z","shell.execute_reply.started":"2026-03-03T17:50:00.674194Z","shell.execute_reply":"2026-03-03T17:58:08.468035Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================================================\n# PIGD + ISOLATION FOREST HYBRID (OPTIMIZED)\n# K-Fold=3 | Early Stopping | Final Metrics Only\n# Designed to push PR-AUC & ROC-AUC higher\n# =========================================================\n\nimport os\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport torch\nfrom torch import nn, optim\nfrom torch.utils.data import DataLoader\n\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.ensemble import IsolationForest\nfrom sklearn.metrics import (\n    roc_auc_score,\n    average_precision_score,\n    precision_score,\n    recall_score,\n    roc_curve,\n    precision_recall_curve,\n    confusion_matrix,\n    f1_score\n)\n\nfrom sklearn.model_selection import KFold\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(\"Device:\", device)\n\n# =========================================================\n# DATA PREPARATION\n# =========================================================\ndef prepare_data(path):\n\n    df = pd.read_csv(path)\n    df[\"timestamp\"] = pd.to_datetime(df[\"timestamp\"], format=\"mixed\", errors=\"coerce\")\n    df = df.dropna(subset=[\"timestamp\"])\n    df = df.sort_values([\"building_id\",\"timestamp\"])\n\n    df[\"meter_log\"] = np.log1p(df[\"meter_reading\"])\n    df[\"shock\"] = df.groupby(\"building_id\")[\"meter_log\"].diff().fillna(0)\n    df[\"kinetic_s\"] = df.groupby(\"building_id\")[\"shock\"].diff().fillna(0)\n\n    df[\"temp_inertia\"] = df.groupby(\"building_id\")[\"air_temperature\"]\\\n        .transform(lambda x: x.rolling(6,min_periods=1).mean())\n\n    df[\"thermal_p\"] = df[\"air_temperature\"] - df[\"temp_inertia\"]\n\n    features = [\"shock\",\"thermal_p\",\"kinetic_s\",\"air_temperature\",\"square_feet\"]\n    df[features] = df[features].replace([np.inf,-np.inf],0).fillna(0)\n\n    scaler = StandardScaler()\n    X = scaler.fit_transform(df[features].values)\n    y = df[\"anomaly\"].values\n\n    return df, X, y\n\n# =========================================================\n# STRONGER PIGD\n# =========================================================\nclass PIGD(nn.Module):\n    def __init__(self,input_dim):\n        super().__init__()\n        self.net = nn.Sequential(\n            nn.Linear(input_dim,512),\n            nn.ReLU(),\n            nn.Linear(512,256),\n            nn.ReLU(),\n            nn.Linear(256,128),\n            nn.ReLU(),\n            nn.Linear(128,input_dim)\n        )\n\n    def forward(self,x):\n        return self.net(x)\n\n# =========================================================\n# EARLY STOPPING TRAINER\n# =========================================================\ndef train_pigd(model, loader, epochs=30, patience=5):\n\n    optimizer = optim.Adam(model.parameters(), lr=1e-3)\n    loss_fn = nn.MSELoss()\n\n    best_loss = np.inf\n    counter = 0\n\n    for epoch in range(epochs):\n        total_loss = 0\n        model.train()\n\n        for batch in loader:\n            batch = batch.to(device)\n            optimizer.zero_grad()\n            recon = model(batch)\n            loss = loss_fn(recon, batch)\n            loss.backward()\n            torch.nn.utils.clip_grad_norm_(model.parameters(),1.0)\n            optimizer.step()\n            total_loss += loss.item()\n\n        if total_loss < best_loss:\n            best_loss = total_loss\n            counter = 0\n        else:\n            counter += 1\n            if counter >= patience:\n                break\n\n# =========================================================\n# MAIN PIPELINE\n# =========================================================\ndef run_pipeline(df,X,y):\n\n    kf = KFold(n_splits=3, shuffle=True, random_state=42)\n\n    final_scores = np.zeros(len(y))\n\n    for fold,(train_idx,test_idx) in enumerate(kf.split(X)):\n\n        X_train = X[train_idx][y[train_idx]==0]  # only normal\n        X_test = X[test_idx]\n\n        # ---------------- PIGD ----------------\n        pigd = PIGD(X.shape[1]).to(device)\n        loader = DataLoader(\n            torch.tensor(X_train,dtype=torch.float32),\n            batch_size=512,\n            shuffle=True\n        )\n        train_pigd(pigd, loader)\n\n        pigd.eval()\n        with torch.no_grad():\n            X_test_tensor = torch.tensor(X_test,dtype=torch.float32).to(device)\n            pigd_score = ((pigd(X_test_tensor)-X_test_tensor)**2)\\\n                .mean(dim=1).cpu().numpy()\n\n        # ---------------- Isolation Forest ----------------\n        iso = IsolationForest(\n            n_estimators=400,\n            contamination=0.03,  # rare anomalies\n            max_samples=\"auto\",\n            random_state=42,\n            n_jobs=-1\n        )\n        iso.fit(X_train)\n\n        iso_score = -iso.score_samples(X_test)  # higher = more anomalous\n\n        # ---------------- Ensemble (boost anomaly sensitivity) ----------------\n        final_scores[test_idx] = 0.75*pigd_score + 0.25*iso_score\n\n    # =====================================================\n    # FINAL METRICS (OVERALL)\n    # =====================================================\n    roc_auc = roc_auc_score(y, final_scores)\n    pr_auc = average_precision_score(y, final_scores)\n\n    # F1-optimal threshold\n    thresholds = np.linspace(final_scores.min(), final_scores.max(), 300)\n    best_f1 = 0\n    best_thr = thresholds[0]\n\n    for thr in thresholds:\n        pred = (final_scores>thr).astype(int)\n        f1 = f1_score(y,pred)\n        if f1>best_f1:\n            best_f1 = f1\n            best_thr = thr\n\n    y_pred = (final_scores>best_thr).astype(int)\n\n    precision = precision_score(y,y_pred)\n    recall = recall_score(y,y_pred)\n\n    tn,fp,fn,tp = confusion_matrix(y,y_pred).ravel()\n    tpr = tp/(tp+fn)\n    fpr = fp/(fp+tn)\n\n    print(\"\\n📊 FINAL RESULTS\")\n    print(\"ROC-AUC :\",roc_auc)\n    print(\"PR-AUC  :\",pr_auc)\n    print(\"Precision:\",precision)\n    print(\"Recall:\",recall)\n    print(\"TPR:\",tpr)\n    print(\"FPR:\",fpr)\n\n    # ROC Curve\n    fpr_curve,tpr_curve,_ = roc_curve(y,final_scores)\n    plt.figure()\n    plt.plot(fpr_curve,tpr_curve)\n    plt.title(\"ROC Curve\")\n    plt.show()\n\n    # PR Curve\n    prec_curve,rec_curve,_ = precision_recall_curve(y,final_scores)\n    plt.figure()\n    plt.plot(rec_curve,prec_curve)\n    plt.title(\"PR Curve\")\n    plt.show()\n\n    # Sample Building Plot\n    sample_building = df[\"building_id\"].unique()[0]\n    mask = df[\"building_id\"]==sample_building\n\n    plt.figure(figsize=(12,5))\n    plt.plot(df[mask][\"timestamp\"], final_scores[mask], label=\"Predicted Score\")\n    plt.scatter(df[mask][\"timestamp\"],\n                df[mask][\"anomaly\"]*final_scores.max(),\n                color=\"red\",label=\"Actual Anomaly\")\n    plt.legend()\n    plt.title(f\"Building {sample_building} - Predicted vs Actual\")\n    plt.xticks(rotation=45)\n    plt.show()\n\n# =========================================================\n# RUN\n# =========================================================\npath = \"/kaggle/input/competitions/energy-anomaly-detection/train_features.csv\"\n\nif os.path.exists(path):\n    df,X,y = prepare_data(path)\n    run_pipeline(df,X,y)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-03T16:38:31.130659Z","iopub.execute_input":"2026-03-03T16:38:31.131006Z","iopub.status.idle":"2026-03-03T16:51:25.100857Z","shell.execute_reply.started":"2026-03-03T16:38:31.130977Z","shell.execute_reply":"2026-03-03T16:51:25.100179Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**PIGD + TEMPORAL TRANSFORMER**","metadata":{}},{"cell_type":"code","source":"# =========================================================\n# PIGD + TEMPORAL TRANSFORMER (STABLE RESEARCH VERSION)\n# =========================================================\n\nimport os\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom torch import nn, optim\nfrom torch.utils.data import DataLoader, Dataset\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import (\n    roc_auc_score,\n    average_precision_score,\n    precision_score,\n    recall_score,\n    roc_curve,\n    precision_recall_curve\n)\nfrom sklearn.model_selection import KFold\nimport matplotlib.pyplot as plt\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(\"Device:\", device)\n\n# =========================================================\n# 1. DATA PREPARATION (PHYSICS UNCHANGED)\n# =========================================================\n\ndef prepare_data(path):\n\n    df = pd.read_csv(path)\n\n    df[\"timestamp\"] = pd.to_datetime(df[\"timestamp\"], format=\"mixed\", errors=\"coerce\")\n    df = df.dropna(subset=[\"timestamp\"])\n    df = df.sort_values([\"building_id\", \"timestamp\"])\n\n    df[\"meter_log\"] = np.log1p(df[\"meter_reading\"])\n    df[\"hour\"] = df[\"timestamp\"].dt.hour\n    df[\"dow\"] = df[\"timestamp\"].dt.dayofweek\n\n    norm = (\n        df[df[\"anomaly\"] == 0]\n        .groupby([\"building_id\", \"dow\", \"hour\"])[\"meter_log\"]\n        .agg([\"median\", \"std\"])\n        .reset_index()\n    )\n    norm.columns = [\"building_id\", \"dow\", \"hour\", \"h_med\", \"h_std\"]\n    df = df.merge(norm, on=[\"building_id\", \"dow\", \"hour\"], how=\"left\")\n\n    df[\"h_med\"] = df[\"h_med\"].fillna(\n        df.groupby(\"building_id\")[\"meter_log\"].transform(\"median\")\n    )\n\n    df[\"temp_inertia\"] = (\n        df.groupby(\"building_id\")[\"air_temperature\"]\n        .transform(lambda x: x.rolling(6, min_periods=1).mean())\n    )\n\n    df[\"thermal_p\"] = df[\"air_temperature\"] - df[\"temp_inertia\"]\n\n    df[\"shock\"] = df[\"meter_log\"] - df[\"h_med\"]\n    df[\"shock_v\"] = df.groupby(\"building_id\")[\"shock\"].diff().fillna(0)\n    df[\"kinetic_s\"] = df.groupby(\"building_id\")[\"shock_v\"].diff().fillna(0)\n\n    df[\"shock_scaled\"] = df.groupby(\"building_id\")[\"shock\"].transform(\n        lambda x: (x - x.median()) /\n        (x.quantile(0.75) - x.quantile(0.25) + 1e-6)\n    )\n\n    feat_cols = [\"shock_scaled\", \"thermal_p\", \"kinetic_s\", \"air_temperature\"]\n    df[feat_cols] = df[feat_cols].replace([np.inf, -np.inf], 0).fillna(0)\n\n    le_b, le_s, le_u = LabelEncoder(), LabelEncoder(), LabelEncoder()\n    df[\"b_idx\"] = le_b.fit_transform(df[\"building_id\"])\n    df[\"s_idx\"] = le_s.fit_transform(df[\"site_id\"])\n    df[\"u_idx\"] = le_u.fit_transform(df[\"primary_use\"])\n\n    return df, feat_cols, le_b, le_s, le_u\n\n# =========================================================\n# 2. SEQUENCE WINDOWING\n# =========================================================\n\nSEQ_LEN = 24\n\nclass EnergyDataset(Dataset):\n    def __init__(self, df, feat_cols):\n\n        self.X, self.y, self.b, self.s, self.u, self.bid = [], [], [], [], [], []\n\n        for bid, group in df.groupby(\"building_id\"):\n            group = group.sort_values(\"timestamp\")\n            feats = group[feat_cols].values\n            labels = group[\"anomaly\"].values\n            b_idx = group[\"b_idx\"].values\n            s_idx = group[\"s_idx\"].values\n            u_idx = group[\"u_idx\"].values\n\n            for i in range(len(group) - SEQ_LEN):\n                self.X.append(feats[i:i+SEQ_LEN])\n                self.y.append(labels[i+SEQ_LEN-1])\n                self.b.append(b_idx[i+SEQ_LEN-1])\n                self.s.append(s_idx[i+SEQ_LEN-1])\n                self.u.append(u_idx[i+SEQ_LEN-1])\n                self.bid.append(bid)\n\n        self.X = torch.tensor(np.array(self.X), dtype=torch.float32)\n        self.y = np.array(self.y)\n        self.b = torch.tensor(self.b)\n        self.s = torch.tensor(self.s)\n        self.u = torch.tensor(self.u)\n        self.bid = np.array(self.bid)\n\n    def __len__(self):\n        return len(self.X)\n\n    def __getitem__(self, idx):\n        return self.X[idx], self.y[idx], self.b[idx], self.s[idx], self.u[idx], self.bid[idx]\n\n# =========================================================\n# 3. MODEL\n# =========================================================\n\nclass TemporalTransformer(nn.Module):\n    def __init__(self, input_dim, d_model=128):\n        super().__init__()\n        self.proj = nn.Linear(input_dim, d_model)\n\n        enc_layer = nn.TransformerEncoderLayer(\n            d_model=d_model,\n            nhead=8,\n            dim_feedforward=256,\n            dropout=0.1,\n            batch_first=True\n        )\n\n        self.encoder = nn.TransformerEncoder(enc_layer, num_layers=3)\n        self.norm = nn.LayerNorm(d_model)\n\n    def forward(self, x):\n        x = self.proj(x)\n        x = self.encoder(x)\n        x = self.norm(x)\n        return x[:, -1, :]\n\nclass PIGD_Block(nn.Module):\n    def __init__(self, d, c):\n        super().__init__()\n        self.ln = nn.LayerNorm(d)\n        self.g = nn.Linear(c, d)\n        self.b = nn.Linear(c, d)\n        self.gate = nn.Sequential(nn.Linear(d, d), nn.Sigmoid())\n        self.net = nn.Sequential(\n            nn.Linear(d, d*2),\n            nn.SiLU(),\n            nn.Linear(d*2, d)\n        )\n\n    def forward(self, x, ctx):\n        h = self.ln(x)\n        h = h * (1 + self.g(ctx)) + self.b(ctx)\n        return x + self.gate(h) * self.net(h)\n\nclass PIGD_Transformer(nn.Module):\n    def __init__(self, inp, nb, ns, nu):\n        super().__init__()\n\n        self.temporal = TemporalTransformer(inp)\n\n        self.b = nn.Embedding(nb, 64)\n        self.s = nn.Embedding(ns, 16)\n        self.u = nn.Embedding(nu, 16)\n        self.t = nn.Sequential(nn.Linear(1, 32), nn.SiLU())\n\n        ctx_dim = 64 + 16 + 16 + 32\n\n        self.inp = nn.Linear(128, 256)\n        self.blocks = nn.ModuleList([PIGD_Block(256, ctx_dim) for _ in range(6)])\n        self.out = nn.Linear(256, inp)\n\n    def forward(self, x, t, b, s, u):\n        x = self.temporal(x)\n\n        te = self.t(t.float().unsqueeze(1) / 100)\n        ctx = torch.cat([self.b(b), self.s(s), self.u(u), te], dim=1)\n\n        h = self.inp(x)\n        for blk in self.blocks:\n            h = blk(h, ctx)\n\n        return self.out(h)\n\n# =========================================================\n# 4. TRAINING\n# =========================================================\n\ndef train_kfold(df, feat_cols, le_b, le_s, le_u):\n\n    dataset = EnergyDataset(df, feat_cols)\n    kf = KFold(n_splits=3, shuffle=True, random_state=42)\n\n    for fold, (train_idx, val_idx) in enumerate(kf.split(dataset.X)):\n\n        print(f\"\\n🔥 Fold {fold+1}\")\n\n        model = PIGD_Transformer(\n            len(feat_cols),\n            len(le_b.classes_),\n            len(le_s.classes_),\n            len(le_u.classes_)\n        ).to(device)\n\n        optimizer = optim.AdamW(model.parameters(), lr=1e-3)\n\n        best_pr = 0\n        patience = 4\n        counter = 0\n\n        train_loader = DataLoader(torch.utils.data.Subset(dataset, train_idx),\n                                  batch_size=1024, shuffle=True)\n\n        val_loader = DataLoader(torch.utils.data.Subset(dataset, val_idx),\n                                batch_size=2048)\n\n        for epoch in range(20):\n\n            model.train()\n            for x, y, b, s, u, _ in train_loader:\n\n                x, b, s, u = x.to(device), b.to(device), s.to(device), u.to(device)\n                t = torch.randint(0, 100, (x.size(0),), device=device)\n\n                recon = model(x, t, b, s, u)\n\n                loss = (\n                    0.6 * torch.abs(x[:, -1, 0] - recon[:, 0]) +\n                    0.3 * torch.abs(x[:, -1, 2] - recon[:, 2]) +\n                    0.1 * torch.abs(x[:, -1, 1] - recon[:, 1])\n                ).mean()\n\n                optimizer.zero_grad()\n                loss.backward()\n                torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n                optimizer.step()\n\n            # VALIDATION\n            model.eval()\n            all_scores, all_y = [], []\n\n            with torch.no_grad():\n                for x, y, b, s, u, _ in val_loader:\n\n                    x, b, s, u = x.to(device), b.to(device), s.to(device), u.to(device)\n                    t = torch.full((x.size(0),), 15, device=device)\n\n                    recon = model(x, t, b, s, u)\n\n                    err = (\n                        torch.abs(x[:, -1, 0] - recon[:, 0]) +\n                        0.75 * torch.abs(x[:, -1, 2] - recon[:, 2])\n                    ).cpu().numpy()\n\n                    err = np.nan_to_num(err)\n                    all_scores.extend(err)\n                    all_y.extend(y.numpy())\n\n            pr = average_precision_score(all_y, all_scores)\n            print(f\"Epoch {epoch+1} | Val PR-AUC: {pr:.4f}\")\n\n            if pr > best_pr:\n                best_pr = pr\n                counter = 0\n                torch.save(model.state_dict(), f\"best_fold{fold}.pt\")\n            else:\n                counter += 1\n\n            if counter >= patience:\n                print(\"Early stopping.\")\n                break\n\n# =========================================================\n# 5. FINAL ENSEMBLE + METRICS + PLOTS\n# =========================================================\n\ndef final_evaluation(df, feat_cols, le_b, le_s, le_u):\n\n    dataset = EnergyDataset(df, feat_cols)\n    loader = DataLoader(dataset, batch_size=2048, shuffle=False)\n\n    fold_predictions = []\n\n    for fold in range(3):\n\n        model = PIGD_Transformer(\n            len(feat_cols),\n            len(le_b.classes_),\n            len(le_s.classes_),\n            len(le_u.classes_)\n        ).to(device)\n\n        model.load_state_dict(torch.load(f\"best_fold{fold}.pt\"))\n        model.eval()\n\n        scores = []\n\n        with torch.no_grad():\n            for x, y, b, s, u, _ in loader:\n\n                x, b, s, u = x.to(device), b.to(device), s.to(device), u.to(device)\n                t = torch.full((x.size(0),), 15, device=device)\n\n                recon = model(x, t, b, s, u)\n\n                err = (\n                    torch.abs(x[:, -1, 0] - recon[:, 0]) +\n                    0.75 * torch.abs(x[:, -1, 2] - recon[:, 2])\n                ).cpu().numpy()\n\n                err = np.nan_to_num(err)\n                scores.extend(err)\n\n        fold_predictions.append(np.array(scores))\n\n    ensemble_scores = np.mean(np.vstack(fold_predictions), axis=0)\n\n    y_true = dataset.y\n    building_ids = dataset.bid\n\n    roc_auc = roc_auc_score(y_true, ensemble_scores)\n    pr_auc = average_precision_score(y_true, ensemble_scores)\n\n    precision_curve, recall_curve, thresholds = precision_recall_curve(\n        y_true, ensemble_scores\n    )\n\n    f1_scores = 2 * (precision_curve * recall_curve) / (\n        precision_curve + recall_curve + 1e-8\n    )\n\n    best_idx = np.argmax(f1_scores)\n    best_threshold = thresholds[best_idx]\n\n    y_pred = (ensemble_scores > best_threshold).astype(int)\n\n    precision = precision_score(y_true, y_pred)\n    recall = recall_score(y_true, y_pred)\n    tpr = recall\n    fpr = np.sum((y_pred == 1) & (y_true == 0)) / np.sum(y_true == 0)\n\n    print(\"\\n🏆 FINAL RESULTS\")\n    print(f\"ROC-AUC : {roc_auc:.4f}\")\n    print(f\"PR-AUC  : {pr_auc:.4f}\")\n    print(f\"Precision: {precision:.4f}\")\n    print(f\"Recall   : {recall:.4f}\")\n    print(f\"TPR      : {tpr:.4f}\")\n    print(f\"FPR      : {fpr:.4f}\")\n\n    # ROC\n    fpr_curve, tpr_curve, _ = roc_curve(y_true, ensemble_scores)\n    plt.figure(figsize=(6,6))\n    plt.plot(fpr_curve, tpr_curve)\n    plt.title(\"ROC Curve\")\n    plt.show()\n\n    # PR\n    plt.figure(figsize=(6,6))\n    plt.plot(recall_curve, precision_curve)\n    plt.title(\"PR Curve\")\n    plt.show()\n\n    # Sample Building Plot\n    sample_building = building_ids[0]\n    mask = building_ids == sample_building\n\n    plt.figure(figsize=(14,5))\n    plt.plot(ensemble_scores[mask], label=\"Predicted Score\")\n    plt.scatter(\n        np.where(y_true[mask] == 1),\n        ensemble_scores[mask][y_true[mask] == 1],\n        marker=\"x\",\n        label=\"Actual Anomaly\"\n    )\n    plt.title(f\"Building {sample_building}\")\n    plt.legend()\n    plt.show()\n\n# =========================================================\n# RUN\n# =========================================================\n\nif __name__ == \"__main__\":\n    path = \"/kaggle/input/competitions/energy-anomaly-detection/train_features.csv\"\n    df, feat_cols, le_b, le_s, le_u = prepare_data(path)\n    train_kfold(df, feat_cols, le_b, le_s, le_u)\n    final_evaluation(df, feat_cols, le_b, le_s, le_u)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-03T19:11:28.621187Z","iopub.execute_input":"2026-03-03T19:11:28.621931Z","iopub.status.idle":"2026-03-03T19:39:20.147829Z","shell.execute_reply.started":"2026-03-03T19:11:28.621901Z","shell.execute_reply":"2026-03-03T19:39:20.147253Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================================================\n# 0. IMPORTS\n# =========================================================\nimport os\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport torch\nfrom torch import nn, optim\nfrom torch.utils.data import DataLoader, TensorDataset\n\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import (\n    roc_auc_score,\n    average_precision_score,\n    precision_score,\n    recall_score,\n    roc_curve,\n    precision_recall_curve\n)\nfrom sklearn.model_selection import KFold\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(\"Device:\", device)\ntorch.cuda.empty_cache()\n\n# =========================================================\n# 1. DATA PREPARATION (PHYSICS UNCHANGED)\n# =========================================================\nSEQ_LEN = 16\n\ndef prepare_data(path):\n    df = pd.read_csv(path)\n\n    df[\"timestamp\"] = pd.to_datetime(df[\"timestamp\"], format=\"mixed\", errors=\"coerce\")\n    df = df.dropna(subset=[\"timestamp\"])\n    df = df.sort_values([\"building_id\", \"timestamp\"])\n\n    df[\"meter_log\"] = np.log1p(df[\"meter_reading\"])\n    df[\"hour\"] = df[\"timestamp\"].dt.hour\n    df[\"dow\"] = df[\"timestamp\"].dt.dayofweek\n\n    norm = (\n        df[df[\"anomaly\"] == 0]\n        .groupby([\"building_id\", \"dow\", \"hour\"])[\"meter_log\"]\n        .agg([\"median\", \"std\"])\n        .reset_index()\n    )\n    norm.columns = [\"building_id\", \"dow\", \"hour\", \"h_med\", \"h_std\"]\n    df = df.merge(norm, on=[\"building_id\", \"dow\", \"hour\"], how=\"left\")\n\n    df[\"h_med\"] = df[\"h_med\"].fillna(\n        df.groupby(\"building_id\")[\"meter_log\"].transform(\"median\")\n    )\n\n    df[\"temp_inertia\"] = (\n        df.groupby(\"building_id\")[\"air_temperature\"]\n        .transform(lambda x: x.rolling(6, min_periods=1).mean())\n    )\n\n    df[\"thermal_p\"] = df[\"air_temperature\"] - df[\"temp_inertia\"]\n\n    df[\"shock\"] = df[\"meter_log\"] - df[\"h_med\"]\n    df[\"shock_v\"] = df.groupby(\"building_id\")[\"shock\"].diff().fillna(0)\n    df[\"kinetic_s\"] = df.groupby(\"building_id\")[\"shock_v\"].diff().fillna(0)\n\n    df[\"shock_scaled\"] = df.groupby(\"building_id\")[\"shock\"].transform(\n        lambda x: (x - x.median()) /\n        (x.quantile(0.75) - x.quantile(0.25) + 1e-6)\n    )\n\n    feat_cols = [\"shock_scaled\", \"thermal_p\", \"kinetic_s\", \"air_temperature\"]\n    df[feat_cols] = df[feat_cols].replace([np.inf, -np.inf], 0).fillna(0)\n\n    le_b, le_s, le_u = LabelEncoder(), LabelEncoder(), LabelEncoder()\n    df[\"b_idx\"] = le_b.fit_transform(df[\"building_id\"])\n    df[\"s_idx\"] = le_s.fit_transform(df[\"site_id\"])\n    df[\"u_idx\"] = le_u.fit_transform(df[\"primary_use\"])\n\n    return df, feat_cols, le_b, le_s, le_u\n\n# =========================================================\n# 2. SEQUENCE WINDOWING\n# =========================================================\ndef create_sequences(df, feat_cols):\n    X, b, s, u, y, bid = [], [], [], [], [], []\n\n    for building_id, group in df.groupby(\"building_id\"):\n        group = group.sort_values(\"timestamp\")\n\n        feats = group[feat_cols].values\n        labels = group[\"anomaly\"].values\n        b_idx = group[\"b_idx\"].values\n        s_idx = group[\"s_idx\"].values\n        u_idx = group[\"u_idx\"].values\n\n        for i in range(len(group) - SEQ_LEN):\n            X.append(feats[i:i+SEQ_LEN])\n            y.append(labels[i+SEQ_LEN-1])\n            b.append(b_idx[i+SEQ_LEN-1])\n            s.append(s_idx[i+SEQ_LEN-1])\n            u.append(u_idx[i+SEQ_LEN-1])\n            bid.append(building_id)\n\n    return (\n        torch.tensor(np.array(X), dtype=torch.float32),\n        torch.tensor(b),\n        torch.tensor(s),\n        torch.tensor(u),\n        np.array(y),\n        np.array(bid)\n    )\n\n# =========================================================\n# 3. LIGHTWEIGHT TRANSFORMER MODEL\n# =========================================================\nclass TemporalTransformer(nn.Module):\n    def __init__(self, input_dim):\n        super().__init__()\n        self.proj = nn.Linear(input_dim, 96)\n\n        encoder_layer = nn.TransformerEncoderLayer(\n            d_model=96,\n            nhead=4,\n            dim_feedforward=192,\n            dropout=0.1,\n            batch_first=True\n        )\n        self.encoder = nn.TransformerEncoder(encoder_layer, num_layers=2)\n        self.norm = nn.LayerNorm(96)\n\n    def forward(self, x):\n        x = self.proj(x)\n        x = self.encoder(x)\n        x = self.norm(x[:, -1, :])\n        return torch.nan_to_num(x, nan=0.0, posinf=1.0, neginf=-1.0)\n\nclass PIGD_Transformer(nn.Module):\n    def __init__(self, inp, nb, ns, nu):\n        super().__init__()\n\n        self.temporal = TemporalTransformer(inp)\n\n        self.b = nn.Embedding(nb, 64)\n        self.s = nn.Embedding(ns, 16)\n        self.u = nn.Embedding(nu, 16)\n        self.t = nn.Sequential(nn.Linear(1, 32), nn.SiLU())\n\n        self.inp = nn.Linear(96, 192)\n\n        self.blocks = nn.ModuleList([\n            nn.Sequential(\n                nn.LayerNorm(192),\n                nn.Linear(192, 384),\n                nn.SiLU(),\n                nn.Linear(384, 192)\n            ) for _ in range(3)\n        ])\n\n        self.out = nn.Linear(192, inp)\n\n    def forward(self, x, t, b, s, u):\n        x = self.temporal(x)\n\n        te = self.t(t.float().unsqueeze(1)/100)\n        ctx = torch.cat([self.b(b), self.s(s), self.u(u), te], dim=1)\n\n        h = self.inp(x)\n        for blk in self.blocks:\n            h = h + blk(h)\n\n        out = self.out(h)\n        return torch.nan_to_num(out, nan=0.0, posinf=1.0, neginf=-1.0)\n\n# =========================================================\n# 4. TRAINING WITH K-FOLD + EARLY STOP + ENSEMBLE\n# =========================================================\ndef train_kfold(X, b, s, u, y, bid, nb, ns, nu):\n\n    kf = KFold(n_splits=3, shuffle=True, random_state=42)\n    ensemble_scores = np.zeros(len(y))\n\n    for fold, (train_idx, val_idx) in enumerate(kf.split(X)):\n        print(f\"\\n🔥 Fold {fold+1}\")\n\n        model = PIGD_Transformer(X.shape[2], nb, ns, nu).to(device)\n        opt = optim.AdamW(model.parameters(), lr=1e-3)\n\n        best_pr = 0\n        patience = 5\n        counter = 0\n\n        train_loader = DataLoader(\n            TensorDataset(X[train_idx], b[train_idx], s[train_idx], u[train_idx]),\n            batch_size=512,\n            shuffle=True\n        )\n\n        for epoch in range(25):\n            model.train()\n            for xb, bb, sb, ub in train_loader:\n                xb, bb, sb, ub = xb.to(device), bb.to(device), sb.to(device), ub.to(device)\n\n                target = xb[:, -1, :]\n                t = torch.zeros(xb.size(0), device=device)\n\n                pred = model(xb, t, bb, sb, ub)\n                loss = nn.MSELoss()(pred, target)\n\n                if torch.isfinite(loss):\n                    opt.zero_grad()\n                    loss.backward()\n                    torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n                    opt.step()\n\n            # Validation\n            model.eval()\n            val_loader = DataLoader(\n                TensorDataset(X[val_idx], b[val_idx], s[val_idx], u[val_idx]),\n                batch_size=512,\n                shuffle=False\n            )\n\n            val_err = []\n            with torch.no_grad():\n                for xb, bb, sb, ub in val_loader:\n                    xb, bb, sb, ub = xb.to(device), bb.to(device), sb.to(device), ub.to(device)\n                    t = torch.zeros(xb.size(0), device=device)\n                    pred = model(xb, t, bb, sb, ub)\n                    err = torch.abs(pred - xb[:, -1, :]).mean(dim=1)\n                    val_err.append(err.cpu())\n\n            val_err = torch.cat(val_err).numpy()\n            val_err = np.nan_to_num(val_err, nan=0.0)\n\n            pr = average_precision_score(y[val_idx], val_err)\n            print(f\"Epoch {epoch+1} | PR: {pr:.4f}\")\n\n            if pr > best_pr:\n                best_pr = pr\n                best_state = model.state_dict()\n                counter = 0\n            else:\n                counter += 1\n\n            if counter >= patience:\n                print(\"Early stopping\")\n                break\n\n        model.load_state_dict(best_state)\n\n        # Full inference batched\n        full_loader = DataLoader(\n            TensorDataset(X, b, s, u),\n            batch_size=512,\n            shuffle=False\n        )\n\n        fold_scores = []\n        with torch.no_grad():\n            for xb, bb, sb, ub in full_loader:\n                xb, bb, sb, ub = xb.to(device), bb.to(device), sb.to(device), ub.to(device)\n                t = torch.zeros(xb.size(0), device=device)\n                pred = model(xb, t, bb, sb, ub)\n                err = torch.abs(pred - xb[:, -1, :]).mean(dim=1)\n                fold_scores.append(err.cpu())\n\n        fold_scores = torch.cat(fold_scores).numpy()\n        fold_scores = np.nan_to_num(fold_scores, nan=0.0)\n\n        ensemble_scores += fold_scores / 3\n\n    return ensemble_scores\n\n# =========================================================\n# 5. EVALUATION\n# =========================================================\ndef evaluate(scores, y, bid):\n\n    scores = np.nan_to_num(scores, nan=0.0)\n\n    auc = roc_auc_score(y, scores)\n    pr = average_precision_score(y, scores)\n\n    thresh = np.percentile(scores, 95)\n    preds = (scores > thresh).astype(int)\n\n    precision = precision_score(y, preds)\n    recall = recall_score(y, preds)\n    fpr = np.sum((preds==1)&(y==0)) / np.sum(y==0)\n\n    print(\"\\n🏆 FINAL RESULTS\")\n    print(\"ROC-AUC :\", auc)\n    print(\"PR-AUC  :\", pr)\n    print(\"Precision:\", precision)\n    print(\"Recall:\", recall)\n    print(\"TPR:\", recall)\n    print(\"FPR:\", fpr)\n\n    fpr_c, tpr_c, _ = roc_curve(y, scores)\n    plt.plot(fpr_c, tpr_c)\n    plt.title(\"ROC Curve\")\n    plt.show()\n\n    p_c, r_c, _ = precision_recall_curve(y, scores)\n    plt.plot(r_c, p_c)\n    plt.title(\"PR Curve\")\n    plt.show()\n\n    sample = bid[0]\n    mask = (bid == sample)\n\n    plt.figure(figsize=(12,5))\n    plt.plot(scores[mask], label=\"Predicted\")\n    plt.scatter(np.where(y[mask]==1), scores[mask][y[mask]==1],\n                color=\"red\", label=\"Actual Anomaly\")\n    plt.legend()\n    plt.title(f\"Building {sample}\")\n    plt.show()\n\n# =========================================================\n# 6. RUN\n# =========================================================\npath = \"/kaggle/input/competitions/energy-anomaly-detection/train_features.csv\"\n\ndf, feats, le_b, le_s, le_u = prepare_data(path)\nX, b, s, u, y, bid = create_sequences(df, feats)\n\nscores = train_kfold(\n    X, b, s, u, y, bid,\n    len(le_b.classes_),\n    len(le_s.classes_),\n    len(le_u.classes_)\n)\n\nevaluate(scores, y, bid)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-03T20:10:08.302679Z","iopub.execute_input":"2026-03-03T20:10:08.302983Z","iopub.status.idle":"2026-03-03T20:27:02.115544Z","shell.execute_reply.started":"2026-03-03T20:10:08.302957Z","shell.execute_reply":"2026-03-03T20:27:02.114716Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================================================\n# 0. IMPORTS\n# =========================================================\nimport os\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport torch\nfrom torch import nn, optim\nfrom torch.utils.data import DataLoader, TensorDataset\n\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import (\n    roc_auc_score,\n    average_precision_score,\n    precision_score,\n    recall_score,\n    roc_curve,\n    precision_recall_curve\n)\nfrom sklearn.model_selection import KFold\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(\"Device:\", device)\ntorch.cuda.empty_cache()\n\n# =========================================================\n# 1. DATA PREPARATION (PHYSICS UNCHANGED)\n# =========================================================\nSEQ_LEN = 16\n\ndef prepare_data(path):\n    df = pd.read_csv(path)\n\n    df[\"timestamp\"] = pd.to_datetime(df[\"timestamp\"], format=\"mixed\", errors=\"coerce\")\n    df = df.dropna(subset=[\"timestamp\"])\n    df = df.sort_values([\"building_id\", \"timestamp\"])\n\n    df[\"meter_log\"] = np.log1p(df[\"meter_reading\"])\n    df[\"hour\"] = df[\"timestamp\"].dt.hour\n    df[\"dow\"] = df[\"timestamp\"].dt.dayofweek\n\n    norm = (\n        df[df[\"anomaly\"] == 0]\n        .groupby([\"building_id\", \"dow\", \"hour\"])[\"meter_log\"]\n        .agg([\"median\", \"std\"])\n        .reset_index()\n    )\n    norm.columns = [\"building_id\", \"dow\", \"hour\", \"h_med\", \"h_std\"]\n    df = df.merge(norm, on=[\"building_id\", \"dow\", \"hour\"], how=\"left\")\n\n    df[\"h_med\"] = df[\"h_med\"].fillna(\n        df.groupby(\"building_id\")[\"meter_log\"].transform(\"median\")\n    )\n\n    df[\"temp_inertia\"] = (\n        df.groupby(\"building_id\")[\"air_temperature\"]\n        .transform(lambda x: x.rolling(6, min_periods=1).mean())\n    )\n\n    df[\"thermal_p\"] = df[\"air_temperature\"] - df[\"temp_inertia\"]\n\n    df[\"shock\"] = df[\"meter_log\"] - df[\"h_med\"]\n    df[\"shock_v\"] = df.groupby(\"building_id\")[\"shock\"].diff().fillna(0)\n    df[\"kinetic_s\"] = df.groupby(\"building_id\")[\"shock_v\"].diff().fillna(0)\n\n    df[\"shock_scaled\"] = df.groupby(\"building_id\")[\"shock\"].transform(\n        lambda x: (x - x.median()) /\n        (x.quantile(0.75) - x.quantile(0.25) + 1e-6)\n    )\n\n    feat_cols = [\"shock_scaled\", \"thermal_p\", \"kinetic_s\", \"air_temperature\"]\n    df[feat_cols] = df[feat_cols].replace([np.inf, -np.inf], 0).fillna(0)\n\n    le_b, le_s, le_u = LabelEncoder(), LabelEncoder(), LabelEncoder()\n    df[\"b_idx\"] = le_b.fit_transform(df[\"building_id\"])\n    df[\"s_idx\"] = le_s.fit_transform(df[\"site_id\"])\n    df[\"u_idx\"] = le_u.fit_transform(df[\"primary_use\"])\n\n    return df, feat_cols, le_b, le_s, le_u\n\n# =========================================================\n# 2. SEQUENCE WINDOWING\n# =========================================================\ndef create_sequences(df, feat_cols):\n    X, b, s, u, y, bid = [], [], [], [], [], []\n\n    for building_id, group in df.groupby(\"building_id\"):\n        group = group.sort_values(\"timestamp\")\n\n        feats = group[feat_cols].values\n        labels = group[\"anomaly\"].values\n        b_idx = group[\"b_idx\"].values\n        s_idx = group[\"s_idx\"].values\n        u_idx = group[\"u_idx\"].values\n\n        for i in range(len(group) - SEQ_LEN):\n            X.append(feats[i:i+SEQ_LEN])\n            y.append(labels[i+SEQ_LEN-1])\n            b.append(b_idx[i+SEQ_LEN-1])\n            s.append(s_idx[i+SEQ_LEN-1])\n            u.append(u_idx[i+SEQ_LEN-1])\n            bid.append(building_id)\n\n    return (\n        torch.tensor(np.array(X), dtype=torch.float32),\n        torch.tensor(b),\n        torch.tensor(s),\n        torch.tensor(u),\n        np.array(y),\n        np.array(bid)\n    )\n\n# =========================================================\n# 3. TEMPORAL TRANSFORMER\n# =========================================================\nclass TemporalTransformer(nn.Module):\n    def __init__(self, input_dim):\n        super().__init__()\n        self.proj = nn.Linear(input_dim, 96)\n\n        encoder_layer = nn.TransformerEncoderLayer(\n            d_model=96,\n            nhead=4,\n            dim_feedforward=192,\n            dropout=0.1,\n            batch_first=True\n        )\n\n        self.encoder = nn.TransformerEncoder(encoder_layer, num_layers=2)\n        self.norm = nn.LayerNorm(96)\n\n    def forward(self, x):\n        x = self.proj(x)\n        x = self.encoder(x)\n        x = self.norm(x[:, -1, :])\n        return torch.nan_to_num(x)\n\n# =========================================================\n# 4. PIGD + TRANSFORMER WITH DIFFUSION\n# =========================================================\nclass PIGD_Transformer(nn.Module):\n    def __init__(self, inp, nb, ns, nu):\n        super().__init__()\n\n        self.temporal = TemporalTransformer(inp)\n\n        self.b = nn.Embedding(nb, 64)\n        self.s = nn.Embedding(ns, 16)\n        self.u = nn.Embedding(nu, 16)\n        self.t = nn.Sequential(nn.Linear(1, 32), nn.SiLU())\n\n        self.proj = nn.Linear(96, 192)\n\n        self.blocks = nn.ModuleList([\n            nn.Sequential(\n                nn.LayerNorm(192),\n                nn.Linear(192, 384),\n                nn.SiLU(),\n                nn.Linear(384, 192)\n            ) for _ in range(4)\n        ])\n\n        self.out = nn.Linear(192, inp)\n\n    def forward(self, x, t, b, s, u):\n        x = self.temporal(x)\n\n        te = self.t(t.float().unsqueeze(1)/100)\n        ctx = torch.cat([self.b(b), self.s(s), self.u(u), te], dim=1)\n\n        h = self.proj(x)\n        for blk in self.blocks:\n            h = h + blk(h)\n\n        return torch.nan_to_num(self.out(h))\n\n# =========================================================\n# 5. TRAINING (NORMAL ONLY + DIFFUSION)\n# =========================================================\ndef train_kfold(X, b, s, u, y, bid, nb, ns, nu):\n\n    kf = KFold(n_splits=3, shuffle=True, random_state=42)\n    ensemble_scores = np.zeros(len(y))\n\n    for fold, (train_idx, val_idx) in enumerate(kf.split(X)):\n        print(f\"\\n🔥 Fold {fold+1}\")\n\n        model = PIGD_Transformer(X.shape[2], nb, ns, nu).to(device)\n        opt = optim.AdamW(model.parameters(), lr=1e-3)\n\n        best_pr = 0\n        patience = 5\n        counter = 0\n\n        # TRAIN ONLY NORMAL WINDOWS\n        normal_mask = y[train_idx] == 0\n        X_train = X[train_idx][normal_mask]\n        b_train = b[train_idx][normal_mask]\n        s_train = s[train_idx][normal_mask]\n        u_train = u[train_idx][normal_mask]\n\n        train_loader = DataLoader(\n            TensorDataset(X_train, b_train, s_train, u_train),\n            batch_size=512,\n            shuffle=True\n        )\n\n        alphas = torch.linspace(1e-4, 0.02, 50).to(device)\n\n        for epoch in range(25):\n            model.train()\n            for xb, bb, sb, ub in train_loader:\n                xb, bb, sb, ub = xb.to(device), bb.to(device), sb.to(device), ub.to(device)\n\n                target = xb[:, -1, :]\n                t = torch.randint(0, 50, (xb.size(0),), device=device)\n                noise = torch.randn_like(target)\n                alpha = alphas[t].unsqueeze(1)\n\n                noisy = torch.sqrt(1-alpha)*target + torch.sqrt(alpha)*noise\n\n                pred = model(xb, t, bb, sb, ub)\n                loss = nn.MSELoss()(pred, target)\n\n                if torch.isfinite(loss):\n                    opt.zero_grad()\n                    loss.backward()\n                    torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n                    opt.step()\n\n            # VALIDATION\n            model.eval()\n            val_loader = DataLoader(\n                TensorDataset(X[val_idx], b[val_idx], s[val_idx], u[val_idx]),\n                batch_size=512,\n                shuffle=False\n            )\n\n            val_err = []\n            with torch.no_grad():\n                for xb, bb, sb, ub in val_loader:\n                    xb, bb, sb, ub = xb.to(device), bb.to(device), sb.to(device), ub.to(device)\n                    t = torch.zeros(xb.size(0), device=device)\n                    pred = model(xb, t, bb, sb, ub)\n\n                    # PHYSICS WEIGHTED ERROR\n                    err = (\n                        torch.abs(pred[:,0]-xb[:,-1,0]) +\n                        0.75*torch.abs(pred[:,2]-xb[:,-1,2]) +\n                        0.5*torch.abs(pred[:,1]-xb[:,-1,1])\n                    )\n\n                    val_err.append(err.cpu())\n\n            val_err = torch.cat(val_err).numpy()\n            val_err = np.nan_to_num(val_err)\n\n            pr = average_precision_score(y[val_idx], val_err)\n            print(f\"Epoch {epoch+1} | PR: {pr:.4f}\")\n\n            if pr > best_pr:\n                best_pr = pr\n                best_state = model.state_dict()\n                counter = 0\n            else:\n                counter += 1\n\n            if counter >= patience:\n                print(\"Early stopping\")\n                break\n\n        model.load_state_dict(best_state)\n\n        # FULL INFERENCE\n        full_loader = DataLoader(\n            TensorDataset(X, b, s, u),\n            batch_size=512,\n            shuffle=False\n        )\n\n        fold_scores = []\n        with torch.no_grad():\n            for xb, bb, sb, ub in full_loader:\n                xb, bb, sb, ub = xb.to(device), bb.to(device), sb.to(device), ub.to(device)\n                t = torch.zeros(xb.size(0), device=device)\n                pred = model(xb, t, bb, sb, ub)\n\n                err = (\n                    torch.abs(pred[:,0]-xb[:,-1,0]) +\n                    0.75*torch.abs(pred[:,2]-xb[:,-1,2]) +\n                    0.5*torch.abs(pred[:,1]-xb[:,-1,1])\n                )\n\n                fold_scores.append(err.cpu())\n\n        fold_scores = torch.cat(fold_scores).numpy()\n        fold_scores = np.nan_to_num(fold_scores)\n\n        ensemble_scores += fold_scores / 3\n\n    return ensemble_scores\n\n# =========================================================\n# 6. EVALUATION\n# =========================================================\ndef evaluate(scores, y, bid):\n\n    scores = np.nan_to_num(scores)\n\n    auc = roc_auc_score(y, scores)\n    pr = average_precision_score(y, scores)\n\n    thresh = np.percentile(scores, 95)\n    preds = (scores > thresh).astype(int)\n\n    precision = precision_score(y, preds)\n    recall = recall_score(y, preds)\n    fpr = np.sum((preds==1)&(y==0)) / np.sum(y==0)\n\n    print(\"\\n🏆 FINAL RESULTS\")\n    print(\"ROC-AUC :\", auc)\n    print(\"PR-AUC  :\", pr)\n    print(\"Precision:\", precision)\n    print(\"Recall:\", recall)\n    print(\"TPR:\", recall)\n    print(\"FPR:\", fpr)\n\n    fpr_c, tpr_c, _ = roc_curve(y, scores)\n    plt.plot(fpr_c, tpr_c)\n    plt.title(\"ROC Curve\")\n    plt.show()\n\n    p_c, r_c, _ = precision_recall_curve(y, scores)\n    plt.plot(r_c, p_c)\n    plt.title(\"PR Curve\")\n    plt.show()\n\n    sample = bid[0]\n    mask = (bid == sample)\n\n    plt.figure(figsize=(12,5))\n    plt.plot(scores[mask], label=\"Predicted\")\n    plt.scatter(np.where(y[mask]==1), scores[mask][y[mask]==1],\n                color=\"red\", label=\"Actual Anomaly\")\n    plt.legend()\n    plt.title(f\"Building {sample}\")\n    plt.show()\n\n# =========================================================\n# RUN\n# =========================================================\npath = \"/kaggle/input/competitions/energy-anomaly-detection/train_features.csv\"\n\ndf, feats, le_b, le_s, le_u = prepare_data(path)\nX, b, s, u, y, bid = create_sequences(df, feats)\n\nscores = train_kfold(\n    X, b, s, u, y, bid,\n    len(le_b.classes_),\n    len(le_s.classes_),\n    len(le_u.classes_)\n)\n\nevaluate(scores, y, bid)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-03T21:02:31.546918Z","iopub.execute_input":"2026-03-03T21:02:31.547539Z","iopub.status.idle":"2026-03-03T21:19:21.578682Z","shell.execute_reply.started":"2026-03-03T21:02:31.547510Z","shell.execute_reply":"2026-03-03T21:19:21.578105Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================================================\n# 0. IMPORTS\n# =========================================================\nimport os\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport torch\nfrom torch import nn, optim\nfrom torch.utils.data import DataLoader, TensorDataset\n\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import (\n    roc_auc_score,\n    average_precision_score,\n    precision_score,\n    recall_score,\n    roc_curve,\n    precision_recall_curve\n)\nfrom sklearn.model_selection import KFold\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(\"Device:\", device)\ntorch.cuda.empty_cache()\n\n# =========================================================\n# 1. DATA PREPARATION\n# =========================================================\nSEQ_LEN = 16\n\ndef prepare_data(path):\n    df = pd.read_csv(path)\n\n    df[\"timestamp\"] = pd.to_datetime(df[\"timestamp\"], format=\"mixed\", errors=\"coerce\")\n    df = df.dropna(subset=[\"timestamp\"])\n    df = df.sort_values([\"building_id\", \"timestamp\"])\n\n    df[\"meter_log\"] = np.log1p(df[\"meter_reading\"])\n    df[\"hour\"] = df[\"timestamp\"].dt.hour\n    df[\"dow\"] = df[\"timestamp\"].dt.dayofweek\n\n    norm = (\n        df[df[\"anomaly\"] == 0]\n        .groupby([\"building_id\", \"dow\", \"hour\"])[\"meter_log\"]\n        .agg([\"median\", \"std\"])\n        .reset_index()\n    )\n    norm.columns = [\"building_id\", \"dow\", \"hour\", \"h_med\", \"h_std\"]\n    df = df.merge(norm, on=[\"building_id\", \"dow\", \"hour\"], how=\"left\")\n\n    df[\"h_med\"] = df[\"h_med\"].fillna(\n        df.groupby(\"building_id\")[\"meter_log\"].transform(\"median\")\n    )\n\n    df[\"temp_inertia\"] = (\n        df.groupby(\"building_id\")[\"air_temperature\"]\n        .transform(lambda x: x.rolling(6, min_periods=1).mean())\n    )\n\n    df[\"thermal_p\"] = df[\"air_temperature\"] - df[\"temp_inertia\"]\n\n    df[\"shock\"] = df[\"meter_log\"] - df[\"h_med\"]\n    df[\"shock_v\"] = df.groupby(\"building_id\")[\"shock\"].diff().fillna(0)\n    df[\"kinetic_s\"] = df.groupby(\"building_id\")[\"shock_v\"].diff().fillna(0)\n\n    df[\"shock_scaled\"] = df.groupby(\"building_id\")[\"shock\"].transform(\n        lambda x: (x - x.median()) /\n        (x.quantile(0.75) - x.quantile(0.25) + 1e-6)\n    )\n\n    feat_cols = [\"shock_scaled\", \"thermal_p\", \"kinetic_s\", \"air_temperature\"]\n    df[feat_cols] = df[feat_cols].replace([np.inf, -np.inf], 0).fillna(0)\n\n    le_b, le_s, le_u = LabelEncoder(), LabelEncoder(), LabelEncoder()\n    df[\"b_idx\"] = le_b.fit_transform(df[\"building_id\"])\n    df[\"s_idx\"] = le_s.fit_transform(df[\"site_id\"])\n    df[\"u_idx\"] = le_u.fit_transform(df[\"primary_use\"])\n\n    return df, feat_cols, le_b, le_s, le_u\n\n# =========================================================\n# 2. SEQUENCE WINDOWING\n# =========================================================\ndef create_sequences(df, feat_cols):\n    X, b, s, u, y, bid = [], [], [], [], [], []\n\n    for building_id, group in df.groupby(\"building_id\"):\n        group = group.sort_values(\"timestamp\")\n\n        feats = group[feat_cols].values\n        labels = group[\"anomaly\"].values\n        b_idx = group[\"b_idx\"].values\n        s_idx = group[\"s_idx\"].values\n        u_idx = group[\"u_idx\"].values\n\n        for i in range(len(group) - SEQ_LEN):\n            X.append(feats[i:i+SEQ_LEN])\n            y.append(labels[i+SEQ_LEN-1])\n            b.append(b_idx[i+SEQ_LEN-1])\n            s.append(s_idx[i+SEQ_LEN-1])\n            u.append(u_idx[i+SEQ_LEN-1])\n            bid.append(building_id)\n\n    return (\n        torch.tensor(np.array(X), dtype=torch.float32),\n        torch.tensor(b),\n        torch.tensor(s),\n        torch.tensor(u),\n        np.array(y),\n        np.array(bid)\n    )\n\n# =========================================================\n# 3. TEMPORAL TRANSFORMER\n# =========================================================\nclass TemporalTransformer(nn.Module):\n    def __init__(self, input_dim):\n        super().__init__()\n        self.proj = nn.Linear(input_dim, 96)\n\n        encoder_layer = nn.TransformerEncoderLayer(\n            d_model=96,\n            nhead=4,\n            dim_feedforward=192,\n            dropout=0.1,\n            batch_first=True\n        )\n        self.encoder = nn.TransformerEncoder(encoder_layer, num_layers=2)\n        self.norm = nn.LayerNorm(96)\n\n    def forward(self, x):\n        x = self.proj(x)\n        x = self.encoder(x)\n        x = self.norm(x[:, -1, :])\n        return torch.nan_to_num(x)\n\n# =========================================================\n# 4. PIGD + TRANSFORMER\n# =========================================================\nclass PIGD_Transformer(nn.Module):\n    def __init__(self, inp, nb, ns, nu):\n        super().__init__()\n\n        self.temporal = TemporalTransformer(inp)\n\n        self.b = nn.Embedding(nb, 64)\n        self.s = nn.Embedding(ns, 16)\n        self.u = nn.Embedding(nu, 16)\n        self.t = nn.Sequential(nn.Linear(1, 32), nn.SiLU())\n\n        self.proj = nn.Linear(96, 192)\n\n        self.blocks = nn.ModuleList([\n            nn.Sequential(\n                nn.LayerNorm(192),\n                nn.Linear(192, 384),\n                nn.SiLU(),\n                nn.Linear(384, 192)\n            ) for _ in range(4)\n        ])\n\n        self.out = nn.Linear(192, inp)\n\n    def forward(self, x, t, b, s, u):\n        x = self.temporal(x)\n\n        te = self.t(t.float().unsqueeze(1)/100)\n        ctx = torch.cat([self.b(b), self.s(s), self.u(u), te], dim=1)\n\n        h = self.proj(x)\n        for blk in self.blocks:\n            h = h + blk(h)\n\n        return torch.nan_to_num(self.out(h))\n\n# =========================================================\n# 5. TRAINING WITH REAL DIFFUSION\n# =========================================================\ndef train_kfold(X, b, s, u, y, bid, nb, ns, nu):\n\n    kf = KFold(n_splits=3, shuffle=True, random_state=42)\n    ensemble_scores = np.zeros(len(y))\n\n    alphas = torch.linspace(1e-4, 0.02, 50).to(device)\n\n    for fold, (train_idx, val_idx) in enumerate(kf.split(X)):\n        print(f\"\\n🔥 Fold {fold+1}\")\n\n        model = PIGD_Transformer(X.shape[2], nb, ns, nu).to(device)\n        opt = optim.AdamW(model.parameters(), lr=1e-3)\n\n        best_pr = 0\n        patience = 5\n        counter = 0\n\n        normal_mask = y[train_idx] == 0\n        X_train = X[train_idx][normal_mask]\n        b_train = b[train_idx][normal_mask]\n        s_train = s[train_idx][normal_mask]\n        u_train = u[train_idx][normal_mask]\n\n        train_loader = DataLoader(\n            TensorDataset(X_train, b_train, s_train, u_train),\n            batch_size=512,\n            shuffle=True\n        )\n\n        for epoch in range(25):\n            model.train()\n            for xb, bb, sb, ub in train_loader:\n                xb, bb, sb, ub = xb.to(device), bb.to(device), sb.to(device), ub.to(device)\n\n                target = xb[:, -1, :]\n                t = torch.randint(0, 50, (xb.size(0),), device=device)\n\n                noise = torch.randn_like(target)\n                alpha = alphas[t].unsqueeze(1)\n\n                noisy_last = torch.sqrt(1 - alpha) * target + torch.sqrt(alpha) * noise\n                xb_noisy = xb.clone()\n                xb_noisy[:, -1, :] = noisy_last\n\n                pred = model(xb_noisy, t, bb, sb, ub)\n\n                loss = nn.MSELoss()(pred, target)\n\n                if torch.isfinite(loss):\n                    opt.zero_grad()\n                    loss.backward()\n                    torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n                    opt.step()\n\n            # VALIDATION\n            model.eval()\n            val_loader = DataLoader(\n                TensorDataset(X[val_idx], b[val_idx], s[val_idx], u[val_idx]),\n                batch_size=512,\n                shuffle=False\n            )\n\n            val_err = []\n            with torch.no_grad():\n                for xb, bb, sb, ub in val_loader:\n                    xb, bb, sb, ub = xb.to(device), bb.to(device), sb.to(device), ub.to(device)\n\n                    t = torch.zeros(xb.size(0), device=device)\n                    pred = model(xb, t, bb, sb, ub)\n\n                    err = (\n                        torch.abs(pred[:,0]-xb[:,-1,0]) +\n                        0.75*torch.abs(pred[:,2]-xb[:,-1,2]) +\n                        0.5*torch.abs(pred[:,1]-xb[:,-1,1])\n                    )\n\n                    val_err.append(err.cpu())\n\n            val_err = torch.cat(val_err).numpy()\n            val_err = np.nan_to_num(val_err)\n\n            pr = average_precision_score(y[val_idx], val_err)\n            print(f\"Epoch {epoch+1} | PR: {pr:.4f}\")\n\n            if pr > best_pr:\n                best_pr = pr\n                best_state = model.state_dict()\n                counter = 0\n            else:\n                counter += 1\n\n            if counter >= patience:\n                print(\"Early stopping\")\n                break\n\n        model.load_state_dict(best_state)\n\n        # FULL INFERENCE\n        full_loader = DataLoader(\n            TensorDataset(X, b, s, u),\n            batch_size=512,\n            shuffle=False\n        )\n\n        fold_scores = []\n        with torch.no_grad():\n            for xb, bb, sb, ub in full_loader:\n                xb, bb, sb, ub = xb.to(device), bb.to(device), sb.to(device), ub.to(device)\n                t = torch.zeros(xb.size(0), device=device)\n                pred = model(xb, t, bb, sb, ub)\n\n                err = (\n                    torch.abs(pred[:,0]-xb[:,-1,0]) +\n                    0.75*torch.abs(pred[:,2]-xb[:,-1,2]) +\n                    0.5*torch.abs(pred[:,1]-xb[:,-1,1])\n                )\n\n                fold_scores.append(err.cpu())\n\n        fold_scores = torch.cat(fold_scores).numpy()\n        fold_scores = np.nan_to_num(fold_scores)\n\n        ensemble_scores += fold_scores / 3\n\n    return ensemble_scores\n\n# =========================================================\n# 6. EVALUATION\n# =========================================================\ndef evaluate(scores, y, bid):\n\n    scores = np.nan_to_num(scores)\n\n    auc = roc_auc_score(y, scores)\n    pr = average_precision_score(y, scores)\n\n    thresh = np.percentile(scores, 95)\n    preds = (scores > thresh).astype(int)\n\n    precision = precision_score(y, preds)\n    recall = recall_score(y, preds)\n    fpr = np.sum((preds==1)&(y==0)) / np.sum(y==0)\n\n    print(\"\\n🏆 FINAL RESULTS\")\n    print(\"ROC-AUC :\", auc)\n    print(\"PR-AUC  :\", pr)\n    print(\"Precision:\", precision)\n    print(\"Recall:\", recall)\n    print(\"TPR:\", recall)\n    print(\"FPR:\", fpr)\n\n    fpr_c, tpr_c, _ = roc_curve(y, scores)\n    plt.plot(fpr_c, tpr_c)\n    plt.title(\"ROC Curve\")\n    plt.show()\n\n    p_c, r_c, _ = precision_recall_curve(y, scores)\n    plt.plot(r_c, p_c)\n    plt.title(\"PR Curve\")\n    plt.show()\n\n    sample = bid[0]\n    mask = (bid == sample)\n\n    plt.figure(figsize=(12,5))\n    plt.plot(scores[mask], label=\"Predicted\")\n    plt.scatter(np.where(y[mask]==1), scores[mask][y[mask]==1],\n                color=\"red\", label=\"Actual Anomaly\")\n    plt.legend()\n    plt.title(f\"Building {sample}\")\n    plt.show()\n\n# =========================================================\n# RUN\n# =========================================================\npath = \"/kaggle/input/competitions/energy-anomaly-detection/train_features.csv\"\n\ndf, feats, le_b, le_s, le_u = prepare_data(path)\nX, b, s, u, y, bid = create_sequences(df, feats)\n\nscores = train_kfold(\n    X, b, s, u, y, bid,\n    len(le_b.classes_),\n    len(le_s.classes_),\n    len(le_u.classes_)\n)\n\nevaluate(scores, y, bid)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-03T21:28:16.285274Z","iopub.execute_input":"2026-03-03T21:28:16.285571Z","iopub.status.idle":"2026-03-03T21:45:11.656919Z","shell.execute_reply.started":"2026-03-03T21:28:16.285546Z","shell.execute_reply":"2026-03-03T21:45:11.656171Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================================================\n# 0. IMPORTS\n# =========================================================\nimport os\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport torch\nfrom torch import nn, optim\nfrom torch.utils.data import DataLoader, TensorDataset\n\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import (\n    roc_auc_score,\n    average_precision_score,\n    precision_score,\n    recall_score,\n    roc_curve,\n    precision_recall_curve\n)\nfrom sklearn.model_selection import KFold\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(\"Device:\", device)\ntorch.cuda.empty_cache()\n\n# =========================================================\n# 1. DATA PREPARATION\n# =========================================================\nSEQ_LEN = 16\nDIFF_T = 15   # diffusion timestep for inference consistency\nDIFF_STEPS = 50\n\ndef prepare_data(path):\n    df = pd.read_csv(path)\n\n    df[\"timestamp\"] = pd.to_datetime(df[\"timestamp\"], format=\"mixed\", errors=\"coerce\")\n    df = df.dropna(subset=[\"timestamp\"])\n    df = df.sort_values([\"building_id\", \"timestamp\"])\n\n    df[\"meter_log\"] = np.log1p(df[\"meter_reading\"])\n    df[\"hour\"] = df[\"timestamp\"].dt.hour\n    df[\"dow\"] = df[\"timestamp\"].dt.dayofweek\n\n    norm = (\n        df[df[\"anomaly\"] == 0]\n        .groupby([\"building_id\", \"dow\", \"hour\"])[\"meter_log\"]\n        .agg([\"median\", \"std\"])\n        .reset_index()\n    )\n    norm.columns = [\"building_id\", \"dow\", \"hour\", \"h_med\", \"h_std\"]\n    df = df.merge(norm, on=[\"building_id\", \"dow\", \"hour\"], how=\"left\")\n\n    df[\"h_med\"] = df[\"h_med\"].fillna(\n        df.groupby(\"building_id\")[\"meter_log\"].transform(\"median\")\n    )\n\n    df[\"temp_inertia\"] = (\n        df.groupby(\"building_id\")[\"air_temperature\"]\n        .transform(lambda x: x.rolling(6, min_periods=1).mean())\n    )\n\n    df[\"thermal_p\"] = df[\"air_temperature\"] - df[\"temp_inertia\"]\n\n    df[\"shock\"] = df[\"meter_log\"] - df[\"h_med\"]\n    df[\"shock_v\"] = df.groupby(\"building_id\")[\"shock\"].diff().fillna(0)\n    df[\"kinetic_s\"] = df.groupby(\"building_id\")[\"shock_v\"].diff().fillna(0)\n\n    df[\"shock_scaled\"] = df.groupby(\"building_id\")[\"shock\"].transform(\n        lambda x: (x - x.median()) /\n        (x.quantile(0.75) - x.quantile(0.25) + 1e-6)\n    )\n\n    feat_cols = [\"shock_scaled\", \"thermal_p\", \"kinetic_s\", \"air_temperature\"]\n    df[feat_cols] = df[feat_cols].replace([np.inf, -np.inf], 0).fillna(0)\n\n    le_b, le_s, le_u = LabelEncoder(), LabelEncoder(), LabelEncoder()\n    df[\"b_idx\"] = le_b.fit_transform(df[\"building_id\"])\n    df[\"s_idx\"] = le_s.fit_transform(df[\"site_id\"])\n    df[\"u_idx\"] = le_u.fit_transform(df[\"primary_use\"])\n\n    return df, feat_cols, le_b, le_s, le_u\n\n# =========================================================\n# 2. SEQUENCE WINDOWING\n# =========================================================\ndef create_sequences(df, feat_cols):\n    X, b, s, u, y, bid = [], [], [], [], [], []\n\n    for building_id, group in df.groupby(\"building_id\"):\n        group = group.sort_values(\"timestamp\")\n\n        feats = group[feat_cols].values\n        labels = group[\"anomaly\"].values\n        b_idx = group[\"b_idx\"].values\n        s_idx = group[\"s_idx\"].values\n        u_idx = group[\"u_idx\"].values\n\n        for i in range(len(group) - SEQ_LEN):\n            X.append(feats[i:i+SEQ_LEN])\n            y.append(labels[i+SEQ_LEN-1])\n            b.append(b_idx[i+SEQ_LEN-1])\n            s.append(s_idx[i+SEQ_LEN-1])\n            u.append(u_idx[i+SEQ_LEN-1])\n            bid.append(building_id)\n\n    return (\n        torch.tensor(np.array(X), dtype=torch.float32),\n        torch.tensor(b),\n        torch.tensor(s),\n        torch.tensor(u),\n        np.array(y),\n        np.array(bid)\n    )\n\n# =========================================================\n# 3. TEMPORAL TRANSFORMER\n# =========================================================\nclass TemporalTransformer(nn.Module):\n    def __init__(self, input_dim):\n        super().__init__()\n        self.proj = nn.Linear(input_dim, 96)\n\n        encoder_layer = nn.TransformerEncoderLayer(\n            d_model=96,\n            nhead=4,\n            dim_feedforward=192,\n            dropout=0.1,\n            batch_first=True\n        )\n        self.encoder = nn.TransformerEncoder(encoder_layer, num_layers=2)\n        self.norm = nn.LayerNorm(96)\n\n    def forward(self, x):\n        x = self.proj(x)\n        x = self.encoder(x)\n        x = self.norm(x[:, -1, :])\n        return torch.nan_to_num(x)\n\n# =========================================================\n# 4. PIGD + TRANSFORMER\n# =========================================================\nclass PIGD_Transformer(nn.Module):\n    def __init__(self, inp, nb, ns, nu):\n        super().__init__()\n\n        self.temporal = TemporalTransformer(inp)\n\n        self.b = nn.Embedding(nb, 64)\n        self.s = nn.Embedding(ns, 16)\n        self.u = nn.Embedding(nu, 16)\n        self.t = nn.Sequential(nn.Linear(1, 32), nn.SiLU())\n\n        self.proj = nn.Linear(96, 192)\n\n        self.blocks = nn.ModuleList([\n            nn.Sequential(\n                nn.LayerNorm(192),\n                nn.Linear(192, 384),\n                nn.SiLU(),\n                nn.Linear(384, 192)\n            ) for _ in range(4)\n        ])\n\n        self.out = nn.Linear(192, inp)\n\n    def forward(self, x, t, b, s, u):\n        x = self.temporal(x)\n\n        te = self.t(t.float().unsqueeze(1)/100)\n        ctx = torch.cat([self.b(b), self.s(s), self.u(u), te], dim=1)\n\n        h = self.proj(x)\n        for blk in self.blocks:\n            h = h + blk(h)\n\n        return torch.nan_to_num(self.out(h))\n\n# =========================================================\n# 5. TRAINING + VALIDATION + ENSEMBLE\n# =========================================================\ndef train_kfold(X, b, s, u, y, bid, nb, ns, nu):\n\n    kf = KFold(n_splits=3, shuffle=True, random_state=42)\n    ensemble_scores = np.zeros(len(y))\n\n    alphas = torch.linspace(1e-4, 0.02, DIFF_STEPS).to(device)\n\n    for fold, (train_idx, val_idx) in enumerate(kf.split(X)):\n        print(f\"\\n🔥 Fold {fold+1}\")\n\n        model = PIGD_Transformer(X.shape[2], nb, ns, nu).to(device)\n        opt = optim.AdamW(model.parameters(), lr=1e-3)\n\n        best_pr = 0\n        patience = 5\n        counter = 0\n\n        normal_mask = y[train_idx] == 0\n        X_train = X[train_idx][normal_mask]\n        b_train = b[train_idx][normal_mask]\n        s_train = s[train_idx][normal_mask]\n        u_train = u[train_idx][normal_mask]\n\n        train_loader = DataLoader(\n            TensorDataset(X_train, b_train, s_train, u_train),\n            batch_size=512,\n            shuffle=True\n        )\n\n        for epoch in range(25):\n            model.train()\n            for xb, bb, sb, ub in train_loader:\n                xb, bb, sb, ub = xb.to(device), bb.to(device), sb.to(device), ub.to(device)\n\n                target = xb[:, -1, :]\n                t = torch.randint(0, DIFF_STEPS, (xb.size(0),), device=device)\n\n                noise = torch.randn_like(target)\n                alpha = alphas[t].unsqueeze(1)\n\n                noisy_last = torch.sqrt(1 - alpha) * target + torch.sqrt(alpha) * noise\n                xb_noisy = xb.clone()\n                xb_noisy[:, -1, :] = noisy_last\n\n                pred = model(xb_noisy, t, bb, sb, ub)\n                loss = nn.MSELoss()(pred, target)\n\n                if torch.isfinite(loss):\n                    opt.zero_grad()\n                    loss.backward()\n                    torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n                    opt.step()\n\n            # VALIDATION WITH SAME DIFFUSION\n            model.eval()\n            val_loader = DataLoader(\n                TensorDataset(X[val_idx], b[val_idx], s[val_idx], u[val_idx]),\n                batch_size=512,\n                shuffle=False\n            )\n\n            val_err = []\n            with torch.no_grad():\n                for xb, bb, sb, ub in val_loader:\n                    xb, bb, sb, ub = xb.to(device), bb.to(device), sb.to(device), ub.to(device)\n\n                    target = xb[:, -1, :]\n                    t = torch.full((xb.size(0),), DIFF_T, device=device)\n\n                    noise = torch.randn_like(target)\n                    alpha = alphas[t].unsqueeze(1)\n\n                    noisy_last = torch.sqrt(1 - alpha) * target + torch.sqrt(alpha) * noise\n                    xb_noisy = xb.clone()\n                    xb_noisy[:, -1, :] = noisy_last\n\n                    pred = model(xb_noisy, t, bb, sb, ub)\n\n                    err = (\n                        torch.abs(pred[:,0]-target[:,0]) +\n                        0.75*torch.abs(pred[:,2]-target[:,2]) +\n                        0.5*torch.abs(pred[:,1]-target[:,1])\n                    )\n\n                    val_err.append(err.cpu())\n\n            val_err = torch.cat(val_err).numpy()\n            val_err = np.nan_to_num(val_err)\n\n            pr = average_precision_score(y[val_idx], val_err)\n            print(f\"Epoch {epoch+1} | PR: {pr:.4f}\")\n\n            if pr > best_pr:\n                best_pr = pr\n                best_state = model.state_dict()\n                counter = 0\n            else:\n                counter += 1\n\n            if counter >= patience:\n                print(\"Early stopping\")\n                break\n\n        model.load_state_dict(best_state)\n\n        # FULL INFERENCE WITH DIFFUSION\n        full_loader = DataLoader(\n            TensorDataset(X, b, s, u),\n            batch_size=512,\n            shuffle=False\n        )\n\n        fold_scores = []\n        with torch.no_grad():\n            for xb, bb, sb, ub in full_loader:\n                xb, bb, sb, ub = xb.to(device), bb.to(device), sb.to(device), ub.to(device)\n\n                target = xb[:, -1, :]\n                t = torch.full((xb.size(0),), DIFF_T, device=device)\n\n                noise = torch.randn_like(target)\n                alpha = alphas[t].unsqueeze(1)\n\n                noisy_last = torch.sqrt(1 - alpha) * target + torch.sqrt(alpha) * noise\n                xb_noisy = xb.clone()\n                xb_noisy[:, -1, :] = noisy_last\n\n                pred = model(xb_noisy, t, bb, sb, ub)\n\n                err = (\n                    torch.abs(pred[:,0]-target[:,0]) +\n                    0.75*torch.abs(pred[:,2]-target[:,2]) +\n                    0.5*torch.abs(pred[:,1]-target[:,1])\n                )\n\n                fold_scores.append(err.cpu())\n\n        fold_scores = torch.cat(fold_scores).numpy()\n        fold_scores = np.nan_to_num(fold_scores)\n\n        ensemble_scores += fold_scores / 3\n\n    return ensemble_scores\n\n# =========================================================\n# 6. EVALUATION\n# =========================================================\ndef evaluate(scores, y, bid):\n\n    scores = np.nan_to_num(scores)\n\n    auc = roc_auc_score(y, scores)\n    pr = average_precision_score(y, scores)\n\n    thresh = np.percentile(scores, 95)\n    preds = (scores > thresh).astype(int)\n\n    precision = precision_score(y, preds)\n    recall = recall_score(y, preds)\n    fpr = np.sum((preds==1)&(y==0)) / np.sum(y==0)\n\n    print(\"\\n🏆 FINAL RESULTS\")\n    print(\"ROC-AUC :\", auc)\n    print(\"PR-AUC  :\", pr)\n    print(\"Precision:\", precision)\n    print(\"Recall:\", recall)\n    print(\"TPR:\", recall)\n    print(\"FPR:\", fpr)\n\n    fpr_c, tpr_c, _ = roc_curve(y, scores)\n    plt.plot(fpr_c, tpr_c)\n    plt.title(\"ROC Curve\")\n    plt.show()\n\n    p_c, r_c, _ = precision_recall_curve(y, scores)\n    plt.plot(r_c, p_c)\n    plt.title(\"PR Curve\")\n    plt.show()\n\n    sample = bid[0]\n    mask = (bid == sample)\n\n    plt.figure(figsize=(12,5))\n    plt.plot(scores[mask], label=\"Predicted\")\n    plt.scatter(np.where(y[mask]==1), scores[mask][y[mask]==1],\n                color=\"red\", label=\"Actual Anomaly\")\n    plt.legend()\n    plt.title(f\"Building {sample}\")\n    plt.show()\n\n# =========================================================\n# RUN\n# =========================================================\npath = \"/kaggle/input/competitions/energy-anomaly-detection/train_features.csv\"\n\ndf, feats, le_b, le_s, le_u = prepare_data(path)\nX, b, s, u, y, bid = create_sequences(df, feats)\n\nscores = train_kfold(\n    X, b, s, u, y, bid,\n    len(le_b.classes_),\n    len(le_s.classes_),\n    len(le_u.classes_)\n)\n\nevaluate(scores, y, bid)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-03T21:54:34.560027Z","iopub.execute_input":"2026-03-03T21:54:34.560784Z","iopub.status.idle":"2026-03-03T22:11:30.408441Z","shell.execute_reply.started":"2026-03-03T21:54:34.560754Z","shell.execute_reply":"2026-03-03T22:11:30.407855Z"}},"outputs":[],"execution_count":null}]}