{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":94157,"databundleVersionId":13396106,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# =============================================================\n# Baseline v2 — AI-Based Modeling for Energy-Efficient Buildings (Kaggle)\n# Fixes: correct submission format (ID, TARGET_VARIABLE), 10-min grid (Jun–Jul 2025),\n#        3-hour lead-time compliance, no target-lag features in test, robust NA handling.\n# =============================================================\n\n\"\"\"\nQuick start (Kaggle):\n- Data path: /kaggle/input/ai-based-modeling-for-energy-efficient-buildings\n- Target sensor (per competition): B205WC000.AM02  (RETURN TEMPERATURE CHILLED WATER)\n- Train months: Jan–May 2025. Test months (submission): Jun & Jul 2025 at 10-min steps.\n- Output file: submission.csv with columns: ID, TARGET_VARIABLE\n  where ID is formatted as YYYY-MM-DD_HH:MM:SS and covers 2025-06-01 00:00:00 → 2025-07-31 23:50:00.\n\nNotes:\n- We resample all sensors at 10 minutes to match the evaluation grid.\n- We enforce the 3-hour lead by shifting ALL features by +18 steps (18 * 10min = 3h),\n  so features used for time t come from ≤ t-3h.\n- We DO NOT include target-lag features (since target is removed in test months).\n- For prediction, we reindex features onto the exact required grid and fill missing\n  values with forward-fill (causal) and then training medians as a last resort.\n\"\"\"\n\nimport os, sys, glob, warnings, re\nwarnings.filterwarnings(\"ignore\")\nfrom typing import List, Dict, Optional, Tuple\n\nimport numpy as np\nimport pandas as pd\n\ntry:\n    import lightgbm as lgb\nexcept Exception:\n    import subprocess\n    subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"lightgbm\", \"-q\"])\n    import lightgbm as lgb\n\n# -----------------------------\n# Config\n# -----------------------------\nCONFIG = {\n    \"INPUT_DIR\": \"/kaggle/input/ai-based-modeling-for-energy-efficient-buildings\",\n    \"BUILDING_ID\": \"B205\",\n\n    # Month folders present under INPUT_DIR (Jan..Jul 2025)\n    \"MONTH_FOLDERS\": [\n        \"RBHU-2025-01\",\n        \"RBHU-2025-02\",\n        \"RBHU-2025-03\",\n        \"RBHU-2025-04\",\n        \"RBHU-2025-05\",\n        \"RBHU-2025-06\",\n        \"RBHU-2025-07\",\n    ],\n\n    # Indexes into MONTH_FOLDERS — train on Jan..May only\n    \"TRAIN_MONTH_INDEXES\": [0, 1, 2, 3, 4],\n\n    # Resample frequency MUST be 10 minutes to match submission grid\n    \"RESAMPLE_FREQ\": \"10min\",\n\n    # Target is explicitly defined by organizers\n    \"TARGET_OBJECT_IDS\": [\"B205WC000.AM02\"],\n\n    # Feature engineering limits\n    \"MAX_SENSOR_FEATURES\": 60,\n    \"ROLL_WINDOWS\": [3, 6, 12, 36],  # in steps of RESAMPLE_FREQ (10min)\n    \"LAG_STEPS\":    [1, 2, 3, 6, 12, 36],\n    \"MIN_COVERAGE\": 0.60,\n\n    # Lead time: 3 hours → 18 steps at 10min\n    \"LEAD_STEPS\": 18,\n\n    # LightGBM params\n    \"LGB_PARAMS\": {\n        \"objective\": \"regression\",\n        \"metric\": [\"l1\", \"l2\"],\n        \"learning_rate\": 0.05,\n        \"num_leaves\": 64,\n        \"feature_fraction\": 0.9,\n        \"bagging_fraction\": 0.9,\n        \"bagging_freq\": 1,\n        \"min_data_in_leaf\": 50,\n        \"n_estimators\": 5000,\n        \"verbosity\": -1,\n        \"random_state\": 42,\n    },\n}\n\n# -----------------------------\n# IO helpers\n# -----------------------------\n\ndef _join_input(*parts: str) -> str:\n    return os.path.join(CONFIG[\"INPUT_DIR\"], *parts)\n\n\ndef read_metadata() -> pd.DataFrame:\n    p_parquet = _join_input(\"metadata.parquet\")\n    p_xlsx    = _join_input(\"metadata.xlsx\")\n    if os.path.exists(p_parquet):\n        meta = pd.read_parquet(p_parquet)\n    elif os.path.exists(p_xlsx):\n        meta = pd.read_excel(p_xlsx)\n    else:\n        print(\"[WARN] metadata not found — proceeding with filenames only.\")\n        return pd.DataFrame()\n    for c in [\n        \"object_id\",\"description\",\"dimension_text\",\"bde_channel_typ\",\"device_class\",\n        \"device\",\"channel\",\"file\",\"data_group_id\",\"class_id\"\n    ]:\n        if c in meta.columns:\n            meta[c] = meta[c].astype(str)\n    return meta\n\n\ndef list_parquet_files_for_building(month_folder: str, building_id: str) -> List[str]:\n    abs_month = _join_input(month_folder)\n    patterns = [\n        os.path.join(abs_month, \"RBHU\", building_id, \"**\", \"*.parquet\"),\n        os.path.join(abs_month, building_id, \"**\", \"*.parquet\"),  # fallback if nesting differs\n    ]\n    files = []\n    for p in patterns:\n        files.extend(glob.glob(p, recursive=True))\n    return sorted(list(set(files)))\n\n\n# -----------------------------\n# Data loading & feature engineering\n# -----------------------------\n\ndef load_and_resample_sensor(parquet_path: str, freq: str) -> pd.DataFrame:\n    df = pd.read_parquet(parquet_path)\n    ts_col = None\n    for c in df.columns:\n        lc = c.lower()\n        if \"time\" in lc or \"timestamp\" in lc:\n            ts_col = c; break\n    if ts_col is None:\n        ts_col = df.columns[0]\n    val_cols = [c for c in df.columns if c != ts_col]\n    if not val_cols:\n        raise ValueError(f\"No value column in {parquet_path}\")\n    val_col = val_cols[0]\n\n    df = df[[ts_col, val_col]].copy()\n    df[ts_col] = pd.to_datetime(df[ts_col], errors=\"coerce\")\n    df = df.dropna(subset=[ts_col]).set_index(ts_col).sort_index()\n    # Resample to 10min grid to match submission\n    df = df.resample(freq).ffill()\n    df.columns = [\"value\"]\n    return df\n\n\ndef build_panel(files: List[str], meta: pd.DataFrame, freq: str, min_coverage: float, max_sensors: int) -> Tuple[pd.DataFrame, Dict[str,str]]:\n    series_list = []\n    col_map: Dict[str,str] = {}\n\n    for fp in files:\n        try:\n            s = load_and_resample_sensor(fp, freq)\n        except Exception:\n            continue\n        cov = s[\"value\"].notna().mean()\n        if cov < min_coverage:\n            continue\n\n        base = os.path.splitext(os.path.basename(fp))[0]\n        oid_guess = base\n        if not meta.empty and {\"file\",\"object_id\"}.issubset(meta.columns):\n            hit = meta[meta[\"file\"].str.contains(base, na=False)]\n            if len(hit) == 1:\n                oid_guess = hit[\"object_id\"].values[0]\n\n        col_name = oid_guess\n        suf = 1\n        while col_name in col_map:\n            col_name = f\"{oid_guess}_{suf}\"; suf += 1\n\n        s = s.rename(columns={\"value\": col_name})\n        series_list.append(s)\n        col_map[col_name] = oid_guess\n\n        if len(col_map) >= max_sensors:\n            break\n\n    if not series_list:\n        raise RuntimeError(\"No usable sensors loaded. Check INPUT_DIR/BUILDING_ID/MONTH_FOLDERS.\")\n\n    df = pd.concat(series_list, axis=1).sort_index().ffill()\n    drop_cols = [c for c in df.columns if df[c].isna().all()]\n    df = df.drop(columns=drop_cols, errors='ignore')\n    for c in drop_cols:\n        col_map.pop(c, None)\n    return df, col_map\n\n\ndef pick_target_column(df: pd.DataFrame, col_map: Dict[str,str], meta: pd.DataFrame, explicit_object_ids: List[str]) -> str:\n    if explicit_object_ids:\n        for oid in explicit_object_ids:\n            for col, coid in col_map.items():\n                if coid == oid:\n                    return col\n        print(\"[WARN] Provided TARGET_OBJECT_IDS not found, falling back to heuristic.\")\n    # heuristic: look for energy/power keywords or exact id match in names\n    for col, oid in col_map.items():\n        if oid == \"B205WC000.AM02\":\n            return col\n    # fallback: highest variance\n    var = df.var(numeric_only=True).sort_values(ascending=False)\n    return var.index[0]\n\n\ndef time_features(index: pd.DatetimeIndex) -> pd.DataFrame:\n    out = pd.DataFrame(index=index)\n    out[\"minute\"] = index.minute\n    out[\"hour\"] = index.hour\n    out[\"day\"] = index.day\n    out[\"dow\"] = index.dayofweek\n    out[\"week\"] = index.isocalendar().week.astype(int)\n    out[\"month\"] = index.month\n    out[\"is_weekend\"] = (out[\"dow\"] >= 5).astype(int)\n    return out\n\n\ndef add_lag_roll_feats(df: pd.DataFrame, base_cols: List[str], roll_windows: List[int], lag_steps: List[int]) -> pd.DataFrame:\n    X = df.copy()\n    for c in base_cols:\n        for lag in lag_steps:\n            X[f\"{c}_lag{lag}\"] = X[c].shift(lag)\n        for w in roll_windows:\n            X[f\"{c}_rollmean{w}\"] = X[c].rolling(w).mean()\n            X[f\"{c}_rollstd{w}\"]  = X[c].rolling(w).std()\n    return X\n\n\ndef parse_year_month_from_folder(name: str) -> Tuple[Optional[int], Optional[int]]:\n    m = re.search(r\"(20\\d{2})[-_/](0[1-9]|1[0-2])\", name)\n    if not m:\n        m = re.search(r\"(20\\d{2}).*?(0[1-9]|1[0-2])\", name)\n    if m:\n        return int(m.group(1)), int(m.group(2))\n    return None, None\n\n\ndef month_mask(index: pd.DatetimeIndex, ym_list: List[Tuple[int,int]]) -> pd.Series:\n    if len(index) == 0:\n        return pd.Series([], dtype=bool)\n    mask = pd.Series(False, index=index)\n    for y, m in ym_list:\n        mask |= ((index.year == y) & (index.month == m))\n    return mask\n\n\ndef build_id_grid() -> pd.DatetimeIndex:\n    start = pd.Timestamp('2025-06-01 00:00:00')\n    end   = pd.Timestamp('2025-07-31 23:50:00')\n    idx = pd.date_range(start, end, freq='10min')\n    return idx\n\n\n# -----------------------------\n# Training / Prediction utils\n# -----------------------------\n\ndef fit_lgbm(X_tr: pd.DataFrame, y_tr: pd.Series, X_va: Optional[pd.DataFrame], y_va: Optional[pd.Series]) -> lgb.LGBMRegressor:\n    model = lgb.LGBMRegressor(**CONFIG[\"LGB_PARAMS\"])\n    if X_va is None or len(X_va) == 0:\n        model.fit(X_tr, y_tr)\n    else:\n        model.fit(\n            X_tr, y_tr,\n            eval_set=[(X_va, y_va)],\n            eval_metric=[\"l1\", \"l2\"],\n            callbacks=[lgb.early_stopping(200), lgb.log_evaluation(50)],\n        )\n    return model\n\n\ndef make_submission(model, features: List[str], X_all_shifted: pd.DataFrame, out_path: str = \"submission.csv\") -> pd.DataFrame:\n    # Create exact competition grid\n    grid = build_id_grid()\n    # Align features to grid and fill causally\n    Xg = X_all_shifted.reindex(grid)\n    # Forward-fill (causal) then fill remaining NA with train medians captured earlier\n    med = Xg.median(numeric_only=True)\n    Xg = Xg.ffill().fillna(med)\n\n    preds = model.predict(Xg[features])\n    sub = pd.DataFrame({\n        \"ID\": [ts.strftime('%Y-%m-%d_%H:%M:%S') for ts in grid],\n        \"TARGET_VARIABLE\": preds,\n    })\n    sub.to_csv(out_path, index=False)\n    print(f\"Saved submission to {out_path} with {len(sub)} rows (expected 8784).\")\n    return sub\n\n\n# -----------------------------\n# Main\n# -----------------------------\n\ndef main():\n    meta = read_metadata()\n\n    # 1) Collect files for each month\n    month_files: List[List[str]] = []\n    for mf in CONFIG[\"MONTH_FOLDERS\"]:\n        files = list_parquet_files_for_building(mf, CONFIG[\"BUILDING_ID\"])\n        month_files.append(files)\n        print(f\"Found {len(files)} parquet files in {mf} for {CONFIG['BUILDING_ID']}\")\n\n    # 2) Build panel for ALL months (resampled at 10min)\n    all_files = sorted(list(set([f for sub in month_files for f in sub])))\n    panel, col_map = build_panel(\n        files=all_files,\n        meta=meta,\n        freq=CONFIG[\"RESAMPLE_FREQ\"],\n        min_coverage=CONFIG[\"MIN_COVERAGE\"],\n        max_sensors=CONFIG[\"MAX_SENSOR_FEATURES\"],\n    )\n    print(f\"Panel shape (raw): {panel.shape} | sensors: {len(col_map)}\")\n\n    # 3) Target column — explicit per competition\n    target_col = pick_target_column(panel, col_map, meta, CONFIG[\"TARGET_OBJECT_IDS\"])\n    print(f\"Target column: {target_col} (object_id={col_map.get(target_col, target_col)})\")\n\n    # 4) Anti-leak: drop columns identical to target\n    dup_cols = [c for c in panel.columns if c != target_col and panel[c].equals(panel[target_col])]\n    if dup_cols:\n        panel = panel.drop(columns=dup_cols)\n        print(f\"Removed {len(dup_cols)} duplicate-of-target columns.\")\n\n    # 5) Time features\n    tf = time_features(panel.index)\n    panel_aug = pd.concat([panel, tf], axis=1)\n\n    # 6) Determine train period (Jan..May) & compute top sensors by correlation (using TRAIN ONLY)\n    ym_all = [parse_year_month_from_folder(f) for f in CONFIG[\"MONTH_FOLDERS\"]]\n    ym_train = [ym_all[i] for i in CONFIG[\"TRAIN_MONTH_INDEXES\"] if i < len(ym_all)]\n    train_mask = month_mask(panel_aug.index, ym_train)\n\n    panel_train = panel_aug.loc[train_mask]\n    # For correlation we need target present\n    corr_df = panel_train[[c for c in panel.columns if c != target_col] + [target_col]].dropna()\n    def select_predictor_columns_train(df: pd.DataFrame, target_col: str, k: int = 25) -> List[str]:\n        cors: Dict[str,float] = {}\n        for c in df.columns:\n            if c == target_col:\n                continue\n            s = df[[c, target_col]].dropna()\n            if len(s) < 100:\n                continue\n            cors[c] = s[c].corr(s[target_col])\n        top = sorted(cors.items(), key=lambda x: -abs(x[1]))[:k]\n        return [c for c,_ in top]\n    top_sensors = select_predictor_columns_train(corr_df, target_col, k=min(25, panel.shape[1]-1))\n\n    # 7) Feature engineering on ALL months (no target-lags!). Then enforce 3h lead by shifting features.\n    base_for_lags = list(set(top_sensors))  # IMPORTANT: exclude target column\n    X_all = add_lag_roll_feats(panel_aug, base_for_lags, CONFIG[\"ROLL_WINDOWS\"], CONFIG[\"LAG_STEPS\"]) \n    feature_cols = [c for c in X_all.columns if c != target_col]  # exclude target raw\n\n    # Shift ALL features by LEAD_STEPS to guarantee 3h-ahead causality\n    X_all_shifted = X_all[feature_cols].shift(CONFIG[\"LEAD_STEPS\"]).copy()\n\n    # 8) Build training/validation split WITHIN Jan–May (last 20% of time as validation)\n    train_feats = X_all_shifted.loc[train_mask]\n    train_label = panel.loc[train_mask, target_col]\n\n    # Drop rows with NA in either X or y\n    train_df = pd.concat([train_label.rename(\"y\"), train_feats], axis=1).dropna()\n    if len(train_df) == 0:\n        raise RuntimeError(\"Training frame is empty after dropping NAs. Consider reducing LAG/ROLL or MIN_COVERAGE.\")\n\n    # time-based split 80/20\n    train_df = train_df.sort_index()\n    n = len(train_df)\n    cut = int(n * 0.8)\n    tr = train_df.iloc[:cut]\n    va = train_df.iloc[cut:]\n\n    X_tr, y_tr = tr.drop(columns=[\"y\"]), tr[\"y\"]\n    X_va, y_va = va.drop(columns=[\"y\"]), va[\"y\"]\n    print(f\"Train: {X_tr.shape} | Valid: {X_va.shape}\")\n\n    # 9) Fit model\n    model = fit_lgbm(X_tr, y_tr, X_va, y_va)\n\n    # 10) Evaluate\n    if len(X_va) > 0:\n        from sklearn.metrics import mean_absolute_error, mean_squared_error\n        va_pred = model.predict(X_va)\n        mae = mean_absolute_error(y_va, va_pred)\n        rmse = mean_squared_error(y_va, va_pred, squared=False)\n        print(f\"VALID — MAE: {mae:.4f} | RMSE: {rmse:.4f}\")\n    else:\n        print(\"[INFO] No validation set; trained on all training data.\")\n\n    # 11) Submission on exact grid (Jun–Jul 2025 @ 10min)\n    sub = make_submission(model, feature_cols, X_all_shifted, out_path=\"submission.csv\")\n\n    # 12) Feature importance (optional)\n    try:\n        import matplotlib.pyplot as plt\n        imp = pd.Series(model.feature_importances_, index=feature_cols).sort_values(ascending=False)[:30]\n        plt.figure(figsize=(8, 10))\n        imp.iloc[::-1].plot(kind='barh')\n        plt.title('Top 30 Feature Importances (LightGBM)')\n        plt.tight_layout()\n        plt.savefig('feature_importance.png', dpi=160)\n        print(\"Saved feature_importance.png\")\n    except Exception as e:\n        print(f\"[WARN] Could not save feature importance: {e}\")\n\n    return sub\n\n\nif __name__ == \"__main__\":\n    _ = main()\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null}]}