{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":17000,"databundleVersionId":895292,"sourceType":"competition"}],"dockerImageVersionId":31153,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# (1) Installs \n!pip install -q lightgbm catboost\n\n# (2) Imports & Global Setup\nimport warnings, gc, os\nwarnings.filterwarnings(\"ignore\")\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport lightgbm as lgb\nfrom xgboost import XGBClassifier\nfrom sklearn.linear_model import SGDClassifier, LogisticRegression\nfrom sklearn.calibration import CalibratedClassifierCV\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-10-27T16:58:05.625587Z","iopub.execute_input":"2025-10-27T16:58:05.625896Z","iopub.status.idle":"2025-10-27T16:58:19.016596Z","shell.execute_reply.started":"2025-10-27T16:58:05.625876Z","shell.execute_reply":"2025-10-27T16:58:19.014809Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **Setup & Load**","metadata":{}},{"cell_type":"code","source":"\n# ============== Load ==============\n\ntrain = pd.read_csv(\"/kaggle/input/cat-in-the-dat-ii/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/cat-in-the-dat-ii/test.csv\")\nID_COL, TARGET = \"id\", \"target\"\n\nprint(f\"Train shape: {train.shape} | Test shape: {test.shape}\")\ndisplay(train.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-27T16:58:19.018346Z","iopub.execute_input":"2025-10-27T16:58:19.019006Z","iopub.status.idle":"2025-10-27T16:58:25.406955Z","shell.execute_reply.started":"2025-10-27T16:58:19.018981Z","shell.execute_reply":"2025-10-27T16:58:25.405531Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **Clean**","metadata":{}},{"cell_type":"code","source":"# Basic cleaning shared by train/test (no leakage)\ndef clean_bound(df, col, lo, hi, fallback=1, as_int=True):\n    if col in df.columns:\n        s = pd.to_numeric(df[col], errors='coerce')\n        s = s.where((s >= lo) & (s <= hi))\n        fill = s.mode().iloc[0] if s.notna().any() else fallback\n        s = s.fillna(fill)\n        if as_int:\n            s = s.round().astype(int)\n        df[col] = s\n\nfor d in (train, test):\n    clean_bound(d, 'month', 1, 12, fallback=1, as_int=True)\n    clean_bound(d, 'day',   1, 7,  fallback=1, as_int=True)\n    d.replace([np.inf, -np.inf], np.nan, inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-27T16:58:25.407858Z","iopub.execute_input":"2025-10-27T16:58:25.408148Z","iopub.status.idle":"2025-10-27T16:58:28.097773Z","shell.execute_reply.started":"2025-10-27T16:58:25.408117Z","shell.execute_reply":"2025-10-27T16:58:28.096805Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"## EDA\nsns.set_style(\"whitegrid\")\nID_COL, TARGET = \"id\", \"target\"\n\n# Target distribution\nplt.figure(figsize=(6,4))\nsns.countplot(x='target', data=train)\nplt.title('Distribution of Target Variable')\nplt.show()\n\n# Categorize columns by type\nbin_cols = [col for col in train.columns if col.startswith('bin')]\nnom_cols = [col for col in train.columns if col.startswith('nom')]\nord_cols = [col for col in train.columns if col.startswith('ord')]\nday_cols = [col for col in train.columns if col.startswith('day')]\nmonth_cols = [col for col in train.columns if col.startswith('month')]\nprint(f'Binary columns ({len(bin_cols)}): {bin_cols}')\nprint(f'Nominal columns ({len(nom_cols)}): {nom_cols}')\nprint(f'Ordinal columns ({len(ord_cols)}): {ord_cols}')\nprint(f'Day columns: {day_cols}')\nprint(f'Month columns: {month_cols}')\nprint(\"Target ratio:\")\ndisplay(train['target'].value_counts(normalize=True).rename('ratio'))\n\n# Identify high-cardinality vs low-cardinality categorical features\n\nfeature_cols = [c for c in train.columns if c != 'target']\nHIGH_CARD_THRESHOLD = 20\ncat_like_cols = [c for c in feature_cols if (train[c].dtype=='object') or (str(train[c].dtype)=='category')]\nhigh_card_cols = [c for c in cat_like_cols if train[c].nunique(dropna=True) >= HIGH_CARD_THRESHOLD]\nlow_card_cols  = [c for c in cat_like_cols if c not in high_card_cols]\nprint(\"High-card:\", high_card_cols[:10], '...')\nprint(\"Low-card :\", low_card_cols[:10],  '...')\n# Plot example columns\nfig, axs = plt.subplots(1, 2, figsize=(12,5))\nsns.countplot(x='nom_0', data=train, ax=axs[0])\nsns.countplot(x='day', data=train, ax=axs[1])\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-27T16:58:28.100209Z","iopub.execute_input":"2025-10-27T16:58:28.100626Z","iopub.status.idle":"2025-10-27T16:58:29.560025Z","shell.execute_reply.started":"2025-10-27T16:58:28.100587Z","shell.execute_reply":"2025-10-27T16:58:29.558242Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#  Features Engineer ","metadata":{}},{"cell_type":"code","source":"#Helper Functions\n# ============================================================\ndef label_fit(series):\n    cats = pd.Index(series.astype(\"string\").fillna(\"__NA__\").unique())\n    return {cat: i for i, cat in enumerate(cats)}\n\ndef label_apply(series, mapping):\n    return series.astype(\"string\").fillna(\"__NA__\").map(mapping).fillna(-1).astype(int)\n\ndef target_encode_oof(train_col, y, val_col, n_splits=5, smoothing=20):\n    gmean = y.mean()\n    oof_enc = np.zeros(len(train_col))\n    kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n    for tr_idx, va_idx in kf.split(train_col, y):\n        tr_c, va_c = train_col.iloc[tr_idx], train_col.iloc[va_idx]\n        tr_y = y.iloc[tr_idx]\n        stats = tr_c.to_frame(\"key\").join(tr_y.rename(\"y\")).groupby(\"key\")[\"y\"].agg([\"mean\",\"count\"])\n        stats[\"enc\"] = (stats[\"mean\"]*stats[\"count\"] + gmean*smoothing) / (stats[\"count\"]+smoothing)\n        oof_enc[va_idx] = va_c.map(stats[\"enc\"]).fillna(gmean)\n    stats_full = train_col.to_frame(\"key\").join(y.rename(\"y\")).groupby(\"key\")[\"y\"].agg([\"mean\",\"count\"])\n    stats_full[\"enc\"] = (stats_full[\"mean\"]*stats_full[\"count\"] + gmean*smoothing) / (stats_full[\"count\"]+smoothing)\n    val_enc = val_col.map(stats_full[\"enc\"]).fillna(gmean)\n    return oof_enc, val_enc, stats_full[\"enc\"]\n\ndef save_submission(preds, filename):\n    sub = pd.DataFrame({ID_COL: test[ID_COL], TARGET: preds})\n    sub.to_csv(filename, index=False)\n    print(f\"✅ Saved: {filename}\")\n\n# ============================================================\n# (7) Feature Engineering\n# ============================================================\nX = train.drop(columns=[ID_COL, TARGET])\ny = train[TARGET]\nX_test = test.drop(columns=[ID_COL])\n\ncat_cols = [c for c in X.columns if X[c].dtype == \"object\" or str(X[c].dtype) == \"category\"]\nhigh_card_cols = [c for c in cat_cols if X[c].nunique(dropna=True) >= 20]\nlow_card_cols  = [c for c in cat_cols if c not in high_card_cols]\n\n# ============================================================\n# (8) KFold Split\n# ============================================================\nN_SPLITS = 5\nRANDOM_STATE = 42\nkf = StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=RANDOM_STATE)\n\noof_lgb, te_lgb = np.zeros(len(X)), np.zeros(len(X_test))\noof_xgb, te_xgb = np.zeros(len(X)), np.zeros(len(X_test))\noof_sgd, te_sgd = np.zeros(len(X)), np.zeros(len(X_test))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-27T16:58:29.562039Z","iopub.execute_input":"2025-10-27T16:58:29.562678Z","iopub.status.idle":"2025-10-27T16:58:30.117151Z","shell.execute_reply.started":"2025-10-27T16:58:29.562616Z","shell.execute_reply":"2025-10-27T16:58:30.115634Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Machine Learning Models","metadata":{}},{"cell_type":"code","source":"# LightGBM\n# ============================================================\nfor fold, (tr_idx, va_idx) in enumerate(kf.split(X, y), 1):\n    print(f\"\\n========== LightGBM Fold {fold}/{N_SPLITS} ==========\")\n    X_tr, X_va = X.iloc[tr_idx].copy(), X.iloc[va_idx].copy()\n    y_tr, y_va = y.iloc[tr_idx], y.iloc[va_idx]\n    X_te = X_test.copy()\n\n    for c in high_card_cols:\n        oof_enc, va_enc, enc_full = target_encode_oof(X_tr[c], y_tr, X_va[c])\n        X_tr[f\"{c}_te\"] = oof_enc\n        X_va[f\"{c}_te\"] = va_enc\n        X_te[f\"{c}_te\"] = X_te[c].map(enc_full).fillna(y_tr.mean())\n\n    for c in low_card_cols:\n        m = label_fit(X_tr[c])\n        X_tr[c] = label_apply(X_tr[c], m)\n        X_va[c] = label_apply(X_va[c], m)\n        X_te[c] = label_apply(X_te[c], m)\n\n    X_tr.drop(columns=high_card_cols, inplace=True)\n    X_va.drop(columns=high_card_cols, inplace=True)\n    X_te.drop(columns=high_card_cols, inplace=True)\n\n    dtr = lgb.Dataset(X_tr, y_tr)\n    dva = lgb.Dataset(X_va, y_va)\n    params = {\n        \"objective\": \"binary\",\n        \"metric\": \"auc\",\n        \"learning_rate\": 0.04,\n        \"num_leaves\": 31,\n        \"feature_fraction\": 0.85,\n        \"bagging_fraction\": 0.85,\n        \"bagging_freq\": 5,\n        \"lambda_l2\": 2.0,\n        \"seed\": RANDOM_STATE,\n        \"verbosity\": -1,\n    }\n    lgb_model = lgb.train(params, dtr, valid_sets=[dva], num_boost_round=6000,\n                          callbacks=[lgb.early_stopping(300), lgb.log_evaluation(200)])\n    va_lgb = lgb_model.predict(X_va, num_iteration=lgb_model.best_iteration)\n    te_lgb += lgb_model.predict(X_te, num_iteration=lgb_model.best_iteration) / N_SPLITS\n    oof_lgb[va_idx] = va_lgb\n    print(\"LGB AUC:\", roc_auc_score(y_va, va_lgb))\n\nsave_submission(te_lgb, \"submission_lightgbm.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-27T16:58:30.118411Z","iopub.execute_input":"2025-10-27T16:58:30.118654Z","iopub.status.idle":"2025-10-27T17:04:08.972141Z","shell.execute_reply.started":"2025-10-27T16:58:30.118635Z","shell.execute_reply":"2025-10-27T17:04:08.970966Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# XGBoost\n# ============================================================\nfor fold, (tr_idx, va_idx) in enumerate(kf.split(X, y), 1):\n    print(f\"\\n========== XGBoost Fold {fold}/{N_SPLITS} ==========\")\n    X_tr, X_va = X.iloc[tr_idx].copy(), X.iloc[va_idx].copy()\n    y_tr, y_va = y.iloc[tr_idx], y.iloc[va_idx]\n    X_te = X_test.copy()\n\n    for c in high_card_cols:\n        oof_enc, va_enc, enc_full = target_encode_oof(X_tr[c], y_tr, X_va[c])\n        X_tr[f\"{c}_te\"] = oof_enc\n        X_va[f\"{c}_te\"] = va_enc\n        X_te[f\"{c}_te\"] = X_te[c].map(enc_full).fillna(y_tr.mean())\n\n    for c in low_card_cols:\n        m = label_fit(X_tr[c])\n        X_tr[c] = label_apply(X_tr[c], m)\n        X_va[c] = label_apply(X_va[c], m)\n        X_te[c] = label_apply(X_te[c], m)\n\n    X_tr.drop(columns=high_card_cols, inplace=True)\n    X_va.drop(columns=high_card_cols, inplace=True)\n    X_te.drop(columns=high_card_cols, inplace=True)\n\n    model_xgb = XGBClassifier(\n        n_estimators=4000,\n        learning_rate=0.05,\n        max_depth=6,\n        subsample=0.8,\n        colsample_bytree=0.8,\n        reg_lambda=1.0,\n        random_state=RANDOM_STATE,\n        eval_metric=\"auc\",\n        tree_method=\"hist\",\n        n_jobs=-1\n    )\n    model_xgb.fit(X_tr, y_tr, eval_set=[(X_va, y_va)], verbose=False, early_stopping_rounds=300)\n    va_xgb = model_xgb.predict_proba(X_va)[:,1]\n    te_xgb += model_xgb.predict_proba(X_te)[:,1] / N_SPLITS\n    oof_xgb[va_idx] = va_xgb\n    print(\"XGB AUC:\", roc_auc_score(y_va, va_xgb))\n\nsave_submission(te_xgb, \"submission_xgboost.csv\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-27T17:04:08.973304Z","iopub.execute_input":"2025-10-27T17:04:08.973629Z","iopub.status.idle":"2025-10-27T17:09:33.772978Z","shell.execute_reply.started":"2025-10-27T17:04:08.973596Z","shell.execute_reply":"2025-10-27T17:09:33.771505Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# SGD + Calibration\n# ============================================================\nfor fold, (tr_idx, va_idx) in enumerate(kf.split(X, y), 1):\n    print(f\"\\n========== SGD Fold {fold}/{N_SPLITS} ==========\")\n    X_tr, X_va = X.iloc[tr_idx].copy(), X.iloc[va_idx].copy()\n    y_tr, y_va = y.iloc[tr_idx], y.iloc[va_idx]\n    X_te = X_test.copy()\n\n    for c in high_card_cols:\n        oof_enc, va_enc, enc_full = target_encode_oof(X_tr[c], y_tr, X_va[c])\n        X_tr[f\"{c}_te\"] = oof_enc\n        X_va[f\"{c}_te\"] = va_enc\n        X_te[f\"{c}_te\"] = X_te[c].map(enc_full).fillna(y_tr.mean())\n\n    for c in low_card_cols:\n        m = label_fit(X_tr[c])\n        X_tr[c] = label_apply(X_tr[c], m)\n        X_va[c] = label_apply(X_va[c], m)\n        X_te[c] = label_apply(X_te[c], m)\n\n    X_tr.drop(columns=high_card_cols, inplace=True)\n    X_va.drop(columns=high_card_cols, inplace=True)\n    X_te.drop(columns=high_card_cols, inplace=True)\n\n    imp = SimpleImputer(strategy=\"median\")\n    X_tr_imp = imp.fit_transform(X_tr)\n    X_va_imp = imp.transform(X_va)\n    X_te_imp = imp.transform(X_te)\n\n    scaler = StandardScaler()\n    X_tr_s = scaler.fit_transform(X_tr_imp)\n    X_va_s = scaler.transform(X_va_imp)\n    X_te_s = scaler.transform(X_te_imp)\n\n    sgd = SGDClassifier(loss=\"log_loss\", penalty=\"l2\", alpha=1e-4,\n                        max_iter=4000, tol=1e-4, random_state=RANDOM_STATE)\n    cal = CalibratedClassifierCV(sgd, method=\"sigmoid\", cv=3)\n    cal.fit(X_tr_s, y_tr)\n    va_sgd = cal.predict_proba(X_va_s)[:,1]\n    te_sgd += cal.predict_proba(X_te_s)[:,1] / N_SPLITS\n    oof_sgd[va_idx] = va_sgd\n    print(\"SGD AUC:\", roc_auc_score(y_va, va_sgd))\n\nsave_submission(te_sgd, \"submission_sgd.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-27T17:09:33.774179Z","iopub.execute_input":"2025-10-27T17:09:33.774554Z","iopub.status.idle":"2025-10-27T17:10:58.683562Z","shell.execute_reply.started":"2025-10-27T17:09:33.774524Z","shell.execute_reply":"2025-10-27T17:10:58.682579Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Stacking\n# ============================================================\nmeta_train = pd.DataFrame({\"lgb\": oof_lgb, \"xgb\": oof_xgb, \"sgd\": oof_sgd})\nmeta_test  = pd.DataFrame({\"lgb\": te_lgb, \"xgb\": te_xgb, \"sgd\": te_sgd})\nmeta_train[\"lgb_x_xgb\"] = meta_train[\"lgb\"] * meta_train[\"xgb\"]\nmeta_test[\"lgb_x_xgb\"]  = meta_test[\"lgb\"] * meta_test[\"xgb\"]\n\nskf = StratifiedKFold(n_splits=5, shuffle=True, random_state=RANDOM_STATE)\nmeta_oof = np.zeros(len(meta_train))\nmeta_pred = np.zeros(len(meta_test))\n\nfor tr, va in skf.split(meta_train, y):\n    lr = LogisticRegression(max_iter=5000)\n    lr.fit(meta_train.iloc[tr], y.iloc[tr])\n    meta_oof[va] = lr.predict_proba(meta_train.iloc[va])[:,1]\n    meta_pred += lr.predict_proba(meta_test)[:,1] / skf.n_splits\n\nprint(\"Meta OOF AUC:\", roc_auc_score(y, meta_oof))\nsave_submission(meta_pred, \"submission_stacking.csv\")\n\nprint(\"\\n✅ All models completed and submissions saved!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-27T17:10:58.684662Z","iopub.execute_input":"2025-10-27T17:10:58.685005Z","iopub.status.idle":"2025-10-27T17:11:06.335434Z","shell.execute_reply.started":"2025-10-27T17:10:58.684974Z","shell.execute_reply":"2025-10-27T17:11:06.334343Z"}},"outputs":[],"execution_count":null}]}