{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":17000,"databundleVersionId":895292,"sourceType":"competition"}],"dockerImageVersionId":31153,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# (1) Installs \n!pip install -q lightgbm catboost\n\n# (2) Imports & Global Setup\nimport warnings, gc, os\nwarnings.filterwarnings(\"ignore\")\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.linear_model import LogisticRegression\n\nimport lightgbm as lgb\nfrom catboost import CatBoostClassifier\nfrom xgboost import XGBClassifier\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.calibration import CalibratedClassifierCV\nfrom sklearn.preprocessing import StandardScaler\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-10-26T22:48:47.488481Z","iopub.execute_input":"2025-10-26T22:48:47.488955Z","iopub.status.idle":"2025-10-26T22:48:51.661647Z","shell.execute_reply.started":"2025-10-26T22:48:47.488925Z","shell.execute_reply":"2025-10-26T22:48:51.660299Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **Setup & Load**","metadata":{}},{"cell_type":"code","source":"\n# ============== Load ==============\nDATA_DIR = \"/kaggle/input/cat-in-the-dat-ii\"\ntrain = pd.read_csv(os.path.join(DATA_DIR, \"train.csv\"))\ntest  = pd.read_csv(os.path.join(DATA_DIR, \"test.csv\"))\nID_COL, TARGET = \"id\", \"target\"\n\nprint(\"Train:\", train.shape, \" Test:\", test.shape)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-26T22:48:51.663806Z","iopub.execute_input":"2025-10-26T22:48:51.664313Z","iopub.status.idle":"2025-10-26T22:48:56.241821Z","shell.execute_reply.started":"2025-10-26T22:48:51.664274Z","shell.execute_reply":"2025-10-26T22:48:56.240666Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# =============  cleaning  ============= ","metadata":{}},{"cell_type":"code","source":"# (4) Basic cleaning shared by train/test (no leakage)\ndef clean_bound(df, col, lo, hi, fallback=1, as_int=True):\n    if col in df.columns:\n        s = pd.to_numeric(df[col], errors='coerce')\n        s = s.where((s >= lo) & (s <= hi))\n        fill = s.mode().iloc[0] if s.notna().any() else fallback\n        s = s.fillna(fill)\n        if as_int:\n            s = s.round().astype(int)\n        df[col] = s\n\nfor d in (train, test):\n    clean_bound(d, 'month', 1, 12, fallback=1, as_int=True)\n    clean_bound(d, 'day',   1, 7,  fallback=1, as_int=True)\n    d.replace([np.inf, -np.inf], np.nan, inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-26T22:48:57.135094Z","iopub.execute_input":"2025-10-26T22:48:57.135387Z","iopub.status.idle":"2025-10-26T22:48:59.450951Z","shell.execute_reply.started":"2025-10-26T22:48:57.135368Z","shell.execute_reply":"2025-10-26T22:48:59.449765Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# =============  EDA  ============= ","metadata":{}},{"cell_type":"code","source":"## EDA\nsns.set_style(\"whitegrid\")\n\nprint(f\"Train shape: {train.shape} | Test shape: {test.shape}\")\ndisplay(train.head())\n\n# 1) Target distribution\nplt.figure(figsize=(5,3))\nsns.countplot(x='target', data=train)\nplt.title('Target Distribution')\nplt.show()\n\nprint(\"Target ratio:\")\ndisplay(train['target'].value_counts(normalize=True).rename('ratio'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-26T22:49:01.470507Z","iopub.execute_input":"2025-10-26T22:49:01.471755Z","iopub.status.idle":"2025-10-26T22:49:01.709592Z","shell.execute_reply.started":"2025-10-26T22:49:01.471713Z","shell.execute_reply":"2025-10-26T22:49:01.708606Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 2) Column overview by type\nid_col, target_col = 'id', 'target'\nfeature_cols = [c for c in train.columns if c not in [id_col, target_col]]\n\nbin_cols = [c for c in feature_cols if c.startswith('bin')]\nnom_cols = [c for c in feature_cols if c.startswith('nom')]\nord_cols = [c for c in feature_cols if c.startswith('ord')]\nother_cols = [c for c in feature_cols if c not in bin_cols + nom_cols + ord_cols]\n\nprint(f'Binary  ({len(bin_cols)}):', bin_cols[:10], '...')\nprint(f'Nominal ({len(nom_cols)}):', nom_cols[:10], '...')\nprint(f'Ordinal ({len(ord_cols)}):', ord_cols[:10], '...')\nprint(f'Other   ({len(other_cols)}):', other_cols[:10], '...')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-26T22:49:14.071767Z","iopub.execute_input":"2025-10-26T22:49:14.072105Z","iopub.status.idle":"2025-10-26T22:49:14.080792Z","shell.execute_reply.started":"2025-10-26T22:49:14.072079Z","shell.execute_reply":"2025-10-26T22:49:14.079400Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 3) Missing values (Top 20)\nna_ratio = train[feature_cols].isna().mean().sort_values(ascending=False)\nplt.figure(figsize=(8,4))\nsns.barplot(x=na_ratio.values[:20], y=na_ratio.index[:20])\nplt.title('Top Missing-Value Columns (train)')\nplt.xlabel('NaN Ratio')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-26T22:49:18.208430Z","iopub.execute_input":"2025-10-26T22:49:18.208758Z","iopub.status.idle":"2025-10-26T22:49:19.220168Z","shell.execute_reply.started":"2025-10-26T22:49:18.208736Z","shell.execute_reply":"2025-10-26T22:49:19.219196Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 4) Cardinality (unique values per feature)\ncard = train[feature_cols].nunique(dropna=True).sort_values(ascending=False)\nplt.figure(figsize=(8,4))\nsns.barplot(x=card.values[:20], y=card.index[:20])\nplt.title('Top High-Cardinality Columns')\nplt.xlabel('Unique Values')\nplt.tight_layout()\nplt.show()\n\n# Identify high-cardinality vs low-cardinality categorical features\nHIGH_CARD_THRESHOLD = 20\ncat_like_cols = [c for c in feature_cols if (train[c].dtype=='object') or (str(train[c].dtype)=='category')]\nhigh_card_cols = [c for c in cat_like_cols if train[c].nunique(dropna=True) >= HIGH_CARD_THRESHOLD]\nlow_card_cols  = [c for c in cat_like_cols if c not in high_card_cols]\nprint(\"High-card:\", high_card_cols[:10], '...')\nprint(\"Low-card :\", low_card_cols[:10],  '...')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-26T22:49:22.900733Z","iopub.execute_input":"2025-10-26T22:49:22.901020Z","iopub.status.idle":"2025-10-26T22:49:24.731494Z","shell.execute_reply.started":"2025-10-26T22:49:22.901000Z","shell.execute_reply":"2025-10-26T22:49:24.730438Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 7) Time-based analysis (month/day)\nfor col, hi in [('month', 12), ('day', 7)]:\n    if col in train.columns:\n        plt.figure(figsize=(6,3))\n        s = pd.to_numeric(train[col], errors='coerce')\n        sns.barplot(x=s, y=train[target_col], estimator=np.mean, errorbar=None)\n        plt.title(f'{col} vs Target Probability')\n        plt.tight_layout()\n        plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-26T22:49:27.021995Z","iopub.execute_input":"2025-10-26T22:49:27.022476Z","iopub.status.idle":"2025-10-26T22:49:27.622699Z","shell.execute_reply.started":"2025-10-26T22:49:27.022427Z","shell.execute_reply":"2025-10-26T22:49:27.621528Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 8) Correlation heatmap (after quick factorization)\nX_tmp = train[feature_cols].copy()\ny_tmp = train[target_col].copy()\n\n# temporary factorization for categorical features\nfor c in cat_like_cols:\n    X_tmp[c] = pd.factorize(X_tmp[c], sort=True)[0]\n\n# replace NaNs with median for numeric analysis\nnum_cols_tmp = X_tmp.select_dtypes(include=[np.number]).columns\nX_tmp[num_cols_tmp] = X_tmp[num_cols_tmp].fillna(X_tmp[num_cols_tmp].median())\n\ncorr = X_tmp.corrwith(y_tmp).sort_values(ascending=False)\ntop_pos = corr.head(15)\ntop_neg = corr.tail(15)\n\nplt.figure(figsize=(7,5))\ntop_pos.plot(kind='barh'); plt.gca().invert_yaxis()\nplt.title('Top Positive Correlations with Target (EDA-only)')\nplt.tight_layout(); plt.show()\n\nplt.figure(figsize=(7,5))\ntop_neg.plot(kind='barh')\nplt.title('Top Negative Correlations with Target (EDA-only)')\nplt.tight_layout(); plt.show()\n\nprint(\"✅ EDA complete.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-26T22:49:31.229719Z","iopub.execute_input":"2025-10-26T22:49:31.230020Z","iopub.status.idle":"2025-10-26T22:49:33.961325Z","shell.execute_reply.started":"2025-10-26T22:49:31.230000Z","shell.execute_reply":"2025-10-26T22:49:33.960522Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# =============  Label Encode ============= ","metadata":{}},{"cell_type":"code","source":"\n# ============== Helpers ==============\ndef label_fit(series):\n    cats = pd.Index(series.astype(\"string\").fillna(\"__NA__\").unique())\n    return {cat: i for i, cat in enumerate(cats)}\n\ndef label_apply(series, mapping):\n    return series.astype(\"string\").fillna(\"__NA__\").map(mapping).fillna(-1).astype(int)\n\ndef target_encode_oof(train_col, y, val_col, n_splits=5, smoothing=20):\n    gmean = y.mean()\n    oof_enc = np.zeros(len(train_col))\n    kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=RANDOM_STATE)\n    for tr_idx, va_idx in kf.split(train_col, y):\n        tr_c, va_c = train_col.iloc[tr_idx], train_col.iloc[va_idx]\n        tr_y = y.iloc[tr_idx]\n        stats = tr_c.to_frame(\"key\").join(tr_y.rename(\"y\")).groupby(\"key\")[\"y\"].agg([\"mean\",\"count\"])\n        stats[\"enc\"] = (stats[\"mean\"]*stats[\"count\"] + gmean*smoothing) / (stats[\"count\"]+smoothing)\n        oof_enc[va_idx] = va_c.map(stats[\"enc\"]).fillna(gmean)\n    stats_full = train_col.to_frame(\"key\").join(y.rename(\"y\")).groupby(\"key\")[\"y\"].agg([\"mean\",\"count\"])\n    stats_full[\"enc\"] = (stats_full[\"mean\"]*stats_full[\"count\"] + gmean*smoothing) / (stats_full[\"count\"]+smoothing)\n    val_enc = val_col.map(stats_full[\"enc\"]).fillna(gmean)\n    return oof_enc, val_enc, stats_full[\"enc\"]\n\n# ============== Prepare ==============\nX = train.drop(columns=[ID_COL, TARGET])\ny = train[TARGET]\nX_test = test.drop(columns=[ID_COL])\n\ncat_cols = [c for c in X.columns if X[c].dtype == \"object\" or str(X[c].dtype) == \"category\"]\nhigh_card_cols = [c for c in cat_cols if X[c].nunique(dropna=True) >= 20]\nlow_card_cols  = [c for c in cat_cols if c not in high_card_cols]\n\n# ============== Folds ==============\noof_lgb = np.zeros(len(X))\noof_xgb = np.zeros(len(X))\noof_sgd = np.zeros(len(X))\nte_lgb = np.zeros(len(X_test))\nte_xgb = np.zeros(len(X_test))\nte_sgd = np.zeros(len(X_test))\nN_SPLITS = 5 \nRANDOM_STATE = 42\n\nkf = StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=RANDOM_STATE)\n\nfor fold, (tr_idx, va_idx) in enumerate(kf.split(X, y), 1):\n    print(f\"\\n========== Fold {fold}/{N_SPLITS} ==========\")\n    X_tr, X_va = X.iloc[tr_idx].copy(), X.iloc[va_idx].copy()\n    y_tr, y_va = y.iloc[tr_idx], y.iloc[va_idx]\n    X_te = X_test.copy()\n\n    # ===== Target Encoding =====\n    for c in high_card_cols:\n        oof_enc, va_enc, enc_full = target_encode_oof(X_tr[c], y_tr, X_va[c], smoothing=10)\n        X_tr[f\"{c}_te\"] = oof_enc\n        X_va[f\"{c}_te\"] = va_enc\n        X_te[f\"{c}_te\"] = X_te[c].map(enc_full).fillna(y_tr.mean())\n\n    # ===== Label Encode for low-card =====\n    for c in low_card_cols:\n        m = label_fit(X_tr[c])\n        X_tr[c] = label_apply(X_tr[c], m)\n        X_va[c] = label_apply(X_va[c], m)\n        X_te[c] = label_apply(X_te[c], m)\n\n    # حذف النصوص الأصلية (بعد الترميز)\n    X_tr.drop(columns=high_card_cols, inplace=True, errors=\"ignore\")\n    X_va.drop(columns=high_card_cols, inplace=True, errors=\"ignore\")\n    X_te.drop(columns=high_card_cols, inplace=True, errors=\"ignore\")\n\n    # ============= LightGBM =============\n    dtr = lgb.Dataset(X_tr, y_tr)\n    dva = lgb.Dataset(X_va, y_va)\n    params = {\n        \"objective\": \"binary\",\n        \"metric\": \"auc\",\n        \"learning_rate\": 0.04,\n        \"num_leaves\": 31,\n        \"feature_fraction\": 0.85,\n        \"bagging_fraction\": 0.85,\n        \"bagging_freq\": 5,\n        \"lambda_l2\": 2.0,\n        \"seed\": RANDOM_STATE,\n        \"verbosity\": -1,\n    }\n    model_lgb = lgb.train(params, dtr, valid_sets=[dva], num_boost_round=6000,\n                          callbacks=[lgb.early_stopping(300), lgb.log_evaluation(200)])\n    va_lgb = model_lgb.predict(X_va)\n    te_lgb += model_lgb.predict(X_te) / N_SPLITS\n    oof_lgb[va_idx] = va_lgb\n    print(\"LGB AUC:\", roc_auc_score(y_va, va_lgb))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-26T22:50:40.305879Z","iopub.execute_input":"2025-10-26T22:50:40.306538Z","iopub.status.idle":"2025-10-26T22:56:14.641511Z","shell.execute_reply.started":"2025-10-26T22:50:40.306507Z","shell.execute_reply":"2025-10-26T22:56:14.640409Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#     ============= XGBoost =============\n","metadata":{}},{"cell_type":"code","source":"    # ============= XGBoost =============\n    model_xgb = XGBClassifier(\n        n_estimators=4000,\n        learning_rate=0.05,\n        max_depth=6,\n        subsample=0.8,\n        colsample_bytree=0.8,\n        reg_lambda=1.0,\n        random_state=RANDOM_STATE,\n        eval_metric=\"auc\",\n        tree_method=\"hist\",\n        n_jobs=-1\n    )\n    model_xgb.fit(X_tr, y_tr, eval_set=[(X_va, y_va)], verbose=False, early_stopping_rounds=300)\n    va_xgb = model_xgb.predict_proba(X_va)[:,1]\n    te_xgb += model_xgb.predict_proba(X_te)[:,1] / N_SPLITS\n    oof_xgb[va_idx] = va_xgb\n    print(\"XGB AUC:\", roc_auc_score(y_va, va_xgb))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-26T22:56:32.517727Z","iopub.execute_input":"2025-10-26T22:56:32.518050Z","iopub.status.idle":"2025-10-26T22:57:24.940492Z","shell.execute_reply.started":"2025-10-26T22:56:32.518030Z","shell.execute_reply":"2025-10-26T22:57:24.939300Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# =============  SGD  =============","metadata":{}},{"cell_type":"code","source":"# ===== SGD (linear) + calibration — FIX: impute before scaling =====\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.calibration import CalibratedClassifierCV\n\n# 1) Impute (median) للتأكد ما في NaN\nimp = SimpleImputer(strategy=\"median\")\nX_tr_imp = imp.fit_transform(X_tr)   # fit على train-fold فقط\nX_va_imp = imp.transform(X_va)\nX_te_imp = imp.transform(X_te)\n\n# 2) Scale\nscaler = StandardScaler(with_mean=True, with_std=True)\nX_tr_s = scaler.fit_transform(X_tr_imp)\nX_va_s = scaler.transform(X_va_imp)\nX_te_s = scaler.transform(X_te_imp)\n\n# 3) SGD + Calibration\nsgd = SGDClassifier(\n    loss=\"log_loss\",        # احتمالات\n    penalty=\"l2\",\n    alpha=1e-4,\n    max_iter=4000,\n    tol=1e-4,\n    random_state=RANDOM_STATE\n)\ncal = CalibratedClassifierCV(base_estimator=sgd, method=\"sigmoid\", cv=3)\ncal.fit(X_tr_s, y_tr)\n\nva_sgd = cal.predict_proba(X_va_s)[:, 1]\nte_sgd = cal.predict_proba(X_te_s)[:, 1]\n\noof_sgd[va_idx] = va_sgd\nte_sgd += te_sgd / N_SPLITS\n   \nprint(\"SGD (cal) AUC:\", roc_auc_score(y_va, va_sgd))\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-26T22:57:24.942363Z","iopub.execute_input":"2025-10-26T22:57:24.942715Z","iopub.status.idle":"2025-10-26T22:57:35.373647Z","shell.execute_reply.started":"2025-10-26T22:57:24.942694Z","shell.execute_reply":"2025-10-26T22:57:35.372612Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============== Stacking ==============\nmeta_train = pd.DataFrame({\"lgb\": oof_lgb, \"xgb\": oof_xgb, \"sgd\": oof_sgd})\nmeta_test  = pd.DataFrame({\"lgb\": te_lgb, \"xgb\": te_xgb, \"sgd\": te_sgd})\nmeta_train[\"lgb_x_xgb\"] = meta_train[\"lgb\"] * meta_train[\"xgb\"]\nmeta_test[\"lgb_x_xgb\"]  = meta_test[\"lgb\"] * meta_test[\"xgb\"]\n\nskf = StratifiedKFold(n_splits=5, shuffle=True, random_state=RANDOM_STATE)\nmeta_oof = np.zeros(len(meta_train))\nmeta_pred = np.zeros(len(meta_test))\n\nfor tr, va in skf.split(meta_train, y):\n    lr = LogisticRegression(max_iter=5000)\n    lr.fit(meta_train.iloc[tr], y.iloc[tr])\n    meta_oof[va] = lr.predict_proba(meta_train.iloc[va])[:,1]\n    meta_pred += lr.predict_proba(meta_test)[:,1] / skf.n_splits\n\nprint(\"Meta OOF AUC:\", roc_auc_score(y, meta_oof))\n\n# ============== Save ==============\npd.DataFrame({ID_COL: test[ID_COL], TARGET: te_lgb}).to_csv(\"submission_lgb.csv\", index=False)\npd.DataFrame({ID_COL: test[ID_COL], TARGET: te_xgb}).to_csv(\"submission_xgb.csv\", index=False)\npd.DataFrame({ID_COL: test[ID_COL], TARGET: te_sgd}).to_csv(\"submission_sgd.csv\", index=False)\npd.DataFrame({ID_COL: test[ID_COL], TARGET: meta_pred}).to_csv(\"submission_stack.csv\", index=False)\n\nprint(\"\\n✅ Saved: submission_lgb.csv, submission_xgb.csv, submission_sgd.csv, submission_stack.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-26T22:57:44.979706Z","iopub.execute_input":"2025-10-26T22:57:44.980008Z","iopub.status.idle":"2025-10-26T22:57:54.364510Z","shell.execute_reply.started":"2025-10-26T22:57:44.979987Z","shell.execute_reply":"2025-10-26T22:57:54.363451Z"}},"outputs":[],"execution_count":null}]}