{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":131653,"databundleVersionId":15802687}],"dockerImageVersionId":31287,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## 1. Environment Setup & Data Loading","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\n\nimport numpy as np\nimport pandas as pd\nfrom pathlib import Path\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom scipy.stats import rankdata\nfrom scipy.optimize import minimize\nimport lightgbm as lgb\n\ntry:\n    import xgboost as xgb\n    XGB_AVAILABLE = True\nexcept ImportError:\n    XGB_AVAILABLE = False\n    print('XGBoost not installed. Run: pip install xgboost')\n\ntry:\n    from catboost import CatBoostClassifier\n    CB_AVAILABLE = True\nexcept ImportError:\n    CB_AVAILABLE = False\n    print('CatBoost not installed. Run: pip install catboost')\n\ntry:\n    from pytorch_tabnet.tab_model import TabNetClassifier\n    TABNET_AVAILABLE = True\nexcept ImportError:\n    TABNET_AVAILABLE = False\n    print('TabNet not installed. Run: pip install pytorch-tabnet')\n\n# Auto-detect GPU\nHAS_GPU = False\ntry:\n    import torch\n    HAS_GPU = torch.cuda.is_available()\nexcept ImportError:\n    pass\nprint(f'GPU available: {HAS_GPU}')\n\nSEED = 42\nnp.random.seed(SEED)\nN_FOLDS = 5\nSEEDS = [42, 123, 2024]  # Multi-seed for variance reduction\nSUPPLEMENTAL_WEIGHT = 1.0  # Keep full signal from supplemental rows\n\n# Generalization diagnostics are aggregated here from each model cell\nGENERALIZATION = {}\n\n# Colab/Kaggle input path\nINPUT = Path('/kaggle/input/competitions/ai-emergency-medicine-turkiye')\nif not INPUT.exists():\n    INPUT = Path('inputs')\n\nprint('Loading datasets...')\ntrain = pd.read_csv(INPUT / 'train.csv')\ntest  = pd.read_csv(INPUT / 'test.csv')\nsample_sub = pd.read_csv(INPUT / 'sample_submission.csv')\nsupplemental = pd.read_csv(INPUT / 'supplemental_data.csv')\n\nTARGET_COLS = ['adverse_outcome', 'readmission_30d']\n\n# Track source and combine train + supplemental\ntrain['is_supplemental'] = 0\nsupplemental['is_supplemental'] = 1\n\nprint(f'Original train: {train.shape[0]:,} rows')\nprint(f'Supplemental:   {supplemental.shape[0]:,} rows')\ntrain = pd.concat([train, supplemental], ignore_index=True)\nprint(f'Combined train: {train.shape[0]:,} rows')\n\nbefore = len(train)\ntrain = train.dropna(subset=TARGET_COLS).reset_index(drop=True)\nif len(train) < before:\n    print(f'  Dropped {before - len(train)} rows with NaN targets')\n\nprint(f'Train: {train.shape}, Test: {test.shape}')\nfor col in TARGET_COLS:\n    print(f'  {col}: {train[col].mean()*100:.2f}% positive')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-20T14:46:09.236671Z","iopub.execute_input":"2026-03-20T14:46:09.237218Z","iopub.status.idle":"2026-03-20T14:46:35.382964Z","shell.execute_reply.started":"2026-03-20T14:46:09.237189Z","shell.execute_reply":"2026-03-20T14:46:35.382324Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2. Feature Engineering (Leak-Safe Encoding)","metadata":{}},{"cell_type":"code","source":"# Columns to drop (matching baseline)\nDROP_COLS = [\n    'encounter_id',\n    'triage_label',\n    'age_group',\n    'map_mmhg',\n    'pulse_pressure_mmhg',\n    'hematocrit_pct',\n    'anion_gap_meql',\n    'consult_specialty',\n    'primary_diagnosis',\n    'disposition',\n    'pandemic_period',\n    'arrival_day_of_week',\n    'latitude',\n    'longitude',\n]\n\n# Low-cardinality categoricals: label encode\nCAT_LABEL_ENCODE = [\n    'urban_rural', 'hospital_type', 'bed_count_category',\n    'arrival_mode', 'arrival_season', 'insurance_type',\n    'icd10_chapter', 'chapter_name', 'trauma_center_level',\n    'helicopter_ems_available', 'stroke_center',\n]\n\n# High-cardinality categoricals: target encode + frequency encode\nCAT_TARGET_ENCODE = [\n    'city', 'nuts2_region', 'chief_complaint', 'primary_icd10_code',\n    'diagnosis_description',\n]\n\ndef engineer_features(df, is_train=True, encoders=None):\n    \"\"\"Feature engineering with robust categorical handling.\"\"\"\n    df = df.copy()\n\n    # Extract numeric sequence signal from encounter id before dropping raw id\n    if 'encounter_id' in df.columns:\n        df['encounter_num'] = pd.to_numeric(\n            df['encounter_id'].astype(str).str.extract(r'(\\d+)$')[0],\n            errors='coerce'\n        ).fillna(0).astype(np.float32)\n\n    # Drop columns\n    drop_here = [c for c in DROP_COLS if c in df.columns]\n    df.drop(columns=drop_here, inplace=True)\n    \n    # === Derived interaction features (baseline + extras) ===\n    df['age_x_charlson'] = df['age'] * df['charlson_comorbidity_index']\n    df['sofa_x_triage'] = df['sofa_score_approx'] * df['triage_level']\n    df['mews_x_triage'] = df['mews_score'] * df['triage_level']\n    df['lactate_x_sbp'] = df['lactate_mmoll'].fillna(df['lactate_mmoll'].median()) * df['sbp_mmhg']\n    df['troponin_log'] = np.log1p(df['troponin_i_ngml'].fillna(0))\n    df['bnp_log'] = np.log1p(df['bnp_pgl'].fillna(0))\n    df['crp_log'] = np.log1p(df['crp_mgl'])\n    df['creatinine_x_age'] = df['creatinine_mgdl'] * df['age']\n    df['wbc_x_temp'] = df['wbc_k_ul'] * df['temperature_celsius']\n    df['shock_index'] = df['heart_rate_bpm'] / df['sbp_mmhg'].clip(lower=1)\n    df['modified_shock_index'] = df['heart_rate_bpm'] / ((df['sbp_mmhg'] + 2 * df['dbp_mmhg']) / 3).clip(lower=1)\n    df['bmi_age_product'] = df['bmi'] * df['age']\n    df['num_comorbidities'] = df[[c for c in df.columns if c.startswith('comorbidity_')]].sum(axis=1)\n    df['num_medications'] = df[[c for c in df.columns if c.startswith('med_')]].sum(axis=1)\n    df['num_procedures'] = df[[c for c in df.columns if c.startswith('proc_')]].sum(axis=1)\n    df['lab_completeness'] = df[['troponin_i_ngml','bnp_pgl','lactate_mmoll','d_dimer_mgfl',\n                                  'procalcitonin_ngl','albumin_gdl','egfr_ml_min']].notna().sum(axis=1) / 7.0\n    df['spo2_sbp_product'] = df['spo2_pct'] * df['sbp_mmhg']\n    df['gcs_spo2_product'] = df['gcs_total'] * df['spo2_pct']\n    df['respiratory_acidosis_flag'] = ((df['bicarbonate_meql'] < 18) & (df['respiratory_rate_rpm'] > 20)).astype(int)\n    df['hyperglycemia_flag'] = (df['glucose_mgdl'] > 200).astype(int)\n    df['hypoglycemia_flag'] = (df['glucose_mgdl'] < 70).astype(int)\n    df['anemia_flag'] = ((df['hemoglobin_gdl'] < 10) | df['hemoglobin_gdl'].isna()).astype(int)\n    df['aki_flag'] = (df['creatinine_mgdl'] > 2.0).astype(int)\n    df['leukocytosis_flag'] = (df['wbc_k_ul'] > 12).astype(int)\n    df['thrombocytopenia_flag'] = (df['platelets_k_ul'] < 100).astype(int)\n    df['night_weekend_combo'] = df['is_night_shift'] * df['is_weekend']\n    df['ed_los_hours'] = df['ed_los_minutes'] / 60.0\n    \n    # === Additional features for extra signal ===\n    df['sofa_x_charlson'] = df['sofa_score_approx'] * df['charlson_comorbidity_index']\n    df['news2_x_triage'] = df['news2_score'] * df['triage_level']\n    df['lab_completeness_x_sofa'] = df['lab_completeness'] * df['sofa_score_approx']\n    df['age_squared'] = df['age'] ** 2\n    df['lactate_log'] = np.log1p(df['lactate_mmoll'].fillna(0))\n    df['d_dimer_log'] = np.log1p(df['d_dimer_mgfl'].fillna(0))\n    df['procalcitonin_log'] = np.log1p(df['procalcitonin_ngl'].fillna(0))\n    df['severity_composite'] = df['sofa_score_approx'] + df['mews_score'] + df['news2_score']\n    df['hr_rr_ratio'] = df['heart_rate_bpm'] / df['respiratory_rate_rpm'].clip(lower=1)\n    df['spo2_deficit'] = 100 - df['spo2_pct']\n    df['gcs_deficit'] = 15 - df['gcs_total']\n    df['sbp_x_age'] = df['sbp_mmhg'] * df['age']\n    df['total_interventions'] = df['num_comorbidities'] + df['num_medications'] + df['num_procedures']\n\n    # Reference-derived risk/resource features\n    if 'typical_mortality_pct' in df.columns:\n        df['icd_mortality_risk'] = df['typical_mortality_pct'].fillna(df['typical_mortality_pct'].median()) / 100.0\n    if 'typical_los_days' in df.columns:\n        df['icd_los_log'] = np.log1p(df['typical_los_days'].fillna(df['typical_los_days'].median()))\n    if 'total_ed_beds' in df.columns and 'population_estimate' in df.columns:\n        pop = df['population_estimate'].replace(0, np.nan)\n        df['ed_beds_per_100k'] = (df['total_ed_beds'] / pop * 100000).fillna(0)\n    if 'total_icu_beds' in df.columns and 'population_estimate' in df.columns:\n        pop = df['population_estimate'].replace(0, np.nan)\n        df['icu_beds_per_100k'] = (df['total_icu_beds'] / pop * 100000).fillna(0)\n    if 'total_icu_beds' in df.columns and 'total_ed_beds' in df.columns:\n        df['icu_to_ed_ratio'] = df['total_icu_beds'] / (df['total_ed_beds'] + 1)\n    if 'annual_ed_volume' in df.columns and 'total_ed_beds' in df.columns:\n        df['ed_volume_per_bed'] = df['annual_ed_volume'] / (df['total_ed_beds'] + 1)\n\n    # Encode gender as binary\n    if 'gender' in df.columns:\n        df['gender'] = (df['gender'] == 'Male').astype(int)\n    \n    # Label encode low-cardinality categoricals\n    new_encoders = {} if encoders is None else dict(encoders)\n    for col in CAT_LABEL_ENCODE:\n        if col not in df.columns:\n            continue\n        if is_train:\n            le = LabelEncoder()\n            df[col] = le.fit_transform(df[col].astype(str))\n            new_encoders[col] = le\n        else:\n            le = new_encoders[col]\n            known = set(le.classes_)\n            df[col] = df[col].astype(str).apply(lambda x: x if x in known else 'Unknown')\n            if 'Unknown' not in known:\n                le.classes_ = np.append(le.classes_, 'Unknown')\n            df[col] = le.transform(df[col])\n    \n    # Frequency-encode high-card columns; target encoding is added afterward with CV\n    if is_train:\n        new_encoders['high_card_freq'] = {}\n    for col in CAT_TARGET_ENCODE:\n        if col not in df.columns:\n            continue\n        s = df[col].astype(str).fillna('MISSING')\n        if is_train:\n            freq_map = s.value_counts(normalize=True)\n            new_encoders['high_card_freq'][col] = freq_map.to_dict()\n        freq_map = new_encoders['high_card_freq'].get(col, {})\n        df[f'{col}_freq'] = s.map(freq_map).fillna(0.0).astype(np.float32)\n    \n    if is_train:\n        return df, new_encoders\n    return df\n\n\ndef target_encode_cv(train_series, test_series, y, n_folds=5, smooth=50, seed=42, min_count=20):\n    \"\"\"Leak-safe out-of-fold target encoding with smoothing and rare-category handling.\"\"\"\n    global_mean = float(y.mean())\n    tr = train_series.astype(str).fillna('MISSING').copy()\n    te = test_series.astype(str).fillna('MISSING').copy()\n    vc = tr.value_counts()\n    rare = set(vc[vc < min_count].index)\n    tr = tr.where(~tr.isin(rare), '__RARE__')\n    te = te.where(~te.isin(rare), '__RARE__')\n\n    enc_train = pd.Series(np.nan, index=tr.index, dtype=np.float32)\n\n    skf_te = StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=seed)\n    for tr_idx, val_idx in skf_te.split(tr, y):\n        fold_x = tr.iloc[tr_idx]\n        fold_y = y.iloc[tr_idx]\n        stats = pd.DataFrame({'x': fold_x, 'y': fold_y}).groupby('x')['y'].agg(['mean', 'count'])\n        smoothed = (stats['mean'] * stats['count'] + global_mean * smooth) / (stats['count'] + smooth)\n        enc_train.iloc[val_idx] = tr.iloc[val_idx].map(smoothed).astype(np.float32)\n\n    enc_train = enc_train.fillna(global_mean).astype(np.float32)\n\n    full_stats = pd.DataFrame({'x': tr, 'y': y}).groupby('x')['y'].agg(['mean', 'count'])\n    full_smoothed = (full_stats['mean'] * full_stats['count'] + global_mean * smooth) / (full_stats['count'] + smooth)\n    enc_test = te.map(full_smoothed).fillna(global_mean).astype(np.float32)\n    return enc_train, enc_test\n\n\nprint('Merging reference tables...')\nhosp_ref = pd.read_csv(INPUT / 'hospital_reference.csv')\nhosp_cols = [\n    'city', 'nuts2_region', 'population_estimate',\n    'num_state_hospitals', 'num_university_hospitals', 'num_private_hospitals',\n    'num_city_hospitals', 'num_training_research_hospitals',\n    'total_icu_beds', 'total_ed_beds',\n    'helicopter_ems_available', 'stroke_center', 'trauma_center_level',\n]\nhosp_ref = hosp_ref[hosp_cols].drop_duplicates(['city', 'nuts2_region'])\n\nicd_ref = pd.read_csv(INPUT / 'icd10_reference.csv').rename(columns={'icd10_code': 'primary_icd10_code'})\nicd_cols = ['primary_icd10_code', 'diagnosis_description', 'icd10_chapter', 'chapter_name', 'typical_los_days', 'typical_mortality_pct']\nicd_ref = icd_ref[icd_cols].drop_duplicates('primary_icd10_code')\n\ntrain = train.merge(hosp_ref, on=['city', 'nuts2_region'], how='left')\ntest = test.merge(hosp_ref, on=['city', 'nuts2_region'], how='left')\ntrain = train.merge(icd_ref, on='primary_icd10_code', how='left')\ntest = test.merge(icd_ref, on='primary_icd10_code', how='left')\nprint('Reference merge done.')\n\nprint('Engineering features...')\nX_train, encoders = engineer_features(train, is_train=True)\ny1 = X_train.pop('adverse_outcome')\ny2 = X_train.pop('readmission_30d')\nstrata = y1.astype(str) + '_' + y2.astype(str)\ntrain_sample_weight = np.where(train['is_supplemental'].values == 1, SUPPLEMENTAL_WEIGHT, 1.0).astype(np.float32)\nX_train.drop(columns=[c for c in TARGET_COLS if c in X_train.columns], inplace=True, errors='ignore')\n\nX_test = engineer_features(test, is_train=False, encoders=encoders)\n\nprint('Applying target encoding...')\nfor col in CAT_TARGET_ENCODE:\n    if col not in X_train.columns or col not in X_test.columns:\n        continue\n    tr_col = X_train[col]\n    te_col = X_test[col]\n    adv_tr, adv_te = target_encode_cv(tr_col, te_col, y1, n_folds=N_FOLDS, smooth=50, seed=SEED, min_count=20)\n    read_tr, read_te = target_encode_cv(tr_col, te_col, y2, n_folds=N_FOLDS, smooth=50, seed=SEED, min_count=20)\n    X_train[f'{col}_te_adv'] = adv_tr\n    X_test[f'{col}_te_adv'] = adv_te\n    X_train[f'{col}_te_read'] = read_tr\n    X_test[f'{col}_te_read'] = read_te\n    print(f'  Target-encoded: {col}')\n\n# Drop original high-cardinality raw strings after encoding\ndrop_high_card = [c for c in CAT_TARGET_ENCODE if c in X_train.columns]\nX_train.drop(columns=drop_high_card, inplace=True)\nX_test.drop(columns=[c for c in drop_high_card if c in X_test.columns], inplace=True)\n\n# Build target-specific matrices to reduce cross-target noise\nadv_drop = [c for c in X_train.columns if c.endswith('_te_read')]\nread_drop = [c for c in X_train.columns if c.endswith('_te_adv')]\nX_train_adv = X_train.drop(columns=adv_drop)\nX_test_adv = X_test.drop(columns=[c for c in adv_drop if c in X_test.columns])\nX_train_read = X_train.drop(columns=read_drop)\nX_test_read = X_test.drop(columns=[c for c in read_drop if c in X_test.columns])\n\n# Align columns (safety)\nshared_adv = [c for c in X_train_adv.columns if c in X_test_adv.columns]\nX_train_adv = X_train_adv[shared_adv]\nX_test_adv = X_test_adv[shared_adv]\nshared_read = [c for c in X_train_read.columns if c in X_test_read.columns]\nX_train_read = X_train_read[shared_read]\nX_test_read = X_test_read[shared_read]\n\n# Additional method data: CatBoost native categorical matrices\ncb_drop_cols = [c for c in DROP_COLS + TARGET_COLS + ['is_supplemental'] if c in train.columns]\nX_train_cb = train.drop(columns=cb_drop_cols).copy()\nX_test_cb = test.drop(columns=[c for c in DROP_COLS if c in test.columns]).copy()\nshared_cb = [c for c in X_train_cb.columns if c in X_test_cb.columns]\nX_train_cb = X_train_cb[shared_cb]\nX_test_cb = X_test_cb[shared_cb]\n\ncb_cat_cols = [c for c in X_train_cb.columns if X_train_cb[c].dtype == 'object']\nfor c in cb_cat_cols:\n    X_train_cb[c] = X_train_cb[c].astype(str).fillna('MISSING')\n    X_test_cb[c] = X_test_cb[c].astype(str).fillna('MISSING')\ncb_cat_idx = [X_train_cb.columns.get_loc(c) for c in cb_cat_cols]\n\nprint(f'Feature matrix adverse:    {X_train_adv.shape}')\nprint(f'Feature matrix readmit:    {X_train_read.shape}')\nprint(f'Test matrix adverse:       {X_test_adv.shape}')\nprint(f'Test matrix readmit:       {X_test_read.shape}')\nprint(f'CatBoost native matrix:    {X_train_cb.shape}, cats={len(cb_cat_idx)}')\nprint(f'Avg sample weight: {train_sample_weight.mean():.4f}')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3. LightGBM — Multi-Seed 5-Fold CV","metadata":{}},{"cell_type":"code","source":"# LightGBM - lower LR + more trees + multi-seed for variance reduction\nLGBM_PARAMS_ADV = {\n    'objective': 'binary',\n    'metric': 'auc',\n    'learning_rate': 0.02,\n    'num_leaves': 127,\n    'max_depth': -1,\n    'min_child_samples': 50,\n    'subsample': 0.8,\n    'subsample_freq': 5,\n    'colsample_bytree': 0.7,\n    'reg_alpha': 0.1,\n    'reg_lambda': 1.0,\n    'n_estimators': 1500,\n    'random_state': SEED,\n    'verbose': -1,\n    'n_jobs': -1,\n}\n\n# Slightly different params for readmission (harder target, more imbalanced)\nLGBM_PARAMS_READ = {\n    **LGBM_PARAMS_ADV,\n    'num_leaves': 63,\n    'min_child_samples': 80,\n    'colsample_bytree': 0.6,\n    'reg_alpha': 0.5,\n    'reg_lambda': 3.0,\n    'scale_pos_weight': 6.0,\n}\n\nlgb_oof1 = np.zeros(len(X_train_adv))\nlgb_oof2 = np.zeros(len(X_train_read))\nlgb_test1 = np.zeros(len(X_test_adv))\nlgb_test2 = np.zeros(len(X_test_read))\nlgb_fold_auc1 = []\nlgb_fold_auc2 = []\n\nprint('=== LightGBM (Multi-Seed) ===')\nfor si, seed in enumerate(SEEDS):\n    print(f'\\n--- Seed {seed} ({si+1}/{len(SEEDS)}) ---')\n    skf = StratifiedKFold(n_splits=N_FOLDS, shuffle=True, random_state=seed)\n\n    seed_oof1 = np.zeros(len(X_train_adv))\n    seed_oof2 = np.zeros(len(X_train_read))\n    seed_test1 = np.zeros(len(X_test_adv))\n    seed_test2 = np.zeros(len(X_test_read))\n\n    params_adv = {**LGBM_PARAMS_ADV, 'random_state': seed}\n    params_read = {**LGBM_PARAMS_READ, 'random_state': seed}\n\n    for fold, (tr_idx, val_idx) in enumerate(skf.split(X_train_adv, strata)):\n        X_tr_adv, X_val_adv = X_train_adv.iloc[tr_idx], X_train_adv.iloc[val_idx]\n        X_tr_read, X_val_read = X_train_read.iloc[tr_idx], X_train_read.iloc[val_idx]\n        w_tr = train_sample_weight[tr_idx]\n\n        # Target 1: adverse_outcome\n        m1 = lgb.LGBMClassifier(**params_adv)\n        m1.fit(X_tr_adv, y1.iloc[tr_idx],\n               sample_weight=w_tr,\n               eval_set=[(X_val_adv, y1.iloc[val_idx])],\n               callbacks=[lgb.early_stopping(100, verbose=False), lgb.log_evaluation(-1)])\n        seed_oof1[val_idx] = m1.predict_proba(X_val_adv)[:, 1]\n        seed_test1 += m1.predict_proba(X_test_adv)[:, 1] / N_FOLDS\n\n        # Target 2: readmission_30d\n        m2 = lgb.LGBMClassifier(**params_read)\n        m2.fit(X_tr_read, y2.iloc[tr_idx],\n               sample_weight=w_tr,\n               eval_set=[(X_val_read, y2.iloc[val_idx])],\n               callbacks=[lgb.early_stopping(100, verbose=False), lgb.log_evaluation(-1)])\n        seed_oof2[val_idx] = m2.predict_proba(X_val_read)[:, 1]\n        seed_test2 += m2.predict_proba(X_test_read)[:, 1] / N_FOLDS\n\n        a1 = roc_auc_score(y1.iloc[val_idx], seed_oof1[val_idx])\n        a2 = roc_auc_score(y2.iloc[val_idx], seed_oof2[val_idx])\n        lgb_fold_auc1.append(a1)\n        lgb_fold_auc2.append(a2)\n        print(f'  Fold {fold+1} | adverse: {a1:.5f} ({m1.best_iteration_} trees) | readmission: {a2:.5f} ({m2.best_iteration_} trees)')\n\n    lgb_oof1 += seed_oof1 / len(SEEDS)\n    lgb_oof2 += seed_oof2 / len(SEEDS)\n    lgb_test1 += seed_test1 / len(SEEDS)\n    lgb_test2 += seed_test2 / len(SEEDS)\n\nlgb_auc1 = roc_auc_score(y1, lgb_oof1)\nlgb_auc2 = roc_auc_score(y2, lgb_oof2)\nlgb_macro = (lgb_auc1 + lgb_auc2) / 2\nprint(f'\\n  LGB OOF: adverse={lgb_auc1:.5f}, readmission={lgb_auc2:.5f}, macro={lgb_macro:.5f}')\nprint(f'  LGB fold std: adverse={np.std(lgb_fold_auc1):.5f}, readmission={np.std(lgb_fold_auc2):.5f}')\n\nGENERALIZATION['LGB'] = {\n    'adverse_mean': float(np.mean(lgb_fold_auc1)),\n    'adverse_std': float(np.std(lgb_fold_auc1)),\n    'readmission_mean': float(np.mean(lgb_fold_auc2)),\n    'readmission_std': float(np.std(lgb_fold_auc2)),\n    'macro_oof': float(lgb_macro),\n}","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4. XGBoost — 5-Fold CV","metadata":{}},{"cell_type":"code","source":"if XGB_AVAILABLE:\n    XGB_PARAMS_ADV = {\n        'objective': 'binary:logistic',\n        'eval_metric': 'auc',\n        'learning_rate': 0.02,\n        'max_depth': 7,\n        'min_child_weight': 50,\n        'subsample': 0.8,\n        'colsample_bytree': 0.7,\n        'reg_alpha': 0.1,\n        'reg_lambda': 1.0,\n        'n_estimators': 1500,\n        'early_stopping_rounds': 100,\n        'tree_method': 'hist',\n        'device': 'cuda' if HAS_GPU else 'cpu',\n        'random_state': SEED,\n        'verbosity': 0,\n        'n_jobs': -1,\n    }\n\n    XGB_PARAMS_READ = {\n        **XGB_PARAMS_ADV,\n        'max_depth': 6,\n        'min_child_weight': 80,\n        'colsample_bytree': 0.6,\n        'reg_alpha': 0.5,\n        'reg_lambda': 3.0,\n        'scale_pos_weight': 6.0,\n    }\n\n    xgb_oof1 = np.zeros(len(X_train_adv))\n    xgb_oof2 = np.zeros(len(X_train_read))\n    xgb_test1 = np.zeros(len(X_test_adv))\n    xgb_test2 = np.zeros(len(X_test_read))\n    xgb_fold_auc1 = []\n    xgb_fold_auc2 = []\n\n    print('=== XGBoost ===')\n    skf = StratifiedKFold(n_splits=N_FOLDS, shuffle=True, random_state=SEED)\n    for fold, (tr_idx, val_idx) in enumerate(skf.split(X_train_adv, strata)):\n        X_tr_adv, X_val_adv = X_train_adv.iloc[tr_idx], X_train_adv.iloc[val_idx]\n        X_tr_read, X_val_read = X_train_read.iloc[tr_idx], X_train_read.iloc[val_idx]\n        w_tr = train_sample_weight[tr_idx]\n\n        m1 = xgb.XGBClassifier(**XGB_PARAMS_ADV)\n        m1.fit(X_tr_adv, y1.iloc[tr_idx], eval_set=[(X_val_adv, y1.iloc[val_idx])], sample_weight=w_tr, verbose=False)\n        xgb_oof1[val_idx] = m1.predict_proba(X_val_adv)[:, 1]\n        xgb_test1 += m1.predict_proba(X_test_adv)[:, 1] / N_FOLDS\n\n        m2 = xgb.XGBClassifier(**XGB_PARAMS_READ)\n        m2.fit(X_tr_read, y2.iloc[tr_idx], eval_set=[(X_val_read, y2.iloc[val_idx])], sample_weight=w_tr, verbose=False)\n        xgb_oof2[val_idx] = m2.predict_proba(X_val_read)[:, 1]\n        xgb_test2 += m2.predict_proba(X_test_read)[:, 1] / N_FOLDS\n\n        a1 = roc_auc_score(y1.iloc[val_idx], xgb_oof1[val_idx])\n        a2 = roc_auc_score(y2.iloc[val_idx], xgb_oof2[val_idx])\n        xgb_fold_auc1.append(a1)\n        xgb_fold_auc2.append(a2)\n        print(f'  Fold {fold+1} | adverse: {a1:.5f} | readmission: {a2:.5f}')\n\n    xgb_auc1 = roc_auc_score(y1, xgb_oof1)\n    xgb_auc2 = roc_auc_score(y2, xgb_oof2)\n    xgb_macro = (xgb_auc1 + xgb_auc2) / 2\n    print(f'\\n  XGB OOF: adverse={xgb_auc1:.5f}, readmission={xgb_auc2:.5f}, macro={xgb_macro:.5f}')\n    print(f'  XGB fold std: adverse={np.std(xgb_fold_auc1):.5f}, readmission={np.std(xgb_fold_auc2):.5f}')\n\n    GENERALIZATION['XGB'] = {\n        'adverse_mean': float(np.mean(xgb_fold_auc1)),\n        'adverse_std': float(np.std(xgb_fold_auc1)),\n        'readmission_mean': float(np.mean(xgb_fold_auc2)),\n        'readmission_std': float(np.std(xgb_fold_auc2)),\n        'macro_oof': float(xgb_macro),\n    }\nelse:\n    print('XGBoost not available - skipping')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5. CatBoost — 5-Fold CV","metadata":{}},{"cell_type":"code","source":"if CB_AVAILABLE:\n    CB_PARAMS_ADV = {\n        'iterations': 1500,\n        'learning_rate': 0.02,\n        'depth': 7,\n        'l2_leaf_reg': 1.0,\n        'bootstrap_type': 'Bernoulli',\n        'subsample': 0.8,\n        'random_seed': SEED,\n        'verbose': 0,\n        'eval_metric': 'AUC',\n        'early_stopping_rounds': 100,\n        'task_type': 'GPU' if HAS_GPU else 'CPU',\n    }\n    \n    CB_PARAMS_READ = {\n        **CB_PARAMS_ADV,\n        'depth': 6,\n        'l2_leaf_reg': 3.0,\n        'auto_class_weights': 'Balanced',\n    }\n    \n    cb_oof1 = np.zeros(len(X_train_adv))\n    cb_oof2 = np.zeros(len(X_train_read))\n    cb_test1 = np.zeros(len(X_test_adv))\n    cb_test2 = np.zeros(len(X_test_read))\n    \n    print('=== CatBoost (engineered, multi-seed) ===')\n    for si, seed in enumerate(SEEDS):\n        print(f'--- Seed {seed} ({si+1}/{len(SEEDS)}) ---')\n        skf = StratifiedKFold(n_splits=N_FOLDS, shuffle=True, random_state=seed)\n        seed_oof1 = np.zeros(len(X_train_adv))\n        seed_oof2 = np.zeros(len(X_train_read))\n        seed_test1 = np.zeros(len(X_test_adv))\n        seed_test2 = np.zeros(len(X_test_read))\n\n        params_adv = {**CB_PARAMS_ADV, 'random_seed': seed}\n        params_read = {**CB_PARAMS_READ, 'random_seed': seed}\n        for fold, (tr_idx, val_idx) in enumerate(skf.split(X_train_adv, strata)):\n            X_tr_adv, X_val_adv = X_train_adv.iloc[tr_idx], X_train_adv.iloc[val_idx]\n            X_tr_read, X_val_read = X_train_read.iloc[tr_idx], X_train_read.iloc[val_idx]\n            w_tr = train_sample_weight[tr_idx]\n            \n            m1 = CatBoostClassifier(**params_adv)\n            m1.fit(X_tr_adv, y1.iloc[tr_idx], eval_set=(X_val_adv, y1.iloc[val_idx]), sample_weight=w_tr)\n            seed_oof1[val_idx] = m1.predict_proba(X_val_adv)[:, 1]\n            seed_test1 += m1.predict_proba(X_test_adv)[:, 1] / N_FOLDS\n            \n            m2 = CatBoostClassifier(**params_read)\n            m2.fit(X_tr_read, y2.iloc[tr_idx], eval_set=(X_val_read, y2.iloc[val_idx]), sample_weight=w_tr)\n            seed_oof2[val_idx] = m2.predict_proba(X_val_read)[:, 1]\n            seed_test2 += m2.predict_proba(X_test_read)[:, 1] / N_FOLDS\n            \n            a1 = roc_auc_score(y1.iloc[val_idx], seed_oof1[val_idx])\n            a2 = roc_auc_score(y2.iloc[val_idx], seed_oof2[val_idx])\n            print(f'  Fold {fold+1} | adverse: {a1:.5f} | readmission: {a2:.5f}')\n\n        cb_oof1 += seed_oof1 / len(SEEDS)\n        cb_oof2 += seed_oof2 / len(SEEDS)\n        cb_test1 += seed_test1 / len(SEEDS)\n        cb_test2 += seed_test2 / len(SEEDS)\n    \n    cb_auc1 = roc_auc_score(y1, cb_oof1)\n    cb_auc2 = roc_auc_score(y2, cb_oof2)\n    cb_macro = (cb_auc1 + cb_auc2) / 2\n    print(f'  Engineered CB OOF: adverse={cb_auc1:.5f}, readmission={cb_auc2:.5f}, macro={cb_macro:.5f}')\n\n    # Native categorical CatBoost path\n    CBN_PARAMS_ADV = {\n        'iterations': 2200,\n        'learning_rate': 0.02,\n        'depth': 8,\n        'l2_leaf_reg': 3.0,\n        'loss_function': 'Logloss',\n        'eval_metric': 'AUC',\n        'bootstrap_type': 'Bernoulli',\n        'subsample': 0.8,\n        'random_seed': SEED,\n        'verbose': 0,\n        'early_stopping_rounds': 150,\n        'task_type': 'GPU' if HAS_GPU else 'CPU',\n    }\n    CBN_PARAMS_READ = {\n        **CBN_PARAMS_ADV,\n        'depth': 7,\n        'l2_leaf_reg': 5.0,\n        'auto_class_weights': 'Balanced',\n    }\n\n    cbn_oof1 = np.zeros(len(X_train_cb))\n    cbn_oof2 = np.zeros(len(X_train_cb))\n    cbn_test1 = np.zeros(len(X_test_cb))\n    cbn_test2 = np.zeros(len(X_test_cb))\n\n    print('=== CatBoost (native categorical, multi-seed) ===')\n    for si, seed in enumerate(SEEDS):\n        print(f'--- Seed {seed} ({si+1}/{len(SEEDS)}) ---')\n        skf_native = StratifiedKFold(n_splits=N_FOLDS, shuffle=True, random_state=seed)\n        seed_oof1 = np.zeros(len(X_train_cb))\n        seed_oof2 = np.zeros(len(X_train_cb))\n        seed_test1 = np.zeros(len(X_test_cb))\n        seed_test2 = np.zeros(len(X_test_cb))\n\n        params_adv_n = {**CBN_PARAMS_ADV, 'random_seed': seed}\n        params_read_n = {**CBN_PARAMS_READ, 'random_seed': seed}\n        for fold, (tr_idx, val_idx) in enumerate(skf_native.split(X_train_cb, strata)):\n            Xtr, Xva = X_train_cb.iloc[tr_idx], X_train_cb.iloc[val_idx]\n            w_tr = train_sample_weight[tr_idx]\n\n            m1n = CatBoostClassifier(**params_adv_n)\n            m1n.fit(Xtr, y1.iloc[tr_idx],\n                   cat_features=cb_cat_idx,\n                   sample_weight=w_tr,\n                   eval_set=(Xva, y1.iloc[val_idx]))\n            seed_oof1[val_idx] = m1n.predict_proba(Xva)[:, 1]\n            seed_test1 += m1n.predict_proba(X_test_cb)[:, 1] / N_FOLDS\n\n            m2n = CatBoostClassifier(**params_read_n)\n            m2n.fit(Xtr, y2.iloc[tr_idx],\n                   cat_features=cb_cat_idx,\n                   sample_weight=w_tr,\n                   eval_set=(Xva, y2.iloc[val_idx]))\n            seed_oof2[val_idx] = m2n.predict_proba(Xva)[:, 1]\n            seed_test2 += m2n.predict_proba(X_test_cb)[:, 1] / N_FOLDS\n\n            a1n = roc_auc_score(y1.iloc[val_idx], seed_oof1[val_idx])\n            a2n = roc_auc_score(y2.iloc[val_idx], seed_oof2[val_idx])\n            print(f'  Fold {fold+1} | adverse: {a1n:.5f} | readmission: {a2n:.5f}')\n\n        cbn_oof1 += seed_oof1 / len(SEEDS)\n        cbn_oof2 += seed_oof2 / len(SEEDS)\n        cbn_test1 += seed_test1 / len(SEEDS)\n        cbn_test2 += seed_test2 / len(SEEDS)\n\n    cbn_auc1 = roc_auc_score(y1, cbn_oof1)\n    cbn_auc2 = roc_auc_score(y2, cbn_oof2)\n    cbn_macro = (cbn_auc1 + cbn_auc2) / 2\n    print(f'  Native CB OOF: adverse={cbn_auc1:.5f}, readmission={cbn_auc2:.5f}, macro={cbn_macro:.5f}')\nelse:\n    print('CatBoost not available — skipping')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 6. Ensemble — Stable AUC-Weighted Blending","metadata":{}},{"cell_type":"code","source":"def to_rank(arr):\n    \"\"\"Convert predictions to rank-based (uniform) for robust blending.\"\"\"\n    from scipy.stats import rankdata\n    return rankdata(arr) / len(arr)\n\ndef _softmax(x):\n    z = x - np.max(x)\n    e = np.exp(z)\n    return e / np.sum(e)\n\ndef optimize_weights_nm(oof_matrix, y_true):\n    from scipy.optimize import minimize\n    def objective(raw_w):\n        return -roc_auc_score(y_true, oof_matrix @ _softmax(raw_w))\n    res = minimize(objective, np.zeros(oof_matrix.shape[1]), method=\"Nelder-Mead\", options={\"maxiter\": 2000, \"xatol\": 1e-8, \"fatol\": 1e-8})\n    return _softmax(res.x), -res.fun\n\ndef auc_weighted_blend(oof_list, y_true, floor=0.001, power=3.0):\n    \"\"\"Derive blend weights from individual OOF AUC values.\"\"\"\n    aucs = np.array([roc_auc_score(y_true, oof) for oof in oof_list])\n    skill = np.clip(aucs - 0.5, floor, None) ** power\n    weights = skill / skill.sum()\n    return aucs, weights\n\ndef choose_best_prediction(oof_list, test_list, names, y_true, target_name):\n    \"\"\"Pick best method (weighted raw/rank, top2 weighted, or best single) for one target.\"\"\"\n    rank_oof = [to_rank(o) for o in oof_list]\n    rank_test = [to_rank(t) for t in test_list]\n\n    aucs_raw, w_raw = auc_weighted_blend(oof_list, y_true)\n    _, w_rank = auc_weighted_blend(rank_oof, y_true)\n\n    print(f\"\\nPer-model OOF AUC ({target_name}):\")\n    for n, a in zip(names, aucs_raw):\n        print(f\"  {n}: {a:.5f}\")\n    print(f\"Raw weights ({target_name}):  {dict(zip(names, [round(float(x), 4) for x in w_raw]))}\")\n    print(f\"Rank weights ({target_name}): {dict(zip(names, [round(float(x), 4) for x in w_rank]))}\")\n\n    candidates = []\n    \n    # Existing heuristics\n    oof_raw = np.column_stack(oof_list) @ w_raw\n    test_raw = np.column_stack(test_list) @ w_raw\n    candidates.append(('WEIGHTED_RAW', oof_raw, test_raw))\n\n    oof_rank = np.column_stack(rank_oof) @ w_rank\n    test_rank = np.column_stack(rank_test) @ w_rank\n    candidates.append(('WEIGHTED_RANK', oof_rank, test_rank))\n\n    # Top-2 only blend (often better than using weak models)\n    if len(oof_list) >= 2:\n        top2_idx = np.argsort(aucs_raw)[-2:]\n        top2_skill = np.clip(aucs_raw[top2_idx] - 0.5, 1e-6, None) ** 4\n        top2_w = top2_skill / top2_skill.sum()\n        top2_oof = np.column_stack([oof_list[i] for i in top2_idx]) @ top2_w\n        top2_test = np.column_stack([test_list[i] for i in top2_idx]) @ top2_w\n        top2_names = '+'.join([names[i] for i in top2_idx])\n        candidates.append((f'TOP2({top2_names})', top2_oof, top2_test))\n    \n    # Add Nelder-Mead Optimization\n    nm_oof_matrix = np.column_stack(oof_list)\n    nm_test_matrix = np.column_stack(test_list)\n    nm_w, nm_auc = optimize_weights_nm(nm_oof_matrix, y_true)\n    nm_oof = nm_oof_matrix @ nm_w\n    nm_test = nm_test_matrix @ nm_w\n    candidates.append(('NELDER_MEAD_OPT', nm_oof, nm_test))\n    print(f\"Nelder-Mead weights ({target_name}): {dict(zip(names, [round(float(x), 4) for x in nm_w]))}\")\n\n    best_single_idx = int(np.argmax(aucs_raw))\n    candidates.append((f\"SINGLE({names[best_single_idx]})\", oof_list[best_single_idx], test_list[best_single_idx]))\n\n    best = None\n    for method, oof_pred, test_pred in candidates:\n        auc = roc_auc_score(y_true, oof_pred)\n        print(f\"  {target_name} {method}: AUC={auc:.5f}\")\n        if best is None or auc > best[0]:\n            best = (auc, method, oof_pred, test_pred)\n\n    return {\n        'auc': best[0],\n        'method': best[1],\n        'oof': best[2],\n        'test': best[3],\n    }\n\n# Build base model pools\nmodels_oof1 = [lgb_oof1]\nmodels_test1 = [lgb_test1]\nnames1 = ['LGB']\n\nmodels_oof2 = [lgb_oof2]\nmodels_test2 = [lgb_test2]\nnames2 = ['LGB']\n\nif XGB_AVAILABLE:\n    models_oof1.append(xgb_oof1); models_test1.append(xgb_test1); names1.append('XGB')\n    models_oof2.append(xgb_oof2); models_test2.append(xgb_test2); names2.append('XGB')\n\nif CB_AVAILABLE:\n    models_oof1.append(cb_oof1); models_test1.append(cb_test1); names1.append('CB_ENG')\n    models_oof2.append(cb_oof2); models_test2.append(cb_test2); names2.append('CB_ENG')\n    if 'cbn_oof1' in globals():\n        models_oof1.append(cbn_oof1); models_test1.append(cbn_test1); names1.append('CB_NATIVE')\n    if 'cbn_oof2' in globals():\n        models_oof2.append(cbn_oof2); models_test2.append(cbn_test2); names2.append('CB_NATIVE')\n\nprint(f'Adverse pool: {names1}')\nprint(f'Readmission pool: {names2}')\n\n# Choose best method per target independently\nbest_adv = choose_best_prediction(models_oof1, models_test1, names1, y1, 'adverse')\nbest_read = choose_best_prediction(models_oof2, models_test2, names2, y2, 'readmission')\n\nfinal_pred1_source = best_adv['test']\nfinal_pred2_source = best_read['test']\nbest_oof1 = best_adv['oof']\nbest_oof2 = best_read['oof']\n\nauc1 = best_adv['auc']\nauc2 = best_read['auc']\nblend_macro = (auc1 + auc2) / 2\nblend_method = f\"ADV:{best_adv['method']} | READ:{best_read['method']}\"\n\nprint(f\"\\nUsing {blend_method}\")\nprint(f\"  adverse AUC:     {auc1:.5f}\")\nprint(f\"  readmission AUC: {auc2:.5f}\")\nprint(f\"  macro AUC:       {blend_macro:.5f}\")\n\n# Keep compatibility flags for submission cell\nUSE_RANK = False\nw1 = np.array([1.0])\nw2 = np.array([1.0])\nbest_single_idx = (0, 0)\n\nprint(f'\\n=== INDIVIDUAL MODEL COMPARISON ===')\nprint(f'  LGB Macro: {lgb_macro:.5f}')\nif XGB_AVAILABLE: print(f'  XGB Macro: {xgb_macro:.5f}')\nif CB_AVAILABLE: print(f'  CB_ENG Macro: {cb_macro:.5f}')\nif CB_AVAILABLE and 'cbn_macro' in globals(): print(f'  CB_NATIVE Macro: {cbn_macro:.5f}')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 7. Generate Submission","metadata":{}},{"cell_type":"code","source":"# final_pred*_source are prepared in the ensemble cell\nfinal_pred1 = np.clip(final_pred1_source, 0.001, 0.999)\nfinal_pred2 = np.clip(final_pred2_source, 0.001, 0.999)\n\nsubmission = sample_sub.copy()\nsubmission['adverse_outcome'] = final_pred1\nsubmission['readmission_30d'] = final_pred2\n\nassert submission.shape[0] == 200000, f'Expected 200k rows, got {submission.shape[0]}'\nassert not submission.isna().any().any(), 'Submission has NaN!'\n\nsubmission.to_csv('submission_scipy_finley_optimize.csv', index=False)\n\nprint('SUBMISSION SAVED!')\nprint(f'  Method: {blend_method}')\nprint(f'  Shape: {submission.shape}')\nprint(f'  adverse_outcome  — min: {final_pred1.min():.4f}, max: {final_pred1.max():.4f}, mean: {final_pred1.mean():.4f}')\nprint(f'  readmission_30d  — min: {final_pred2.min():.4f}, max: {final_pred2.max():.4f}, mean: {final_pred2.mean():.4f}')\nprint(f'\\nExpected LB Macro AUC (OOF): {blend_macro:.5f}')\ndisplay(submission.head())","metadata":{},"outputs":[],"execution_count":null}]}