{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":103436,"databundleVersionId":12476472,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom catboost import CatBoostClassifier, Pool\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score, precision_score, recall_score, f1_score, accuracy_score, log_loss\nfrom sklearn.preprocessing import MinMaxScaler\nimport gc\nimport os\nfrom typing import List, Tuple, Dict\nimport logging\nfrom datetime import datetime\nimport subprocess\nimport sys\n!pip install unidecode\nfrom unidecode import unidecode\n\n# --- Setup Logging ---\nlogging.basicConfig(\n    level=logging.INFO,\n    format='%(asctime)s - %(levelname)s - %(message)s',\n    handlers=[\n        logging.FileHandler('model_training.log'),\n        logging.StreamHandler()\n    ]\n)\nlogger = logging.getLogger(__name__)\n\n# --- Configuration ---\nDATA_PATH = '/kaggle/input/ds-108-p-21-assigment-06/'\nDELAY_4_6_PATH = os.path.join(DATA_PATH, 'delay_4_6_CONDITION_PRODUCT_SUPPLIER.csv')\nDELAY_7_9_PATH = os.path.join(DATA_PATH, 'delay_7_9_CONDITION_PRODUCT_SUPPLIER.csv')\nNOT_DELAY_4_6_PATH = os.path.join(DATA_PATH, 'not_delay_4_6_CONDITION_PRODUCT_SUPPLIER.csv')\nNOT_DELAY_7_9_PATH = os.path.join(DATA_PATH, 'not_delay_7_9_CONDITION_PRODUCT_SUPPLIER.csv')\nTEST_DATA_PATH = os.path.join(DATA_PATH, 'PILOT_10.csv')\nOUTPUT_PATH = '/kaggle/working/'\n\nTARGET_COL = 'label'\nID_SUBMISSION_COL = 'ID'\nID_RULE_COL = 'GLOBAL_NO'\n\nFILTER_COLS_CONFIG = {\n    'SUBSIDIARY_CD': 'MJP',\n    'MC_PLANT_DIV_CHAR_3': '2',\n    'VOUCHER_DIV_VALUES': [\"110\", \"160\", \"170\", \"180\"],\n    'CUSTSUB_SUBSIDIARY_CD_IS_NULL': True\n}\nFILTER_COLUMN_NAMES = ['SUBSIDIARY_CD', 'MC_PLANT_DIV', 'VOUCHER_DIV', 'CUSTSUB_SUBSIDIARY_CD']\n\nDATETIME_FEATURES = ['Order date', 'VSD']\nNUMERICAL_FEATURES_RAW = [\n    'SUPPLIER_INV_AMOUNT', 'WEIGHT PER PIECE', 'SO QTY', 'Consider count hodiday Saturday'\n]\nCATEGORICAL_FEATURES_ORIGINAL = [\n    'Ship Mode', 'CUST_CD', 'CLASSIFY_CD', 'INNER_CD', 'PACKING_RANK', 'PRODUCT_CD',\n    'OTHER_AREA_SHIP_DIV', 'PRODUCT_ATTRIBUTION', 'SUPPLIER_CD', 'DELI_DIV', 'SUPPLIER_DIV',\n    ID_RULE_COL\n]\n\nUSEFUL_FEATURES_BASE = list(set(DATETIME_FEATURES + NUMERICAL_FEATURES_RAW + CATEGORICAL_FEATURES_ORIGINAL))\nREMOVE_FEATURES_BASE = list(set([\n    'SO_TIME', 'QTUF_RCV_NO', 'SOUF_RCV_NO', 'HEAVY_FLG', 'EXPENSIVE_FLG', 'SPECIAL_DIV',\n    'WEIGHT_UNIT', 'PACK_QTY', 'IO_UNFIT_FLG', 'ACTUAL_SHIP_DAYS', 'SPECIFY_PRODUCTION_DAYS',\n    'SPECIFY_SHIP_DAYS', 'HAZARD_FLG', 'SUPPLIER_CATEGORY_CD', 'PRODUCT_ASSORT', 'REASON_CD',\n    'BRAND_CD', 'Sales order line number', 'Stock class', 'ALLOCATION QTY', 'LOGICAL PLANT',\n    'PURCHASE AMOUNT', 'DIRECT SHIP FLG', 'SHIP DECISION NO'\n] + [fc for fc in FILTER_COLUMN_NAMES if fc not in USEFUL_FEATURES_BASE]))\n\n# Hyperparameters\nRANDOM_STATE = 42\nN_FOLDS = 5\nCATBOOST_BASE_PARAMS = {\n    'iterations': 1200,\n    'learning_rate': 0.03,\n    'depth': 6,\n    'l2_leaf_reg': 3,\n    'random_seed': RANDOM_STATE,\n    'verbose': 200,\n    'early_stopping_rounds': 150,\n    'eval_metric': 'AUC',\n    'task_type': 'CPU'\n}\nFEATURE_SELECTION_TOP_N = 25\nUNDERSAMPLING_RATIO_DELAY_TO_NOT_DELAY = 8\nHIGH_DELAY_SUPPLIER_THRESHOLD = 0.80\nLOW_DELAY_SUPPLIER_THRESHOLD = 0.20\nRULE_PROBABILITY_SMOOTHING = 0.9\nSUBMISSION_THRESHOLD = 0.5\nENSEMBLE_SEEDS = [42, 43, 44]\n\ndef install_unidecode():\n    try:\n        from unidecode import unidecode\n        return unidecode\n    except ImportError:\n        logger.warning(\"unidecode not found, attempting to install...\")\n        try:\n            subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"unidecode\"])\n            from unidecode import unidecode\n            logger.info(\"unidecode installed successfully.\")\n            return unidecode\n        except Exception as e:\n            logger.error(f\"Failed to install unidecode: {e}\")\n            raise\n\nunidecode = install_unidecode()\n\ndef reduce_mem_usage(df: pd.DataFrame, verbose: bool = True) -> pd.DataFrame:\n    start_mem = df.memory_usage().sum() / 1024**2\n    for col in df.columns:\n        col_type = df[col].dtype\n        if str(col_type).startswith(('int', 'float')):\n            c_min, c_max = df[col].min(), df[col].max()\n            if str(col_type).startswith('int'):\n                for dtype in [np.int8, np.int16, np.int32, np.int64]:\n                    if c_min > np.iinfo(dtype).min and c_max < np.iinfo(dtype).max:\n                        df[col] = df[col].astype(dtype)\n                        break\n            else:\n                df[col] = df[col].astype(np.float32)\n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose:\n        logger.info(f'Mem usage decreased to {end_mem:.2f} Mb ({100 * (start_mem - end_mem) / start_mem:.1f}% reduction)')\n    return df\n\ndef validate_columns(df: pd.DataFrame, required_cols: List[str]) -> bool:\n    missing_cols = [col for col in required_cols if col not in df.columns]\n    if missing_cols:\n        logger.error(f\"Missing required columns: {missing_cols}\")\n        return False\n    return True\n\ndef load_and_filter_data(\n    path: str, required_cols: List[str], filter_cols: List[str], filter_config: Dict, is_test: bool = False\n) -> pd.DataFrame:\n    logger.info(f\"Loading data from {path}\")\n    try:\n        df_cols = pd.read_csv(path, nrows=0).columns.tolist()\n        load_cols = list(set(required_cols + filter_cols) & set(df_cols))\n        if is_test:\n            load_cols = list(set(load_cols + [ID_SUBMISSION_COL, ID_RULE_COL]) & set(df_cols))\n        elif TARGET_COL not in load_cols:\n            load_cols.append(TARGET_COL) if TARGET_COL in df_cols else None\n        df = pd.read_csv(path, usecols=load_cols, low_memory=False, on_bad_lines='skip')\n        logger.info(f\"Loaded {len(df)} rows with {len(df.columns)} columns from {path}\")\n    except Exception as e:\n        logger.error(f\"Error loading {path}: {e}\")\n        return pd.DataFrame()\n\n    if df.empty or not validate_columns(df, required_cols):\n        return df\n\n    # Apply filters\n    for col, config in filter_config.items():\n        if col == 'SUBSIDIARY_CD' and col in df:\n            df = df[df[col] == config]\n        elif col == 'MC_PLANT_DIV_CHAR_3' and 'MC_PLANT_DIV' in df:\n            df['MC_PLANT_DIV'] = df['MC_PLANT_DIV'].astype(str)\n            df = df[df['MC_PLANT_DIV'].str[2:3] == config]\n        elif col == 'VOUCHER_DIV_VALUES' and 'VOUCHER_DIV' in df:\n            df['VOUCHER_DIV'] = df['VOUCHER_DIV'].astype(str)\n            df = df[df['VOUCHER_DIV'].isin(config)]\n        elif col == 'CUSTSUB_SUBSIDIARY_CD_IS_NULL' and 'CUSTSUB_SUBSIDIARY_CD' in df:\n            if config:\n                df = df[df['CUSTSUB_SUBSIDIARY_CD'].isnull()]\n        else:\n            logger.warning(f\"Filter column {col} not applicable for {path}\")\n\n    logger.info(f\"Shape after filtering: {df.shape}\")\n    final_cols = list(set(required_cols + ([ID_SUBMISSION_COL, ID_RULE_COL] if is_test else [TARGET_COL, ID_RULE_COL])))\n    final_cols = [col for col in final_cols if col in df.columns]\n    return reduce_mem_usage(df[final_cols])\n\ndef handle_nulls(\n    df: pd.DataFrame, num_feats: List[str], cat_feats: List[str], date_feats: List[str]\n) -> pd.DataFrame:\n    df_copy = df.copy()\n    num_feats = [f for f in num_feats if f in df_copy.columns and f not in [ID_SUBMISSION_COL, ID_RULE_COL]]\n    cat_feats = [f for f in cat_feats if f in df_copy.columns and f != ID_SUBMISSION_COL]\n    date_feats = [f for f in date_feats if f in df_copy.columns]\n\n    for col in num_feats:\n        df_copy[col] = pd.to_numeric(df_copy[col], errors='coerce').fillna(-1).astype(np.float32)\n    for col in cat_feats:\n        df_copy[col] = df_copy[col].astype(str).fillna('missing_value')\n    for col in date_feats:\n        df_copy[col] = pd.to_datetime(df_copy[col], errors='coerce')\n        if df_copy[col].isna().any():\n            df_copy[col] = df_copy[col].ffill().bfill()\n    return df_copy\n\ndef frequency_encoding(df: pd.DataFrame, cat_cols: List[str]) -> Tuple[List[str], pd.DataFrame]:\n    new_cols = []\n    for col in cat_cols:\n        if col in df.columns:\n            freq = df[col].value_counts(normalize=True)\n            df[f'{col}_freq'] = df[col].map(freq).astype(np.float32)\n            new_cols.append(f'{col}_freq')\n    return new_cols, df\n\ndef feature_engineering(\n    train_df: pd.DataFrame, test_df: pd.DataFrame\n) -> Tuple[pd.DataFrame, pd.DataFrame, List[str]]:\n    logger.info(f\"Starting feature engineering. Train shape: {train_df.shape}, Test shape: {test_df.shape}\")\n    train_len = len(train_df)\n    labels = train_df[TARGET_COL].copy() if TARGET_COL in train_df else pd.Series()\n    train_df = train_df.drop(columns=[TARGET_COL], errors='ignore')\n\n    # Align IDs\n    for col in [ID_SUBMISSION_COL, ID_RULE_COL]:\n        if col in test_df and col not in train_df:\n            train_df[col] = f'train_placeholder_{col}'\n\n    combined = pd.concat([train_df, test_df], ignore_index=True)\n    logger.info(f\"Combined shape: {combined.shape}\")\n\n    # Text Normalization\n    for col in CATEGORICAL_FEATURES_ORIGINAL:\n        if col in combined and col != ID_SUBMISSION_COL:\n            combined[col] = combined[col].astype(str).str.lower().apply(\n                lambda x: unidecode(x) if pd.notnull(x) else 'missing_value'\n            ).fillna('missing_value')\n\n    # Specific Categorical Mappings\n    if 'OTHER_AREA_SHIP_DIV' in combined:\n        empty_vals = ['missing_value', 'nan', 'none', '', 'khoang trang']\n        combined['OTHER_AREA_SHIP_DIV'] = combined['OTHER_AREA_SHIP_DIV'].astype(str).str.strip()\n        combined.loc[combined['OTHER_AREA_SHIP_DIV'].isin(empty_vals), 'OTHER_AREA_SHIP_DIV'] = 'own_area'\n        combined.loc[combined['OTHER_AREA_SHIP_DIV'] == '1', 'OTHER_AREA_SHIP_DIV'] = 'other_area'\n        combined.loc[~combined['OTHER_AREA_SHIP_DIV'].isin(['own_area', 'other_area']), 'OTHER_AREA_SHIP_DIV'] = 'unknown_area'\n\n    # Datetime Features\n    derived_date_cols = []\n    for col in DATETIME_FEATURES:\n        if col in combined and pd.api.types.is_datetime64_any_dtype(combined[col]):\n            prefix = col.replace(' ', '_')\n            combined[f'{prefix}_month'] = combined[col].dt.month.fillna(-1).astype(np.int8)\n            combined[f'{prefix}_day'] = combined[col].dt.day.fillna(-1).astype(np.int8)\n            combined[f'{prefix}_dayofweek'] = combined[col].dt.dayofweek.fillna(-1).astype(np.int8)\n            combined[f'{prefix}_week'] = combined[col].dt.isocalendar().week.fillna(-1).astype(np.int8)\n            combined[f'{prefix}_quarter'] = combined[col].dt.quarter.fillna(-1).astype(np.int8)\n            combined[f'{prefix}_is_weekend'] = (combined[col].dt.dayofweek >= 5).astype(np.int8)\n            derived_date_cols.extend([\n                f'{prefix}_month', f'{prefix}_day', f'{prefix}_dayofweek',\n                f'{prefix}_week', f'{prefix}_quarter', f'{prefix}_is_weekend'\n            ])\n\n    if 'VSD' in combined and 'Order date' in combined:\n        combined['days_vsd_order'] = (\n            (combined['VSD'] - combined['Order date']).dt.days.fillna(-1).astype(np.int16)\n        )\n        derived_date_cols.append('days_vsd_order')\n\n    # Numerical Calculations\n    derived_num_cols = []\n    if 'WEIGHT PER PIECE' in combined and 'SO QTY' in combined:\n        combined['WEIGHT_calc'] = (\n            pd.to_numeric(combined['WEIGHT PER PIECE'], errors='coerce').fillna(-1) *\n            pd.to_numeric(combined['SO QTY'], errors='coerce').fillna(-1)\n        ).astype(np.float32)\n        derived_num_cols.append('WEIGHT_calc')\n\n    # Interaction Features\n    if 'SUPPLIER_INV_AMOUNT' in combined and 'WEIGHT_calc' in combined:\n        combined['price_per_weight'] = (\n            combined['SUPPLIER_INV_AMOUNT'].astype(np.float32) /\n            (combined['WEIGHT_calc'] + 1e-6)  # Avoid division by zero\n        ).astype(np.float32)\n        derived_num_cols.append('price_per_weight')\n\n    # Frequency Encoding\n    freq_cols, combined = frequency_encoding(combined, CATEGORICAL_FEATURES_ORIGINAL)\n\n    # Scaling Numerical Features\n    num_cols = list(set(\n        [f for f in NUMERICAL_FEATURES_RAW if f in combined] +\n        derived_date_cols + derived_num_cols + freq_cols\n    ))\n    if num_cols:\n        scaler = MinMaxScaler()\n        combined[num_cols] = pd.DataFrame(\n            scaler.fit_transform(combined[num_cols]),\n            columns=num_cols,\n            index=combined.index\n        ).astype(np.float32)\n\n    # Drop Unused Features\n    cols_to_drop = list(set(\n        DATETIME_FEATURES + ['WEIGHT PER PIECE', 'SO QTY'] +\n        [f for f in REMOVE_FEATURES_BASE if f in combined]\n    ))\n    cols_to_drop = [col for col in cols_to_drop if col not in [ID_SUBMISSION_COL, ID_RULE_COL]]\n    combined = combined.drop(columns=cols_to_drop, errors='ignore')\n\n    # Split back\n    train_fe = combined.iloc[:train_len].copy()\n    test_fe = combined.iloc[train_len:].copy()\n    train_fe[TARGET_COL] = labels.astype(np.int8)\n    if ID_SUBMISSION_COL in train_fe:\n        train_fe = train_fe.drop(columns=[ID_SUBMISSION_COL], errors='ignore')\n\n    logger.info(f\"FE completed. Train shape: {train_fe.shape}, Test shape: {test_fe.shape}\")\n    return train_fe, test_fe, num_cols + freq_cols\n\ndef kfold_catboost(\n    train_df: pd.DataFrame, test_df: pd.DataFrame, num_folds: int, seed: int, run_for_importance: bool = False\n) -> Tuple[np.ndarray, pd.DataFrame, List[str]]:\n    if not run_for_importance and ID_SUBMISSION_COL not in test_df:\n        logger.error(f\"{ID_SUBMISSION_COL} missing in test_df\")\n        raise ValueError(f\"{ID_SUBMISSION_COL} missing\")\n\n    folds = StratifiedKFold(n_splits=num_folds, shuffle=True, random_state=seed)\n    oof_preds = np.zeros(len(train_df), dtype=np.float32)\n    sub_preds = np.zeros(len(test_df), dtype=np.float32)\n    fi_df = pd.DataFrame()\n\n    feats = [f for f in train_df.columns if f not in [TARGET_COL, ID_SUBMISSION_COL]]\n    cat_feats = [f for f in CATEGORICAL_FEATURES_ORIGINAL if f in feats]\n    logger.info(f\"Training on {len(feats)} features, {len(cat_feats)} categorical\")\n\n    scale_pos_weight = train_df[TARGET_COL].value_counts()[0] / train_df[TARGET_COL].value_counts()[1] if 1 in train_df[TARGET_COL].value_counts() else 1.0\n    logger.info(f\"Scale pos weight: {scale_pos_weight:.2f}\")\n\n    for fold, (train_idx, val_idx) in enumerate(folds.split(train_df[feats], train_df[TARGET_COL])):\n        logger.info(f\"Fold {fold+1}/{num_folds}\")\n        train_pool = Pool(train_df.iloc[train_idx][feats], train_df.iloc[train_idx][TARGET_COL], cat_features=cat_feats)\n        val_pool = Pool(train_df.iloc[val_idx][feats], train_df.iloc[val_idx][TARGET_COL], cat_features=cat_feats)\n        test_pool = Pool(test_df[feats], cat_features=cat_feats)\n\n        model = CatBoostClassifier(**{**CATBOOST_BASE_PARAMS, 'random_seed': seed + fold, 'scale_pos_weight': scale_pos_weight})\n        model.fit(train_pool, eval_set=val_pool, use_best_model=True)\n\n        oof_preds[val_idx] = model.predict_proba(val_pool)[:, 1]\n        sub_preds += model.predict_proba(test_pool)[:, 1] / num_folds\n\n        fi_df = pd.concat([fi_df, pd.DataFrame({\n            'feature': feats,\n            'importance': model.get_feature_importance(),\n            'fold': fold + 1\n        })], axis=0)\n\n        if not run_for_importance:\n            auc = roc_auc_score(train_df.iloc[val_idx][TARGET_COL], oof_preds[val_idx])\n            binary_preds = (oof_preds[val_idx] > SUBMISSION_THRESHOLD).astype(np.int8)\n            logger.info(\n                f\"Fold {fold+1} AUC: {auc:.4f}, F1: {f1_score(train_df.iloc[val_idx][TARGET_COL], binary_preds):.4f}, \"\n                f\"Prec: {precision_score(train_df.iloc[val_idx][TARGET_COL], binary_preds, zero_division=0):.4f}, \"\n                f\"Rec: {recall_score(train_df.iloc[val_idx][TARGET_COL], binary_preds, zero_division=0):.4f}\"\n            )\n\n        del train_pool, val_pool, test_pool, model\n        gc.collect()\n\n    if not run_for_importance:\n        auc = roc_auc_score(train_df[TARGET_COL], oof_preds)\n        binary_preds = (oof_preds > SUBMISSION_THRESHOLD).astype(np.int8)\n        logger.info(f\"Full OOF AUC: {auc:.4f}, F1: {f1_score(train_df[TARGET_COL], binary_preds):.4f}\")\n\n    mean_fi = fi_df.groupby('feature')['importance'].mean().sort_values(ascending=False)\n    selected_feats = mean_fi.head(FEATURE_SELECTION_TOP_N).index.tolist()\n    logger.info(f\"Top {FEATURE_SELECTION_TOP_N} features:\\n{mean_fi.head(FEATURE_SELECTION_TOP_N)}\")\n\n    return sub_preds, fi_df, selected_feats\n\ndef ensemble_catboost(\n    train_df: pd.DataFrame, test_df: pd.DataFrame, num_folds: int\n) -> np.ndarray:\n    all_sub_preds = np.zeros(len(test_df))\n    for seed in ENSEMBLE_SEEDS:\n        logger.info(f\"Running CatBoost ensemble with seed {seed}\")\n        sub_preds, _, _ = kfold_catboost(train_df, test_df, num_folds, seed, run_for_importance=False)\n        all_sub_preds += sub_preds / len(ENSEMBLE_SEEDS)\n    return all_sub_preds\n\n# --- Main Execution ---\nos.makedirs(OUTPUT_PATH, exist_ok=True)\nlogger.info(\"Listing input files:\")\nfor root, _, files in os.walk(DATA_PATH):\n    for f in files:\n        logger.info(os.path.join(root, f))\n\ncols_needed_train = list(set(USEFUL_FEATURES_BASE + [TARGET_COL, ID_RULE_COL, 'SUPPLIER_CD'] + FILTER_COLUMN_NAMES))\ncols_needed_test = list(set(USEFUL_FEATURES_BASE + [ID_SUBMISSION_COL, ID_RULE_COL, 'SUPPLIER_CD'] + FILTER_COLUMN_NAMES))\n\nlogger.info(\"Loading training data...\")\ndata_delay_46 = load_and_filter_data(DELAY_4_6_PATH, cols_needed_train, FILTER_COLUMN_NAMES, FILTER_COLS_CONFIG)\ndata_delay_79 = load_and_filter_data(DELAY_7_9_PATH, cols_needed_train, FILTER_COLUMN_NAMES, FILTER_COLS_CONFIG)\ndata_not_delay_46 = load_and_filter_data(NOT_DELAY_4_6_PATH, cols_needed_train, FILTER_COLUMN_NAMES, FILTER_COLS_CONFIG)\ndata_not_delay_79 = load_and_filter_data(NOT_DELAY_7_9_PATH, cols_needed_train, FILTER_COLUMN_NAMES, FILTER_COLS_CONFIG)\n\ndata_delay = pd.concat([data_delay_46, data_delay_79], ignore_index=True).drop_duplicates()\ndata_not_delay = pd.concat([data_not_delay_46, data_not_delay_79], ignore_index=True).drop_duplicates()\ndel data_delay_46, data_delay_79, data_not_delay_46, data_not_delay_79\ngc.collect()\n\nif data_delay.empty or data_not_delay.empty:\n    logger.error(\"Delay or not_delay data is empty. Exiting.\")\n    raise ValueError(\"Empty training data\")\n\nn_not_delay = min(len(data_delay) * UNDERSAMPLING_RATIO_DELAY_TO_NOT_DELAY, len(data_not_delay))\ndata_not_delay = data_not_delay.sample(n=n_not_delay, random_state=RANDOM_STATE)\ndata_delay[TARGET_COL] = 1\ndata_not_delay[TARGET_COL] = 0\n\ntrain_data = pd.concat([data_delay, data_not_delay], ignore_index=True).sample(frac=1, random_state=RANDOM_STATE)\ntrain_data = reduce_mem_usage(train_data)\nlogger.info(f\"Training data shape: {train_data.shape}, Label dist:\\n{train_data[TARGET_COL].value_counts(normalize=True)}\")\n\nlogger.info(\"Loading test data...\")\ntest_data = load_and_filter_data(TEST_DATA_PATH, cols_needed_test, FILTER_COLUMN_NAMES, FILTER_COLS_CONFIG, is_test=True)\ntest_data = reduce_mem_usage(test_data)\nif test_data.empty or ID_SUBMISSION_COL not in test_data or ID_RULE_COL not in test_data:\n    logger.error(\"Test data invalid or missing ID columns\")\n    raise ValueError(\"Invalid test data\")\n\ntrain_data = handle_nulls(train_data, NUMERICAL_FEATURES_RAW, CATEGORICAL_FEATURES_ORIGINAL, DATETIME_FEATURES)\ntest_data = handle_nulls(test_data, NUMERICAL_FEATURES_RAW, CATEGORICAL_FEATURES_ORIGINAL, DATETIME_FEATURES)\n\ntrain_fe, test_fe, derived_num_cols = feature_engineering(train_data.copy(), test_data.copy())\ndel train_data, test_data\ngc.collect()\n\nif train_fe.empty or TARGET_COL not in train_fe:\n    logger.error(\"Training data after FE is empty or missing target\")\n    raise ValueError(\"Invalid training data after FE\")\n\nlogger.info(\"First pass: Feature importance...\")\n_, _, selected_feats = kfold_catboost(train_fe, test_fe, N_FOLDS, RANDOM_STATE, run_for_importance=True)\n\nif not selected_feats:\n    logger.warning(\"No features selected. Using all features for final model.\")\n    selected_feats = [f for f in train_fe.columns if f not in [TARGET_COL, ID_SUBMISSION_COL]]\n\ntrain_final = train_fe[list(set(selected_feats + [TARGET_COL]))]\ntest_final = test_fe[list(set(selected_feats + [ID_SUBMISSION_COL, ID_RULE_COL]))]\nlogger.info(f\"Final train shape: {train_final.shape}, Final test shape: {test_final.shape}\")\n\nlogger.info(\"Second pass: Ensemble training...\")\nsub_preds = ensemble_catboost(train_final, test_final, N_FOLDS)\ndel train_final, test_final\ngc.collect()\n\n# --- Rules ---\nlogger.info(\"Generating rules...\")\nrule_cols = [ID_RULE_COL, 'SUPPLIER_CD']\nrule_delay_46 = load_and_filter_data(DELAY_4_6_PATH, rule_cols, FILTER_COLUMN_NAMES, FILTER_COLS_CONFIG)\nrule_delay_79 = load_and_filter_data(DELAY_7_9_PATH, rule_cols, FILTER_COLUMN_NAMES, FILTER_COLS_CONFIG)\nrule_not_delay_46 = load_and_filter_data(NOT_DELAY_4_6_PATH, rule_cols, FILTER_COLUMN_NAMES, FILTER_COLS_CONFIG)\nrule_not_delay_79 = load_and_filter_data(NOT_DELAY_7_9_PATH, rule_cols, FILTER_COLUMN_NAMES, FILTER_COLS_CONFIG)\n\ndata_delay_rules = pd.concat([rule_delay_46, rule_delay_79], ignore_index=True).drop_duplicates(subset=[ID_RULE_COL, 'SUPPLIER_CD'])\ndata_not_delay_rules = pd.concat([rule_not_delay_46, rule_not_delay_79], ignore_index=True).drop_duplicates(subset=[ID_RULE_COL, 'SUPPLIER_CD'])\ndel rule_delay_46, rule_delay_79, rule_not_delay_46, rule_not_delay_79\ngc.collect()\n\nrules_df = pd.DataFrame(columns=[ID_RULE_COL, 'DELAY_RULE'])\nif not data_delay_rules.empty and not data_not_delay_rules.empty:\n    data_delay_rules[TARGET_COL] = 1\n    data_not_delay_rules[TARGET_COL] = 0\n    rule_data = pd.concat([data_delay_rules, data_not_delay_rules], ignore_index=True)\n    rule_data = rule_data.drop_duplicates(subset=[ID_RULE_COL], keep='first')\n\n    sdr = rule_data.groupby('SUPPLIER_CD')[TARGET_COL].agg(['sum', 'count']).reset_index()\n    sdr['Delay_Ratio'] = sdr['sum'] / sdr['count']\n    high_delay = sdr[sdr['Delay_Ratio'] >= HIGH_DELAY_SUPPLIER_THRESHOLD]['SUPPLIER_CD'].unique()\n    low_delay = sdr[sdr['Delay_Ratio'] <= LOW_DELAY_SUPPLIER_THRESHOLD]['SUPPLIER_CD'].unique()\n\n    supplier_rules = {sc: 'Y' for sc in high_delay}\n    supplier_rules.update({sc: 'N' for sc in low_delay})\n    rules = [\n        {ID_RULE_COL: row[ID_RULE_COL], 'DELAY_RULE': supplier_rules.get(row['SUPPLIER_CD'])}\n        for _, row in rule_data[[ID_RULE_COL, 'SUPPLIER_CD']].drop_duplicates().iterrows()\n        if row['SUPPLIER_CD'] in supplier_rules\n    ]\n    if rules:\n        rules_df = pd.DataFrame(rules).drop_duplicates(subset=[ID_RULE_COL])\n    rules_df.to_csv(os.path.join(OUTPUT_PATH, 'rules.csv'), index=False)\n    logger.info(f\"Generated {len(rules_df)} rules\")\nelse:\n    logger.warning(\"Rule data empty or missing columns. Skipping rule generation.\")\n\n# --- Submission ---\nlogger.info(\"Preparing submission...\")\nsubmission_df = pd.DataFrame({\n    ID_SUBMISSION_COL: test_fe[ID_SUBMISSION_COL].astype(str),\n    ID_RULE_COL: test_fe[ID_RULE_COL].astype(str),\n    'Predicted_Probability': sub_preds\n})\nsubmission_df['Predicted_Probability'] *= RULE_PROBABILITY_SMOOTHING\n\nif not rules_df.empty:\n    submission_df = submission_df.merge(rules_df, on=ID_RULE_COL, how='left')\n    submission_df.loc[submission_df['DELAY_RULE'] == 'Y', 'Predicted_Probability'] = (\n        submission_df['Predicted_Probability'] * 0.2 + 0.8\n    )\n    submission_df.loc[submission_df['DELAY_RULE'] == 'N', 'Predicted_Probability'] = (\n        submission_df['Predicted_Probability'] * 0.2 + 0.2\n    )\n    submission_df = submission_df.drop(columns=['DELAY_RULE'], errors='ignore')\n\nsubmission_df[TARGET_COL] = (submission_df['Predicted_Probability'] > SUBMISSION_THRESHOLD).astype(np.int8)\nfinal_submission = submission_df[[ID_SUBMISSION_COL, TARGET_COL]].rename(columns={ID_SUBMISSION_COL: 'ID'})\nfinal_submission = final_submission.drop_duplicates(subset=['ID'], keep='first')\nfinal_submission.to_csv(os.path.join(OUTPUT_PATH, 'submission.csv'), index=False)\nlogger.info(f\"Submission saved with {len(final_submission)} rows\\n{final_submission.head()}\")\nlogger.info(f\"Label distribution:\\n{final_submission[TARGET_COL].value_counts(normalize=True)}\")\n\nlogger.info(\"Script completed successfully.\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-16T05:00:43.428993Z","iopub.execute_input":"2025-06-16T05:00:43.429445Z"}},"outputs":[],"execution_count":null}]}