{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.13"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":17000,"databundleVersionId":895292,"sourceType":"competition"}],"dockerImageVersionId":31154,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":3202.838851,"end_time":"2025-10-28T08:44:18.486215","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-10-28T07:50:55.647364","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"fcc15fe1-b4d3-465e-b0cd-564056709bab","cell_type":"markdown","source":"# Robust Binary Classification for Complex Categorical Data\nIn this work, I tackle the challenge of binary classification on high-cardinality categorical data. Given the complexity introduced by missing values and the risk of overfitting, my focus in this solution is to ensure processing integrity and completely prevent data leakage during training.\n\nInstead of relying on traditional encoding methods, I applied a **Hybrid Feature Engineering** strategy. Safe transformations (such as date features and interactions) are performed globally, while sensitive techniques like **Frequency Encoding** are applied exclusively within **Cross-Validation** loops. This ensures that no information from the test data leaks into the model.\n\nFinally, the model leverages the **CatBoost** algorithm with **GPU acceleration**, allowing for efficient handling of unseen categories and the detection of complex feature interactions to maximize the **AUC** score.","metadata":{}},{"id":"9a4811bd-5b40-4e9c-9c11-6816a44b67e5","cell_type":"markdown","source":"#  Load the datasets and import the library","metadata":{}},{"id":"1d590f3e","cell_type":"code","source":"# --- Setup and Libraries ---\nprint(\"--- Step 0: Installing CatBoost Library ---\")\n!pip install catboost --quiet\nprint(\"--- CatBoost installed successfully ---\")\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom catboost import CatBoostClassifier\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.preprocessing import OrdinalEncoder\nimport warnings\n\n# Display settings and warning suppression\nwarnings.filterwarnings('ignore')\nsns.set_style('whitegrid')\npd.set_option('display.max_columns', 100) # To display more columns when printing\nprint(\"--- Libraries loaded and settings configured ---\")\n# --- 1. Quick Look at the Data (Loading Data) ---\nprint(\"--- 1. Quick Look at the Data (Loading Data) ---\")\nBASE_PATH = '/kaggle/input/cat-in-the-dat-ii/'\n\ntry:\n    train_df = pd.read_csv(BASE_PATH + 'train.csv')\n    test_df = pd.read_csv(BASE_PATH + 'test.csv')\nexcept FileNotFoundError:\n    print(\"❌ Error: Make sure to add 'cat-in-the-dat-ii' data to the notebook.\")\n    raise\n\n# Separate variables\ny = train_df['target']\ntest_ids = test_df['id']\ntrain_df = train_df.drop(['id', 'target'], axis=1)\ntest_df = test_df.drop('id', axis=1)\n\nprint(f\"✅ Data loaded successfully.\")\nprint(f\"Train data shape: {train_df.shape}\")\nprint(f\"Test data shape: {test_df.shape}\")","metadata":{"execution":{"iopub.status.busy":"2025-11-19T19:21:56.177632Z","iopub.execute_input":"2025-11-19T19:21:56.177892Z","iopub.status.idle":"2025-11-19T19:22:02.368243Z","shell.execute_reply.started":"2025-11-19T19:21:56.177875Z","shell.execute_reply":"2025-11-19T19:22:02.367303Z"},"papermill":{"duration":13.847267,"end_time":"2025-10-28T07:51:13.271321","exception":false,"start_time":"2025-10-28T07:50:59.424054","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"094af8f2","cell_type":"markdown","source":"# Quick look at the data","metadata":{"papermill":{"duration":0.004054,"end_time":"2025-10-28T07:51:13.280175","exception":false,"start_time":"2025-10-28T07:51:13.276121","status":"completed"},"tags":[]}},{"id":"cfaf7949","cell_type":"code","source":"print(\"First 5 rows of the training data:\")\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2025-11-19T19:22:02.369909Z","iopub.execute_input":"2025-11-19T19:22:02.370112Z","iopub.status.idle":"2025-11-19T19:22:02.391309Z","shell.execute_reply.started":"2025-11-19T19:22:02.370095Z","shell.execute_reply":"2025-11-19T19:22:02.390556Z"},"papermill":{"duration":0.031928,"end_time":"2025-10-28T07:51:13.316147","exception":false,"start_time":"2025-10-28T07:51:13.284219","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"e62dadbb","cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2025-11-19T19:22:02.392079Z","iopub.execute_input":"2025-11-19T19:22:02.392283Z","iopub.status.idle":"2025-11-19T19:22:02.842120Z","shell.execute_reply.started":"2025-11-19T19:22:02.392268Z","shell.execute_reply":"2025-11-19T19:22:02.841323Z"},"papermill":{"duration":0.44754,"end_time":"2025-10-28T07:51:13.768066","exception":false,"start_time":"2025-10-28T07:51:13.320526","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"281dc53f","cell_type":"code","source":"train_df.describe()","metadata":{"execution":{"iopub.status.busy":"2025-11-19T19:22:02.843618Z","iopub.execute_input":"2025-11-19T19:22:02.843816Z","iopub.status.idle":"2025-11-19T19:22:03.001240Z","shell.execute_reply.started":"2025-11-19T19:22:02.843801Z","shell.execute_reply":"2025-11-19T19:22:03.000597Z"},"papermill":{"duration":0.192698,"end_time":"2025-10-28T07:51:13.966591","exception":false,"start_time":"2025-10-28T07:51:13.773893","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"f7a3d752","cell_type":"code","source":"train_df.describe(include='O')","metadata":{"execution":{"iopub.status.busy":"2025-11-19T19:22:03.002387Z","iopub.execute_input":"2025-11-19T19:22:03.002652Z","iopub.status.idle":"2025-11-19T19:22:04.149984Z","shell.execute_reply.started":"2025-11-19T19:22:03.002635Z","shell.execute_reply":"2025-11-19T19:22:04.149246Z"},"papermill":{"duration":1.150547,"end_time":"2025-10-28T07:51:15.122678","exception":false,"start_time":"2025-10-28T07:51:13.972131","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"448d8f68","cell_type":"code","source":"train_df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2025-11-19T19:22:04.150777Z","iopub.execute_input":"2025-11-19T19:22:04.150978Z","iopub.status.idle":"2025-11-19T19:22:04.598159Z","shell.execute_reply.started":"2025-11-19T19:22:04.150963Z","shell.execute_reply":"2025-11-19T19:22:04.597452Z"},"papermill":{"duration":0.431185,"end_time":"2025-10-28T07:51:15.558794","exception":false,"start_time":"2025-10-28T07:51:15.127609","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"488cb554","cell_type":"code","source":"train_df.duplicated().sum()","metadata":{"execution":{"iopub.status.busy":"2025-11-19T19:22:04.598929Z","iopub.execute_input":"2025-11-19T19:22:04.599140Z","iopub.status.idle":"2025-11-19T19:22:05.457805Z","shell.execute_reply.started":"2025-11-19T19:22:04.599125Z","shell.execute_reply":"2025-11-19T19:22:05.457021Z"},"papermill":{"duration":0.888234,"end_time":"2025-10-28T07:51:16.452019","exception":false,"start_time":"2025-10-28T07:51:15.563785","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"b3c5446c","cell_type":"markdown","source":"#  EDA (Exploratory Data Analysis)","metadata":{"papermill":{"duration":0.004606,"end_time":"2025-10-28T07:51:16.461994","exception":false,"start_time":"2025-10-28T07:51:16.457388","status":"completed"},"tags":[]}},{"id":"092ef879","cell_type":"code","source":"# --- 1. Calculation (Fixes NameError) ---\n# Calculate the absolute counts of each target class (0 and 1)\ntarget_counts = y.value_counts()\ntotal_count = len(y)\n# Calculate the percentage distribution\ntarget_percentage = (target_counts / total_count) * 100\n\nprint(\"--- Target (y) Distribution Counts ---\")\nprint(target_counts)\nprint(\"\\n--- Target Class Percentage ---\")\nprint(target_percentage.round(2).astype(str) + '%')\n\n# --- 2. Visualization (Countplot) ---\nplt.figure(figsize=(6, 4))\n# Note: The 'palette' uses distinct colors for clear visualization.\nsns.countplot(x=y, palette=['#1f77b4', '#ff7f0e']) \nplt.title('Target Variable Distribution (Target: 0 and 1)', fontsize=14)\nplt.xlabel('Target Value')\nplt.ylabel('Count')\n\n# Adding the percentage directly above the bars\nfor i, count in enumerate(target_counts):\n    percentage = target_percentage.iloc[i]\n    # Placing the text slightly above the bar (count + 1000) for visibility.\n    plt.text(i, count + 1000, f'{percentage:.2f}%', \n             ha='center', va='bottom', fontsize=12)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-11-19T19:22:05.458681Z","iopub.execute_input":"2025-11-19T19:22:05.458966Z","iopub.status.idle":"2025-11-19T19:22:05.639032Z","shell.execute_reply.started":"2025-11-19T19:22:05.458942Z","shell.execute_reply":"2025-11-19T19:22:05.638248Z"},"papermill":{"duration":0.257994,"end_time":"2025-10-28T07:51:16.724829","exception":false,"start_time":"2025-10-28T07:51:16.466835","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"018144f5","cell_type":"code","source":"# --- 1. Data Loading and Initial Setup ---\nBASE_PATH = '/kaggle/input/cat-in-the-dat-ii/'\n\ntry:\n    # Load and drop unnecessary ID/Target columns immediately\n    train_df = pd.read_csv(BASE_PATH + 'train.csv').drop(['id', 'target'], axis=1, errors='ignore')\n    test_df = pd.read_csv(BASE_PATH + 'test.csv').drop('id', axis=1, errors='ignore')\nexcept FileNotFoundError:\n    print(\"Error: Ensure data is accessible at the specified path.\")\n    raise\n\n# Concatenate data for combined analysis\nfull_df = pd.concat([train_df, test_df], axis=0, ignore_index=True)\n\n# --- 2. Calculate and Filter Missing Data ---\n# Calculate the percentage of missing values for all columns\nmissing_percentage = full_df.isnull().mean() * 100\n\n# Filter to keep only columns with missing data and sort them\nmissing_data = missing_percentage[missing_percentage > 0].sort_values(ascending=False)\n\n# --- 3. Visualization and Output ---\nif not missing_data.empty:\n    plt.figure(figsize=(10, 8))\n    \n    # Plotting the missing percentage for all relevant features\n    sns.barplot(x=missing_data.values, y=missing_data.index, palette=\"viridis\")\n\n    plt.title('Percentage of Missing Values Per Feature', fontsize=16)\n    plt.xlabel('Missing Percentage (%)', fontsize=12)\n    plt.ylabel('Features', fontsize=12)\n    plt.grid(axis='x', alpha=0.6) # Added slight transparency to the grid\n    plt.tight_layout()\n    plt.show()\n    \n    print(\"\\n--- Top Missing Features ---\")\n    # Display the top features and their missing percentage\n    print(missing_data.head(10).apply(lambda x: f\"{x:.2f}%\").to_string())\nelse:\n    print(\"No missing values found in the combined dataset.\")","metadata":{"execution":{"iopub.status.busy":"2025-11-19T19:22:05.639859Z","iopub.execute_input":"2025-11-19T19:22:05.640079Z","iopub.status.idle":"2025-11-19T19:22:09.782739Z","shell.execute_reply.started":"2025-11-19T19:22:05.640062Z","shell.execute_reply":"2025-11-19T19:22:09.782079Z"},"papermill":{"duration":4.432873,"end_time":"2025-10-28T07:51:21.163322","exception":false,"start_time":"2025-10-28T07:51:16.730449","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"b49d3c74","cell_type":"code","source":"# Assuming 'train_eda' (full original train_df) is available\n\n# Recalculate train_eda just to be safe (if run standalone)\nBASE_PATH = '/kaggle/input/cat-in-the-dat-ii/'\ntrain_eda = pd.read_csv(BASE_PATH + 'train.csv')\n\n\nprint(\"--- 3. Ordinal Features (ord_2) vs. Target Rate ---\")\n\n# Calculate P(Target=1) for each category in ord_2\ntarget_rate_ord2 = train_eda.groupby('ord_2')['target'].mean().sort_values()\n\nplt.figure(figsize=(8, 5))\n# Plot the mean of the target, which is the probability P(Target=1)\ntarget_rate_ord2.plot(kind='bar', color='darkgreen')\nplt.title('Target Rate by Category: ord_2 (P(Target=1))', fontsize=14)\nplt.xlabel('ord_2 Categories (Ordered)')\nplt.ylabel('Probability of Target=1')\nplt.xticks(rotation=45, ha='right')\nplt.grid(axis='y', alpha=0.5)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-11-19T19:22:09.785755Z","iopub.execute_input":"2025-11-19T19:22:09.786400Z","iopub.status.idle":"2025-11-19T19:22:11.691484Z","shell.execute_reply.started":"2025-11-19T19:22:09.786375Z","shell.execute_reply":"2025-11-19T19:22:11.690713Z"},"papermill":{"duration":2.019449,"end_time":"2025-10-28T07:51:23.189701","exception":false,"start_time":"2025-10-28T07:51:21.170252","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"b97ba40e","cell_type":"code","source":"# Assuming 'train_df' is available (features only)\n\nprint(\"--- 4. Nominal Feature Cardinality ---\")\n\n# Select only nominal columns for cardinality check\nnominal_cols = [col for col in train_df.columns if 'nom_' in col]\ncardinality = train_df[nominal_cols].nunique().sort_values(ascending=False)\n\nplt.figure(figsize=(10, 6))\nsns.barplot(x=cardinality.index, y=cardinality.values, palette=\"rocket\")\nplt.title('Unique Values Count (Cardinality) for Nominal Features', fontsize=16)\nplt.xlabel('Nominal Features')\nplt.ylabel('Number of Unique Values')\nplt.xticks(rotation=45, ha='right')\nplt.grid(axis='y', alpha=0.6)\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-11-19T19:22:11.692323Z","iopub.execute_input":"2025-11-19T19:22:11.692510Z","iopub.status.idle":"2025-11-19T19:22:12.236865Z","shell.execute_reply.started":"2025-11-19T19:22:11.692484Z","shell.execute_reply":"2025-11-19T19:22:12.236003Z"},"papermill":{"duration":0.487212,"end_time":"2025-10-28T07:51:23.684097","exception":false,"start_time":"2025-10-28T07:51:23.196885","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"67a7d2f8","cell_type":"code","source":"# Assuming 'train_eda' (full original train_df) is available\n\n# Recalculate train_eda just to be safe\nBASE_PATH = '/kaggle/input/cat-in-the-dat-ii/'\ntrain_eda = pd.read_csv(BASE_PATH + 'train.csv')\n\n\nprint(\"--- 5. Cyclical Feature Impact (Monthly Rate) ---\")\n\n# Calculate the mean target rate for each month\nmonthly_target_rate = train_eda.groupby('month')['target'].mean()\n\nplt.figure(figsize=(8, 5))\n# Use a line plot to show the cyclical nature\nmonthly_target_rate.plot(kind='line', marker='o', color='darkblue')\nplt.title('Average P(Target=1) Rate Across Months', fontsize=14)\nplt.xlabel('Month')\nplt.ylabel('Probability of Target=1')\nplt.xticks(range(1, 13)) # Ensure all months (1-12) are labeled\nplt.grid(True, linestyle='--', alpha=0.6)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-11-19T19:22:12.237787Z","iopub.execute_input":"2025-11-19T19:22:12.238046Z","iopub.status.idle":"2025-11-19T19:22:14.163699Z","shell.execute_reply.started":"2025-11-19T19:22:12.238018Z","shell.execute_reply":"2025-11-19T19:22:14.162951Z"},"papermill":{"duration":1.855252,"end_time":"2025-10-28T07:51:25.547642","exception":false,"start_time":"2025-10-28T07:51:23.692390","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"26ab82cb","cell_type":"markdown","source":"# Feature Engineering","metadata":{"papermill":{"duration":0.00877,"end_time":"2025-10-28T07:51:25.566044","exception":false,"start_time":"2025-10-28T07:51:25.557274","status":"completed"},"tags":[]}},{"id":"b45b812f","cell_type":"code","source":"# --- 3. Feature Engineering ---\nprint(\"\\n--- 3. Feature Engineering ---\")\n\n# Concatenate data\nfull_df = pd.concat([train_df, test_df], axis=0, ignore_index=True)\n","metadata":{"execution":{"iopub.status.busy":"2025-11-19T19:22:14.164586Z","iopub.execute_input":"2025-11-19T19:22:14.164787Z","iopub.status.idle":"2025-11-19T19:22:14.393701Z","shell.execute_reply.started":"2025-11-19T19:22:14.164770Z","shell.execute_reply":"2025-11-19T19:22:14.392940Z"},"papermill":{"duration":0.106395,"end_time":"2025-10-28T07:51:25.681270","exception":false,"start_time":"2025-10-28T07:51:25.574875","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"649d1bc9","cell_type":"code","source":"# 3.1: Handling Missing Values and Creating Indicators\nprint(\"--- 3.1: Handling Missing Values and Creating Indicators ---\")\nfor col in full_df.columns:\n    if full_df[col].isnull().any():\n        full_df[f'{col}_was_missing'] = full_df[col].isnull().astype(int)\n        \n        if pd.api.types.is_numeric_dtype(full_df[col]):\n            full_df[col] = full_df[col].fillna(-1)  \n        else:\n            full_df[col] = full_df[col].fillna('NONE')","metadata":{"execution":{"iopub.status.busy":"2025-11-19T19:22:14.394442Z","iopub.execute_input":"2025-11-19T19:22:14.394695Z","iopub.status.idle":"2025-11-19T19:22:16.922687Z","shell.execute_reply.started":"2025-11-19T19:22:14.394673Z","shell.execute_reply":"2025-11-19T19:22:16.921951Z"},"papermill":{"duration":2.395612,"end_time":"2025-10-28T07:51:28.086995","exception":false,"start_time":"2025-10-28T07:51:25.691383","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"d81697af","cell_type":"code","source":"# 3.2: Ordinal Feature Encoding (ord_*)\nprint(\"--- 3.2: Ordinal Feature Encoding (ord_*) ---\")\nord_1_mapping = {'Novice': 1, 'Contributor': 2, 'Expert': 3, 'Master': 4, 'Grandmaster': 5, 'NONE': 0, -1: 0}\nord_2_mapping = {'Freezing': 1, 'Cold': 2, 'Warm': 3, 'Hot': 4, 'Boiling Hot': 5, 'Lava Hot': 6, 'NONE': 0, -1: 0}\n\nfull_df['ord_1'] = full_df['ord_1'].map(ord_1_mapping)\nfull_df['ord_2'] = full_df['ord_2'].map(ord_2_mapping)\n\nord_3_col = ['ord_3', 'ord_4', 'ord_5']\nfor col in ord_3_col:\n    # Use OrdinalEncoder after filling missing values\n    full_df[col] = full_df[col].replace('NONE', chr(ord('a') - 1)).fillna(chr(ord('a') - 1))\n    encoder = OrdinalEncoder(categories=[sorted(full_df[col].unique())], dtype=int)\n    full_df[col] = encoder.fit_transform(full_df[[col]])","metadata":{"execution":{"iopub.status.busy":"2025-11-19T19:22:16.923511Z","iopub.execute_input":"2025-11-19T19:22:16.923749Z","iopub.status.idle":"2025-11-19T19:22:18.064877Z","shell.execute_reply.started":"2025-11-19T19:22:16.923725Z","shell.execute_reply":"2025-11-19T19:22:18.064245Z"},"papermill":{"duration":1.113706,"end_time":"2025-10-28T07:51:29.211592","exception":false,"start_time":"2025-10-28T07:51:28.097886","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"43182ef6","cell_type":"code","source":"# 3.3: Frequency Encoding\nprint(\"--- 3.3: Frequency Encoding ---\")\nall_cat_cols = [col for col in full_df.columns if 'nom_' in col or 'bin_' in col or 'day' in col or 'month' in col]\nfor col in all_cat_cols:\n    freq_map = full_df[col].value_counts().to_dict()\n    full_df[f'{col}_freq'] = full_df[col].map(freq_map)","metadata":{"execution":{"iopub.status.busy":"2025-11-19T19:22:18.065986Z","iopub.execute_input":"2025-11-19T19:22:18.066210Z","iopub.status.idle":"2025-11-19T19:22:19.836264Z","shell.execute_reply.started":"2025-11-19T19:22:18.066173Z","shell.execute_reply":"2025-11-19T19:22:19.835662Z"},"papermill":{"duration":1.733218,"end_time":"2025-10-28T07:51:30.954939","exception":false,"start_time":"2025-10-28T07:51:29.221721","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"0f8a6646","cell_type":"code","source":"# 3.4: Cyclical Feature Engineering\nprint(\"--- 3.4: Cyclical Feature Engineering (day, month) ---\")\nfull_df['day_sin'] = np.sin(2 * np.pi * full_df['day'] / 7.0)\nfull_df['day_cos'] = np.cos(2 * np.pi * full_df['day'] / 7.0)\nfull_df['month_sin'] = np.sin(2 * np.pi * full_df['month'] / 12.0)\nfull_df['month_cos'] = np.cos(2 * np.pi * full_df['month'] / 12.0)\nfull_df = full_df.drop(['day', 'month'], axis=1)\n","metadata":{"execution":{"iopub.status.busy":"2025-11-19T19:22:19.836969Z","iopub.execute_input":"2025-11-19T19:22:19.837145Z","iopub.status.idle":"2025-11-19T19:22:20.356379Z","shell.execute_reply.started":"2025-11-19T19:22:19.837131Z","shell.execute_reply":"2025-11-19T19:22:20.355543Z"},"papermill":{"duration":0.526631,"end_time":"2025-10-28T07:51:31.491445","exception":false,"start_time":"2025-10-28T07:51:30.964814","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"a1af6a5b","cell_type":"code","source":"# 3.5: Feature Interactions\nprint(\"--- 3.5: Feature Interactions ---\")\nfull_df['ord_1_nom_3'] = full_df['ord_1'].astype(str) + '_' + full_df['nom_3'].astype(str)\nfull_df['ord_1_ord_2'] = full_df['ord_1'].astype(str) + '_' + full_df['ord_2'].astype(str)\n","metadata":{"execution":{"iopub.status.busy":"2025-11-19T19:22:20.357331Z","iopub.execute_input":"2025-11-19T19:22:20.357593Z","iopub.status.idle":"2025-11-19T19:22:21.406223Z","shell.execute_reply.started":"2025-11-19T19:22:20.357574Z","shell.execute_reply":"2025-11-19T19:22:21.405625Z"},"papermill":{"duration":1.06001,"end_time":"2025-10-28T07:51:32.561240","exception":false,"start_time":"2025-10-28T07:51:31.501230","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"145723bf","cell_type":"code","source":"# 3.6: Converting Categorical Columns to 'category' (for CatBoost)\nprint(\"--- 3.6: Converting Categorical Columns to 'category' ---\")\n# Ensure bin_x columns are string type before converting to category\nbinary_cols_to_fix = [col for col in full_df.columns if 'bin_' in col]\nfor col in binary_cols_to_fix:\n    full_df[col] = full_df[col].astype(str)\n\nnominal_cols = [col for col in full_df.columns if 'nom_' in col or 'bin_' in col]\ninteraction_cols = ['ord_1_nom_3', 'ord_1_ord_2']\nfor col in nominal_cols + interaction_cols:\n    full_df[col] = full_df[col].astype('category')","metadata":{"execution":{"iopub.status.busy":"2025-11-19T19:22:21.406933Z","iopub.execute_input":"2025-11-19T19:22:21.407144Z","iopub.status.idle":"2025-11-19T19:22:28.515884Z","shell.execute_reply.started":"2025-11-19T19:22:21.407127Z","shell.execute_reply":"2025-11-19T19:22:28.515245Z"},"papermill":{"duration":6.897547,"end_time":"2025-10-28T07:51:39.468829","exception":false,"start_time":"2025-10-28T07:51:32.571282","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"489c554b","cell_type":"code","source":"# Separate data back\nX = full_df.iloc[:len(train_df)]\nX_test = full_df.iloc[len(train_df):]","metadata":{"execution":{"iopub.status.busy":"2025-11-19T19:22:28.516716Z","iopub.execute_input":"2025-11-19T19:22:28.517275Z","iopub.status.idle":"2025-11-19T19:22:28.523441Z","shell.execute_reply.started":"2025-11-19T19:22:28.517246Z","shell.execute_reply":"2025-11-19T19:22:28.522500Z"},"papermill":{"duration":0.01527,"end_time":"2025-10-28T07:51:39.494042","exception":false,"start_time":"2025-10-28T07:51:39.478772","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"4759ed7a","cell_type":"code","source":"# Define Categorical Columns for CatBoost\ncategorical_features = [col for col in X.columns if X[col].dtype.name == 'category']\nprint(f\"Total number of features: {len(X.columns)}\")\nprint(f\"Number of categorical features for CatBoost: {len(categorical_features)}\")\nprint(\"--- Feature Engineering completed. ---\")","metadata":{"execution":{"iopub.status.busy":"2025-11-19T19:22:28.524591Z","iopub.execute_input":"2025-11-19T19:22:28.524854Z","iopub.status.idle":"2025-11-19T19:22:28.542794Z","shell.execute_reply.started":"2025-11-19T19:22:28.524829Z","shell.execute_reply":"2025-11-19T19:22:28.542046Z"},"papermill":{"duration":0.016625,"end_time":"2025-10-28T07:51:39.520062","exception":false,"start_time":"2025-10-28T07:51:39.503437","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"5bbf6e4e","cell_type":"markdown","source":"# Train ML model","metadata":{"papermill":{"duration":0.009143,"end_time":"2025-10-28T07:51:39.538580","exception":false,"start_time":"2025-10-28T07:51:39.529437","status":"completed"},"tags":[]}},{"id":"821b9551","cell_type":"code","source":"## --- 4. Train ML Model (CatBoost Training) ---\nprint(\"\\n--- 4. Train ML Model (Training with CatBoost) ---\")\nprint(\"🔥 Using StratifiedKFold and GPU training (requires GPU environment).\")\n\n# CatBoost parameters definition\ncat_params = {\n    'iterations': 3000,\n    'learning_rate': 0.02,\n    'depth': 6,\n    'eval_metric': 'AUC',\n    'random_seed': 42,\n    'verbose': 0,\n    'early_stopping_rounds': 1000,\n    'task_type': 'GPU',     # Modification to enable GPU acceleration\n    'metric_period': 20     # Reduce number of output metric prints to minimize slowdown\n}\n\nN_SPLITS = 10\nskf = StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=42)\n\noof_preds_cat = np.zeros(len(X))\ntest_preds_cat = np.zeros(len(X_test))","metadata":{"execution":{"iopub.status.busy":"2025-11-19T19:22:28.543575Z","iopub.execute_input":"2025-11-19T19:22:28.543775Z","iopub.status.idle":"2025-11-19T19:22:28.557462Z","shell.execute_reply.started":"2025-11-19T19:22:28.543752Z","shell.execute_reply":"2025-11-19T19:22:28.556736Z"},"papermill":{"duration":0.016546,"end_time":"2025-10-28T07:51:39.564266","exception":false,"start_time":"2025-10-28T07:51:39.547720","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"5c45fe51","cell_type":"code","source":"# Training and Cross-Validation loop\nfor fold, (train_index, val_index) in enumerate(skf.split(X, y)):\n  \n    print(f\"\\n--- FOLD {fold + 1}/{N_SPLITS} ---\") \n    \n    X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n    y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n    \n    model_cat = CatBoostClassifier(**cat_params)\n    \n    model_cat.fit(X_train, y_train,\n                  eval_set=[(X_val, y_val)],\n                  cat_features=categorical_features,\n                  verbose=0\n                  )\n    \n    val_preds = model_cat.predict_proba(X_val)[:, 1]\n    oof_preds_cat[val_index] = val_preds\n    test_preds_cat += model_cat.predict_proba(X_test)[:, 1] / N_SPLITS\n    \n    print(f\"Fold {fold + 1} AUC: {roc_auc_score(y_val, val_preds):.6f}\")\n    \n        # The preceding code block calculates val_preds and roc_auc_score\n    print(f\"Fold {fold+1} AUC: {roc_auc_score(y_val, val_preds)}\")\n    \n \noverall_auc_cat = roc_auc_score(y, oof_preds_cat)\nprint(f\"\\n===================================\")\nprint(f\"Overall OOF AUC (CatBoost): {overall_auc_cat:.6f}\")\nprint(f\"===================================\")","metadata":{"execution":{"iopub.status.busy":"2025-11-19T19:22:28.558231Z","iopub.execute_input":"2025-11-19T19:22:28.558490Z","iopub.status.idle":"2025-11-19T20:15:29.266478Z","shell.execute_reply.started":"2025-11-19T19:22:28.558468Z","shell.execute_reply":"2025-11-19T20:15:29.265588Z"},"papermill":{"duration":3157.313503,"end_time":"2025-10-28T08:44:16.887333","exception":false,"start_time":"2025-10-28T07:51:39.573830","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"86e6fd30","cell_type":"code","source":"# --- 5. Submission (Creating Submission File) ---\nprint(\"\\n--- 5. Submission (Creating Submission File) ---\")\nsubmission_df_cat = pd.DataFrame({'id': test_ids, 'target': test_preds_cat})\nsubmission_df_cat.to_csv('submission.csv', index=False)\n\nprint(\"✅ 'submission.csv' file created successfully.\")","metadata":{"execution":{"iopub.status.busy":"2025-11-19T20:15:29.267300Z","iopub.execute_input":"2025-11-19T20:15:29.267557Z","iopub.status.idle":"2025-11-19T20:15:30.064939Z","shell.execute_reply.started":"2025-11-19T20:15:29.267538Z","shell.execute_reply":"2025-11-19T20:15:30.063996Z"},"papermill":{"duration":0.759795,"end_time":"2025-10-28T08:44:17.657553","exception":false,"start_time":"2025-10-28T08:44:16.897758","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null}]}