{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":17000,"databundleVersionId":895292,"sourceType":"competition"}],"dockerImageVersionId":31154,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 1. Import Libraries ---\nimport numpy as np\nimport pandas as pd\nfrom catboost import CatBoostClassifier\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\nimport warnings\nimport gc  # Garbage Collector for memory management\n\n# Suppress unnecessary warnings\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 2. Configuration and Setup ---\n\n# Set a random seed for reproducibility (so results are the same every time)\nSEED = 42\nnp.random.seed(SEED)\n\n# Set the number of folds for cross-validation\n# 10 is robust, 3 or 5 is faster for testing\nN_SPLITS = 10\n\n# Define the path to your data files\nDATA_PATH = '../input/cat-in-the-dat-ii/'","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 3. Load Data ---\nprint(\"Loading data...\")\ntry:\n    # Read the training, test, and submission files\n    train_df = pd.read_csv(f'{DATA_PATH}train.csv')\n    test_df = pd.read_csv(f'{DATA_PATH}test.csv')\n    submission_df = pd.read_csv(f'{DATA_PATH}sample_submission.csv')\n\n    print(f\"Train shape: {train_df.shape}\")\n    print(f\"Test shape: {test_df.shape}\")\n\nexcept FileNotFoundError:\n    print(f\"Error: Data files not found in {DATA_PATH}\")\n    # Handle the error, maybe stop the script\n    # For now, we'll assume the files loaded","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 4. Exploratory Data Analysis (EDA) ---\n# This section is for understanding the data\n\nprint(\"\\n--- Starting EDA ---\")\n\n# Check the target variable distribution\nprint(\"\\nTarget Distribution:\")\nprint(train_df['target'].value_counts(normalize=True))\n\n# Check for columns that have missing (null) values\nprint(\"\\nMissing Values (Train):\")\nmissing_train = train_df.isnull().sum()\nprint(missing_train[missing_train > 0])\n\n# Check the number of unique values (cardinality) for each feature\nprint(\"\\nFeature Cardinality (Unique Values):\")\nfor col in train_df.columns:\n    if col not in ['id', 'target']:\n        print(f\"{col}: {train_df[col].nunique()} unique values\")\n\nprint(\"--- EDA Complete ---\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 5. Feature Engineering and Preprocessing ---\nprint(\"\\n--- Starting Feature Engineering ---\")\n\n# Create copies for processing\n# X = features, y = target\nX = train_df.drop(['id', 'target'], axis=1)\ny = train_df['target']\nX_test = test_df.drop('id', axis=1)\n\n# Save the test IDs for the final submission file\nX_test_ids = test_df['id']\n\n# --- 5a. Handle Missing Values ---\nprint(\"Handling missing values...\")\n# Loop through all columns\nfor col in X.columns:\n    # Find columns that are 'object' (text/string) type\n    if X[col].dtype == 'object':\n        # Fill missing values with the string \"MISSING\"\n        # This treats \"missing\" as its own category\n        X[col] = X[col].fillna(\"MISSING\")\n        X_test[col] = X_test[col].fillna(\"MISSING\")\n\n# Note: We leave numeric NaNs (like in bin_0) as is.\n# CatBoost can handle them natively.\n\n# --- 5b. Manual Feature Mapping ---\nprint(\"Mapping manual ordinal/binary features...\")\n# These features have a clear order, so we map them to numbers\n\n# ord_1: Novice (0) < Contributor (1) < ...\nord_1_mapping = {\n    'Novice': 0, 'Contributor': 1, 'Expert': 2,\n    'Master': 3, 'Grandmaster': 4, \"MISSING\": -1\n}\nX['ord_1'] = X['ord_1'].map(ord_1_mapping)\nX_test['ord_1'] = X_test['ord_1'].map(ord_1_mapping)\n\n# ord_2: Freezing (0) < Cold (1) < ...\nord_2_mapping = {\n    'Freezing': 0, 'Cold': 1, 'Warm': 2, 'Hot': 3,\n    'Boiling Hot': 4, 'Lava Hot': 5, \"MISSING\": -1\n}\nX['ord_2'] = X['ord_2'].map(ord_2_mapping)\nX_test['ord_2'] = X_test['ord_2'].map(ord_2_mapping)\n\n# bin_3: T/F to 1/0\nX['bin_3'] = X['bin_3'].map({'T': 1, 'F': 0, 'MISSING': -1})\nX_test['bin_3'] = X_test['bin_3'].map({'T': 1, 'F': 0, 'MISSING': -1})\n\n# bin_4: Y/N to 1/0\nX['bin_4'] = X['bin_4'].map({'Y': 1, 'N': 0, 'MISSING': -1})\nX_test['bin_4'] = X_test['bin_4'].map({'Y': 1, 'N': 0, 'MISSING': -1})\n\n# --- 5c. Cyclic Feature Engineering ---\nprint(\"Creating cyclic features for day and month...\")\n# This helps the model understand that day 7 is next to day 1,\n# and month 12 is next to month 1.\n\n# Fill any missing day/month with 0 (a neutral placeholder)\nX['day'] = X['day'].fillna(0)\nX_test['day'] = X_test['day'].fillna(0)\nX['month'] = X['month'].fillna(0)\nX_test['month'] = X_test['month'].fillna(0)\n\n# Create sin/cos features for day of the week (max=7)\nX['day_sin'] = np.sin(2 * np.pi * X['day'] / 7.0)\nX['day_cos'] = np.cos(2 * np.pi * X['day'] / 7.0)\nX_test['day_sin'] = np.sin(2 * np.pi * X_test['day'] / 7.0)\nX_test['day_cos'] = np.cos(2 * np.pi * X_test['day'] / 7.0)\n\n# Create sin/cos features for month of the year (max=12)\nX['month_sin'] = np.sin(2 * np.pi * X['month'] / 12.0)\nX['month_cos'] = np.cos(2 * np.pi * X['month'] / 12.0)\nX_test['month_sin'] = np.sin(2 * np.pi * X_test['month'] / 12.0)\nX_test['month_cos'] = np.cos(2 * np.pi * X_test['month'] / 12.0)\n\n# Drop the original 'day' and 'month' columns\nX = X.drop(['day', 'month'], axis=1)\nX_test = X_test.drop(['day', 'month'], axis=1)\n\n# --- 5d. Advanced 'ord_5' Splitting ---\nprint(\"Splitting ord_5...\")\n# 'ord_5' has values like 'aZ', 'bA'.\n# Splitting them into two separate features can be powerful.\nX['ord_5_1'] = X['ord_5'].str[0]\nX['ord_5_2'] = X['ord_5'].str[1]\nX_test['ord_5_1'] = X_test['ord_5'].str[0]\nX_test['ord_5_2'] = X_test['ord_5'].str[1]\n\n# Drop the original 'ord_5' column\nX = X.drop('ord_5', axis=1)\nX_test = X_test.drop('ord_5', axis=1)\n\n# --- 5e. Define and Prepare Categorical Features ---\nprint(\"Defining and converting categorical features...\")\n\n# List all features we want CatBoost to treat as categories\ncat_features = [\n    'bin_0', 'bin_1', 'bin_2',\n    'nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4',\n    'nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9',\n    'ord_0', 'ord_3', 'ord_4',\n    'ord_5_1', 'ord_5_2'  # Our new engineered features\n]\n\n# Convert all these categorical columns to 'str' type\n# This is a safety step to ensure CatBoost treats them as categories,\n# even if they look like numbers (e.g., bin_0).\nfor col in cat_features:\n    X[col] = X[col].astype(str)\n    X_test[col] = X_test[col].astype(str)\n\n# Fill NaNs in numeric columns that we didn't map\n# (CatBoost can handle NaNs, but filling with a placeholder is explicit)\nnum_cols = ['bin_0', 'bin_1', 'bin_2']\nX[num_cols] = X[num_cols].fillna(-1)\nX_test[num_cols] = X_test[num_cols].fillna(-1)\n\nprint(\"--- Preprocessing and FE Complete ---\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 6. Model Training (Tuned CatBoost) ---\nprint(f\"\\n--- Starting CatBoost {N_SPLITS}-Fold Training ---\")\n\n# Setup the K-Fold cross-validation\n# StratifiedKFold ensures each fold has the same balance of targets (0s and 1s)\nskf = StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED)\n\n# Create arrays to store out-of-fold (OOF) predictions and test predictions\noof_preds = np.zeros(len(X))\ntest_preds = np.zeros(len(X_test))\nmodels = []  # List to store the trained model from each fold\n\n# Start the cross-validation loop\nfor fold, (train_idx, val_idx) in enumerate(skf.split(X, y)):\n    print(f\"--- Fold {fold+1}/{N_SPLITS} ---\")\n\n    # Split the data into training and validation sets for this fold\n    X_train, y_train = X.iloc[train_idx], y.iloc[train_idx]\n    X_val, y_val = X.iloc[val_idx], y.iloc[val_idx]\n\n    # Initialize the CatBoostClassifier with tuned parameters\n    model = CatBoostClassifier(\n        iterations=3000,\n        learning_rate=0.02,\n        depth=7,\n        eval_metric='AUC',\n        random_seed=SEED,\n        task_type=\"GPU\",  # Use \"GPU\" or \"CPU\"\n        early_stopping_rounds=200,  # Stop if AUC doesn't improve for 200 rounds\n        verbose=1000,  # Print progress every 500 trees\n        cat_features=cat_features,\n        nan_mode='Min',  # How to handle missing values in numeric features\n\n        # --- Regularization Parameters ---\n        l2_leaf_reg=3.0,  # L2 regularization\n        subsample=0.8,  # Use 80% of data for training each tree\n        bootstrap_type='Poisson',  # Required for 'subsample' on GPU\n    )\n\n    # Train the model\n    model.fit(\n        X_train, y_train,\n        eval_set=(X_val, y_val),\n        use_best_model=True  # Save the model from the best iteration\n    )\n\n    # Make predictions on the validation set\n    val_preds = model.predict_proba(X_val)[:, 1]\n    oof_preds[val_idx] = val_preds\n\n    # Make predictions on the test set\n    # We average the predictions from all folds\n    test_preds += model.predict_proba(X_test)[:, 1] / N_SPLITS\n    models.append(model)\n\n    print(f\"Fold {fold+1} AUC: {roc_auc_score(y_val, val_preds)}\")\n\n    # Clean up memory\n    del X_train, y_train, X_val, y_val\n    gc.collect()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 7. Final Results ---\nprint(\"\\n--- Training Complete ---\")\n\n# Calculate and print the overall Out-of-Fold (OOF) AUC score\n# This is our most reliable measure of the model's performance\noverall_auc = roc_auc_score(y, oof_preds)\nprint(f\"Overall CV AUC (Tuned CatBoost): {overall_auc}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 8. Create Submission File ---\nprint(\"Creating submission file...\")\n\n# Create a DataFrame with the test IDs and our averaged predictions\nsubmission_df_cb = pd.DataFrame({'id': X_test_ids, 'target': test_preds})\n\n# Save the DataFrame to a CSV file\nsubmission_df_cb.to_csv('submission.csv', index=False)\n\nprint(\"\\nAll steps complete! 'submission.csv' is ready.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}