{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":100725,"databundleVersionId":12103520,"sourceType":"competition"},{"sourceId":115000,"databundleVersionId":13897013,"sourceType":"competition"},{"sourceId":266181212,"sourceType":"kernelVersion"}],"dockerImageVersionId":31153,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **My Approaches :** \n 1) Ensemble of XGBoost and CatBoost (Using VotingRegressor) : Tree-based ensembles can learn complex patterns, interactions, and non-linear relationships .\n 2) Including LightGBM Regressor and Ridge as Meta Model: Chooses the Split of best gain for further consideration in levels. Ridge uses L2 Regularization\n 3) Stacking ensemble of XGBoost and Catboost with Hyperparameter Tuning Using Optuna : Giving Importance to that model more with more correct Predictions.","metadata":{}},{"cell_type":"markdown","source":"# First Approach (Ensemble of XGBoost and CatBoost):","metadata":{}},{"cell_type":"markdown","source":"## **Data Loading**","metadata":{}},{"cell_type":"code","source":"# Ensemble of XGBoost and CatBoost\n\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import mean_absolute_error\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import VotingRegressor\n\ntrain_df = pd.read_csv(\"/kaggle/input/carnival-risk-analytics-challenge/train.csv\")\ntest_df = pd.read_csv(\"/kaggle/input/carnival-risk-analytics-challenge/test.csv\")\n\nprint(\"Train shape:\", train_df.shape)\nprint(\"Test shape:\", test_df.shape)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-10T21:23:08.751525Z","iopub.execute_input":"2025-10-10T21:23:08.752057Z","iopub.status.idle":"2025-10-10T21:23:17.919063Z","shell.execute_reply.started":"2025-10-10T21:23:08.752034Z","shell.execute_reply":"2025-10-10T21:23:17.918383Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **Data Cleaning and Preprocessing:**","metadata":{}},{"cell_type":"code","source":"target_col = \"Premium Amount\"\n\n# Drop rows with missing target\ntrain_df = train_df.dropna(subset=[target_col])\n\n# Combine train + test for consistent preprocessing\ntrain_df[\"is_train\"] = 1\ntest_df[\"is_train\"] = 0\ncombined = pd.concat([train_df, test_df], ignore_index=True)\n\n# Handle datetime features\nif \"Policy Start Date\" in combined.columns:\n    combined[\"Policy Start Date\"] = pd.to_datetime(combined[\"Policy Start Date\"], errors='coerce')\n    combined[\"Policy_Year\"] = combined[\"Policy Start Date\"].dt.year\n    combined[\"Policy_Month\"] = combined[\"Policy Start Date\"].dt.month\n    combined.drop(columns=[\"Policy Start Date\"], inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-10T21:23:23.695270Z","iopub.execute_input":"2025-10-10T21:23:23.695587Z","iopub.status.idle":"2025-10-10T21:23:24.792760Z","shell.execute_reply.started":"2025-10-10T21:23:23.695565Z","shell.execute_reply":"2025-10-10T21:23:24.792026Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Encode categorical features\ncat_cols = combined.select_dtypes(include=[\"object\"]).columns\nfor col in cat_cols:\n    combined[col] = combined[col].fillna(\"Unknown\")\n    le = LabelEncoder()\n    combined[col] = le.fit_transform(combined[col])\n\n# Fill numeric NaNs\nnum_cols = combined.select_dtypes(include=[np.number]).columns\ncombined[num_cols] = combined[num_cols].fillna(combined[num_cols].median())\n\n# Split back\ntrain_df = combined[combined[\"is_train\"] == 1].drop(columns=[\"is_train\"])\ntest_df = combined[combined[\"is_train\"] == 0].drop(columns=[\"is_train\", target_col], errors=\"ignore\")\n\n# Features and target\nX = train_df.drop(columns=[target_col])\ny = train_df[target_col]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-10T21:23:28.725869Z","iopub.execute_input":"2025-10-10T21:23:28.726178Z","iopub.status.idle":"2025-10-10T21:23:33.589902Z","shell.execute_reply.started":"2025-10-10T21:23:28.726156Z","shell.execute_reply":"2025-10-10T21:23:33.588782Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, random_state=42)\n\n\nxgb_model = XGBRegressor(\n    n_estimators=600,\n    learning_rate=0.05,\n    max_depth=8,\n    subsample=0.8,\n    colsample_bytree=0.8,\n    random_state=42,\n    tree_method=\"hist\"\n)\n\ncat_model = CatBoostRegressor(\n    iterations=800,\n    learning_rate=0.05,\n    depth=8,\n    random_seed=42,\n    verbose=0\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-10T21:23:38.496800Z","iopub.execute_input":"2025-10-10T21:23:38.497882Z","iopub.status.idle":"2025-10-10T21:23:38.764734Z","shell.execute_reply.started":"2025-10-10T21:23:38.497840Z","shell.execute_reply":"2025-10-10T21:23:38.763991Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **Training:**","metadata":{}},{"cell_type":"code","source":"ensemble = VotingRegressor([\n    (\"xgb\", xgb_model),\n    (\"cat\", cat_model)\n])\n\nensemble.fit(X_train, y_train)\n\npreds_valid = ensemble.predict(X_valid)\nmae = mean_absolute_error(y_valid, preds_valid)\nprint(f\"Validation MAE: {mae:.4f}\")\n\ntest_preds = ensemble.predict(test_df)\n\nsubmission = pd.DataFrame({\n    \"id\": test_df[\"id\"],\n    \"Premium Amount\": test_preds\n})\n\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(\"submission.csv generated successfully!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-10T21:26:09.993215Z","iopub.execute_input":"2025-10-10T21:26:09.993568Z","iopub.status.idle":"2025-10-10T21:27:27.182454Z","shell.execute_reply.started":"2025-10-10T21:26:09.993548Z","shell.execute_reply":"2025-10-10T21:27:27.181549Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(submission.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-10T21:27:33.854310Z","iopub.execute_input":"2025-10-10T21:27:33.854590Z","iopub.status.idle":"2025-10-10T21:27:33.865055Z","shell.execute_reply.started":"2025-10-10T21:27:33.854572Z","shell.execute_reply":"2025-10-10T21:27:33.864037Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(submission.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-10T21:27:40.079371Z","iopub.execute_input":"2025-10-10T21:27:40.079704Z","iopub.status.idle":"2025-10-10T21:27:40.084820Z","shell.execute_reply.started":"2025-10-10T21:27:40.079676Z","shell.execute_reply":"2025-10-10T21:27:40.083949Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Second Approach (Including LightGBM Regressor and Ridge as Meta Model):","metadata":{}},{"cell_type":"markdown","source":"## **Data Loading:**","metadata":{}},{"cell_type":"code","source":" # Includes Feature Engineering + K-Fold + XGB, CatBoost, LightGBM + Ridge\n\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.linear_model import Ridge\nfrom sklearn.metrics import mean_absolute_error\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor, Pool\nimport lightgbm as lgb\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n#Preprocessing nd Feature Engineering\n\nTARGET = \"Premium Amount\"\nID_COL = \"id\"\n\ndf_train = train_df.copy()\ndf_test = test_df.copy()\n\n# Drop rows with missing target\ndf_train = df_train.dropna(subset=[TARGET])\n\n# Parse dates\nif \"Policy Start Date\" in df_train.columns:\n    for df in [df_train, df_test]:\n        df[\"Policy Start Date\"] = pd.to_datetime(df[\"Policy Start Date\"], errors=\"coerce\")\n        df[\"policy_year\"] = df[\"Policy Start Date\"].dt.year\n        df[\"policy_month\"] = df[\"Policy Start Date\"].dt.month\n        df[\"policy_day\"] = df[\"Policy Start Date\"].dt.day\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-10T21:27:43.472199Z","iopub.execute_input":"2025-10-10T21:27:43.472544Z","iopub.status.idle":"2025-10-10T21:27:48.866401Z","shell.execute_reply.started":"2025-10-10T21:27:43.472520Z","shell.execute_reply":"2025-10-10T21:27:48.865410Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **Feature Engineering:**","metadata":{}},{"cell_type":"code","source":"# Feature interactions\nfor df in [df_train, df_test]:\n    df[\"Income_per_Dependent\"] = df[\"Annual Income\"] / (df[\"Number of Children\"].fillna(0)+1)\n    df[\"Health_per_Age\"] = df[\"Health Score\"] / (df[\"Age\"]+1)\n    df[\"Vehicle_per_Duration\"] = df[\"Vehicle Age\"] / (df[\"Insurance Duration\"]+1)\n    df[\"Income_per_VehicleAge\"] = df[\"Annual Income\"] / (df[\"Vehicle Age\"] + 1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-10T21:28:08.669202Z","iopub.execute_input":"2025-10-10T21:28:08.669571Z","iopub.status.idle":"2025-10-10T21:28:08.699544Z","shell.execute_reply.started":"2025-10-10T21:28:08.669548Z","shell.execute_reply":"2025-10-10T21:28:08.698170Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Binning skewed numeric features\nfor df in [df_train, df_test]:\n    df[\"Income_bin\"] = pd.qcut(df[\"Annual Income\"].rank(method='first'), 10, labels=False)\n    df[\"CreditScore_bin\"] = pd.qcut(df[\"Credit Score\"].rank(method='first'), 10, labels=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-10T21:28:13.531366Z","iopub.execute_input":"2025-10-10T21:28:13.531988Z","iopub.status.idle":"2025-10-10T21:28:14.180835Z","shell.execute_reply.started":"2025-10-10T21:28:13.531961Z","shell.execute_reply":"2025-10-10T21:28:14.179902Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **Data Cleaning and Preprocessing:**","metadata":{}},{"cell_type":"code","source":"# Separate numeric & categorical\nnum_cols = df_train.select_dtypes(include=[np.number]).columns.tolist()\nnum_cols = [c for c in num_cols if c not in [TARGET, ID_COL]]\ncat_cols = df_train.select_dtypes(include=[\"object\"]).columns.tolist()\n\n# Fill missing\nfor c in num_cols:\n    median_val = df_train[c].median()\n    df_train[c] = df_train[c].fillna(median_val)\n    df_test[c] = df_test[c].fillna(median_val)\n\nfor c in cat_cols:\n    df_train[c] = df_train[c].fillna(\"Unknown\").astype(str)\n    df_test[c] = df_test[c].fillna(\"Unknown\").astype(str)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-10T21:28:18.017071Z","iopub.execute_input":"2025-10-10T21:28:18.017391Z","iopub.status.idle":"2025-10-10T21:28:18.624125Z","shell.execute_reply.started":"2025-10-10T21:28:18.017369Z","shell.execute_reply":"2025-10-10T21:28:18.623215Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Encode categorical features for XGB/LightGBM\nle_dict = {}\nfor c in cat_cols:\n    le = LabelEncoder()\n    le.fit(pd.concat([df_train[c], df_test[c]], axis=0))\n    df_train[c] = le.transform(df_train[c])\n    df_test[c] = le.transform(df_test[c])\n    le_dict[c] = le","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-10T21:28:22.636961Z","iopub.execute_input":"2025-10-10T21:28:22.637379Z","iopub.status.idle":"2025-10-10T21:28:22.643967Z","shell.execute_reply.started":"2025-10-10T21:28:22.637353Z","shell.execute_reply":"2025-10-10T21:28:22.642866Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **Training and KFold Iterations:**","metadata":{}},{"cell_type":"code","source":"FEATURES = num_cols + cat_cols\nX = df_train[FEATURES].copy()\ny = np.log1p(df_train[TARGET].values)  #log transform target\nX_test = df_test[FEATURES].copy()\n\n\nN_SPLITS = 5\nkf = KFold(n_splits=N_SPLITS, shuffle=True, random_state=42)\n\n#Out of fold predictions\noof_preds = np.zeros(X.shape[0])\ntest_preds_xgb = np.zeros(X_test.shape[0])\ntest_preds_cb = np.zeros(X_test.shape[0])\ntest_preds_lgb = np.zeros(X_test.shape[0])\n\nfor fold, (train_idx, val_idx) in enumerate(kf.split(X, y)):\n    print(f\"\\n=== Fold {fold+1} ===\")\n    \n    X_tr, X_val = X.iloc[train_idx], X.iloc[val_idx]\n    y_tr, y_val = y[train_idx], y[val_idx]\n    \n    xgb_model = XGBRegressor(\n        n_estimators=1000,\n        learning_rate=0.05,\n        max_depth=7,\n        subsample=0.8,\n        colsample_bytree=0.7,\n        random_state=42,\n        tree_method=\"hist\"\n    )\n    xgb_model.fit(\n        X_tr, y_tr,\n        eval_set=[(X_val, y_val)],\n        early_stopping_rounds=50,\n        verbose=50\n    )\n    val_pred_xgb = xgb_model.predict(X_val)\n    test_preds_xgb += xgb_model.predict(X_test) / N_SPLITS\n    \n    cat_model = CatBoostRegressor(\n        iterations=1000,\n        learning_rate=0.05,\n        depth=6,\n        random_seed=42,\n        early_stopping_rounds=50,\n        verbose=50\n    )\n    cat_model.fit(\n        Pool(X_tr, y_tr),\n        eval_set=Pool(X_val, y_val)\n    )\n    val_pred_cb = cat_model.predict(X_val)\n    test_preds_cb += cat_model.predict(X_test) / N_SPLITS\n    lgb_model = lgb.LGBMRegressor(\n        n_estimators=1000,\n        learning_rate=0.05,\n        max_depth=7,\n        subsample=0.8,\n        colsample_bytree=0.7,\n        random_state=42\n    )\n    lgb_model.fit(\n        X_tr, y_tr,\n        eval_set=[(X_val, y_val)],\n        eval_metric='rmse',\n        callbacks=[lgb.early_stopping(stopping_rounds=50)]\n    )\n    val_pred_lgb = lgb_model.predict(X_val)\n    test_preds_lgb += lgb_model.predict(X_test) / N_SPLITS\n    \n    val_pred_ensemble = (val_pred_xgb + val_pred_cb + val_pred_lgb)/3\n    oof_preds[val_idx] = val_pred_ensemble\n    \n    fold_mae = mean_absolute_error(np.expm1(y_val), np.expm1(val_pred_ensemble))\n    print(f\"Fold {fold+1} MAE (original scale): {fold_mae:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-10T21:28:26.007487Z","iopub.execute_input":"2025-10-10T21:28:26.007784Z","iopub.status.idle":"2025-10-10T21:31:33.531798Z","shell.execute_reply.started":"2025-10-10T21:28:26.007764Z","shell.execute_reply":"2025-10-10T21:31:33.530862Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **Ridge as Meta Model and Stacking Ensemble:**","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import Ridge\n\nmeta_train = np.vstack([oof_preds, oof_preds, oof_preds]).T  # simple stacking\nmeta_test = np.vstack([test_preds_xgb, test_preds_cb, test_preds_lgb]).T\n\nridge_meta = Ridge(alpha=1.0)\nridge_meta.fit(meta_train, np.log1p(df_train[TARGET]))\nfinal_test_preds_log = ridge_meta.predict(meta_test)\nfinal_test_preds = np.expm1(final_test_preds_log)\n\nsubmission = pd.DataFrame({\n    ID_COL: df_test[ID_COL].values,\n    TARGET: final_test_preds\n})\nsubmission[TARGET] = submission[TARGET].clip(lower=0)\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(\"\\nSubmission saved as submission.csv\")\ndisplay(submission.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-10T21:31:59.732016Z","iopub.execute_input":"2025-10-10T21:31:59.732678Z","iopub.status.idle":"2025-10-10T21:32:00.931209Z","shell.execute_reply.started":"2025-10-10T21:31:59.732651Z","shell.execute_reply":"2025-10-10T21:32:00.930487Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Final Approach (Adding Hyperparameter Tuning Using Optuna):","metadata":{}},{"cell_type":"markdown","source":"## **Data Loading:**","metadata":{}},{"cell_type":"code","source":"#Ensemble of XGBoost and CatBoost + Optuna Tuning for XGBoost\n\n\nimport pandas as pd\nimport numpy as np\nimport optuna\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import mean_absolute_error\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import VotingRegressor\n\n\ntrain_df = pd.read_csv(\"/kaggle/input/carnival-risk-analytics-challenge/train.csv\")\ntest_df = pd.read_csv(\"/kaggle/input/carnival-risk-analytics-challenge/test.csv\")\n\nprint(\"Train shape:\", train_df.shape)\nprint(\"Test shape:\", test_df.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-10T21:33:56.363353Z","iopub.execute_input":"2025-10-10T21:33:56.363625Z","iopub.status.idle":"2025-10-10T21:34:01.231496Z","shell.execute_reply.started":"2025-10-10T21:33:56.363607Z","shell.execute_reply":"2025-10-10T21:34:01.230642Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **Data Cleaning and Preprocessing:**","metadata":{}},{"cell_type":"code","source":"#Preprocess\n\ntarget_col = \"Premium Amount\"\ntrain_df = train_df.dropna(subset=[target_col])\n\ntrain_df[\"is_train\"] = 1\ntest_df[\"is_train\"] = 0\ncombined = pd.concat([train_df, test_df], ignore_index=True)\n\nif \"Policy Start Date\" in combined.columns:\n    combined[\"Policy Start Date\"] = pd.to_datetime(combined[\"Policy Start Date\"], errors='coerce')\n    combined[\"Policy_Year\"] = combined[\"Policy Start Date\"].dt.year\n    combined[\"Policy_Month\"] = combined[\"Policy Start Date\"].dt.month\n    combined.drop(columns=[\"Policy Start Date\"], inplace=True)\n\ncat_cols = combined.select_dtypes(include=[\"object\"]).columns\nfor col in cat_cols:\n    combined[col] = combined[col].fillna(\"Unknown\")\n    le = LabelEncoder()\n    combined[col] = le.fit_transform(combined[col])\n\nnum_cols = combined.select_dtypes(include=[np.number]).columns\ncombined[num_cols] = combined[num_cols].fillna(combined[num_cols].median())\n\ntrain_df = combined[combined[\"is_train\"] == 1].drop(columns=[\"is_train\"])\ntest_df = combined[combined[\"is_train\"] == 0].drop(columns=[\"is_train\", target_col], errors=\"ignore\")\n\nX = train_df.drop(columns=[target_col])\ny = train_df[target_col]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-10T21:34:35.145747Z","iopub.execute_input":"2025-10-10T21:34:35.146544Z","iopub.status.idle":"2025-10-10T21:34:40.756053Z","shell.execute_reply.started":"2025-10-10T21:34:35.146514Z","shell.execute_reply":"2025-10-10T21:34:40.755129Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Train / Validation Split\nX_train, X_valid, y_train, y_valid = train_test_split(\n    X, y, test_size=0.2, random_state=42\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-10T21:35:21.523860Z","iopub.execute_input":"2025-10-10T21:35:21.524255Z","iopub.status.idle":"2025-10-10T21:35:21.765930Z","shell.execute_reply.started":"2025-10-10T21:35:21.524205Z","shell.execute_reply":"2025-10-10T21:35:21.765145Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **Hyperparameter Tuning of XGBoost:**","metadata":{}},{"cell_type":"code","source":"#Optuna Hyperparameter Tuning (XGBoost)\n\ndef objective(trial):\n    params = {\n        \"n_estimators\": trial.suggest_int(\"n_estimators\", 500, 2000),\n        \"learning_rate\": trial.suggest_float(\"learning_rate\", 0.01, 0.3, log=True),\n        \"max_depth\": trial.suggest_int(\"max_depth\", 4, 12),\n        \"min_child_weight\": trial.suggest_int(\"min_child_weight\", 1, 10),\n        \"subsample\": trial.suggest_float(\"subsample\", 0.6, 1.0),\n        \"colsample_bytree\": trial.suggest_float(\"colsample_bytree\", 0.6, 1.0),\n        \"gamma\": trial.suggest_float(\"gamma\", 0, 5),\n        \"reg_alpha\": trial.suggest_float(\"reg_alpha\", 0, 10),\n        \"reg_lambda\": trial.suggest_float(\"reg_lambda\", 0, 10),\n        \"random_state\": 42,\n        \"tree_method\": \"hist\"\n    }\n\n    model = XGBRegressor(**params)\n    model.fit(X_train, y_train, eval_set=[(X_valid, y_valid)], verbose=False)\n    preds = model.predict(X_valid)\n    mae = mean_absolute_error(y_valid, preds)\n    return mae\n\n\nstudy = optuna.create_study(direction=\"minimize\")\nstudy.optimize(objective, n_trials=25, show_progress_bar=True)\nprint(\"Best parameters:\", study.best_params)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-10T21:36:46.056587Z","iopub.execute_input":"2025-10-10T21:36:46.057475Z","iopub.status.idle":"2025-10-10T21:59:41.252855Z","shell.execute_reply.started":"2025-10-10T21:36:46.057446Z","shell.execute_reply":"2025-10-10T21:59:41.252138Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Final Models (Using Best XGB Params)\n\nxgb_best = XGBRegressor(**study.best_params)\n\ncat_model = CatBoostRegressor(\n    iterations=800,\n    learning_rate=0.05,\n    depth=8,\n    random_seed=42,\n    verbose=0\n)\n\n# Ensemble(Voting Regressor)\nensemble = VotingRegressor([\n    (\"xgb\", xgb_best),\n    (\"cat\", cat_model)\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-10T22:00:28.502349Z","iopub.execute_input":"2025-10-10T22:00:28.502667Z","iopub.status.idle":"2025-10-10T22:00:28.517548Z","shell.execute_reply.started":"2025-10-10T22:00:28.502647Z","shell.execute_reply":"2025-10-10T22:00:28.516437Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **Ensemble after Selecting Best Hyperparameters and Final Submission:**","metadata":{}},{"cell_type":"code","source":"ensemble.fit(X_train, y_train)\n\npreds_valid = ensemble.predict(X_valid)\nmae = mean_absolute_error(y_valid, preds_valid)\nprint(f\"Validation MAE after tuning: {mae:.4f}\")\n\n\ntest_preds = ensemble.predict(test_df)\n\nsubmission = pd.DataFrame({\n    \"id\": test_df[\"id\"],\n    \"Premium Amount\": test_preds\n})\n\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(\"submission.csv generated successfully!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-10T22:03:45.806643Z","iopub.execute_input":"2025-10-10T22:03:45.806970Z","iopub.status.idle":"2025-10-10T22:05:32.678080Z","shell.execute_reply.started":"2025-10-10T22:03:45.806945Z","shell.execute_reply":"2025-10-10T22:05:32.677139Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(submission.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-10T22:06:27.950024Z","iopub.execute_input":"2025-10-10T22:06:27.950896Z","iopub.status.idle":"2025-10-10T22:06:27.957875Z","shell.execute_reply.started":"2025-10-10T22:06:27.950862Z","shell.execute_reply":"2025-10-10T22:06:27.956690Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Conclusion :\n## 1) StratifiedKFold ensures equal class proportions in each fold that help model ensemble drastically reduce MAE after each fold iteration during training.\n## 2) Stacking ensemble with Ridge ensures giving high importance to XGBoost that have given more predictions to this train dataset.\n## 3) Hyperparamter Tuning provided a good performance of the model ensemble ","metadata":{}}]}