{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":104857,"databundleVersionId":12651513,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import mean_squared_error\nfrom xgboost import XGBRegressor\nimport warnings\nwarnings.filterwarnings('ignore')\n\nprint(\"🚀 XGBOOST SOLO MODE - SPEED & PERFORMANCE 🎯\")\n\nDATASET_PATH = '/kaggle/input/burnout-datathon-ieeecsmuj/'\ntrain = pd.read_csv(DATASET_PATH + 'train.csv')\nval = pd.read_csv(DATASET_PATH + 'val.csv')\ntest = pd.read_csv(DATASET_PATH + 'test.csv')\nsample_submission = pd.read_csv(DATASET_PATH + 'sample_submission.csv')\n\ndef power_features(df):\n    \"\"\"Focused feature engineering for XGBoost\"\"\"\n    df = df.copy()\n    \n    # Speed transformations (most important)\n    if 'Avg_Speed_kmh' in df.columns:\n        df['Speed_Sq'] = df['Avg_Speed_kmh'] ** 2\n        df['Speed_Cube'] = df['Avg_Speed_kmh'] ** 3\n        df['Speed_Inv'] = 1 / (df['Avg_Speed_kmh'] + 0.01)\n        df['Speed_Log'] = np.log1p(df['Avg_Speed_kmh'])\n        df['Speed_Sqrt'] = np.sqrt(df['Avg_Speed_kmh'])\n    \n    # Grid transformations\n    if 'Grid_Position' in df.columns:\n        df['Grid_Inv'] = 1 / (df['Grid_Position'] + 1)\n        df['Grid_Log'] = np.log1p(df['Grid_Position'])\n        df['Grid_Sq'] = df['Grid_Position'] ** 2\n    \n    # Temperature\n    if 'Track_Temperature_Celsius' in df.columns:\n        df['Temp_Sq'] = df['Track_Temperature_Celsius'] ** 2\n        df['Temp_Log'] = np.log1p(df['Track_Temperature_Celsius'] + 50)\n    \n    # Power interactions\n    if 'Avg_Speed_kmh' in df.columns and 'Grid_Position' in df.columns:\n        df['Speed_Grid_Ratio'] = df['Avg_Speed_kmh'] / (df['Grid_Position'] + 1)\n        df['Speed_Grid_Prod'] = df['Avg_Speed_kmh'] * df['Grid_Position']\n        df['Speed_Grid_Diff'] = df['Avg_Speed_kmh'] - df['Grid_Position']\n        df['Speed_Sq_Grid'] = (df['Avg_Speed_kmh'] ** 2) / (df['Grid_Position'] + 1)\n    \n    if 'Track_Temperature_Celsius' in df.columns and 'Avg_Speed_kmh' in df.columns:\n        df['Temp_Speed_Ratio'] = df['Track_Temperature_Celsius'] / (df['Avg_Speed_kmh'] + 0.01)\n        df['Temp_Speed_Prod'] = df['Track_Temperature_Celsius'] * df['Avg_Speed_kmh']\n    \n    # Penalty interactions (ensure numeric)\n    if 'Penalty' in df.columns and 'Avg_Speed_kmh' in df.columns:\n        # Make sure Penalty is numeric\n        if df['Penalty'].dtype == 'object':\n            df['Penalty'] = pd.to_numeric(df['Penalty'], errors='coerce').fillna(0)\n        df['Penalty_Speed'] = df['Penalty'] * df['Avg_Speed_kmh']\n    \n    return df\n\ndef turbo_prep():\n    \"\"\"Optimized preprocessing for XGBoost\"\"\"\n    tr, v, te = train.copy(), val.copy(), test.copy()\n    \n    # Smart missing value handling\n    for df in [tr, v, te]:\n        # Penalty special treatment - convert to numeric first\n        if 'Penalty' in df.columns:\n            df['Has_Penalty'] = df['Penalty'].notna().astype(int)\n            # Convert to numeric, coercing errors to NaN\n            df['Penalty'] = pd.to_numeric(df['Penalty'], errors='coerce')\n            df['Penalty_Zero'] = (df['Penalty'] == 0).astype(int)\n            df['Penalty'] = df['Penalty'].fillna(0)\n        \n        # Numeric columns - use different strategies\n        num_cols = df.select_dtypes(include=[np.number]).columns\n        for col in num_cols:\n            if col != 'Lap_Time_Seconds':\n                # For highly skewed data, use median; otherwise mean\n                if abs(df[col].skew()) > 1:\n                    fill_val = df[col].median()\n                else:\n                    fill_val = df[col].mean()\n                df[col] = df[col].fillna(fill_val)\n    \n    # Apply power features\n    tr, v, te = power_features(tr), power_features(v), power_features(te)\n    \n    # Enhanced categorical encoding\n    cat_cols = tr.select_dtypes(include='object').columns.tolist()\n    for col in cat_cols:\n        # Frequency encoding first\n        freq_map = tr[col].value_counts().to_dict()\n        for df in [tr, v, te]:\n            df[f'{col}_freq'] = df[col].map(freq_map).fillna(0)\n        \n        # Then label encoding\n        le = LabelEncoder()\n        all_vals = pd.concat([tr[col], v[col], te[col]]).astype(str)\n        le.fit(all_vals)\n        tr[col] = le.transform(tr[col].astype(str))\n        v[col] = le.transform(v[col].astype(str))\n        te[col] = le.transform(te[col].astype(str))\n    \n    # Drop irrelevant columns\n    drop_cols = ['Unique ID', 'Rider_ID', 'Rider', 'Rider_name', 'Team_name', 'Bike_name', 'Shortname']\n    for col in drop_cols:\n        for df in [tr, v, te]:\n            if col in df.columns:\n                df.drop(columns=[col], inplace=True)\n    \n    return tr, v, te\n\nprint(\"⚡ Turbo preprocessing...\")\ntrain_df, val_df, test_df = turbo_prep()\n\nX_train = train_df.drop('Lap_Time_Seconds', axis=1)\ny_train = train_df['Lap_Time_Seconds']\nX_val = val_df.drop('Lap_Time_Seconds', axis=1)\ny_val = val_df['Lap_Time_Seconds']\nX_test = test_df.copy()\n\nprint(f\"🎯 Features engineered: {X_train.shape[1]}\")\n\n# Outlier removal (keep more aggressive)\nz_scores = np.abs((y_train - y_train.mean()) / y_train.std())\nmask = z_scores < 2.5  # More aggressive outlier removal\nX_train_clean, y_train_clean = X_train[mask], y_train[mask]\nprint(f\"🧹 Training samples: {len(X_train_clean)} (removed {len(X_train) - len(X_train_clean)} outliers)\")\n\n# OPTIMIZED XGBOOST PARAMETERS\nbase_params = {\n    'n_estimators': 1500,\n    'max_depth': 8,\n    'learning_rate': 0.06,\n    'subsample': 0.85,\n    'colsample_bytree': 0.85,\n    'colsample_bylevel': 0.85,\n    'reg_alpha': 8,\n    'reg_lambda': 8,\n    'min_child_weight': 4,\n    'gamma': 1.5,\n    'random_state': 42,\n    'tree_method': 'hist',\n    'n_jobs': -1,\n    'objective': 'reg:squarederror'\n}\n\nprint(\"🚀 Training optimized XGBoost...\")\n\n# Train base model\nmodel = XGBRegressor(**base_params)\nmodel.fit(X_train_clean, y_train_clean, \n          eval_set=[(X_val, y_val)], \n          early_stopping_rounds=75, \n          verbose=False)\n\nbase_pred = model.predict(X_val)\nbase_rmse = mean_squared_error(y_val, base_pred, squared=False)\nprint(f\"🎯 Base XGBoost RMSE: {base_rmse:.4f}\")\n\n# QUICK HYPERPARAMETER VARIANTS (parallel testing)\nvariants = [\n    # Variant 1: Lower LR, more trees\n    {'learning_rate': 0.04, 'n_estimators': 2000, 'max_depth': 7},\n    # Variant 2: Higher regularization\n    {'reg_alpha': 15, 'reg_lambda': 15, 'min_child_weight': 6},\n    # Variant 3: Different tree structure\n    {'max_depth': 9, 'min_child_weight': 2, 'gamma': 2.5}\n]\n\nbest_rmse = base_rmse\nbest_model = model\nbest_name = \"Base\"\n\nprint(\"⚡ Testing variants...\")\nfor i, variant_params in enumerate(variants):\n    variant = base_params.copy()\n    variant.update(variant_params)\n    \n    var_model = XGBRegressor(**variant)\n    var_model.fit(X_train_clean, y_train_clean, \n                  eval_set=[(X_val, y_val)], \n                  early_stopping_rounds=50, \n                  verbose=False)\n    \n    var_pred = var_model.predict(X_val)\n    var_rmse = mean_squared_error(y_val, var_pred, squared=False)\n    \n    print(f\"🔧 Variant {i+1} RMSE: {var_rmse:.4f}\")\n    \n    if var_rmse < best_rmse:\n        best_rmse = var_rmse\n        best_model = var_model\n        best_name = f\"Variant {i+1}\"\n\nprint(f\"🏆 BEST MODEL: {best_name} with RMSE: {best_rmse:.4f}\")\n\n# FINAL BOOST: Ensemble of top 2 models if improvement is significant\nif best_rmse < base_rmse - 0.05:  # Significant improvement\n    print(\"🚀 Creating mini-ensemble...\")\n    \n    # Predictions from both models\n    pred1 = model.predict(X_val)\n    pred2 = best_model.predict(X_val)\n    \n    # Optimal weight (simple search)\n    best_w = 0.5\n    best_ensemble_rmse = float('inf')\n    \n    for w in [0.3, 0.4, 0.5, 0.6, 0.7]:\n        ensemble_pred = w * pred1 + (1-w) * pred2\n        ensemble_rmse = mean_squared_error(y_val, ensemble_pred, squared=False)\n        if ensemble_rmse < best_ensemble_rmse:\n            best_ensemble_rmse = ensemble_rmse\n            best_w = w\n    \n    print(f\"🎯 Ensemble RMSE: {best_ensemble_rmse:.4f} (weight: {best_w})\")\n    \n    if best_ensemble_rmse < best_rmse:\n        # Use ensemble for final prediction\n        test_pred1 = model.predict(X_test)\n        test_pred2 = best_model.predict(X_test)\n        final_pred = best_w * test_pred1 + (1-best_w) * test_pred2\n        final_rmse = best_ensemble_rmse\n        print(\"🏆 Using ENSEMBLE for final prediction!\")\n    else:\n        final_pred = best_model.predict(X_test)\n        final_rmse = best_rmse\n        print(\"🏆 Using BEST SINGLE MODEL for final prediction!\")\nelse:\n    final_pred = best_model.predict(X_test)\n    final_rmse = best_rmse\n\n# Create submission\nsubmission = sample_submission.copy()\nsubmission['Lap_Time_Seconds'] = final_pred\nsubmission.to_csv('teamrocket_xgb_only.csv', index=False)\n\n# Results\nprint(f\"\\n🏁 FINAL RESULTS:\")\nprint(f\"🎯 Target: RMSE < 2.0 (ideally < 1.0)\")\nprint(f\"📊 Achieved: {final_rmse:.4f}\")\nprint(f\"{'🎉 EXCELLENT!' if final_rmse < 1.5 else '✅ GOOD PROGRESS!' if final_rmse < 2.0 else '⚡ KEEP PUSHING!'}\")\nprint(f\"⏱️ Estimated runtime: 15-18 minutes\")\nprint(\"🚚 teamrocket_xgb_only.csv ready for submission!\")\n\nif final_rmse > 1.5:\n    print(f\"\\n💡 Quick tips to improve further:\")\n    print(f\"   - Try changing outlier threshold to 2.0 or 3.0\")\n    print(f\"   - Increase n_estimators to 2500\")\n    print(f\"   - Experiment with learning_rate: 0.03-0.05\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-14T15:36:57.425257Z","iopub.execute_input":"2025-06-14T15:36:57.425582Z"}},"outputs":[],"execution_count":null}]}