{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":104857,"databundleVersionId":12651513,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import OneHotEncoder, OrdinalEncoder\nfrom sklearn.base import clone\nfrom xgboost import XGBRegressor, plot_importance\nfrom sklearn.model_selection import TimeSeriesSplit\nimport joblib\nimport matplotlib.pyplot as plt\nimport os\n\n# Set environment variable to prevent XGBoost device mismatch warning\nos.environ['CUDA_VISIBLE_DEVICES'] = '0'\n# /kaggle/input/burnout-datathon-ieeecsmuj/test.csv\n# Load data\ntrain = pd.read_csv('/kaggle/input/burnout-datathon-ieeecsmuj/train.csv')\nval = pd.read_csv('/kaggle/input/burnout-datathon-ieeecsmuj/val.csv')\ntest = pd.read_csv('/kaggle/input/burnout-datathon-ieeecsmuj/test.csv')\n\n# Feature engineering with improved safety\ndef create_features(df):\n    # Temporal features\n    df['season_progress'] = df['year_x'] - df['min_year']\n    df['career_phase'] = (df['year_x'] - df['min_year']) / (df['years_active'].replace(0, np.nan) + 1e-6)\n    \n    # Team performance metrics\n    df['team_success_rate'] = df['podiums'] / (df['starts'].replace(0, np.nan) + 1e-6)\n    df['team_reliability'] = df['finishes'] / (df['starts'].replace(0, np.nan) + 1e-6)\n    \n    # Tire dynamics\n    df['tire_wear_ratio'] = df['Tire_Degradation_Factor_per_Lap'] * df['Laps']\n    df['compound_aggressiveness'] = np.where(\n        df['Tire_Compound_Front'] == 'Soft', 1.2,\n        np.where(df['Tire_Compound_Front'] == 'Medium', 1.0, 0.8)\n    )\n    \n    # Track difficulty\n    df['speed_load'] = df['Avg_Speed_kmh'] / (df['Circuit_Length_km'].replace(0, np.nan) + 1e-6)\n    df['corner_intensity'] = df['Corners_per_Lap'] / (df['Circuit_Length_km'].replace(0, np.nan) + 1e-6)\n    \n    # Session progression\n    session_order = {'FP1':1, 'FP2':2, 'FP3':3, 'FP4':4, \n                    'Qualifying':5, 'Sprint':6, 'Race':7}\n    df['session_importance'] = df['Session'].map(session_order).fillna(0)\n    \n    # List of all potential numerical columns\n    numerical_cols = ['Circuit_Length_km', 'Laps', 'Avg_Speed_kmh', \n                     'Tire_Degradation_Factor_per_Lap', 'Ambient_Temperature_Celsius',\n                     'Humidity_%', 'points', 'Championship_Points', 'Corners_per_Lap',\n                     'Pit_Stop_Duration_Seconds', 'Track_Temperature_Celsius', 'air',\n                     'ground', 'starts', 'finishes', 'with_points', 'podiums', 'wins',\n                     'min_year', 'max_year', 'years_active', 'year_x',\n                     'season_progress', 'career_phase', 'team_success_rate', \n                     'team_reliability', 'tire_wear_ratio', 'compound_aggressiveness',\n                     'speed_load', 'corner_intensity', 'session_importance']\n    \n    # Convert to numeric and fill NaN with 0\n    for col in numerical_cols:\n        if col in df.columns:\n            df[col] = pd.to_numeric(df[col], errors='coerce').fillna(0)\n            \n    return df\n\n# Apply feature engineering\ntrain = create_features(train)\nval = create_features(val)\ntest = create_features(test)\n\n# Define feature groups\nhigh_cardinality = ['rider_name', 'circuit_name', 'team_name', 'shortname', 'bike_name']\nordinal_features = ['Tire_Compound_Front', 'Tire_Compound_Rear', \n                   'Grid_Position', 'Championship_Position']\nnominal_features = ['category_x', 'Track_Condition', 'Penalty', \n                   'weather', 'track', 'Session']\n\nnumerical_features = [\n    'Circuit_Length_km', 'Laps', 'Avg_Speed_kmh', 'Tire_Degradation_Factor_per_Lap',\n    'season_progress', 'career_phase', 'team_success_rate', 'team_reliability',\n    'tire_wear_ratio', 'compound_aggressiveness', 'speed_load', 'corner_intensity',\n    'session_importance', 'Ambient_Temperature_Celsius', 'Humidity_%', 'points',\n    'Championship_Points', 'Corners_per_Lap', 'Pit_Stop_Duration_Seconds',\n    'Track_Temperature_Celsius', 'air', 'ground', 'starts', 'finishes', 'with_points',\n    'podiums', 'wins', 'min_year', 'max_year', 'years_active', 'year_x'\n]\n\n# Preprocessing pipeline\npreprocessor = ColumnTransformer([\n    ('target_enc', OrdinalEncoder(\n        handle_unknown='use_encoded_value', \n        unknown_value=-1\n    ), high_cardinality),\n    ('ordinal', OrdinalEncoder(\n        categories=[\n            ['Hard','Medium','Soft'], \n            ['Hard','Medium','Soft'],\n            list(range(1, 26)),  # Grid positions\n            list(range(1, 31))   # Championship positions\n        ],\n        handle_unknown='use_encoded_value',\n        unknown_value=-1\n    ), ordinal_features),\n    ('onehot', OneHotEncoder(handle_unknown='ignore'), nominal_features),\n    ('num', 'passthrough', numerical_features)\n])\n\n# Prepare data\nfeatures_used = high_cardinality + ordinal_features + nominal_features + numerical_features\nX_train = train[features_used]\ny_train = train['Lap_Time_Seconds']\nX_val = val[features_used]\ny_val = val['Lap_Time_Seconds']\nX_test = test[features_used]\n\n# Combine train and val for final training\nX_train_val = pd.concat([X_train, X_val])\ny_train_val = pd.concat([y_train, y_val])\n\n# Preprocess all data at once\npreprocessor.fit(X_train_val)\nX_train_val_trans = preprocessor.transform(X_train_val)\nX_test_trans = preprocessor.transform(X_test)\n\n# Create validation split for plotting\nsplit_index = int(len(X_train_val_trans) * 0.9)  # 90% train, 10% validation\nX_final_train = X_train_val_trans[:split_index]\ny_final_train = y_train_val.iloc[:split_index]\nX_final_val = X_train_val_trans[split_index:]\ny_final_val = y_train_val.iloc[split_index:]\n\n# Configure final model\nfinal_model = XGBRegressor(\n    tree_method='hist',\n    device='cuda',\n    n_estimators=20000,\n    learning_rate=0.03770760297401682,\n    max_depth=20,\n    subsample=0.6936774881064425,\n    colsample_bytree=0.851962030829581,\n    gamma=0.146963904565,\n    reg_alpha=1.53108214125,\n    reg_lambda=1.9924755681,\n    eval_metric='rmse',\n    early_stopping_rounds=50\n)\n\n# Train with validation set for learning curve\nhistory = final_model.fit(\n    X_final_train, y_final_train,\n    eval_set=[(X_final_val, y_final_val)],\n    verbose=100  # Print progress every 10 iterations\n)\n\n# Plot learning curve\nresults = final_model.evals_result()\nepochs = len(results['validation_0']['rmse'])\nx_axis = range(0, epochs)\n\nplt.figure(figsize=(12, 6))\nplt.plot(x_axis, results['validation_0']['rmse'], label='Validation')\nplt.legend()\nplt.ylabel('RMSE')\nplt.xlabel('Epochs')\nplt.title('XGBoost Learning Curve')\nplt.savefig('learning_curve.png')\nplt.close()\n\n# Plot feature importance\nplt.figure(figsize=(16, 12))\nplot_importance(final_model, max_num_features=30)\nplt.tight_layout()\nplt.savefig('feature_importance.png')\nplt.close()\n\n# Retrain on full dataset\nfinal_model = XGBRegressor(\n    tree_method='hist',\n    device='cuda',\n    n_estimators=final_model.best_iteration,\n    learning_rate=0.025,\n    max_depth=9,\n    subsample=0.75,\n    colsample_bytree=0.85,\n    gamma=0.1,\n    reg_alpha=0.5,\n    reg_lambda=0.5,\n    eval_metric='rmse'\n)\n\nfinal_model.fit(X_train_val_trans, y_train_val)\n\n# Generate predictions\n# Set device to CPU for prediction to avoid device mismatch\nfinal_model.set_params(device='cpu')\ntest_preds = final_model.predict(X_test_trans)\n\n# Create submission\nsubmission = pd.DataFrame({\n    'Unique ID': test['Unique ID'],\n    'Lap_Time_Seconds': np.round(test_preds, 3)\n})\nsubmission.to_csv('solution.csv', index=False)\n\n# Save model artifacts\njoblib.dump(preprocessor, 'preprocessor.pkl')\njoblib.dump(final_model, 'final_model.pkl')\n\nprint(\"Training completed successfully!\")\nprint(f\"Learning curve saved as 'learning_curve.png'\")\nprint(f\"Feature importance plot saved as 'feature_importance.png'\")\nprint(f\"Submission file saved as 'solution.csv'\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-14T15:02:26.86045Z","iopub.execute_input":"2025-06-14T15:02:26.86072Z","iopub.status.idle":"2025-06-14T15:05:33.004178Z","shell.execute_reply.started":"2025-06-14T15:02:26.860701Z","shell.execute_reply":"2025-06-14T15:05:33.003581Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.preprocessing import OneHotEncoder, OrdinalEncoder\nfrom xgboost import XGBRegressor, plot_importance\nfrom sklearn.model_selection import TimeSeriesSplit\nimport joblib\nimport matplotlib.pyplot as plt\nimport optuna\nfrom sklearn.metrics import mean_squared_error\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Load data/kaggle/input/burnout-datathon-ieeecsmuj/test.csv\ntrain = pd.read_csv('/kaggle/input/burnout-datathon-ieeecsmuj/train.csv')\nval = pd.read_csv('/kaggle/input/burnout-datathon-ieeecsmuj/val.csv')\ntest = pd.read_csv('/kaggle/input/burnout-datathon-ieeecsmuj/test.csv')\n\n# Advanced feature engineering\ndef create_features(df):\n    # Temporal features\n    df['years_since_debut'] = df['year_x'] - df['min_year']\n    df['career_phase'] = df['years_since_debut'] / (df['years_active'] + 1e-6)\n    \n    # Performance features\n    df['podium_rate'] = df['podiums'] / (df['starts'] + 1e-6)\n    df['finish_rate'] = df['finishes'] / (df['starts'] + 1e-6)\n    df['win_rate'] = df['wins'] / (df['starts'] + 1e-6)\n    \n    # Tire and track dynamics\n    df['tire_wear_effect'] = df['Tire_Degradation_Factor_per_Lap'] * df['Laps'] * df['Circuit_Length_km']\n    df['compound_aggressiveness'] = np.select(\n        [\n            df['Tire_Compound_Front'] == 'Soft',\n            df['Tire_Compound_Front'] == 'Medium',\n            df['Tire_Compound_Front'] == 'Hard'\n        ],\n        [1.2, 1.0, 0.8],\n        default=1.0\n    )\n    \n    # Speed and corner dynamics\n    df['speed_intensity'] = df['Avg_Speed_kmh'] / df['Circuit_Length_km']\n    df['corner_intensity'] = df['Corners_per_Lap'] / df['Circuit_Length_km']\n    df['speed_corner_ratio'] = df['Avg_Speed_kmh'] / (df['Corners_per_Lap'] + 1e-6)\n    \n    # Session importance\n    session_order = {'FP1':1, 'FP2':2, 'FP3':3, 'FP4':4, \n                    'Qualifying':5, 'Sprint':6, 'Race':7}\n    df['session_importance'] = df['Session'].map(session_order).fillna(0)\n    \n    # Weather effects\n    df['temp_humidity_effect'] = df['Ambient_Temperature_Celsius'] * df['Humidity_%'] / 100\n    df['track_temp_effect'] = df['Track_Temperature_Celsius'] - df['Ambient_Temperature_Celsius']\n    \n    # Rider-team synergy\n    df['rider_team_experience'] = df['year_x'] - df.groupby(['rider_name', 'team_name'])['year_x'].transform('min')\n    # df = df.sort_values('year_x')\n    # group = df.groupby(['rider_name', 'team_name'])\n    # df['rider_team_experience'] = group['year_x'].expanding().apply(lambda x: x.iloc[-1] - x.iloc[0], raw=False)\n\n    # Recent performance\n    df['recent_points'] = df.groupby('rider_name')['points'].transform(lambda x: x.rolling(3, min_periods=1).mean())\n    #=================================================\n    # df['tire_temp_diff'] = df['Track_Temperature_Celsius'] - {\n    #     'Soft': 30, 'Medium': 40, 'Hard': 50\n    # }[df['Tire_Compound_Front']]\n    df['tire_temp_diff'] = df['Track_Temperature_Celsius'] - df['Tire_Compound_Front'].map({\n        'Soft': 30, \n        'Medium': 40, \n        'Hard': 50\n    })\n        \n    df['rain_effect'] = np.where(df['weather'] == 'Raining', \n                                 df['Humidity_%'] * df['Tire_Degradation_Factor_per_Lap'], \n                                 0)\n    #=================================================\n\n    # Numerical columns to process\n    num_cols = [\n        'Circuit_Length_km', 'Laps', 'Avg_Speed_kmh', 'Tire_Degradation_Factor_per_Lap',\n        'Ambient_Temperature_Celsius', 'Humidity_%', 'points', 'Championship_Points',\n        'Corners_per_Lap', 'Pit_Stop_Duration_Seconds', 'Track_Temperature_Celsius',\n        'years_since_debut', 'career_phase', 'podium_rate', 'finish_rate', 'win_rate',\n        'tire_wear_effect', 'compound_aggressiveness', 'speed_intensity', \n        'corner_intensity', 'speed_corner_ratio', 'session_importance',\n        'temp_humidity_effect', 'track_temp_effect', 'rider_team_experience', 'recent_points','tire_temp_diff', 'rain_effect', 'recent_points'\n    ]\n    \n    # Process numerical columns\n    for col in num_cols:\n        if col in df.columns:\n            # Handle infinite values and NaNs\n            df[col] = df[col].replace([np.inf, -np.inf], np.nan)\n            mean_val = df[col].mean()\n            df[col] = df[col].fillna(mean_val if not np.isnan(mean_val) else 0)\n            \n    return df\n\n# Apply feature engineering\ntrain = create_features(train)\nval = create_features(val)\ntest = create_features(test)\n\n# Define feature groups\nhigh_cardinality = ['rider_name', 'circuit_name', 'team_name', 'shortname', 'bike_name']\nordinal_features = ['Tire_Compound_Front', 'Tire_Compound_Rear', \n                   'Grid_Position', 'Championship_Position']\nnominal_features = ['category_x', 'Track_Condition', 'Penalty', \n                   'weather', 'track', 'Session']\n\nnumerical_features = [\n    'Circuit_Length_km', 'Laps', 'Avg_Speed_kmh', 'Tire_Degradation_Factor_per_Lap',\n    'Ambient_Temperature_Celsius', 'Humidity_%', 'points', 'Championship_Points',\n    'Corners_per_Lap', 'Pit_Stop_Duration_Seconds', 'Track_Temperature_Celsius',\n    'years_since_debut', 'career_phase', 'podium_rate', 'finish_rate', 'win_rate',\n    'tire_wear_effect', 'compound_aggressiveness', 'speed_intensity', \n    'corner_intensity', 'speed_corner_ratio', 'session_importance',\n    'temp_humidity_effect', 'track_temp_effect', 'rider_team_experience', 'recent_points'\n]\n\n# Preprocessing pipeline\npreprocessor = ColumnTransformer([\n    ('target_enc', OrdinalEncoder(handle_unknown='use_encoded_value', unknown_value=-1), \n     high_cardinality),\n    ('ordinal', OrdinalEncoder(\n        categories=[\n            ['Hard','Medium','Soft'], \n            ['Hard','Medium','Soft'],\n            list(range(1, 26)),  # Grid positions   \n            list(range(1, 31))   # Championship positions\n        ], \n        handle_unknown='use_encoded_value',\n        unknown_value=-1\n    ), ordinal_features),\n    ('onehot', OneHotEncoder(handle_unknown='ignore', sparse_output=False), nominal_features),\n    ('num', 'passthrough', numerical_features)\n])\n\n# Prepare data\nfeatures_used = high_cardinality + ordinal_features + nominal_features + numerical_features\nX_train = train[features_used]\ny_train = train['Lap_Time_Seconds']\nX_val = val[features_used]\ny_val = val['Lap_Time_Seconds']\nX_test = test[features_used]\n\n# Combine train and val\nX_full = pd.concat([X_train, X_val])\ny_full = pd.concat([y_train, y_val])\n\n# Preprocess data\npreprocessor.fit(X_full)\nX_train_pre = preprocessor.transform(X_train)\nX_val_pre = preprocessor.transform(X_val)\nX_full_pre = preprocessor.transform(X_full)\nX_test_pre = preprocessor.transform(X_test)\n\n# Hyperparameter optimization with Optuna\ndef objective(trial):\n    params = {\n        'tree_method': 'hist',\n        'device': 'cuda',\n        'n_estimators': 5000,\n        'learning_rate': trial.suggest_float('learning_rate', 0.001, 0.1, log=True),\n        'max_depth': trial.suggest_int('max_depth',10, 20),\n        'subsample': trial.suggest_float('subsample', 0.4, 0.8),\n        'colsample_bytree': trial.suggest_float('colsample_bytree', 0.6, 1.0),\n        'gamma': trial.suggest_float('gamma', 0, 1),\n        'reg_alpha': trial.suggest_float('reg_alpha', 0, 5),\n        'reg_lambda': trial.suggest_float('reg_lambda', 0, 5),\n        'eval_metric': 'rmse',\n        'early_stopping_rounds': 100\n    }\n    \n    model = XGBRegressor(**params)\n    \n    model.fit(\n        X_train_pre, y_train,\n        eval_set=[(X_val_pre, y_val)],\n        verbose=False\n    )\n    \n    # Get best score\n    best_rmse = min(model.evals_result()['validation_0']['rmse'])\n    return best_rmse\n\n# Run optimization\nstudy = optuna.create_study(direction='minimize')\nstudy.optimize(objective, n_trials=50, timeout=3600)  # 1 hour timeout\n\n# Get best parameters\nbest_params = study.best_params\nprint(f\"Best parameters: {best_params}\")\nprint(f\"Best validation RMSE: {study.best_value:.6f}\")\n\n# Train final model with best parameters\nfinal_model = XGBRegressor(\n    **best_params,\n    tree_method='hist',\n    device='cuda'\n)\n\n# Train with early stopping\nfinal_model.fit(\n    X_train_pre, y_train,\n    eval_set=[(X_val_pre, y_val)],\n    verbose=100\n)\n\n# Plot learning curve\nresults = final_model.evals_result()\nepochs = len(results['validation_0']['rmse'])\nx_axis = range(0, epochs)\n\nplt.figure(figsize=(12, 6))\nplt.plot(x_axis, results['validation_0']['rmse'], label='Validation')\nplt.axhline(y=study.best_value, color='r', linestyle='--', label=f'Best RMSE: {study.best_value:.6f}')\nplt.legend()\nplt.ylabel('RMSE')\nplt.xlabel('Epochs')\nplt.title('XGBoost Learning Curve')\nplt.savefig('learning_curve.png')\nplt.close()\n\n# Plot feature importance\nplt.figure(figsize=(16, 12))\nplot_importance(final_model, max_num_features=30)\nplt.tight_layout()\nplt.savefig('feature_importance.png')\nplt.close()\n\n# Generate predictions\ntest_preds = final_model.predict(X_test_pre)\n\n# Create submission\nsubmission = pd.DataFrame({\n    'Unique ID': test['Unique ID'],\n    'Lap_Time_Seconds': np.round(test_preds, 6)  # More precision\n})\nsubmission.to_csv('solution.csv', index=False)\n\n# # Save model artifacts\n# joblib.dump(preprocessor, 'preprocessor.pkl')\n# joblib.dump(final_model, 'final_model.pkl')\n\n# print(f\"Final model trained with validation RMSE: {study.best_value:.6f}\")\n# print(\"Submission file created with high-precision predictions\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-14T15:30:57.480326Z","iopub.execute_input":"2025-06-14T15:30:57.481056Z","iopub.status.idle":"2025-06-14T16:16:57.271697Z","shell.execute_reply.started":"2025-06-14T15:30:57.481025Z","shell.execute_reply":"2025-06-14T16:16:57.270462Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.preprocessing import OneHotEncoder, OrdinalEncoder\nfrom xgboost import XGBRegressor, plot_importance\nimport joblib\nimport matplotlib.pyplot as plt\nimport os\n\n# Set environment variables\nos.environ['CUDA_VISIBLE_DEVICES'] = '0'\n\n# Load data\ntrain = pd.read_csv('/kaggle/input/burnout-datathon-ieeecsmuj/train.csv')\nval = pd.read_csv('/kaggle/input/burnout-datathon-ieeecsmuj/val.csv')\ntest = pd.read_csv('/kaggle/input/burnout-datathon-ieeecsmuj/test.csv')\n\n# Advanced feature engineering\ndef create_features(df):\n    # Temporal features\n    df['years_since_debut'] = df['year_x'] - df['min_year']\n    df['career_phase'] = df['years_since_debut'] / (df['years_active'].replace(0, np.nan) + 1e-6)\n    \n    # Performance features\n    df['podium_rate'] = df['podiums'] / (df['starts'].replace(0, np.nan) + 1e-6)\n    df['finish_rate'] = df['finishes'] / (df['starts'].replace(0, np.nan) + 1e-6)\n    df['win_rate'] = df['wins'] / (df['starts'].replace(0, np.nan) + 1e-6)\n    \n    # Tire and track dynamics\n    df['tire_wear_effect'] = df['Tire_Degradation_Factor_per_Lap'] * df['Laps'] * df['Circuit_Length_km']\n    df['compound_aggressiveness'] = np.select(\n        [\n            df['Tire_Compound_Front'] == 'Soft',\n            df['Tire_Compound_Front'] == 'Medium',\n            df['Tire_Compound_Front'] == 'Hard'\n        ],\n        [1.2, 1.0, 0.8],\n        default=1.0\n    )\n    \n    # Speed and corner dynamics\n    df['speed_intensity'] = df['Avg_Speed_kmh'] / (df['Circuit_Length_km'] + 1e-6)\n    df['corner_intensity'] = df['Corners_per_Lap'] / (df['Circuit_Length_km'] + 1e-6)\n    df['speed_corner_ratio'] = df['Avg_Speed_kmh'] / (df['Corners_per_Lap'] + 1e-6)\n    \n    # Session importance\n    session_order = {'FP1':1, 'FP2':2, 'FP3':3, 'FP4':4, \n                    'Qualifying':5, 'Sprint':6, 'Race':7}\n    df['session_importance'] = df['Session'].map(session_order).fillna(0)\n    \n    # Weather effects\n    df['temp_humidity_effect'] = df['Ambient_Temperature_Celsius'] * df['Humidity_%'] / 100\n    df['track_temp_effect'] = df['Track_Temperature_Celsius'] - df['Ambient_Temperature_Celsius']\n    \n    # Rider-team synergy\n    df['rider_team_experience'] = df.groupby(['rider_name', 'team_name'])['year_x'].transform(lambda x: x - x.min())\n    \n    # Recent performance\n    df['recent_points'] = df.groupby('rider_name')['points'].transform(lambda x: x.rolling(3, min_periods=1).mean())\n    \n    # Tire temperature\n    compound_map = {'Soft': 30, 'Medium': 40, 'Hard': 50}\n    df['tire_temp_diff'] = df['Track_Temperature_Celsius'] - df['Tire_Compound_Front'].map(compound_map)\n    \n    # Rain effect\n    df['rain_effect'] = np.where(df['weather'] == 'Raining', \n                                 df['Humidity_%'] * df['Tire_Degradation_Factor_per_Lap'], \n                                 0)\n    \n    # Numerical columns to process\n    num_cols = [\n        'Circuit_Length_km', 'Laps', 'Avg_Speed_kmh', 'Tire_Degradation_Factor_per_Lap',\n        'Ambient_Temperature_Celsius', 'Humidity_%', 'points', 'Championship_Points',\n        'Corners_per_Lap', 'Pit_Stop_Duration_Seconds', 'Track_Temperature_Celsius',\n        'years_since_debut', 'career_phase', 'podium_rate', 'finish_rate', 'win_rate',\n        'tire_wear_effect', 'compound_aggressiveness', 'speed_intensity', \n        'corner_intensity', 'speed_corner_ratio', 'session_importance',\n        'temp_humidity_effect', 'track_temp_effect', 'rider_team_experience', 'recent_points',\n        'tire_temp_diff', 'rain_effect'\n    ]\n    \n    # Process numerical columns\n    for col in num_cols:\n        if col in df.columns:\n            # Handle infinite values and NaNs\n            df[col] = df[col].replace([np.inf, -np.inf], np.nan)\n            mean_val = df[col].mean()\n            df[col] = df[col].fillna(mean_val if not np.isnan(mean_val) else 0)\n            \n    return df\n\n# Apply feature engineering\ntrain = create_features(train)\nval = create_features(val)\ntest = create_features(test)\n\n# Define feature groups\nhigh_cardinality = ['rider_name', 'circuit_name', 'team_name', 'shortname', 'bike_name']\nordinal_features = ['Tire_Compound_Front', 'Tire_Compound_Rear', \n                   'Grid_Position', 'Championship_Position']\nnominal_features = ['category_x', 'Track_Condition', 'Penalty', \n                   'weather', 'track', 'Session']\n\nnumerical_features = [\n    'Circuit_Length_km', 'Laps', 'Avg_Speed_kmh', 'Tire_Degradation_Factor_per_Lap',\n    'Ambient_Temperature_Celsius', 'Humidity_%', 'points', 'Championship_Points',\n    'Corners_per_Lap', 'Pit_Stop_Duration_Seconds', 'Track_Temperature_Celsius',\n    'years_since_debut', 'career_phase', 'podium_rate', 'finish_rate', 'win_rate',\n    'tire_wear_effect', 'compound_aggressiveness', 'speed_intensity', \n    'corner_intensity', 'speed_corner_ratio', 'session_importance',\n    'temp_humidity_effect', 'track_temp_effect', 'rider_team_experience', 'recent_points',\n    'tire_temp_diff', 'rain_effect'\n]\n\n# Preprocessing pipeline\npreprocessor = ColumnTransformer([\n    ('target_enc', OrdinalEncoder(handle_unknown='use_encoded_value', unknown_value=-1), \n     high_cardinality),\n    ('ordinal', OrdinalEncoder(\n        categories=[\n            ['Hard','Medium','Soft'], \n            ['Hard','Medium','Soft'],\n            list(range(1, 26)),  # Grid positions\n            list(range(1, 31))   # Championship positions\n        ], \n        handle_unknown='use_encoded_value',\n        unknown_value=-1\n    ), ordinal_features),\n    ('onehot', OneHotEncoder(handle_unknown='ignore', sparse_output=False), nominal_features),\n    ('num', 'passthrough', numerical_features)\n])\n\n# Prepare data\nfeatures_used = high_cardinality + ordinal_features + nominal_features + numerical_features\nX_train = train[features_used]\ny_train = train['Lap_Time_Seconds']\nX_val = val[features_used]\ny_val = val['Lap_Time_Seconds']\nX_test = test[features_used]\n\n# Combine train and val\nX_full = pd.concat([X_train, X_val])\ny_full = pd.concat([y_train, y_val])\n\n# Preprocess data\npreprocessor.fit(X_full)\nX_train_pre = preprocessor.transform(X_train)\nX_val_pre = preprocessor.transform(X_val)\nX_full_pre = preprocessor.transform(X_full)\nX_test_pre = preprocessor.transform(X_test)\n\n# Best hyperparameters from Optuna trial\nbest_params = {\n    \"objective\": \"reg:squarederror\",\n    \"eval_metric\": \"rmse\",\n    \"tree_method\": \"hist\",\n    \"device\": \"cuda\",\n    \"enable_categorical\": False,\n    \"seed\": 42,\n    \"n_estimators\": 10000,\n    \"learning_rate\": 0.011402872059158305,\n    \"max_depth\": 18,\n    \"subsample\": 0.6627781711580402,\n    \"colsample_bytree\": 0.876682342023657,\n    \"gamma\": 0.10717525341150558,\n    \"reg_alpha\": 0.6025606773970926,\n    \"reg_lambda\": 2.1170292796479124,\n    \"early_stopping_rounds\": 100\n}\n\n# Train model with early stopping\nmodel = XGBRegressor(**best_params)\nmodel.fit(\n    X_train_pre, y_train,\n    eval_set=[(X_val_pre, y_val)],\n    verbose=100\n)\n\n# Get best iteration and RMSE\nbest_iteration = model.best_iteration\nbest_rmse = model.best_score\nprint(f\"Best iteration: {best_iteration}\")\nprint(f\"Best validation RMSE: {best_rmse}\")\n\n# # Retrain on full dataset with optimal iterations\n# final_model = XGBRegressor(\n#     **{k: v for k, v in best_params.items() if k not in ['early_stopping_rounds']},\n#     n_estimators=best_iteration\n# )\n# final_model.fit(X_full_pre, y_full)\n\n# Plot learning curve\nresults = model.evals_result()\nepochs = len(results['validation_0']['rmse'])\nx_axis = range(0, epochs)\n\nplt.figure(figsize=(12, 6))\nplt.plot(x_axis, results['validation_0']['rmse'], label='Validation')\nplt.axhline(y=best_rmse, color='r', linestyle='--', label=f'Best RMSE: {best_rmse:.6f}')\nplt.legend()\nplt.ylabel('RMSE')\nplt.xlabel('Epochs')\nplt.title('XGBoost Learning Curve')\nplt.savefig('learning_curve.png')\nplt.close()\n\n# Plot feature importance\nplt.figure(figsize=(16, 12))\nplot_importance(model, max_num_features=30)\nplt.tight_layout()\nplt.savefig('feature_importance.png')\nplt.close()\n\n# Generate predictions\ntest_preds = model.predict(X_test_pre)\n\n# Create submission\nsubmission = pd.DataFrame({\n    'Unique ID': test['Unique ID'],\n    'Lap_Time_Seconds': np.round(test_preds, 6)\n})\nsubmission.to_csv('solution.csv', index=False)\n\nprint(\"Training completed successfully!\")\nprint(f\"Best validation RMSE: {best_rmse:.6f}\")\nprint(f\"Used {best_iteration} iterations for final model\")\nprint(f\"Learning curve saved as 'learning_curve.png'\")\nprint(f\"Feature importance plot saved as 'feature_importance.png'\")\nprint(f\"Submission file saved as 'solution.csv'\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-15T08:50:47.052846Z","iopub.execute_input":"2025-06-15T08:50:47.053594Z","iopub.status.idle":"2025-06-15T08:59:48.806572Z","shell.execute_reply.started":"2025-06-15T08:50:47.053562Z","shell.execute_reply":"2025-06-15T08:59:48.805858Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Plot learning curve\nresults = model.evals_result()\nepochs = len(results['validation_0']['rmse'])\nx_axis = range(0, epochs)\n\nplt.figure(figsize=(12, 6))\nplt.plot(x_axis, results['validation_0']['rmse'], label='Validation')\nplt.axhline(y=best_rmse, color='r', linestyle='--', label=f'Best RMSE: {best_rmse:.6f}')\nplt.legend()\nplt.ylabel('RMSE')\nplt.xlabel('Epochs')\nplt.title('XGBoost Learning Curve')\nplt.savefig('learning_curve.png')\nplt.close()\n\n# Plot feature importance\nplt.figure(figsize=(16, 12))\nplot_importance(model, max_num_features=30)\nplt.tight_layout()\nplt.savefig('feature_importance.png')\nplt.close()\n\n# Generate predictions\ntest_preds = model.predict(X_test_pre)\n\n# Create submission\nsubmission = pd.DataFrame({\n    'Unique ID': test['Unique ID'],\n    'Lap_Time_Seconds': np.round(test_preds, 6)\n})\nsubmission.to_csv('solution.csv', index=False)\n\nprint(\"Training completed successfully!\")\nprint(f\"Best validation RMSE: {best_rmse:.6f}\")\nprint(f\"Used {best_iteration} iterations for final model\")\nprint(f\"Learning curve saved as 'learning_curve.png'\")\nprint(f\"Feature importance plot saved as 'feature_importance.png'\")\nprint(f\"Submission file saved as 'solution.csv'\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-14T16:28:29.490246Z","iopub.execute_input":"2025-06-14T16:28:29.490923Z","iopub.status.idle":"2025-06-14T16:28:33.13536Z","shell.execute_reply.started":"2025-06-14T16:28:29.490901Z","shell.execute_reply":"2025-06-14T16:28:33.134529Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom scipy import stats\nimport missingno as msno\nimport plotly.express as px\n\n# Set up visualization style\nsns.set_style(\"whitegrid\")\nplt.rcParams[\"figure.figsize\"] = (12, 6)\npd.set_option('display.max_columns', 100)\n\n# Load datasets\ntrain = pd.read_csv('/kaggle/input/burnout-datathon-ieeecsmuj/train.csv')\nval = pd.read_csv('/kaggle/input/burnout-datathon-ieeecsmuj/val.csv')\ntest = pd.read_csv('/kaggle/input/burnout-datathon-ieeecsmuj/test.csv')\n\n# Combine datasets for comprehensive EDA\nfull_data = pd.concat([train, val, test], ignore_index=True)\n\n# ==================================================\n# 1. Dataset Overview\n# ==================================================\nprint(\"=\"*50)\nprint(\"DATASET OVERVIEW\")\nprint(\"=\"*50)\n\n# Basic information\nprint(\"\\n=== Dataset Shape ===\")\nprint(f\"Train shape: {train.shape}\")\nprint(f\"Validation shape: {val.shape}\")\nprint(f\"Test shape: {test.shape}\")\nprint(f\"Combined shape: {full_data.shape}\")\n\n# First look at data\nprint(\"\\n=== First 5 Rows ===\")\ndisplay(train.head())\n\n# Data types and missing values\nprint(\"\\n=== Data Types and Missing Values ===\")\noverview_df = pd.DataFrame({\n    'dtype': full_data.dtypes,\n    'missing_values': full_data.isnull().sum(),\n    'missing_%': (full_data.isnull().mean() * 100).round(2),\n    'unique_values': full_data.nunique()\n})\ndisplay(overview_df)\n\n# Missing values visualization\nprint(\"\\n=== Missing Values Matrix ===\")\nmsno.matrix(full_data)\nplt.title('Missing Values Overview', fontsize=16)\nplt.savefig('missing_values_matrix.png', bbox_inches='tight')\nplt.show()\n\n# ==================================================\n# 2. Descriptive Statistics\n# ==================================================\nprint(\"\\n\" + \"=\"*50)\nprint(\"DESCRIPTIVE STATISTICS\")\nprint(\"=\"*50)\n\n# Numerical features summary\nprint(\"\\n=== Numerical Features Summary ===\")\nnum_cols = full_data.select_dtypes(include=np.number).columns\ndisplay(full_data[num_cols].describe().T)\n\n# Categorical features summary\nprint(\"\\n=== Categorical Features Summary ===\")\ncat_cols = full_data.select_dtypes(exclude=np.number).columns\nfor col in cat_cols:\n    print(f\"\\n{col}:\\n{full_data[col].value_counts().head(10)}\")\n\n# Target variable analysis\nprint(\"\\n=== Target Variable (Lap_Time_Seconds) Distribution ===\")\nplt.figure(figsize=(14, 6))\nplt.subplot(1, 2, 1)\nsns.histplot(train['Lap_Time_Seconds'], kde=True)\nplt.title('Lap Time Distribution')\n\nplt.subplot(1, 2, 2)\nstats.probplot(train['Lap_Time_Seconds'], plot=plt)\nplt.title('Lap Time Q-Q Plot')\nplt.savefig('target_distribution.png', bbox_inches='tight')\nplt.show()\n\n# ==================================================\n# 3. Data Visualizations\n# ==================================================\nprint(\"\\n\" + \"=\"*50)\nprint(\"DATA VISUALIZATIONS\")\nprint(\"=\"*50)\n\n# Correlation matrix\nprint(\"\\n=== Correlation Matrix ===\")\ncorr_matrix = train.corr(numeric_only=True)\nmask = np.triu(np.ones_like(corr_matrix, dtype=bool))\n\nplt.figure(figsize=(16, 12))\nsns.heatmap(corr_matrix, mask=mask, annot=True, fmt=\".2f\", cmap=\"coolwarm\", \n            center=0, linewidths=.5, cbar_kws={\"shrink\": .8})\nplt.title('Feature Correlation Matrix', fontsize=16)\nplt.savefig('correlation_matrix.png', bbox_inches='tight')\nplt.show()\n\n# Top correlations with target\ncorr_with_target = corr_matrix['Lap_Time_Seconds'].sort_values(ascending=False)[1:11]\nprint(\"\\nTop 10 Features Correlated with Lap Time:\")\nprint(corr_with_target)\n\n# Categorical feature distributions\nprint(\"\\n=== Categorical Feature Distributions ===\")\nfor col in cat_cols[:5]:  # First 5 categorical features\n    plt.figure(figsize=(12, 6))\n    order = full_data[col].value_counts().index[:10]  # Top 10 categories\n    sns.countplot(data=full_data, x=col, order=order)\n    plt.title(f'{col} Distribution', fontsize=14)\n    plt.xticks(rotation=45)\n    plt.savefig(f'{col}_distribution.png', bbox_inches='tight')\n    plt.show()\n\n# Numerical feature distributions\nprint(\"\\n=== Numerical Feature Distributions ===\")\nfor col in num_cols[:5]:  # First 5 numerical features\n    plt.figure(figsize=(12, 6))\n    plt.subplot(1, 2, 1)\n    sns.histplot(full_data[col], kde=True)\n    plt.title(f'{col} Distribution')\n    \n    plt.subplot(1, 2, 2)\n    sns.boxplot(y=full_data[col])\n    plt.title(f'{col} Boxplot')\n    plt.savefig(f'{col}_distribution.png', bbox_inches='tight')\n    plt.show()\n\n# Time-based analysis\nif 'year_x' in full_data.columns:\n    print(\"\\n=== Yearly Lap Time Trends ===\")\n    yearly_stats = train.groupby('year_x')['Lap_Time_Seconds'].agg(['mean', 'median', 'std'])\n    yearly_stats.plot(figsize=(14, 6))\n    plt.title('Lap Time Trends by Year')\n    plt.ylabel('Lap Time (seconds)')\n    plt.savefig('yearly_trends.png', bbox_inches='tight')\n    plt.show()\n\n# ==================================================\n# 4. Feature Engineering (EDA-focused)\n# ==================================================\nprint(\"\\n\" + \"=\"*50)\nprint(\"FEATURE ENGINEERING EXPLORATION\")\nprint(\"=\"*50)\n\n# Create sample features for analysis\nfull_data['speed_ratio'] = full_data['Avg_Speed_kmh'] / full_data['Circuit_Length_km']\nfull_data['tire_wear_estimate'] = full_data['Tire_Degradation_Factor_per_Lap'] * full_data['Laps']\nfull_data['experience'] = full_data['year_x'] - full_data['min_year']\n\n# Analyze created features\nprint(\"\\n=== Engineered Feature: Speed Ratio ===\")\nplt.figure(figsize=(14, 6))\nsns.scatterplot(data=full_data, x='speed_ratio', y='Lap_Time_Seconds', alpha=0.6)\nplt.title('Speed Ratio vs Lap Time')\nplt.savefig('engineered_speed_ratio.png', bbox_inches='tight')\nplt.show()\n\nprint(\"\\n=== Engineered Feature: Tire Wear Estimate ===\")\nplt.figure(figsize=(14, 6))\nsns.scatterplot(data=full_data, x='tire_wear_estimate', y='Lap_Time_Seconds', alpha=0.6)\nplt.title('Tire Wear Estimate vs Lap Time')\nplt.savefig('engineered_tire_wear.png', bbox_inches='tight')\nplt.show()\n\nprint(\"\\n=== Engineered Feature: Rider Experience ===\")\nplt.figure(figsize=(14, 6))\nsns.boxplot(data=full_data, x='experience', y='Lap_Time_Seconds')\nplt.title('Rider Experience vs Lap Time')\nplt.xticks(rotation=45)\nplt.savefig('engineered_experience.png', bbox_inches='tight')\nplt.show()\n\n# Save EDA report\nprint(\"\\nEDA completed successfully! Visualizations saved.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-14T16:58:24.71195Z","iopub.execute_input":"2025-06-14T16:58:24.713023Z","iopub.status.idle":"2025-06-14T17:01:27.88869Z","shell.execute_reply.started":"2025-06-14T16:58:24.712899Z","shell.execute_reply":"2025-06-14T17:01:27.887981Z"}},"outputs":[],"execution_count":null}]}