{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":104857,"databundleVersionId":12651513,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-14T13:04:54.002027Z","iopub.execute_input":"2025-06-14T13:04:54.002212Z","iopub.status.idle":"2025-06-14T13:04:56.848312Z","shell.execute_reply.started":"2025-06-14T13:04:54.002196Z","shell.execute_reply":"2025-06-14T13:04:56.847696Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-14T13:04:58.814961Z","iopub.execute_input":"2025-06-14T13:04:58.815655Z","iopub.status.idle":"2025-06-14T13:04:58.818775Z","shell.execute_reply.started":"2025-06-14T13:04:58.815633Z","shell.execute_reply":"2025-06-14T13:04:58.818024Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df=pd.read_csv('/kaggle/input/burnout-datathon-ieeecsmuj/train.csv')\ntrain_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-14T13:05:00.854937Z","iopub.execute_input":"2025-06-14T13:05:00.855200Z","iopub.status.idle":"2025-06-14T13:05:16.583831Z","shell.execute_reply.started":"2025-06-14T13:05:00.855180Z","shell.execute_reply":"2025-06-14T13:05:16.583096Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df=pd.read_csv('/kaggle/input/burnout-datathon-ieeecsmuj/test.csv')\ntest_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-14T13:05:19.345379Z","iopub.execute_input":"2025-06-14T13:05:19.345930Z","iopub.status.idle":"2025-06-14T13:05:24.148188Z","shell.execute_reply.started":"2025-06-14T13:05:19.345907Z","shell.execute_reply":"2025-06-14T13:05:24.147469Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"val_df=pd.read_csv('/kaggle/input/burnout-datathon-ieeecsmuj/val.csv')\nval_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-14T13:05:26.854832Z","iopub.execute_input":"2025-06-14T13:05:26.855676Z","iopub.status.idle":"2025-06-14T13:05:29.257299Z","shell.execute_reply.started":"2025-06-14T13:05:26.855646Z","shell.execute_reply":"2025-06-14T13:05:29.256589Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.isnull().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-14T13:05:31.174806Z","iopub.execute_input":"2025-06-14T13:05:31.175084Z","iopub.status.idle":"2025-06-14T13:05:32.366162Z","shell.execute_reply.started":"2025-06-14T13:05:31.175062Z","shell.execute_reply":"2025-06-14T13:05:32.365475Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train_df.info())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-14T13:05:33.464971Z","iopub.execute_input":"2025-06-14T13:05:33.465481Z","iopub.status.idle":"2025-06-14T13:05:33.493694Z","shell.execute_reply.started":"2025-06-14T13:05:33.465456Z","shell.execute_reply":"2025-06-14T13:05:33.492940Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df['Penalty']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-14T13:05:36.850265Z","iopub.execute_input":"2025-06-14T13:05:36.850868Z","iopub.status.idle":"2025-06-14T13:05:36.856515Z","shell.execute_reply.started":"2025-06-14T13:05:36.850844Z","shell.execute_reply":"2025-06-14T13:05:36.855984Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"with pd.option_context('display.max_columns', None):\n    print(train_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-14T13:05:37.864890Z","iopub.execute_input":"2025-06-14T13:05:37.865176Z","iopub.status.idle":"2025-06-14T13:05:37.878477Z","shell.execute_reply.started":"2025-06-14T13:05:37.865157Z","shell.execute_reply":"2025-06-14T13:05:37.877809Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train_df['category_x'].value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-14T13:05:40.054931Z","iopub.execute_input":"2025-06-14T13:05:40.055226Z","iopub.status.idle":"2025-06-14T13:05:40.182301Z","shell.execute_reply.started":"2025-06-14T13:05:40.055204Z","shell.execute_reply":"2025-06-14T13:05:40.181310Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"team_means1 = train_df.groupby('circuit_name')['Lap_Time_Seconds'].mean()\nteam_means2 = train_df.groupby('shortname')['Lap_Time_Seconds'].mean()\nteam_means3 = train_df.groupby('team_name')['Lap_Time_Seconds'].mean()\nteam_means4 = train_df.groupby('rider_name')['Lap_Time_Seconds'].mean()\nteam_means5 = train_df.groupby('bike_name')['Lap_Time_Seconds'].mean()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-14T13:05:41.575067Z","iopub.execute_input":"2025-06-14T13:05:41.575310Z","iopub.status.idle":"2025-06-14T13:05:42.171473Z","shell.execute_reply.started":"2025-06-14T13:05:41.575294Z","shell.execute_reply":"2025-06-14T13:05:42.170693Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def feature_engineering(df):\n    df = pd.get_dummies(df, columns=['category_x'], prefix='moto', dtype=int)\n    df = pd.get_dummies(df, columns=['Track_Condition'], prefix='Track_Condition', dtype=int,drop_first=True)\n    df = pd.get_dummies(df, columns=['Tire_Compound_Rear'], prefix='rear_tire', dtype=int, drop_first=True)\n    df = pd.get_dummies(df, columns=['Tire_Compound_Front'], prefix='front_tire', dtype=int, drop_first=True)\n    df = pd.get_dummies(df, columns=['Penalty'], prefix='Penalty', dtype=int, drop_first=True)\n    df = pd.get_dummies(df, columns=['Session'], prefix='Session', dtype=int, drop_first=True)\n    df = pd.get_dummies(df, columns=['weather'], prefix='weather', dtype=int, drop_first=True)\n    df = pd.get_dummies(df, columns=['track'], prefix='track', dtype=int, drop_first=True)\n    team_means1 = df.groupby('circuit_name')['Lap_Time_Seconds'].mean()\n    df['Circuit_name_encoded'] = df['circuit_name'].map(team_means1)\n    team_means2 = df.groupby('shortname')['Lap_Time_Seconds'].mean()\n    df['Short_name_encoded'] = df['shortname'].map(team_means2)\n    team_means3 = df.groupby('team_name')['Lap_Time_Seconds'].mean()\n    df['team_encoded'] = df['team_name'].map(team_means3)\n    team_means4 = df.groupby('rider_name')['Lap_Time_Seconds'].mean()\n    df['rider_name_encoded'] = df['rider_name'].map(team_means4)\n    team_means5 = df.groupby('bike_name')['Lap_Time_Seconds'].mean()\n    df['bike_name_encoded'] = df['bike_name'].map(team_means5)\n    df=df.drop([\"circuit_name\",\"shortname\",\"team_name\",\"rider_name\",\"bike_name\",\n                            \"Unique ID\",\"Rider_ID\"],axis=1)\n    return df\n    \ntrain_df = feature_engineering(train_df)\nval_df = feature_engineering(val_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-14T13:05:55.700200Z","iopub.execute_input":"2025-06-14T13:05:55.700492Z","iopub.status.idle":"2025-06-14T13:06:08.550821Z","shell.execute_reply.started":"2025-06-14T13:05:55.700470Z","shell.execute_reply":"2025-06-14T13:06:08.550271Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def feature_engineering(df):\n    df = pd.get_dummies(df, columns=['category_x'], prefix='moto', dtype=int)\n    df = pd.get_dummies(df, columns=['Track_Condition'], prefix='Track_Condition', dtype=int,drop_first=True)\n    df = pd.get_dummies(df, columns=['Tire_Compound_Rear'], prefix='rear_tire', dtype=int, drop_first=True)\n    df = pd.get_dummies(df, columns=['Tire_Compound_Front'], prefix='front_tire', dtype=int, drop_first=True)\n    df = pd.get_dummies(df, columns=['Penalty'], prefix='Penalty', dtype=int, drop_first=True)\n    df = pd.get_dummies(df, columns=['Session'], prefix='Session', dtype=int, drop_first=True)\n    df = pd.get_dummies(df, columns=['weather'], prefix='weather', dtype=int, drop_first=True)\n    df = pd.get_dummies(df, columns=['track'], prefix='track', dtype=int, drop_first=True)\n    df['Circuit_name_encoded'] = df['circuit_name'].map(team_means1)\n    df['Short_name_encoded'] = df['shortname'].map(team_means2)\n    df['team_encoded'] = df['team_name'].map(team_means3)\n    df['rider_name_encoded'] = df['rider_name'].map(team_means4)\n    df['bike_name_encoded'] = df['bike_name'].map(team_means5)\n    df=df.drop([\"circuit_name\",\"shortname\",\"team_name\",\"rider_name\",\"bike_name\",\n                            \"Unique ID\",\"Rider_ID\"],axis=1)\n    return df\n\ntest_df = feature_engineering(test_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-14T13:06:11.460370Z","iopub.execute_input":"2025-06-14T13:06:11.460652Z","iopub.status.idle":"2025-06-14T13:06:14.198185Z","shell.execute_reply.started":"2025-06-14T13:06:11.460632Z","shell.execute_reply":"2025-06-14T13:06:14.197220Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\nimport numpy as np\n\nX_train = train_df.drop(['Lap_Time_Seconds'], axis=1)  # Remove target and original categorical columns\ny_train = train_df['Lap_Time_Seconds']\n\n\nX_val = val_df.drop(['Lap_Time_Seconds'], axis=1)\ny_val = val_df['Lap_Time_Seconds']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-14T13:06:22.615534Z","iopub.execute_input":"2025-06-14T13:06:22.615818Z","iopub.status.idle":"2025-06-14T13:06:25.399686Z","shell.execute_reply.started":"2025-06-14T13:06:22.615798Z","shell.execute_reply":"2025-06-14T13:06:25.398802Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tensorflow.keras import regularizers\nfrom tensorflow.keras.optimizers import AdamW\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nmodel1 = keras.Sequential([\n    layers.Input(shape=(57,)),\n    layers.Dense(units=512,kernel_initializer='he_normal',kernel_regularizer=regularizers.l2(1e-4),activation=None),\n    layers.BatchNormalization(),\n    #layers.Activation('relu'),\n    layers.LeakyReLU(alpha=0.1),\n    layers.Dropout(0.1),\n    layers.Dense(units=512,kernel_regularizer=regularizers.l2(1e-4),activation=None),\n    layers.BatchNormalization(),\n    #layers.Activation('relu'),\n    layers.LeakyReLU(alpha=0.1),\n    layers.Dropout(0.1),\n    layers.Dense(units=128,activation=None),\n    layers.BatchNormalization(),\n    #layers.Activation('relu'),\n    layers.LeakyReLU(alpha=0.1),\n    layers.Dropout(0.1),\n    layers.Dense(units=128,activation=None),\n    layers.BatchNormalization(),\n    #layers.Activation('relu'),\n    layers.LeakyReLU(alpha=0.1),\n    layers.Dropout(0.1),\n    layers.Dense(units=16,activation=None),\n    layers.BatchNormalization(),\n    #layers.Activation('relu'),\n    layers.LeakyReLU(alpha=0.1),\n   # layers.Dropout(0.1),\n    layers.Dense(units=1, activation=None)\n])\nmodel1.compile(\n    optimizer=AdamW(learning_rate=1e-3, weight_decay=1e-4),  # Lower learning rate\n    loss='mse',\n    metrics=['mae']\n)\n\n# More patient early stopping\nearly_stopping = keras.callbacks.EarlyStopping(\n    patience=25,           # Increased from 15\n    min_delta=0.00001,     # Smaller threshold (was 0.0001)\n    restore_best_weights=True,\n    monitor='val_loss'\n)\n\n# More patient learning rate scheduler\nlr_scheduler = keras.callbacks.ReduceLROnPlateau(\n    monitor='val_loss',\n    factor=0.8,            # Less aggressive reduction (was 0.7)\n    patience=15,           # Increased from 10\n    min_lr=1e-7,           # Lower minimum (was 1e-6)\n    verbose=1\n)\nhistory1=model1.fit(\n    X_train,y_train,\n    validation_data=(X_val,y_val),\n    batch_size=1024,\n    epochs=500,\n    callbacks=[early_stopping,lr_scheduler],\n    verbose=1\n)\n\nfrom sklearn.metrics import mean_squared_error\n\ntrain_pred = model1.predict(X_train)\nval_pred = model1.predict(X_val)\n\ntrain_rmse = np.sqrt(mean_squared_error(y_train, train_pred))\nval_rmse = np.sqrt(mean_squared_error(y_val, val_pred))\n\nprint(f\"Train RMSE: {train_rmse:.4f}\")\nprint(f\"Validation RMSE: {val_rmse:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-14T13:14:56.885967Z","iopub.execute_input":"2025-06-14T13:14:56.886591Z","iopub.status.idle":"2025-06-14T14:10:00.508047Z","shell.execute_reply.started":"2025-06-14T13:14:56.886556Z","shell.execute_reply":"2025-06-14T14:10:00.507240Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_submission = pd.read_csv('/kaggle/input/burnout-datathon-ieeecsmuj/sample_submission.csv')\n\n# Check the structure of sample submission\nprint(\"Sample submission structure:\")\nprint(f\"Shape: {sample_submission.shape}\")\nprint(f\"Columns: {sample_submission.columns.tolist()}\")\nprint(\"\\nFirst few rows:\")\nprint(sample_submission.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-14T14:14:44.085274Z","iopub.execute_input":"2025-06-14T14:14:44.086014Z","iopub.status.idle":"2025-06-14T14:14:44.340917Z","shell.execute_reply.started":"2025-06-14T14:14:44.085989Z","shell.execute_reply":"2025-06-14T14:14:44.340288Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"id_column = sample_submission['Unique ID']\n\nprint(f\"\\nID column extracted:\")\nprint(f\"Column name: 'Unique ID'\")\nprint(f\"Number of IDs: {len(id_column)}\")\nprint(f\"First few IDs: {id_column.head().tolist()}\")\n\n# Generate predictions on your test data\n# Make sure test_df has the same 57 features in the same order as training\ntest_predictions = model1.predict(test_df)\n\n# Create submission using the exact same format as sample\nsubmission = pd.DataFrame({\n    'Unique ID': id_column,\n    'Lap_Time_Seconds': test_predictions.flatten()\n})\n\n# Save submission\nsubmission.to_csv('submission.csv', index=False)\n\nprint(f\"\\nSubmission created with columns: {submission.columns.tolist()}\")\nprint(\"Submission saved as 'submission.csv'\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-14T14:14:46.784668Z","iopub.execute_input":"2025-06-14T14:14:46.785202Z","iopub.status.idle":"2025-06-14T14:15:21.475154Z","shell.execute_reply.started":"2025-06-14T14:14:46.785182Z","shell.execute_reply":"2025-06-14T14:15:21.474510Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nfrom sklearn.decomposition import PCA\nfrom sklearn.preprocessing import StandardScaler\n\n# 1. Standardize your data (important!)\nscaler = StandardScaler()\nX_scaled = scaler.fit_transform(X_train)  # X is your feature matrix\n\n# 2. Apply PCA\npca = PCA(n_components=0.95)  # Keep 95% of variance\nX_pca = pca.fit_transform(X_scaled)\n\nX_val_scaled = scaler.transform(X_val)  # Note: transform, not fit_transform\nX_val_pca = pca.transform(X_val_scaled)\n\n# 3. Check results\nprint(f\"Original features: {X_train.shape[1]}\")\nprint(f\"PCA features: {X_pca.shape[1]}\")\nprint(f\"Variance explained: {pca.explained_variance_ratio_.sum():.3f}\")\n'''","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-14T14:17:21.670265Z","iopub.execute_input":"2025-06-14T14:17:21.670764Z","iopub.status.idle":"2025-06-14T14:17:21.676303Z","shell.execute_reply.started":"2025-06-14T14:17:21.670742Z","shell.execute_reply":"2025-06-14T14:17:21.675632Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nimport lightgbm as lgb\nfrom sklearn.metrics import mean_squared_error, mean_absolute_error\ntrain_data = lgb.Dataset(X_pca, label=y_train)\nval_data = lgb.Dataset(X_val_pca, label=y_val, reference=train_data)\n\n# 6. Set LightGBM parameters\nparams = {\n    'objective': 'regression',\n    'metric': ['rmse', 'mae'],\n    'boosting_type': 'gbdt',\n    'num_leaves': 31,\n    'learning_rate': 0.05,\n    'feature_fraction': 0.9,\n    'bagging_fraction': 0.8,\n    'bagging_freq': 5,\n    'verbose': 0,\n    'random_state': 42\n}\n\n# 7. Train the model\nprint(\"Training LightGBM model...\")\nmodel = lgb.train(\n    params,\n    train_data,\n    valid_sets=[train_data, val_data],\n    valid_names=['train', 'val'],\n    num_boost_round=1000,\n    callbacks=[\n        lgb.early_stopping(stopping_rounds=50),\n        lgb.log_evaluation(period=100)\n    ]\n)\n\n# 8. Make predictions\ntrain_pred = model.predict(X_pca, num_iteration=model.best_iteration)\nval_pred = model.predict(X_val_pca, num_iteration=model.best_iteration)\n\n# 9. Calculate metrics\ntrain_rmse = np.sqrt(mean_squared_error(y_train, train_pred))\nval_rmse = np.sqrt(mean_squared_error(y_val, val_pred))\ntrain_mae = mean_absolute_error(y_train, train_pred)\nval_mae = mean_absolute_error(y_val, val_pred)\n\nprint(f\"\\n=== Model Performance ===\")\nprint(f\"Training RMSE: {train_rmse:.4f}\")\nprint(f\"Validation RMSE: {val_rmse:.4f}\")\nprint(f\"Training MAE: {train_mae:.4f}\")\nprint(f\"Validation MAE: {val_mae:.4f}\")\n\n# 10. Feature importance (top 10 PCA components)\nfeature_importance = model.feature_importance(importance_type='gain')\nprint(f\"\\n=== Top 10 PCA Components by Importance ===\")\nfor i in range(min(10, len(feature_importance))):\n    idx = np.argsort(feature_importance)[::-1][i]\n    print(f\"PCA Component {idx}: {feature_importance[idx]:.0f}\")\n\n# Optional: Plot feature importance\nimport matplotlib.pyplot as plt\nlgb.plot_importance(model, max_num_features=15, importance_type='gain')\nplt.title('Top 15 PCA Components Feature Importance')\nplt.tight_layout()\nplt.show()\n'''","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-14T14:17:23.224855Z","iopub.execute_input":"2025-06-14T14:17:23.225544Z","iopub.status.idle":"2025-06-14T14:17:23.230835Z","shell.execute_reply.started":"2025-06-14T14:17:23.225520Z","shell.execute_reply":"2025-06-14T14:17:23.230126Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"'''\nimport lightgbm as lgb\nfrom sklearn.metrics import mean_squared_error\nimport numpy as np\nimport pandas as pd\nfrom itertools import product\n\nprint(\"=\" * 60)\nprint(\"LIGHTGBM STAGED GRID SEARCH HYPERPARAMETER TUNING\")\nprint(\"=\" * 60)\n\ndef train_and_evaluate(params, X_train, y_train, X_val, y_val):\n    \"\"\"Train model and return validation RMSE\"\"\"\n    train_data = lgb.Dataset(X_train, label=y_train)\n    val_data = lgb.Dataset(X_val, label=y_val, reference=train_data)\n    \n    model = lgb.train(\n        params,\n        train_data,\n        valid_sets=[val_data],\n        valid_names=['val'],\n        num_boost_round=1000,\n        callbacks=[\n            lgb.early_stopping(stopping_rounds=50, verbose=False),\n            lgb.log_evaluation(period=0)\n        ]\n    )\n    \n    val_pred = model.predict(X_val, num_iteration=model.best_iteration)\n    val_rmse = np.sqrt(mean_squared_error(y_val, val_pred))\n    return val_rmse, model\n\n# Stage 1: Find best num_leaves and learning_rate\nprint(\"Stage 1: Tuning num_leaves and learning_rate...\")\nstage1_params = {\n    'objective': 'regression',\n    'metric': 'rmse',\n    'boosting_type': 'gbdt',\n    'feature_fraction': 0.8,\n    'bagging_fraction': 0.8,\n    'bagging_freq': 5,\n    'verbose': -1,\n    'random_state': 42\n}\n\nstage1_grid = {\n    'num_leaves': [10, 15, 20, 25, 31],\n    'learning_rate': [0.01, 0.03, 0.05, 0.1, 0.15]\n}\n\nbest_stage1_rmse = float('inf')\nbest_stage1_params = None\n\nresults_stage1 = []\ntotal_combinations = len(stage1_grid['num_leaves']) * len(stage1_grid['learning_rate'])\ncurrent_combo = 0\n\nfor num_leaves, lr in product(stage1_grid['num_leaves'], stage1_grid['learning_rate']):\n    current_combo += 1\n    params = stage1_params.copy()\n    params.update({'num_leaves': num_leaves, 'learning_rate': lr})\n    \n    rmse, _ = train_and_evaluate(params, X_pca, y_train, X_val_pca, y_val)\n    results_stage1.append({\n        'num_leaves': num_leaves,\n        'learning_rate': lr,\n        'rmse': rmse\n    })\n    \n    print(f\"  [{current_combo:2d}/{total_combinations}] num_leaves: {num_leaves:2d}, lr: {lr:.3f} -> RMSE: {rmse:.4f}\")\n    \n    if rmse < best_stage1_rmse:\n        best_stage1_rmse = rmse\n        best_stage1_params = {'num_leaves': num_leaves, 'learning_rate': lr}\n        print(f\"    ✓ New best!\")\n\nprint(f\"\\nBest Stage 1 RMSE: {best_stage1_rmse:.4f}\")\nprint(f\"Best num_leaves: {best_stage1_params['num_leaves']}\")\nprint(f\"Best learning_rate: {best_stage1_params['learning_rate']}\")\n\n# Stage 2: Tune regularization with best num_leaves and learning_rate\nprint(\"\\n\" + \"-\" * 60)\nprint(\"Stage 2: Tuning regularization...\")\nstage2_params = stage1_params.copy()\nstage2_params.update(best_stage1_params)\n\nstage2_grid = {\n    'lambda_l1': [0, 0.1, 0.3, 0.5, 1.0],\n    'lambda_l2': [0, 0.1, 0.3, 0.5, 1.0]\n}\n\nbest_stage2_rmse = float('inf')\nbest_stage2_params = None\n\nresults_stage2 = []\ntotal_combinations = len(stage2_grid['lambda_l1']) * len(stage2_grid['lambda_l2'])\ncurrent_combo = 0\n\nfor l1, l2 in product(stage2_grid['lambda_l1'], stage2_grid['lambda_l2']):\n    current_combo += 1\n    params = stage2_params.copy()\n    params.update({'lambda_l1': l1, 'lambda_l2': l2})\n    \n    rmse, _ = train_and_evaluate(params, X_pca, y_train, X_val_pca, y_val)\n    results_stage2.append({\n        'lambda_l1': l1,\n        'lambda_l2': l2,\n        'rmse': rmse\n    })\n    \n    print(f\"  [{current_combo:2d}/{total_combinations}] L1: {l1:.1f}, L2: {l2:.1f} -> RMSE: {rmse:.4f}\")\n    \n    if rmse < best_stage2_rmse:\n        best_stage2_rmse = rmse\n        best_stage2_params = {'lambda_l1': l1, 'lambda_l2': l2}\n        print(f\"    ✓ New best!\")\n\nprint(f\"\\nBest Stage 2 RMSE: {best_stage2_rmse:.4f}\")\nprint(f\"Best lambda_l1: {best_stage2_params['lambda_l1']}\")\nprint(f\"Best lambda_l2: {best_stage2_params['lambda_l2']}\")\n\n# Stage 3: Fine-tune sampling parameters\nprint(\"\\n\" + \"-\" * 60)\nprint(\"Stage 3: Tuning sampling parameters...\")\nstage3_params = stage2_params.copy()\nstage3_params.update(best_stage2_params)\n\nstage3_grid = {\n    'feature_fraction': [0.6, 0.7, 0.8, 0.9],\n    'bagging_fraction': [0.6, 0.7, 0.8, 0.9],\n    'min_data_in_leaf': [10, 20, 30, 50]\n}\n\nbest_stage3_rmse = float('inf')\nbest_stage3_params = None\nbest_model = None\n\nresults_stage3 = []\ntotal_combinations = len(stage3_grid['feature_fraction']) * len(stage3_grid['bagging_fraction']) * len(stage3_grid['min_data_in_leaf'])\ncurrent_combo = 0\n\nfor ff, bf, mdil in product(stage3_grid['feature_fraction'], \n                           stage3_grid['bagging_fraction'],\n                           stage3_grid['min_data_in_leaf']):\n    current_combo += 1\n    params = stage3_params.copy()\n    params.update({\n        'feature_fraction': ff,\n        'bagging_fraction': bf,\n        'min_data_in_leaf': mdil\n    })\n    \n    rmse, model = train_and_evaluate(params, X_pca, y_train, X_val_pca, y_val)\n    results_stage3.append({\n        'feature_fraction': ff,\n        'bagging_fraction': bf,\n        'min_data_in_leaf': mdil,\n        'rmse': rmse\n    })\n    \n    print(f\"  [{current_combo:2d}/{total_combinations}] FF: {ff:.1f}, BF: {bf:.1f}, MDL: {mdil:2d} -> RMSE: {rmse:.4f}\")\n    \n    if rmse < best_stage3_rmse:\n        best_stage3_rmse = rmse\n        best_stage3_params = {\n            'feature_fraction': ff,\n            'bagging_fraction': bf,\n            'min_data_in_leaf': mdil\n        }\n        best_model = model\n        print(f\"    ✓ New best!\")\n\nprint(f\"\\nBest Stage 3 RMSE: {best_stage3_rmse:.4f}\")\nprint(f\"Best feature_fraction: {best_stage3_params['feature_fraction']}\")\nprint(f\"Best bagging_fraction: {best_stage3_params['bagging_fraction']}\")\nprint(f\"Best min_data_in_leaf: {best_stage3_params['min_data_in_leaf']}\")\n\n# Final best parameters\nfinal_best_params = stage3_params.copy()\nfinal_best_params.update(best_stage1_params)\nfinal_best_params.update(best_stage2_params)\nfinal_best_params.update(best_stage3_params)\n\nprint(\"\\n\" + \"=\" * 60)\nprint(\"FINAL RESULTS\")\nprint(\"=\" * 60)\nprint(f\"Original RMSE: 10.3841\")\nprint(f\"Best Grid Search RMSE: {best_stage3_rmse:.4f}\")\nprint(f\"Improvement: {10.3841 - best_stage3_rmse:.4f} ({((10.3841 - best_stage3_rmse)/10.3841)*100:.2f}%)\")\n\nprint(\"\\nFinal Best Parameters:\")\nfor param, value in final_best_params.items():\n    if param not in ['objective', 'metric', 'boosting_type', 'verbose', 'random_state']:\n        print(f\"  {param}: {value}\")\n\n# Train final model and get detailed metrics\nprint(\"\\nTraining final model with best parameters...\")\nfinal_train_pred = best_model.predict(X_pca, num_iteration=best_model.best_iteration)\nfinal_val_pred = best_model.predict(X_val_pca, num_iteration=best_model.best_iteration)\n\nfinal_train_rmse = np.sqrt(mean_squared_error(y_train, final_train_pred))\nfinal_val_rmse = np.sqrt(mean_squared_error(y_val, final_val_pred))\nfinal_train_mae = np.mean(np.abs(y_train - final_train_pred))\nfinal_val_mae = np.mean(np.abs(y_val - final_val_pred))\n\nprint(f\"\\nFinal Model Performance:\")\nprint(f\"Training RMSE: {final_train_rmse:.4f}\")\nprint(f\"Validation RMSE: {final_val_rmse:.4f}\")\nprint(f\"Training MAE: {final_train_mae:.4f}\")\nprint(f\"Validation MAE: {final_val_mae:.4f}\")\nprint(f\"Overfitting Gap: {final_val_rmse - final_train_rmse:.4f}\")\n\n# Display top 5 parameter combinations from each stage\nprint(\"\\n\" + \"=\" * 60)\nprint(\"TOP 5 COMBINATIONS FROM EACH STAGE\")\nprint(\"=\" * 60)\n\nprint(\"\\nStage 1 - Top 5 (num_leaves, learning_rate):\")\ndf_stage1 = pd.DataFrame(results_stage1).sort_values('rmse')\nfor i, row in df_stage1.head().iterrows():\n    print(f\"  {row['num_leaves']:2d}, {row['learning_rate']:.3f} -> RMSE: {row['rmse']:.4f}\")\n\nprint(\"\\nStage 2 - Top 5 (lambda_l1, lambda_l2):\")\ndf_stage2 = pd.DataFrame(results_stage2).sort_values('rmse')\nfor i, row in df_stage2.head().iterrows():\n    print(f\"  {row['lambda_l1']:.1f}, {row['lambda_l2']:.1f} -> RMSE: {row['rmse']:.4f}\")\n\nprint(\"\\nStage 3 - Top 5 (feature_frac, bagging_frac, min_data_leaf):\")\ndf_stage3 = pd.DataFrame(results_stage3).sort_values('rmse')\nfor i, row in df_stage3.head().iterrows():\n    print(f\"  {row['feature_fraction']:.1f}, {row['bagging_fraction']:.1f}, {row['min_data_in_leaf']:2d} -> RMSE: {row['rmse']:.4f}\")\n\nprint(f\"\\n\" + \"=\" * 60)\nprint(\"COPY-PASTE BEST PARAMETERS:\")\nprint(\"=\" * 60)\nprint(\"params = {\")\nfor param, value in final_best_params.items():\n    if isinstance(value, str):\n        print(f\"    '{param}': '{value}',\")\n    else:\n        print(f\"    '{param}': {value},\")\nprint(\"}\")\n'''","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-14T14:17:25.840504Z","iopub.execute_input":"2025-06-14T14:17:25.841227Z","iopub.status.idle":"2025-06-14T14:17:25.850718Z","shell.execute_reply.started":"2025-06-14T14:17:25.841205Z","shell.execute_reply":"2025-06-14T14:17:25.850041Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}