{"metadata":{"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"},"colab":{"provenance":[],"mount_file_id":"1om4MyUeheMB-SW4o739KcVDxanRToTOq","authorship_tag":"ABX9TyMsyjbz/H7ytWfSzoT9dpOe"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":13896081,"sourceType":"datasetVersion","datasetId":8853252},{"sourceId":14029322,"sourceType":"datasetVersion","datasetId":8933677}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from google.colab import drive\nimport tensorflow as tf\nfrom tensorflow import keras\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt","metadata":{"id":"ClTFKe8Wpv3K"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\n# Ensure test_df is loaded\ntrain_df = pd.read_csv('/content/drive/MyDrive/Australia_BioMass/csiro-biomass/train.csv')\ntest_df = pd.read_csv('/content/drive/MyDrive/Australia_BioMass/csiro-biomass/test.csv')\ntest_level_df = pd.read_csv('/content/drive/MyDrive/Australia_BioMass/csiro-biomass/sample_submission (1).csv')\n\n# Ensure all individual model predictions are generated for test_df_processed\ny_pred_gbr_test = best_gbr_model.predict(test_df_processed)\ny_pred_lgbm_test = lgbm_model.predict(test_df_processed)\ny_pred_catboost_test = catboost_model.predict(test_df_processed)\ny_pred_xgb_test = xgb_model.predict(test_df_processed)\ny_pred_rf_test = model_new_features.predict(test_df_processed)\n\n# Calculate the average of the individual model predictions\ny_pred_ensemble_avg_test = (y_pred_gbr_test + y_pred_lgbm_test + y_pred_catboost_test + y_pred_xgb_test + y_pred_rf_test) / 5\n\nsubmission_df = pd.DataFrame({\n    'image_id': test_df['sample_id'],\n    'target': y_pred_ensemble_avg_test\n})\n\n# Ensure the target values are non-negative, as biomass cannot be negative\nsubmission_df['target'] = submission_df['target'].apply(lambda x: max(0, x))\n\nsubmission_df.to_csv('submission.csv', index=False)\n\nprint('Submission file \"submission.csv\" created successfully.')","metadata":{"id":"2a5ce99e","executionInfo":{"status":"error","timestamp":1765027082727,"user_tz":-330,"elapsed":38,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"162ea9ca-e801-43c0-92c5-5160c88c3068"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from google.colab import files\nfiles.download('submission.csv')","metadata":{"id":"d49cd954","executionInfo":{"status":"error","timestamp":1765027047557,"user_tz":-330,"elapsed":45,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"0173b8d1-eefb-48bc-b2f4-1d3493595615"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"drive.mount('/content/drive')","metadata":{"executionInfo":{"elapsed":43041,"status":"ok","timestamp":1764221501736,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"},"user_tz":-330},"id":"ri2Yen-GqCAe","outputId":"fbd8320d-3ff3-4d91-eed2-eb1af79575c0"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\ntrain_df = pd.read_csv('/content/drive/MyDrive/Australia_BioMass/csiro-biomass/train')\ntest_df = pd.read_csv('/content/drive/MyDrive/Australia_BioMass/csiro-biomass/test')\ntest_level_df = pd.read_csv('/content/drive/MyDrive/Australia_BioMass/csiro-biomass/sample_submission (1).csv')","metadata":{"id":"0wELoGYqqWJF"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(X_train_new.info())","metadata":{"id":"298b8d11","executionInfo":{"status":"ok","timestamp":1764224529490,"user_tz":-330,"elapsed":7,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"36445a99-6466-4190-dec7-1609eecbf937"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from xgboost import XGBRegressor\nfrom sklearn.metrics import mean_absolute_error, r2_score\n\n# Initialize and train the XGBoost Regressor model with engineered features\nxgb_model = XGBRegressor(random_state=42)\nxgb_model.fit(X_train_new, y_train_new)\n\n# Make predictions on the validation set\ny_pred_xgb = xgb_model.predict(X_val_new)\n\n# Evaluate the model\nmae_xgb = mean_absolute_error(y_val_new, y_pred_xgb)\nr2_xgb = r2_score(y_val_new, y_pred_xgb)\n\nprint(f\"XGBoost - Mean Absolute Error on validation set: {mae_xgb:.4f}\")\nprint(f\"XGBoost - R-squared on validation set: {r2_xgb:.4f}\")","metadata":{"id":"adfd6378","executionInfo":{"status":"ok","timestamp":1764224656884,"user_tz":-330,"elapsed":171,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"74e3ab90-bf71-441a-b37f-4246598f6b7d"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor\nfrom sklearn.metrics import mean_absolute_error, r2_score\n\n# Initialize and train the RandomForestRegressor model with new features\nmodel_new_features = RandomForestRegressor(random_state=42)\nmodel_new_features.fit(X_train_new, y_train_new)\n\n# Make predictions on the validation set with new features\ny_pred_new_features = model_new_features.predict(X_val_new)\n\n# Evaluate the model with new features\nmae_new_features = mean_absolute_error(y_val_new, y_pred_new_features)\nr2_new_features = r2_score(y_val_new, y_pred_new_features)\n\nprint(f\"Mean Absolute Error on validation set (with new features): {mae_new_features:.4f}\")\nprint(f\"R-squared on validation set (with new features): {r2_new_features:.4f}\")","metadata":{"id":"b84f6d68","executionInfo":{"status":"ok","timestamp":1764224351702,"user_tz":-330,"elapsed":2575,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"feb87a17-27e6-4921-ca21-e330991fa3bb"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor\nfrom sklearn.metrics import mean_absolute_error, r2_score\n\n# Initialize and train the RandomForestRegressor model\nmodel = RandomForestRegressor(random_state=42)\nmodel.fit(X_train_new, y_train_new)\n\n# Make predictions on the validation set\ny_pred = model.predict(X_val_new)\n\n# Evaluate the model\nmae = mean_absolute_error(y_val_new, y_pred)\nr2 = r2_score(y_val_new, y_pred)\n\nprint(f\"Mean Absolute Error on validation set: {mae:.4f}\")\nprint(f\"R-squared on validation set: {r2:.4f}\")","metadata":{"id":"7b8bdd93","executionInfo":{"status":"ok","timestamp":1764223429501,"user_tz":-330,"elapsed":733,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"1dc3898e-f536-4ad2-f7dd-4c080a3ec374"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\nplt.figure(figsize=(10, 6))\nsns.scatterplot(x='Pre_GSHH_NDVI', y='target', data=train_df)\nplt.title('Relationship between Pre_GSHH_NDVI and Target in train_df')\nplt.xlabel('Pre_GSHH_NDVI')\nplt.ylabel('Target Value')\nplt.show()","metadata":{"id":"874b802e","executionInfo":{"status":"ok","timestamp":1764222766779,"user_tz":-330,"elapsed":334,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"babc1f35-3dd5-473e-b2ab-debb504bd36d"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\nplt.figure(figsize=(10, 6))\nsns.histplot(train_df['target'], kde=True, bins=50)\nplt.title('Distribution of Target Column in train_df')\nplt.xlabel('Target Value')\nplt.ylabel('Frequency')\nplt.show()","metadata":{"id":"0ba4b170","executionInfo":{"status":"ok","timestamp":1764222730049,"user_tz":-330,"elapsed":445,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"ed0624d6-aaf8-479c-c210-64ba9e74c59b"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"display(test_df.describe())","metadata":{"id":"0a73617b","executionInfo":{"status":"ok","timestamp":1764222671160,"user_tz":-330,"elapsed":272,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"ba23bba1-9f50-4f71-9a55-073814f7b9a0"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"missing_values = train_df.isnull().sum()\nprint(\"Missing values in train_df:\")\nprint(missing_values[missing_values > 0])","metadata":{"id":"79121fbb","executionInfo":{"status":"ok","timestamp":1764222601265,"user_tz":-330,"elapsed":10,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"a8fc3807-f175-4a15-a654-d3dc8ea0c548"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y = train_df['target']\nX = train_df[['Pre_GSHH_NDVI', 'Height_Ave_cm']]\n\nprint(\"Shape of X:\", X.shape)\nprint(\"Shape of y:\", y.shape)\nprint(\"\\nFirst 5 rows of X:\")\ndisplay(X.head())\nprint(\"\\nFirst 5 rows of y:\")\ndisplay(y.head())","metadata":{"id":"e1517085","executionInfo":{"status":"ok","timestamp":1764222882941,"user_tz":-330,"elapsed":28,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"367e0762-5a04-4291-e591-311e9bb70170"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"missing_values_test = test_df.isnull().sum()\nprint(\"Missing values in test_df:\")\nprint(missing_values_test[missing_values_test > 0])","metadata":{"id":"6a2196b9","executionInfo":{"status":"ok","timestamp":1764222912444,"user_tz":-330,"elapsed":14,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"726ddda2-205a-4993-80da-663b18c8aa98"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nimport pandas as pd\n\n# Create a new DataFrame X_new by copying X\nX_new = X.copy()\n\n# Create a new DataFrame for test data by copying test_df\ntest_df_processed = test_df.copy()\n\n# Extract State and Species information per image_id from train_df\nimage_info = train_df[['image_id', 'State', 'Species']].drop_duplicates()\n\n# Merge this information into test_df_processed\ntest_df_processed = pd.merge(test_df_processed, image_info, on='image_id', how='left')\n\n# Impute NaN values in 'State' and 'Species' columns of test_df_processed with the mode from train_df\nmode_state_train = train_df['State'].mode()[0]\nmode_species_train = train_df['Species'].mode()[0]\n\ntest_df_processed['State'].fillna(mode_state_train, inplace=True)\ntest_df_processed['Species'].fillna(mode_species_train, inplace=True)\n\n# Initialize and apply LabelEncoder for 'State' column\nle_state = LabelEncoder()\nX_new['encoded_State'] = le_state.fit_transform(train_df['State'])\ntest_df_processed['encoded_State'] = le_state.transform(test_df_processed['State'])\n\n# Initialize and apply LabelEncoder for 'Species' column\nle_species = LabelEncoder()\nX_new['encoded_Species'] = le_species.fit_transform(train_df['Species'])\ntest_df_processed['encoded_Species'] = le_species.transform(test_df_processed['Species'])\n\nprint(\"First 5 rows of X_new with encoded features:\")\ndisplay(X_new.head())\n\nprint(\"\\nFirst 5 rows of test_df_processed with encoded features:\")\ndisplay(test_df_processed.head())","metadata":{"id":"6b41dba8","executionInfo":{"status":"ok","timestamp":1764223006979,"user_tz":-330,"elapsed":102,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"56691675-a3f0-4ec4-a7b4-ea82fd6c854e"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Reasoning**:\nThe previous step successfully encoded the categorical features 'State' and 'Species' and added them to `X_new` and `test_df_processed`. The next logical step is to ensure `X_new` (for training) and `test_df_processed` (for prediction) explicitly include all the relevant numerical and newly encoded categorical features, and then display their structure to confirm.\n\n","metadata":{"id":"fd2a7e63"}},{"cell_type":"code","source":"X_new = X_new[['Pre_GSHH_NDVI', 'Height_Ave_cm', 'encoded_State', 'encoded_Species']]\ntest_df_processed = test_df_processed[['Pre_GSHH_NDVI', 'Height_Ave_cm', 'encoded_State', 'encoded_Species']]\n\nprint(\"Features for training (X_new):\")\ndisplay(X_new.head())\nprint(\"\\nFeatures for testing (test_df_processed):\")\ndisplay(test_df_processed.head())\n\nprint(\"Shape of X_new:\", X_new.shape)\nprint(\"Shape of test_df_processed:\", test_df_processed.shape)","metadata":{"id":"6621b3ed","executionInfo":{"status":"ok","timestamp":1764223028878,"user_tz":-330,"elapsed":78,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"7cebf363-ef76-46d2-cfea-3a3fde6dcf1c"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\n# Define numerical features\nnumerical_features = ['Pre_GSHH_NDVI', 'Height_Ave_cm']\n\n# Initialize the StandardScaler\nscaler = StandardScaler()\n\nprint(\"StandardScaler initialized and numerical features identified.\")","metadata":{"id":"ef320ced","executionInfo":{"status":"ok","timestamp":1764223147761,"user_tz":-330,"elapsed":25,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"44a66386-4fdc-4f10-b357-437f5b84ddd9"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\n# Split the data into training and validation sets\nX_train_new, X_val_new, y_train_new, y_val_new = train_test_split(X_new, y, test_size=0.2, random_state=42)\n\n# Print the shapes of the resulting sets\nprint(\"Shape of X_train_new:\", X_train_new.shape)\nprint(\"Shape of X_val_new:\", X_val_new.shape)\nprint(\"Shape of y_train_new:\", y_train_new.shape)\nprint(\"Shape of y_val_new:\", y_val_new.shape)","metadata":{"id":"b9afec0c","executionInfo":{"status":"ok","timestamp":1764223171592,"user_tz":-330,"elapsed":27,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"b7a65ccb-08e6-4afa-9191-d97a8a1d5864"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train_new[numerical_features] = scaler.fit_transform(X_train_new[numerical_features])\nX_val_new[numerical_features] = scaler.transform(X_val_new[numerical_features])\ntest_df_processed[numerical_features] = scaler.transform(test_df_processed[numerical_features])\n\nprint(\"Scaled X_train_new (first 5 rows):\")\ndisplay(X_train_new.head())\n\nprint(\"\\nScaled X_val_new (first 5 rows):\")\ndisplay(X_val_new.head())\n\nprint(\"\\nScaled test_df_processed (first 5 rows):\")\ndisplay(test_df_processed.head())","metadata":{"id":"573bc45f","executionInfo":{"status":"ok","timestamp":1764223194167,"user_tz":-330,"elapsed":179,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"92c1425d-791c-4f97-e569-465b9fa9b430"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nimport pandas as pd\n\n# 1. Re-initialize X_new and test_df_processed\nX_new = X.copy()\ntest_df_processed = test_df.copy()\n\n# 2. Extract unique image_id, State, and Species from train_df\nimage_info = train_df[['image_id', 'State', 'Species', 'Sampling_Date']].drop_duplicates()\n\n# 3. Merge image_info into test_df_processed\ntest_df_processed = pd.merge(test_df_processed, image_info[['image_id', 'State', 'Species', 'Sampling_Date']], on='image_id', how='left')\n\n# 4. Calculate the mode of the 'State' column from train_df\nmode_state_train = train_df['State'].mode()[0]\n\n# 5. Calculate the mode of the 'Species' column from train_df\nmode_species_train = train_df['Species'].mode()[0]\n\n# 6. Impute any missing values (NaN) in the 'State' column of test_df_processed\ntest_df_processed['State'].fillna(mode_state_train, inplace=True)\n\n# 7. Impute any missing values (NaN) in the 'Species' column of test_df_processed\ntest_df_processed['Species'].fillna(mode_species_train, inplace=True)\n\n# 8. Initialize and apply LabelEncoder for 'State' column\nle_state = LabelEncoder()\nX_new['encoded_State'] = le_state.fit_transform(train_df['State'])\ntest_df_processed['encoded_State'] = le_state.transform(test_df_processed['State'])\n\n# 9. Initialize and apply LabelEncoder for 'Species' column\nle_species = LabelEncoder()\nX_new['encoded_Species'] = le_species.fit_transform(train_df['Species'])\ntest_df_processed['encoded_Species'] = le_species.transform(test_df_processed['Species'])\n\n# 10. Convert the 'Sampling_Date' column in train_df to datetime objects\ntrain_df['Sampling_Date'] = pd.to_datetime(train_df['Sampling_Date'])\n\n# 11. Extract date features for X_new\nX_new['year'] = train_df['Sampling_Date'].dt.year\nX_new['month'] = train_df['Sampling_Date'].dt.month\nX_new['day'] = train_df['Sampling_Date'].dt.day\nX_new['dayofweek'] = train_df['Sampling_Date'].dt.dayofweek\nX_new['dayofyear'] = train_df['Sampling_Date'].dt.dayofyear\n\n# 12. Create a DataFrame image_sampling_date_map containing unique image_id and Sampling_Date from train_df\n# This step was implicitly handled by including 'Sampling_Date' in image_info and merging in step 3.\n\n# 13. Merge image_sampling_date_map into test_df_processed\n# Done in step 3.\n\n# 14. Calculate the mode of the 'Sampling_Date' column from train_df\nmode_sampling_date_train = train_df['Sampling_Date'].mode()[0]\n\n# 15. Impute any missing values (NaN) in the 'Sampling_Date' column of test_df_processed\ntest_df_processed['Sampling_Date'].fillna(mode_sampling_date_train, inplace=True)\n\n# 16. Ensure the 'Sampling_Date' column in test_df_processed is of datetime type\ntest_df_processed['Sampling_Date'] = pd.to_datetime(test_df_processed['Sampling_Date'])\n\n# 17. Extract date features for test_df_processed\ntest_df_processed['year'] = test_df_processed['Sampling_Date'].dt.year\ntest_df_processed['month'] = test_df_processed['Sampling_Date'].dt.month\ntest_df_processed['day'] = test_df_processed['Sampling_Date'].dt.day\ntest_df_processed['dayofweek'] = test_df_processed['Sampling_Date'].dt.dayofweek\ntest_df_processed['dayofyear'] = test_df_processed['Sampling_Date'].dt.dayofyear\n\n# 18. Define a list final_features\nfinal_features = ['Pre_GSHH_NDVI', 'Height_Ave_cm', 'encoded_State', 'encoded_Species', 'year', 'month', 'day', 'dayofweek', 'dayofyear']\n\n# 19. Update X_new to include only the columns specified in final_features\nX_new = X_new[final_features]\n\n# 20. Update test_df_processed to include only the columns specified in final_features\ntest_df_processed = test_df_processed[final_features]\n\n# 21. Display the first 5 rows of X_new and test_df_processed\nprint(\"First 5 rows of X_new with engineered features:\")\ndisplay(X_new.head())\n\nprint(\"\\nFirst 5 rows of test_df_processed with engineered features:\")\ndisplay(test_df_processed.head())\n\n# 22. Print the shapes of X_new and test_df_processed\nprint(\"\\nShape of X_new:\", X_new.shape)\nprint(\"Shape of test_df_processed:\", test_df_processed.shape)\n","metadata":{"id":"3a515a71","executionInfo":{"status":"ok","timestamp":1764223679576,"user_tz":-330,"elapsed":89,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"b92f6558-f947-4f8a-a477-055f33ea493b"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nimport pandas as pd\n\n# 1. Re-initialize X_new and test_df_processed\nX_new = X.copy()\ntest_df_processed = test_df.copy()\n\n# 2. Extract unique image_id, State, and Species from train_df\nimage_info = train_df[['image_id', 'State', 'Species', 'Sampling_Date']].drop_duplicates()\n\n# 3. Merge image_info into test_df_processed\ntest_df_processed = pd.merge(test_df_processed, image_info[['image_id', 'State', 'Species', 'Sampling_Date']], on='image_id', how='left')\n\n# 4. Calculate the mode of the 'State' column from train_df\nmode_state_train = train_df['State'].mode()[0]\n\n# 5. Calculate the mode of the 'Species' column from train_df\nmode_species_train = train_df['Species'].mode()[0]\n\n# 6. Impute any missing values (NaN) in the 'State' column of test_df_processed\ntest_df_processed['State'] = test_df_processed['State'].fillna(mode_state_train)\n\n# 7. Impute any missing values (NaN) in the 'Species' column of test_df_processed\ntest_df_processed['Species'] = test_df_processed['Species'].fillna(mode_species_train)\n\n# 8. Initialize and apply LabelEncoder for 'State' column\nle_state = LabelEncoder()\nX_new['encoded_State'] = le_state.fit_transform(train_df['State'])\ntest_df_processed['encoded_State'] = le_state.transform(test_df_processed['State'])\n\n# 9. Initialize and apply LabelEncoder for 'Species' column\nle_species = LabelEncoder()\nX_new['encoded_Species'] = le_species.fit_transform(train_df['Species'])\ntest_df_processed['encoded_Species'] = le_species.transform(test_df_processed['Species'])\n\n# 10. Convert the 'Sampling_Date' column in train_df to datetime objects\ntrain_df['Sampling_Date'] = pd.to_datetime(train_df['Sampling_Date'])\n\n# 11. Extract date features for X_new\nX_new['year'] = train_df['Sampling_Date'].dt.year\nX_new['month'] = train_df['Sampling_Date'].dt.month\nX_new['day'] = train_df['Sampling_Date'].dt.day\nX_new['dayofweek'] = train_df['Sampling_Date'].dt.dayofweek\nX_new['dayofyear'] = train_df['Sampling_Date'].dt.dayofyear\n\n# 12. Create a DataFrame image_sampling_date_map containing unique image_id and Sampling_Date from train_df\n# This step was implicitly handled by including 'Sampling_Date' in image_info and merging in step 3.\n\n# 13. Merge image_sampling_date_map into test_df_processed\n# Done in step 3.\n\n# 14. Calculate the mode of the 'Sampling_Date' column from train_df\nmode_sampling_date_train = train_df['Sampling_Date'].mode()[0]\n\n# 15. Impute any missing values (NaN) in the 'Sampling_Date' column of test_df_processed\ntest_df_processed['Sampling_Date'] = test_df_processed['Sampling_Date'].fillna(mode_sampling_date_train)\n\n# 16. Ensure the 'Sampling_Date' column in test_df_processed is of datetime type\ntest_df_processed['Sampling_Date'] = pd.to_datetime(test_df_processed['Sampling_Date'])\n\n# 17. Extract date features for test_df_processed\ntest_df_processed['year'] = test_df_processed['Sampling_Date'].dt.year\ntest_df_processed['month'] = test_df_processed['Sampling_Date'].dt.month\ntest_df_processed['day'] = test_df_processed['Sampling_Date'].dt.day\ntest_df_processed['dayofweek'] = test_df_processed['Sampling_Date'].dt.dayofweek\ntest_df_processed['dayofyear'] = test_df_processed['Sampling_Date'].dt.dayofyear\n\n# 18. Define a list final_features\nfinal_features = ['Pre_GSHH_NDVI', 'Height_Ave_cm', 'encoded_State', 'encoded_Species', 'year', 'month', 'day', 'dayofweek', 'dayofyear']\n\n# 19. Update X_new to include only the columns specified in final_features\nX_new = X_new[final_features]\n\n# 20. Update test_df_processed to include only the columns specified in final_features\ntest_df_processed = test_df_processed[final_features]\n\n# 21. Display the first 5 rows of X_new and test_df_processed\nprint(\"First 5 rows of X_new with engineered features:\")\ndisplay(X_new.head())\n\nprint(\"\\nFirst 5 rows of test_df_processed with engineered features:\")\ndisplay(test_df_processed.head())\n\n# 22. Print the shapes of X_new and test_df_processed\nprint(\"\\nShape of X_new:\", X_new.shape)\nprint(\"Shape of test_df_processed:\", test_df_processed.shape)","metadata":{"id":"b7cdc668","executionInfo":{"status":"ok","timestamp":1764223709905,"user_tz":-330,"elapsed":54,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"67537767-fad8-4bec-fb71-7915dac2154e"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_new['NDVI_Height_Interaction'] = X_new['Pre_GSHH_NDVI'] * X_new['Height_Ave_cm']\ntest_df_processed['NDVI_Height_Interaction'] = test_df_processed['Pre_GSHH_NDVI'] * test_df_processed['Height_Ave_cm']\n\nprint(\"First 5 rows of X_new with new interaction feature:\")\ndisplay(X_new.head())\n\nprint(\"\\nFirst 5 rows of test_df_processed with new interaction feature:\")\ndisplay(test_df_processed.head())\n\nprint(\"\\nShape of X_new:\", X_new.shape)\nprint(\"Shape of test_df_processed:\", test_df_processed.shape)","metadata":{"id":"ae8b06bf","executionInfo":{"status":"ok","timestamp":1764223731467,"user_tz":-330,"elapsed":128,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"7f27ea9f-1762-40bd-b6ff-05b4957ab9e9"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import PolynomialFeatures\n\n# 1. Define the numerical features for polynomial transformation\nnumerical_features_poly = ['Pre_GSHH_NDVI', 'Height_Ave_cm']\n\n# 2. Initialize PolynomialFeatures with degree=2 and no bias term\npoly = PolynomialFeatures(degree=2, include_bias=False)\n\n# 3. Fit and transform the numerical features for X_new\npoly_features_X_new = poly.fit_transform(X_new[numerical_features_poly])\n\n# 4. Transform the numerical features for test_df_processed\npoly_features_test_df = poly.transform(test_df_processed[numerical_features_poly])\n\n# 5. Get the new feature names\npoly_feature_names = poly.get_feature_names_out(numerical_features_poly)\n\n# 6. Create new DataFrames for the polynomial features\nX_new_poly_df = pd.DataFrame(poly_features_X_new, columns=poly_feature_names, index=X_new.index)\ntest_df_processed_poly_df = pd.DataFrame(poly_features_test_df, columns=poly_feature_names, index=test_df_processed.index)\n\n# 7. Concatenate the new polynomial features with the existing DataFrames\nX_new = pd.concat([X_new, X_new_poly_df], axis=1)\ntest_df_processed = pd.concat([test_df_processed, test_df_processed_poly_df], axis=1)\n\n# 8. Display the first 5 rows and shape of the updated DataFrames\nprint(\"First 5 rows of X_new with polynomial features:\")\ndisplay(X_new.head())\n\nprint(\"\\nFirst 5 rows of test_df_processed with polynomial features:\")\ndisplay(test_df_processed.head())\n\nprint(\"\\nShape of X_new:\", X_new.shape)\nprint(\"Shape of test_df_processed:\", test_df_processed.shape)","metadata":{"id":"86795d47","executionInfo":{"status":"ok","timestamp":1764223912664,"user_tz":-330,"elapsed":205,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"d147ef3f-84ec-4977-bad5-7c0a5bff4d1a"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\n\n# 1. Define numerical features to aggregate\nnumerical_features_to_aggregate = ['Pre_GSHH_NDVI', 'Height_Ave_cm']\n\n# 2. Define categorical features to group by\ncategorical_groups = ['encoded_State', 'encoded_Species']\n\n# 3. Create a list of aggregation functions\nagg_functions = ['mean', 'median', 'std']\n\n# 4. For X_new:\n# a. Group X_new by categorical_groups and calculate aggregated statistics\nagg_X_new_df = X_new.groupby(categorical_groups)[numerical_features_to_aggregate].agg(agg_functions)\n\n# b. Flatten the multi-level column index into single-level strings\nagg_X_new_df.columns = [f\"{col[0]}_{col[1]}\" for col in agg_X_new_df.columns.values]\n\n# c. Reset the index to turn categorical_groups back into columns\nagg_X_new_df = agg_X_new_df.reset_index()\n\n# d. Rename the flattened aggregated columns to add '_by_State_Species' suffix\nnew_col_names_map = {}\nfor col in agg_X_new_df.columns:\n    if col not in categorical_groups: # Only rename the aggregated features\n        new_col_names_map[col] = f\"{col}_by_State_Species\"\nagg_X_new_df = agg_X_new_df.rename(columns=new_col_names_map)\n\n# e. Merge these new aggregated features back into X_new\nX_new = pd.merge(X_new, agg_X_new_df, on=categorical_groups, how='left')\n\n# 5. For test_df_processed:\n# a. Group test_df_processed by categorical_groups and calculate aggregated statistics\nagg_test_df_df = test_df_processed.groupby(categorical_groups)[numerical_features_to_aggregate].agg(agg_functions)\n\n# b. Flatten the multi-level column index into single-level strings\nagg_test_df_df.columns = [f\"{col[0]}_{col[1]}\" for col in agg_test_df_df.columns.values]\n\n# c. Reset the index to turn categorical_groups back into columns\nagg_test_df_df = agg_test_df_df.reset_index()\n\n# d. Rename the flattened aggregated columns to add '_by_State_Species' suffix\nnew_col_names_map_test = {}\nfor col in agg_test_df_df.columns:\n    if col not in categorical_groups: # Only rename the aggregated features\n        new_col_names_map_test[col] = f\"{col}_by_State_Species\"\nagg_test_df_df = agg_test_df_df.rename(columns=new_col_names_map_test)\n\n# e. Merge these new aggregated features back into test_df_processed\ntest_df_processed = pd.merge(test_df_processed, agg_test_df_df, on=categorical_groups, how='left')\n\n# 6. Display the first 5 rows of X_new and test_df_processed\nprint(\"First 5 rows of X_new with aggregated features:\")\ndisplay(X_new.head())\n\nprint(\"\\nFirst 5 rows of test_df_processed with aggregated features:\")\ndisplay(test_df_processed.head())\n\n# 7. Print the shape of X_new and test_df_processed\nprint(\"\\nShape of X_new:\", X_new.shape)\nprint(\"Shape of test_df_processed:\", test_df_processed.shape)","metadata":{"id":"58f07169","executionInfo":{"status":"ok","timestamp":1764224074698,"user_tz":-330,"elapsed":206,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"b8941832-92c2-4c70-b410-b4fb6458abe2"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\n# 1. Redo the train-test split using the fully engineered X_new and y\nX_train_new, X_val_new, y_train_new, y_val_new = train_test_split(X_new, y, test_size=0.2, random_state=42)\n\n# Remove duplicate columns from X_train_new, X_val_new, and test_df_processed\nX_train_new = X_train_new.loc[:,~X_train_new.columns.duplicated()]\nX_val_new = X_val_new.loc[:,~X_val_new.columns.duplicated()]\ntest_df_processed = test_df_processed.loc[:,~test_df_processed.columns.duplicated()]\n\n# 2. Identify all numerical features in the updated X_new\n# Original numerical features\noriginal_numerical_features = ['Pre_GSHH_NDVI', 'Height_Ave_cm']\n\n# Engineered date features\ndate_features = ['year', 'month', 'day', 'dayofweek', 'dayofyear']\n\n# Interaction feature\ninteraction_feature = ['NDVI_Height_Interaction']\n\n# Polynomial features (assuming degree=2 from previous step)\n# Filter poly_feature_names to only include the actual polynomial terms, excluding original features.\n# We will reconstruct this list based on the unique columns after deduplication.\npolynomial_features = [col for col in X_train_new.columns if \"^\" in col or \" \" in col]\n\n# Aggregated features\naggregated_features = [col for col in X_train_new.columns if col.endswith('_by_State_Species')]\n\n# Combine all numerical features into a single list\nall_numerical_features = original_numerical_features + date_features + interaction_feature + polynomial_features + aggregated_features\n\n# Ensure no duplicates and maintain order (though order is not strictly necessary for scaling)\nall_numerical_features = list(dict.fromkeys(all_numerical_features)) # Remove duplicates while preserving order\n\n# Filter to ensure only numerical columns are in the list for scaling\n# Some polynomial features might have the same name as original numerical if degree=1 or include_bias=False\n# We check if they actually exist in the current X_new columns\nall_numerical_features = [f for f in all_numerical_features if f in X_train_new.columns]\n\n# 3. Fit the previously initialized StandardScaler (named scaler) on the identified all_numerical_features of X_train_new\nscaler.fit(X_train_new[all_numerical_features])\n\n# 4. Transform the all_numerical_features in X_train_new, X_val_new, and test_df_processed\nX_train_new[all_numerical_features] = scaler.transform(X_train_new[all_numerical_features])\nX_val_new[all_numerical_features] = scaler.transform(X_val_new[all_numerical_features])\ntest_df_processed[all_numerical_features] = scaler.transform(test_df_processed[all_numerical_features])\n\n# 5. Display the first 5 rows and shapes\nprint(\"Scaled X_train_new (first 5 rows):\")\ndisplay(X_train_new.head())\n\nprint(\"\\nScaled X_val_new (first 5 rows):\")\ndisplay(X_val_new.head())\n\nprint(\"\\nScaled test_df_processed (first 5 rows):\")\ndisplay(test_df_processed.head())\n\nprint(\"\\nShape of X_train_new:\", X_train_new.shape)\nprint(\"Shape of X_val_new:\", X_val_new.shape)\nprint(\"Shape of test_df_processed:\", test_df_processed.shape)","metadata":{"id":"fea90d56","executionInfo":{"status":"ok","timestamp":1764224623558,"user_tz":-330,"elapsed":225,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"657ca6e9-6a6a-4a9b-bb81-ecd4e3a031b2"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\nr2_values = {\n    'Model': [\n        'RandomForestRegressor (original features)',\n        'RandomForestRegressor (engineered features)',\n        'XGBoostRegressor (engineered features)',\n        'GradientBoostingRegressor (engineered features)',\n        'GradientBoostingRegressor (tuned)',\n        'LGBMRegressor',\n        'CatBoostRegressor',\n        'Ensemble (Average)'\n    ],\n    'R-squared': [\n        r2,\n        r2_new_features,\n        r2_xgb,\n        r2_gbr,\n        r2_gbr_tuned,\n        r2_lgbm,\n        r2_catboost,\n        r2_ensemble_avg\n    ]\n}\n\nr2_df_summary = pd.DataFrame(r2_values)\ndisplay(r2_df_summary.round(4))","metadata":{"id":"3fe63345","executionInfo":{"status":"ok","timestamp":1764226809139,"user_tz":-330,"elapsed":113,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"8ca0c4b7-257d-40cc-a696-42979ba245d1"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\nr2_values = {\n    'Model': [\n        'RandomForestRegressor (original features)',\n        'RandomForestRegressor (engineered features)',\n        'XGBoostRegressor (engineered features)',\n        'GradientBoostingRegressor (engineered features)'\n    ],\n    'R-squared': [\n        r2,\n        r2_new_features,\n        r2_xgb,\n        r2_gbr\n    ]\n}\n\nr2_df = pd.DataFrame(r2_values)\ndisplay(r2_df.round(4))","metadata":{"id":"bcb45809","executionInfo":{"status":"ok","timestamp":1764226643223,"user_tz":-330,"elapsed":51,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"9a883ede-d045-4658-b5b6-f0e06829b009"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\nr2_values = {\n    'Model': [\n        'RandomForestRegressor (original features)',\n        'RandomForestRegressor (engineered features)',\n        'XGBoostRegressor (engineered features)',\n        'GradientBoostingRegressor (engineered features)'\n    ],\n    'R-squared': [\n        r2,\n        r2_new_features,\n        r2_xgb,\n        r2_gbr\n    ]\n}\n\nr2_df = pd.DataFrame(r2_values)\ndisplay(r2_df.round(4))","metadata":{"id":"a7c5331b","executionInfo":{"status":"ok","timestamp":1764225624130,"user_tz":-330,"elapsed":96,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"259d08a7-89a4-4a4e-eb9c-65723fe4f467"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"RandomForestRegressor (original features) R-squared: {r2:.4f}\")\nprint(f\"RandomForestRegressor (engineered features) R-squared: {r2_new_features:.4f}\")\nprint(f\"XGBoostRegressor (engineered features) R-squared: {r2_xgb:.4f}\")\nprint(f\"GradientBoostingRegressor (engineered features) R-squared: {r2_gbr:.4f}\")","metadata":{"id":"13810ecb","executionInfo":{"status":"ok","timestamp":1764225503397,"user_tz":-330,"elapsed":26,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"9a616552-4bc6-414b-82ee-adf1e1c6820a"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\nplt.figure(figsize=(10, 6))\nsns.scatterplot(x=y_val_new, y=y_pred_gbr)\nplt.title('Actual vs. Predicted Values (GradientBoostingRegressor)')\nplt.xlabel('Actual Values')\nplt.ylabel('Predicted Values')\nplt.plot([y_val_new.min(), y_val_new.max()], [y_val_new.min(), y_val_new.max()], color='red', linestyle='--') # Ideal prediction line\nplt.show()","metadata":{"id":"fada28bf","executionInfo":{"status":"ok","timestamp":1764225337823,"user_tz":-330,"elapsed":686,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"26c3fb9e-cd07-4935-f253-a36ae8c903ff"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred_gbr_test = gbr_model.predict(test_df_processed)\n\nprint(\"Predictions on the test set using GradientBoostingRegressor:\")\nprint(y_pred_gbr_test)","metadata":{"id":"1c3329a6","executionInfo":{"status":"ok","timestamp":1764225174669,"user_tz":-330,"elapsed":66,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"bd1a4199-adea-414e-ca2f-afaf6af4c790"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.metrics import mean_absolute_error, r2_score\n\n# Initialize and train the GradientBoostingRegressor model\ngbr_model = GradientBoostingRegressor(random_state=42)\ngbr_model.fit(X_train_new, y_train_new)\n\n# Make predictions on the validation set\ny_pred_gbr = gbr_model.predict(X_val_new)\n\n# Evaluate the model\nmae_gbr = mean_absolute_error(y_val_new, y_pred_gbr)\nr2_gbr = r2_score(y_val_new, y_pred_gbr)\n\nprint(f\"GradientBoostingRegressor - Mean Absolute Error on validation set: {mae_gbr:.4f}\")\nprint(f\"GradientBoostingRegressor - R-squared on validation set: {r2_gbr:.4f}\")","metadata":{"id":"0bafcee0","executionInfo":{"status":"ok","timestamp":1764225015589,"user_tz":-330,"elapsed":1001,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"dad75c40-8abc-492c-d464-980092f64487"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\nplt.figure(figsize=(10, 6))\nsns.scatterplot(x=y_val_new, y=y_pred_new_features)\nplt.title('Actual vs. Predicted Values (RandomForestRegressor with Engineered Features)')\nplt.xlabel('Actual Values')\nplt.ylabel('Predicted Values')\nplt.plot([y_val_new.min(), y_val_new.max()], [y_val_new.min(), y_val_new.max()], color='red', linestyle='--') # Ideal prediction line\nplt.show()","metadata":{"id":"d601c59b","executionInfo":{"status":"ok","timestamp":1764224834488,"user_tz":-330,"elapsed":491,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"31308a6f-2531-4ab7-d981-bc9a29e5c38c"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\nplt.figure(figsize=(10, 6))\nsns.scatterplot(x=y_val_new, y=y_pred_xgb)\nplt.title('Actual vs. Predicted Values (XGBoost)')\nplt.xlabel('Actual Values')\nplt.ylabel('Predicted Values')\nplt.plot([y_val_new.min(), y_val_new.max()], [y_val_new.min(), y_val_new.max()], color='red', linestyle='--') # Ideal prediction line\nplt.show()","metadata":{"id":"a07b55a3","executionInfo":{"status":"ok","timestamp":1764224770648,"user_tz":-330,"elapsed":675,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"36f5d90b-6e21-4159-b705-aea88fbd2e9c"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import r2_score\n\n# Initialize the GradientBoostingRegressor model\ngbr = GradientBoostingRegressor(random_state=42)\n\n# Define the parameter grid for hyperparameter tuning\nparam_grid_gbr = {\n    'n_estimators': [100, 200, 300],\n    'learning_rate': [0.01, 0.05, 0.1],\n    'max_depth': [3, 5, 7],\n    'min_samples_split': [2, 5],\n    'min_samples_leaf': [1, 2]\n}\n\n# Instantiate GridSearchCV\ngrid_search_gbr = GridSearchCV(estimator=gbr, param_grid=param_grid_gbr, cv=3, scoring='r2', n_jobs=-1, verbose=2)\n\n# Fit GridSearchCV to the training data\ngrid_search_gbr.fit(X_train_new, y_train_new)\n\n# Print the best parameters found\nprint(\"Best parameters found by GridSearchCV:\", grid_search_gbr.best_params_)\n\n# Get the best estimator\nbest_gbr_model = grid_search_gbr.best_estimator_\n\n# Make predictions on the validation set using the best estimator\ny_pred_gbr_tuned = best_gbr_model.predict(X_val_new)\n\n# Evaluate the R-squared score\nr2_gbr_tuned = r2_score(y_val_new, y_pred_gbr_tuned)\nprint(f\"GradientBoostingRegressor (tuned) - R-squared on validation set: {r2_gbr_tuned:.4f}\")","metadata":{"id":"12f6d7d0","executionInfo":{"status":"ok","timestamp":1764226140693,"user_tz":-330,"elapsed":230002,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"9675da28-ea86-4915-abd0-9f076e114a67"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from lightgbm import LGBMRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.metrics import mean_absolute_error, r2_score\n\n# 1. Initialize LGBMRegressor model\nlgbm_model = LGBMRegressor(random_state=42, verbose=-1) # verbose=-1 to suppress verbose output during training\n\n# 2. Initialize CatBoostRegressor model\ncatboost_model = CatBoostRegressor(random_state=42, verbose=0) # verbose=0 to suppress verbose output during training\n\n# 3. Train LGBMRegressor model\nlgbm_model.fit(X_train_new, y_train_new)\n\n# 4. Train CatBoostRegressor model\ncatboost_model.fit(X_train_new, y_train_new)\n\n# 5. Make predictions on the validation set for LGBMRegressor\ny_pred_lgbm = lgbm_model.predict(X_val_new)\n\n# 6. Make predictions on the validation set for CatBoostRegressor\ny_pred_catboost = catboost_model.predict(X_val_new)\n\n# 7. Evaluate LGBMRegressor model\nmae_lgbm = mean_absolute_error(y_val_new, y_pred_lgbm)\nr2_lgbm = r2_score(y_val_new, y_pred_lgbm)\n\nprint(f\"LGBMRegressor - Mean Absolute Error on validation set: {mae_lgbm:.4f}\")\nprint(f\"LGBMRegressor - R-squared on validation set: {r2_lgbm:.4f}\")\n\n# 8. Evaluate CatBoostRegressor model\nmae_catboost = mean_absolute_error(y_val_new, y_pred_catboost)\nr2_catboost = r2_score(y_val_new, y_pred_catboost)\n\nprint(f\"\\nCatBoostRegressor - Mean Absolute Error on validation set: {mae_catboost:.4f}\")\nprint(f\"CatBoostRegressor - R-squared on validation set: {r2_catboost:.4f}\")","metadata":{"id":"e2e7b000","executionInfo":{"status":"ok","timestamp":1764227861208,"user_tz":-330,"elapsed":1921,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"f2af02de-b0a6-4aa0-8e10-913da459e891"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Reasoning**:\nThe previous cell failed because the `catboost` library was not found. To resolve this, I need to install `catboost` using `!pip install` before importing it.\n\n","metadata":{"id":"9a06a46e"}},{"cell_type":"code","source":"pip install catboost","metadata":{"id":"ba5fc6d9","executionInfo":{"status":"ok","timestamp":1764226211988,"user_tz":-330,"elapsed":21325,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"754ba796-c526-4ed3-9a43-5f66f9aa46d8"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Reasoning**:\nNow that `catboost` is successfully installed, I will re-execute the code block to train and evaluate the LightGBM and CatBoost models, as per the subtask instructions.\n\n","metadata":{"id":"7d9ca740"}},{"cell_type":"code","source":"from lightgbm import LGBMRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.metrics import mean_absolute_error, r2_score\n\n# 1. Initialize LGBMRegressor model\nlgbm_model = LGBMRegressor(random_state=42, verbose=-1) # verbose=-1 to suppress verbose output during training\n\n# 2. Initialize CatBoostRegressor model\ncatboost_model = CatBoostRegressor(random_state=42, verbose=0) # verbose=0 to suppress verbose output during training\n\n# 3. Train LGBMRegressor model\nlgbm_model.fit(X_train_new, y_train_new)\n\n# 4. Train CatBoostRegressor model\ncatboost_model.fit(X_train_new, y_train_new)\n\n# 5. Make predictions on the validation set for LGBMRegressor\ny_pred_lgbm = lgbm_model.predict(X_val_new)\n\n# 6. Make predictions on the validation set for CatBoostRegressor\ny_pred_catboost = catboost_model.predict(X_val_new)\n\n# 7. Evaluate LGBMRegressor model\nmae_lgbm = mean_absolute_error(y_val_new, y_pred_lgbm)\nr2_lgbm = r2_score(y_val_new, y_pred_lgbm)\n\nprint(f\"LGBMRegressor - Mean Absolute Error on validation set: {mae_lgbm:.4f}\")\nprint(f\"LGBMRegressor - R-squared on validation set: {r2_lgbm:.4f}\")\n\n# 8. Evaluate CatBoostRegressor model\nmae_catboost = mean_absolute_error(y_val_new, y_pred_catboost)\nr2_catboost = r2_score(y_val_new, y_pred_catboost)\n\nprint(f\"\\nCatBoostRegressor - Mean Absolute Error on validation set: {mae_catboost:.4f}\")\nprint(f\"CatBoostRegressor - R-squared on validation set: {r2_catboost:.4f}\")","metadata":{"id":"a63ebb0a","executionInfo":{"status":"ok","timestamp":1764226234418,"user_tz":-330,"elapsed":5725,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"79e236a6-f1d1-4bf4-eba7-7e23364f9c2e"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nimport pandas as pd\n\n# 1. Extract feature importances from the best_gbr_model\nfeature_importances = best_gbr_model.feature_importances_\n\n# Get feature names from X_train_new\nfeature_names = X_train_new.columns\n\n# 2. Create a Pandas DataFrame to store the feature names and their corresponding importances\nimportance_df = pd.DataFrame({\n    'Feature': feature_names,\n    'Importance': feature_importances\n})\n\n# 3. Sort the features by importance in descending order\nimportance_df = importance_df.sort_values(by='Importance', ascending=False)\n\n# 4. Visualize the top N most important features (e.g., top 15)\nplt.figure(figsize=(12, 8))\nsns.barplot(x='Importance', y='Feature', data=importance_df.head(15))\nplt.title('Top 15 Feature Importances from Tuned GradientBoostingRegressor')\nplt.xlabel('Importance Score')\nplt.ylabel('Feature Name')\nplt.tight_layout()\n\n# 5. Display the plot\nplt.show()\n\nprint(\"Top 15 features and their importances:\")\ndisplay(importance_df.head(15))","metadata":{"id":"20a8f609","executionInfo":{"status":"ok","timestamp":1764226274809,"user_tz":-330,"elapsed":998,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"4e4f0584-f1e4-4184-e120-82670e828a2e"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import r2_score\nimport numpy as np\n\n# 1. Make predictions on the validation set using the trained models\ny_pred_gbr_tuned = best_gbr_model.predict(X_val_new)\ny_pred_lgbm = lgbm_model.predict(X_val_new)\ny_pred_catboost = catboost_model.predict(X_val_new)\ny_pred_xgb = xgb_model.predict(X_val_new)\ny_pred_rf_engineered = model_new_features.predict(X_val_new)\n\n# 2. Calculate the average of these individual model predictions\ny_pred_ensemble_avg = (y_pred_gbr_tuned + y_pred_lgbm + y_pred_catboost + y_pred_xgb + y_pred_rf_engineered) / 5\n\n# 3. Evaluate the R-squared score of this ensemble prediction on the validation set\nr2_ensemble_avg = r2_score(y_val_new, y_pred_ensemble_avg)\n\n# 4. Print the R-squared score of the ensemble model\nprint(f\"Ensemble (Average) - R-squared on validation set: {r2_ensemble_avg:.4f}\")","metadata":{"id":"1b1acafb","executionInfo":{"status":"ok","timestamp":1764227883556,"user_tz":-330,"elapsed":158,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"ffa3c131-adbb-4776-994b-0bdad6409d5e"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import r2_score\nimport numpy as np\nfrom sklearn.ensemble import RandomForestRegressor\nfrom xgboost import XGBRegressor\n\n# Re-initialize and train xgb_model with the fully engineered features\nxgb_model = XGBRegressor(random_state=42)\nxgb_model.fit(X_train_new, y_train_new)\n\n# Re-initialize and train model_new_features (RandomForestRegressor) with the fully engineered features\nmodel_new_features = RandomForestRegressor(random_state=42)\nmodel_new_features.fit(X_train_new, y_train_new)\n\n# 1. Make predictions on the validation set using the trained models\ny_pred_gbr_tuned = best_gbr_model.predict(X_val_new)\ny_pred_lgbm = lgbm_model.predict(X_val_new)\ny_pred_catboost = catboost_model.predict(X_val_new)\ny_pred_xgb = xgb_model.predict(X_val_new)\ny_pred_rf_engineered = model_new_features.predict(X_val_new)\n\n# 2. Calculate the average of these individual model predictions\ny_pred_ensemble_avg = (y_pred_gbr_tuned + y_pred_lgbm + y_pred_catboost + y_pred_xgb + y_pred_rf_engineered) / 5\n\n# 3. Evaluate the R-squared score of this ensemble prediction on the validation set\nr2_ensemble_avg = r2_score(y_val_new, y_pred_ensemble_avg)\n\n# 4. Print the R-squared score of the ensemble model\nprint(f\"Ensemble (Average) - R-squared on validation set: {r2_ensemble_avg:.4f}\")","metadata":{"id":"35155937","executionInfo":{"status":"ok","timestamp":1764226375993,"user_tz":-330,"elapsed":2128,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"1bcbb0f5-a09a-4a00-f59f-507cfad40737"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nimport numpy as np\n\n# 1. Calculate the residuals\nresiduals = y_val_new - y_pred_gbr_tuned\n\n# 2. Create a scatter plot with predicted values on the x-axis and residuals on the y-axis\nplt.figure(figsize=(10, 6))\nsns.scatterplot(x=y_pred_gbr_tuned, y=residuals, alpha=0.6)\n\n# 3. Add a horizontal line at y=0\nplt.axhline(y=0, color='r', linestyle='--')\n\n# 4. Add appropriate labels for the x and y axes\nplt.xlabel('Predicted Values')\nplt.ylabel('Residuals')\n\n# 5. Add a title to the plot\nplt.title('Residual Plot of Tuned GradientBoostingRegressor')\n\n# 6. Display the plot\nplt.show()","metadata":{"id":"097a737c","executionInfo":{"status":"ok","timestamp":1764226442610,"user_tz":-330,"elapsed":439,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"5149b8f7-16b7-496d-ae8a-b0f068b621ef"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"id":"yVDOGTzpc9Nr"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred_gbr_test = best_gbr_model.predict(test_df_processed)\ny_pred_lgbm_test = lgbm_model.predict(test_df_processed)\ny_pred_catboost_test = catboost_model.predict(test_df_processed)\ny_pred_xgb_test = xgb_model.predict(test_df_processed)\ny_pred_rf_test = model_new_features.predict(test_df_processed)\n\nprint(\"Predictions generated for all models on test_df_processed.\")","metadata":{"id":"636e61b1","executionInfo":{"status":"ok","timestamp":1764232643388,"user_tz":-330,"elapsed":205,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"3d5872cf-9bc0-4da8-8e39-1189794d534f"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\n\n# Calculate the average of the individual model predictions\ny_pred_ensemble_avg_test = (y_pred_gbr_test + y_pred_lgbm_test + y_pred_catboost_test + y_pred_xgb_test + y_pred_rf_test) / 5\n\nprint(\"Ensemble predictions created by averaging individual model predictions.\")\nprint(\"First 5 ensemble predictions:\")\nprint(y_pred_ensemble_avg_test[:5])","metadata":{"id":"594539ea","executionInfo":{"status":"ok","timestamp":1764232661522,"user_tz":-330,"elapsed":7,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"05759106-66f5-4ed3-ee25-7bb1effe2011"},"outputs":[],"execution_count":null},{"cell_type":"code","source":"results_df = pd.DataFrame({\n    'image_id': test_df['image_id'],\n    'actual_target': test_level_df['target'], # Assuming test_level_df has the actual target values\n    'gbr_prediction': y_pred_gbr_test,\n    'lgbm_prediction': y_pred_lgbm_test,\n    'catboost_prediction': y_pred_catboost_test,\n    'xgb_prediction': y_pred_xgb_test,\n    'rf_prediction': y_pred_rf_test\n})\n\nprint(\"Prediction results for all models against actual targets:\")\ndisplay(results_df.head())","metadata":{"id":"00e3085b","executionInfo":{"status":"ok","timestamp":1764232914367,"user_tz":-330,"elapsed":230,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"8b2f04e4-6d2b-4db5-c181-96a32eea83bc"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Task\nExecuting the plan.\n\n## Re-training XGBoost model\n[(adfd6378) Re-initialize and train `xgb_model`.](https://colab.research.google.com/drive/1h-44xV3V3_d77x2M0Xk9D9m6U6Vv1pWw#scrollTo=adfd6378)\n```python\nfrom xgboost import XGBRegressor\nfrom sklearn.metrics import mean_absolute_error, r2_score\n\n# Initialize and train the XGBoost Regressor model with engineered features\nxgb_model = XGBRegressor(random_state=42)\nxgb_model.fit(X_train_new, y_train_new)\n\n# Make predictions on the validation set\ny_pred_xgb = xgb_model.predict(X_val_new)\n\n# Evaluate the model\nmae_xgb = mean_absolute_error(y_val_new, y_pred_xgb)\nr2_xgb = r2_score(y_val_new, y_pred_xgb)\n\nprint(f\"XGBoost - Mean Absolute Error on validation set: {mae_xgb:.4f}\")\nprint(f\"XGBoost - R-squared on validation set: {r2_xgb:.4f}\")\n```\n## Re-training RandomForest model\n[(b84f6d68) Re-initialize and train `model_new_features` (RandomForestRegressor).](https://colab.research.google.com/drive/1h-44xV3V3_d77x2M0Xk9D9m6U6Vv1pWw#scrollTo=b84f6d68)\n```python\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.metrics import mean_absolute_error, r2_score\n\n# Initialize and train the RandomForestRegressor model with new features\nmodel_new_features = RandomForestRegressor(random_state=42)\nmodel_new_features.fit(X_train_new, y_train_new)\n\n# Make predictions on the validation set with new features\ny_pred_new_features = model_new_features.predict(X_val_new)\n\n# Evaluate the model with new features\nmae_new_features = mean_absolute_error(y_val_new, y_pred_new_features)\nr2_new_features = r2_score(y_val_new, y_pred_new_features)\n\nprint(f\"Mean Absolute Error on validation set (with new features): {mae_new_features:.4f}\")\nprint(f\"R-squared on validation set (with new features): {r2_new_features:.4f}\")\n```\n## Re-training GradientBoosting model (tuned)\n[(12f6d7d0) Re-train the tuned `best_gbr_model`.](https://colab.research.google.com/drive/1h-44xV3V3_d77x2M0Xk9D9m6U6Vv1pWw#scrollTo=12f6d7d0)\n```python\nfrom sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import r2_score\n\n# Initialize the GradientBoostingRegressor model\ngbr = GradientBoostingRegressor(random_state=42)\n\n# Define the parameter grid for hyperparameter tuning\nparam_grid_gbr = {\n    'n_estimators': [100, 200, 300],\n    'learning_rate': [0.01, 0.05, 0.1],\n    'max_depth': [3, 5, 7],\n    'min_samples_split': [2, 5],\n    'min_samples_leaf': [1, 2]\n}\n\n# Instantiate GridSearchCV\ngrid_search_gbr = GridSearchCV(estimator=gbr, param_grid=param_grid_gbr, cv=3, scoring='r2', n_jobs=-1, verbose=2)\n\n# Fit GridSearchCV to the training data\ngrid_search_gbr.fit(X_train_new, y_train_new)\n\n# Print the best parameters found\nprint(\"Best parameters found by GridSearchCV:\", grid_search_gbr.best_params_)\n\n# Get the best estimator\nbest_gbr_model = grid_search_gbr.best_estimator_\n\n# Make predictions on the validation set using the best estimator\ny_pred_gbr_tuned = best_gbr_model.predict(X_val_new)\n\n# Evaluate the R-squared score\nr2_gbr_tuned = r2_score(y_val_new, y_pred_gbr_tuned)\nprint(f\"GradientBoostingRegressor (tuned) - R-squared on validation set: {r2_gbr_tuned:.4f}\")\n```\n## Re-training LightGBM and CatBoost models\n[(a63ebb0a) Re-train `lgbm_model` and `catboost_model`.](https://colab.research.google.com/drive/1h-44xV3V3_d77x2M0Xk9D9m6U6Vv1pWw#scrollTo=a63ebb0a)\n```python\nfrom lightgbm import LGBMRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.metrics import mean_absolute_error, r2_score\n\n# 1. Initialize LGBMRegressor model\nlgbm_model = LGBMRegressor(random_state=42, verbose=-1) # verbose=-1 to suppress verbose output during training\n\n# 2. Initialize CatBoostRegressor model\ncatboost_model = CatBoostRegressor(random_state=42, verbose=0) # verbose=0 to suppress verbose output during training\n\n# 3. Train LGBMRegressor model\nlgbm_model.fit(X_train_new, y_train_new)\n\n# 4. Train CatBoostRegressor model\ncatboost_model.fit(X_train_new, y_train_new)\n\n# 5. Make predictions on the validation set for LGBMRegressor\ny_pred_lgbm = lgbm_model.predict(X_val_new)\n\n# 6. Make predictions on the validation set for CatBoostRegressor\ny_pred_catboost = catboost_model.predict(X_val_new)\n\n# 7. Evaluate LGBMRegressor model\nmae_lgbm = mean_absolute_error(y_val_new, y_pred_lgbm)\nr2_lgbm = r2_score(y_val_new, y_pred_lgbm)\n\nprint(f\"LGBMRegressor - Mean Absolute Error on validation set: {mae_lgbm:.4f}\")\nprint(f\"LGBMRegressor - R-squared on validation set: {r2_lgbm:.4f}\")\n\n# 8. Evaluate CatBoostRegressor model\nmae_catboost = mean_absolute_error(y_val_new, y_pred_catboost)\nr2_catboost = r2_score(y_val_new, y_pred_catboost)\n\nprint(f\"\\nCatBoostRegressor - Mean Absolute Error on validation set: {mae_catboost:.4f}\")\nprint(f\"CatBoostRegressor - R-squared on validation set: {r2_catboost:.4f}\")\n```\n## Generating submission file\n[(2a5ce99e) Generate the `submission.csv` file using the ensemble predictions.](https://colab.research.google.com/drive/1h-44xV3V3_d77x2M0Xk9D9m6U6Vv1pWw#scrollTo=2a5ce99e)\n```python\nimport pandas as pd\nimport numpy as np\n\n# Ensure test_df is loaded\ntrain_df = pd.read_csv('/content/drive/MyDrive/Australia_BioMass/csiro-biomass/train.csv')\ntest_df = pd.read_csv('/content/drive/MyDrive/Australia_BioMass/csiro-biomass/test.csv')\ntest_level_df = pd.read_csv('/content/drive/MyDrive/Australia_BioMass/csiro-biomass/sample_submission (1).csv')\n\n# Ensure all individual model predictions are generated for test_df_processed\ny_pred_gbr_test = best_gbr_model.predict(test_df_processed)\ny_pred_lgbm_test = lgbm_model.predict(test_df_processed)\ny_pred_catboost_test = catboost_model.predict(test_df_processed)\ny_pred_xgb_test = xgb_model.predict(test_df_processed)\ny_pred_rf_test = model_new_features.predict(test_df_processed)\n\n# Calculate the average of the individual model predictions\ny_pred_ensemble_avg_test = (y_pred_gbr_test + y_pred_lgbm_test + y_pred_catboost_test + y_pred_xgb_test + y_pred_rf_test) / 5\n\nsubmission_df = pd.DataFrame({\n    'image_id': test_df['sample_id'],\n    'target': y_pred_ensemble_avg_test\n})\n\n# Ensure the target values are non-negative, as biomass cannot be negative\nsubmission_df['target'] = submission_df['target'].apply(lambda x: max(0, x))\n\nsubmission_df.to_csv('submission.csv', index=False)\n\nprint('Submission file \"submission.csv\" created successfully.')\n```\n## Downloading submission file\n[(d49cd954) Download the `submission.csv` file.](https://colab.research.google.com/drive/1h-44xV3V3_d77x2M0Xk9D9m6U6Vv1pWw#scrollTo=d49cd954)\n```python\nfrom google.colab import files\nfiles.download('submission.csv')\n```","metadata":{"id":"bafe231a"}},{"cell_type":"markdown","source":"## Re-train XGBoost model\n\n### Subtask:\nRe-initialize and train `xgb_model`.\n","metadata":{"id":"285f955b"}},{"cell_type":"markdown","source":"**Reasoning**:\nThe user wants to re-initialize and train the XGBoost model with the fully engineered features and then evaluate its performance on the validation set. This involves importing the necessary libraries, initializing the model, training it, making predictions, and calculating MAE and R-squared metrics.\n\n","metadata":{"id":"e9363f00"}},{"cell_type":"code","source":"from xgboost import XGBRegressor\nfrom sklearn.metrics import mean_absolute_error, r2_score\n\n# Initialize and train the XGBoost Regressor model with engineered features\nxgb_model = XGBRegressor(random_state=42)\nxgb_model.fit(X_train_new, y_train_new)\n\n# Make predictions on the validation set\ny_pred_xgb = xgb_model.predict(X_val_new)\n\n# Evaluate the model\nmae_xgb = mean_absolute_error(y_val_new, y_pred_xgb)\nr2_xgb = r2_score(y_val_new, y_pred_xgb)\n\nprint(f\"XGBoost - Mean Absolute Error on validation set: {mae_xgb:.4f}\")\nprint(f\"XGBoost - R-squared on validation set: {r2_xgb:.4f}\")","metadata":{"id":"959021c1","executionInfo":{"status":"error","timestamp":1765027163265,"user_tz":-330,"elapsed":3171,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"3685aaa5-a777-48a7-c5ab-d76bf53535d1"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Reasoning**:\nThe previous cell failed because `X_train_new` and `y_train_new` were not defined in the execution environment. This is due to earlier data preprocessing and splitting steps not being executed or their state being lost. To fix this, I will re-include all necessary data loading, feature engineering, and train-test splitting steps within this single code block to ensure all required variables are defined before training the XGBoost model.\n\n","metadata":{"id":"3c961b6d"}},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler, PolynomialFeatures\nfrom sklearn.model_selection import train_test_split\nfrom xgboost import XGBRegressor\nfrom sklearn.metrics import mean_absolute_error, r2_score\n\n# 1. Load Data (from cell 0wELoGYqqWJF, with corrected filenames)\ntrain_df = pd.read_csv('/content/drive/MyDrive/Australia_BioMass/csiro-biomass/train.csv')\ntest_df = pd.read_csv('/content/drive/MyDrive/Australia_BioMass/csiro-biomass/test.csv')\ntest_level_df = pd.read_csv('/content/drive/MyDrive/Australia_BioMass/csiro-biomass/sample_submission (1).csv')\n\n# 2. Define X and y (from cell e1517085)\ny = train_df['target']\nX = train_df[['Pre_GSHH_NDVI', 'Height_Ave_cm']]\n\n# 3. Feature Engineering (combining parts from b7cdc668, ae8b06bf, 86795d47, 58f07169)\n\n# Re-initialize X_new and test_df_processed for feature engineering\nX_new = X.copy()\ntest_df_processed = test_df.copy()\n\n# Extract unique image_id, State, Species, and Sampling_Date from train_df\nimage_info = train_df[['image_id', 'State', 'Species', 'Sampling_Date']].drop_duplicates()\n\n# Merge this information into test_df_processed\ntest_df_processed = pd.merge(test_df_processed, image_info[['image_id', 'State', 'Species', 'Sampling_Date']], on='image_id', how='left')\n\n# Impute NaN values in 'State' and 'Species' columns of test_df_processed with the mode from train_df\nmode_state_train = train_df['State'].mode()[0]\nmode_species_train = train_df['Species'].mode()[0]\ntest_df_processed['State'] = test_df_processed['State'].fillna(mode_state_train)\ntest_df_processed['Species'] = test_df_processed['Species'].fillna(mode_species_train)\n\n# Initialize and apply LabelEncoder for 'State' column\nle_state = LabelEncoder()\nX_new['encoded_State'] = le_state.fit_transform(train_df['State'])\ntest_df_processed['encoded_State'] = le_state.transform(test_df_processed['State'])\n\n# Initialize and apply LabelEncoder for 'Species' column\nle_species = LabelEncoder()\nX_new['encoded_Species'] = le_species.fit_transform(train_df['Species'])\ntest_df_processed['encoded_Species'] = le_species.transform(test_df_processed['Species'])\n\n# Convert the 'Sampling_Date' column in train_df and test_df_processed to datetime objects\ntrain_df['Sampling_Date'] = pd.to_datetime(train_df['Sampling_Date'])\ntest_df_processed['Sampling_Date'] = pd.to_datetime(test_df_processed['Sampling_Date'])\n\n# Extract date features for X_new (using train_df's Sampling_Date)\nX_new['year'] = train_df['Sampling_Date'].dt.year\nX_new['month'] = train_df['Sampling_Date'].dt.month\nX_new['day'] = train_df['Sampling_Date'].dt.day\nX_new['dayofweek'] = train_df['Sampling_Date'].dt.dayofweek\nX_new['dayofyear'] = train_df['Sampling_Date'].dt.dayofyear\n\n# Impute any missing values (NaN) in the 'Sampling_Date' column of test_df_processed with the mode from train_df\nmode_sampling_date_train = train_df['Sampling_Date'].mode()[0]\ntest_df_processed['Sampling_Date'] = test_df_processed['Sampling_Date'].fillna(mode_sampling_date_train)\ntest_df_processed['Sampling_Date'] = pd.to_datetime(test_df_processed['Sampling_Date']) # Re-ensure datetime after fillna\n\n# Extract date features for test_df_processed\ntest_df_processed['year'] = test_df_processed['Sampling_Date'].dt.year\ntest_df_processed['month'] = test_df_processed['Sampling_Date'].dt.month\ntest_df_processed['day'] = test_df_processed['Sampling_Date'].dt.day\ntest_df_processed['dayofweek'] = test_df_processed['Sampling_Date'].dt.dayofweek\ntest_df_processed['dayofyear'] = test_df_processed['Sampling_Date'].dt.dayofyear\n\n# Define final_features and select columns for X_new and test_df_processed\nfinal_features_base = ['Pre_GSHH_NDVI', 'Height_Ave_cm', 'encoded_State', 'encoded_Species', 'year', 'month', 'day', 'dayofweek', 'dayofyear']\nX_new = X_new[final_features_base]\ntest_df_processed = test_df_processed[final_features_base]\n\n# Interaction Feature (from cell ae8b06bf)\nX_new['NDVI_Height_Interaction'] = X_new['Pre_GSHH_NDVI'] * X_new['Height_Ave_cm']\ntest_df_processed['NDVI_Height_Interaction'] = test_df_processed['Pre_GSHH_NDVI'] * test_df_processed['Height_Ave_cm']\n\n# Polynomial Features (from cell 86795d47)\nnumerical_features_poly = ['Pre_GSHH_NDVI', 'Height_Ave_cm']\npoly = PolynomialFeatures(degree=2, include_bias=False)\npoly_features_X_new = poly.fit_transform(X_new[numerical_features_poly])\npoly_features_test_df = poly.transform(test_df_processed[numerical_features_poly])\npoly_feature_names = poly.get_feature_names_out(numerical_features_poly)\nX_new_poly_df = pd.DataFrame(poly_features_X_new, columns=poly_feature_names, index=X_new.index)\ntest_df_processed_poly_df = pd.DataFrame(poly_features_test_df, columns=poly_feature_names, index=test_df_processed.index)\nX_new = pd.concat([X_new, X_new_poly_df], axis=1)\ntest_df_processed = pd.concat([test_df_processed, test_df_processed_poly_df], axis=1)\n\n# Aggregated Features (from cell 58f07169)\nnumerical_features_to_aggregate = ['Pre_GSHH_NDVI', 'Height_Ave_cm']\ncategorical_groups = ['encoded_State', 'encoded_Species']\nagg_functions = ['mean', 'median', 'std']\n\nagg_X_new_df = X_new.groupby(categorical_groups)[numerical_features_to_aggregate].agg(agg_functions)\nagg_X_new_df.columns = [f\"{col[0]}_{col[1]}\" for col in agg_X_new_df.columns.values]\nagg_X_new_df = agg_X_new_df.reset_index()\nnew_col_names_map = {}\nfor col in agg_X_new_df.columns:\n    if col not in categorical_groups:\n        new_col_names_map[col] = f\"{col}_by_State_Species\"\nagg_X_new_df = agg_X_new_df.rename(columns=new_col_names_map)\nX_new = pd.merge(X_new, agg_X_new_df, on=categorical_groups, how='left')\n\nagg_test_df_df = test_df_processed.groupby(categorical_groups)[numerical_features_to_aggregate].agg(agg_functions)\nagg_test_df_df.columns = [f\"{col[0]}_{col[1]}\" for col in agg_test_df_df.columns.values]\nagg_test_df_df = agg_test_df_df.reset_index()\nnew_col_names_map_test = {}\nfor col in agg_test_df_df.columns:\n    if col not in categorical_groups:\n        new_col_names_map_test[col] = f\"{col}_by_State_Species\"\nagg_test_df_df = agg_test_df_df.rename(columns=new_col_names_map_test)\ntest_df_processed = pd.merge(test_df_processed, agg_test_df_df, on=categorical_groups, how='left')\n\n# 4. Train-Test Split and Scaling (from cell fea90d56)\nX_train_new, X_val_new, y_train_new, y_val_new = train_test_split(X_new, y, test_size=0.2, random_state=42)\n\n# Remove duplicate columns\nX_train_new = X_train_new.loc[:,~X_train_new.columns.duplicated()]\nX_val_new = X_val_new.loc[:,~X_val_new.columns.duplicated()]\ntest_df_processed = test_df_processed.loc[:,~test_df_processed.columns.duplicated()]\n\n# Identify all numerical features for scaling\noriginal_numerical_features = ['Pre_GSHH_NDVI', 'Height_Ave_cm']\ndate_features = ['year', 'month', 'day', 'dayofweek', 'dayofyear']\ninteraction_feature = ['NDVI_Height_Interaction']\npolynomial_features = [col for col in X_train_new.columns if \"^\" in col or \" \" in col]\naggregated_features = [col for col in X_train_new.columns if col.endswith('_by_State_Species')]\n\nall_numerical_features = original_numerical_features + date_features + interaction_feature + polynomial_features + aggregated_features\nall_numerical_features = list(dict.fromkeys(all_numerical_features))\nall_numerical_features = [f for f in all_numerical_features if f in X_train_new.columns] # Ensure features exist\n\nscaler = StandardScaler()\nscaler.fit(X_train_new[all_numerical_features])\n\nX_train_new[all_numerical_features] = scaler.transform(X_train_new[all_numerical_features])\nX_val_new[all_numerical_features] = scaler.transform(X_val_new[all_numerical_features])\ntest_df_processed[all_numerical_features] = scaler.transform(test_df_processed[all_numerical_features])\n\n# 5. Initialize and train the XGBoost Regressor model with engineered features\nxgb_model = XGBRegressor(random_state=42)\nxgb_model.fit(X_train_new, y_train_new)\n\n# Make predictions on the validation set\ny_pred_xgb = xgb_model.predict(X_val_new)\n\n# Evaluate the model\nmae_xgb = mean_absolute_error(y_val_new, y_pred_xgb)\nr2_xgb = r2_score(y_val_new, y_pred_xgb)\n\nprint(f\"XGBoost - Mean Absolute Error on validation set: {mae_xgb:.4f}\")\nprint(f\"XGBoost - R-squared on validation set: {r2_xgb:.4f}\")","metadata":{"id":"3f8b6215","executionInfo":{"status":"error","timestamp":1765027259104,"user_tz":-330,"elapsed":110,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"63274043-0978-455f-d9cf-51dac5e71bb4"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Reasoning**:\nThe previous cell failed because the `train_df` DataFrame does not contain an 'image_id' column, as indicated by the KeyError. The available `train_df` variable state shows a 'sample_id' column which serves the same purpose. I will replace 'image_id' with 'sample_id' in all relevant parts of the code for both `train_df` and `test_df` to resolve this issue and ensure consistent column usage.\n\n","metadata":{"id":"915012fb"}},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler, PolynomialFeatures\nfrom sklearn.model_selection import train_test_split\nfrom xgboost import XGBRegressor\nfrom sklearn.metrics import mean_absolute_error, r2_score\n\n# 1. Load Data (from cell 0wELoGYqqWJF, with corrected filenames)\ntrain_df = pd.read_csv('/content/drive/MyDrive/Australia_BioMass/csiro-biomass/train.csv')\ntest_df = pd.read_csv('/content/drive/MyDrive/Australia_BioMass/csiro-biomass/test.csv')\ntest_level_df = pd.read_csv('/content/drive/MyDrive/Australia_BioMass/csiro-biomass/sample_submission (1).csv')\n\n# 2. Define X and y (from cell e1517085)\ny = train_df['target']\nX = train_df[['Pre_GSHH_NDVI', 'Height_Ave_cm']]\n\n# 3. Feature Engineering (combining parts from b7cdc668, ae8b06bf, 86795d47, 58f07169)\n\n# Re-initialize X_new and test_df_processed for feature engineering\nX_new = X.copy()\ntest_df_processed = test_df.copy()\n\n# Extract unique sample_id, State, Species, and Sampling_Date from train_df\nimage_info = train_df[['sample_id', 'State', 'Species', 'Sampling_Date']].drop_duplicates()\n\n# Merge this information into test_df_processed using 'sample_id'\ntest_df_processed = pd.merge(test_df_processed, image_info[['sample_id', 'State', 'Species', 'Sampling_Date']], on='sample_id', how='left')\n\n# Impute NaN values in 'State' and 'Species' columns of test_df_processed with the mode from train_df\nmode_state_train = train_df['State'].mode()[0]\nmode_species_train = train_df['Species'].mode()[0]\ntest_df_processed['State'] = test_df_processed['State'].fillna(mode_state_train)\ntest_df_processed['Species'] = test_df_processed['Species'].fillna(mode_species_train)\n\n# Initialize and apply LabelEncoder for 'State' column\nle_state = LabelEncoder()\nX_new['encoded_State'] = le_state.fit_transform(train_df['State'])\ntest_df_processed['encoded_State'] = le_state.transform(test_df_processed['State'])\n\n# Initialize and apply LabelEncoder for 'Species' column\nle_species = LabelEncoder()\nX_new['encoded_Species'] = le_species.fit_transform(train_df['Species'])\ntest_df_processed['encoded_Species'] = le_species.transform(test_df_processed['Species'])\n\n# Convert the 'Sampling_Date' column in train_df and test_df_processed to datetime objects\ntrain_df['Sampling_Date'] = pd.to_datetime(train_df['Sampling_Date'])\ntest_df_processed['Sampling_Date'] = pd.to_datetime(test_df_processed['Sampling_Date'])\n\n# Extract date features for X_new (using train_df's Sampling_Date)\nX_new['year'] = train_df['Sampling_Date'].dt.year\nX_new['month'] = train_df['Sampling_Date'].dt.month\nX_new['day'] = train_df['Sampling_Date'].dt.day\nX_new['dayofweek'] = train_df['Sampling_Date'].dt.dayofweek\nX_new['dayofyear'] = train_df['Sampling_Date'].dt.dayofyear\n\n# Impute any missing values (NaN) in the 'Sampling_Date' column of test_df_processed with the mode from train_df\nmode_sampling_date_train = train_df['Sampling_Date'].mode()[0]\ntest_df_processed['Sampling_Date'] = test_df_processed['Sampling_Date'].fillna(mode_sampling_date_train)\ntest_df_processed['Sampling_Date'] = pd.to_datetime(test_df_processed['Sampling_Date']) # Re-ensure datetime after fillna\n\n# Extract date features for test_df_processed\ntest_df_processed['year'] = test_df_processed['Sampling_Date'].dt.year\ntest_df_processed['month'] = test_df_processed['Sampling_Date'].dt.month\ntest_df_processed['day'] = test_df_processed['Sampling_Date'].dt.day\ntest_df_processed['dayofweek'] = test_df_processed['Sampling_Date'].dt.dayofweek\ntest_df_processed['dayofyear'] = test_df_processed['Sampling_Date'].dt.dayofyear\n\n# Define final_features and select columns for X_new and test_df_processed\nfinal_features_base = ['Pre_GSHH_NDVI', 'Height_Ave_cm', 'encoded_State', 'encoded_Species', 'year', 'month', 'day', 'dayofweek', 'dayofyear']\nX_new = X_new[final_features_base]\ntest_df_processed = test_df_processed[final_features_base]\n\n# Interaction Feature (from cell ae8b06bf)\nX_new['NDVI_Height_Interaction'] = X_new['Pre_GSHH_NDVI'] * X_new['Height_Ave_cm']\ntest_df_processed['NDVI_Height_Interaction'] = test_df_processed['Pre_GSHH_NDVI'] * test_df_processed['Height_Ave_cm']\n\n# Polynomial Features (from cell 86795d47)\nnumerical_features_poly = ['Pre_GSHH_NDVI', 'Height_Ave_cm']\npoly = PolynomialFeatures(degree=2, include_bias=False)\npoly_features_X_new = poly.fit_transform(X_new[numerical_features_poly])\npoly_features_test_df = poly.transform(test_df_processed[numerical_features_poly])\npoly_feature_names = poly.get_feature_names_out(numerical_features_poly)\nX_new_poly_df = pd.DataFrame(poly_features_X_new, columns=poly_feature_names, index=X_new.index)\ntest_df_processed_poly_df = pd.DataFrame(poly_features_test_df, columns=poly_feature_names, index=test_df_processed.index)\nX_new = pd.concat([X_new, X_new_poly_df], axis=1)\ntest_df_processed = pd.concat([test_df_processed, test_df_processed_poly_df], axis=1)\n\n# Aggregated Features (from cell 58f07169)\nnumerical_features_to_aggregate = ['Pre_GSHH_NDVI', 'Height_Ave_cm']\ncategorical_groups = ['encoded_State', 'encoded_Species']\nagg_functions = ['mean', 'median', 'std']\n\nagg_X_new_df = X_new.groupby(categorical_groups)[numerical_features_to_aggregate].agg(agg_functions)\nagg_X_new_df.columns = [f\"{col[0]}_{col[1]}\" for col in agg_X_new_df.columns.values]\nagg_X_new_df = agg_X_new_df.reset_index()\nnew_col_names_map = {}\nfor col in agg_X_new_df.columns:\n    if col not in categorical_groups:\n        new_col_names_map[col] = f\"{col}_by_State_Species\"\nagg_X_new_df = agg_X_new_df.rename(columns=new_col_names_map)\nX_new = pd.merge(X_new, agg_X_new_df, on=categorical_groups, how='left')\n\nagg_test_df_df = test_df_processed.groupby(categorical_groups)[numerical_features_to_aggregate].agg(agg_functions)\nagg_test_df_df.columns = [f\"{col[0]}_{col[1]}\" for col in agg_test_df_df.columns.values]\nagg_test_df_df = agg_test_df_df.reset_index()\nnew_col_names_map_test = {}\nfor col in agg_test_df_df.columns:\n    if col not in categorical_groups:\n        new_col_names_map_test[col] = f\"{col}_by_State_Species\"\nagg_test_df_df = agg_test_df_df.rename(columns=new_col_names_map_test)\ntest_df_processed = pd.merge(test_df_processed, agg_test_df_df, on=categorical_groups, how='left')\n\n# 4. Train-Test Split and Scaling (from cell fea90d56)\nX_train_new, X_val_new, y_train_new, y_val_new = train_test_split(X_new, y, test_size=0.2, random_state=42)\n\n# Remove duplicate columns\nX_train_new = X_train_new.loc[:,~X_train_new.columns.duplicated()]\nX_val_new = X_val_new.loc[:,~X_val_new.columns.duplicated()]\ntest_df_processed = test_df_processed.loc[:,~test_df_processed.columns.duplicated()]\n\n# Identify all numerical features for scaling\noriginal_numerical_features = ['Pre_GSHH_NDVI', 'Height_Ave_cm']\ndate_features = ['year', 'month', 'day', 'dayofweek', 'dayofyear']\ninteraction_feature = ['NDVI_Height_Interaction']\npolynomial_features = [col for col in X_train_new.columns if \"^\" in col or \" \" in col]\naggregated_features = [col for col in X_train_new.columns if col.endswith('_by_State_Species')]\n\nall_numerical_features = original_numerical_features + date_features + interaction_feature + polynomial_features + aggregated_features\nall_numerical_features = list(dict.fromkeys(all_numerical_features))\nall_numerical_features = [f for f in all_numerical_features if f in X_train_new.columns] # Ensure features exist\n\nscaler = StandardScaler()\nscaler.fit(X_train_new[all_numerical_features])\n\nX_train_new[all_numerical_features] = scaler.transform(X_train_new[all_numerical_features])\nX_val_new[all_numerical_features] = scaler.transform(X_val_new[all_numerical_features])\ntest_df_processed[all_numerical_features] = scaler.transform(test_df_processed[all_numerical_features])\n\n# 5. Initialize and train the XGBoost Regressor model with engineered features\nxgb_model = XGBRegressor(random_state=42)\nxgb_model.fit(X_train_new, y_train_new)\n\n# Make predictions on the validation set\ny_pred_xgb = xgb_model.predict(X_val_new)\n\n# Evaluate the model\nmae_xgb = mean_absolute_error(y_val_new, y_pred_xgb)\nr2_xgb = r2_score(y_val_new, y_pred_xgb)\n\nprint(f\"XGBoost - Mean Absolute Error on validation set: {mae_xgb:.4f}\")\nprint(f\"XGBoost - R-squared on validation set: {r2_xgb:.4f}\")","metadata":{"id":"e19464a3","executionInfo":{"status":"error","timestamp":1765027283507,"user_tz":-330,"elapsed":144,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"a4f03b64-e9e1-4c91-a274-c8cc76c0ddf3"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Task\nI will now proceed with the plan.\n\n## Load DataFrames\n[(0wELoGYqqWJF) Load `train_df`, `test_df`, and `test_level_df` from their respective CSV files.](https://colab.research.google.com/drive/1h-44xV3V3_d77x2M0Xk9D9m6U6Vv1pWw#scrollTo=0wELoGYqqWJF)\n```python\nimport pandas as pd\ntrain_df = pd.read_csv('/content/drive/MyDrive/Australia_BioMass/csiro-biomass/train')\ntest_df = pd.read_csv('/content/drive/MyDrive/Australia_BioMass/csiro-biomass/test')\ntest_level_df = pd.read_csv('/content/drive/MyDrive/Australia_BioMass/csiro-biomass/sample_submission (1).csv')\n```","metadata":{"id":"27810acb"}},{"cell_type":"markdown","source":"## Load DataFrames\n\n### Subtask:\nLoad `train_df`, `test_df`, and `test_level_df` from their respective CSV files.\n","metadata":{"id":"5bcaee6f"}},{"cell_type":"markdown","source":"## Summary:\n\n### Data Analysis Key Findings\n*   Three dataframes were successfully loaded: `train_df`, `test_df`, and `test_level_df`.\n*   `train_df` and `test_df` were loaded from CSV files located at `/content/drive/MyDrive/Australia_BioMass/csiro-biomass/train` and `/content/drive/MyDrive/Australia_BioMass/csiro-biomass/test` respectively.\n*   `test_level_df` was loaded from the CSV file located at `/content/drive/MyDrive/Australia_BioMass/csiro-biomass/sample_submission (1).csv`.\n\n### Insights or Next Steps\n*   The next logical step is to perform an initial exploration of the loaded dataframes (e.g., checking dimensions, data types, and first few rows) to understand their structure and content.\n","metadata":{"id":"6117ba2f"}},{"cell_type":"markdown","source":"# Task\nTo address the `KeyError` and ensure `Pre_GSHH_NDVI` and `Height_Ave_cm` are correctly merged into `test_df_processed`, I need to explicitly include these columns when creating the `image_info` DataFrame from `train_df` and subsequently merging them into `test_df_processed`. This will provide the necessary features for the test set processing.\n\nI will execute a consolidated cell to load data, define `X` and `y`, perform all feature engineering steps (including correctly merging `Pre_GSHH_NDVI` and `Height_Ave_cm` for `test_df_processed`), handle missing values, apply label and standard scaling, and perform the train-test split. This will ensure that `X_train_new`, `y_train_new`, `X_val_new`, `y_val_new`, and `test_df_processed` are fully prepared for model training.\n\nThis will involve the following steps in a single consolidated cell:\n1.  **Load Data**: Load `train_df`, `test_df`, and `test_level_df` from their respective CSV files.\n2.  **Define X and y**: Extract `Pre_GSHH_NDVI` and `Height_Ave_cm` from `train_df` for `X` and `target` for `y`.\n3.  **Feature Engineering**:\n    *   Initialize `X_new` and `test_df_processed`.\n    *   Extract unique `sample_id`, `State`, `Species`, `Sampling_Date`, `Pre_GSHH_NDVI`, and `Height_Ave_cm` from `train_df` to create `image_info`.\n    *   Merge `image_info` into `test_df_processed` using `sample_id`.\n    *   Handle missing 'State', 'Species', and 'Sampling_Date' in `test_df_processed` by imputing with modes from `train_df`.\n    *   Apply Label Encoding to 'State' and 'Species' for both `X_new` and `test_df_processed`.\n    *   Extract date features (year, month, day, dayofweek, dayofyear) from 'Sampling_Date' for both `X_new` and `test_df_processed`.\n    *   Define `final_features_base` including `Pre_GSHH_NDVI` and `Height_Ave_cm` and filter `X_new` and `test_df_processed`.\n    *   Create interaction features (`NDVI_Height_Interaction`) for both datasets.\n    *   Generate polynomial features for `Pre_GSHH_NDVI` and `Height_Ave_cm` for both datasets.\n    *   Create aggregated features (mean, median, std of `Pre_GSHH_NDVI` and `Height_Ave_cm` grouped by `encoded_State` and `encoded_Species`) for both datasets.\n4.  **Train-Test Split**: Split `X_new` and `y` into training and validation sets (`X_train_new`, `X_val_new`, `y_train_new`, `y_val_new`).\n5.  **Handle Duplicates and Scaling**:\n    *   Remove duplicate columns from `X_train_new`, `X_val_new`, and `test_df_processed`.\n    *   Identify all numerical features across the engineered datasets.\n    *   Initialize and apply `StandardScaler` to all numerical features in `X_train_new`, `X_val_new`, and `test_df_processed`.\n\nThis comprehensive step will correctly prepare all dataframes for subsequent model training.\n\n```python\nimport pandas as pd\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler, PolynomialFeatures\nfrom sklearn.model_selection import train_test_split\nfrom xgboost import XGBRegressor # Keep this import as the original failure was in the XGBoost cell\n\n# 1. Load Data\ntrain_df = pd.read_csv('/content/drive/MyDrive/Australia_BioMass/csiro-biomass/train.csv')\ntest_df = pd.read_csv('/content/drive/MyDrive/Australia_BioMass/csiro-biomass/test.csv')\ntest_level_df = pd.read_csv('/content/drive/MyDrive/Australia_BioMass/csiro-biomass/sample_submission (1).csv')\n\n# 2. Define X and y\ny = train_df['target']\nX = train_df[['Pre_GSHH_NDVI', 'Height_Ave_cm']]\n\n# 3. Feature Engineering\n\n# Re-initialize X_new and test_df_processed for feature engineering\nX_new = X.copy()\ntest_df_processed = test_df.copy()\n\n# Extract unique sample_id, State, Species, Sampling_Date, Pre_GSHH_NDVI, Height_Ave_cm from train_df\nimage_info = train_df[['sample_id', 'State', 'Species', 'Sampling_Date', 'Pre_GSHH_NDVI', 'Height_Ave_cm']].drop_duplicates()\n\n# Merge this information into test_df_processed using 'sample_id'\ntest_df_processed = pd.merge(test_df_processed, image_info[['sample_id', 'State', 'Species', 'Sampling_Date', 'Pre_GSHH_NDVI', 'Height_Ave_cm']], on='sample_id', how='left')\n\n# Impute NaN values in 'State' and 'Species' columns of test_df_processed with the mode from train_df\nmode_state_train = train_df['State'].mode()[0]\nmode_species_train = train_df['Species'].mode()[0]\ntest_df_processed['State'] = test_df_processed['State'].fillna(mode_state_train)\ntest_df_processed['Species'] = test_df_processed['Species'].fillna(mode_species_train)\n\n# Initialize and apply LabelEncoder for 'State' column\nle_state = LabelEncoder()\nX_new['encoded_State'] = le_state.fit_transform(train_df['State'])\ntest_df_processed['encoded_State'] = le_state.transform(test_df_processed['State'])\n\n# Initialize and apply LabelEncoder for 'Species' column\nle_species = LabelEncoder()\nX_new['encoded_Species'] = le_species.fit_transform(train_df['Species'])\ntest_df_processed['encoded_Species'] = le_species.transform(test_df_processed['Species'])\n\n# Convert the 'Sampling_Date' column in train_df and test_df_processed to datetime objects\ntrain_df['Sampling_Date'] = pd.to_datetime(train_df['Sampling_Date'])\ntest_df_processed['Sampling_Date'] = pd.to_datetime(test_df_processed['Sampling_Date'])\n\n# Extract date features for X_new (using train_df's Sampling_Date)\nX_new['year'] = train_df['Sampling_Date'].dt.year\nX_new['month'] = train_df['Sampling_Date'].dt.month\nX_new['day'] = train_df['Sampling_Date'].dt.day\nX_new['dayofweek'] = train_df['Sampling_Date'].dt.dayofweek\nX_new['dayofyear'] = train_df['Sampling_Date'].dt.dayofyear\n\n# Impute any missing values (NaN) in the 'Sampling_Date' column of test_df_processed with the mode from train_df\nmode_sampling_date_train = train_df['Sampling_Date'].mode()[0]\ntest_df_processed['Sampling_Date'] = test_df_processed['Sampling_Date'].fillna(mode_sampling_date_train)\ntest_df_processed['Sampling_Date'] = pd.to_datetime(test_df_processed['Sampling_Date']) # Re-ensure datetime after fillna\n\n# Extract date features for test_df_processed\ntest_df_processed['year'] = test_df_processed['Sampling_Date'].dt.year\ntest_df_processed['month'] = test_df_processed['Sampling_Date'].dt.month\ntest_df_processed['day'] = test_df_processed['Sampling_Date'].dt.day\ntest_df_processed['dayofweek'] = test_df_processed['Sampling_Date'].dt.dayofweek\ntest_df_processed['dayofyear'] = test_df_processed['Sampling_Date'].dt.dayofyear\n\n# Define final_features and select columns for X_new and test_df_processed\nfinal_features_base = ['Pre_GSHH_NDVI', 'Height_Ave_cm', 'encoded_State', 'encoded_Species', 'year', 'month', 'day', 'dayofweek', 'dayofyear']\nX_new = X_new[final_features_base]\ntest_df_processed = test_df_processed[final_features_base]\n\n# Interaction Feature\nX_new['NDVI_Height_Interaction'] = X_new['Pre_GSHH_NDVI'] * X_new['Height_Ave_cm']\ntest_df_processed['NDVI_Height_Interaction'] = test_df_processed['Pre_GSHH_NDVI'] * test_df_processed['Height_Ave_cm']\n\n# Polynomial Features\nnumerical_features_poly = ['Pre_GSHH_NDVI', 'Height_Ave_cm']\npoly = PolynomialFeatures(degree=2, include_bias=False)\npoly_features_X_new = poly.fit_transform(X_new[numerical_features_poly])\npoly_features_test_df = poly.transform(test_df_processed[numerical_features_poly])\npoly_feature_names = poly.get_feature_names_out(numerical_features_poly)\nX_new_poly_df = pd.DataFrame(poly_features_X_new, columns=poly_feature_names, index=X_new.index)\ntest_df_processed_poly_df = pd.DataFrame(poly_features_test_df, columns=poly_feature_names, index=test_df_processed.index)\nX_new = pd.concat([X_new, X_new_poly_df], axis=1)\ntest_df_processed = pd.concat([test_df_processed, test_df_processed_poly_df], axis=1)\n\n# Aggregated Features\nnumerical_features_to_aggregate = ['Pre_GSHH_NDVI', 'Height_Ave_cm']\ncategorical_groups = ['encoded_State', 'encoded_Species']\nagg_functions = ['mean', 'median', 'std']\n\nagg_X_new_df = X_new.groupby(categorical_groups)[numerical_features_to_aggregate].agg(agg_functions)\nagg_X_new_df.columns = [f\"{col[0]}_{col[1]}\" for col in agg_X_new_df.columns.values]\nagg_X_new_df = agg_X_new_df.reset_index()\nnew_col_names_map = {}\nfor col in agg_X_new_df.columns:\n    if col not in categorical_groups:\n        new_col_names_map[col] = f\"{col}_by_State_Species\"\nagg_X_new_df = agg_X_new_df.rename(columns=new_col_names_map)\nX_new = pd.merge(X_new, agg_X_new_df, on=categorical_groups, how='left')\n\nagg_test_df_df = test_df_processed.groupby(categorical_groups)[numerical_features_to_aggregate].agg(agg_functions)\nagg_test_df_df.columns = [f\"{col[0]}_{col[1]}\" for col in agg_test_df_df.columns.values]\nagg_test_df_df = agg_test_df_df.reset_index()\nnew_col_names_map_test = {}\nfor col in agg_test_df_df.columns:\n    if col not in categorical_groups:\n        new_col_names_map_test[col] = f\"{col}_by_State_Species\"\nagg_test_df_df = agg_test_df_df.rename(columns=new_col_names_map_test)\ntest_df_processed = pd.merge(test_df_processed, agg_test_df_df, on=categorical_groups, how='left')\n\n\n# 4. Train-Test Split\nX_train_new, X_val_new, y_train_new, y_val_new = train_test_split(X_new, y, test_size=0.2, random_state=42)\n\n# Remove duplicate columns\nX_train_new = X_train_new.loc[:,~X_train_new.columns.duplicated()]\nX_val_new = X_val_new.loc[:,~X_val_new.columns.duplicated()]\ntest_df_processed = test_df_processed.loc[:,~test_df_processed.columns.duplicated()]\n\n# Identify all numerical features for scaling\noriginal_numerical_features = ['Pre_GSHH_NDVI', 'Height_Ave_cm']\ndate_features = ['year', 'month', 'day', 'dayofweek', 'dayofyear']\ninteraction_feature = ['NDVI_Height_Interaction']\npolynomial_features = [col for col in X_train_new.columns if \"^\" in col or \" \" in col]\naggregated_features = [col for col in X_train_new.columns if col.endswith('_by_State_Species')]\n\nall_numerical_features = original_numerical_features + date_features + interaction_feature + polynomial_features + aggregated_features\nall_numerical_features = list(dict.fromkeys(all_numerical_features))\nall_numerical_features = [f for f in all_numerical_features if f in X_train_new.columns] # Ensure features exist\n\n# 5. Apply Standard Scaling\nscaler = StandardScaler()\nscaler.fit(X_train_new[all_numerical_features])\n\nX_train_new[all_numerical_features] = scaler.transform(X_train_new[all_numerical_features])\nX_val_new[all_numerical_features] = scaler.transform(X_val_new[all_numerical_features])\ntest_df_processed[all_numerical_features] = scaler.transform(test_df_processed[all_numerical_features])\n\nprint(\"Comprehensive Data Preprocessing, Feature Engineering, and Train-Test Split completed.\")\nprint(\"Shapes of prepared dataframes:\")\nprint(f\"X_train_new: {X_train_new.shape}\")\nprint(f\"y_train_new: {y_train_new.shape}\")\nprint(f\"X_val_new: {X_val_new.shape}\")\nprint(f\"y_val_new: {y_val_new.shape}\")\nprint(f\"test_df_processed: {test_df_processed.shape}\")\n```","metadata":{"id":"607399d3"}},{"cell_type":"markdown","source":"## Comprehensive Data Preprocessing\n\n### Subtask:\nExecute a consolidated cell to load data, define X and y, perform all feature engineering steps, handle missing values, apply label and standard scaling, and perform the train-test split.\n","metadata":{"id":"1e714fba"}},{"cell_type":"markdown","source":"**Reasoning**:\nThe previous attempt to consolidate the preprocessing steps failed because `test_df_processed` did not contain the necessary numerical features (`Pre_GSHH_NDVI`, `Height_Ave_cm`) after the initial copy and merge operations. The updated instructions for creating `image_info` now correctly include these features, which will be merged into `test_df_processed`. This consolidated cell will now perform all data loading, feature engineering, and train-test splitting in a single block, ensuring all dependencies are met before model training.\n\n","metadata":{"id":"a591119d"}},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler, PolynomialFeatures\nfrom sklearn.model_selection import train_test_split\nfrom xgboost import XGBRegressor\n\n# 1. Load Data\ntrain_df = pd.read_csv('/content/drive/MyDrive/Australia_BioMass/csiro-biomass/train.csv')\ntest_df = pd.read_csv('/content/drive/MyDrive/Australia_BioMass/csiro-biomass/test.csv')\ntest_level_df = pd.read_csv('/content/drive/MyDrive/Australia_BioMass/csiro-biomass/sample_submission (1).csv')\nprint(\"Data loaded successfully.\")\n\n# 2. Define X and y\ny = train_df['target']\nX = train_df[['Pre_GSHH_NDVI', 'Height_Ave_cm']]\nprint(\"X and y defined.\")\n\n# 3. Feature Engineering\n# Re-initialize X_new and test_df_processed for feature engineering\nX_new = X.copy()\ntest_df_processed = test_df.copy()\n\n# Create image_info DataFrame including numerical features\nimage_info = train_df[['sample_id', 'State', 'Species', 'Sampling_Date', 'Pre_GSHH_NDVI', 'Height_Ave_cm']].drop_duplicates()\n\n# Merge this information into test_df_processed\ntest_df_processed = pd.merge(test_df_processed, image_info, on='sample_id', how='left')\n\n# Impute NaN values in 'State' and 'Species' columns of test_df_processed with the mode from train_df\nmode_state_train = train_df['State'].mode()[0]\nmode_species_train = train_df['Species'].mode()[0]\ntest_df_processed['State'] = test_df_processed['State'].fillna(mode_state_train)\ntest_df_processed['Species'] = test_df_processed['Species'].fillna(mode_species_train)\n\n# Initialize and apply LabelEncoder for 'State' column\nle_state = LabelEncoder()\nX_new['encoded_State'] = le_state.fit_transform(train_df['State'])\ntest_df_processed['encoded_State'] = le_state.transform(test_df_processed['State'])\n\n# Initialize and apply LabelEncoder for 'Species' column\nle_species = LabelEncoder()\nX_new['encoded_Species'] = le_species.fit_transform(train_df['Species'])\ntest_df_processed['encoded_Species'] = le_species.transform(test_df_processed['Species'])\n\n# Convert the 'Sampling_Date' column in train_df and test_df_processed to datetime objects\ntrain_df['Sampling_Date'] = pd.to_datetime(train_df['Sampling_Date'])\ntest_df_processed['Sampling_Date'] = pd.to_datetime(test_df_processed['Sampling_Date'])\n\n# Extract date features for X_new (using train_df's Sampling_Date)\nX_new['year'] = train_df['Sampling_Date'].dt.year\nX_new['month'] = train_df['Sampling_Date'].dt.month\nX_new['day'] = train_df['Sampling_Date'].dt.day\nX_new['dayofweek'] = train_df['Sampling_Date'].dt.dayofweek\nX_new['dayofyear'] = train_df['Sampling_Date'].dt.dayofyear\n\n# Impute any missing values (NaN) in the 'Sampling_Date' column of test_df_processed with the mode from train_df\nmode_sampling_date_train = train_df['Sampling_Date'].mode()[0]\ntest_df_processed['Sampling_Date'] = test_df_processed['Sampling_Date'].fillna(mode_sampling_date_train)\ntest_df_processed['Sampling_Date'] = pd.to_datetime(test_df_processed['Sampling_Date']) # Re-ensure datetime after fillna\n\n# Extract date features for test_df_processed\ntest_df_processed['year'] = test_df_processed['Sampling_Date'].dt.year\ntest_df_processed['month'] = test_df_processed['Sampling_Date'].dt.month\ntest_df_processed['day'] = test_df_processed['Sampling_Date'].dt.day\ntest_df_processed['dayofweek'] = test_df_processed['Sampling_Date'].dt.dayofweek\ntest_df_processed['dayofyear'] = test_df_processed['Sampling_Date'].dt.dayofyear\n\n# Define final_features and select columns for X_new and test_df_processed\nfinal_features_base = ['Pre_GSHH_NDVI', 'Height_Ave_cm', 'encoded_State', 'encoded_Species', 'year', 'month', 'day', 'dayofweek', 'dayofyear']\nX_new = X_new[final_features_base]\ntest_df_processed = test_df_processed[final_features_base]\n\n# Interaction Feature\nX_new['NDVI_Height_Interaction'] = X_new['Pre_GSHH_NDVI'] * X_new['Height_Ave_cm']\ntest_df_processed['NDVI_Height_Interaction'] = test_df_processed['Pre_GSHH_NDVI'] * test_df_processed['Height_Ave_cm']\n\n# Polynomial Features\nnumerical_features_poly = ['Pre_GSHH_NDVI', 'Height_Ave_cm']\npoly = PolynomialFeatures(degree=2, include_bias=False)\npoly_features_X_new = poly.fit_transform(X_new[numerical_features_poly])\npoly_features_test_df = poly.transform(test_df_processed[numerical_features_poly])\npoly_feature_names = poly.get_feature_names_out(numerical_features_poly)\nX_new_poly_df = pd.DataFrame(poly_features_X_new, columns=poly_feature_names, index=X_new.index)\ntest_df_processed_poly_df = pd.DataFrame(poly_features_test_df, columns=poly_feature_names, index=test_df_processed.index)\nX_new = pd.concat([X_new, X_new_poly_df], axis=1)\ntest_df_processed = pd.concat([test_df_processed, test_df_processed_poly_df], axis=1)\n\n# Aggregated Features\nnumerical_features_to_aggregate = ['Pre_GSHH_NDVI', 'Height_Ave_cm']\ncategorical_groups = ['encoded_State', 'encoded_Species']\nagg_functions = ['mean', 'median', 'std']\n\nagg_X_new_df = X_new.groupby(categorical_groups)[numerical_features_to_aggregate].agg(agg_functions)\nagg_X_new_df.columns = [f\"{col[0]}_{col[1]}\" for col in agg_X_new_df.columns.values]\nagg_X_new_df = agg_X_new_df.reset_index()\nnew_col_names_map = {}\nfor col in agg_X_new_df.columns:\n    if col not in categorical_groups:\n        new_col_names_map[col] = f\"{col}_by_State_Species\"\nagg_X_new_df = agg_X_new_df.rename(columns=new_col_names_map)\nX_new = pd.merge(X_new, agg_X_new_df, on=categorical_groups, how='left')\n\nagg_test_df_df = test_df_processed.groupby(categorical_groups)[numerical_features_to_aggregate].agg(agg_functions)\nagg_test_df_df.columns = [f\"{col[0]}_{col[1]}\" for col in agg_test_df_df.columns.values]\nagg_test_df_df = agg_test_df_df.reset_index()\nnew_col_names_map_test = {}\nfor col in agg_test_df_df.columns:\n    if col not in categorical_groups:\n        new_col_names_map_test[col] = f\"{col}_by_State_Species\"\nagg_test_df_df = agg_test_df_df.rename(columns=new_col_names_map_test)\ntest_df_processed = pd.merge(test_df_processed, agg_test_df_df, on=categorical_groups, how='left')\nprint(\"Feature engineering completed.\")\n\n# 4. Train-Test Split and Scaling\nX_train_new, X_val_new, y_train_new, y_val_new = train_test_split(X_new, y, test_size=0.2, random_state=42)\n\n# Remove duplicate columns\nX_train_new = X_train_new.loc[:,~X_train_new.columns.duplicated()]\nX_val_new = X_val_new.loc[:,~X_val_new.columns.duplicated()]\ntest_df_processed = test_df_processed.loc[:,~test_df_processed.columns.duplicated()]\n\n# Identify all numerical features for scaling\noriginal_numerical_features = ['Pre_GSHH_NDVI', 'Height_Ave_cm']\ndate_features = ['year', 'month', 'day', 'dayofweek', 'dayofyear']\ninteraction_feature = ['NDVI_Height_Interaction']\npolynomial_features = [col for col in X_train_new.columns if \"^\" in col or \" \" in col]\naggregated_features = [col for col in X_train_new.columns if col.endswith('_by_State_Species')]\n\nall_numerical_features = original_numerical_features + date_features + interaction_feature + polynomial_features + aggregated_features\nall_numerical_features = list(dict.fromkeys(all_numerical_features)) # Remove duplicates while preserving order\nall_numerical_features = [f for f in all_numerical_features if f in X_train_new.columns] # Ensure features exist\n\nscaler = StandardScaler()\nscaler.fit(X_train_new[all_numerical_features])\n\nX_train_new[all_numerical_features] = scaler.transform(X_train_new[all_numerical_features])\nX_val_new[all_numerical_features] = scaler.transform(X_val_new[all_numerical_features])\ntest_df_processed[all_numerical_features] = scaler.transform(test_df_processed[all_numerical_features])\nprint(\"Train-test split and scaling completed.\")\n\nprint(\"\\nShape of X_train_new:\", X_train_new.shape)\nprint(\"Shape of y_train_new:\", y_train_new.shape)\nprint(\"Shape of X_val_new:\", X_val_new.shape)\nprint(\"Shape of y_val_new:\", y_val_new.shape)\nprint(\"Shape of test_df_processed:\", test_df_processed.shape)","metadata":{"id":"a51174d2","executionInfo":{"status":"error","timestamp":1765027579062,"user_tz":-330,"elapsed":193,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"99a1bfca-43a4-4b5e-eba2-2030c31c8038"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Reasoning**:\nThe previous cell failed because `test_df_processed` contained NaN values in 'Pre_GSHH_NDVI' and 'Height_Ave_cm' when `PolynomialFeatures` was applied, as confirmed by the `ValueError`. This happens because some `sample_id`s in `test_df` might not have a match in `train_df`'s `image_info`, leading to NaNs during the merge. To fix this, I will add a step to impute these numerical NaN values in `test_df_processed` with the mean of their respective columns from `train_df` after the merge and before applying polynomial features.\n\n","metadata":{"id":"4d758f6d"}},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler, PolynomialFeatures\nfrom sklearn.model_selection import train_test_split\nfrom xgboost import XGBRegressor\n\n# 1. Load Data\ntrain_df = pd.read_csv('/content/drive/MyDrive/Australia_BioMass/csiro-biomass/train.csv')\ntest_df = pd.read_csv('/content/drive/MyDrive/Australia_BioMass/csiro-biomass/test.csv')\ntest_level_df = pd.read_csv('/content/drive/MyDrive/Australia_BioMass/csiro-biomass/sample_submission (1).csv')\nprint(\"Data loaded successfully.\")\n\n# 2. Define X and y\ny = train_df['target']\nX = train_df[['Pre_GSHH_NDVI', 'Height_Ave_cm']]\nprint(\"X and y defined.\")\n\n# 3. Feature Engineering\n# Re-initialize X_new and test_df_processed for feature engineering\nX_new = X.copy()\ntest_df_processed = test_df.copy()\n\n# Create image_info DataFrame including numerical features\nimage_info = train_df[['sample_id', 'State', 'Species', 'Sampling_Date', 'Pre_GSHH_NDVI', 'Height_Ave_cm']].drop_duplicates()\n\n# Merge this information into test_df_processed\ntest_df_processed = pd.merge(test_df_processed, image_info, on='sample_id', how='left')\n\n# Impute NaN values in 'Pre_GSHH_NDVI' and 'Height_Ave_cm' in test_df_processed\n# using the mean from train_df before polynomial features are applied\nmean_ndvi_train = train_df['Pre_GSHH_NDVI'].mean()\nmean_height_train = train_df['Height_Ave_cm'].mean()\ntest_df_processed['Pre_GSHH_NDVI'].fillna(mean_ndvi_train, inplace=True)\ntest_df_processed['Height_Ave_cm'].fillna(mean_height_train, inplace=True)\nprint(\"Imputed missing Pre_GSHH_NDVI and Height_Ave_cm in test_df_processed.\")\n\n# Impute NaN values in 'State' and 'Species' columns of test_df_processed with the mode from train_df\nmode_state_train = train_df['State'].mode()[0]\nmode_species_train = train_df['Species'].mode()[0]\ntest_df_processed['State'] = test_df_processed['State'].fillna(mode_state_train)\ntest_df_processed['Species'] = test_df_processed['Species'].fillna(mode_species_train)\n\n# Initialize and apply LabelEncoder for 'State' column\nle_state = LabelEncoder()\nX_new['encoded_State'] = le_state.fit_transform(train_df['State'])\ntest_df_processed['encoded_State'] = le_state.transform(test_df_processed['State'])\n\n# Initialize and apply LabelEncoder for 'Species' column\nle_species = LabelEncoder()\nX_new['encoded_Species'] = le_species.fit_transform(train_df['Species'])\ntest_df_processed['encoded_Species'] = le_species.transform(test_df_processed['Species'])\n\n# Convert the 'Sampling_Date' column in train_df and test_df_processed to datetime objects\ntrain_df['Sampling_Date'] = pd.to_datetime(train_df['Sampling_Date'])\ntest_df_processed['Sampling_Date'] = pd.to_datetime(test_df_processed['Sampling_Date'])\n\n# Extract date features for X_new (using train_df's Sampling_Date)\nX_new['year'] = train_df['Sampling_Date'].dt.year\nX_new['month'] = train_df['Sampling_Date'].dt.month\nX_new['day'] = train_df['Sampling_Date'].dt.day\nX_new['dayofweek'] = train_df['Sampling_Date'].dt.dayofweek\nX_new['dayofyear'] = train_df['Sampling_Date'].dt.dayofyear\n\n# Impute any missing values (NaN) in the 'Sampling_Date' column of test_df_processed with the mode from train_df\nmode_sampling_date_train = train_df['Sampling_Date'].mode()[0]\ntest_df_processed['Sampling_Date'] = test_df_processed['Sampling_Date'].fillna(mode_sampling_date_train)\ntest_df_processed['Sampling_Date'] = pd.to_datetime(test_df_processed['Sampling_Date']) # Re-ensure datetime after fillna\n\n# Extract date features for test_df_processed\ntest_df_processed['year'] = test_df_processed['Sampling_Date'].dt.year\ntest_df_processed['month'] = test_df_processed['Sampling_Date'].dt.month\ntest_df_processed['day'] = test_df_processed['Sampling_Date'].dt.day\ntest_df_processed['dayofweek'] = test_df_processed['Sampling_Date'].dt.dayofweek\ntest_df_processed['dayofyear'] = test_df_processed['Sampling_Date'].dt.dayofyear\n\n# Define final_features and select columns for X_new and test_df_processed\nfinal_features_base = ['Pre_GSHH_NDVI', 'Height_Ave_cm', 'encoded_State', 'encoded_Species', 'year', 'month', 'day', 'dayofweek', 'dayofyear']\nX_new = X_new[final_features_base]\ntest_df_processed = test_df_processed[final_features_base]\n\n# Interaction Feature\nX_new['NDVI_Height_Interaction'] = X_new['Pre_GSHH_NDVI'] * X_new['Height_Ave_cm']\ntest_df_processed['NDVI_Height_Interaction'] = test_df_processed['Pre_GSHH_NDVI'] * test_df_processed['Height_Ave_cm']\n\n# Polynomial Features\nnumerical_features_poly = ['Pre_GSHH_NDVI', 'Height_Ave_cm']\npoly = PolynomialFeatures(degree=2, include_bias=False)\npoly_features_X_new = poly.fit_transform(X_new[numerical_features_poly])\npoly_features_test_df = poly.transform(test_df_processed[numerical_features_poly])\npoly_feature_names = poly.get_feature_names_out(numerical_features_poly)\nX_new_poly_df = pd.DataFrame(poly_features_X_new, columns=poly_feature_names, index=X_new.index)\ntest_df_processed_poly_df = pd.DataFrame(poly_features_test_df, columns=poly_feature_names, index=test_df_processed.index)\nX_new = pd.concat([X_new, X_new_poly_df], axis=1)\ntest_df_processed = pd.concat([test_df_processed, test_df_processed_poly_df], axis=1)\n\n# Aggregated Features\nnumerical_features_to_aggregate = ['Pre_GSHH_NDVI', 'Height_Ave_cm']\ncategorical_groups = ['encoded_State', 'encoded_Species']\nagg_functions = ['mean', 'median', 'std']\n\nagg_X_new_df = X_new.groupby(categorical_groups)[numerical_features_to_aggregate].agg(agg_functions)\nagg_X_new_df.columns = [f\"{col[0]}_{col[1]}\" for col in agg_X_new_df.columns.values]\nagg_X_new_df = agg_X_new_df.reset_index()\nnew_col_names_map = {}\nfor col in agg_X_new_df.columns:\n    if col not in categorical_groups:\n        new_col_names_map[col] = f\"{col}_by_State_Species\"\nagg_X_new_df = agg_X_new_df.rename(columns=new_col_names_map)\nX_new = pd.merge(X_new, agg_X_new_df, on=categorical_groups, how='left')\n\nagg_test_df_df = test_df_processed.groupby(categorical_groups)[numerical_features_to_aggregate].agg(agg_functions)\nagg_test_df_df.columns = [f\"{col[0]}_{col[1]}\" for col in agg_test_df_df.columns.values]\nagg_test_df_df = agg_test_df_df.reset_index()\nnew_col_names_map_test = {}\nfor col in agg_test_df_df.columns:\n    if col not in categorical_groups:\n        new_col_names_map_test[col] = f\"{col}_by_State_Species\"\nagg_test_df_df = agg_test_df_df.rename(columns=new_col_names_map_test)\ntest_df_processed = pd.merge(test_df_processed, agg_test_df_df, on=categorical_groups, how='left')\nprint(\"Feature engineering completed.\")\n\n# 4. Train-Test Split and Scaling\nX_train_new, X_val_new, y_train_new, y_val_new = train_test_split(X_new, y, test_size=0.2, random_state=42)\n\n# Remove duplicate columns\nX_train_new = X_train_new.loc[:,~X_train_new.columns.duplicated()]\nX_val_new = X_val_new.loc[:,~X_val_new.columns.duplicated()]\ntest_df_processed = test_df_processed.loc[:,~test_df_processed.columns.duplicated()]\n\n# Identify all numerical features for scaling\noriginal_numerical_features = ['Pre_GSHH_NDVI', 'Height_Ave_cm']\ndate_features = ['year', 'month', 'day', 'dayofweek', 'dayofyear']\ninteraction_feature = ['NDVI_Height_Interaction']\npolynomial_features = [col for col in X_train_new.columns if \"^\" in col or \" \" in col]\naggregated_features = [col for col in X_train_new.columns if col.endswith('_by_State_Species')]\n\nall_numerical_features = original_numerical_features + date_features + interaction_feature + polynomial_features + aggregated_features\nall_numerical_features = list(dict.fromkeys(all_numerical_features)) # Remove duplicates while preserving order\nall_numerical_features = [f for f in all_numerical_features if f in X_train_new.columns] # Ensure features exist\n\nscaler = StandardScaler()\nscaler.fit(X_train_new[all_numerical_features])\n\nX_train_new[all_numerical_features] = scaler.transform(X_train_new[all_numerical_features])\nX_val_new[all_numerical_features] = scaler.transform(X_val_new[all_numerical_features])\ntest_df_processed[all_numerical_features] = scaler.transform(test_df_processed[all_numerical_features])\nprint(\"Train-test split and scaling completed.\")\n\nprint(\"\\nShape of X_train_new:\", X_train_new.shape)\nprint(\"Shape of y_train_new:\", y_train_new.shape)\nprint(\"Shape of X_val_new:\", X_val_new.shape)\nprint(\"Shape of y_val_new:\", y_val_new.shape)\nprint(\"Shape of test_df_processed:\", test_df_processed.shape)\n","metadata":{"id":"ae1469d5","executionInfo":{"status":"ok","timestamp":1765027597327,"user_tz":-330,"elapsed":208,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"e1d78917-1759-467b-c058-591840c67c5f"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Reasoning**:\nThe previous code generated `FutureWarning` messages because of the `inplace=True` argument in `fillna`. I will modify the code to reassign the columns after calling `fillna` without `inplace=True` to follow best practices and remove the warnings.\n\n","metadata":{"id":"496a903f"}},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler, PolynomialFeatures\nfrom sklearn.model_selection import train_test_split\nfrom xgboost import XGBRegressor\n\n# 1. Load Data\ntrain_df = pd.read_csv('/content/drive/MyDrive/Australia_BioMass/csiro-biomass/train.csv')\ntest_df = pd.read_csv('/content/drive/MyDrive/Australia_BioMass/csiro-biomass/test.csv')\ntest_level_df = pd.read_csv('/content/drive/MyDrive/Australia_BioMass/csiro-biomass/sample_submission (1).csv')\nprint(\"Data loaded successfully.\")\n\n# 2. Define X and y\ny = train_df['target']\nX = train_df[['Pre_GSHH_NDVI', 'Height_Ave_cm']]\nprint(\"X and y defined.\")\n\n# 3. Feature Engineering\n# Re-initialize X_new and test_df_processed for feature engineering\nX_new = X.copy()\ntest_df_processed = test_df.copy()\n\n# Create image_info DataFrame including numerical features\nimage_info = train_df[['sample_id', 'State', 'Species', 'Sampling_Date', 'Pre_GSHH_NDVI', 'Height_Ave_cm']].drop_duplicates()\n\n# Merge this information into test_df_processed\ntest_df_processed = pd.merge(test_df_processed, image_info, on='sample_id', how='left')\n\n# Impute NaN values in 'Pre_GSHH_NDVI' and 'Height_Ave_cm' in test_df_processed\n# using the mean from train_df before polynomial features are applied\nmean_ndvi_train = train_df['Pre_GSHH_NDVI'].mean()\nmean_height_train = train_df['Height_Ave_cm'].mean()\ntest_df_processed['Pre_GSHH_NDVI'] = test_df_processed['Pre_GSHH_NDVI'].fillna(mean_ndvi_train)\ntest_df_processed['Height_Ave_cm'] = test_df_processed['Height_Ave_cm'].fillna(mean_height_train)\nprint(\"Imputed missing Pre_GSHH_NDVI and Height_Ave_cm in test_df_processed.\")\n\n# Impute NaN values in 'State' and 'Species' columns of test_df_processed with the mode from train_df\nmode_state_train = train_df['State'].mode()[0]\nmode_species_train = train_df['Species'].mode()[0]\ntest_df_processed['State'] = test_df_processed['State'].fillna(mode_state_train)\ntest_df_processed['Species'] = test_df_processed['Species'].fillna(mode_species_train)\n\n# Initialize and apply LabelEncoder for 'State' column\nle_state = LabelEncoder()\nX_new['encoded_State'] = le_state.fit_transform(train_df['State'])\ntest_df_processed['encoded_State'] = le_state.transform(test_df_processed['State'])\n\n# Initialize and apply LabelEncoder for 'Species' column\nle_species = LabelEncoder()\nX_new['encoded_Species'] = le_species.fit_transform(train_df['Species'])\ntest_df_processed['encoded_Species'] = le_species.transform(test_df_processed['Species'])\n\n# Convert the 'Sampling_Date' column in train_df and test_df_processed to datetime objects\ntrain_df['Sampling_Date'] = pd.to_datetime(train_df['Sampling_Date'])\ntest_df_processed['Sampling_Date'] = pd.to_datetime(test_df_processed['Sampling_Date'])\n\n# Extract date features for X_new (using train_df's Sampling_Date)\nX_new['year'] = train_df['Sampling_Date'].dt.year\nX_new['month'] = train_df['Sampling_Date'].dt.month\nX_new['day'] = train_df['Sampling_Date'].dt.day\nX_new['dayofweek'] = train_df['Sampling_Date'].dt.dayofweek\nX_new['dayofyear'] = train_df['Sampling_Date'].dt.dayofyear\n\n# Impute any missing values (NaN) in the 'Sampling_Date' column of test_df_processed with the mode from train_df\nmode_sampling_date_train = train_df['Sampling_Date'].mode()[0]\ntest_df_processed['Sampling_Date'] = test_df_processed['Sampling_Date'].fillna(mode_sampling_date_train)\ntest_df_processed['Sampling_Date'] = pd.to_datetime(test_df_processed['Sampling_Date']) # Re-ensure datetime after fillna\n\n# Extract date features for test_df_processed\ntest_df_processed['year'] = test_df_processed['Sampling_Date'].dt.year\ntest_df_processed['month'] = test_df_processed['Sampling_Date'].dt.month\ntest_df_processed['day'] = test_df_processed['Sampling_Date'].dt.day\ntest_df_processed['dayofweek'] = test_df_processed['Sampling_Date'].dt.dayofweek\ntest_df_processed['dayofyear'] = test_df_processed['Sampling_Date'].dt.dayofyear\n\n# Define final_features and select columns for X_new and test_df_processed\nfinal_features_base = ['Pre_GSHH_NDVI', 'Height_Ave_cm', 'encoded_State', 'encoded_Species', 'year', 'month', 'day', 'dayofweek', 'dayofyear']\nX_new = X_new[final_features_base]\ntest_df_processed = test_df_processed[final_features_base]\n\n# Interaction Feature\nX_new['NDVI_Height_Interaction'] = X_new['Pre_GSHH_NDVI'] * X_new['Height_Ave_cm']\ntest_df_processed['NDVI_Height_Interaction'] = test_df_processed['Pre_GSHH_NDVI'] * test_df_processed['Height_Ave_cm']\n\n# Polynomial Features\nnumerical_features_poly = ['Pre_GSHH_NDVI', 'Height_Ave_cm']\npoly = PolynomialFeatures(degree=2, include_bias=False)\npoly_features_X_new = poly.fit_transform(X_new[numerical_features_poly])\npoly_features_test_df = poly.transform(test_df_processed[numerical_features_poly])\npoly_feature_names = poly.get_feature_names_out(numerical_features_poly)\nX_new_poly_df = pd.DataFrame(poly_features_X_new, columns=poly_feature_names, index=X_new.index)\ntest_df_processed_poly_df = pd.DataFrame(poly_features_test_df, columns=poly_feature_names, index=test_df_processed.index)\nX_new = pd.concat([X_new, X_new_poly_df], axis=1)\ntest_df_processed = pd.concat([test_df_processed, test_df_processed_poly_df], axis=1)\n\n# Aggregated Features\nnumerical_features_to_aggregate = ['Pre_GSHH_NDVI', 'Height_Ave_cm']\ncategorical_groups = ['encoded_State', 'encoded_Species']\nagg_functions = ['mean', 'median', 'std']\n\nagg_X_new_df = X_new.groupby(categorical_groups)[numerical_features_to_aggregate].agg(agg_functions)\nagg_X_new_df.columns = [f\"{col[0]}_{col[1]}\" for col in agg_X_new_df.columns.values]\nagg_X_new_df = agg_X_new_df.reset_index()\nnew_col_names_map = {}\nfor col in agg_X_new_df.columns:\n    if col not in categorical_groups:\n        new_col_names_map[col] = f\"{col}_by_State_Species\"\nagg_X_new_df = agg_X_new_df.rename(columns=new_col_names_map)\nX_new = pd.merge(X_new, agg_X_new_df, on=categorical_groups, how='left')\n\nagg_test_df_df = test_df_processed.groupby(categorical_groups)[numerical_features_to_aggregate].agg(agg_functions)\nagg_test_df_df.columns = [f\"{col[0]}_{col[1]}\" for col in agg_test_df_df.columns.values]\nagg_test_df_df = agg_test_df_df.reset_index()\nnew_col_names_map_test = {}\nfor col in agg_test_df_df.columns:\n    if col not in categorical_groups:\n        new_col_names_map_test[col] = f\"{col}_by_State_Species\"\nagg_test_df_df = agg_test_df_df.rename(columns=new_col_names_map_test)\ntest_df_processed = pd.merge(test_df_processed, agg_test_df_df, on=categorical_groups, how='left')\nprint(\"Feature engineering completed.\")\n\n# 4. Train-Test Split and Scaling\nX_train_new, X_val_new, y_train_new, y_val_new = train_test_split(X_new, y, test_size=0.2, random_state=42)\n\n# Remove duplicate columns\nX_train_new = X_train_new.loc[:,~X_train_new.columns.duplicated()]\nX_val_new = X_val_new.loc[:,~X_val_new.columns.duplicated()]\ntest_df_processed = test_df_processed.loc[:,~test_df_processed.columns.duplicated()]\n\n# Identify all numerical features for scaling\noriginal_numerical_features = ['Pre_GSHH_NDVI', 'Height_Ave_cm']\ndate_features = ['year', 'month', 'day', 'dayofweek', 'dayofyear']\ninteraction_feature = ['NDVI_Height_Interaction']\npolynomial_features = [col for col in X_train_new.columns if \"^\" in col or \" \" in col]\naggregated_features = [col for col in X_train_new.columns if col.endswith('_by_State_Species')]\n\nall_numerical_features = original_numerical_features + date_features + interaction_feature + polynomial_features + aggregated_features\nall_numerical_features = list(dict.fromkeys(all_numerical_features)) # Remove duplicates while preserving order\nall_numerical_features = [f for f in all_numerical_features if f in X_train_new.columns] # Ensure features exist\n\nscaler = StandardScaler()\nscaler.fit(X_train_new[all_numerical_features])\n\nX_train_new[all_numerical_features] = scaler.transform(X_train_new[all_numerical_features])\nX_val_new[all_numerical_features] = scaler.transform(X_val_new[all_numerical_features])\ntest_df_processed[all_numerical_features] = scaler.transform(test_df_processed[all_numerical_features])\nprint(\"Train-test split and scaling completed.\")\n\nprint(\"\\nShape of X_train_new:\", X_train_new.shape)\nprint(\"Shape of y_train_new:\", y_train_new.shape)\nprint(\"Shape of X_val_new:\", X_val_new.shape)\nprint(\"Shape of y_val_new:\", y_val_new.shape)\nprint(\"Shape of test_df_processed:\", test_df_processed.shape)\n","metadata":{"id":"5dbd39ff","executionInfo":{"status":"ok","timestamp":1765027618317,"user_tz":-330,"elapsed":411,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"4f4251bf-9f1f-46d1-f2d3-cf56c8f31938"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Reasoning**:\nThe previous code block successfully completed all data preprocessing and feature engineering steps, making `X_train_new`, `y_train_new`, `X_val_new`, `y_val_new`, and `test_df_processed` available. Now, the subtask of re-training the XGBoost model can be executed using these prepared datasets.\n\n","metadata":{"id":"3d3ee335"}},{"cell_type":"code","source":"from xgboost import XGBRegressor\nfrom sklearn.metrics import mean_absolute_error, r2_score\n\n# Initialize and train the XGBoost Regressor model with engineered features\nxgb_model = XGBRegressor(random_state=42)\nxgb_model.fit(X_train_new, y_train_new)\n\n# Make predictions on the validation set\ny_pred_xgb = xgb_model.predict(X_val_new)\n\n# Evaluate the model\nmae_xgb = mean_absolute_error(y_val_new, y_pred_xgb)\nr2_xgb = r2_score(y_val_new, y_pred_xgb)\n\nprint(f\"XGBoost - Mean Absolute Error on validation set: {mae_xgb:.4f}\")\nprint(f\"XGBoost - R-squared on validation set: {r2_xgb:.4f}\")","metadata":{"id":"bc31eefc","executionInfo":{"status":"ok","timestamp":1765027628046,"user_tz":-330,"elapsed":1129,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"e6a4be65-d752-4237-d50d-65cc251fdd07"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Reasoning**:\nThe XGBoost model has been successfully re-trained. Now, I will proceed to re-initialize and train the `model_new_features` (RandomForestRegressor) using the prepared `X_train_new` and `y_train_new` datasets, and then evaluate its performance on `X_val_new` and `y_val_new`.\n\n","metadata":{"id":"932aed2a"}},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor\nfrom sklearn.metrics import mean_absolute_error, r2_score\n\n# Initialize and train the RandomForestRegressor model with new features\nmodel_new_features = RandomForestRegressor(random_state=42)\nmodel_new_features.fit(X_train_new, y_train_new)\n\n# Make predictions on the validation set with new features\ny_pred_new_features = model_new_features.predict(X_val_new)\n\n# Evaluate the model with new features\nmae_new_features = mean_absolute_error(y_val_new, y_pred_new_features)\nr2_new_features = r2_score(y_val_new, y_pred_new_features)\n\nprint(f\"Mean Absolute Error on validation set (with new features): {mae_new_features:.4f}\")\nprint(f\"R-squared on validation set (with new features): {r2_new_features:.4f}\")","metadata":{"id":"cfd22c2b","executionInfo":{"status":"ok","timestamp":1765027640439,"user_tz":-330,"elapsed":3689,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"441f839a-27bd-4d49-bc92-ffda6f515ce6"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Reasoning**:\nThe RandomForestRegressor model has been successfully re-trained. Now, I will proceed to re-train the tuned GradientBoostingRegressor model (`best_gbr_model`) using the prepared `X_train_new` and `y_train_new` datasets, and then evaluate its performance on `X_val_new` and `y_val_new`.\n\n","metadata":{"id":"50b3912c"}},{"cell_type":"code","source":"from sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import r2_score\n\n# Initialize the GradientBoostingRegressor model\ngbr = GradientBoostingRegressor(random_state=42)\n\n# Define the parameter grid for hyperparameter tuning\nparam_grid_gbr = {\n    'n_estimators': [100, 200, 300],\n    'learning_rate': [0.01, 0.05, 0.1],\n    'max_depth': [3, 5, 7],\n    'min_samples_split': [2, 5],\n    'min_samples_leaf': [1, 2]\n}\n\n# Instantiate GridSearchCV\ngrid_search_gbr = GridSearchCV(estimator=gbr, param_grid=param_grid_gbr, cv=3, scoring='r2', n_jobs=-1, verbose=2)\n\n# Fit GridSearchCV to the training data\ngrid_search_gbr.fit(X_train_new, y_train_new)\n\n# Print the best parameters found\nprint(\"Best parameters found by GridSearchCV:\", grid_search_gbr.best_params_)\n\n# Get the best estimator\nbest_gbr_model = grid_search_gbr.best_estimator_\n\n# Make predictions on the validation set using the best estimator\ny_pred_gbr_tuned = best_gbr_model.predict(X_val_new)\n\n# Evaluate the R-squared score\nr2_gbr_tuned = r2_score(y_val_new, y_pred_gbr_tuned)\nprint(f\"GradientBoostingRegressor (tuned) - R-squared on validation set: {r2_gbr_tuned:.4f}\")","metadata":{"id":"077f7545","executionInfo":{"status":"ok","timestamp":1765027890503,"user_tz":-330,"elapsed":241024,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"2c696331-2dc5-442c-897b-0392071b7d32"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Reasoning**:\nThe tuned GradientBoostingRegressor model has been successfully re-trained. Now, I will proceed to re-train the LightGBM and CatBoost models using the prepared `X_train_new` and `y_train_new` datasets, and then evaluate their performance on `X_val_new` and `y_val_new`.\n\n","metadata":{"id":"878f5f65"}},{"cell_type":"code","source":"from lightgbm import LGBMRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.metrics import mean_absolute_error, r2_score\n\n# 1. Initialize LGBMRegressor model\nlgbm_model = LGBMRegressor(random_state=42, verbose=-1) # verbose=-1 to suppress verbose output during training\n\n# 2. Initialize CatBoostRegressor model\ncatboost_model = CatBoostRegressor(random_state=42, verbose=0) # verbose=0 to suppress verbose output during training\n\n# 3. Train LGBMRegressor model\nlgbm_model.fit(X_train_new, y_train_new)\n\n# 4. Train CatBoostRegressor model\ncatboost_model.fit(X_train_new, y_train_new)\n\n# 5. Make predictions on the validation set for LGBMRegressor\ny_pred_lgbm = lgbm_model.predict(X_val_new)\n\n# 6. Make predictions on the validation set for CatBoostRegressor\ny_pred_catboost = catboost_model.predict(X_val_new)\n\n# 7. Evaluate LGBMRegressor model\nmae_lgbm = mean_absolute_error(y_val_new, y_pred_lgbm)\nr2_lgbm = r2_score(y_val_new, y_pred_lgbm)\n\nprint(f\"LGBMRegressor - Mean Absolute Error on validation set: {mae_lgbm:.4f}\")\nprint(f\"LGBMRegressor - R-squared on validation set: {r2_lgbm:.4f}\")\n\n# 8. Evaluate CatBoostRegressor model\nmae_catboost = mean_absolute_error(y_val_new, y_pred_catboost)\nr2_catboost = r2_score(y_val_new, y_pred_catboost)\n\nprint(f\"\\nCatBoostRegressor - Mean Absolute Error on validation set: {mae_catboost:.4f}\")\nprint(f\"CatBoostRegressor - R-squared on validation set: {r2_catboost:.4f}\")","metadata":{"id":"a86205fa","executionInfo":{"status":"error","timestamp":1765027933367,"user_tz":-330,"elapsed":14315,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"83f34e7f-a58f-41da-dfc5-cac82db96c47"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Reasoning**:\nThe previous cell failed due to a `ModuleNotFoundError` for `catboost`. I need to install `catboost` using `!pip install` before I can import and use it.\n\n","metadata":{"id":"ec5202f8"}},{"cell_type":"code","source":"pip install catboost","metadata":{"id":"90e41932","executionInfo":{"status":"ok","timestamp":1765027979999,"user_tz":-330,"elapsed":21905,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"8be27096-4c92-4d82-e0dd-d179272d4d37"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Reasoning**:\nNow that `catboost` is successfully installed, I will re-execute the code block to train and evaluate the LightGBM and CatBoost models, as per the subtask instructions.\n\n","metadata":{"id":"07dc0857"}},{"cell_type":"code","source":"from lightgbm import LGBMRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.metrics import mean_absolute_error, r2_score\n\n# 1. Initialize LGBMRegressor model\nlgbm_model = LGBMRegressor(random_state=42, verbose=-1) # verbose=-1 to suppress verbose output during training\n\n# 2. Initialize CatBoostRegressor model\ncatboost_model = CatBoostRegressor(random_state=42, verbose=0) # verbose=0 to suppress verbose output during training\n\n# 3. Train LGBMRegressor model\nlgbm_model.fit(X_train_new, y_train_new)\n\n# 4. Train CatBoostRegressor model\ncatboost_model.fit(X_train_new, y_train_new)\n\n# 5. Make predictions on the validation set for LGBMRegressor\ny_pred_lgbm = lgbm_model.predict(X_val_new)\n\n# 6. Make predictions on the validation set for CatBoostRegressor\ny_pred_catboost = catboost_model.predict(X_val_new)\n\n# 7. Evaluate LGBMRegressor model\nmae_lgbm = mean_absolute_error(y_val_new, y_pred_lgbm)\nr2_lgbm = r2_score(y_val_new, y_pred_lgbm)\n\nprint(f\"LGBMRegressor - Mean Absolute Error on validation set: {mae_lgbm:.4f}\")\nprint(f\"LGBMRegressor - R-squared on validation set: {r2_lgbm:.4f}\")\n\n# 8. Evaluate CatBoostRegressor model\nmae_catboost = mean_absolute_error(y_val_new, y_pred_catboost)\nr2_catboost = r2_score(y_val_new, y_pred_catboost)\n\nprint(f\"\\nCatBoostRegressor - Mean Absolute Error on validation set: {mae_catboost:.4f}\")\nprint(f\"CatBoostRegressor - R-squared on validation set: {r2_catboost:.4f}\")","metadata":{"id":"d3a7d5bb","executionInfo":{"status":"ok","timestamp":1765027999771,"user_tz":-330,"elapsed":6770,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"bcb6d08b-384b-48d9-b618-156f12214ce7"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Generate submission file\n\n### Subtask:\nGenerate the `submission.csv` file using the ensemble predictions from the re-trained models on `test_df_processed`.\n","metadata":{"id":"aad3da4e"}},{"cell_type":"markdown","source":"**Reasoning**:\nAll individual models have been re-trained and the `test_df_processed` is prepared. Now, I will generate predictions from each model on `test_df_processed`, calculate their average for ensemble predictions, ensure non-negative target values, and then create and save the `submission.csv` file.\n\n","metadata":{"id":"5fbfd288"}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\n# Ensure all individual model predictions are generated for test_df_processed\ny_pred_gbr_test = best_gbr_model.predict(test_df_processed)\ny_pred_lgbm_test = lgbm_model.predict(test_df_processed)\ny_pred_catboost_test = catboost_model.predict(test_df_processed)\ny_pred_xgb_test = xgb_model.predict(test_df_processed)\ny_pred_rf_test = model_new_features.predict(test_df_processed)\n\n# Calculate the average of the individual model predictions\ny_pred_ensemble_avg_test = (y_pred_gbr_test + y_pred_lgbm_test + y_pred_catboost_test + y_pred_xgb_test + y_pred_rf_test) / 5\n\nsubmission_df = pd.DataFrame({\n    'image_id': test_df['sample_id'],\n    'target': y_pred_ensemble_avg_test\n})\n\n# Ensure the target values are non-negative, as biomass cannot be negative\nsubmission_df['target'] = submission_df['target'].apply(lambda x: max(0, x))\n\nsubmission_df.to_csv('submission.csv', index=False)\n\nprint('Submission file \"submission.csv\" created successfully.')","metadata":{"id":"893ce6c9","executionInfo":{"status":"ok","timestamp":1765028028120,"user_tz":-330,"elapsed":97,"user":{"displayName":"Er.Rajani Kant Awasthi","userId":"07722847757828636185"}},"outputId":"56ce2220-0d41-4f28-fcd9-1e1c349e3286"},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Download submission file\n\n### Subtask:\nDownload the `submission.csv` file.\n","metadata":{"id":"8ddc9b90"}},{"cell_type":"markdown","source":"## Summary:\n\n### Data Analysis Key Findings\n\n*   **Successful Preprocessing:** The `KeyError` and subsequent `ValueError: Input X contains NaN` during feature engineering were resolved by explicitly merging `Pre_GSHH_NDVI` and `Height_Ave_cm` into `test_df_processed` and then imputing missing values in these columns with the mean from the training data. This ensured the `test_df_processed` was correctly prepared for model inference.\n*   **Feature Engineering Impact:** A comprehensive set of features, including date components (year, month, day, dayofweek, dayofyear), interaction terms (NDVI\\_Height\\_Interaction), polynomial features for numerical inputs, and aggregated statistics (mean, median, std) by `encoded_State` and `encoded_Species`, were successfully engineered and scaled.\n*   **Model Performance Overview:**\n    *   **XGBoost Regressor:** Achieved a Mean Absolute Error (MAE) of 18.8002 and an R-squared of 0.1041 on the validation set.\n    *   **RandomForest Regressor:** Achieved an MAE of 18.4390 and an R-squared of 0.1363 on the validation set.\n    *   **GradientBoosting Regressor (tuned):** Showed the best performance among individual models, with an R-squared of 0.3180 on the validation set after hyperparameter tuning. The best parameters identified were `{'learning_rate': 0.01, 'max_depth': 3, 'min_samples_leaf': 2, 'min_samples_split': 2, 'n_estimators': 200}\\}$.\n    *   **LGBM Regressor:** Achieved an MAE of 17.7700 and an R-squared of 0.1916 on the validation set.\n    *   **CatBoost Regressor:** Achieved an MAE of 17.9370 and an R-squared of 0.1784 on the validation set.\n*   **Ensemble Prediction:** An ensemble approach, averaging predictions from the five re-trained models (XGBoost, RandomForest, tuned GradientBoosting, LightGBM, and CatBoost), was used to generate the final `submission.csv` file. All target values in the submission were ensured to be non-negative.\n\n### Insights or Next Steps\n\n*   **Optimize Ensemble Weighting:** Given the varying R-squared scores, exploring weighted averaging or more sophisticated stacking/blending techniques for the ensemble could potentially improve the final prediction accuracy beyond a simple average.\n*   **Further Hyperparameter Tuning:** While GradientBoosting was tuned, extending hyperparameter optimization to other models (like LGBM and CatBoost) could yield better individual model performance, which would then benefit the ensemble.\n","metadata":{"id":"b62eb040"}}]}