{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":628057,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":472619,"modelId":488505}],"dockerImageVersionId":31154,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !pip download -d packages transformers\n!pip install -q -U transformers --no-index -f /kaggle/input/metadino-v3-convnext/pytorch/default/3/packages\n# !pip install -q -U transformers xgboost opencv-python 'numpy<2.0' 'pandas>=2.2'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-11-03T17:37:07.684421Z","iopub.execute_input":"2025-11-03T17:37:07.684763Z","iopub.status.idle":"2025-11-03T17:37:32.563777Z","shell.execute_reply.started":"2025-11-03T17:37:07.684733Z","shell.execute_reply":"2025-11-03T17:37:32.561939Z"},"_kg_hide-output":true,"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import huggingface_hub\n# huggingface_hub.login()","metadata":{"trusted":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-11-03T17:37:32.566965Z","iopub.execute_input":"2025-11-03T17:37:32.567301Z","iopub.status.idle":"2025-11-03T17:37:32.573689Z","shell.execute_reply.started":"2025-11-03T17:37:32.567272Z","shell.execute_reply":"2025-11-03T17:37:32.572361Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport glob\nimport random\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nfrom tqdm.auto import tqdm\n\nimport torch\nimport warnings\n\nwarnings.simplefilter(action='ignore', category=FutureWarning)\nprint(f\"PyTorch: {torch.__version__}\")\nprint(f\"Device: {torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'CPU'}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T17:37:32.575296Z","iopub.execute_input":"2025-11-03T17:37:32.575661Z","iopub.status.idle":"2025-11-03T17:37:40.277534Z","shell.execute_reply.started":"2025-11-03T17:37:32.575627Z","shell.execute_reply":"2025-11-03T17:37:40.275581Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Explore Data","metadata":{}},{"cell_type":"code","source":"PATH_DATA = '/kaggle/input/csiro-biomass'\nPATH_TRAIN_CSV = os.path.join(PATH_DATA, 'train.csv')\nPATH_TRAIN_IMG = os.path.join(PATH_DATA, 'train')\nPATH_TEST_IMG = os.path.join(PATH_DATA, 'test')\n\ndf = pd.read_csv(PATH_TRAIN_CSV)\nprint(f\"Dataset size: {df.shape}\")\ndisplay(df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T17:37:40.278818Z","iopub.execute_input":"2025-11-03T17:37:40.279352Z","iopub.status.idle":"2025-11-03T17:37:40.355711Z","shell.execute_reply.started":"2025-11-03T17:37:40.279291Z","shell.execute_reply":"2025-11-03T17:37:40.353883Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"TARGET_COLS = [c for c in df.columns if c not in ['image_id', 'Image']]\nprint(f\"Target columns: {TARGET_COLS}\")\nprint(f\"Number of targets: {len(TARGET_COLS)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T17:37:40.362371Z","iopub.execute_input":"2025-11-03T17:37:40.362751Z","iopub.status.idle":"2025-11-03T17:37:40.373244Z","shell.execute_reply.started":"2025-11-03T17:37:40.362726Z","shell.execute_reply":"2025-11-03T17:37:40.371908Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Plot target distribution","metadata":{}},{"cell_type":"code","source":"# Exclude non-numeric or identifier columns from histogram plotting\ncols_to_plot = [col for col in TARGET_COLS if col not in ['sample_id', 'image_path', 'State', 'target_name']]\n\nfor col in cols_to_plot:\n    plt.figure(figsize=(8, 3)) # Create a new figure for each histogram\n    plt.hist(df[col].dropna(), bins=50, edgecolor='black', alpha=0.7)\n    plt.xlabel(col, fontsize=12)\n    plt.ylabel('Count', fontsize=12)\n    plt.title(f'{col} Distribution', fontsize=14, fontweight='bold')\n    plt.grid(alpha=0.3)\n    plt.xticks(rotation=45, ha=\"right\") # Rotate x-axis labels\n    plt.tight_layout() # Adjust layout to prevent overlap\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T17:37:40.374517Z","iopub.execute_input":"2025-11-03T17:37:40.375173Z","iopub.status.idle":"2025-11-03T17:37:42.597119Z","shell.execute_reply.started":"2025-11-03T17:37:40.375055Z","shell.execute_reply":"2025-11-03T17:37:42.595852Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cols_to_plot = ['State', 'target_name']\nn_rows, n_cols = 1, len(cols_to_plot)\nfig, axes = plt.subplots(n_rows, n_cols, figsize=(4 * n_cols, 4 * n_rows))\n\n# Ensure axes is an array even for a single subplot\naxes = axes.flatten()\n\nfor ax, col in zip(axes, cols_to_plot):\n    counts = df[col].value_counts()\n    ax.pie(counts, labels=counts.index, autopct='%1.1f%%', startangle=140)\n    ax.set_title(f'Distribution of {col}', fontsize=14, fontweight='bold')\n    ax.axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle.\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T17:37:42.598259Z","iopub.execute_input":"2025-11-03T17:37:42.598598Z","iopub.status.idle":"2025-11-03T17:37:42.882840Z","shell.execute_reply.started":"2025-11-03T17:37:42.598574Z","shell.execute_reply":"2025-11-03T17:37:42.881468Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\ngrouped_train_data = df.groupby('target_name')\n\nplt.figure(figsize=(12, 8))\nfor target_name, group_data in grouped_train_data:\n    sns.histplot(data=group_data, x='target', kde=True, label=target_name)\n\nplt.title('Distribution of Target for Each Target Name Class')\nplt.xlabel('Target')\nplt.ylabel('Frequency')\nplt.legend(title='Target Name')\nplt.grid(True) # Added grid here\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T17:37:42.884017Z","iopub.execute_input":"2025-11-03T17:37:42.884508Z","iopub.status.idle":"2025-11-03T17:37:45.253023Z","shell.execute_reply.started":"2025-11-03T17:37:42.884481Z","shell.execute_reply":"2025-11-03T17:37:45.251246Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Show sample images","metadata":{}},{"cell_type":"code","source":"def show_images(df_sample, n=12, path_img=PATH_DATA):\n    \"\"\"Displays a linear sampling of images sorted by target value.\"\"\"\n\n    # Sort the DataFrame by the 'target' column\n    df_sorted = df_sample.sort_values(by='target').reset_index(drop=True)\n\n    # Perform linear sampling\n    indices_to_show = np.linspace(0, len(df_sorted) - 1, n, dtype=int)\n    df_to_show = df_sorted.iloc[indices_to_show]\n\n    # Determine the number of rows and columns for subplots\n    n_cols = 3  # You can adjust this number\n    n_rows = (n + n_cols - 1) // n_cols\n\n    fig, axes = plt.subplots(n_rows, n_cols, figsize=(16, 4 * n_rows))\n    axes = axes.flatten()\n\n    # Remove unused subplots if any\n    for i in range(n, len(axes)):\n        fig.delaxes(axes[i])\n\n    for i, (idx, row) in enumerate(df_to_show.iterrows()):\n        # Use image_path directly (includes train/ID....jpg)\n        img_path = os.path.join(path_img, row['image_path'])\n\n        if os.path.exists(img_path):\n            img = Image.open(img_path).convert('RGB')\n            axes[i].imshow(img)\n            # Include the target value in the title\n            title = f\"ID: {row['sample_id']}\\nTarget: {row['target']:.2f}\"\n            axes[i].set_title(title, fontsize=10)\n        axes[i].axis('off')\n\n    plt.tight_layout()\n    plt.show()\n\n# Example usage: Show 12 images linearly sampled based on target value\nshow_images(df, n=12)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T17:37:45.253933Z","iopub.execute_input":"2025-11-03T17:37:45.254584Z","iopub.status.idle":"2025-11-03T17:37:52.236166Z","shell.execute_reply.started":"2025-11-03T17:37:45.254553Z","shell.execute_reply":"2025-11-03T17:37:52.233800Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature extraction","metadata":{}},{"cell_type":"code","source":"from transformers import pipeline\n\n# Define the feature extraction pipeline\nfeature_extractor = pipeline(\n    # model=\"facebook/dinov3-convnext-tiny-pretrain-lvd1689m\",\n    model=\"/kaggle/input/metadino-v3-convnext/pytorch/default/3/dinov3-convnext-tiny-pretrain-lvd1689m\",\n    task=\"image-feature-extraction\",\n    device=0 if torch.cuda.is_available() else -1 # Use GPU if available\n)\n\ndef extract_image_features_pipeline(image_path, feature_extractor, path_data):\n    \"\"\"\n    Splits an image into two 1000x1000 parts and extracts features from each using a pipeline.\n    \"\"\"\n    full_image_path = os.path.join(path_data, image_path)\n    img = Image.open(full_image_path)#.convert('RGB')\n\n    # Split the image into two 1000x1000 parts\n    width, height = img.size\n    img1 = img.crop((0, 0, width // 2, height))\n    img2 = img.crop((width // 2, 0, width, height))\n\n    # The pipeline expects PIL Images or paths, use 'inputs'\n    extracted_features = feature_extractor(inputs=[img1, img2], pool=True)\n    # extracted_features = feature_extractor(inputs=[img], pool=True)\n    # print(f\"extracted_features: {np.array(extracted_features).shape}\")\n\n    # Concatenate features from both parts\n    combined_features = list(np.array(extracted_features).flatten())\n    # print(f\"combined_features: {len(combined_features)}\")\n\n    return combined_features\n\nprint(\"Image feature extraction function defined using pipeline.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T17:37:52.238546Z","iopub.execute_input":"2025-11-03T17:37:52.239128Z","iopub.status.idle":"2025-11-03T17:38:27.479982Z","shell.execute_reply.started":"2025-11-03T17:37:52.239075Z","shell.execute_reply":"2025-11-03T17:38:27.478640Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Extract features for all images\nimage_features = {}\nfor index, row in tqdm(df.iterrows(), total=df.shape[0]):\n    image_path = row['image_path']\n    # Use the pipeline function\n    features = extract_image_features_pipeline(image_path, feature_extractor, PATH_DATA)\n    # Use the sample_id as the key for the features\n    image_features[row['sample_id']] = features\n\nprint(\"Image feature extraction complete.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T17:38:27.481747Z","iopub.execute_input":"2025-11-03T17:38:27.482607Z","iopub.status.idle":"2025-11-03T17:50:30.472884Z","shell.execute_reply.started":"2025-11-03T17:38:27.482574Z","shell.execute_reply":"2025-11-03T17:50:30.471920Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Convert the image features dictionary to a DataFrame\nimage_features_df = pd.DataFrame.from_dict(image_features, orient='index')\nimage_features_df.index.name = 'sample_id'\nimage_features_df.columns = [f'img_feature_{i}' for i in range(image_features_df.shape[1])]\n\n# One-hot encode the 'target_name' column\ndf_one_hot = pd.get_dummies(df['target_name'], prefix='target_name').astype(int) # Convert to int\n\n# Merge the image features and one-hot encoded features with the original DataFrame\ndf_combined = df.merge(image_features_df, on='sample_id', how='left')\ndf_combined = df_combined.merge(df_one_hot, left_index=True, right_index=True)\n\nprint(\"Image features and one-hot encoded target names combined with the original DataFrame.\")\ndisplay(df_combined.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T17:50:30.473948Z","iopub.execute_input":"2025-11-03T17:50:30.474294Z","iopub.status.idle":"2025-11-03T17:50:31.410863Z","shell.execute_reply.started":"2025-11-03T17:50:30.474261Z","shell.execute_reply":"2025-11-03T17:50:31.409949Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Flatten all image features into a single Series\nimage_features_cols = [col for col in df_combined.columns if col.startswith(\"img_feature_\")]\nall_features_flat = image_features_df[image_features_cols].values.flatten()\nall_features_series = pd.Series(all_features_flat)\n\n# Plot a single histogram for all features\nplt.figure(figsize=(10, 6))\nsns.histplot(all_features_series, kde=True, bins=100) # Adjust bins as needed\nplt.title('Distribution of All Extracted Image Feature Values')\nplt.xlabel('Feature Value')\nplt.ylabel('Frequency')\nplt.grid(True, alpha=0.3)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T17:50:31.411975Z","iopub.execute_input":"2025-11-03T17:50:31.412433Z","iopub.status.idle":"2025-11-03T17:50:44.697462Z","shell.execute_reply.started":"2025-11-03T17:50:31.412409Z","shell.execute_reply":"2025-11-03T17:50:44.696033Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train Regressor","metadata":{}},{"cell_type":"code","source":"from sklearn.decomposition import PCA\nfrom sklearn.preprocessing import StandardScaler\n\n# Separate image features and other features\nimage_feature_cols = [col for col in df_combined.columns if col.startswith('img_feature_')]\nother_features_cols = [col for col in df_combined.columns if col.startswith('target_name_')]\n\nX_image_features = df_combined[image_feature_cols]\nX_other_features = df_combined[other_features_cols]\n\n# Standardize the image features before applying PCA\nscaler = StandardScaler()\nX_image_scaled = scaler.fit_transform(X_image_features)\n\n# Apply PCA to reduce dimensionality\n# You can adjust the number of components (n_components)\npca = PCA(n_components=75) # Example: Reduce components by 20x\nX_image_pca = pca.fit_transform(X_image_scaled)\n\n# Convert the PCA reduced features to a DataFrame\nX_image_pca_df = pd.DataFrame(\n    X_image_pca, index=df_combined.index,\n    columns=[f'pca_img_feature_{i}' for i in range(pca.n_components)])\n\n# Combine the PCA reduced image features with the other features\nX_combined_pca = pd.concat([X_other_features, X_image_pca_df], axis=1)\n\nprint(f\"Original number of image features: {X_image_features.shape[1]}\")\nprint(f\"Reduced number of image features after PCA: {X_image_pca_df.shape[1]}\")\nprint(f\"Total number of features for regression after PCA: {X_combined_pca.shape[1]}\")\n\n# Display the first few rows of the combined feature DataFrame after PCA\ndisplay(X_combined_pca.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T17:50:44.701360Z","iopub.execute_input":"2025-11-03T17:50:44.701673Z","iopub.status.idle":"2025-11-03T17:50:45.830033Z","shell.execute_reply.started":"2025-11-03T17:50:44.701652Z","shell.execute_reply":"2025-11-03T17:50:45.829077Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import xgboost as xgb\nfrom sklearn.model_selection import train_test_split, RandomizedSearchCV\nfrom sklearn.metrics import mean_squared_error\nfrom scipy.stats import randint, uniform, loguniform\nimport numpy as np\n\n# Define features (X) and target (y) using the PCA-reduced image features\n# Exclude identifier columns and the original target column\n# Use X_combined_pca which contains the one-hot encoded target_name and PCA image features\nfeatures = X_combined_pca.columns.tolist() # Use all columns from X_combined_pca\nprint(f\"training features: {features}\")\ntarget = 'target'\n\nX = X_combined_pca\ny = df_combined[target] # Target remains the same\n\n# Split data into training and testing sets\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# Initialize and train the XGBoost Regressor\n# The features include binary (from one-hot encoding) and float types.\n# XGBoost can handle this mix of feature types effectively.\n\n# Define a distribution for hyperparameters to sample from\n# Using distributions allows for linear or log-uniform (experimental/exponential) sampling\nparam_dist = {\n    'n_estimators': randint(100, 1000),\n    'learning_rate': loguniform(0.001, 0.2),        # Log-uniform sampling\n    'max_depth': randint(3, 11),\n    'colsample_bytree': uniform(0.5, 0.5),          # Uniform sampling [0.5, 1.0]\n    'reg_alpha': loguniform(1e-4, 10.0),            # Log-uniform sampling\n    'reg_lambda': loguniform(1.0, 10.0)             # Log-uniform sampling\n}\n\nxgb_model = xgb.XGBRegressor(objective='reg:squarederror', random_state=42)\n\n# Use RandomizedSearchCV to find the best parameters\n# n_iter controls the number of parameter settings that are sampled\nrandom_search = RandomizedSearchCV(\n    estimator=xgb_model,\n    param_distributions=param_dist,\n    n_iter=1200,\n    scoring='neg_mean_squared_error',\n    cv=3,\n    verbose=2,\n    random_state=42,\n    n_jobs=-1,\n)\n\nrandom_search.fit(X_train, y_train)\n\n# Get the best parameters and the best model\nbest_params = random_search.best_params_\nxgb_model = random_search.best_estimator_\n\nprint(\"Randomized search complete.\")\nprint(f\"Best parameters found: {best_params}\")\nprint(\"XGBoost model training complete with best parameters.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T18:35:37.189937Z","iopub.execute_input":"2025-11-03T18:35:37.190469Z","iopub.status.idle":"2025-11-03T18:36:11.941160Z","shell.execute_reply.started":"2025-11-03T18:35:37.190343Z","shell.execute_reply":"2025-11-03T18:36:11.940088Z"},"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Get feature importances from the best model found by GridSearchCV\n# If you didn't use GridSearchCV, use xgb_model.feature_importances_\nfeature_importances = xgb_model.feature_importances_\n\n# Create a pandas Series for easier handling and sorting\nfeature_importances_series = pd.Series(feature_importances, index=X_train.columns)\n\n# Sort features by importance\nsorted_feature_importances = feature_importances_series.sort_values(ascending=False)\n\n# Plot the top N feature importances (e.g., top 20)\nn_top_features = 20\nplt.figure(figsize=(10, 8))\nsns.barplot(x=sorted_feature_importances.head(n_top_features).values,\n            y=sorted_feature_importances.head(n_top_features).index,\n            palette='viridis')\nplt.title(f'Top {n_top_features} Feature Importances (XGBoost)', fontsize=14)\nplt.xlabel('Importance', fontsize=12)\nplt.ylabel('Feature', fontsize=12)\nplt.tight_layout()\nplt.show()\n\n# You can also display the sorted feature importances\nprint(\"Sorted Feature Importances:\")\ndisplay(sorted_feature_importances)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T18:37:43.688003Z","iopub.execute_input":"2025-11-03T18:37:43.688437Z","iopub.status.idle":"2025-11-03T18:37:44.090712Z","shell.execute_reply.started":"2025-11-03T18:37:43.688409Z","shell.execute_reply":"2025-11-03T18:37:44.089570Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Make predictions on the test data\ny_pred = xgb_model.predict(X_test)\n\nprint(\"Predictions on test data complete.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T18:37:48.232274Z","iopub.execute_input":"2025-11-03T18:37:48.233465Z","iopub.status.idle":"2025-11-03T18:37:48.250759Z","shell.execute_reply.started":"2025-11-03T18:37:48.233429Z","shell.execute_reply":"2025-11-03T18:37:48.249421Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import mean_absolute_error, r2_score\n\n# Calculate Root Mean Squared Error (RMSE)\nrmse = np.sqrt(mean_squared_error(y_test, y_pred))\nprint(f\"Root Mean Squared Error (RMSE) on the test set: {rmse}\")\n\n# Calculate Mean Absolute Error (MAE)\nmae = mean_absolute_error(y_test, y_pred)\nprint(f\"Mean Absolute Error (MAE) on the test set: {mae}\")\n\n# Calculate R-squared\nr2 = r2_score(y_test, y_pred)\nprint(f\"R-squared on the test set: {r2}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T18:37:51.485298Z","iopub.execute_input":"2025-11-03T18:37:51.486280Z","iopub.status.idle":"2025-11-03T18:37:51.495162Z","shell.execute_reply.started":"2025-11-03T18:37:51.486247Z","shell.execute_reply":"2025-11-03T18:37:51.494107Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Test predictions","metadata":{}},{"cell_type":"code","source":"# 1. Load test data\nPATH_TEST_CSV = os.path.join(PATH_DATA, 'test.csv')\ndf_test = pd.read_csv(PATH_TEST_CSV)\nprint(f\"Test Dataset size: {df_test.shape}\")\ndisplay(df_test.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T18:37:53.257471Z","iopub.execute_input":"2025-11-03T18:37:53.257765Z","iopub.status.idle":"2025-11-03T18:37:53.280367Z","shell.execute_reply.started":"2025-11-03T18:37:53.257747Z","shell.execute_reply":"2025-11-03T18:37:53.279090Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 2. Extract test image features\ntest_image_features = {}\nfor index, row in tqdm(df_test.iterrows(), total=df_test.shape[0]):\n    image_path = row['image_path']\n    # Use the pipeline function\n    features = extract_image_features_pipeline(image_path, feature_extractor, PATH_DATA)\n    # Use the sample_id as the key for the features\n    test_image_features[row['sample_id']] = features\n\nprint(\"Test Image feature extraction complete.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T18:37:55.732649Z","iopub.execute_input":"2025-11-03T18:37:55.732997Z","iopub.status.idle":"2025-11-03T18:37:57.829946Z","shell.execute_reply.started":"2025-11-03T18:37:55.732974Z","shell.execute_reply":"2025-11-03T18:37:57.828787Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 3. Combine test features\ntest_image_features_df = pd.DataFrame.from_dict(test_image_features, orient='index')\ntest_image_features_df.index.name = 'sample_id'\ntest_image_features_df.columns = [f'img_feature_{i}' for i in range(test_image_features_df.shape[1])]\n\n# One-hot encode the 'target_name' column for the test set\ntest_df_one_hot = pd.get_dummies(df_test['target_name'], prefix='target_name').astype(int)\n\n# Merge the test image features and one-hot encoded features with the original test DataFrame\ndf_test_combined = df_test.merge(test_image_features_df, on='sample_id', how='left')\ndf_test_combined = df_test_combined.merge(test_df_one_hot, left_index=True, right_index=True)\n\nprint(\"Test image features and one-hot encoded target names combined.\")\ndisplay(df_test_combined.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T18:38:01.534573Z","iopub.execute_input":"2025-11-03T18:38:01.534910Z","iopub.status.idle":"2025-11-03T18:38:01.588950Z","shell.execute_reply.started":"2025-11-03T18:38:01.534885Z","shell.execute_reply":"2025-11-03T18:38:01.587873Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 4. Scale and PCA transform test image features\n# Separate image features and other features for the test set\ntest_image_feature_cols = [col for col in df_test_combined.columns if col.startswith('img_feature_')]\ntest_other_features_cols = [col for col in df_test_combined.columns if col.startswith('target_name_')]\n\nX_test_image_features = df_test_combined[test_image_feature_cols]\nX_test_other_features = df_test_combined[test_other_features_cols]\n\n# Apply the *trained* scaler and PCA to the test image features\nX_test_image_scaled = scaler.transform(X_test_image_features) # Use the fitted scaler\nX_test_image_pca = pca.transform(X_test_image_scaled)       # Use the fitted pca\n\n# Convert the PCA reduced features to a DataFrame\nX_test_image_pca_df = pd.DataFrame(\n    X_test_image_pca, index=df_test_combined.index,\n    columns=[f'pca_img_feature_{i}' for i in range(pca.n_components)])\n\nprint(f\"Scaled and PCA-transformed test image features shape: {X_test_image_pca_df.shape}\")\n\n# Display the first few rows of the PCA reduced test image features\ndisplay(X_test_image_pca_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T18:38:04.217211Z","iopub.execute_input":"2025-11-03T18:38:04.218006Z","iopub.status.idle":"2025-11-03T18:38:04.287422Z","shell.execute_reply.started":"2025-11-03T18:38:04.217967Z","shell.execute_reply":"2025-11-03T18:38:04.285875Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 5. Prepare test data for inference\n# Combine the PCA reduced image features with the other features (one-hot encoded target_name)\nX_test_final = pd.concat([X_test_other_features, X_test_image_pca_df], axis=1)\n\nprint(f\"Final test feature DataFrame shape: {X_test_final.shape}\")\n\n# Display the first few rows of the final test feature DataFrame\ndisplay(X_test_final.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T18:38:07.879713Z","iopub.execute_input":"2025-11-03T18:38:07.880518Z","iopub.status.idle":"2025-11-03T18:38:07.902637Z","shell.execute_reply.started":"2025-11-03T18:38:07.880485Z","shell.execute_reply":"2025-11-03T18:38:07.901606Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 6. Make predictions\ntest_predictions = xgb_model.predict(X_test_final)\n\nprint(\"Predictions on test data complete.\")\ndisplay(test_predictions)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T18:38:10.452634Z","iopub.execute_input":"2025-11-03T18:38:10.452964Z","iopub.status.idle":"2025-11-03T18:38:10.471516Z","shell.execute_reply.started":"2025-11-03T18:38:10.452929Z","shell.execute_reply":"2025-11-03T18:38:10.470414Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Prepare submission","metadata":{}},{"cell_type":"code","source":"# 7. Generate submission file\nsubmission_df = pd.DataFrame({'sample_id': df_test['sample_id'], 'target': test_predictions})\n\n# Ensure predictions are non-negative if the target represents a physical quantity\nsubmission_df['target'] = submission_df['target'].clip(lower=0)\n\nsubmission_path = 'submission.csv'\nsubmission_df.to_csv(submission_path, index=False)\n\nprint(f\"Submission file generated at: {submission_path}\")\ndisplay(submission_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T18:38:12.581101Z","iopub.execute_input":"2025-11-03T18:38:12.581453Z","iopub.status.idle":"2025-11-03T18:38:12.605459Z","shell.execute_reply.started":"2025-11-03T18:38:12.581429Z","shell.execute_reply":"2025-11-03T18:38:12.604377Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!head submission.csv","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T18:38:15.239444Z","iopub.execute_input":"2025-11-03T18:38:15.239776Z","iopub.status.idle":"2025-11-03T18:38:15.447566Z","shell.execute_reply.started":"2025-11-03T18:38:15.239754Z","shell.execute_reply":"2025-11-03T18:38:15.446139Z"}},"outputs":[],"execution_count":null}]}