{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":4534,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":3326,"modelId":986}],"dockerImageVersionId":31153,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"I just swapped Dino v3 for Dino v2 as it is freely available and added caching while creating image features for faster processing. Thanks to @jirkaborovec for his original work [here](https://www.kaggle.com/code/jirkaborovec/csiro-img2bio-eda-xgboost-pca-image-features)","metadata":{}},{"cell_type":"code","source":"import os\nimport glob\nimport random\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nfrom tqdm.auto import tqdm\n\nimport torch\nimport warnings\n\nwarnings.simplefilter(action='ignore', category=FutureWarning)\nprint(f\"PyTorch: {torch.__version__}\")\nprint(f\"Device: {torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'CPU'}\")","metadata":{"trusted":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-11-03T15:52:40.136654Z","iopub.execute_input":"2025-11-03T15:52:40.136901Z","iopub.status.idle":"2025-11-03T15:52:45.955138Z","shell.execute_reply.started":"2025-11-03T15:52:40.136881Z","shell.execute_reply":"2025-11-03T15:52:45.954134Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load Data","metadata":{}},{"cell_type":"code","source":"PATH_DATA = '/kaggle/input/csiro-biomass'\nPATH_TRAIN_CSV = os.path.join(PATH_DATA, 'train.csv')\nPATH_TRAIN_IMG = os.path.join(PATH_DATA, 'train')\nPATH_TEST_IMG = os.path.join(PATH_DATA, 'test')\n\ndf = pd.read_csv(PATH_TRAIN_CSV)\nprint(f\"Dataset size: {df.shape}\")\ndisplay(df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T15:52:45.956482Z","iopub.execute_input":"2025-11-03T15:52:45.956863Z","iopub.status.idle":"2025-11-03T15:52:46.002930Z","shell.execute_reply.started":"2025-11-03T15:52:45.956832Z","shell.execute_reply":"2025-11-03T15:52:46.002280Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"TARGET_COLS = [c for c in df.columns if c not in ['image_id', 'Image']]\nprint(f\"Target columns: {TARGET_COLS}\")\nprint(f\"Number of targets: {len(TARGET_COLS)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T15:52:46.003628Z","iopub.execute_input":"2025-11-03T15:52:46.003807Z","iopub.status.idle":"2025-11-03T15:52:46.008554Z","shell.execute_reply.started":"2025-11-03T15:52:46.003792Z","shell.execute_reply":"2025-11-03T15:52:46.007702Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature extraction","metadata":{}},{"cell_type":"code","source":"from transformers import pipeline\n\n# Define the feature extraction pipeline\nfeature_extractor = pipeline(\n    # try \"facebook/dinov3\" model\n    model=\"/kaggle/input/dinov2/pytorch/base/1\",\n    task=\"image-feature-extraction\",\n    device=0 if torch.cuda.is_available() else -1 # Use GPU if available\n)\n\ndef extract_image_features_pipeline(image_path, feature_extractor, path_data):\n    \"\"\"\n    Splits an image into two 1000x1000 parts and extracts features from each using a pipeline.\n    \"\"\"\n    full_image_path = os.path.join(path_data, image_path)\n    img = Image.open(full_image_path)#.convert('RGB')\n\n    # Split the image into two 1000x1000 parts\n    width, height = img.size\n    img1 = img.crop((0, 0, width // 2, height))\n    img2 = img.crop((width // 2, 0, width, height))\n\n    # The pipeline expects PIL Images or paths, use 'inputs'\n    extracted_features = feature_extractor(inputs=[img1, img2], pool=True)\n    # extracted_features = feature_extractor(inputs=[img], pool=True)\n    # print(f\"extracted_features: {np.array(extracted_features).shape}\")\n\n    # Concatenate features from both parts\n    combined_features = list(np.array(extracted_features).flatten())\n    # print(f\"combined_features: {len(combined_features)}\")\n\n    return combined_features\n\nprint(\"Image feature extraction function defined using pipeline.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T15:52:46.009432Z","iopub.execute_input":"2025-11-03T15:52:46.009688Z","iopub.status.idle":"2025-11-03T15:53:16.409311Z","shell.execute_reply.started":"2025-11-03T15:52:46.009659Z","shell.execute_reply":"2025-11-03T15:53:16.408313Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tqdm import tqdm\n\n# Dictionary to cache image features by image path\nimage_cache = {}\nimage_features = {}\n\nfor index, row in tqdm(df.iterrows(), total=df.shape[0]):\n    image_path = row['image_path']\n    \n    # Check if features for this image_path are already cached\n    if image_path in image_cache:\n        features = image_cache[image_path]\n    else:\n        # Compute features only once per unique image\n        features = extract_image_features_pipeline(image_path, feature_extractor, PATH_DATA)\n        image_cache[image_path] = features  # Cache the result\n    \n    # Store by sample_id (or whatever your unique identifier is)\n    image_features[row['sample_id']] = features\n\nprint(f\"Image feature extraction complete. Cached {len(image_cache)} unique images.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T15:53:16.411758Z","iopub.execute_input":"2025-11-03T15:53:16.412009Z","iopub.status.idle":"2025-11-03T15:54:18.757924Z","shell.execute_reply.started":"2025-11-03T15:53:16.411993Z","shell.execute_reply":"2025-11-03T15:54:18.757051Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Convert the image features dictionary to a DataFrame\nimage_features_df = pd.DataFrame.from_dict(image_features, orient='index')\nimage_features_df.index.name = 'sample_id'\nimage_features_df.columns = [f'img_feature_{i}' for i in range(image_features_df.shape[1])]\n\n# One-hot encode the 'target_name' column\ndf_one_hot = pd.get_dummies(df['target_name'], prefix='target_name').astype(int) # Convert to int\n\n# Merge the image features and one-hot encoded features with the original DataFrame\ndf_combined = df.merge(image_features_df, on='sample_id', how='left')\ndf_combined = df_combined.merge(df_one_hot, left_index=True, right_index=True)\n\nprint(\"Image features and one-hot encoded target names combined with the original DataFrame.\")\ndisplay(df_combined.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T15:54:18.758851Z","iopub.execute_input":"2025-11-03T15:54:18.759198Z","iopub.status.idle":"2025-11-03T15:54:19.418382Z","shell.execute_reply.started":"2025-11-03T15:54:18.759169Z","shell.execute_reply":"2025-11-03T15:54:19.417566Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train Regressor","metadata":{}},{"cell_type":"code","source":"from sklearn.decomposition import PCA\nfrom sklearn.preprocessing import StandardScaler\n\n# Separate image features and other features\nimage_feature_cols = [col for col in df_combined.columns if col.startswith('img_feature_')]\nother_features_cols = [col for col in df_combined.columns if col.startswith('target_name_')]\n\nX_image_features = df_combined[image_feature_cols]\nX_other_features = df_combined[other_features_cols]\n\n# Standardize the image features before applying PCA\nscaler = StandardScaler()\nX_image_scaled = scaler.fit_transform(X_image_features)\n\n# Apply PCA to reduce dimensionality\n# You can adjust the number of components (n_components)\npca = PCA(n_components=75) # Example: Reduce components by 20x\nX_image_pca = pca.fit_transform(X_image_scaled)\n\n# Convert the PCA reduced features to a DataFrame\nX_image_pca_df = pd.DataFrame(\n    X_image_pca, index=df_combined.index,\n    columns=[f'pca_img_feature_{i}' for i in range(pca.n_components)])\n\n# Combine the PCA reduced image features with the other features\nX_combined_pca = pd.concat([X_other_features, X_image_pca_df], axis=1)\n\nprint(f\"Original number of image features: {X_image_features.shape[1]}\")\nprint(f\"Reduced number of image features after PCA: {X_image_pca_df.shape[1]}\")\nprint(f\"Total number of features for regression after PCA: {X_combined_pca.shape[1]}\")\n\n# Display the first few rows of the combined feature DataFrame after PCA\ndisplay(X_combined_pca.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T15:54:19.419319Z","iopub.execute_input":"2025-11-03T15:54:19.419609Z","iopub.status.idle":"2025-11-03T15:54:20.234516Z","shell.execute_reply.started":"2025-11-03T15:54:19.419590Z","shell.execute_reply":"2025-11-03T15:54:20.233842Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import xgboost as xgb\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\n\n# Define features (X) and target (y) using the PCA-reduced image features\n# Exclude identifier columns and the original target column\n# Use X_combined_pca which contains the one-hot encoded target_name and PCA image features\nfeatures = X_combined_pca.columns.tolist() # Use all columns from X_combined_pca\nprint(f\"training features: {features}\")\ntarget = 'target'\n\nX = X_combined_pca\ny = df_combined[target] # Target remains the same\n\n# Split data into training and testing sets\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# Initialize and train the XGBoost Regressor\n# The features include binary (from one-hot encoding) and float types.\n# XGBoost can handle this mix of feature types effectively.\nxgb_model = xgb.XGBRegressor(\n    objective='reg:squarederror', # Regression objective\n    n_estimators=100,           # Number of boosting rounds\n    learning_rate=0.1,          # Step size shrinkage\n    max_depth=6,                # Maximum depth of a tree\n    random_state=42,\n)\n\nxgb_model.fit(X_train, y_train)\n\nprint(\"XGBoost model training complete.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T15:54:20.235429Z","iopub.execute_input":"2025-11-03T15:54:20.235649Z","iopub.status.idle":"2025-11-03T15:54:21.724340Z","shell.execute_reply.started":"2025-11-03T15:54:20.235630Z","shell.execute_reply":"2025-11-03T15:54:21.723454Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Make predictions on the test data\ny_pred = xgb_model.predict(X_test)\n\nprint(\"Predictions on test data complete.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T15:54:21.725336Z","iopub.execute_input":"2025-11-03T15:54:21.725638Z","iopub.status.idle":"2025-11-03T15:54:21.740464Z","shell.execute_reply.started":"2025-11-03T15:54:21.725620Z","shell.execute_reply":"2025-11-03T15:54:21.739648Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import mean_absolute_error, r2_score\n\n# Calculate Root Mean Squared Error (RMSE)\nrmse = np.sqrt(mean_squared_error(y_test, y_pred))\nprint(f\"Root Mean Squared Error (RMSE) on the test set: {rmse}\")\n\n# Calculate Mean Absolute Error (MAE)\nmae = mean_absolute_error(y_test, y_pred)\nprint(f\"Mean Absolute Error (MAE) on the test set: {mae}\")\n\n# Calculate R-squared\nr2 = r2_score(y_test, y_pred)\nprint(f\"R-squared on the test set: {r2}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T15:54:21.741234Z","iopub.execute_input":"2025-11-03T15:54:21.741448Z","iopub.status.idle":"2025-11-03T15:54:21.752349Z","shell.execute_reply.started":"2025-11-03T15:54:21.741431Z","shell.execute_reply":"2025-11-03T15:54:21.751558Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### After evaluation, train on complete data","metadata":{}},{"cell_type":"code","source":"xgb_model.fit(X, y)\n\nprint(\"Final XGBoost model training complete.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Test predictions","metadata":{}},{"cell_type":"code","source":"# 1. Load test data\nPATH_TEST_CSV = os.path.join(PATH_DATA, 'test.csv')\ndf_test = pd.read_csv(PATH_TEST_CSV)\nprint(f\"Test Dataset size: {df_test.shape}\")\ndisplay(df_test.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T15:54:21.753139Z","iopub.execute_input":"2025-11-03T15:54:21.753395Z","iopub.status.idle":"2025-11-03T15:54:21.774033Z","shell.execute_reply.started":"2025-11-03T15:54:21.753374Z","shell.execute_reply":"2025-11-03T15:54:21.773381Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 2. Extract test image features\ntest_image_features = {}\nfor index, row in tqdm(df_test.iterrows(), total=df_test.shape[0], desc=\"Extracting test features\"):\n    image_path = row['image_path']\n    \n    if image_path in image_cache:\n        features = image_cache[image_path]\n    else:\n        features = extract_image_features_pipeline(image_path, feature_extractor, PATH_DATA)\n        image_cache[image_path] = features  # Add to cache\n    \n    test_image_features[row['sample_id']] = features\n\nprint(f\"Test image feature extraction complete. Total cached unique images: {len(image_cache)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T15:54:21.774767Z","iopub.execute_input":"2025-11-03T15:54:21.775028Z","iopub.status.idle":"2025-11-03T15:54:21.974344Z","shell.execute_reply.started":"2025-11-03T15:54:21.775005Z","shell.execute_reply":"2025-11-03T15:54:21.973626Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 3. Combine test features\ntest_image_features_df = pd.DataFrame.from_dict(test_image_features, orient='index')\ntest_image_features_df.index.name = 'sample_id'\ntest_image_features_df.columns = [f'img_feature_{i}' for i in range(test_image_features_df.shape[1])]\n\n# One-hot encode the 'target_name' column for the test set\ntest_df_one_hot = pd.get_dummies(df_test['target_name'], prefix='target_name').astype(int)\n\n# Merge the test image features and one-hot encoded features with the original test DataFrame\ndf_test_combined = df_test.merge(test_image_features_df, on='sample_id', how='left')\ndf_test_combined = df_test_combined.merge(test_df_one_hot, left_index=True, right_index=True)\n\nprint(\"Test image features and one-hot encoded target names combined.\")\ndisplay(df_test_combined.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T15:54:21.975157Z","iopub.execute_input":"2025-11-03T15:54:21.975846Z","iopub.status.idle":"2025-11-03T15:54:22.014712Z","shell.execute_reply.started":"2025-11-03T15:54:21.975820Z","shell.execute_reply":"2025-11-03T15:54:22.014110Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 4. Scale and PCA transform test image features\n# Separate image features and other features for the test set\ntest_image_feature_cols = [col for col in df_test_combined.columns if col.startswith('img_feature_')]\ntest_other_features_cols = [col for col in df_test_combined.columns if col.startswith('target_name_')]\n\nX_test_image_features = df_test_combined[test_image_feature_cols]\nX_test_other_features = df_test_combined[test_other_features_cols]\n\n# Apply the *trained* scaler and PCA to the test image features\nX_test_image_scaled = scaler.transform(X_test_image_features) # Use the fitted scaler\nX_test_image_pca = pca.transform(X_test_image_scaled)       # Use the fitted pca\n\n# Convert the PCA reduced features to a DataFrame\nX_test_image_pca_df = pd.DataFrame(\n    X_test_image_pca, index=df_test_combined.index,\n    columns=[f'pca_img_feature_{i}' for i in range(pca.n_components)])\n\nprint(f\"Scaled and PCA-transformed test image features shape: {X_test_image_pca_df.shape}\")\n\n# Display the first few rows of the PCA reduced test image features\ndisplay(X_test_image_pca_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T15:54:22.017175Z","iopub.execute_input":"2025-11-03T15:54:22.017617Z","iopub.status.idle":"2025-11-03T15:54:22.067975Z","shell.execute_reply.started":"2025-11-03T15:54:22.017599Z","shell.execute_reply":"2025-11-03T15:54:22.067114Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 5. Prepare test data for inference\n# Combine the PCA reduced image features with the other features (one-hot encoded target_name)\nX_test_final = pd.concat([X_test_other_features, X_test_image_pca_df], axis=1)\n\nprint(f\"Final test feature DataFrame shape: {X_test_final.shape}\")\n\n# Display the first few rows of the final test feature DataFrame\ndisplay(X_test_final.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T15:54:22.068932Z","iopub.execute_input":"2025-11-03T15:54:22.069278Z","iopub.status.idle":"2025-11-03T15:54:22.087197Z","shell.execute_reply.started":"2025-11-03T15:54:22.069250Z","shell.execute_reply":"2025-11-03T15:54:22.086553Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 6. Make predictions\ntest_predictions = xgb_model.predict(X_test_final)\n\nprint(\"Predictions on test data complete.\")\ndisplay(test_predictions)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T15:54:22.088107Z","iopub.execute_input":"2025-11-03T15:54:22.088404Z","iopub.status.idle":"2025-11-03T15:54:22.108439Z","shell.execute_reply.started":"2025-11-03T15:54:22.088378Z","shell.execute_reply":"2025-11-03T15:54:22.107644Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Prepare submission","metadata":{}},{"cell_type":"code","source":"# 7. Generate submission file\nsubmission_df = pd.DataFrame({'sample_id': df_test['sample_id'], 'target': test_predictions})\n\n# Ensure predictions are non-negative if the target represents a physical quantity\nsubmission_df['target'] = submission_df['target'].clip(lower=0)\n\nsubmission_path = 'submission.csv'\nsubmission_df.to_csv(submission_path, index=False)\n\nprint(f\"Submission file generated at: {submission_path}\")\ndisplay(submission_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T15:54:22.109254Z","iopub.execute_input":"2025-11-03T15:54:22.109488Z","iopub.status.idle":"2025-11-03T15:54:22.128582Z","shell.execute_reply.started":"2025-11-03T15:54:22.109469Z","shell.execute_reply":"2025-11-03T15:54:22.127826Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!head submission.csv","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-03T15:54:22.129496Z","iopub.execute_input":"2025-11-03T15:54:22.129700Z","iopub.status.idle":"2025-11-03T15:54:22.289472Z","shell.execute_reply.started":"2025-11-03T15:54:22.129663Z","shell.execute_reply":"2025-11-03T15:54:22.288650Z"}},"outputs":[],"execution_count":null}]}