{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":268942,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":230141,"modelId":251887}],"dockerImageVersionId":31153,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# About\n- Feature preparation using SigLip\n- Predictions using GradientBoosting\n- No deep learning\n\n## Update version 8:\n- Added PCA trick with idea that predicted values are linearly dependent ([here](https://www.kaggle.com/code/none00000/lb-0-57-infer-model-code))\n- My trick is different from the previous approaches, where people only predict 3 targets\n- It gives around +0.01 LB\n- PCA trick is in `cross_validate`","metadata":{}},{"cell_type":"code","source":"import argparse\nfrom pathlib import Path\nimport sys\nfrom tqdm.auto import tqdm\nimport json\nfrom copy import deepcopy\nimport polars as pl\nimport numpy as np\nimport os\n\nimport torch\nfrom PIL import Image\nfrom transformers import AutoProcessor, AutoImageProcessor, AutoModel, Siglip2Model, Siglip2ImageProcessor, SiglipModel, SiglipImageProcessor","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T18:40:47.454527Z","iopub.execute_input":"2025-11-05T18:40:47.454880Z","iopub.status.idle":"2025-11-05T18:41:24.608466Z","shell.execute_reply.started":"2025-11-05T18:40:47.454846Z","shell.execute_reply":"2025-11-05T18:41:24.607809Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import KFold, GroupKFold\nfrom sklearn.linear_model import LinearRegression, Ridge, Lasso\nfrom sklearn.preprocessing import Normalizer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.dummy import DummyRegressor\nfrom sklearn.decomposition import PCA\n\nimport catboost","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T18:50:48.254993Z","iopub.execute_input":"2025-11-05T18:50:48.255612Z","iopub.status.idle":"2025-11-05T18:50:49.131764Z","shell.execute_reply.started":"2025-11-05T18:50:48.255589Z","shell.execute_reply":"2025-11-05T18:50:49.130977Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Prepare features","metadata":{}},{"cell_type":"code","source":"device = 'cuda' if torch.cuda.is_available() else 'cpu'\nprint(device)\n\n# Initialize model\nmodel_name = \"/kaggle/input/google-siglip-so400m-patch14-384/transformers/default/1/\"\nmodel = AutoModel.from_pretrained(\n    model_name,\n)\nmodel = model.to(device)\nmodel.eval()\nprocessor = AutoImageProcessor.from_pretrained(model_name)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T18:41:25.069497Z","iopub.execute_input":"2025-11-05T18:41:25.069709Z","iopub.status.idle":"2025-11-05T18:42:12.165484Z","shell.execute_reply.started":"2025-11-05T18:41:25.069692Z","shell.execute_reply":"2025-11-05T18:42:12.164739Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_path = Path('/kaggle/input/csiro-biomass')\n\nlabels = [\n  \"Dry_Clover_g\",\n  \"Dry_Dead_g\",\n  \"Dry_Green_g\",\n  \"Dry_Total_g\",\n  \"GDM_g\"\n]\n\ntrain = pl.read_csv(data_path / 'train.csv')\n\ndf = (\n    train\n    .with_columns([\n        pl.when(pl.col('target_name') == label).then(pl.col('target')).alias(label)\n        for label in labels\n    ])\n    .group_by('image_path')\n    .agg([\n        pl.col(label).mean()\n        for label in labels\n    ] + [\n        pl.concat_str([\"Sampling_Date\", \"State\"], separator=\" \")\n        .alias(\"group\")\n        .first()\n    ])\n    .sort('image_path')\n)\n\ndf","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T18:42:12.167516Z","iopub.execute_input":"2025-11-05T18:42:12.167789Z","iopub.status.idle":"2025-11-05T18:42:12.521364Z","shell.execute_reply.started":"2025-11-05T18:42:12.167764Z","shell.execute_reply":"2025-11-05T18:42:12.520677Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test = pl.read_csv(data_path / 'test.csv')\n# test = train.select('sample_id', 'image_path', 'target_name')\n\ndf_test = (\n    test\n    .group_by('image_path')\n    .len()\n    .sort('image_path')\n)\n\ndf_test","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T18:42:12.522048Z","iopub.execute_input":"2025-11-05T18:42:12.522264Z","iopub.status.idle":"2025-11-05T18:42:12.539283Z","shell.execute_reply.started":"2025-11-05T18:42:12.522249Z","shell.execute_reply":"2025-11-05T18:42:12.538689Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def compute_features(images, save_path):\n    batch_size = 20\n    with torch.no_grad(), open(save_path, 'w') as f:\n        for i in tqdm(range(0, len(images), batch_size)):\n            batch_paths = images[i:i + batch_size]\n            batch = [Image.open(data_path / p) for p in batch_paths]\n            \n            inputs = processor(images=batch, return_tensors=\"pt\").to(model.device)\n            features = model.get_image_features(**inputs)\n            \n            for line in features:\n                data = {f'x_{j}': line[j].item() for j in range(len(line))}\n                f.write(json.dumps(data) + '\\n')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T18:42:12.539934Z","iopub.execute_input":"2025-11-05T18:42:12.540184Z","iopub.status.idle":"2025-11-05T18:42:12.545873Z","shell.execute_reply.started":"2025-11-05T18:42:12.540146Z","shell.execute_reply":"2025-11-05T18:42:12.545027Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"compute_features(df['image_path'], 'features.ndjson')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T18:42:12.546782Z","iopub.execute_input":"2025-11-05T18:42:12.547064Z","iopub.status.idle":"2025-11-05T18:44:20.361855Z","shell.execute_reply.started":"2025-11-05T18:42:12.547034Z","shell.execute_reply":"2025-11-05T18:44:20.360945Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"compute_features(df_test['image_path'], 'features_test.ndjson')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T18:44:20.362755Z","iopub.execute_input":"2025-11-05T18:44:20.363019Z","iopub.status.idle":"2025-11-05T18:44:20.810459Z","shell.execute_reply.started":"2025-11-05T18:44:20.362996Z","shell.execute_reply":"2025-11-05T18:44:20.809621Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"responses = pl.read_ndjson('features.ndjson')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T18:44:20.811219Z","iopub.execute_input":"2025-11-05T18:44:20.811414Z","iopub.status.idle":"2025-11-05T18:44:20.930008Z","shell.execute_reply.started":"2025-11-05T18:44:20.811398Z","shell.execute_reply":"2025-11-05T18:44:20.929218Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"responses_test = pl.read_ndjson('features_test.ndjson')\nresponses_test","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T18:44:20.932350Z","iopub.execute_input":"2025-11-05T18:44:20.932803Z","iopub.status.idle":"2025-11-05T18:44:20.942891Z","shell.execute_reply.started":"2025-11-05T18:44:20.932785Z","shell.execute_reply":"2025-11-05T18:44:20.942345Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_aug = pl.concat(\n    [df, responses], how='horizontal'\n)\ndf_aug","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T18:44:20.943602Z","iopub.execute_input":"2025-11-05T18:44:20.944130Z","iopub.status.idle":"2025-11-05T18:44:20.975876Z","shell.execute_reply.started":"2025-11-05T18:44:20.944113Z","shell.execute_reply":"2025-11-05T18:44:20.975109Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test_aug = pl.concat(\n    [df_test, responses_test], how='horizontal'\n)\ndf_test_aug","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T18:44:20.976609Z","iopub.execute_input":"2025-11-05T18:44:20.976853Z","iopub.status.idle":"2025-11-05T18:44:20.995415Z","shell.execute_reply.started":"2025-11-05T18:44:20.976833Z","shell.execute_reply":"2025-11-05T18:44:20.994796Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Set up validation","metadata":{}},{"cell_type":"code","source":"weights = {\n    'Dry_Green_g': 0.1,\n    'Dry_Dead_g': 0.1,\n    'Dry_Clover_g': 0.1,\n    'GDM_g': 0.2,\n    'Dry_Total_g': 0.5,\n}\n\ndef competition_metric(y_true, y_pred) -> float:\n    y_weighted = 0\n    for l, label in enumerate(labels):\n        y_weighted = y_weighted + y_true[:, l].mean() * weights[label]\n\n    ss_res = 0\n    ss_tot = 0\n    for l, label in enumerate(labels):\n        ss_res = ss_res + ((y_true[:, l] - y_pred[:, l])**2).mean() * weights[label]\n        ss_tot = ss_tot + ((y_true[:, l] - y_weighted)**2).mean() * weights[label]\n\n    return 1 - ss_res / ss_tot","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T18:44:20.996015Z","iopub.execute_input":"2025-11-05T18:44:20.996268Z","iopub.status.idle":"2025-11-05T18:44:21.011621Z","shell.execute_reply.started":"2025-11-05T18:44:20.996253Z","shell.execute_reply":"2025-11-05T18:44:21.011098Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def cross_validate(model, data, data_test, x_columns, random_state=42) -> float:\n    assert not any((col in labels for col in x_columns))\n    X = data.select(x_columns).to_numpy()\n    X_test = data_test.select(x_columns).to_numpy()\n    y_true = data.select(labels).to_numpy()\n    y_pred = np.zeros([len(X), len(labels)])\n    y_pred_test = np.zeros([len(X_test), len(labels)])\n\n    n_splits = 5\n    kf = GroupKFold(n_splits=5)\n    groups = data.select('group')\n\n    for i, (train_index, test_index) in enumerate(kf.split(X, groups=groups)):\n        pca = PCA(3).fit(y_true[train_index])\n        for l in range(len(labels)):\n            m = deepcopy(model)\n            m.fit(X[train_index], y_true[train_index, l])\n            y_pred[test_index, l] = m.predict(X[test_index]).clip(0)\n            y_pred_test[:, l] += m.predict(X_test).clip(0) / n_splits\n\n        # Apply the PCA trick\n        y_pred[test_index] = pca.inverse_transform(pca.transform(y_pred[test_index])).clip(0)\n        y_pred_test = pca.inverse_transform(pca.transform(y_pred_test)).clip(0)\n\n        print(f'Fold {i}:', competition_metric(y_true[test_index], y_pred[test_index]))\n\n    print('Full CV:', competition_metric(y_true, y_pred))\n\n    return y_pred, y_pred_test","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T18:44:21.012468Z","iopub.execute_input":"2025-11-05T18:44:21.012770Z","iopub.status.idle":"2025-11-05T18:44:21.027781Z","shell.execute_reply.started":"2025-11-05T18:44:21.012750Z","shell.execute_reply":"2025-11-05T18:44:21.026946Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Choose a model","metadata":{}},{"cell_type":"code","source":"cross_validate(DummyRegressor(), df_aug, df_test_aug, sorted(responses.columns));","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T18:49:30.039993Z","iopub.execute_input":"2025-11-05T18:49:30.040608Z","iopub.status.idle":"2025-11-05T18:49:30.079945Z","shell.execute_reply.started":"2025-11-05T18:49:30.040586Z","shell.execute_reply":"2025-11-05T18:49:30.079238Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cross_validate(Ridge(), df_aug, df_test_aug, sorted(responses.columns));","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T18:49:34.858944Z","iopub.execute_input":"2025-11-05T18:49:34.859327Z","iopub.status.idle":"2025-11-05T18:49:35.318788Z","shell.execute_reply.started":"2025-11-05T18:49:34.859306Z","shell.execute_reply":"2025-11-05T18:49:35.318194Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cross_validate(Lasso(), df_aug, df_test_aug, sorted(responses.columns));","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T18:49:37.841850Z","iopub.execute_input":"2025-11-05T18:49:37.842488Z","iopub.status.idle":"2025-11-05T18:49:38.103946Z","shell.execute_reply.started":"2025-11-05T18:49:37.842464Z","shell.execute_reply":"2025-11-05T18:49:38.103178Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_, pred_test_gb = cross_validate(GradientBoostingRegressor(), df_aug, df_test_aug, sorted(responses.columns))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T19:08:15.400299Z","iopub.execute_input":"2025-11-05T19:08:15.400881Z","iopub.status.idle":"2025-11-05T19:11:36.373862Z","shell.execute_reply.started":"2025-11-05T19:08:15.400857Z","shell.execute_reply":"2025-11-05T19:11:36.373199Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_, pred_test_cb = cross_validate(catboost.CatBoostRegressor(verbose=False, iterations=100), df_aug, df_test_aug, sorted(responses.columns))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T19:11:36.374940Z","iopub.execute_input":"2025-11-05T19:11:36.375170Z","iopub.status.idle":"2025-11-05T19:17:21.426993Z","shell.execute_reply.started":"2025-11-05T19:11:36.375139Z","shell.execute_reply":"2025-11-05T19:17:21.426111Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"I choose the last two models","metadata":{}},{"cell_type":"code","source":"pred_test = (\n    pred_test_gb\n    + pred_test_cb\n) / 2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T19:17:21.427806Z","iopub.execute_input":"2025-11-05T19:17:21.428018Z","iopub.status.idle":"2025-11-05T19:17:21.431777Z","shell.execute_reply.started":"2025-11-05T19:17:21.428003Z","shell.execute_reply":"2025-11-05T19:17:21.431125Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Save predictions","metadata":{}},{"cell_type":"code","source":"pred_with_id = pl.concat([\n    df_test,\n    pl.DataFrame(pred_test, schema=labels),\n], how='horizontal')\npred_with_id","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T19:17:21.433307Z","iopub.execute_input":"2025-11-05T19:17:21.433912Z","iopub.status.idle":"2025-11-05T19:17:21.453144Z","shell.execute_reply.started":"2025-11-05T19:17:21.433891Z","shell.execute_reply":"2025-11-05T19:17:21.452437Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pred_save = (\n    test\n    .join(pred_with_id, on='image_path')\n    .with_columns(\n        pl.coalesce(*[\n            pl.when(pl.col('target_name') == col).then(pl.col(col))\n            for col in labels\n        ]).alias('target')\n    )\n    .select('sample_id', 'target')\n)\npred_save","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T19:17:21.453826Z","iopub.execute_input":"2025-11-05T19:17:21.454032Z","iopub.status.idle":"2025-11-05T19:17:21.472562Z","shell.execute_reply.started":"2025-11-05T19:17:21.454017Z","shell.execute_reply":"2025-11-05T19:17:21.471885Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pred_save.write_csv('submission.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-05T19:17:21.473221Z","iopub.execute_input":"2025-11-05T19:17:21.473411Z","iopub.status.idle":"2025-11-05T19:17:21.484976Z","shell.execute_reply.started":"2025-11-05T19:17:21.473398Z","shell.execute_reply":"2025-11-05T19:17:21.484200Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}