{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":289551848,"sourceType":"kernelVersion"},{"sourceId":290126653,"sourceType":"kernelVersion"}],"dockerImageVersionId":31234,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n# from scipy.stats import chi2_contingency\n# import matplotlib.pyplot as plt\n# import seaborn as sns\n\npd.set_option('display.max_columns', None)\n\ndirectory = '/kaggle/input/csiro-biomass/'\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-05T10:51:24.010926Z","iopub.execute_input":"2026-01-05T10:51:24.011248Z","iopub.status.idle":"2026-01-05T10:51:25.105039Z","shell.execute_reply.started":"2026-01-05T10:51:24.011219Z","shell.execute_reply":"2026-01-05T10:51:25.103943Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import cv2\nimport numpy as np\nimport pandas as pd\n\nfrom skimage.feature import graycomatrix, graycoprops, local_binary_pattern\nfrom scipy.stats import skew, kurtosis, entropy\nfrom scipy.fft import fft2\n\nimport torch\nimport torchvision.models as models\nimport torchvision.transforms as T\n\nfrom lightgbm import LGBMRegressor\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\n\n\ndef load_image(path):\n    img = cv2.imread(path)\n    if img is None:\n        raise ValueError(f\"Cannot read image: {path}\")\n    return cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n\n    \ndef rgb_stats(img):\n    feats = {}\n    for i, c in enumerate([\"R\",\"G\",\"B\"]):\n        ch = img[:,:,i].astype(np.float32)\n        feats[f\"mean_{c}\"] = ch.mean()\n        feats[f\"std_{c}\"] = ch.std()\n        feats[f\"skew_{c}\"] = skew(ch.flatten())\n        feats[f\"kurt_{c}\"] = kurtosis(ch.flatten())\n    return feats\n\n\ndef color_space_features(img):\n    feats = {}\n    hsv = cv2.cvtColor(img, cv2.COLOR_RGB2HSV)\n    lab = cv2.cvtColor(img, cv2.COLOR_RGB2LAB)\n    for name, ch in zip([\"H\",\"S\",\"V\"], cv2.split(hsv)):\n        feats[f\"mean_{name}\"] = ch.mean()\n        feats[f\"std_{name}\"] = ch.std()\n        feats[f\"entropy_{name}\"] = entropy(np.histogram(ch,256)[0] + 1)\n    for name, ch in zip([\"L\",\"a\",\"b\"], cv2.split(lab)):\n        feats[f\"mean_{name}\"] = ch.mean()\n        feats[f\"std_{name}\"] = ch.std()\n    return feats\n\n\ndef vegetation_indices(img):\n    R, G, B = img[:,:,0].astype(float), img[:,:,1].astype(float), img[:,:,2].astype(float)\n    eps = 1e-6\n    ExG   = 2*G - R - B\n    ExR   = 1.4*R - G\n    VARI  = (G - R) / (G + R - B + eps)\n    NGRDI = (G - R) / (G + R + eps)\n    feats = {}\n    for name, arr in zip(\n        [\"ExG\",\"ExR\",\"VARI\",\"NGRDI\"],\n        [ExG, ExR, VARI, NGRDI]\n    ):\n        feats[f\"{name}_mean\"] = arr.mean()\n        feats[f\"{name}_std\"]  = arr.std()\n        feats[f\"{name}_p75\"]  = np.percentile(arr, 75)\n    return feats\n\n\ndef pixel_fractions(img):\n    hsv = cv2.cvtColor(img, cv2.COLOR_RGB2HSV)\n    H, S, V = cv2.split(hsv)\n    green = ((H > 35) & (H < 85) & (S > 40))\n    brown = ((H > 10) & (H < 35) & (S > 30))\n    total = img.shape[0] * img.shape[1]\n    return {\n        \"green_pixel_fraction\": green.sum() / total,\n        \"brown_pixel_fraction\": brown.sum() / total\n    }\n\n\ndef glcm_features(img):\n    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n    gray = cv2.resize(gray, (256,256))\n    glcm = graycomatrix(\n        gray,\n        distances=[1,3],\n        angles=[0, np.pi/4, np.pi/2],\n        levels=256,\n        symmetric=True,\n        normed=True\n    )\n    feats = {}\n    for prop in [\"contrast\",\"homogeneity\",\"energy\",\"correlation\"]:\n        feats[f\"glcm_{prop}\"] = graycoprops(glcm, prop).mean()\n    return feats\n \n\ndef lbp_features(img):\n    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n    lbp = local_binary_pattern(gray, P=8, R=1, method=\"uniform\")\n    hist, _ = np.histogram(lbp, bins=10)\n    hist = hist / (hist.sum() + 1e-6)\n    return {\n        \"lbp_entropy\": entropy(hist),\n        \"lbp_variance\": np.var(hist)\n    }\n\n\ndef edge_features(img):\n    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n    edges = cv2.Canny(gray, 50, 150)\n    return {\n        \"edge_density\": edges.mean(),\n        \"edge_pixel_count\": edges.sum()\n    }\n\n\ndef shape_features(img):\n    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n    _, bw = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)\n    contours, _ = cv2.findContours(bw, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n    areas = [cv2.contourArea(c) for c in contours if cv2.contourArea(c) > 20]\n    if len(areas) == 0:\n        return {\"num_components\": 0, \"mean_component_area\": 0}\n    return {\n        \"num_components\": len(areas),\n        \"mean_component_area\": np.mean(areas),\n        \"max_component_area\": np.max(areas)\n    }\n\n\ndef frequency_features(img):\n    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n    f = np.abs(fft2(gray))\n    return {\n        \"fft_mean\": f.mean(),\n        \"fft_std\": f.std(),\n        \"fft_entropy\": entropy(np.histogram(f.flatten(),256)[0] + 1)\n    }\n\n\n_resnet = None\ndef cnn_features(img):\n    global _resnet\n    if _resnet is None:\n        model = models.resnet18(pretrained=True)\n        model.fc = torch.nn.Identity()\n        model.eval()\n        _resnet = model\n    transform = T.Compose([\n        T.ToPILImage(),\n        T.Resize((224,224)),\n        T.ToTensor(),\n        T.Normalize(\n            mean=[0.485,0.456,0.406],\n            std=[0.229,0.224,0.225]\n        )\n    ])\n    with torch.no_grad():\n        x = transform(img).unsqueeze(0)\n        feats = _resnet(x).numpy().flatten()\n    return {f\"cnn_{i}\": v for i, v in enumerate(feats)}\n\ndef extract_all_features_handcrafted(image_path, use_cnn=True):\n    img = load_image(image_path)\n    feats = {}\n    feats.update(rgb_stats(img))\n    feats.update(color_space_features(img))\n    feats.update(vegetation_indices(img))\n    feats.update(pixel_fractions(img))\n    feats.update(glcm_features(img))\n    feats.update(lbp_features(img))\n    feats.update(edge_features(img))\n    feats.update(shape_features(img))\n    feats.update(frequency_features(img))\n\n    return {k: float(v) for k, v in feats.items()}\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T10:51:25.107201Z","iopub.execute_input":"2026-01-05T10:51:25.107826Z","iopub.status.idle":"2026-01-05T10:51:47.249075Z","shell.execute_reply.started":"2026-01-05T10:51:25.107792Z","shell.execute_reply":"2026-01-05T10:51:47.248160Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport cv2\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\n\nimport torch\nimport timm\n\nfrom PIL import Image\nfrom torchvision import transforms\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error\nfrom lightgbm import LGBMRegressor\nfrom scipy.stats import skew, kurtosis\n\n\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\n# model = timm.create_model(\n#     \"vit_base_patch14_dinov2.lvd142m\",\n#     pretrained=True,\n#     num_classes=0,\n#     dynamic_img_size=True\n# ).to(device).eval()\nimport joblib\nimport os\n\nLOAD_DIR = \"/kaggle/input/dinov2/artifacts\"\n\nmodel = joblib.load(f\"{LOAD_DIR}/model.joblib\")\n\n\ntransform = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.ToTensor(),\n    transforms.Normalize(\n        mean=[0.485, 0.456, 0.406],\n        std =[0.229, 0.224, 0.225]\n    )\n])\n\n@torch.no_grad()\ndef extract_dino_features(img_path):\n    img = Image.open(img_path).convert(\"RGB\")\n    x = transform(img).unsqueeze(0).to(device)\n\n    out = model.forward_features(x)\n\n    # -------- Handle timm output safely --------\n    if isinstance(out, dict):\n        # Newer timm returns dict\n        tokens = out.get(\"x_norm_patchtokens\", None)\n        cls = out.get(\"x_norm_clstoken\", None)\n\n        if tokens is None:\n            tokens = out[\"x\"]  # fallback\n\n        if cls is None:\n            cls = tokens[:, 0]     # first token is CLS\n            tokens = tokens[:, 1:]\n        else:\n            tokens = tokens.squeeze(0)\n            cls = cls.squeeze(0)\n        \n\n    else:\n        # Older / alternative timm path\n        tokens = out              # (B, N, D)\n        cls = tokens[:, 0]        # CLS\n        tokens = tokens[:, 1:]    # patches\n\n    tokens = tokens.squeeze(0).cpu().numpy()\n    cls = cls.squeeze(0).cpu().numpy()\n\n    feats = {}\n\n    # ---- CLS embedding\n    for i, v in enumerate(cls):\n        feats[f\"dino_cls_{i}\"] = float(v)\n\n    # ---- Patch statistics (CRITICAL)\n    feats[\"dino_patch_mean\"] = tokens.mean()\n    feats[\"dino_patch_std\"] = tokens.std()\n    feats[\"dino_patch_p90\"] = np.percentile(tokens, 90)\n    feats[\"dino_patch_entropy\"] = -np.mean(\n        tokens * np.log(np.abs(tokens) + 1e-6)\n    )\n\n    if isinstance(out, dict):\n        tokens = out[\"x_norm_patchtokens\"]  # (1, N, D)\n    else:\n        tokens = out[:, 1:]  # fallback\n\n    tokens = tokens.squeeze(0).cpu().numpy()  # (N, D)\n    flat = tokens.flatten()\n        \n    feats[\"patch_mean\"] = flat.mean()\n    feats[\"patch_std\"] = flat.std()\n    feats[\"patch_p90\"] = np.percentile(flat, 90)\n    feats[\"patch_skew\"] = skew(flat)\n    feats[\"patch_kurt\"] = kurtosis(flat)\n    feats[\"patch_entropy\"] = -np.mean(flat * np.log(np.abs(flat) + 1e-6))\n\n    # 🔑 LOW vs HIGH FREQUENCY SPLIT\n    mid = tokens.shape[1] // 2\n    low_freq  = tokens[:, :mid].flatten()\n    high_freq = tokens[:, mid:].flatten()\n\n    feats[\"patch_low_std\"]  = low_freq.std()\n    feats[\"patch_high_std\"] = high_freq.std()\n\n    return feats\n\ndef vegetation_fractions(img_path):\n    img = cv2.imread(img_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)\n\n    H, S, V = cv2.split(img)\n    total = img.shape[0] * img.shape[1]\n\n    green = np.sum((H > 35) & (H < 85) & (S > 40))\n    brown = np.sum((H > 10) & (H < 30))\n    soil  = np.sum((S < 30) & (V > 50))\n\n    return {\n        \"green_frac\": green / total,\n        \"dead_frac\": brown / total,\n        \"soil_frac\": soil / total\n    }\n\n\ndef texture_features(img_path):\n    img = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)\n    img = cv2.resize(img, (256, 256))\n\n    # Contrast proxy\n    contrast = img.var()\n\n    # Edge density → height / biomass proxy\n    edges = cv2.Canny(img, 50, 150)\n    edge_density = edges.mean()\n\n    # Entropy\n    hist = cv2.calcHist([img], [0], None, [256], [0,256])\n    p = hist / hist.sum()\n    entropy = -np.sum(p * np.log2(p + 1e-9))\n\n    return {\n        \"tex_contrast\": contrast,\n        \"tex_edge_density\": edge_density,\n        \"tex_entropy\": entropy\n    }\n\n\ndef fft_features(img_path):\n    img = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)\n    img = cv2.resize(img, (256, 256))\n\n    f = np.fft.fft2(img)\n    fshift = np.abs(np.fft.fftshift(f))\n\n    return {\n        \"fft_mean\": fshift.mean(),\n        \"fft_std\": fshift.std()\n    }\n\n\ndef extract_all_features(img_path):\n    feats = {}\n    feats.update(extract_dino_features(directory+img_path))\n    feats.update(vegetation_fractions(directory+img_path))\n    feats.update(texture_features(directory+img_path))\n    feats.update(fft_features(directory+img_path))\n    return feats\n\n\ndef build_feature_table(df):\n\n    # 1️⃣ Deduplicate images\n    img_df = (\n        df[[\"image_path\"]]\n        .drop_duplicates()\n        .reset_index(drop=True)\n    )\n\n    tqdm.pandas()\n\n    # 2️⃣ Extract BOTH feature families\n    feats = img_df[\"image_path\"].progress_apply(\n        lambda x: {\n            **extract_all_features(x),              # DINO + aux\n            **extract_all_features_handcrafted(f\"{directory}/{x}\",   # handcrafted\n                                              use_cnn=False)\n        }\n    )\n\n    feat_df = pd.DataFrame(feats.tolist())\n\n    img_feat_df = pd.concat([img_df, feat_df], axis=1)\n\n\n    # 4️⃣ Merge back to original df\n    out = df.merge(\n        img_feat_df,\n        on=\"image_path\",\n        how=\"left\"\n    )\n\n    return out\n\ndef prepare_groups(df):\n    if \"image_id\" not in df.columns:\n        df = df.copy()\n        df[\"image_id\"] = df[\"sample_id\"].str.split(\"__\").str[0]\n    return df\n\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.metrics import mean_squared_error\nfrom catboost import CatBoostRegressor\nimport numpy as np\n\ndef cv_mse_catboost(\n    df,\n    feature_cols,\n    params,\n    n_splits=5\n):\n    X = df[feature_cols]\n    y = df[\"target\"]\n    groups = df[\"image_id\"]\n\n    gkf = GroupKFold(n_splits=n_splits)\n    scores = []\n\n    for tr_idx, va_idx in gkf.split(X, y, groups):\n        model = CatBoostRegressor(\n            **params,\n            loss_function=\"RMSE\",\n            eval_metric=\"RMSE\",\n            random_seed=42,\n            verbose=False,\n            allow_writing_files=False\n        )\n\n        model.fit(\n            X.iloc[tr_idx],\n            y.iloc[tr_idx],\n            eval_set=(X.iloc[va_idx], y.iloc[va_idx]),\n            use_best_model=True\n        )\n\n        preds = model.predict(X.iloc[va_idx])\n        # scores.append(mean_squared_error(y.iloc[va_idx], preds))\n        scores.append(mean_squared_error(y.iloc[va_idx], preds))\n\n\n    return np.mean(scores)\n\n\ndef tune_catboost_per_target(\n    df,\n    feature_cols,\n    target_name,\n    param_grid\n):\n    df_tgt = df[df[\"target_name\"] == target_name]\n\n    best_mse = np.inf\n    best_params = None\n\n    for params in param_grid:\n        mse = cv_mse_catboost(df_tgt, feature_cols, params)\n        print(f\"{target_name} | {params} | CV MSE = {mse:.6f}\")\n\n        if mse < best_mse:\n            best_mse = mse\n            best_params = params\n\n    print(f\"✅ BEST {target_name}: {best_params}\")\n    return best_params\n\n\ndef train_final_catboost_model(\n    df,\n    feature_cols,\n    best_params\n):\n    X = df[feature_cols]\n    y = df[\"target\"]\n\n    model = CatBoostRegressor(\n        **best_params,\n        loss_function=\"RMSE\",\n        eval_metric=\"RMSE\",\n        random_seed=42,\n        verbose=False,\n        allow_writing_files=False\n    )\n\n    model.fit(X, y)\n    return model\n\n\ndef train_models_per_target(\n    df_feat,\n    all_features,\n    target_feature_map,\n    param_grid\n):\n    df_feat = prepare_groups(df_feat)\n\n    trained_models = {}\n\n    for tgt in df_feat[\"target_name\"].unique():\n        print(f\"\\n🔍 Training target: {tgt}\")\n\n        tgt_features = target_feature_map[tgt]\n        df_tgt = df_feat[df_feat[\"target_name\"] == tgt]\n\n        best_params = tune_catboost_per_target(\n            df=df_tgt,\n            feature_cols=tgt_features,\n            target_name=tgt,\n            param_grid=param_grid[tgt]   # ✅ FIX\n        )\n\n\n        model = train_final_catboost_model(\n            df=df_tgt,\n            feature_cols=tgt_features,\n            best_params=best_params\n        )\n\n        trained_models[tgt] = {\n            \"model\": model,\n            \"features\": tgt_features\n        }\n\n    return trained_models\n\ndef predict_biomass(\n    df_feat,\n    trained_models\n):\n    df_feat = df_feat.copy()\n    df_feat[\"prediction\"] = np.nan\n\n    for tgt, bundle in trained_models.items():\n        model = bundle[\"model\"]\n        features = bundle[\"features\"]\n\n        mask = df_feat[\"target_name\"] == tgt\n        if mask.sum() == 0:\n            continue\n\n        X = df_feat.loc[mask, features]\n        df_feat.loc[mask, \"prediction\"] = model.predict(X)\n\n    return df_feat\n\n\ndef apply_physics_constraints(df):\n    df = df.copy()\n    df[\"image_id\"] = df[\"sample_id\"].str.split(\"__\").str[0]\n\n    wide = df.pivot(\n        index=\"image_id\",\n        columns=\"target_name\",\n        values=\"prediction\"\n    )\n\n    # wide[\"Dry_Total_g\"] = (\n    #     wide[\"Dry_Green_g\"]\n    #     + wide[\"Dry_Dead_g\"]\n    #     + wide[\"Dry_Clover_g\"]\n    # )\n    wide[\"Dry_Dead_g\"] = wide[\"Dry_Total_g\"] - wide[\"GDM_g\"]\n    # wide[\"GDM_g\"] = wide[\"Dry_Green_g\"]\n\n    out = wide.stack().reset_index()\n    out.columns = [\"image_id\", \"target_name\", \"prediction\"]\n\n    return df.merge(\n        out,\n        on=[\"image_id\", \"target_name\"],\n        how=\"left\",\n        suffixes=(\"\", \"_phys\")\n    ).assign(\n        prediction=lambda x: x[\"prediction_phys\"]\n    )[[\"sample_id\",\"target_name\", \"prediction\"]]\n\nfrom scipy.stats import spearmanr\n\nfrom scipy.stats import spearmanr\nimport numpy as np\nimport pandas as pd\n\ndef select_features_per_target(\n    df,\n    feature_cols,\n    top_k=180\n):\n    \"\"\"\n    Select top features per target using Spearman correlation.\n\n    Args:\n        df : DataFrame with columns [\"target_name\", \"target\", features...]\n        feature_cols : list of candidate feature names\n        top_k : int OR dict {target_name: k}\n\n    Returns:\n        dict: { target_name -> list of selected features }\n    \"\"\"\n    feature_map = {}\n\n    for tgt in df[\"target_name\"].unique():\n        df_tgt = df[df[\"target_name\"] == tgt]\n        y = df_tgt[\"target\"].values\n\n        # 🔑 allow per-target top_k\n        k = top_k[tgt] if isinstance(top_k, dict) else top_k\n\n        scores = []\n\n        for f in feature_cols:\n            x = df_tgt[f].values\n\n            # skip near-constant features\n            if np.std(x) < 1e-6:\n                continue\n\n            corr = spearmanr(x, y).correlation\n            if np.isnan(corr):\n                continue\n\n            scores.append((f, abs(corr)))\n\n        ranked = (\n            pd.DataFrame(scores, columns=[\"feature\", \"score\"])\n            .sort_values(\"score\", ascending=False)\n        )\n\n        feature_map[tgt] = ranked.head(k)[\"feature\"].tolist()\n\n        print(f\"{tgt}: selected {len(feature_map[tgt])} features\")\n\n    return feature_map\n\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T10:51:47.250417Z","iopub.execute_input":"2026-01-05T10:51:47.251307Z","iopub.status.idle":"2026-01-05T10:52:24.892332Z","shell.execute_reply.started":"2026-01-05T10:51:47.251260Z","shell.execute_reply":"2026-01-05T10:52:24.891371Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\ntrain = pd.read_csv('/kaggle/input/csiro-biomass/train.csv')\ntrain['Sampling_Date'] = pd.to_datetime(train['Sampling_Date'])\n\ntest = pd.read_csv('/kaggle/input/csiro-biomass/test.csv')    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T10:52:24.894567Z","iopub.execute_input":"2026-01-05T10:52:24.895374Z","iopub.status.idle":"2026-01-05T10:52:24.945109Z","shell.execute_reply.started":"2026-01-05T10:52:24.895340Z","shell.execute_reply":"2026-01-05T10:52:24.944057Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import joblib\nimport os\n\n# LOAD_DIR = \"/kaggle/input/criso-model-1/artifacts\"\nLOAD_DIR = \"/kaggle/input/dinov2/artifacts\"\n# LOAD_DIR = \"/kaggle/input/criso-catboost/artifacts\"\n# LOAD_DIR = \"/kaggle/input/criso-trees/artifacts\"\n\n# Load features\n# feature_cols = joblib.load(f\"{LOAD_DIR}/feature_cols.joblib\")\n\n# Load models\ntrained_models = joblib.load(f\"{LOAD_DIR}/trained_models.joblib\")\n\nprint(\"✅ Models and features loaded\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T10:52:24.946685Z","iopub.execute_input":"2026-01-05T10:52:24.947094Z","iopub.status.idle":"2026-01-05T10:52:25.011339Z","shell.execute_reply.started":"2026-01-05T10:52:24.947061Z","shell.execute_reply":"2026-01-05T10:52:25.010102Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_feat = build_feature_table(\n    test\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T10:52:25.012953Z","iopub.execute_input":"2026-01-05T10:52:25.013345Z","iopub.status.idle":"2026-01-05T10:52:34.414002Z","shell.execute_reply.started":"2026-01-05T10:52:25.013306Z","shell.execute_reply":"2026-01-05T10:52:34.412747Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# PREDICT\ntest_pred = predict_biomass(\n    df_feat=test_feat,\n    trained_models=trained_models,\n    # feature_cols=feature_cols\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T10:52:34.415387Z","iopub.execute_input":"2026-01-05T10:52:34.415781Z","iopub.status.idle":"2026-01-05T10:52:34.528393Z","shell.execute_reply.started":"2026-01-05T10:52:34.415751Z","shell.execute_reply":"2026-01-05T10:52:34.527503Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_pred = test_pred[[\n    \"sample_id\",\n    \"target_name\",\n    # \"target\",\n    \"prediction\"\n]]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T10:52:34.529641Z","iopub.execute_input":"2026-01-05T10:52:34.529990Z","iopub.status.idle":"2026-01-05T10:52:34.536741Z","shell.execute_reply.started":"2026-01-05T10:52:34.529953Z","shell.execute_reply":"2026-01-05T10:52:34.535392Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_pred","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T10:52:34.539324Z","iopub.execute_input":"2026-01-05T10:52:34.539652Z","iopub.status.idle":"2026-01-05T10:52:34.580417Z","shell.execute_reply.started":"2026-01-05T10:52:34.539624Z","shell.execute_reply":"2026-01-05T10:52:34.579313Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# PHYSICS\nsubmission = apply_physics_constraints(test_pred)\nsubmission = test_pred[['sample_id','prediction']].copy()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T10:52:34.581537Z","iopub.execute_input":"2026-01-05T10:52:34.581867Z","iopub.status.idle":"2026-01-05T10:52:34.615801Z","shell.execute_reply.started":"2026-01-05T10:52:34.581841Z","shell.execute_reply":"2026-01-05T10:52:34.614430Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T10:52:34.616879Z","iopub.execute_input":"2026-01-05T10:52:34.617176Z","iopub.status.idle":"2026-01-05T10:52:34.629519Z","shell.execute_reply.started":"2026-01-05T10:52:34.617149Z","shell.execute_reply":"2026-01-05T10:52:34.628399Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T10:52:34.630790Z","iopub.execute_input":"2026-01-05T10:52:34.631175Z","iopub.status.idle":"2026-01-05T10:52:34.649602Z","shell.execute_reply.started":"2026-01-05T10:52:34.631135Z","shell.execute_reply":"2026-01-05T10:52:34.648559Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### CATBOOST - HandCraft","metadata":{}},{"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/csiro-biomass/test.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T04:59:46.569048Z","iopub.execute_input":"2026-01-04T04:59:46.570710Z","iopub.status.idle":"2026-01-04T04:59:46.595387Z","shell.execute_reply.started":"2026-01-04T04:59:46.570587Z","shell.execute_reply":"2026-01-04T04:59:46.594225Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import cv2\nimport numpy as np\nimport pandas as pd\n\nfrom skimage.feature import graycomatrix, graycoprops, local_binary_pattern\nfrom scipy.stats import skew, kurtosis, entropy\nfrom scipy.fft import fft2\n\nimport torch\nimport torchvision.models as models\nimport torchvision.transforms as T\n\nfrom lightgbm import LGBMRegressor\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\n\n\ndef load_image(path):\n    img = cv2.imread(path)\n    if img is None:\n        raise ValueError(f\"Cannot read image: {path}\")\n    return cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n\n    \ndef rgb_stats(img):\n    feats = {}\n    for i, c in enumerate([\"R\",\"G\",\"B\"]):\n        ch = img[:,:,i].astype(np.float32)\n        feats[f\"mean_{c}\"] = ch.mean()\n        feats[f\"std_{c}\"] = ch.std()\n        feats[f\"skew_{c}\"] = skew(ch.flatten())\n        feats[f\"kurt_{c}\"] = kurtosis(ch.flatten())\n    return feats\n\n\ndef color_space_features(img):\n    feats = {}\n    hsv = cv2.cvtColor(img, cv2.COLOR_RGB2HSV)\n    lab = cv2.cvtColor(img, cv2.COLOR_RGB2LAB)\n    for name, ch in zip([\"H\",\"S\",\"V\"], cv2.split(hsv)):\n        feats[f\"mean_{name}\"] = ch.mean()\n        feats[f\"std_{name}\"] = ch.std()\n        feats[f\"entropy_{name}\"] = entropy(np.histogram(ch,256)[0] + 1)\n    for name, ch in zip([\"L\",\"a\",\"b\"], cv2.split(lab)):\n        feats[f\"mean_{name}\"] = ch.mean()\n        feats[f\"std_{name}\"] = ch.std()\n    return feats\n\n\ndef vegetation_indices(img):\n    R, G, B = img[:,:,0].astype(float), img[:,:,1].astype(float), img[:,:,2].astype(float)\n    eps = 1e-6\n    ExG   = 2*G - R - B\n    ExR   = 1.4*R - G\n    VARI  = (G - R) / (G + R - B + eps)\n    NGRDI = (G - R) / (G + R + eps)\n    feats = {}\n    for name, arr in zip(\n        [\"ExG\",\"ExR\",\"VARI\",\"NGRDI\"],\n        [ExG, ExR, VARI, NGRDI]\n    ):\n        feats[f\"{name}_mean\"] = arr.mean()\n        feats[f\"{name}_std\"]  = arr.std()\n        feats[f\"{name}_p75\"]  = np.percentile(arr, 75)\n    return feats\n\n\ndef pixel_fractions(img):\n    hsv = cv2.cvtColor(img, cv2.COLOR_RGB2HSV)\n    H, S, V = cv2.split(hsv)\n    green = ((H > 35) & (H < 85) & (S > 40))\n    brown = ((H > 10) & (H < 35) & (S > 30))\n    total = img.shape[0] * img.shape[1]\n    return {\n        \"green_pixel_fraction\": green.sum() / total,\n        \"brown_pixel_fraction\": brown.sum() / total\n    }\n\n\ndef glcm_features(img):\n    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n    gray = cv2.resize(gray, (256,256))\n    glcm = graycomatrix(\n        gray,\n        distances=[1,3],\n        angles=[0, np.pi/4, np.pi/2],\n        levels=256,\n        symmetric=True,\n        normed=True\n    )\n    feats = {}\n    for prop in [\"contrast\",\"homogeneity\",\"energy\",\"correlation\"]:\n        feats[f\"glcm_{prop}\"] = graycoprops(glcm, prop).mean()\n    return feats\n \n\ndef lbp_features(img):\n    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n    lbp = local_binary_pattern(gray, P=8, R=1, method=\"uniform\")\n    hist, _ = np.histogram(lbp, bins=10)\n    hist = hist / (hist.sum() + 1e-6)\n    return {\n        \"lbp_entropy\": entropy(hist),\n        \"lbp_variance\": np.var(hist)\n    }\n\n\ndef edge_features(img):\n    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n    edges = cv2.Canny(gray, 50, 150)\n    return {\n        \"edge_density\": edges.mean(),\n        \"edge_pixel_count\": edges.sum()\n    }\n\n\ndef shape_features(img):\n    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n    _, bw = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)\n    contours, _ = cv2.findContours(bw, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n    areas = [cv2.contourArea(c) for c in contours if cv2.contourArea(c) > 20]\n    if len(areas) == 0:\n        return {\"num_components\": 0, \"mean_component_area\": 0}\n    return {\n        \"num_components\": len(areas),\n        \"mean_component_area\": np.mean(areas),\n        \"max_component_area\": np.max(areas)\n    }\n\n\ndef frequency_features(img):\n    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n    f = np.abs(fft2(gray))\n    return {\n        \"fft_mean\": f.mean(),\n        \"fft_std\": f.std(),\n        \"fft_entropy\": entropy(np.histogram(f.flatten(),256)[0] + 1)\n    }\n\n\n_resnet = None\ndef cnn_features(img):\n    global _resnet\n    if _resnet is None:\n        model = models.resnet18(pretrained=True)\n        model.fc = torch.nn.Identity()\n        model.eval()\n        _resnet = model\n    transform = T.Compose([\n        T.ToPILImage(),\n        T.Resize((224,224)),\n        T.ToTensor(),\n        T.Normalize(\n            mean=[0.485,0.456,0.406],\n            std=[0.229,0.224,0.225]\n        )\n    ])\n    with torch.no_grad():\n        x = transform(img).unsqueeze(0)\n        feats = _resnet(x).numpy().flatten()\n    return {f\"cnn_{i}\": v for i, v in enumerate(feats)}\n\ndef extract_all_features(image_path, use_cnn=True):\n    img = load_image(image_path)\n    feats = {}\n    feats.update(rgb_stats(img))\n    feats.update(color_space_features(img))\n    feats.update(vegetation_indices(img))\n    feats.update(pixel_fractions(img))\n    feats.update(glcm_features(img))\n    feats.update(lbp_features(img))\n    feats.update(edge_features(img))\n    feats.update(shape_features(img))\n    feats.update(frequency_features(img))\n    if use_cnn:\n        feats.update(cnn_features(img))\n\n    return pd.DataFrame([feats])\n\n\nimport numpy as np\nimport pandas as pd\nfrom heapq import heappush, heappushpop\nfrom tqdm import tqdm\nfrom scipy.cluster.hierarchy import linkage, fcluster\nfrom scipy.spatial.distance import squareform\n\n\ndef generate_and_score_features(\n    df,\n    base_features,\n    target_col,\n    eps=1e-6\n):\n    \"\"\"\n    Returns:\n        feature_scores_df : feature, corr, abs_corr\n        df_all_features   : df with all generated features\n    \"\"\"\n    y = df[target_col].values\n    df_out = df.copy()\n    scores = []\n    def score_feature(name, x):\n        if np.isnan(x).all():\n            return\n        corr = np.corrcoef(x, y)[0, 1]\n        if np.isnan(corr):\n            return\n        scores.append((name, corr, abs(corr)))\n    # ------------------------\n    # 1. Single features\n    # ------------------------\n    for f in tqdm(base_features, desc=\"Scoring base features\"):\n        score_feature(f, df_out[f].values)\n    # ------------------------\n    # 2. Pairwise crosses\n    # ------------------------\n    n = len(base_features)\n    new_cols = {}\n    for i in tqdm(range(n), desc=\"Generating crossed features\"):\n        f1 = base_features[i]\n        x1 = df_out[f1].values\n        for j in range(i + 1, n):\n            f2 = base_features[j]\n            x2 = df_out[f2].values\n            # +\n            name = f\"{f1}_plus_{f2}\"\n            new_cols[name] = x1 + x2\n            score_feature(name, new_cols[name])\n            # *\n            name = f\"{f1}_mul_{f2}\"\n            new_cols[name] = x1 * x2\n            score_feature(name, new_cols[name])\n            # /\n            name = f\"{f1}_div_{f2}\"\n            new_cols[name] = x1 / (x2 + eps)\n            score_feature(name, new_cols[name])\n    df_out = pd.concat([df_out, pd.DataFrame(new_cols)], axis=1)\n    feature_scores_df = (\n        pd.DataFrame(scores, columns=[\"feature\", \"corr\", \"abs_corr\"])\n        .sort_values(\"abs_corr\", ascending=False)\n        .reset_index(drop=True)\n    )\n    return feature_scores_df, df_out\n\n\ndef cluster_features_by_correlation(df, features, K):\n    # Compute absolute correlation\n    corr = df[features].corr().abs()\n    # Replace NaNs (from constant columns)\n    corr = corr.fillna(0.0)\n    # Numerical safety: clip to [0, 1]\n    corr = corr.clip(0.0, 1.0)\n    # Distance matrix\n    dist = 1.0 - corr\n    # Ensure exact zeros on diagonal\n    np.fill_diagonal(dist.values, 0.0)\n    # Convert to condensed distance\n    dist_condensed = squareform(dist.values, checks=False)\n    # Hierarchical clustering\n    Z = linkage(dist_condensed, method=\"average\")\n    clusters = fcluster(Z, K, criterion=\"maxclust\")\n    return pd.DataFrame({\n        \"feature\": features,\n        \"cluster\": clusters\n    })\n\n\ndef build_top_k_diverse_features(\n    df,\n    base_features,\n    target_col,\n    K\n):\n    fundamental_features = [col for col in df.columns if col not in base_features]\n    \n    # 1. Generate & score all features\n    feature_scores_df, df_all = generate_and_score_features(\n        df, base_features, target_col\n    )\n    all_features = feature_scores_df[\"feature\"].tolist()\n    # 2. Cluster features\n    cluster_df = cluster_features_by_correlation(\n        df_all, all_features, K\n    )\n    # 3. Select representatives\n    selected_df = select_cluster_representatives(\n        feature_scores_df, cluster_df\n    )\n    selected_features = selected_df[\"feature\"].tolist()\n    # 4. Final dataframe\n    final_df = df_all[fundamental_features + selected_features].copy()\n    print(final_df.shape)\n    return {\n        \"final_df\": final_df,\n        \"selected_features\": selected_features,\n        \"feature_cluster_table\": selected_df\n    }\n\n\ndef generate_selected_features_only(\n    df,\n    selected_features,\n    directory,\n    use_cnn=False,\n    eps=1e-6\n):\n    \"\"\"\n    Generate only the features required by the trained model.\n    Works even when 'target' is NOT present.\n    \"\"\"\n\n    df = df.reset_index(drop=True)\n\n    # --- extract base image features ---\n    img_df = df[['image_path']].drop_duplicates().reset_index(drop=True)\n    tqdm.pandas()\n    feats = img_df['image_path'].progress_apply(\n        lambda x: extract_all_features(f\"{directory}/{x}\", use_cnn=use_cnn)\n    )\n    feats = pd.concat(feats.tolist(), ignore_index=True)\n    img_df = pd.concat([img_df, feats], axis=1)\n\n    df = df.merge(img_df, on='image_path', how='left')\n    df = drop_duplicate_columns_strict(df)\n\n    # --- materialize ONLY required features ---\n    new_cols = {}\n\n    for feat in selected_features:\n        if feat in df.columns:\n            continue\n\n        if \"_plus_\" in feat:\n            f1, f2 = feat.split(\"_plus_\")\n            new_cols[feat] = df[f1] + df[f2]\n\n        elif \"_mul_\" in feat:\n            f1, f2 = feat.split(\"_mul_\")\n            new_cols[feat] = df[f1] * df[f2]\n\n        elif \"_div_\" in feat:\n            f1, f2 = feat.split(\"_div_\")\n            new_cols[feat] = df[f1] / (df[f2] + eps)\n\n    if new_cols:\n        df = pd.concat([df, pd.DataFrame(new_cols)], axis=1)\n\n    return df\n\n\nfrom tqdm import tqdm\nimport pandas as pd\n\ndef add_features(df, top_k=100):\n    df = df.reset_index(drop=True)\n\n    # Extract image features\n    img_df = df[['image_path']].drop_duplicates().reset_index(drop=True)\n    tqdm.pandas()\n    feats = img_df['image_path'].progress_apply(\n        lambda x: extract_all_features(f\"{directory}/{x}\", use_cnn=False)\n    )\n    feats = pd.concat(feats.tolist(), ignore_index=True)\n    img_df = pd.concat([img_df, feats], axis=1)\n\n    df = df.merge(img_df, on='image_path', how='left')\n    df = drop_duplicate_columns_strict(df)\n\n    float_cols = df.select_dtypes(include=[\"float32\", \"float64\"]).columns\n    exclude_cols = [\"target\", \"Pre_GSHH_NDVI\", \"Height_Ave_cm\"]\n    base_features = canonicalize_features(\n        [c for c in float_cols if c not in exclude_cols]\n    )\n\n    # 🔥 clustering-based selection\n    result = build_top_k_diverse_features(\n        df=df,\n        base_features=base_features,\n        target_col=\"target\",\n        K=top_k\n    )\n\n    final_df          = result[\"final_df\"]\n    selected_features = result[\"selected_features\"]\n    cluster_table     = result[\"feature_cluster_table\"]\n\n    return final_df, selected_features, cluster_table, base_features\n\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\nfrom lightgbm import LGBMRegressor\n\ndef train_test_mse(df_sub, feature_cols):\n    X = df_sub[feature_cols]\n    y = df_sub[\"target\"]\n    X_train, X_test, y_train, y_test = train_test_split(\n        X, y, test_size=0.2, random_state=42\n    )\n    model = LGBMRegressor(\n        n_estimators=1000,\n        learning_rate=0.005,\n        max_depth=10,\n        subsample=0.8,\n        colsample_bytree=0.8,\n        random_state=42\n    )\n    model.fit(X_train, y_train)\n    y_pred = model.predict(X_test)\n    mse = mean_squared_error(y_test, y_pred)\n    return mse, model\n\n\ndef train_model(df):\n    df_feat, feature_cols, cluster_table, base_features = add_features(df)\n\n    feature_cols = list(dict.fromkeys(feature_cols))  # safety\n\n    trained_models = {}\n    results = {}\n\n    for tgt in df_feat['target_name'].unique():\n        df_tgt = df_feat[df_feat[\"target_name\"] == tgt]\n        mse, model = train_test_mse(df_tgt, feature_cols)\n        trained_models[tgt] = model\n        results[tgt] = mse\n        print(f\"{tgt} | Test MSE = {mse:.6f}\")\n\n    return trained_models, results, feature_cols, cluster_table, base_features\n\n\ndef predict_biomass(df, trained_models, feature_cols, base_features):\n    df_feat, _, _, _ = add_features(\n        df,\n        top_k=0   # no re-clustering\n    )\n\n    df_feat[\"prediction\"] = np.nan\n\n    for tgt, model in trained_models.items():\n        mask = df_feat[\"target_name\"] == tgt\n        if mask.sum() == 0:\n            continue\n        X = df_feat.loc[mask, feature_cols]\n        df_feat.loc[mask, \"prediction\"] = model.predict(X)\n\n    return df_feat\n\ndef predict_biomass_unseen(\n    df_unseen,\n    trained_models,\n    selected_features,\n    directory\n):\n    \"\"\"\n    Predict biomass on unseen data where target is NOT present.\n    \"\"\"\n\n    df_feat = generate_selected_features_only(\n        df=df_unseen,\n        selected_features=selected_features,\n        directory=directory\n    )\n\n    df_feat[\"prediction\"] = np.nan\n\n    for tgt, model in trained_models.items():\n        mask = df_feat[\"target_name\"] == tgt\n        if mask.sum() == 0:\n            continue\n\n        X = df_feat.loc[mask, selected_features]\n        df_feat.loc[mask, \"prediction\"] = model.predict(X)\n\n    return df_feat\n\n\n\ndef canonicalize_features(base_features):\n    \"\"\"\n    Remove _x/_y duplicates by keeping only one version.\n    \"\"\"\n    seen = {}\n    clean = []\n    for f in base_features:\n        root = f.replace(\"_x\", \"\").replace(\"_y\", \"\")\n        if root not in seen:\n            seen[root] = f\n            clean.append(f)\n    return clean\n\ndef drop_duplicate_columns_strict(df):\n    \"\"\"\n    Remove duplicate column NAMES (not just _x/_y).\n    Keeps first occurrence.\n    \"\"\"\n    return df.loc[:, ~df.columns.duplicated()]\n\n\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T04:59:46.944140Z","iopub.execute_input":"2026-01-04T04:59:46.944545Z","iopub.status.idle":"2026-01-04T04:59:47.159866Z","shell.execute_reply.started":"2026-01-04T04:59:46.944499Z","shell.execute_reply":"2026-01-04T04:59:47.158704Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import joblib\n# import os\n\n# # LOAD_DIR = \"/kaggle/input/criso-model-1/artifacts\"\n# LOAD_DIR = \"/kaggle/input/criso-catboost/artifacts\"\n# # LOAD_DIR = \"/kaggle/input/criso-catboost/artifacts\"\n# # LOAD_DIR = \"/kaggle/input/criso-trees/artifacts\"\n\n# # Load features\n# feature_cols = joblib.load(f\"{LOAD_DIR}/feature_cols.joblib\")\n\n# # Load models\n# trained_models = {}\n# for fname in os.listdir(LOAD_DIR):\n#     if fname.startswith(\"lgbm_\") and fname.endswith(\".joblib\"):\n#         tgt = fname.replace(\"lgbm_\", \"\").replace(\".joblib\", \"\")\n#         trained_models[tgt] = joblib.load(f\"{LOAD_DIR}/{fname}\")\n\n# print(\"✅ Models and features loaded\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T05:00:00.705384Z","iopub.execute_input":"2026-01-04T05:00:00.706722Z","iopub.status.idle":"2026-01-04T05:00:00.900337Z","shell.execute_reply.started":"2026-01-04T05:00:00.706674Z","shell.execute_reply":"2026-01-04T05:00:00.899317Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# df_preds = predict_biomass_unseen(\n#     df_unseen=test,\n#     trained_models=trained_models,\n#     selected_features=feature_cols,\n#     directory=directory\n# )\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T05:00:04.006404Z","iopub.execute_input":"2026-01-04T05:00:04.006787Z","iopub.status.idle":"2026-01-04T05:00:05.497480Z","shell.execute_reply.started":"2026-01-04T05:00:04.006749Z","shell.execute_reply":"2026-01-04T05:00:05.496355Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# df_preds[[\n#     \"sample_id\",\n#     \"target_name\",\n#     # \"target\",\n#     \"prediction\"\n# ]]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T05:00:07.198850Z","iopub.execute_input":"2026-01-04T05:00:07.199186Z","iopub.status.idle":"2026-01-04T05:00:07.212711Z","shell.execute_reply.started":"2026-01-04T05:00:07.199156Z","shell.execute_reply":"2026-01-04T05:00:07.211771Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# df_preds.rename(columns = {'prediction':'target'}, inplace = True)\n# df_preds = df_preds[['sample_id','target']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T05:00:52.688734Z","iopub.execute_input":"2026-01-04T05:00:52.689099Z","iopub.status.idle":"2026-01-04T05:00:52.695794Z","shell.execute_reply.started":"2026-01-04T05:00:52.689065Z","shell.execute_reply":"2026-01-04T05:00:52.694861Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Identify rows in submission where sample_id contains 'Dry_Dead'\n# mask = submission['sample_id'].str.contains('Dry_Dead', case=False)\n\n# # Create a mapping from df_preds\n# target_map = df_preds.set_index('sample_id')['target']\n\n# # Update 'prediction' in submission only for 'Dry_Dead' rows\n# submission.loc[mask, 'prediction'] = submission.loc[mask, 'sample_id'].map(target_map)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T05:04:36.916913Z","iopub.execute_input":"2026-01-04T05:04:36.917311Z","iopub.status.idle":"2026-01-04T05:04:36.928215Z","shell.execute_reply.started":"2026-01-04T05:04:36.917275Z","shell.execute_reply":"2026-01-04T05:04:36.926980Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# submission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T05:04:40.963444Z","iopub.execute_input":"2026-01-04T05:04:40.963783Z","iopub.status.idle":"2026-01-04T05:04:40.973291Z","shell.execute_reply.started":"2026-01-04T05:04:40.963754Z","shell.execute_reply":"2026-01-04T05:04:40.972336Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# submission = submission[['sample_id','target']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-03T14:58:32.313542Z","iopub.execute_input":"2026-01-03T14:58:32.314172Z","iopub.status.idle":"2026-01-03T14:58:32.319128Z","shell.execute_reply.started":"2026-01-03T14:58:32.314141Z","shell.execute_reply":"2026-01-03T14:58:32.318320Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# submission.to_csv('submission.csv', index = False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T05:05:53.909254Z","iopub.execute_input":"2026-01-04T05:05:53.909650Z","iopub.status.idle":"2026-01-04T05:05:53.916727Z","shell.execute_reply.started":"2026-01-04T05:05:53.909611Z","shell.execute_reply":"2026-01-04T05:05:53.915805Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# submission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T05:05:19.394574Z","iopub.execute_input":"2026-01-04T05:05:19.395674Z","iopub.status.idle":"2026-01-04T05:05:19.400054Z","shell.execute_reply.started":"2026-01-04T05:05:19.395625Z","shell.execute_reply":"2026-01-04T05:05:19.398929Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}