{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":14451968,"sourceType":"datasetVersion","datasetId":9230857},{"sourceId":3033643,"sourceType":"datasetVersion","datasetId":1857775},{"sourceId":4535,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":3327,"modelId":986}],"dockerImageVersionId":31260,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\nimport os\nimport sys\nimport glob\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nimport torch\nfrom sklearn.linear_model import Ridge\nfrom sklearn.model_selection import KFold, cross_val_predict\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.impute import SimpleImputer\nfrom transformers import AutoModel, AutoImageProcessor\nimport timm\nimport warnings\nfrom sklearn.metrics import r2_score\nfrom sklearn.ensemble import RandomForestRegressor, ExtraTreesRegressor, StackingRegressor\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.decomposition import PCA\nfrom tqdm.auto import tqdm # Import tqdm for progress bars\n\n# Imports for Level 2 Stacking\nfrom sklearn.ensemble import HistGradientBoostingRegressor\nfrom sklearn.multioutput import MultiOutputRegressor\n\nwarnings.filterwarnings(\"ignore\")\n\n# ---------------- GLOBAL CONFIG ----------------\nDATA_DIR_DEFAULT = \"/kaggle/input/ns_image_metadata_challenge\"\nSEED = 42\nN_FOLDS = 5\nBATCH_SIZE = 4 # Prilagodi po potrebi (npr. 8, 16) zavisno od VRAM-a\n\nTARGETS = [\"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\", \"GDM_g\", \"Dry_Total_g\"]\nMETA_FEATURES = ['Pre_GSHH_NDVI', 'Height_Ave_cm']\n\n# Kalibracioni parametri (ostavljeni iz originalnog koda)\nCALIB_PARAMS = {\n    'Dry_Green_g':  {'a': 0.78771466, 'b': 7.1499996},\n    'Dry_Dead_g':   {'a': 0.92350906, 'b': 2.7188596},\n    'Dry_Clover_g': {'a': 1.15238631, 'b': 0.8628082},\n    'GDM_g':        {'a': 0.97447401, 'b': 2.9059562},\n    'Dry_Total_g':  {'a': 0.96117550, 'b': 4.0000915}\n}\n\n# Smanjen broj alpha vrednosti za bržu pretragu, ali možete eksperimentisati\nEXTENDED_RIDGE_ALPHAS = [1.0, 10.0, 50.0, 100.0, 500.0]\n\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(\"DEVICE:\", DEVICE)\n\n# ---------------- HELPERS ----------------\ndef find_dinov2_path():\n    patterns = [\n        \"/kaggle/input/facebookdinov2-large/facebook_dinov2-large\",\n        \"/kaggle/input/dinov2/pytorch/base/1\",\n        \"/kaggle/input/dinov2\",\n        \"/kaggle/input/facebookdinov2-large\" # Dodaj kao fallback\n    ]\n    for p in patterns:\n        if p and os.path.exists(p):\n            if os.path.exists(os.path.join(p, \"config.json\")):\n                return p\n    # Pokušaj generičku pretragu za 'dinov2' u imenu foldera\n    for root, dirs, files in os.walk(\"/kaggle/input\"):\n        for dir_name in dirs:\n            if \"dinov2\" in dir_name.lower():\n                full_path = os.path.join(root, dir_name)\n                if os.path.exists(os.path.join(full_path, \"config.json\")):\n                    return full_path\n    return None\n\ndef find_convnext_weight():\n    files = glob.glob(\"/kaggle/input/**/convnext*.pth\", recursive=True)\n    return files[0] if files else None\n\ndef tta_images(pil_img, tta_enabled=True):\n    if not tta_enabled:\n        return [pil_img]\n    return [\n        pil_img,\n        pil_img.transpose(Image.FLIP_LEFT_RIGHT),\n        pil_img.transpose(Image.FLIP_TOP_BOTTOM),\n        pil_img.transpose(Image.ROTATE_180)\n    ]\n\ndef resolve_image_path(base_dirs, rel):\n    name = os.path.basename(rel)\n    # Lista potencijalnih putanja, uključujući originalnu i varijante unutar podfoldera\n    candidates = [\n        rel, # Originalna relativna putanja\n        os.path.join(base_dirs[0], name), # Prvi base_dir (npr. DATA_BASE/test)\n        os.path.join(base_dirs[0], \"images\", name),\n        os.path.join(base_dirs[0], \"test\", name),\n        os.path.join(base_dirs[1], name), # Drugi base_dir (npr. DATA_BASE)\n        os.path.join(base_dirs[1], \"images\", name),\n        os.path.join(base_dirs[1], \"test\", name)\n    ]\n    for c in candidates:\n        if os.path.exists(c):\n            return c\n    # Fallback za originalnu relativnu putanju ako nijedna ne postoji\n    # U suprotnom, ovo bi moglo dovesti do FileNotFoundError kasnije\n    print(f\"WARNING: Image path not found for {rel}. Returning original relative path as fallback.\")\n    return rel\n\n# ---------------- DATA LOCATION HANDLING ----------------\ndef locate_data_paths(user_defined_dir=None):\n    candidates = []\n    if user_defined_dir:\n        candidates.append(user_defined_dir)\n\n    candidates += [\n        DATA_DIR_DEFAULT,\n        \"/kaggle/input\",\n        \"/workspace/datasets\"\n    ]\n\n    for base in candidates:\n        train_path = os.path.join(base, \"train.csv\")\n        test_path = os.path.join(base, \"test.csv\")\n        if os.path.exists(train_path) and os.path.exists(test_path):\n            return train_path, test_path, base\n\n    for root, dirs, files in os.walk(\"/kaggle/input\", topdown=True):\n        if \"train.csv\" in files and \"test.csv\" in files:\n            train_path = os.path.join(root, \"train.csv\")\n            test_path = os.path.join(root, \"test.csv\")\n            return train_path, test_path, root\n\n    if os.path.exists(\"train.csv\") and os.path.exists(\"test.csv\"):\n        return \"train.csv\", \"test.csv\", os.getcwd()\n\n    raise FileNotFoundError(\"Cannot locate train.csv and test.csv. Please provide path or ensure dataset is mounted.\")\n\n# ---------------- MODEL LOADING ----------------\nDINOV2_PATH = find_dinov2_path()\nif DINOV2_PATH is None:\n    print(\"WARNING: DINOv2 not found via standard paths. Attempting default hardcoded path.\")\n    DINOV2_PATH = \"/kaggle/input/facebookdinov2-large/facebook_dinov2-large\"\n\nprint(f\"Loading DINOv2 from: {DINOV2_PATH}\")\ntry:\n    processor = AutoImageProcessor.from_pretrained(DINOV2_PATH, local_files_only=True)\n    dinov2 = AutoModel.from_pretrained(DINOV2_PATH, local_files_only=True).to(DEVICE)\n    dinov2.eval()\nexcept Exception as e:\n    print(f\"CRITICAL ERROR loading DINOv2: {e}\")\n    sys.exit(1)\n\nconvnext_name = \"convnext_base.fb_in22k\"\nconv = timm.create_model(convnext_name, pretrained=False, num_classes=0)\nconv_weight = find_convnext_weight()\nif conv_weight:\n    print(f\"Loading ConvNeXt weights from: {conv_weight}\")\n    sd = torch.load(conv_weight, map_location=\"cpu\")\n    if isinstance(sd, dict):\n        for k in [\"model\", \"state_dict\", \"model_state\"]:\n            if k in sd:\n                sd = sd[k]\n                break\n    conv.load_state_dict(sd, strict=False)\nelse:\n    print(\"WARNING: ConvNeXt weights not found! Running with random init (may affect performance).\")\n\nconv.to(DEVICE)\nconv.eval()\n\n# ---------------- FEATURE EXTRACTION ----------------\ndef extract_features(img_paths_list):\n    dino_feats = []\n    conv_feats = []\n    print(f\"Extracting features for {len(img_paths_list)} images...\")\n    for i in tqdm(range(0, len(img_paths_list), BATCH_SIZE), desc=\"Feature Extraction\"):\n        batch_paths = img_paths_list[i:i+BATCH_SIZE]\n        pil_imgs = []\n        tta_counts = []\n        for p in batch_paths:\n            try:\n                img = Image.open(p).convert(\"RGB\")\n            except Exception as e:\n                print(f\"Error loading image {p}: {e}. Creating dummy image.\")\n                img = Image.new(\"RGB\", (384, 384), (0, 0, 0)) # Koristi 384x384 za konzistentnost sa ConvNeXt\n            variants = tta_images(img, True)\n            pil_imgs.extend(variants)\n            tta_counts.append(len(variants))\n\n        # DINOv2\n        if pil_imgs:\n            inputs = processor(images=pil_imgs, return_tensors=\"pt\").to(DEVICE)\n            with torch.no_grad():\n                outputs = dinov2(**inputs).last_hidden_state[:, 0, :]\n            curr_idx = 0\n            outputs_np = outputs.cpu().numpy()\n            for count in tta_counts:\n                feat_avg = outputs_np[curr_idx : curr_idx+count].mean(axis=0)\n                dino_feats.append(feat_avg)\n                curr_idx += count\n        else:\n            for count in tta_counts:\n                dino_feats.append(np.zeros(dinov2.config.hidden_size))\n\n        # ConvNeXt\n        tensors = []\n        mean = torch.tensor([0.485, 0.456, 0.406]).view(3,1,1).to(DEVICE)\n        std = torch.tensor([0.229, 0.224, 0.225]).view(3,1,1).to(DEVICE)\n        if pil_imgs:\n            for img in pil_imgs:\n                # Resize na 384x384 za ConvNeXt\n                img_r = img.resize((384, 384))\n                arr = torch.from_numpy(np.array(img_r)).permute(2,0,1).float().to(DEVICE) / 255.0\n                arr = (arr - mean) / std\n                tensors.append(arr)\n            batch_t = torch.stack(tensors)\n            with torch.no_grad():\n                out_c = conv(batch_t)\n            out_c_np = out_c.cpu().numpy()\n            curr_idx = 0\n            for count in tta_counts:\n                feat_avg = out_c_np[curr_idx : curr_idx+count].mean(axis=0)\n                conv_feats.append(feat_avg)\n                curr_idx += count\n        else:\n            for count in tta_counts:\n                conv_feats.append(np.zeros(conv.num_features))\n\n    return np.array(dino_feats), np.array(conv_feats)\n\ndef get_metadata(df, image_col=\"image_path\", is_train=True, imputer=None, scaler=None):\n    meta_cols_in_df = [f for f in META_FEATURES if f in df.columns]\n\n    if is_train:\n        if not meta_cols_in_df:\n            print(\"WARNING: No configured metadata features found in train set. Returning empty metadata.\")\n            return np.zeros((len(df[image_col].unique()), 0), dtype=np.float32), df[image_col].unique(), None, None\n\n        meta_df = df.groupby(image_col)[meta_cols_in_df].first().reset_index()\n        X_meta_raw = meta_df[meta_cols_in_df].values.astype(np.float32)\n        imputer = SimpleImputer(strategy=\"mean\")\n        scaler = StandardScaler()\n        X_meta_processed = imputer.fit_transform(X_meta_raw)\n        X_meta_processed = scaler.fit_transform(X_meta_processed)\n        return X_meta_processed, meta_df[image_col].values, imputer, scaler\n    else:\n        if not meta_cols_in_df or imputer is None or scaler is None:\n            print(\"WARNING: No configured metadata features or imputer/scaler not fitted during training. Returning empty metadata for test.\")\n            return np.zeros((len(df[image_col].unique()), 0), dtype=np.float32), df[image_col].unique()\n\n        meta_df = df.groupby(image_col)[meta_cols_in_df].first().reset_index()\n        X_meta_raw = meta_df[meta_cols_in_df].values.astype(np.float32)\n        X_meta_processed = imputer.transform(X_meta_raw)\n        X_meta_processed = scaler.transform(X_meta_processed)\n        return X_meta_processed, meta_df[image_col].values\n\n# ---------------- MAIN PIPELINE: DATA LOADING & FEATURE ENGINEERING ----------------\ntry:\n    train_csv, test_csv, DATA_BASE = locate_data_paths()\n    print(f\"Detected dataset base: {DATA_BASE}\")\n    print(f\"Using train.csv: {train_csv}\")\n    print(f\"Using test.csv:  {test_csv}\")\nexcept Exception as e:\n    print(str(e))\n    print(\"Molimo vas da navedete tačnu putanju ili proverite dataset. Kraj rada.\")\n    sys.exit(1)\n\ntrain_df = pd.read_csv(train_csv)\ntest_df = pd.read_csv(test_csv)\n\nrequired = [\"image_path\", \"target_name\", \"target\"]\nfor col in required:\n    if col not in train_df.columns:\n        raise ValueError(f\"train.csv must contain column: {col}\")\n\n# unique_imgs_test treba da sadrži samo image_path stringove\nunique_imgs_test = test_df['image_path'].unique()\ntest_dirs = [os.path.join(DATA_BASE, \"test\"), DATA_BASE] # Redosled je bitan, test images bi trebalo biti u DATA_BASE/test\nresolved_paths_test = [resolve_image_path(test_dirs, p) for p in unique_imgs_test]\n\nprint(\"Preparing training data (pivot to wide) ...\")\ntrain_wide = train_df.pivot(index=\"image_path\", columns=\"target_name\", values=\"target\").reset_index()\ntrain_wide[TARGETS] = np.log1p(train_wide[TARGETS])\n\nmetadata_cols = [col for col in train_df.columns if col in META_FEATURES]\nif metadata_cols:\n    metadata_df = train_df.groupby(\"image_path\")[metadata_cols].first().reset_index()\n    train_wide = train_wide.merge(metadata_df, on=\"image_path\", how=\"left\")\n    print(f\"✅ Merged {len(metadata_cols)} metadata columns back into train_wide: {metadata_cols}\")\n\ntrain_paths = [resolve_image_path([os.path.join(DATA_BASE, \"train\"), DATA_BASE], p) for p in train_wide[\"image_path\"]]\n\nXt_dino, Xt_conv = extract_features(train_paths)\nX_dino, X_conv = extract_features(resolved_paths_test)\n\nXt_meta_from_train, unique_imgs_train_meta, meta_imputer, meta_scaler = get_metadata(\n    train_wide, image_col=\"image_path\", is_train=True\n)\ntrained_meta_dim = Xt_meta_from_train.shape[1]\nprint(f\"Extracted {trained_meta_dim} metadata features from training set.\")\n\nprint(\"\\nProcessing Metadata for Test and Final Fusion...\")\nX_meta_from_test, unique_imgs_test_meta_for_meta_extraction = get_metadata( # Promenjeno ime varijable da ne bi bilo konfuzije\n    test_df, image_col=\"image_path\", is_train=False, imputer=meta_imputer, scaler=meta_scaler\n)\n\ndef align_test_metadata(X_meta_test_in, trained_dim, n_test_samples):\n    if X_meta_test_in.shape[1] == trained_dim:\n        return X_meta_test_in\n\n    if X_meta_test_in.shape[1] < trained_dim:\n        pad_width = trained_dim - X_meta_test_in.shape[1]\n        return np.pad(X_meta_test_in, ((0, 0), (0, pad_width)), 'constant', constant_values=0)\n\n    return X_meta_test_in[:, :trained_dim]\n\nX_meta_from_test_aligned = align_test_metadata(X_meta_from_test, trained_meta_dim, len(unique_imgs_test))\n\n# Final feature matrices pre PCA\nXt_full_raw = np.hstack([Xt_dino, Xt_conv, Xt_meta_from_train])\nX_full_raw  = np.hstack([X_dino, X_conv, X_meta_from_test_aligned])\n\nprint(f\"Initial Feature Shape (Train): {Xt_full_raw.shape}\")\nprint(f\"Initial Feature Shape (Test):  {X_full_raw.shape}\")\n\n# --- GLOBAL PCA REDUKCIJA DIMENZIONALNOSTI ---\nprint(\"Applying Global PCA for dimensionality reduction...\")\npca_transformer = PCA(n_components=0.95, random_state=SEED)\nXt_full = pca_transformer.fit_transform(Xt_full_raw)\nX_full = pca_transformer.transform(X_full_raw)\n\nprint(f\"Reduced Feature Shape (Train after PCA): {Xt_full.shape}\")\nprint(f\"Reduced Feature Shape (Test after PCA):  {X_full.shape}\")\n\n# ---------------- Level 1 Stacking Boost per target ----------------\nprint(\"\\n--- Starting Stacking Boost: Ridge + RF + ET (no LightGBM) per target ---\")\n\nkf = KFold(n_splits=N_FOLDS, shuffle=True, random_state=SEED)\n\ndef build_base_estimators(alpha_val):\n    estimators = []\n    estimators.append(('ridge', Ridge(alpha=alpha_val, random_state=SEED)))\n    estimators.append(('rf', RandomForestRegressor(n_estimators=100, random_state=SEED, n_jobs=-1)))\n    estimators.append(('et', ExtraTreesRegressor(n_estimators=200, random_state=SEED, n_jobs=-1)))\n    return estimators\n\n# INICIJALIZACIJA REČNIKA ZA LEVEL 2 STACKING\noof_predictions_from_level1_stacking = {}\ntest_predictions_from_level1_stacking = {}\n\nfor t in tqdm(TARGETS, desc=\"Processing Targets\"):\n    print(f\"\\nSTACKING BOOST: Target={t}\")\n    y = train_wide[t].values\n\n    best_alpha = None\n    best_r2 = -np.inf\n    best_oof_pred_for_target = None # Varijabla za čuvanje najboljih OOF predikcija\n\n    for alpha_val in tqdm(EXTENDED_RIDGE_ALPHAS, desc=f\"  Tuning alpha for {t}\", leave=False):\n        estimators = build_base_estimators(alpha_val)\n\n        stacking_model = StackingRegressor(\n            estimators=estimators,\n            final_estimator=Ridge(alpha=1.0, random_state=SEED),\n            cv=kf,\n            n_jobs=-1\n        )\n\n        oof_pred = cross_val_predict(stacking_model, Xt_full, y, cv=kf, n_jobs=-1)\n\n        r2 = r2_score(y, oof_pred)\n        print(f\"  Alpha={alpha_val:.2f}: OOF R2={r2:.4f}\")\n        if r2 > best_r2:\n            best_r2 = r2\n            best_alpha = alpha_val\n            best_oof_pred_for_target = oof_pred # Sačuvaj OOF predikcije za najbolju alfu\n\n    # Popuni oof_predictions_from_level1_stacking sa najboljim OOF predikcijama za ovaj target\n    oof_predictions_from_level1_stacking[t] = best_oof_pred_for_target\n\n    # Fit final model on full training data with best_alpha and predict on test\n    estimators_final = build_base_estimators(best_alpha)\n    final_model = StackingRegressor(\n        estimators=estimators_final,\n        final_estimator=Ridge(alpha=1.0, random_state=SEED),\n        cv=kf,\n        n_jobs=-1\n    )\n    final_model.fit(Xt_full, y)\n\n    test_pred_raw = final_model.predict(X_full)\n\n    # Kalibracija\n    preds_expm1 = np.expm1(test_pred_raw)\n    p = CALIB_PARAMS[t]\n    preds_cal = (preds_expm1 * p['a']) + p['b']\n    preds_cal = np.maximum(preds_cal, 0)\n\n    # Popuni test_predictions_from_level1_stacking sa kalibrisanim predikcijama za ovaj target\n    test_predictions_from_level1_stacking[t] = preds_cal\n\n    print(f\"  -> Best alpha for {t}: {best_alpha} (OOF R2: {best_r2:.4f})\")\n\n# ---------------- Level 2 Stacking (Meta-Model) ----------------\nprint(\"\\n--- Starting Level 2 Stacking (Meta-Model) ---\")\n\n# --- KORAK 1: Pripremi meta-feature (X_meta) i meta-target (y_meta) za trening ---\nX_meta_train = np.column_stack([oof_predictions_from_level1_stacking[t] for t in TARGETS])\ny_meta_train = train_wide[TARGETS].values\n\nprint(f\"Meta-features shape (train): {X_meta_train.shape}\")\nprint(f\"Meta-targets shape (train): {y_meta_train.shape}\")\n\n# --- KORAK 2: Treniraj Level 2 Meta-Model (MultiOutputRegressor) ---\nmeta_base_estimator = HistGradientBoostingRegressor(\n    random_state=SEED,\n    max_iter=500,\n    max_depth=5,\n    learning_rate=0.05,\n)\n\nlevel2_meta_model = MultiOutputRegressor(meta_base_estimator, n_jobs=-1)\n\nprint(\"Training Level 2 Meta-Model...\")\nlevel2_meta_model.fit(X_meta_train, y_meta_train)\nprint(\"Level 2 Meta-Model training complete.\")\n\n# --- KORAK 3: Evaluacija OOF R2 meta-modela ---\nmeta_oof_preds = level2_meta_model.predict(X_meta_train)\nprint(\"\\nMeta-Model OOF R2 na Level 1 predikcijama (trening set):\")\nfor i, target in enumerate(TARGETS):\n    r2 = r2_score(y_meta_train[:, i], meta_oof_preds[:, i])\n    print(f\"  R2 za {target}: {r2:.4f}\")\n\n# --- KORAK 4: Pripremi meta-feature za test set ---\nX_meta_test = np.column_stack([test_predictions_from_level1_stacking[t] for t in TARGETS])\nprint(f\"\\nMeta-features shape (test): {X_meta_test.shape}\")\n\n# --- KORAK 5: Generiši finalne predikcije sa Level 2 Meta-Modelom ---\nfinal_predictions = level2_meta_model.predict(X_meta_test)\n\nfinal_predictions[final_predictions < 0] = 0\n\n# --- KORAK 6: Generiši submission fajl (KONAČNA I PROVERENA VERZIJA) ---\n\n# Mapiranje targeta na indeks kolone u final_predictions (npr. 'Dry_Green_g': 0)\ntarget_to_col_idx = {target: i for i, target in enumerate(TARGETS)} # ISPRAVLJENO: Koristi GLOBALNU 'TARGETS' listu\n\n# Mapiranje rešenih putanja slika na indeks reda u final_predictions (npr. 'st/ID1001187975.jpg': 0)\n# unique_imgs_test sadrži rešene putanje slika (npr. 'st/ID1001187975.jpg')\nimage_path_to_row_idx = {path: i for i, path in enumerate(resolved_paths_test)} # Korigovano: Koristi resolved_paths_test\n\nfinal_submission_rows = []\n\n# Iteriramo kroz ORIGINALNI test_df, jer on sadrži sve sample_id i target_name parove\nfor _, row in test_df.iterrows():\n    original_sample_id = row['sample_id']\n    original_image_path = row['image_path']\n    target_name_for_row = row['target_name']\n\n    # Moramo da rešimo putanju slike iz test_df da bi se poklapala sa ključevima u image_path_to_row_idx\n    resolved_img_path = resolve_image_path(test_dirs, original_image_path)\n\n    predicted_value = 0.0\n\n    # Pronađi indeks reda i kolone za finalnu predikciju\n    row_idx_in_predictions = image_path_to_row_idx.get(resolved_img_path)\n    col_idx_in_predictions = target_to_col_idx.get(target_name_for_row)\n\n    if row_idx_in_predictions is not None and col_idx_in_predictions is not None:\n        predicted_value = final_predictions[row_idx_in_predictions, col_idx_in_predictions]\n\n    final_submission_rows.append({'sample_id': original_sample_id, 'target': predicted_value})\n\n# Kreiraj DataFrame iz liste rečnika\nfinal_submission_df = pd.DataFrame(final_submission_rows)\n\n# Sortiranje submission DataFrame-a radi konzistentnog redosleda (kao što Kaggle očekuje)\nif not final_submission_df.empty:\n    final_submission_df['image_id_numeric'] = final_submission_df['sample_id'].apply(lambda x: int(x.split('__')[0][2:]))\n    final_submission_df['target_name_category'] = pd.Categorical(final_submission_df['sample_id'].apply(lambda x: x.split('__')[1]), categories=TARGETS, ordered=True) # ISPRAVLJENO: Koristi GLOBALNU 'TARGETS' listu\n    final_submission_df = final_submission_df.sort_values(by=['image_id_numeric', 'target_name_category']).drop(columns=['image_id_numeric', 'target_name_category'])\n\n\nsubmission_filename = 'submission_level2_stacked.csv'\nfinal_submission_df.to_csv(submission_filename, index=False)\n\nprint(f\"\\n🎯 Final Level 2 Stacking submission written: {submission_filename}\")\nprint(\"Prikaz prvih 5 redova submission fajla (Long Format):\")\nprint(final_submission_df.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-31T12:26:08.299952Z","iopub.execute_input":"2026-01-31T12:26:08.300242Z","iopub.status.idle":"2026-01-31T12:46:13.251441Z","shell.execute_reply.started":"2026-01-31T12:26:08.300210Z","shell.execute_reply":"2026-01-31T12:46:13.250777Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ---------------------- KAGGLE-COMPATIBLE FINAL SUBMISSION BLOCK ----------------------\n# Upotrebljava promenljive iz tvoje skripte: \n#   - train_wide, test_df, TARGETS\n#   - oof_predictions_from_level1_stacking, test_predictions_from_level1_stacking\n#   - resolved_paths_test (ili unique_imgs_test), resolve_image_path, test_dirs\n# Ne trenira/ucitava nista novo.\n\nimport numpy as np\nimport pandas as pd\nfrom pathlib import Path\n\nprint(\"\\n--- Creating Kaggle-compatible submission(s) (no retrain) ---\")\n\n# --- Helper: locate sample_submission (if available) to match exact order --- \n# Prefer to use sample_submission.csv if present in dataset base to guarantee exact row order.\nsample_sub_paths = [\n    Path(DATA_BASE) / \"sample_submission.csv\" if 'DATA_BASE' in globals() else None,\n    Path(\"/kaggle/input/csiro-biomass/sample_submission.csv\"),\n    Path(\"/kaggle/input\") / \"sample_submission.csv\"\n]\nsample_submission_path = None\nfor p in sample_sub_paths:\n    if p is not None and p.exists():\n        sample_submission_path = p\n        break\n\nif sample_submission_path is not None:\n    print(f\"Using sample_submission to enforce row order: {sample_submission_path}\")\n    sample_sub_df = pd.read_csv(sample_submission_path)\n    expected_order = sample_sub_df['sample_id'].values\nelse:\n    # Fallback: use test_df's sample_id order (should match sample_submission format)\n    print(\"sample_submission.csv not found in inputs — using test_df ordering as fallback.\")\n    # Build expected order directly from test_df\n    expected_order = test_df['sample_id'].values\n\n# --- Build original-space matrices (no retrain) ---\neps = 1e-9\n# original-space y_train\ny_train_orig = np.expm1(train_wide[TARGETS].values)\n\n# OOF meta preds -> convert from log-space (if those OOF are in log-space)\nX_oof_orig = np.column_stack([np.expm1(oof_predictions_from_level1_stacking[t]) for t in TARGETS])\n\n# Test meta preds from your level1 stacking pipeline; expected to already be in original space\nX_test_orig = np.column_stack([test_predictions_from_level1_stacking[t] for t in TARGETS])\n\nprint(f\"Shapes: y_train_orig {y_train_orig.shape}, X_oof_orig {X_oof_orig.shape}, X_test_orig {X_test_orig.shape}\")\n\n# ---------------- STEP 1: Target-wise bias correction (mean ratio) ----------------\nbias_factors = {}\nfor i, t in enumerate(TARGETS):\n    mu_train = float(y_train_orig[:, i].mean())\n    mu_oof   = float(X_oof_orig[:, i].mean())\n    factor = mu_train / (mu_oof + eps)\n    bias_factors[t] = factor\n    print(f\"Bias factor for {t}: {factor:.4f}\")\n\nX_test_bias = X_test_orig.copy()\nfor i, t in enumerate(TARGETS):\n    X_test_bias[:, i] *= bias_factors[t]\n\n# ---------------- STEP 2: Soft shrinkage toward train median (LB-stabilizer) ----------------\nshrink_strength = 0.15  # tweakable: 0.10-0.25 typical for LB improvements\ntrain_medians = np.median(y_train_orig, axis=0)\nX_test_shrunk = (1.0 - shrink_strength) * X_test_bias + shrink_strength * train_medians\n\n# ---------------- STEP 3: Clip predictions to train min/max (prevent extremes) -------------\ntrain_min = y_train_orig.min(axis=0)\ntrain_max = y_train_orig.max(axis=0)\nX_test_final = np.clip(X_test_shrunk, train_min, train_max)\n\n# ---------------- Helper: Resolve image path list used to index X_test_final ----------------\nif 'resolved_paths_test' in globals() and len(resolved_paths_test) == X_test_final.shape[0]:\n    img_list = resolved_paths_test\nelif 'unique_imgs_test' in globals() and len(unique_imgs_test) == X_test_final.shape[0]:\n    img_list = list(unique_imgs_test)\nelse:\n    # Emulate mapping from test_df unique image_path order (try to keep consistent)\n    img_list = []\n    for p in test_df['image_path'].unique():\n        img_list.append(resolve_image_path(test_dirs, p) if 'resolve_image_path' in globals() else p)\n\n# Map image -> row idx\nimg_to_idx = {p: i for i, p in enumerate(img_list)}\ntarget_to_idx = {t: i for i, t in enumerate(TARGETS)}\n\n# ---------------- Build a helper that creates long-format submission DF from a matrix ----------------\ndef build_submission_from_matrix(Xmat, out_name):\n    rows = []\n    for sid in expected_order:\n        # sid format: 'IDxxxxx__TargetName'\n        # recover components\n        try:\n            parts = sid.split('__')\n            img_id_part = parts[0]\n            tgt_name = parts[1]\n        except Exception:\n            # fallback parse using test_df mapping\n            row = test_df[test_df['sample_id'] == sid].iloc[0]\n            tgt_name = row['target_name']\n            orig_img = row['image_path']\n            resolved_img = resolve_image_path(test_dirs, orig_img) if 'resolve_image_path' in globals() else orig_img\n            idx = img_to_idx.get(resolved_img, None)\n            val = float(Xmat[idx, target_to_idx[tgt_name]]) if idx is not None else 0.0\n            rows.append({'sample_id': sid, 'target': max(val, 0.0)})\n            continue\n\n        # reconstruct image path token used in mapping: find matching test_df rows with same image id\n        # image path in test_df is something like 'st/ID1001187975.jpg' — we need to find the one that contains img_id_part\n        # search in test_df for any image_path containing img_id_part\n        matches = test_df[test_df['sample_id'].str.startswith(img_id_part)]\n        if len(matches) > 0:\n            orig_img = matches.iloc[0]['image_path']\n            resolved_img = resolve_image_path(test_dirs, orig_img) if 'resolve_image_path' in globals() else orig_img\n            idx = img_to_idx.get(resolved_img, None)\n            col = target_to_idx.get(tgt_name, None)\n            val = 0.0\n            if idx is not None and col is not None and idx < Xmat.shape[0]:\n                val = float(Xmat[idx, col])\n        else:\n            val = 0.0\n\n        rows.append({'sample_id': sid, 'target': max(val, 0.0)})\n\n    out_df = pd.DataFrame(rows)\n    # final safety sort to match Kaggle expected order (sample_submission ordering)\n    out_df = out_df.set_index('sample_id').loc[expected_order].reset_index()\n    out_df.to_csv(out_name, index=False)\n    print(f\"Saved: {out_name}  (n_rows={len(out_df)})\")\n    return out_df\n\n# ---------------- Save baseline (if present) and boosted submission ----------------\n# Baseline if you previously saved final_submission_df variable:\nif 'final_submission_df' in globals():\n    try:\n        baseline_name = \"submission_level2_stacked_baseline_saved.csv\"\n        final_submission_df.to_csv(baseline_name, index=False)\n        print(f\"Baseline saved as {baseline_name}\")\n    except Exception:\n        pass\n\n# Save boosted Kaggle-ready submission (this is the file you'll upload)\nkaggle_submission_name = \"submission.csv\"  # Kaggle expects 'submission.csv' or similar; sample_submission naming is enforced by platform\nsubmission_boosted_df = build_submission_from_matrix(X_test_final, kaggle_submission_name)\n\n# Also save a named copy for traceability\nsubmission_boosted_copy = \"submission_PUBLIC_LB_boosted.csv\"\nsubmission_boosted_df.to_csv(submission_boosted_copy, index=False)\nprint(f\"Also saved human-readable copy: {submission_boosted_copy}\")\n\n# ---------------- Diagnostics: print per-target min/mean/max for final predictions ----------------\nprint(\"\\nPer-target stats for final (boosted) predictions:\")\nfor i, t in enumerate(TARGETS):\n    col = X_test_final[:, i]\n    print(f\"  {t}: min={col.min():.4f}, mean={col.mean():.4f}, max={col.max():.4f}\")\n\nprint(\"\\nFinished. 'submission.csv' is Kaggle-compatible (sample_id, target) — ready to upload.\")\n# -----------------------------------------------------------------------------------------------\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-31T13:02:37.331865Z","iopub.execute_input":"2026-01-31T13:02:37.332686Z","iopub.status.idle":"2026-01-31T13:02:37.387245Z","shell.execute_reply.started":"2026-01-31T13:02:37.332649Z","shell.execute_reply":"2026-01-31T13:02:37.386621Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# -------------------- APPEND: GENERATE 5 PUBLIC-LB VARIANTS (NO RETRAIN) --------------------\n# Inserts 5 post-hoc LB-oriented submission variants using combinations of:\n#  - bias correction (mean ratio)\n#  - varying shrink strengths toward train median\n#  - small per-target multipliers for controlled exploration\n#\n# Relies ONLY on existing in-memory variables from your run:\n#   train_wide, test_df, TARGETS,\n#   oof_predictions_from_level1_stacking, test_predictions_from_level1_stacking,\n#   resolved_paths_test (optional), resolve_image_path (optional), test_dirs (optional), DATA_BASE (optional)\n#\n# Copy/paste this block at the end of your script. It will NOT retrain or reload any model weights.\n\nimport numpy as np\nimport pandas as pd\nfrom pathlib import Path\n\nprint(\"\\n🔬 Generating 5 public-LB submission variants (post-hoc, no retrain)...\")\n\n# --- find sample_submission ordering if available (preferred) ---\nsample_submission_path = None\ncandidates = []\nif 'DATA_BASE' in globals():\n    candidates.append(Path(DATA_BASE) / \"sample_submission.csv\")\ncandidates += [\n    Path(\"/kaggle/input/csiro-biomass/sample_submission.csv\"),\n    Path(\"/kaggle/input/sample_submission.csv\"),\n    Path(\"sample_submission.csv\")\n]\nfor p in candidates:\n    if p and p.exists():\n        sample_submission_path = p\n        break\n\nif sample_submission_path is not None:\n    expected_order = pd.read_csv(sample_submission_path)['sample_id'].values\n    print(f\"Using sample_submission for ordering: {sample_submission_path}\")\nelse:\n    expected_order = test_df['sample_id'].values\n    print(\"sample_submission not found — using test_df sample_id order as fallback.\")\n\n# --- Build original-space matrices (with sensible fallbacks) ---\neps = 1e-9\n# y_train original-space\ny_train_orig = np.expm1(train_wide[TARGETS].values)\n\n# OOF meta preds -> convert from log-space if present\ntry:\n    X_oof_orig = np.column_stack([np.expm1(oof_predictions_from_level1_stacking[t]) for t in TARGETS])\nexcept Exception as e:\n    print(\"Warning building X_oof_orig:\", e)\n    X_oof_orig = np.tile(y_train_orig.mean(axis=0, keepdims=True), (y_train_orig.shape[0], 1))\n\n# Test meta preds (level1/test) - expected to already be in original space\ntry:\n    X_test_orig = np.column_stack([test_predictions_from_level1_stacking[t] for t in TARGETS])\nexcept Exception as e:\n    print(\"Warning building X_test_orig:\", e)\n    X_test_orig = np.tile(y_train_orig.mean(axis=0, keepdims=True), (1 if 'resolved_paths_test' not in globals() else len(resolved_paths_test), 1))\n\nprint(f\"Shapes -> y_train_orig: {y_train_orig.shape}, X_oof_orig: {X_oof_orig.shape}, X_test_orig: {X_test_orig.shape}\")\n\n# --- Compute bias factors (train mean / oof mean) once ---\nbias_factors = {}\nfor i, t in enumerate(TARGETS):\n    mu_train = float(np.mean(y_train_orig[:, i]))\n    mu_oof = float(np.mean(X_oof_orig[:, i])) if X_oof_orig.shape[0] > 0 else 1.0\n    bias_factors[t] = mu_train / (mu_oof + eps)\nprint(\"Bias factors:\", {k: round(v,4) for k,v in bias_factors.items()})\n\n# --- Prepare image list mapping used to index X_test_orig (try to reuse resolved_paths_test) ---\nif 'resolved_paths_test' in globals() and len(resolved_paths_test) == X_test_orig.shape[0]:\n    img_list = resolved_paths_test\nelif 'unique_imgs_test' in globals() and len(unique_imgs_test) == X_test_orig.shape[0]:\n    img_list = list(unique_imgs_test)\nelse:\n    # fallback: use unique image_path order from test_df and resolve\n    uniq = list(test_df['image_path'].unique())\n    img_list = [resolve_image_path(test_dirs, p) if 'resolve_image_path' in globals() else p for p in uniq]\n\nimg_to_idx = {p: i for i, p in enumerate(img_list)}\ntarget_to_idx = {t: i for i, t in enumerate(TARGETS)}\n\n# --- Define 5 variant configurations (shrink_strength, per-target multipliers) ---\nvariants = []\nshrinkvals = [0.05, 0.10, 0.15, 0.20, 0.30]  # five shrink strengths\n# base multipliers (no change) and a couple of mild adjustments for other variants\nmultipliers_list = [\n    {t: 1.0 for t in TARGETS},  # variant 0: baseline bias+shrink only\n    {**{t: 1.0 for t in TARGETS}, 'Dry_Dead_g': 1.02, 'Dry_Clover_g': 1.02},  # slight up for dead/clover\n    {**{t: 1.0 for t in TARGETS}, 'Dry_Clover_g': 1.03},  # slight up clover\n    {**{t: 1.0 for t in TARGETS}, 'Dry_Dead_g': 1.04, 'Dry_Total_g': 1.02},  # more up for dead\n    {**{t: 1.0 for t in TARGETS}, 'Dry_Green_g': 0.98, 'GDM_g': 1.02}  # small rebalancing\n]\nfor i in range(5):\n    variants.append({\n        'name': f'var_{i+1}',\n        'shrink': shrinkvals[i],\n        'multipliers': multipliers_list[i]\n    })\n\n# --- utility: function to convert matrix -> Kaggle long-format DF using expected_order ---\ndef matrix_to_kaggle_df(Xmat, expected_order_local):\n    rows = []\n    # Build quick mapping from image id token (IDxxxx) to resolved image path\n    # We'll scan test_df to get mapping from sample_id prefix to image_path\n    sample_prefix_to_img = {}\n    for _, r in test_df.iterrows():\n        sid = r['sample_id']\n        prefix = sid.split('__')[0]\n        sample_prefix_to_img.setdefault(prefix, r['image_path'])\n    for sid in expected_order_local:\n        try:\n            prefix, tgt = sid.split('__')\n        except Exception:\n            # fallback: search test_df\n            rec = test_df[test_df['sample_id'] == sid]\n            if rec.shape[0] == 0:\n                rows.append({'sample_id': sid, 'target': 0.0})\n                continue\n            tgt = rec.iloc[0]['target_name']\n            orig_img = rec.iloc[0]['image_path']\n            resolved_img = resolve_image_path(test_dirs, orig_img) if 'resolve_image_path' in globals() else orig_img\n            idx = img_to_idx.get(resolved_img, None)\n            val = float(Xmat[idx, target_to_idx[tgt]]) if idx is not None else 0.0\n            rows.append({'sample_id': sid, 'target': max(val, 0.0)})\n            continue\n\n        orig_img = sample_prefix_to_img.get(prefix, None)\n        if orig_img is None:\n            # fallback: try find any test_df row that contains prefix in image_path\n            matches = test_df[test_df['image_path'].str.contains(prefix, na=False)]\n            if matches.shape[0] > 0:\n                orig_img = matches.iloc[0]['image_path']\n        resolved_img = resolve_image_path(test_dirs, orig_img) if ('resolve_image_path' in globals() and orig_img is not None) else orig_img\n        idx = img_to_idx.get(resolved_img, None)\n        val = 0.0\n        if idx is not None and tgt in target_to_idx and idx < Xmat.shape[0]:\n            val = float(Xmat[idx, target_to_idx[tgt]])\n        rows.append({'sample_id': sid, 'target': max(val, 0.0)})\n    out_df = pd.DataFrame(rows)\n    # Ensure ordering matches expected_order_local exactly\n    out_df = out_df.set_index('sample_id').loc[expected_order_local].reset_index()\n    return out_df\n\n# --- Run each variant and save CSV ---\nsaved_files = []\ntrain_medians = np.median(y_train_orig, axis=0)\ntrain_min = y_train_orig.min(axis=0)\ntrain_max = y_train_orig.max(axis=0)\n\nfor idx, cfg in enumerate(variants, start=1):\n    name = cfg['name']\n    shrink = cfg['shrink']\n    multipliers = cfg['multipliers']\n\n    # 1) bias-correct\n    X_bias = X_test_orig.copy()\n    for j, t in enumerate(TARGETS):\n        X_bias[:, j] *= bias_factors[t]\n\n    # 2) apply per-target multipliers (elementwise)\n    for j, t in enumerate(TARGETS):\n        X_bias[:, j] *= multipliers.get(t, 1.0)\n\n    # 3) shrink toward train median\n    X_shrunk = (1.0 - shrink) * X_bias + shrink * train_medians\n\n    # 4) clip to train min/max\n    X_final = np.clip(X_shrunk, train_min, train_max)\n\n    # Save matrix -> kaggle DF\n    fname = f\"submission_PUBLIC_LB_variant_{idx}_{name}.csv\"\n    df_out = matrix_to_kaggle_df(X_final, expected_order)\n    df_out.to_csv(fname, index=False)\n    saved_files.append(fname)\n\n    # print variant diagnostics\n    print(f\"\\nVariant {idx} ({name}): shrink={shrink}, multipliers={multipliers}\")\n    for j, t in enumerate(TARGETS):\n        col = X_final[:, j]\n        print(f\"  {t}: min={col.min():.4f}, mean={col.mean():.4f}, max={col.max():.4f}\")\n    print(f\"Saved -> {fname}\")\n\n# Also optionally save a small JSON summary of variants\ntry:\n    import json\n    summary = {'variants': []}\n    for i, cfg in enumerate(variants, start=1):\n        summary['variants'].append({\n            'file': saved_files[i-1],\n            'shrink': cfg['shrink'],\n            'multipliers': cfg['multipliers']\n        })\n    Path(\"lb_variants_summary.json\").write_text(json.dumps(summary, indent=2))\n    print(\"\\nSaved lb_variants_summary.json\")\nexcept Exception:\n    pass\n\nprint(\"\\n✅ Created variants:\")\nfor f in saved_files:\n    print(\" -\", f)\nprint(\"\\nUpload these to public LB and keep the best-performing one. Done. ✅\")\n# ----------------------------------------------------------------------------------------------\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-31T13:05:11.526409Z","iopub.execute_input":"2026-01-31T13:05:11.527060Z","iopub.status.idle":"2026-01-31T13:05:11.573922Z","shell.execute_reply.started":"2026-01-31T13:05:11.527033Z","shell.execute_reply":"2026-01-31T13:05:11.573024Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ----------------- FINAL FIXED SUBMISSION BLOCK (ONE CSV) -----------------\nimport numpy as np\nimport pandas as pd\nfrom pathlib import Path\n\nprint(\"\\n--- Running fixed final submission block (one CSV) ---\")\n\n# 0) Load sample_submission to enforce exact order (Kaggle)\nsample_sub_path = Path(\"/kaggle/input/csiro-biomass/sample_submission.csv\")\nif not sample_sub_path.exists():\n    raise FileNotFoundError(f\"Expected sample_submission.csv at {sample_sub_path} not found.\")\nsample_sub = pd.read_csv(sample_sub_path)\nexpected_order = sample_sub[\"sample_id\"].values\nprint(f\"Using sample_submission for ordering, rows={len(expected_order)}\")\n\n# 1) Prepare original-space train / meta matrices\ny_train_orig = np.expm1(train_wide[TARGETS].values)  # (n_train_images, n_targets)\n\n# OOF meta preds — convert from log-space (safe)\nX_oof_orig = np.column_stack([np.expm1(oof_predictions_from_level1_stacking[t]) for t in TARGETS])\n\n# Test meta preds (level1/test) — expected to be in original space already\nX_test_orig = np.column_stack([test_predictions_from_level1_stacking[t] for t in TARGETS])\n\nprint(\"Shapes -> y_train_orig:\", y_train_orig.shape, \"X_oof_orig:\", X_oof_orig.shape, \"X_test_orig:\", X_test_orig.shape)\n\n# 2) Bias correction (train mean / oof mean)\neps = 1e-9\nbias_factors = {}\nfor i, t in enumerate(TARGETS):\n    denom = X_oof_orig[:, i].mean()\n    bias_factors[t] = float(y_train_orig[:, i].mean() / (denom + eps)) if denom != 0 else 1.0\nprint(\"Bias factors:\", {k: round(v,4) for k,v in bias_factors.items()})\n\n# 3) Apply bias -> (no variants here, single canonical boosted submission)\nX_test_biased = X_test_orig.copy()\nfor i, t in enumerate(TARGETS):\n    X_test_biased[:, i] *= bias_factors[t]\n\n# Optional light shrink toward train median and clipping (keeps safe)\nshrink_strength = 0.15\ntrain_medians = np.median(y_train_orig, axis=0)\nX_test_shrunk = (1.0 - shrink_strength) * X_test_biased + shrink_strength * train_medians\n\ntrain_min = y_train_orig.min(axis=0)\ntrain_max = y_train_orig.max(axis=0)\nX_test_final = np.clip(X_test_shrunk, train_min, train_max)\n\n# 4) Resolve test image paths robustly (use resolve_image_path if available)\n# We'll build a mapping resolved_path -> row index for X_test_final\nresolved_test_imgs = []\ntry:\n    # Prefer using resolved_paths_test if it exists and matches shape\n    if 'resolved_paths_test' in globals() and len(resolved_paths_test) == X_test_final.shape[0]:\n        resolved_test_imgs = list(resolved_paths_test)\n    else:\n        # Build from unique test_df image_path order and resolve each\n        uniq = list(test_df['image_path'].unique())\n        resolved_test_imgs = [resolve_image_path(test_dirs, p) if 'resolve_image_path' in globals() else p for p in uniq]\n        # If sizes mismatch, try using resolved_paths_test if longer/shorter\n        if len(resolved_test_imgs) != X_test_final.shape[0]:\n            if 'resolved_paths_test' in globals():\n                # fallback to resolved_paths_test trimmed/padded\n                rp = list(resolved_paths_test)\n                if len(rp) >= X_test_final.shape[0]:\n                    resolved_test_imgs = rp[:X_test_final.shape[0]]\n                else:\n                    # pad by repeating last (degenerate but safe)\n                    resolved_test_imgs = rp + [rp[-1]] * (X_test_final.shape[0] - len(rp))\nexcept Exception as e:\n    # Ultimate fallback: use the raw unique image_path strings\n    print(\"Warning resolving test image paths:\", e)\n    resolved_test_imgs = list(test_df['image_path'].unique())[:X_test_final.shape[0]]\n\n# Ensure length matches\nif len(resolved_test_imgs) != X_test_final.shape[0]:\n    # as last resort, create an index-based list\n    resolved_test_imgs = [f\"__IDX_{i}\" for i in range(X_test_final.shape[0])]\n\n# Map resolved image path -> row index\npred_map = {resolved_test_imgs[i]: X_test_final[i] for i in range(len(resolved_test_imgs))}\nprint(f\"Built pred_map with {len(pred_map)} entries (expected {X_test_final.shape[0]})\")\n\n# 5) Build final submission strictly following sample_submission order\ntarget_to_idx = {t: i for i,t in enumerate(TARGETS)}\nrows = []\n# For faster lookup: map from sample_id prefix (ID...) to rows in test_df\nprefix_to_rows = {}\nfor _, r in test_df.iterrows():\n    sid = r['sample_id']\n    prefix = sid.split('__')[0]\n    prefix_to_rows.setdefault(prefix, []).append(r)\n\nfor sid in expected_order:\n    # sid format: 'IDxxxxxx__TargetName'\n    try:\n        img_prefix, tgt_name = sid.split(\"__\")\n    except Exception:\n        # malformed sample id; safe fallback\n        rows.append({'sample_id': sid, 'target': 0.0})\n        continue\n\n    # Find corresponding test_df row (should exist)\n    matched_rows = prefix_to_rows.get(img_prefix, None)\n    if matched_rows is None or len(matched_rows) == 0:\n        # fallback: try find by startswith in test_df.image_path\n        candidates = test_df[test_df['image_path'].str.contains(img_prefix, na=False)]\n        if candidates.shape[0] > 0:\n            orig_img = candidates.iloc[0]['image_path']\n            resolved_img = resolve_image_path(test_dirs, orig_img) if 'resolve_image_path' in globals() else orig_img\n            pred = pred_map.get(resolved_img, None)\n            val = float(pred[target_to_idx[tgt_name]]) if (pred is not None and tgt_name in target_to_idx) else 0.0\n            rows.append({'sample_id': sid, 'target': max(val, 0.0)})\n            continue\n        else:\n            rows.append({'sample_id': sid, 'target': 0.0})\n            continue\n\n    # Usually there's exactly one matching row per prefix\n    row0 = matched_rows[0]\n    orig_img = row0['image_path']\n    # resolve it the same way we resolved the test images\n    resolved_img = resolve_image_path(test_dirs, orig_img) if 'resolve_image_path' in globals() else orig_img\n\n    pred = pred_map.get(resolved_img, None)\n    if pred is None:\n        # last fallback: try matching by endswith (maybe keys include folder)\n        found = None\n        for k in pred_map.keys():\n            if isinstance(k, str) and k.endswith(img_prefix + \".jpg\"):\n                found = pred_map[k]; break\n            if isinstance(k, str) and img_prefix in k:\n                found = pred_map[k]; break\n        pred = found\n\n    if pred is not None and tgt_name in target_to_idx:\n        val = float(pred[target_to_idx[tgt_name]])\n    else:\n        val = 0.0\n\n    rows.append({'sample_id': sid, 'target': max(val, 0.0)})\n\nsubmission_df = pd.DataFrame(rows)\n\n# 6) Sanity checks\nif submission_df.shape[0] != sample_sub.shape[0]:\n    raise RuntimeError(f\"Submission row count mismatch: expected {sample_sub.shape[0]}, got {submission_df.shape[0]}\")\n\nif submission_df.isna().any().any():\n    raise RuntimeError(\"Submission contains NaN values. Abort.\")\n\nprint(\"Non-zero targets present:\", submission_df['target'].gt(0).any())\nprint(submission_df.head())\n\n# 7) Save single Kaggle-ready CSV\nout_name = \"submission.csv\"\nsubmission_df.to_csv(out_name, index=False)\nprint(f\"\\n✅ SINGLE FINAL SUBMISSION WRITTEN: {out_name} (rows={len(submission_df)})\")\n# -------------------------------------------------------------------------\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-31T13:12:56.727919Z","iopub.execute_input":"2026-01-31T13:12:56.728215Z","iopub.status.idle":"2026-01-31T13:12:56.760735Z","shell.execute_reply.started":"2026-01-31T13:12:56.728192Z","shell.execute_reply":"2026-01-31T13:12:56.760131Z"}},"outputs":[],"execution_count":null}]}