{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":13620649,"sourceType":"datasetVersion","datasetId":8656316},{"sourceId":13635908,"sourceType":"datasetVersion","datasetId":8667286},{"sourceId":13671598,"sourceType":"datasetVersion","datasetId":8693006},{"sourceId":13698366,"sourceType":"datasetVersion","datasetId":8713496},{"sourceId":13737495,"sourceType":"datasetVersion","datasetId":8740812},{"sourceId":13743997,"sourceType":"datasetVersion","datasetId":8745224},{"sourceId":13754791,"sourceType":"datasetVersion","datasetId":8752591},{"sourceId":4533,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":3325,"modelId":986},{"sourceId":4534,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":3326,"modelId":986},{"sourceId":268942,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":230141,"modelId":251887}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport gc\nimport random\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport cv2\nfrom sklearn.linear_model import Ridge, ElasticNet \nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.metrics import r2_score, mean_squared_error, mean_absolute_error\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.cross_decomposition import PLSRegression\nimport joblib\nfrom tqdm import tqdm\nimport timm\nimport torch\nfrom PIL import Image\nimport lightgbm as lgb\nimport re\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nwarnings.filterwarnings('ignore')\n\n# --------------------- CONFIG ---------------------\nclass CFG:\n    SEED = 42\n    DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    \n    MODELS = [\n        ('convnext_large.fb_in22k_ft_in1k_384', 0.35, 1536, 384, '/kaggle/input/last-datset/convnext_large.fb_in22k_ft_in1k_384.pth'),\n        ('densenet169.tv_in1k', 0.35, 1664, 224, '/kaggle/input/last-datset/densenet169.tv_in1k.pth'),\n        ('tf_efficientnet_b7', 0.30, 2560, 300, '/kaggle/input/last-datset/tf_efficientnet_b7.pth')\n    ]\n    \n    IMG_SIZE = 384\n    \n    # Regularization\n    RIDGE_ALPHA = 150.0\n    ELASTICNET_ALPHA = 75.0\n    ELASTICNET_L1_RATIO = 0.5\n    PLS_COMPONENTS = 5\n    \n    LGB_PARAMS = {\n        'n_estimators': 350, 'learning_rate': 0.01, 'max_depth': 7, \n        'num_leaves': 31, 'subsample': 0.7, 'colsample_bytree': 0.7, \n        'reg_alpha': 1.0, 'reg_lambda': 1.0, 'random_state': SEED, \n        'n_jobs': -1, 'verbosity': -1\n    }\n    \n    NUM_FOLDS = 5\n    TARGETS = ['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'GDM_g', 'Dry_Total_g']\n    TARGET_WEIGHTS = [0.08, 0.08, 0.08, 0.16, 0.60]\n    \n    TRAIN_CSV = '/kaggle/input/csiro-biomass/train.csv'\n    TRAIN_DIR = '/kaggle/input/csiro-biomass/train'\n    TEST_CSV = '/kaggle/input/csiro-biomass/test.csv'\n    TEST_DIR = '/kaggle/input/csiro-biomass/test'\n    OUT_DIR = '/kaggle/working/ultimate_fixed_ensemble'\n    \n    os.makedirs(OUT_DIR, exist_ok=True)\n\n# --------------------- UTILITIES ---------------------\ndef competition_score(true, pred):\n    weights = np.array(CFG.TARGET_WEIGHTS)\n    r2_scores = []\n    for i in range(true.shape[1]):\n        try:\n            r2 = r2_score(true[:, i], pred[:, i])\n            r2_scores.append(max(0, r2))\n        except:\n            r2_scores.append(0.0)\n    return np.average(r2_scores, weights=weights)\n\n# --------------------- PLOTTING FUNCTIONS ---------------------\ndef plot_target_performance(y_true, y_pred, fold_scores, cv_score):\n    \"\"\"Plot comprehensive performance analysis\"\"\"\n    print(\"\\n📊 GENERATING PERFORMANCE PLOTS...\")\n    \n    # 1. Target-wise Performance\n    fig, axes = plt.subplots(2, 3, figsize=(20, 12))\n    axes = axes.ravel()\n    \n    target_performance = []\n    for i, target in enumerate(CFG.TARGETS):\n        true_vals = y_true[:, i]\n        pred_vals = y_pred[:, i]\n        \n        r2 = r2_score(true_vals, pred_vals)\n        rmse = np.sqrt(mean_squared_error(true_vals, pred_vals))\n        mae = mean_absolute_error(true_vals, pred_vals)\n        \n        target_performance.append({\n            'Target': target, 'R²': r2, 'RMSE': rmse, 'MAE': mae\n        })\n        \n        axes[i].scatter(true_vals, pred_vals, alpha=0.6, s=30, color='blue')\n        max_val = max(true_vals.max(), pred_vals.max())\n        axes[i].plot([0, max_val], [0, max_val], 'r--', lw=2)\n        axes[i].set_xlabel(f'True {target}')\n        axes[i].set_ylabel(f'Predicted {target}')\n        axes[i].set_title(f'{target}\\nR² = {r2:.3f}, RMSE = {rmse:.2f}')\n        axes[i].grid(True, alpha=0.3)\n    \n    for i in range(len(CFG.TARGETS), 6):\n        axes[i].set_visible(False)\n    \n    plt.tight_layout()\n    plt.savefig(f'{CFG.OUT_DIR}/target_performance.png', dpi=300, bbox_inches='tight')\n    plt.show()\n    \n    # 2. Fold Performance\n    plt.figure(figsize=(10, 6))\n    bars = plt.bar(range(1, CFG.NUM_FOLDS + 1), fold_scores, \n                   color=['#ff9999', '#66b3ff', '#99ff99', '#ffcc99', '#ff99cc'],\n                   alpha=0.7, edgecolor='black')\n    \n    mean_score = np.mean(fold_scores)\n    plt.axhline(y=mean_score, color='red', linestyle='--', \n                linewidth=2, label=f'Mean CV: {mean_score:.4f}')\n    \n    plt.xlabel('Fold Number')\n    plt.ylabel('Competition Score')\n    plt.title(f'Fold-wise Performance (Final CV: {cv_score:.4f})')\n    plt.legend()\n    \n    for bar, score in zip(bars, fold_scores):\n        plt.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.01, \n                f'{score:.4f}', ha='center', va='bottom', fontweight='bold')\n    \n    plt.grid(True, alpha=0.3)\n    plt.savefig(f'{CFG.OUT_DIR}/fold_performance.png', dpi=300, bbox_inches='tight')\n    plt.show()\n    \n    # 3. Residual Analysis\n    fig, axes = plt.subplots(2, 3, figsize=(20, 12))\n    axes = axes.ravel()\n    \n    for i, target in enumerate(CFG.TARGETS):\n        true_vals = y_true[:, i]\n        pred_vals = y_pred[:, i]\n        residuals = true_vals - pred_vals\n        \n        axes[i].scatter(pred_vals, residuals, alpha=0.6, s=30, color='purple')\n        axes[i].axhline(y=0, color='red', linestyle='--', linewidth=2)\n        axes[i].set_xlabel(f'Predicted {target}')\n        axes[i].set_ylabel('Residuals')\n        axes[i].set_title(f'Residuals - {target}\\n(Mean: {np.mean(residuals):.2f}, Std: {np.std(residuals):.2f})')\n        axes[i].grid(True, alpha=0.3)\n    \n    for i in range(len(CFG.TARGETS), 6):\n        axes[i].set_visible(False)\n    \n    plt.tight_layout()\n    plt.savefig(f'{CFG.OUT_DIR}/residual_analysis.png', dpi=300, bbox_inches='tight')\n    plt.show()\n    \n    return target_performance\n\ndef plot_model_analysis(fold_models):\n    \"\"\"Plot model weights and performance analysis\"\"\"\n    model_weights_data = []\n    \n    for fold in range(1, CFG.NUM_FOLDS + 1):\n        fold_model_dict = joblib.load(f\"{CFG.OUT_DIR}/fold_{fold}_models.pkl\")\n        for model_name, model_info in fold_model_dict.items():\n            model_weights_data.append({\n                'Fold': fold, 'Model': model_name, 'Weight': model_info['weight'],\n                'Avg_R2': np.mean(model_info['scores'])\n            })\n    \n    weights_df = pd.DataFrame(model_weights_data)\n    \n    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6))\n    \n    # Model weights\n    sns.boxplot(data=weights_df, x='Model', y='Weight', ax=ax1, palette='viridis')\n    ax1.set_title('Model Weights Distribution Across Folds')\n    ax1.set_ylabel('Weight')\n    ax1.tick_params(axis='x', rotation=45)\n    \n    # Model R2 scores\n    sns.boxplot(data=weights_df, x='Model', y='Avg_R2', ax=ax2, palette='viridis')\n    ax2.set_title('Model R² Scores Distribution Across Folds')\n    ax2.set_ylabel('Average R² Score')\n    ax2.tick_params(axis='x', rotation=45)\n    \n    plt.tight_layout()\n    plt.savefig(f'{CFG.OUT_DIR}/model_analysis.png', dpi=300, bbox_inches='tight')\n    plt.show()\n    \n    return weights_df\n\n# --------------------- IMPROVED GROUPING LOGIC ---------------------\ndef get_robust_group_id(path):\n    basename = os.path.basename(path)\n    id_part = re.sub(r'_(left|right)', '', basename)\n    id_part = re.sub(r'_[LR]\\.(jpg|png|jpeg)', '', id_part, flags=re.IGNORECASE)\n    id_part = re.sub(r'\\.(jpg|png|jpeg)', '', id_part, flags=re.IGNORECASE)\n    return id_part.split('_')[0]\n\ndef create_simple_groups(feature_df):\n    \"\"\"Improved grouping logic with detailed analysis\"\"\"\n    print(\"🔍 Creating robust groups...\")\n    groups = feature_df['image_path'].apply(get_robust_group_id)\n    unique_groups = len(groups.unique())\n    \n    group_counts = groups.value_counts()\n    stereo_groups = group_counts[group_counts > 1]\n    \n    print(f\"📊 UNIQUE GROUPS: {unique_groups}\")\n    print(f\"📊 Group distribution: Avg size: {group_counts.mean():.2f}, Max size: {group_counts.max()}\")\n    print(f\"📊 Groups with stereo pairs: {len(stereo_groups)}\")\n    \n    # Print sample stereo groups\n    print(\"📊 Sample stereo groups:\")\n    for group_id, count in list(stereo_groups.items())[:3]:\n        sample_images = feature_df[groups == group_id]['image_path'].tolist()\n        print(f\"   - {group_id}: {count} images\")\n        for img in sample_images[:2]:\n            print(f\"     → {os.path.basename(img)}\")\n    \n    groups = groups.astype(str)\n    return groups\n\n# --------------------- STABLE FEATURE EXTRACTOR ---------------------\nclass StableFeatureExtractor:\n    def __init__(self):\n        self.models = {}\n        print(\"🚀 Loading Models (Using Local .pth Files)...\")\n        \n        for model_name, weight, feat_dim, img_size, weights_path in CFG.MODELS:\n            print(f\"📦 Loading {model_name} from {weights_path}...\")\n            try:\n                model = timm.create_model(model_name, pretrained=False, num_classes=0)\n                \n                if not os.path.exists(weights_path):\n                    raise FileNotFoundError(f\"Local weights file not found at {weights_path}\")\n                \n                state_dict = torch.load(weights_path, map_location=CFG.DEVICE)\n                model.load_state_dict(state_dict)\n                \n                model.to(CFG.DEVICE)\n                model.eval()\n                actual_feat_dim = model.num_features \n                self.models[model_name] = (model, weight, actual_feat_dim, img_size)\n                print(f\" ✅ Loaded successfully. Actual Features: {actual_feat_dim}\")\n            except Exception as e:\n                print(f\" ❌ FAILED TO LOAD {model_name}: {e}\")\n                continue\n    \n    def find_stereo_pair(self, img_path, is_train=True):\n        dir_path = CFG.TRAIN_DIR if is_train else CFG.TEST_DIR\n        basename = os.path.basename(img_path)\n        right_basename = basename\n        \n        stereo_pairs = [\n            ('_left', '_right'), ('_right', '_left'),\n            ('_L.', '_R.'), ('_R.', '_L.'),\n            ('_L_', '_R_'), ('_R_', '_L_')\n        ]\n        \n        for left_ind, right_ind in stereo_pairs:\n            if left_ind in basename:\n                right_basename = basename.replace(left_ind, right_ind)\n                break\n            elif right_ind in basename:\n                right_basename = basename.replace(right_ind, left_ind)\n                break\n        \n        left_path = os.path.join(dir_path, basename)\n        right_path = os.path.join(dir_path, right_basename)\n        \n        if not os.path.exists(right_path) or left_path == right_path:\n            right_path = left_path \n        \n        return left_path, right_path\n    \n    def extract_single_features(self, img_path, model):\n        feat_dim = getattr(model, 'num_features', 1024)\n        try:\n            if not os.path.exists(img_path):\n                return np.random.normal(0, 0.001, feat_dim)\n            img_pil = Image.open(img_path).convert('RGB')\n            data_config = timm.data.resolve_model_data_config(model)\n            transform = timm.data.create_transform(**data_config, is_training=False)\n            img_tensor = transform(img_pil).unsqueeze(0).to(CFG.DEVICE)\n            with torch.no_grad():\n                features = model(img_tensor).cpu().numpy().flatten()\n            return features\n        except Exception as e:\n            return np.random.normal(0, 0.001, feat_dim)\n    \n    def extract_features(self, df, is_train=True):\n        print(\"🔍 Extracting Features...\")\n        all_features = []\n        all_paths = df['image_path'].tolist()\n        \n        for model_name, (model, weight, feat_dim, img_size) in self.models.items():\n            print(f\"📸 Processing {model_name}...\")\n            current_feat_dim = model.num_features\n            model_features = []\n            \n            for img_path in tqdm(all_paths, desc=model_name[:15]):\n                try:\n                    left_path, right_path = self.find_stereo_pair(img_path, is_train)\n                    left_feat = self.extract_single_features(left_path, model)\n                    right_feat = self.extract_single_features(right_path, model)\n                    \n                    # Feature fusion\n                    if left_feat.shape[0] != right_feat.shape[0]:\n                        stereo_features = left_feat \n                    else:\n                        stereo_features = (left_feat + right_feat) / 2  # Average fusion\n                    \n                    weighted_features = stereo_features * weight\n                    model_features.append(weighted_features)\n                except Exception as e:\n                    model_features.append(np.random.normal(0, 0.001, current_feat_dim))\n            \n            model_features = np.array(model_features)\n            all_features.append(model_features)\n            print(f\"   ✅ {model_name}: {model_features.shape}\")\n        \n        features_concat = np.concatenate(all_features, axis=1)\n        print(f\"🎯 Final features shape: {features_concat.shape}\")\n        return features_concat, all_paths\n\n# --------------------- PIPELINE FUNCTIONS ---------------------\ndef extract_features(is_train=True):\n    df = pd.read_csv(CFG.TRAIN_CSV if is_train else CFG.TEST_CSV)\n    if is_train:\n        wide = df.pivot_table(index='image_path', columns='target_name', values='target').reset_index()\n        wide = wide[['image_path'] + CFG.TARGETS]\n    else:\n        wide = df[['image_path']].drop_duplicates().reset_index(drop=True)\n    \n    extractor = StableFeatureExtractor()\n    features, paths = extractor.extract_features(wide, is_train)\n    \n    feature_cols = [f'feat_{i}' for i in range(features.shape[1])]\n    feature_df = pd.DataFrame(features, columns=feature_cols)\n    feature_df['image_path'] = paths\n    \n    if is_train:\n        result_df = wide.merge(feature_df, on='image_path')\n    else:\n        result_df = feature_df\n    \n    del extractor\n    gc.collect()\n    return result_df\n\n# --------------------- STABLE ENSEMBLE ---------------------\nclass StableEnsemble:\n    def __init__(self):\n        self.base_models = {\n            'ridge': Ridge(alpha=CFG.RIDGE_ALPHA, random_state=CFG.SEED),\n            'elasticnet': ElasticNet(\n                alpha=CFG.ELASTICNET_ALPHA, l1_ratio=CFG.ELASTICNET_L1_RATIO, \n                random_state=CFG.SEED, max_iter=10000\n            ),\n            'lgb': lgb.LGBMRegressor(**CFG.LGB_PARAMS) \n        }\n    \n    def train_fold(self, X_pls, y_train, X_val_pls, y_val):\n        fold_preds = np.zeros((X_val_pls.shape[0], len(CFG.TARGETS)))\n        model_weights = {}\n        fold_models = {}\n        \n        for name, model in self.base_models.items():\n            target_preds = np.zeros((X_val_pls.shape[0], len(CFG.TARGETS)))\n            target_models = []\n            model_scores = []\n            \n            for i in range(len(CFG.TARGETS)):\n                try:\n                    model_instance = model.__class__(**model.get_params())\n                    model_instance.fit(X_pls, y_train[:, i])\n                    pred = model_instance.predict(X_val_pls)\n                    target_preds[:, i] = pred\n                    target_models.append(model_instance)\n                    target_r2 = r2_score(y_val[:, i], pred)\n                    model_scores.append(max(0.0, target_r2))\n                except:\n                    target_preds[:, i] = np.mean(y_train[:, i])\n                    target_models.append(None)\n                    model_scores.append(0.0)\n            \n            avg_score = np.mean(model_scores)\n            weight = max(0.1, avg_score)\n            model_weights[name] = weight\n            fold_preds += target_preds * weight\n            \n            fold_models[name] = {'models': target_models, 'weight': weight, 'scores': model_scores}\n        \n        total_weight = sum(model_weights.values())\n        if total_weight > 0: \n            fold_preds /= total_weight\n        \n        fold_score = competition_score(y_val, fold_preds)\n        print(f\" 🎯 Model Weights: { {k: f'{v:.3f}' for k, v in model_weights.items()} }\")\n        \n        return fold_preds, fold_models, fold_score\n\n# --------------------- MAIN TRAINING WITH PLOTS ---------------------\ndef train_stable_ensemble(feature_df):\n    print(\"🎯 Training STABLE Ensemble...\")\n    feature_cols = [col for col in feature_df.columns if col.startswith('feat_')]\n    X = feature_df[feature_cols].values\n    y = feature_df[CFG.TARGETS].values\n    print(f\"📊 Features: {X.shape}, Targets: {y.shape}\")\n    \n    # Scaling\n    scaler = StandardScaler()\n    X_scaled = scaler.fit_transform(X)\n    X_scaled = np.nan_to_num(X_scaled, nan=0.0, posinf=1.0, neginf=-1.0)\n    joblib.dump(scaler, f\"{CFG.OUT_DIR}/scaler.pkl\")\n    \n    # PLS Transformation\n    try:\n        pls = PLSRegression(n_components=min(CFG.PLS_COMPONENTS, X_scaled.shape[1]-1), \n                           scale=False, max_iter=1000)\n        X_pls = pls.fit_transform(X_scaled, y)[0]\n        joblib.dump(pls, f\"{CFG.OUT_DIR}/pls.pkl\")\n        print(f\"📊 PLS features: {X_pls.shape}\")\n    except Exception as e:\n        print(f\"⚠️ PLS failed: {e}, using original features\")\n        X_pls = X_scaled\n        joblib.dump(None, f\"{CFG.OUT_DIR}/pls.pkl\")\n    \n    # Group KFold\n    groups = create_simple_groups(feature_df)\n    kf = GroupKFold(n_splits=CFG.NUM_FOLDS)\n    \n    all_scores, all_oof_preds, all_oof_true, fold_models_list = [], [], [], []\n    \n    fold = 0\n    for train_idx, val_idx in kf.split(X_scaled, y, groups=groups):\n        fold += 1\n        print(f\"\\n🔥 Fold {fold}/{CFG.NUM_FOLDS}\")\n        \n        X_train_pls, X_val_pls = X_pls[train_idx], X_pls[val_idx]\n        y_train, y_val = y[train_idx], y[val_idx]\n        \n        ensemble = StableEnsemble()\n        fold_preds, fold_model_dict, fold_score = ensemble.train_fold(\n            X_train_pls, y_train, X_val_pls, y_val\n        )\n        \n        all_scores.append(fold_score)\n        all_oof_preds.append(fold_preds)\n        all_oof_true.append(y_val)\n        fold_models_list.append(fold_model_dict)\n        \n        print(f\" ✅ Fold Score: {fold_score:.4f}\")\n        joblib.dump(fold_model_dict, f\"{CFG.OUT_DIR}/fold_{fold}_models.pkl\")\n    \n    # Combine OOF predictions\n    y_oof = np.concatenate(all_oof_true, axis=0)\n    preds_oof = np.concatenate(all_oof_preds, axis=0)\n    \n    # Improved clipping based on training data distribution\n    for i in range(preds_oof.shape[1]):\n        upper = np.percentile(y_oof[:, i], 99.5)  # More conservative\n        lower = np.percentile(y_oof[:, i], 0.5)\n        preds_oof[:, i] = np.clip(preds_oof[:, i], lower, upper)\n    \n    final_cv = competition_score(y_oof, preds_oof)\n    \n    # Generate plots\n    target_performance = plot_target_performance(y_oof, preds_oof, all_scores, final_cv)\n    weights_df = plot_model_analysis(fold_models_list)\n    \n    # Performance summary\n    print(f\"\\n{'='*60}\")\n    print(\"🎯 TRAINING COMPLETED WITH ANALYSIS!\")\n    print(f\"{'='*60}\")\n    \n    performance_df = pd.DataFrame(target_performance)\n    print(\"\\n📊 TARGET-WISE PERFORMANCE:\")\n    print(performance_df.round(4))\n    \n    print(f\"\\n📈 FOLD PERFORMANCE:\")\n    for fold, score in enumerate(all_scores, 1):\n        print(f\"  Fold {fold}: {score:.4f}\")\n    \n    print(f\"\\n🏆 FINAL OOF CV: {final_cv:.5f}\")\n    print(f\"📊 CV Stats: Mean = {np.mean(all_scores):.5f} ± {np.std(all_scores):.5f}\")\n    \n    # Overfitting analysis\n    cv_std = np.std(all_scores)\n    if cv_std < 0.03:\n        overfitting_status = \"✅ EXCELLENT - Low overfitting\"\n        estimated_lb = final_cv * 0.95  # Conservative estimate\n    elif cv_std < 0.06:\n        overfitting_status = \"✅ GOOD - Moderate overfitting\"  \n        estimated_lb = final_cv * 0.92\n    else:\n        overfitting_status = \"⚠️  MODERATE - Some overfitting\"\n        estimated_lb = final_cv * 0.88\n    \n    print(f\"\\n🔍 OVERFITTING ANALYSIS:\")\n    print(f\"   {overfitting_status}\")\n    print(f\"   Estimated LB Score: {estimated_lb:.4f}\")\n    \n    if final_cv > 0.75:\n        print(\"🚀 PHENOMENAL! Top 10% leaderboard potential!\")\n    elif final_cv > 0.70:\n        print(\"📈 EXCELLENT! Strong leaderboard contender!\")\n    elif final_cv > 0.65:\n        print(\"💪 VERY GOOD! Competitive performance!\")\n    else:\n        print(\"👍 GOOD! Solid baseline!\")\n    \n    return final_cv, fold_models_list\n\n# --------------------- SUBMISSION ---------------------\ndef create_submission(test_df, fold_models):\n    print(\"🔮 Creating Submission...\")\n    test_features_df = extract_features(is_train=False)\n    feature_cols = [col for col in test_features_df.columns if col.startswith('feat_')]\n    X_test = test_features_df[feature_cols].values\n    \n    scaler = joblib.load(f\"{CFG.OUT_DIR}/scaler.pkl\")\n    X_test_scaled = scaler.transform(X_test)\n    X_test_scaled = np.nan_to_num(X_test_scaled, nan=0.0, posinf=1.0, neginf=-1.0)\n    \n    pls = joblib.load(f\"{CFG.OUT_DIR}/pls.pkl\")\n    X_test_pls = pls.transform(X_test_scaled) if pls is not None else X_test_scaled\n    \n    test_preds = np.zeros((X_test_pls.shape[0], len(CFG.TARGETS)))\n    \n    for fold in range(1, CFG.NUM_FOLDS + 1):\n        fold_model_dict = joblib.load(f\"{CFG.OUT_DIR}/fold_{fold}_models.pkl\")\n        fold_preds = np.zeros((X_test_pls.shape[0], len(CFG.TARGETS)))\n        total_weight = sum(model_info['weight'] for model_info in fold_model_dict.values())\n        \n        for name, model_info in fold_model_dict.items():\n            target_models = model_info['models']\n            weight = model_info['weight']\n            for i in range(len(CFG.TARGETS)):\n                model = target_models[i]\n                if model is not None:\n                    try:\n                        fold_preds[:, i] += model.predict(X_test_pls) * weight\n                    except: \n                        pass\n        \n        if total_weight > 0: \n            fold_preds /= total_weight\n        test_preds += fold_preds / CFG.NUM_FOLDS\n    \n    # Conservative clipping for test predictions\n    for i in range(test_preds.shape[1]):\n        upper = np.percentile(test_preds[:, i], 99)\n        lower = np.percentile(test_preds[:, i], 1)\n        test_preds[:, i] = np.clip(test_preds[:, i], lower, upper)\n    \n    predictions_dict = {}\n    for idx, row in test_features_df.iterrows():\n        predictions_dict[os.path.basename(row['image_path'])] = test_preds[idx]\n    \n    final_predictions = []\n    for _, row in test_df.iterrows():\n        img_path = os.path.basename(row['image_path'])\n        target_name = row['target_name']\n        if img_path in predictions_dict:\n            target_idx = CFG.TARGETS.index(target_name)\n            final_predictions.append(max(0, predictions_dict[img_path][target_idx]))\n        else: \n            final_predictions.append(0.0)\n    \n    submission = test_df[['sample_id']].copy()\n    submission['target'] = final_predictions\n    submission.to_csv('/kaggle/working/submission.csv', index=False)\n    print(\"✅ Submission created: /kaggle/working/submission.csv\")\n    return submission\n\n# --------------------- MAIN EXECUTION ---------------------\ndef main():\n    print(\"🚀 STARTING ULTIMATE STABLE ENSEMBLE WITH COMPREHENSIVE ANALYSIS!\")\n    print(\"📊 This version includes performance plots and overfitting analysis\")\n    \n    random.seed(CFG.SEED)\n    np.random.seed(CFG.SEED)\n    torch.manual_seed(CFG.SEED)\n    \n    # Training with analysis\n    train_df = extract_features(is_train=True)\n    cv_score, fold_models = train_stable_ensemble(train_df)\n    \n    # Create submission\n    test_df = pd.read_csv(CFG.TEST_CSV)\n    submission = create_submission(test_df, fold_models)\n    \n    print(f\"\\n{'='*60}\")\n    print(\"🎉 PIPELINE COMPLETED!\")\n    print(f\"📊 Final CV Score: {cv_score:.5f}\")\n    print(\"📁 Submission: /kaggle/working/submission.csv\") \n    print(\"📁 Analysis plots saved in output directory\")\n    print(f\"{'='*60}\")\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-17T13:04:55.739308Z","iopub.execute_input":"2025-11-17T13:04:55.740108Z","iopub.status.idle":"2025-11-17T13:09:43.870801Z","shell.execute_reply.started":"2025-11-17T13:04:55.740079Z","shell.execute_reply":"2025-11-17T13:09:43.870129Z"}},"outputs":[],"execution_count":null}]}