{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"# **Biomass Train Data Mutual Importance**","metadata":{}},{"cell_type":"markdown","source":"## Related notebooks for the 2nd approach\n\n**check mutual importance to plan strategy for this compe(new, this notebook)** <br>\nhttps://www.kaggle.com/code/stpeteishii/biomass-train-data-mutual-importance<br>\n\nfit model to predict target using tabular data (use the former one)<br>\nhttps://www.kaggle.com/code/stpeteishii/biomass-train-data-visualize-importance<br>\n\nfit model to predict pre-gshh-ndvi/height-ave-cm using images (use the former ones)<br>\nhttps://www.kaggle.com/code/stpeteishii/pre-gshh-ndvi-pytorch-lightning-cnn-regressor<br>\nhttps://www.kaggle.com/code/stpeteishii/height-ave-cm-pytorch-lightning-cnn-regressor<br>\n\nfit model to predict species using tabular data (new, not published)<br>\nhttps://www.kaggle.com/code/stpeteishii/biomass-train-data-wo-target-species-importance<br>\n\npredict test species and test target (new, not published) <br>\nhttps://www.kaggle.com/code/stpeteishii/biomass-test-inference-the-2nd-approach<br>","metadata":{}},{"cell_type":"code","source":"# Mutual Importance Analysis for All Features\nimport os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom time import time\nfrom tqdm import tqdm\nimport lightgbm as lgbm\nimport joblib\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import LabelEncoder\n\n# ============================================================================\n# Timer Utility\n# ============================================================================\nclass Timer:\n    def __init__(self, logger=None, format_str='{:.3f}[s]', prefix=None, suffix=None, sep=' '):\n        if prefix: format_str = str(prefix) + sep + format_str\n        if suffix: format_str = format_str + sep + str(suffix)\n        self.format_str = format_str\n        self.logger = logger\n        self.start = None\n        self.end = None\n\n    @property\n    def duration(self):\n        if self.end is None:\n            return 0\n        return self.end - self.start\n\n    def __enter__(self):\n        self.start = time()\n\n    def __exit__(self, exc_type, exc_val, exc_tb):\n        self.end = time()\n        out_str = self.format_str.format(self.duration)\n        if self.logger:\n            self.logger.info(out_str)\n        else:\n            print(out_str)\n\n# ============================================================================\n# Data Preparation\n# ============================================================================\ndata0 = pd.read_csv(\"/kaggle/input/csiro-biomass/train.csv\")\n\n# Delete unnecessary columns\ndelete_cols = ['sample_id', 'image_path', 'Sampling_Date', 'State']\ndata0 = data0.drop(columns=delete_cols, axis=1)\n\n# Encode target_name\ntarget_names = sorted(data0['target_name'].unique().tolist())\ntarget_name_mapping = dict(zip(target_names, list(range(len(target_names)))))\ndata0['target_name'] = data0['target_name'].map(target_name_mapping)\n\n# Label encode categorical columns\ndef labelencoder(df):\n    for c in df.columns:\n        if df[c].dtype == 'object': \n            df[c] = df[c].fillna('N')\n            lbl = LabelEncoder()\n            lbl.fit(list(df[c].values))\n            df[c] = lbl.transform(df[c].values)\n    return df\n\ndata1 = labelencoder(data0)\n\n# ============================================================================\n# Model Training Function\n# ============================================================================\ndef fit_lgbm(X, y, cv, params: dict = None, verbose: int = 50):\n    \"\"\"Train LightGBM model with cross-validation\"\"\"\n    if params is None:\n        params = {}\n\n    models = []\n    oof_pred = np.zeros_like(y, dtype=float)\n\n    for i, (idx_train, idx_valid) in enumerate(cv): \n        x_train, y_train = X[idx_train], y[idx_train]\n        x_valid, y_valid = X[idx_valid], y[idx_valid]\n\n        clf = lgbm.LGBMRegressor(**params)\n        \n        with Timer(prefix=f'Fit fold={i} '):\n            clf.fit(x_train, y_train, \n                    eval_set=[(x_valid, y_valid)],\n                    callbacks=[lgbm.early_stopping(stopping_rounds=50, verbose=False)])\n\n        pred_i = clf.predict(x_valid)\n        oof_pred[idx_valid] = pred_i\n        models.append(clf)\n        print(f'Fold {i} RMSE: {mean_squared_error(y_valid, pred_i) ** .5:.4f}')\n\n    score = mean_squared_error(y, oof_pred) ** .5\n    print(f'Overall RMSE: {score:.4f}\\n')\n    return oof_pred, models\n\n# ============================================================================\n# Mutual Importance Calculation\n# ============================================================================\ndef calculate_mutual_importance(data, target_col=None, n_splits=5, random_state=42):\n    \"\"\"\n    Calculate mutual importance matrix for all features\n    \n    Args:\n        data: DataFrame with all features\n        target_col: Optional target column name to exclude from predictors\n        n_splits: Number of CV folds\n        random_state: Random seed\n        \n    Returns:\n        importance_matrix: DataFrame where [i,j] represents importance of feature j \n                          when predicting feature i\n    \"\"\"\n    \n    # LightGBM parameters\n    params = {\n        'objective': 'rmse', \n        'learning_rate': 0.1,\n        'reg_lambda': 1.0,\n        'reg_alpha': 0.1,\n        'max_depth': 5, \n        'n_estimators': 500, \n        'colsample_bytree': 0.5, \n        'min_child_samples': 10,\n        'subsample_freq': 3,\n        'subsample': 0.9,\n        'importance_type': 'gain', \n        'random_state': random_state,\n        'num_leaves': 31,\n        'verbose': -1\n    }\n    \n    # Prepare columns\n    if target_col and target_col in data.columns:\n        columns = [col for col in data.columns if col != target_col]\n    else:\n        columns = data.columns.tolist()\n    \n    # Initialize importance matrix\n    importance_matrix = pd.DataFrame(\n        np.zeros((len(columns), len(columns))),\n        index=columns,\n        columns=columns\n    )\n    \n    # Calculate importance for each feature as target\n    for target_feature in tqdm(columns, desc=\"Calculating mutual importance\"):\n        # Prepare data: use all other features to predict target_feature\n        predictor_cols = [col for col in columns if col != target_feature]\n        X = data[predictor_cols].values\n        y = data[target_feature].values\n        \n        # Cross-validation\n        fold = KFold(n_splits=n_splits, shuffle=True, random_state=random_state)\n        cv = list(fold.split(X, y))\n        \n        # Train model\n        print(f\"\\n{'='*60}\")\n        print(f\"Target Feature: {target_feature}\")\n        print(f\"{'='*60}\")\n        oof, models = fit_lgbm(X, y, cv, params=params)\n        \n        # Average feature importance across folds\n        avg_importance = np.mean([model.feature_importances_ for model in models], axis=0)\n        \n        # Store in matrix\n        importance_matrix.loc[target_feature, predictor_cols] = avg_importance\n    \n    return importance_matrix\n\n# ============================================================================\n# Visualization Functions\n# ============================================================================\ndef visualize_importance_heatmap(importance_matrix, figsize=(14, 12), cmap='viridis'):\n    \"\"\"Visualize mutual importance matrix as heatmap\"\"\"\n    \n    fig, ax = plt.subplots(figsize=figsize)\n    \n    # Create heatmap\n    sns.heatmap(importance_matrix, \n                annot=False,\n                fmt='.2f',\n                cmap=cmap,\n                cbar_kws={'label': 'Feature Importance'},\n                ax=ax)\n    \n    ax.set_title('Mutual Feature Importance Matrix\\n(Row: Target Feature, Column: Predictor Feature)', \n                 fontsize=14, pad=20)\n    ax.set_xlabel('Predictor Features', fontsize=12)\n    ax.set_ylabel('Target Features', fontsize=12)\n    \n    plt.xticks(rotation=45, ha='right')\n    plt.yticks(rotation=0)\n    plt.tight_layout()\n    \n    return fig, ax\n\ndef visualize_top_relationships(importance_matrix, top_n=20):\n    \"\"\"Visualize top N feature relationships\"\"\"\n    \n    # Get all pairwise relationships (excluding diagonal)\n    relationships = []\n    for target in importance_matrix.index:\n        for predictor in importance_matrix.columns:\n            if target != predictor:\n                relationships.append({\n                    'target': target,\n                    'predictor': predictor,\n                    'importance': importance_matrix.loc[target, predictor]\n                })\n    \n    # Sort and get top N\n    df_relationships = pd.DataFrame(relationships)\n    df_relationships = df_relationships.sort_values('importance', ascending=False).head(top_n)\n    \n    # Create labels\n    df_relationships['pair'] = df_relationships['predictor'] + ' → ' + df_relationships['target']\n    \n    # Plot\n    fig, ax = plt.subplots(figsize=(10, max(6, top_n * 0.3)))\n    \n    sns.barplot(data=df_relationships, \n                y='pair', \n                x='importance',\n                palette='rocket',\n                ax=ax)\n    \n    ax.set_title(f'Top {top_n} Feature Relationships by Importance', fontsize=14, pad=15)\n    ax.set_xlabel('Feature Importance', fontsize=12)\n    ax.set_ylabel('Feature Relationship', fontsize=12)\n    ax.grid(axis='x', alpha=0.3)\n    \n    plt.tight_layout()\n    \n    return fig, ax, df_relationships\n\ndef analyze_feature_connectivity(importance_matrix, threshold=None):\n    \"\"\"Analyze how connected each feature is to others\"\"\"\n    \n    if threshold is None:\n        threshold = importance_matrix.values[importance_matrix.values > 0].mean()\n    \n    # Count strong connections for each feature\n    # As predictor (how many features depend on it)\n    as_predictor = (importance_matrix > threshold).sum(axis=0)\n    # As target (how many features it depends on)\n    as_target = (importance_matrix > threshold).sum(axis=1)\n    \n    connectivity_df = pd.DataFrame({\n        'Feature': importance_matrix.index,\n        'As_Predictor': as_predictor.values,\n        'As_Target': as_target.values,\n        'Total_Connections': as_predictor.values + as_target.values\n    }).sort_values('Total_Connections', ascending=False)\n    \n    return connectivity_df\n\n# ============================================================================\n# Main Analysis\n# ============================================================================\nif __name__ == \"__main__\":\n    \n    # Calculate mutual importance INCLUDING 'target'\n    print(\"\\n\" + \"=\"*70)\n    print(\"STARTING MUTUAL IMPORTANCE ANALYSIS (INCLUDING 'target')\")\n    print(\"=\"*70 + \"\\n\")\n    \n    print(\"⚠ NOTE: 'target' is included in the analysis.\")\n    print(\"This will reveal:\")\n    print(\"  1. How features predict 'target' (forward direction)\")\n    print(\"  2. How 'target' predicts features (reverse direction)\")\n    print(\"  3. Hidden relationships between all variables\\n\")\n    \n    importance_matrix = calculate_mutual_importance(\n        data1, \n        target_col=None,  # Include ALL columns including 'target'\n        n_splits=3,  # Use fewer splits for faster computation\n        random_state=42\n    )\n    \n    # Save results\n    os.makedirs('mutual_importance_results', exist_ok=True)\n    importance_matrix.to_csv('mutual_importance_results/importance_matrix.csv')\n    print(\"\\n✓ Importance matrix saved to 'mutual_importance_results/importance_matrix.csv'\")\n    \n    # Visualize heatmap\n    print(\"\\n\" + \"=\"*70)\n    print(\"CREATING VISUALIZATIONS\")\n    print(\"=\"*70 + \"\\n\")\n    \n    fig1, ax1 = visualize_importance_heatmap(importance_matrix)\n    fig1.savefig('mutual_importance_results/importance_heatmap.png', dpi=150, bbox_inches='tight')\n    print(\"✓ Heatmap saved to 'mutual_importance_results/importance_heatmap.png'\")\n    \n    # Visualize top relationships\n    fig2, ax2, top_rels = visualize_top_relationships(importance_matrix, top_n=30)\n    fig2.savefig('mutual_importance_results/top_relationships.png', dpi=150, bbox_inches='tight')\n    print(\"✓ Top relationships saved to 'mutual_importance_results/top_relationships.png'\")\n    \n    # Analyze connectivity\n    connectivity = analyze_feature_connectivity(importance_matrix)\n    connectivity.to_csv('mutual_importance_results/feature_connectivity.csv', index=False)\n    print(\"✓ Connectivity analysis saved to 'mutual_importance_results/feature_connectivity.csv'\")\n    \n    # Display summary statistics\n    print(\"\\n\" + \"=\"*70)\n    print(\"SUMMARY STATISTICS\")\n    print(\"=\"*70)\n    print(f\"\\nMean importance: {importance_matrix.values.mean():.4f}\")\n    print(f\"Max importance: {importance_matrix.values.max():.4f}\")\n    print(f\"Std importance: {importance_matrix.values.std():.4f}\")\n    \n    print(\"\\n\" + \"=\"*70)\n    print(\"TOP 10 MOST CONNECTED FEATURES\")\n    print(\"=\"*70)\n    print(connectivity.head(10).to_string(index=False))\n    \n    print(\"\\n\" + \"=\"*70)\n    print(\"TOP 10 FEATURE RELATIONSHIPS\")\n    print(\"=\"*70)\n    print(top_rels.head(10).to_string(index=False))\n    \n    plt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### **From the result above, we are planning another strategy. To improve accuracy of prediction of species, we will use predicted valus of Pre_GSHH_NDVI, Height_Ave_cm, but not images.**\n<br>\n\n#### **Images → Pre_GSHH_NDVI, Height_Ave_cm**\n####         ↓\n#### **Tabular (with Pre_GSHH_NDVI, Height_Ave_cm) → Species**\n####         ↓\n#### **Tabular (with Species, Pre_GSHH_NDVI, Height_Ave_cm) → target**","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}