{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"# **Biomass Train Data Visualize Importance**","metadata":{}},{"cell_type":"markdown","source":"### Strategy for this competition\n\nThe test tabular data is missing some items compared to the training tabular data.\n\n**First, in the training phase, we verify that we can predict targets using only the tabular data (#1, this notebook).**\n\nWe also verify that image data can predict items that are present in the train tabular data but not in the test tabular data (#2).\n\nIn the testing phase, we first predict the missing items in the test tabular data from the images. Finally, we use the model already trained on the training data to predict targets using the complete test tabular data (#3).\n\n1. https://www.kaggle.com/code/stpeteishii/biomass-train-data-visualize-importance<br>\n2. https://www.kaggle.com/code/stpeteishii/pre-gshh-ndvi-pytorch-lightning-cnn-regressor<br>\nhttps://www.kaggle.com/code/stpeteishii/height-ave-cm-pytorch-lightning-cnn-regressor<br>\nhttps://www.kaggle.com/code/stpeteishii/species-pytorch-lightning-cnn-classifier<br>\n3. https://www.kaggle.com/code/stpeteishii/biomass-test-inference<br>","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport random\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom contextlib import contextmanager\nfrom time import time\nfrom tqdm import tqdm\nimport lightgbm as lgbm\nimport category_encoders as ce\nfrom tensorflow.keras.utils import to_categorical\nfrom sklearn.metrics import classification_report, log_loss, accuracy_score\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold","metadata":{"papermill":{"duration":8.958811,"end_time":"2021-06-21T06:51:56.66964","exception":false,"start_time":"2021-06-21T06:51:47.710829","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-10-29T15:31:48.674812Z","iopub.execute_input":"2025-10-29T15:31:48.675315Z","iopub.status.idle":"2025-10-29T15:31:48.682054Z","shell.execute_reply.started":"2025-10-29T15:31:48.675285Z","shell.execute_reply":"2025-10-29T15:31:48.68089Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data preparation","metadata":{"papermill":{"duration":0.024602,"end_time":"2021-06-21T06:51:56.719864","exception":false,"start_time":"2021-06-21T06:51:56.695262","status":"completed"},"tags":[]}},{"cell_type":"code","source":"data0 = pd.read_csv(\"/kaggle/input/csiro-biomass/train.csv\")\ndisplay(data0[0:3].T)\nprint(data0.columns.tolist())\ntest0=pd.read_csv('/kaggle/input/csiro-biomass/test.csv')\ndisplay(test0[0:3].T)\nprint(test0.columns.tolist())\ndelete_cols=['sample_id','image_path','Sampling_Date','State']\ndata0=data0.drop(columns=delete_cols,axis=1)\ndisplay(data0[0:3].T)\nprint(data0.columns.tolist())\nprint(set(data0.columns.tolist())-set(test0.columns.tolist()))\n\n# In test data,'Species', 'Pre_GSHH_NDVI', and 'Height_Ave_cm' will be predicted \n# from test image data.","metadata":{"papermill":{"duration":0.075867,"end_time":"2021-06-21T06:51:56.820913","exception":false,"start_time":"2021-06-21T06:51:56.745046","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-10-29T15:33:16.88404Z","iopub.execute_input":"2025-10-29T15:33:16.885076Z","iopub.status.idle":"2025-10-29T15:33:16.920533Z","shell.execute_reply.started":"2025-10-29T15:33:16.885053Z","shell.execute_reply":"2025-10-29T15:33:16.919613Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"target_names=sorted(data0['target_name'].unique().tolist())\ntarget_name_mapping=dict(zip(target_names,list(range(len(target_names)))))\ndata0['target_name']=data0['target_name'].map(target_name_mapping)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\ndef labelencoder(df):\n    for c in df.columns:\n        if df[c].dtype=='object': \n            df[c] = df[c].fillna('N')\n            lbl = LabelEncoder()\n            lbl.fit(list(df[c].values))\n            df[c] = lbl.transform(df[c].values)\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-29T15:31:48.740855Z","iopub.status.idle":"2025-10-29T15:31:48.741249Z","shell.execute_reply.started":"2025-10-29T15:31:48.741014Z","shell.execute_reply":"2025-10-29T15:31:48.741032Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data1=labelencoder(data0)","metadata":{"papermill":{"duration":0.052272,"end_time":"2021-06-21T06:51:57.219751","exception":false,"start_time":"2021-06-21T06:51:57.167479","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-10-29T15:31:48.742697Z","iopub.status.idle":"2025-10-29T15:31:48.743507Z","shell.execute_reply.started":"2025-10-29T15:31:48.74334Z","shell.execute_reply":"2025-10-29T15:31:48.743358Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Target setting","metadata":{"papermill":{"duration":0.027891,"end_time":"2021-06-21T06:51:57.343874","exception":false,"start_time":"2021-06-21T06:51:57.315983","status":"completed"},"tags":[]}},{"cell_type":"code","source":"target='target'\ndataY=data1[target]\ndataX=data1.drop(target,axis=1)","metadata":{"papermill":{"duration":0.039864,"end_time":"2021-06-21T06:51:57.411692","exception":false,"start_time":"2021-06-21T06:51:57.371828","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-10-29T15:31:48.744212Z","iopub.status.idle":"2025-10-29T15:31:48.744468Z","shell.execute_reply.started":"2025-10-29T15:31:48.744352Z","shell.execute_reply":"2025-10-29T15:31:48.744363Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_columns = list(dataX.columns)\nprint(df_columns)","metadata":{"papermill":{"duration":0.039658,"end_time":"2021-06-21T06:51:57.556135","exception":false,"start_time":"2021-06-21T06:51:57.516477","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-10-29T15:31:48.745963Z","iopub.status.idle":"2025-10-29T15:31:48.746415Z","shell.execute_reply.started":"2025-10-29T15:31:48.746209Z","shell.execute_reply":"2025-10-29T15:31:48.746229Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrainX, testX, trainY, testY = train_test_split(dataX, dataY, test_size=0.1, random_state=42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-29T15:31:48.747992Z","iopub.status.idle":"2025-10-29T15:31:48.748429Z","shell.execute_reply.started":"2025-10-29T15:31:48.748287Z","shell.execute_reply":"2025-10-29T15:31:48.748302Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df=trainX","metadata":{"papermill":{"duration":0.037505,"end_time":"2021-06-21T06:51:57.694983","exception":false,"start_time":"2021-06-21T06:51:57.657478","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-10-29T15:31:48.750296Z","iopub.status.idle":"2025-10-29T15:31:48.75074Z","shell.execute_reply.started":"2025-10-29T15:31:48.750525Z","shell.execute_reply":"2025-10-29T15:31:48.750543Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_numeric_feature(input_df):\n    use_columns = df_columns \n    return input_df[use_columns].copy()","metadata":{"papermill":{"duration":0.037641,"end_time":"2021-06-21T06:51:57.828748","exception":false,"start_time":"2021-06-21T06:51:57.791107","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-10-29T15:31:48.751963Z","iopub.status.idle":"2025-10-29T15:31:48.752391Z","shell.execute_reply.started":"2025-10-29T15:31:48.752185Z","shell.execute_reply":"2025-10-29T15:31:48.752202Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from contextlib import contextmanager\nfrom time import time\n\nclass Timer:\n    def __init__(self, logger=None, format_str='{:.3f}[s]', prefix=None, suffix=None, sep=' '):\n\n        if prefix: format_str = str(prefix) + sep + format_str\n        if suffix: format_str = format_str + sep + str(suffix)\n        self.format_str = format_str\n        self.logger = logger\n        self.start = None\n        self.end = None\n\n    @property\n    def duration(self):\n        if self.end is None:\n            return 0\n        return self.end - self.start\n\n    def __enter__(self):\n        self.start = time()\n\n    def __exit__(self, exc_type, exc_val, exc_tb):\n        self.end = time()\n        out_str = self.format_str.format(self.duration)\n        if self.logger:\n            self.logger.info(out_str)\n        else:\n            print(out_str)","metadata":{"papermill":{"duration":0.041214,"end_time":"2021-06-21T06:51:57.899433","exception":false,"start_time":"2021-06-21T06:51:57.858219","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-10-29T15:31:48.754677Z","iopub.status.idle":"2025-10-29T15:31:48.755069Z","shell.execute_reply.started":"2025-10-29T15:31:48.754868Z","shell.execute_reply":"2025-10-29T15:31:48.754884Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tqdm import tqdm\n\ndef to_feature(input_df):\n\n    processors = [\n        create_numeric_feature,\n    ]\n    \n    out_df = pd.DataFrame()\n    \n    for func in tqdm(processors, total=len(processors)):\n        with Timer(prefix='create' + func.__name__ + ' '):\n            _df = func(input_df)\n\n        assert len(_df) == len(input_df), func.__name__\n        out_df = pd.concat([out_df, _df], axis=1)\n        \n    return out_df","metadata":{"papermill":{"duration":0.039575,"end_time":"2021-06-21T06:51:57.968414","exception":false,"start_time":"2021-06-21T06:51:57.928839","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-10-29T15:31:48.756211Z","iopub.status.idle":"2025-10-29T15:31:48.756586Z","shell.execute_reply.started":"2025-10-29T15:31:48.756395Z","shell.execute_reply":"2025-10-29T15:31:48.756412Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_feat_df = to_feature(train_df)\n#test_feat_df = to_feature(test_df)","metadata":{"papermill":{"duration":0.046135,"end_time":"2021-06-21T06:51:58.044269","exception":false,"start_time":"2021-06-21T06:51:57.998134","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-10-29T15:31:48.859937Z","iopub.execute_input":"2025-10-29T15:31:48.860317Z","iopub.status.idle":"2025-10-29T15:31:48.87266Z","shell.execute_reply.started":"2025-10-29T15:31:48.860293Z","shell.execute_reply":"2025-10-29T15:31:48.871625Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model","metadata":{"papermill":{"duration":0.030413,"end_time":"2021-06-21T06:51:58.107898","exception":false,"start_time":"2021-06-21T06:51:58.077485","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import lightgbm as lgbm\nfrom sklearn.metrics import mean_squared_error\n\ndef fit_lgbm(X, y, cv, \n             params: dict=None, \n             verbose: int=50):\n\n    if params is None:\n        params = {}\n\n    models = []\n    oof_pred = np.zeros_like(y, dtype=float)\n\n    for i, (idx_train, idx_valid) in enumerate(cv): \n        x_train, y_train = X[idx_train], y[idx_train]\n        x_valid, y_valid = X[idx_valid], y[idx_valid]\n\n        clf = lgbm.LGBMRegressor(**params)\n        \n        with Timer(prefix='fit fold={} '.format(i)):\n            clf.fit(x_train, y_train, \n                    eval_set=[(x_valid, y_valid)])\n\n        pred_i = clf.predict(x_valid)\n        oof_pred[idx_valid] = pred_i\n        models.append(clf)\n        print(f'Fold {i} RMSLE: {mean_squared_error(y_valid, pred_i) ** .5:.4f}')\n        print()\n\n    score = mean_squared_error(y, oof_pred) ** .5\n    print('-' * 50)\n    print('FINISHED | Whole RMSLE: {:.4f}'.format(score))\n    return oof_pred, models","metadata":{"papermill":{"duration":0.047557,"end_time":"2021-06-21T06:51:58.186048","exception":false,"start_time":"2021-06-21T06:51:58.138491","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-10-29T15:31:48.87471Z","iopub.execute_input":"2025-10-29T15:31:48.874993Z","iopub.status.idle":"2025-10-29T15:31:48.890503Z","shell.execute_reply.started":"2025-10-29T15:31:48.874972Z","shell.execute_reply":"2025-10-29T15:31:48.889269Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"params = {\n    'objective': 'rmse', \n    'learning_rate': .1,\n    'reg_lambda': 1.,\n    'reg_alpha': .1,\n    'max_depth': 5, \n    'n_estimators': 1000, \n    'colsample_bytree': .5, \n    'min_child_samples': 10,\n    'subsample_freq': 3,\n    'subsample': .9,\n    'importance_type': 'gain', \n    'random_state': 71,\n    'num_leaves': 62\n}","metadata":{"papermill":{"duration":0.04017,"end_time":"2021-06-21T06:51:58.257181","exception":false,"start_time":"2021-06-21T06:51:58.217011","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-10-29T15:31:48.891492Z","iopub.execute_input":"2025-10-29T15:31:48.891811Z","iopub.status.idle":"2025-10-29T15:31:48.908209Z","shell.execute_reply.started":"2025-10-29T15:31:48.891782Z","shell.execute_reply":"2025-10-29T15:31:48.907069Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y = trainY\nydf=pd.DataFrame(y)\ndisplay(ydf)","metadata":{"papermill":{"duration":0.039211,"end_time":"2021-06-21T06:51:58.32736","exception":false,"start_time":"2021-06-21T06:51:58.288149","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-10-29T15:31:48.909083Z","iopub.execute_input":"2025-10-29T15:31:48.909452Z","iopub.status.idle":"2025-10-29T15:31:48.933441Z","shell.execute_reply.started":"2025-10-29T15:31:48.909425Z","shell.execute_reply":"2025-10-29T15:31:48.932081Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import joblib\nfrom sklearn.model_selection import KFold\nimport os\n\nos.makedirs('models', exist_ok=True)\n\nMODELS = []\nfor i in range(1):\n    fold = KFold(n_splits=5, shuffle=True, random_state=71)\n    ydfi = ydf.iloc[:, i]\n    y = np.array(ydfi)\n    cv = list(fold.split(train_feat_df, y))\n    oof, models = fit_lgbm(train_feat_df.values, y, cv, params=params)\n    MODELS += [models]\n\n    for fold_idx, model in enumerate(models):\n        filename = f'models/model_target{i}_fold{fold_idx}.joblib'\n        joblib.dump(model, filename)\n\n    joblib.dump(models, f'models/all_models_target{i}.joblib')\n    \n    fig, ax = plt.subplots(figsize=(6,6))\n    ax.set_title(target, fontsize=20)\n    ax.set_xlabel('true', fontsize=12)\n    ax.set_ylabel('pred', fontsize=12)\n    ax.scatter(y, oof, alpha=0.3)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Visualize Importance","metadata":{"papermill":{"duration":0.042694,"end_time":"2021-06-21T06:52:02.133811","exception":false,"start_time":"2021-06-21T06:52:02.091117","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def visualize_importance(models, feat_train_df):\n\n    feature_importance_df = pd.DataFrame()\n    for i, model in enumerate(models):\n        _df = pd.DataFrame()\n        _df['feature_importance'] = model.feature_importances_\n        _df['column'] = feat_train_df.columns\n        _df['fold'] = i + 1\n        feature_importance_df = pd.concat([feature_importance_df, _df], \n                                          axis=0, ignore_index=True)\n\n    order = feature_importance_df.groupby('column')\\\n        .sum()[['feature_importance']]\\\n        .sort_values('feature_importance', ascending=False).index[:50]\n\n    fig, ax = plt.subplots(figsize=(8, max(6, len(order) * .25)))\n    sns.boxenplot(data=feature_importance_df, \n                  x='feature_importance', \n                  y='column', \n                  order=order, \n                  ax=ax, \n                  palette='viridis', \n                  orient='h')\n    \n    ax.tick_params(axis='x', rotation=0)\n    #ax.set_title('Importance')\n    ax.grid()\n    fig.tight_layout()\n    \n    return fig,ax\n\n#fig, ax = visualize_importance(models, train_feat_df)","metadata":{"papermill":{"duration":0.053724,"end_time":"2021-06-21T06:52:02.229723","exception":false,"start_time":"2021-06-21T06:52:02.175999","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-10-29T15:31:49.245982Z","iopub.execute_input":"2025-10-29T15:31:49.2463Z","iopub.status.idle":"2025-10-29T15:31:49.254084Z","shell.execute_reply.started":"2025-10-29T15:31:49.246269Z","shell.execute_reply":"2025-10-29T15:31:49.253215Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i in range(1):\n    models=MODELS[i]\n    fold = KFold(n_splits=5, shuffle=True, random_state=71)\n    ydfi=ydf.iloc[:,i]\n    y=np.array(ydfi)\n    cv = list(fold.split(train_feat_df, y))\n    oof, models = fit_lgbm(train_feat_df.values, y, cv, params=params)\n\n    fig, ax = visualize_importance(models, train_feat_df)\n    ax.set_title(target+' Imortance',fontsize=20)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-29T15:31:49.255005Z","iopub.execute_input":"2025-10-29T15:31:49.25533Z","iopub.status.idle":"2025-10-29T15:31:49.64546Z","shell.execute_reply.started":"2025-10-29T15:31:49.255299Z","shell.execute_reply":"2025-10-29T15:31:49.64427Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**The results shows items which are not included in test tabular data, 'Species', 'Pre_GSHH_NDVI' and 'Height_Ave_cm', are important to predict target.**","metadata":{}},{"cell_type":"markdown","source":"    # Confirming Model Structure\n    loaded_models = joblib.load('models/all_models_target0.joblib')\n    \n    print(type(loaded_models))  # <class 'list'>\n    print(len(loaded_models))   # 5 (because n_splits=5)\n    \n    # Accessing each fold's model\n    for fold_idx, model in enumerate(loaded_models):\n        print(f\"Fold {fold_idx}: {type(model)}\")\n        # For inference, predict individually with each model\n        pred_fold = model.predict(X_test)","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}