{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31153,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 📋Table of contents\n* [Features](#features)\n* [Targets](#targets)\n* [Targets vs Features](#targets_vs_features)\n* [Examples](#examples)\n* [Pivot Table and Target Correlations](#pivot)","metadata":{}},{"cell_type":"code","source":"# packages\n\n# standard\nimport numpy as np\nimport pandas as pd\nimport time\n\n# plots\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# images\nfrom PIL import Image","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-11-01T17:06:32.636465Z","iopub.execute_input":"2025-11-01T17:06:32.636777Z","iopub.status.idle":"2025-11-01T17:06:32.641917Z","shell.execute_reply.started":"2025-11-01T17:06:32.636754Z","shell.execute_reply":"2025-11-01T17:06:32.640869Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# configs\npd.set_option('display.max_columns', None) # we want to display all columns in this notebook\npd.set_option('display.max_rows', 100) # increase rows to be displayed\npd.set_option('display.max_colwidth', None) # show full cell contents\n\n# random seed\nmy_random_seed = 123\n\n# aesthetics\ndefault_color_1 = 'darkblue'\ndefault_color_2 = 'darkgreen'\ndefault_color_3 = 'darkred'\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T17:06:32.643497Z","iopub.execute_input":"2025-11-01T17:06:32.643766Z","iopub.status.idle":"2025-11-01T17:06:32.667324Z","shell.execute_reply.started":"2025-11-01T17:06:32.643745Z","shell.execute_reply":"2025-11-01T17:06:32.666128Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# load data\nt1 = time.time()\ndf_train = pd.read_csv('/kaggle/input/csiro-biomass/train.csv', low_memory=False)\ndf_test = pd.read_csv('/kaggle/input/csiro-biomass/test.csv', low_memory=False)\ndf_sub = pd.read_csv('/kaggle/input/csiro-biomass/sample_submission.csv', low_memory=False)\nt2 = time.time()\nprint('Elapsed time [s]:', np.round(t2-t1,4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T17:06:32.668415Z","iopub.execute_input":"2025-11-01T17:06:32.669029Z","iopub.status.idle":"2025-11-01T17:06:32.701579Z","shell.execute_reply.started":"2025-11-01T17:06:32.668996Z","shell.execute_reply":"2025-11-01T17:06:32.700459Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# preview\ndf_train.head(10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T17:06:32.703303Z","iopub.execute_input":"2025-11-01T17:06:32.703561Z","iopub.status.idle":"2025-11-01T17:06:32.717614Z","shell.execute_reply.started":"2025-11-01T17:06:32.703541Z","shell.execute_reply":"2025-11-01T17:06:32.716676Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# show structure of data - train\ndf_train.info(show_counts=True, verbose=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T17:06:32.718554Z","iopub.execute_input":"2025-11-01T17:06:32.718774Z","iopub.status.idle":"2025-11-01T17:06:32.747527Z","shell.execute_reply.started":"2025-11-01T17:06:32.718757Z","shell.execute_reply":"2025-11-01T17:06:32.746396Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# show structure of data - test\ndf_test.info(show_counts=True, verbose=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T17:06:32.748311Z","iopub.execute_input":"2025-11-01T17:06:32.748549Z","iopub.status.idle":"2025-11-01T17:06:32.771621Z","shell.execute_reply.started":"2025-11-01T17:06:32.748530Z","shell.execute_reply":"2025-11-01T17:06:32.770513Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# convert date\ndf_train['Sampling_Date'] = pd.to_datetime(df_train['Sampling_Date'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T17:06:32.772699Z","iopub.execute_input":"2025-11-01T17:06:32.773049Z","iopub.status.idle":"2025-11-01T17:06:32.794073Z","shell.execute_reply.started":"2025-11-01T17:06:32.773025Z","shell.execute_reply":"2025-11-01T17:06:32.792858Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# define target and predictors\ntarget = 'target'\n\n# numerical features\nfeatures_num = ['Pre_GSHH_NDVI', 'Height_Ave_cm']\n\n# categorical features\nfeatures_cat = ['State', 'Species']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T17:06:32.794999Z","iopub.execute_input":"2025-11-01T17:06:32.795323Z","iopub.status.idle":"2025-11-01T17:06:32.813013Z","shell.execute_reply.started":"2025-11-01T17:06:32.795297Z","shell.execute_reply":"2025-11-01T17:06:32.811920Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<a id='features'></a>\n# Features","metadata":{}},{"cell_type":"code","source":"# plot histograms (train only);\n# note that each observation (=image) occurs 5 times (one for each target)!\nn_bins = 20\nfor f in features_num:\n    plt.figure(figsize=(7,3))\n    df_train[f].plot(kind='hist', bins=n_bins, color=default_color_1)\n    plt.title(f + ' - Train')\n    plt.grid()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T17:06:32.817211Z","iopub.execute_input":"2025-11-01T17:06:32.817476Z","iopub.status.idle":"2025-11-01T17:06:33.309986Z","shell.execute_reply.started":"2025-11-01T17:06:32.817456Z","shell.execute_reply":"2025-11-01T17:06:33.309015Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# bivariate scatter\nplt.figure(figsize=(4,4))\nplt.scatter(df_train['Pre_GSHH_NDVI'], df_train['Height_Ave_cm'],\n            color=default_color_1, s=10)\nplt.xlabel('Pre_GSHH_NDVI')\nplt.ylabel('Height_Ave_cm')\nplt.grid()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T17:06:33.310775Z","iopub.execute_input":"2025-11-01T17:06:33.311036Z","iopub.status.idle":"2025-11-01T17:06:33.474730Z","shell.execute_reply.started":"2025-11-01T17:06:33.311011Z","shell.execute_reply":"2025-11-01T17:06:33.473774Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# plot categorical feature distributions (train only);\n# note that each observation (=image) occurs 5 times (one for each target)!\nfor f in features_cat:\n    plt.figure(figsize=(12,3))\n    df_train[f].value_counts().sort_index().plot(kind='bar', color=default_color_1)\n    plt.title(f + ' - Train')\n    plt.grid()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T17:06:33.475651Z","iopub.execute_input":"2025-11-01T17:06:33.475929Z","iopub.status.idle":"2025-11-01T17:06:33.945120Z","shell.execute_reply.started":"2025-11-01T17:06:33.475902Z","shell.execute_reply":"2025-11-01T17:06:33.944189Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# bivariate table of categories;\n# note that each observation (=image) occurs 5 times (one for each target)!\npd.crosstab(df_train['Species'], df_train['State'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T17:06:33.946267Z","iopub.execute_input":"2025-11-01T17:06:33.946556Z","iopub.status.idle":"2025-11-01T17:06:33.966068Z","shell.execute_reply.started":"2025-11-01T17:06:33.946529Z","shell.execute_reply":"2025-11-01T17:06:33.964948Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# dates evaluation\nprint(df_train.Sampling_Date.describe())\n\nplt.figure(figsize=(10,3))\nplt.hist(df_train.Sampling_Date, bins=n_bins, color=default_color_1)\nplt.title('Sampling_Date')\nplt.grid()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T17:06:33.967128Z","iopub.execute_input":"2025-11-01T17:06:33.967431Z","iopub.status.idle":"2025-11-01T17:06:34.232215Z","shell.execute_reply.started":"2025-11-01T17:06:33.967406Z","shell.execute_reply":"2025-11-01T17:06:34.231017Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<a id='targets'></a>\n# Targets","metadata":{}},{"cell_type":"code","source":"# plot target distributions - option 1\nsns.histplot(data=df_train, x=target, hue='target_name')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T17:06:34.233395Z","iopub.execute_input":"2025-11-01T17:06:34.233737Z","iopub.status.idle":"2025-11-01T17:06:34.742140Z","shell.execute_reply.started":"2025-11-01T17:06:34.233708Z","shell.execute_reply":"2025-11-01T17:06:34.740939Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# plot target distributions - option 2\nsns.histplot(data=df_train, x=target, y='target_name')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T17:06:34.743045Z","iopub.execute_input":"2025-11-01T17:06:34.743356Z","iopub.status.idle":"2025-11-01T17:06:34.912145Z","shell.execute_reply.started":"2025-11-01T17:06:34.743335Z","shell.execute_reply":"2025-11-01T17:06:34.911270Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# check frequency of different targets\ntarget_freqs = df_train.target_name.value_counts()\ntarget_freqs","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T17:06:34.913067Z","iopub.execute_input":"2025-11-01T17:06:34.913370Z","iopub.status.idle":"2025-11-01T17:06:34.921299Z","shell.execute_reply.started":"2025-11-01T17:06:34.913350Z","shell.execute_reply":"2025-11-01T17:06:34.920360Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<a id='targets_vs_features'></a>\n# Targets vs Features","metadata":{}},{"cell_type":"code","source":"# get target names\ntargets = target_freqs.index.tolist()\ntargets","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T17:06:34.922263Z","iopub.execute_input":"2025-11-01T17:06:34.922639Z","iopub.status.idle":"2025-11-01T17:06:34.940849Z","shell.execute_reply.started":"2025-11-01T17:06:34.922618Z","shell.execute_reply":"2025-11-01T17:06:34.939817Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# scatter plot target vs features\nfor t in targets:\n    df_temp = df_train[df_train.target_name==t]\n    for f in features_num:\n        title = 'Target ' + t + ' vs ' + f\n        plt.figure(figsize=(4,3))\n        plt.scatter(df_temp[f], df_temp.target, color=default_color_3, s=10)\n        plt.title(title)\n        plt.grid()\n        plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T17:06:34.941883Z","iopub.execute_input":"2025-11-01T17:06:34.942455Z","iopub.status.idle":"2025-11-01T17:06:36.593465Z","shell.execute_reply.started":"2025-11-01T17:06:34.942423Z","shell.execute_reply":"2025-11-01T17:06:36.592519Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<a id='examples'></a>\n# Examples","metadata":{}},{"cell_type":"code","source":"# pick an example via id\nmy_id = 'ID1011485656'\n\n# corresponding pathes\nimage_path = 'train/' + my_id + '.jpg'\nfull_path = '/kaggle/input/csiro-biomass/' + image_path\n\n# select only this example from the data frame\ndf_ex = df_train[df_train.image_path == image_path]\ndf_ex","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T17:06:36.594415Z","iopub.execute_input":"2025-11-01T17:06:36.594658Z","iopub.status.idle":"2025-11-01T17:06:36.610598Z","shell.execute_reply.started":"2025-11-01T17:06:36.594638Z","shell.execute_reply":"2025-11-01T17:06:36.609293Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# show corresponding image\nimage = Image.open(full_path)\nimage","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T17:06:36.611815Z","iopub.execute_input":"2025-11-01T17:06:36.612118Z","iopub.status.idle":"2025-11-01T17:06:37.312559Z","shell.execute_reply.started":"2025-11-01T17:06:36.612097Z","shell.execute_reply":"2025-11-01T17:06:37.311229Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# pick another example - this case shows clover\nmy_id = 'ID1025234388'\n\n# corresponding pathes\nimage_path = 'train/' + my_id + '.jpg'\nfull_path = '/kaggle/input/csiro-biomass/' + image_path\n\n# select only this example from the data frame\ndf_ex = df_train[df_train.image_path == image_path]\ndf_ex","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T17:06:37.313713Z","iopub.execute_input":"2025-11-01T17:06:37.314081Z","iopub.status.idle":"2025-11-01T17:06:37.340015Z","shell.execute_reply.started":"2025-11-01T17:06:37.314050Z","shell.execute_reply":"2025-11-01T17:06:37.338863Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# show corresponding image\nimage = Image.open(full_path)\nimage","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T17:06:37.341194Z","iopub.execute_input":"2025-11-01T17:06:37.341465Z","iopub.status.idle":"2025-11-01T17:06:37.893738Z","shell.execute_reply.started":"2025-11-01T17:06:37.341445Z","shell.execute_reply":"2025-11-01T17:06:37.892817Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<a id='pivot'></a>\n# Pivot Table and Target Correlations","metadata":{}},{"cell_type":"code","source":"# create pivot table with different targets in the columns\ndf_pivot = df_train.pivot(index='image_path', columns='target_name', values=target)\ndf_pivot","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T17:06:37.894617Z","iopub.execute_input":"2025-11-01T17:06:37.894856Z","iopub.status.idle":"2025-11-01T17:06:37.919533Z","shell.execute_reply.started":"2025-11-01T17:06:37.894836Z","shell.execute_reply":"2025-11-01T17:06:37.918717Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# multivariate scatter plot of targets\nsns.pairplot(df_pivot, \n             plot_kws = { 'color' : default_color_3, 's' : 15},\n             diag_kws = { 'color' : default_color_3})\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T17:06:37.923910Z","iopub.execute_input":"2025-11-01T17:06:37.924452Z","iopub.status.idle":"2025-11-01T17:06:44.158896Z","shell.execute_reply.started":"2025-11-01T17:06:37.924412Z","shell.execute_reply":"2025-11-01T17:06:44.157813Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# correlations\ncorr_pearson = df_pivot[targets].corr(method='pearson')\ncorr_spearman = df_pivot[targets].corr(method='spearman')\n\nplt.figure(figsize=(15,5))\nax1 = plt.subplot(1,2,1)\nsns.heatmap(corr_pearson, annot=True, cmap='RdYlGn', vmin=-1, vmax=+1,\n            fmt='.2g', linecolor='black', linewidths=0.5)\nplt.title('Pearson Correlation')\n\nax2 = plt.subplot(1,2,2, sharex=ax1)\nsns.heatmap(corr_spearman, annot=True, cmap='RdYlGn', vmin=-1, vmax=+1,\n            fmt='.2g', linecolor='black', linewidths=0.5)\nplt.title('Spearman Correlation')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T17:06:44.159983Z","iopub.execute_input":"2025-11-01T17:06:44.160268Z","iopub.status.idle":"2025-11-01T17:06:44.814101Z","shell.execute_reply.started":"2025-11-01T17:06:44.160248Z","shell.execute_reply":"2025-11-01T17:06:44.813087Z"}},"outputs":[],"execution_count":null}]}