{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nimport cv2\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-03T08:11:13.038474Z","iopub.execute_input":"2025-12-03T08:11:13.038701Z","iopub.status.idle":"2025-12-03T08:11:16.690500Z","shell.execute_reply.started":"2025-12-03T08:11:13.038680Z","shell.execute_reply":"2025-12-03T08:11:16.689511Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train, test = pd.read_csv('/kaggle/input/csiro-biomass/train.csv'), pd.read_csv('/kaggle/input/csiro-biomass/test.csv')\npath = '/kaggle/input/csiro-biomass/'\nprint('*****'*5, 'train', '*****'*5)\nprint(train.head(6))\nprint('*****'*5, 'test', '*****'*5)\nprint(test.head(6))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T08:11:16.692414Z","iopub.execute_input":"2025-12-03T08:11:16.692860Z","iopub.status.idle":"2025-12-03T08:11:16.756882Z","shell.execute_reply.started":"2025-12-03T08:11:16.692835Z","shell.execute_reply":"2025-12-03T08:11:16.756034Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train and test insigths\n**They come in stacks of 5 where each picture is composed by their dry_mass presented in the picture**","metadata":{}},{"cell_type":"code","source":"print(train['Species'].unique())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T08:11:16.757905Z","iopub.execute_input":"2025-12-03T08:11:16.758326Z","iopub.status.idle":"2025-12-03T08:11:16.769130Z","shell.execute_reply.started":"2025-12-03T08:11:16.758293Z","shell.execute_reply":"2025-12-03T08:11:16.767613Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 11 species, and images with multiple species ","metadata":{}},{"cell_type":"code","source":"%matplotlib inline\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport math\n\nimage_paths = list(train['image_path'])\n\ndef plot_6_imgs(path, image_paths):\n\n    rows = 3\n    cols = 2\n    n_images = rows * cols\n\n\n    image_paths = image_paths[:n_images]\n\n\n    fig, axes = plt.subplots(rows, cols, figsize=(15, 15))\n\n\n    axes = axes.flatten()\n\n    for ax, img_name in zip(axes, image_paths):\n        img_path = os.path.join(path, img_name)\n        img = mpimg.imread(img_path)\n        ax.imshow(img)\n        ax.set_title(os.path.basename(img_name), fontsize=8)\n        ax.axis('off')\n\n\n    for ax in axes[len(image_paths):]:\n        ax.axis('off')\n    \n    plt.tight_layout()\n    plt.show()\n\nplot_6_imgs(path, image_paths)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T08:11:16.770439Z","iopub.execute_input":"2025-12-03T08:11:16.770741Z","iopub.status.idle":"2025-12-03T08:11:20.319253Z","shell.execute_reply.started":"2025-12-03T08:11:16.770710Z","shell.execute_reply":"2025-12-03T08:11:20.317508Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# apply pca descomposition","metadata":{}},{"cell_type":"code","source":"from PIL import Image\nimages_path = list(set(train['image_path']))\n\n\ndef pca_descomp(path, images_path, output_dir):\n    counter = len(images_path)\n    os.makedirs(output_dir, exist_ok=True)\n    \n    for i, img_rel_path in enumerate(images_path):\n        \n        # Load PIL image\n        im = Image.open(os.path.join(path, img_rel_path))\n        new_size = (150, 120)\n        im = im.resize(new_size, Image.Resampling.LANCZOS)\n        # Convert to numpy scaled\n        img_np = np.array(im)\n        scaled_image = img_np / img_np.max()\n\n        # Split channels\n        blue = scaled_image[:, :, 0]\n        green = scaled_image[:, :, 1]\n        red = scaled_image[:, :, 2]\n\n        # Covariance matrices\n        red_cov_mat = np.cov(red.T)\n        green_cov_mat = np.cov(green.T)\n        blue_cov_mat = np.cov(blue.T)\n\n        # Eigendecomp\n        red_eig_vals, red_eig_vecs = np.linalg.eig(red_cov_mat)\n        green_eig_vals, green_eig_vecs = np.linalg.eig(green_cov_mat)\n        blue_eig_vals, blue_eig_vecs = np.linalg.eig(blue_cov_mat)\n\n        # Sort eigenvectors\n        red_sorted_indices = np.argsort(red_eig_vals)[::-1]\n        green_sorted_indices = np.argsort(green_eig_vals)[::-1]\n        blue_sorted_indices = np.argsort(blue_eig_vals)[::-1]\n\n        k = 115\n        red_selected_eig_vecs = red_eig_vecs[:, red_sorted_indices[:k]]\n        green_selected_eig_vecs = green_eig_vecs[:, green_sorted_indices[:k]]\n        blue_selected_eig_vecs = blue_eig_vecs[:, blue_sorted_indices[:k]]\n\n        # Projection\n        red_reduced = red @ red_selected_eig_vecs\n        green_reduced = green @ green_selected_eig_vecs\n        blue_reduced = blue @ blue_selected_eig_vecs\n\n        # Reconstruction\n        red_reconstructed = red_reduced @ red_selected_eig_vecs.T\n        green_reconstructed = green_reduced @ green_selected_eig_vecs.T\n        blue_reconstructed = blue_reduced @ blue_selected_eig_vecs.T\n\n        # Stack back\n        reconstructed_image = np.dstack((blue_reconstructed, green_reconstructed, red_reconstructed))\n        \n        # Remove imaginary noise\n        reconstructed_image = np.real(reconstructed_image)\n\n        # Normalize back to 0–255\n        reconstructed_image = reconstructed_image - reconstructed_image.min()\n        reconstructed_image = reconstructed_image / reconstructed_image.max()\n        reconstructed_image = (reconstructed_image * 255).astype(np.uint8)\n\n        # Denoising\n        # denois_img = cv2.boxFilter(reconstructed_image, -1, (3, 3), normalize=True)\n        denois_img = cv2.fastNlMeansDenoisingColored(reconstructed_image, None, 10, 10, 7, 21)\n        # cv2.boxFilter(reconstructed_image, -1, (3, 3), normalize=True)\n        # Convert to PIL\n        im_out = Image.fromarray(denois_img)\n\n        # Construct save path (preserving folder structure)\n        save_path = os.path.join(output_dir, img_rel_path)\n        os.makedirs(os.path.dirname(save_path), exist_ok=True)\n\n        # Save output image\n        im_out.save(save_path)\n        #counter -= 1 \n        #print(f\"{counter}\")\n\n    print(\"\\nProcessing complete!\")\n\n    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T08:11:20.320656Z","iopub.execute_input":"2025-12-03T08:11:20.321066Z","iopub.status.idle":"2025-12-03T08:11:20.339976Z","shell.execute_reply.started":"2025-12-03T08:11:20.321024Z","shell.execute_reply":"2025-12-03T08:11:20.339044Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Pending for apply pca to all Images","metadata":{}},{"cell_type":"markdown","source":"# **Create a mask to cover green pixels**\n\n**uncomment next cell to avoid kaggle run in bucle**","metadata":{}},{"cell_type":"code","source":"import cv2\nfrom numpy import inf\nimport math\nfrom ipywidgets import interact, IntSlider\nfrom IPython.display import display\n\n# Load your image\nimg = cv2.imread(f'{path}{images_path[30]}')\nimg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\nhsv = cv2.cvtColor(img, cv2.COLOR_RGB2HSV)\n\"\"\"\ndef update_mask(lower_H=40, lower_S=50, lower_V=50, upper_H=90, upper_S=200, upper_V=200):\n\n    # Define bounds\n    lower_bound = np.array([lower_H, lower_S, lower_V])\n    upper_bound = np.array([upper_H, upper_S, upper_V])\n\n    # Create mask and result\n    mask = cv2.inRange(hsv, lower_bound, upper_bound)\n    result = cv2.bitwise_and(img, img, mask=mask)\n\n    # Plot original, mask, and result\n    fig, axes = plt.subplots(1, 3, figsize=(15, 5))\n    axes[0].imshow(img)\n    axes[0].set_title(\"Original Image\")\n    axes[0].axis(\"off\")\n\n    axes[1].imshow(mask, cmap='gray')\n    axes[1].set_title(\"Mask\")\n    axes[1].axis(\"off\")\n\n    axes[2].imshow(result)\n    axes[2].set_title(\"Result\")\n    axes[2].axis(\"off\")\n\n    plt.show()\n\"\"\"\n# Create interactive sliders\n\"\"\"\ninteract(\n    update_mask,\n    lower_H=IntSlider(min=0, max=179, step=1, value=40, description='Lower H'),\n    lower_S=IntSlider(min=0, max=255, step=1, value=50, description='Lower S'),\n    lower_V=IntSlider(min=0, max=255, step=1, value=50, description='Lower V'),\n    upper_H=IntSlider(min=0, max=179, step=1, value=90, description='Upper H'),\n    upper_S=IntSlider(min=0, max=255, step=1, value=255, description='Upper S'),\n    upper_V=IntSlider(min=0, max=255, step=1, value=255, description='Upper V')\n)\n\"\"\"\n# 40, 50, 50, 90, 200, 200","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T08:11:20.340931Z","iopub.execute_input":"2025-12-03T08:11:20.341284Z","iopub.status.idle":"2025-12-03T08:11:20.728879Z","shell.execute_reply.started":"2025-12-03T08:11:20.341250Z","shell.execute_reply":"2025-12-03T08:11:20.727728Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def apply_mask(lh, ls, lv, uh, us, uv, img):\n    # Convert to HSV\n    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)\n    \n    # Bounds\n    lower_bound = np.array([lh, ls, lv])\n    upper_bound = np.array([uh, us, uv])\n\n    # Create mask and result\n    mask = cv2.inRange(hsv, lower_bound, upper_bound)\n    result = cv2.bitwise_and(img, img, mask=mask)\n    return result\n\nfig, axes = plt.subplots(figsize=(15, 5))\n# Load your image\nmasked_img = apply_mask(40, 50, 50, 90, 200, 200, img)\n\nplt.imshow(masked_img)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T08:11:20.732215Z","iopub.execute_input":"2025-12-03T08:11:20.732494Z","iopub.status.idle":"2025-12-03T08:11:21.515145Z","shell.execute_reply.started":"2025-12-03T08:11:20.732471Z","shell.execute_reply":"2025-12-03T08:11:21.514078Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"OUTPUT_DIR = '/kaggle/working/masked_images/'\nos.makedirs(OUTPUT_DIR, exist_ok=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T08:11:21.516259Z","iopub.execute_input":"2025-12-03T08:11:21.517136Z","iopub.status.idle":"2025-12-03T08:11:21.522413Z","shell.execute_reply.started":"2025-12-03T08:11:21.516852Z","shell.execute_reply":"2025-12-03T08:11:21.521237Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def mask_images(path, images_path, OUTPUT_DIR):\n    counter = len(images_path)\n    for i in range(len(images_path)):\n    # Read the image\n        img = cv2.imread(os.path.join(path, images_path[i]))\n        \n        if img is None:\n            print(f\"Error: Could not read image {images_path[i]}\")\n            continue\n            \n        # Apply the mask \n        masked_img = apply_mask(35, 45, 45, 90, 200, 200, img)\n        \n        # Build output path including subfolders\n        output_filepath = os.path.join(OUTPUT_DIR, images_path[i])\n        \n        # Create directories needed for this file\n        os.makedirs(os.path.dirname(output_filepath), exist_ok=True)\n        \n        # Save masked image\n        cv2.imwrite(output_filepath, masked_img)\n        \n        counter -= 1 \n\n        #print(f'{counter} Remaining Images for applying mask')\n        \n    print(f\"Successfully processed and saved {len(images_path)} images to: {OUTPUT_DIR}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T08:11:21.524052Z","iopub.execute_input":"2025-12-03T08:11:21.524401Z","iopub.status.idle":"2025-12-03T08:11:21.547276Z","shell.execute_reply.started":"2025-12-03T08:11:21.524370Z","shell.execute_reply":"2025-12-03T08:11:21.546036Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mask_images(path, images_path,OUTPUT_DIR)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T08:11:21.548169Z","iopub.execute_input":"2025-12-03T08:11:21.548499Z","iopub.status.idle":"2025-12-03T08:12:06.173128Z","shell.execute_reply.started":"2025-12-03T08:11:21.548467Z","shell.execute_reply":"2025-12-03T08:12:06.172151Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pca_descomp(OUTPUT_DIR, images_path, OUTPUT_DIR)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T08:12:06.174012Z","iopub.execute_input":"2025-12-03T08:12:06.174288Z","iopub.status.idle":"2025-12-03T08:13:03.789288Z","shell.execute_reply.started":"2025-12-03T08:12:06.174268Z","shell.execute_reply":"2025-12-03T08:13:03.788223Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_6_imgs(OUTPUT_DIR, list(set(image_paths))[40:46])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T08:13:03.790678Z","iopub.execute_input":"2025-12-03T08:13:03.791572Z","iopub.status.idle":"2025-12-03T08:13:04.468934Z","shell.execute_reply.started":"2025-12-03T08:13:03.791539Z","shell.execute_reply":"2025-12-03T08:13:04.467916Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"output_path = '/kaggle/working/edge_images/'\ndef edge_detection(path, images_path, output_path):\n    os.makedirs(output_path, exist_ok=True)\n    count = len(images_path)\n    for i in range(len(images_path)):\n        img = cv2.imread(os.path.join(path, images_path[i]))\n        gray_img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n        min_threshold = 200\n        max_threshold = 255\n        edges = cv2.Canny(gray_img, min_threshold, max_threshold)\n        # Build output path including subfolders\n        output_filepath = os.path.join(output_path, images_path[i])\n        # Create directories needed for this file\n        os.makedirs(os.path.dirname(output_filepath), exist_ok=True)\n        # save image\n        cv2.imwrite(output_filepath, edges)\n        count -= 1 \n\n        #print(f'{count} remaining images')\n    print(f'Images save in {output_path}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T08:13:04.469877Z","iopub.execute_input":"2025-12-03T08:13:04.470152Z","iopub.status.idle":"2025-12-03T08:13:04.477413Z","shell.execute_reply.started":"2025-12-03T08:13:04.470126Z","shell.execute_reply":"2025-12-03T08:13:04.475964Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"edge_detection(OUTPUT_DIR, images_path, output_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T08:13:04.478497Z","iopub.execute_input":"2025-12-03T08:13:04.478744Z","iopub.status.idle":"2025-12-03T08:13:04.892699Z","shell.execute_reply.started":"2025-12-03T08:13:04.478723Z","shell.execute_reply":"2025-12-03T08:13:04.891582Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_6_imgs(output_path, list(set(image_paths))[40:46])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T08:13:04.894023Z","iopub.execute_input":"2025-12-03T08:13:04.894394Z","iopub.status.idle":"2025-12-03T08:13:05.579437Z","shell.execute_reply.started":"2025-12-03T08:13:04.894364Z","shell.execute_reply":"2025-12-03T08:13:05.578538Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def transform_to_np(path, images_path):\n    np_array_imgs = {}\n    for i, img in enumerate(images_path):\n        path_img = os.path.join(path, images_path[i])\n        image_array = cv2.imread(path_img, 0)\n        np_array_imgs[f'{images_path[i]}'] = image_array\n    return np_array_imgs\nmasked_images = '/kaggle/working/masked_images'\nedge_images = '/kaggle/working/edge_images'\nmask_to_array=transform_to_np(masked_images, images_path)\nedge_to_array=transform_to_np(edge_images, images_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T08:13:05.580605Z","iopub.execute_input":"2025-12-03T08:13:05.580970Z","iopub.status.idle":"2025-12-03T08:13:05.741557Z","shell.execute_reply.started":"2025-12-03T08:13:05.580946Z","shell.execute_reply":"2025-12-03T08:13:05.740629Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train.head(1))\nprint(mask_to_array.get(f'{train[\"image_path\"][0]}'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T08:13:05.742541Z","iopub.execute_input":"2025-12-03T08:13:05.743124Z","iopub.status.idle":"2025-12-03T08:13:05.753275Z","shell.execute_reply.started":"2025-12-03T08:13:05.743075Z","shell.execute_reply":"2025-12-03T08:13:05.751942Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['masked_np'] = train['image_path'].map(mask_to_array)\ntrain['edge_np'] = train['image_path'].map(edge_to_array)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T08:13:05.754367Z","iopub.execute_input":"2025-12-03T08:13:05.754622Z","iopub.status.idle":"2025-12-03T08:13:05.786039Z","shell.execute_reply.started":"2025-12-03T08:13:05.754602Z","shell.execute_reply":"2025-12-03T08:13:05.784851Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head(1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T08:13:05.787061Z","iopub.execute_input":"2025-12-03T08:13:05.787413Z","iopub.status.idle":"2025-12-03T08:13:05.943647Z","shell.execute_reply.started":"2025-12-03T08:13:05.787380Z","shell.execute_reply":"2025-12-03T08:13:05.942568Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder as le\n\nle_target_name = le()\n\nle_target_name.fit(train['target_name'].unique())\n\nprint(le_target_name.classes_)\n\n\ntrain['target_name'] = le_target_name.transform(train['target_name'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T08:13:05.944834Z","iopub.execute_input":"2025-12-03T08:13:05.945644Z","iopub.status.idle":"2025-12-03T08:13:06.652036Z","shell.execute_reply.started":"2025-12-03T08:13:05.945613Z","shell.execute_reply":"2025-12-03T08:13:06.651157Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"selected_columns = [\"sample_id\",\"image_path\",\"target_name\",\"masked_np\",\"edge_np\", \"target\"]\ntrain = train[selected_columns]\nprint(train.head(6))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T08:13:06.653511Z","iopub.execute_input":"2025-12-03T08:13:06.654048Z","iopub.status.idle":"2025-12-03T08:13:07.028687Z","shell.execute_reply.started":"2025-12-03T08:13:06.654016Z","shell.execute_reply":"2025-12-03T08:13:07.027449Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train[\"masked_np\"] = [arr.flatten() for arr in train['masked_np']]\ntrain[\"edge_np\"] = [arr.flatten() for arr in train[\"edge_np\"]]\nprint(train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T08:13:07.032689Z","iopub.execute_input":"2025-12-03T08:13:07.033005Z","iopub.status.idle":"2025-12-03T08:13:07.084900Z","shell.execute_reply.started":"2025-12-03T08:13:07.032984Z","shell.execute_reply":"2025-12-03T08:13:07.083925Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **Important datafunction **\n* \"new_size = (150, 120)\"\n* len(train = 1785)","metadata":{}},{"cell_type":"code","source":"# Use boolean indexing to create a separate DataFrame for each target class\ntrain_expert_0 = train[train['target_name'] == 0].copy()\ntrain_expert_1 = train[train['target_name'] == 1].copy()\ntrain_expert_2 = train[train['target_name'] == 2].copy()\ntrain_expert_3 = train[train['target_name'] == 3].copy()\ntrain_expert_4 = train[train['target_name'] == 4].copy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T08:13:07.085985Z","iopub.execute_input":"2025-12-03T08:13:07.086559Z","iopub.status.idle":"2025-12-03T08:13:07.104789Z","shell.execute_reply.started":"2025-12-03T08:13:07.086524Z","shell.execute_reply":"2025-12-03T08:13:07.102992Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_expert_3.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T08:13:07.106447Z","iopub.execute_input":"2025-12-03T08:13:07.106842Z","iopub.status.idle":"2025-12-03T08:13:07.148859Z","shell.execute_reply.started":"2025-12-03T08:13:07.106803Z","shell.execute_reply":"2025-12-03T08:13:07.147484Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ndef create_train_data(df):\n    \n    X = np.stack(\n        (\n            np.array(df['masked_np'].tolist()),  # Shape (N, H, W)\n            np.array(df['edge_np'].tolist())     # Shape (N, H, W)\n        ), \n        axis=-1\n    ) \n    \n    y = df['target'].values\n    \n    X_train, X_test, y_train, y_test = train_test_split(\n        X, y, test_size=0.1, random_state=42)\n    return X_train, X_test, y_train, y_test","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T08:13:07.149944Z","iopub.execute_input":"2025-12-03T08:13:07.150279Z","iopub.status.idle":"2025-12-03T08:13:07.342596Z","shell.execute_reply.started":"2025-12-03T08:13:07.150256Z","shell.execute_reply":"2025-12-03T08:13:07.341548Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **Build train data for mixture of experts**","metadata":{}},{"cell_type":"code","source":"train_expert_0_X_train, train_expert_0_X_test, train_expert_0_y_train, train_expert_0_y_test = create_train_data(train_expert_0) \ntrain_expert_1_X_train, train_expert_1_X_test, train_expert_1_y_train, train_expert_1_y_test = create_train_data(train_expert_1) \ntrain_expert_2_X_train, train_expert_2_X_test, train_expert_2_y_train, train_expert_2_y_test = create_train_data(train_expert_2) \ntrain_expert_3_X_train, train_expert_3_X_test, train_expert_3_y_train, train_expert_3_y_test = create_train_data(train_expert_3) \ntrain_expert_4_X_train, train_expert_4_X_test, train_expert_4_y_train, train_expert_4_y_test = create_train_data(train_expert_4) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T08:13:07.343444Z","iopub.execute_input":"2025-12-03T08:13:07.343755Z","iopub.status.idle":"2025-12-03T08:13:07.460697Z","shell.execute_reply.started":"2025-12-03T08:13:07.343733Z","shell.execute_reply":"2025-12-03T08:13:07.459644Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_expert_0_X_train.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T08:13:07.461758Z","iopub.execute_input":"2025-12-03T08:13:07.462070Z","iopub.status.idle":"2025-12-03T08:13:07.470387Z","shell.execute_reply.started":"2025-12-03T08:13:07.462047Z","shell.execute_reply":"2025-12-03T08:13:07.469242Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pip install protobuf==3.20.3","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T08:13:07.471566Z","iopub.execute_input":"2025-12-03T08:13:07.471834Z","iopub.status.idle":"2025-12-03T08:13:15.477565Z","shell.execute_reply.started":"2025-12-03T08:13:07.471814Z","shell.execute_reply":"2025-12-03T08:13:15.476314Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, BatchNormalization, Dropout\n# --- Assuming X_train_reshaped and X_test_reshaped are available ---\n# (You must still perform the reshaping step: (N, H, W, C) for your data)\n\nORIGINAL_HEIGHT = 120\nORIGINAL_WIDTH = 150\n\nN_train = train_expert_0_X_train.shape[0]\nN_test = train_expert_0_X_test.shape[0]\n\ntrain_expert_0_X_train = train_expert_0_X_train.reshape(N_train, ORIGINAL_HEIGHT, ORIGINAL_WIDTH, 2)\ntrain_expert_0_X_test = train_expert_0_X_test.reshape(N_test, ORIGINAL_HEIGHT, ORIGINAL_WIDTH, 2)\n\nINPUT_SHAPE = train_expert_0_X_train.shape[1:]\n\n# Define the model for REGRESSION\ntf_model = Sequential([\n    # 1. Input/Initial Convolutional Block\n    Conv2D(32, (3, 3), activation='relu', input_shape=INPUT_SHAPE),\n    BatchNormalization(), # Normalizes inputs across the batch\n    MaxPooling2D((2, 2)),\n\n    # 2. Second Convolutional Block\n    Conv2D(64, (3, 3), activation='relu'),\n    MaxPooling2D((2, 2)),\n    \n    # 3. Third Convolutional Block\n    Conv2D(64, (3, 3), activation='relu'),\n    MaxPooling2D((2, 2)),\n    \n    # --- Transition to Fully Connected Layers ---\n    Flatten(), # Prepares output for the Dense layers\n\n    # 4. Dense Layer 1 with Dropout\n    Dense(64, activation='relu'),\n    # Dropout (0.3 rate means 30% of neurons are randomly set to zero during training)\n    Dropout(0.3), \n    \n    # 5. Final Output Layer for Regression\n    # 1 unit and 'linear' activation for single continuous prediction\n    Dense(1, activation='linear') \n])\n# CRITICAL CHANGE 2: Loss function is MSE (Mean Squared Error) or MAE (Mean Absolute Error)\ntf_model.compile(\n    optimizer='adam',\n    loss='mse',\n    metrics=['mae'] \n)\n\nprint(\"--- TensorFlow Regression Model Summary ---\")\ntf_model.summary()\n\n# Train the model\nhistory = tf_model.fit(\n    train_expert_0_X_train, train_expert_0_y_train, # y_train must now contain float values\n    epochs=20, \n    batch_size=32,\n    validation_data=(train_expert_0_X_test, train_expert_0_y_test)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T09:02:23.387523Z","iopub.execute_input":"2025-12-03T09:02:23.387862Z","iopub.status.idle":"2025-12-03T09:06:32.638728Z","shell.execute_reply.started":"2025-12-03T09:02:23.387836Z","shell.execute_reply":"2025-12-03T09:06:32.637825Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\n# --- Generate predictions ---\ny_pred_raw = tf_model.predict(train_expert_0_X_test)\n\n# Flatten the output from shape (N, 1) to (N,)\ny_pred = y_pred_raw.flatten()\n\n# --- FIX: Flatten the true values as well ---\n# y_true is likely still (N, 1) and must be converted to (N,)\ny_true = train_expert_0_y_test.flatten() \n\n# --- Display Predictions ---\nresults_df = pd.DataFrame({\n    'True_Value': y_true,\n    'Predicted_Value': y_pred\n})\n\n# --- Calculate and Display MSE ---\nmse = mean_squared_error(y_true, y_pred)\nprint(f\"\\n--- Mean Squared Error (MSE) on Test Set: {mse:.4f} ---\")\n\nprint(\"\\n--- Sample Predictions from Test Set ---\")\nprint(results_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-03T09:06:42.341849Z","iopub.execute_input":"2025-12-03T09:06:42.342183Z","iopub.status.idle":"2025-12-03T09:06:42.941609Z","shell.execute_reply.started":"2025-12-03T09:06:42.342158Z","shell.execute_reply":"2025-12-03T09:06:42.940365Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}