{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"},{"sourceId":12797608,"sourceType":"datasetVersion","datasetId":8091268},{"sourceId":4534,"sourceType":"modelInstanceVersion","modelInstanceId":3326,"modelId":986}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-20T02:50:57.850274Z","iopub.execute_input":"2025-08-20T02:50:57.850585Z","iopub.status.idle":"2025-08-20T02:50:59.917362Z","shell.execute_reply.started":"2025-08-20T02:50:57.850560Z","shell.execute_reply":"2025-08-20T02:50:59.916548Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Behavior Detection with Sensor Data: A Hybrid EfficientNet Approach\nWelcome to this notebook. Here, we introduce a Hybrid EfficientNet-based model for performing multi-label classification on wearable sensor data.\n\n### Why EfficientNet?\nEfficientNet is known for achieving high accuracy with significantly lower computational costs compared to other state-of-the-art models. This efficiency is crucial for implementation on wearable devices with limited resources.\n\n### Multi-Task Approach\nThis model is designed as a multi-task classification model to simultaneously predict four different targets: orientation, behavior, gesture, and phase. This allows for a more comprehensive understanding of user actions compared to predicting a single label.\n\n### Feature Engineering Strategy\nThe dataset consists of multiple sensors, and we process the data for each sequence (sequence_id).\n\nToF (Time of Flight) Sensor: Creates a 3D tensor (matrix data).\n\nIMU (Inertial Measurement Unit) Sensor: Converted into image data using the Gramian Angular Field (GAF) algorithm.\n\nTemperature Sensor: Converted into a spectrogram.\n\nThis diverse data representation allows the model to learn from various features.\n\n### Notebook Structure\nLibrary Imports: Import all necessary packages.\n\nConfiguration Class: A class to manage all parameters, paths, and hyperparameters.\n\nData Preprocessing: Load, preprocess, and generate/save tensors for training.\n\nDataLoader: Implement a custom PyTorch Dataset and DataLoader.\n\nModel Architecture: A detailed explanation of the custom 3D EfficientNet and the final hybrid model.\n\nTraining and Validation: The training loop, evaluation function, and the main training execution part.\n\nResult Visualization: Visualize the training results (loss trends) with a graph.","metadata":{}},{"cell_type":"markdown","source":"# Library Imports","metadata":{}},{"cell_type":"code","source":"!pip install pyts -q\n!pip install torchinfo hiddenlayer -q\n!pip install -q torchsummary","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-20T03:29:11.012891Z","iopub.execute_input":"2025-08-20T03:29:11.013653Z","iopub.status.idle":"2025-08-20T03:29:20.467655Z","shell.execute_reply.started":"2025-08-20T03:29:11.013624Z","shell.execute_reply":"2025-08-20T03:29:20.466551Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport gc\nimport pickle\nimport random\nimport json\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nfrom tqdm.auto import tqdm\nimport matplotlib.pyplot as plt\n\n# Scikit-learn\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.metrics import log_loss, accuracy_score, precision_recall_fscore_support, roc_auc_score, hamming_loss\n\n# Signal & Image Processing\n!pip install pyts -q\nfrom pyts.image import GramianAngularField\nfrom scipy.signal import spectrogram\nfrom skimage.transform import resize\n\n# PyTorch & Torchvision\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.cuda.amp import autocast, GradScaler\nimport torch.nn.functional as F\nfrom torchvision import models\n\n# Model Summary & Visualization Libraries\nfrom torchsummary import summary\nimport hiddenlayer as hl\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-20T03:30:24.031997Z","iopub.execute_input":"2025-08-20T03:30:24.032750Z","iopub.status.idle":"2025-08-20T03:30:27.245758Z","shell.execute_reply.started":"2025-08-20T03:30:24.032716Z","shell.execute_reply":"2025-08-20T03:30:27.244585Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Configuration\n","metadata":{}},{"cell_type":"code","source":"class CFG:\n    # General\n    DEBUG = False # Debug mode. If True, only 1 epoch is run on a subset of the data.\n    \n    # --- Paths ---\n    INPUT_DIR = '/kaggle/input/cmi-detect-behavior-with-sensor-data/'\n    TRAIN_CSV_PATH = os.path.join(INPUT_DIR, 'train.csv')\n    WORK_DIR = '/kaggle/working/'\n    PREPROCESSED_DIR = os.path.join(WORK_DIR, 'processed_data/') # Directory to save preprocessed data\n    MODEL_OUTPUT_DIR = os.path.join(WORK_DIR, 'models/') # Directory to save trained models\n\n    # --- Data & Preprocessing Parameters ---\n    SEQUENCE_LENGTH = 64\n    IMG_SIZE_2D = 128\n    GAF_IMG_SIZE = 32\n    SPECTROGRAM_IMG_SIZE = 32\n    \n    # --- Column Lists & Class Counts ---\n    TARGET_COLS = ['orientation', 'behavior', 'gesture', 'phase']\n    N_CLASSES = [] # This will be set dynamically after creating label_encoders\n    IMU_COLS = ['acc_x', 'acc_y', 'acc_z', 'rot_w', 'rot_x', 'rot_y', 'rot_z']\n    THM_COLS = [f'thm_{i}' for i in range(1, 6)]\n    TOF_COLS = [f'tof_{s}_v{v}' for s in range(1, 6) for v in range(64)]\n\n    # --- Model Hyperparameters ---\n    MODEL_WIDTH_MULT = 1.0\n    MODEL_DEPTH_MULT = 1.0\n    DROPOUT_RATE = 0.3\n    \n    # --- Training Parameters ---\n    EPOCHS = 10\n    BATCH_SIZE = 32\n    LEARNING_RATE = 1e-4\n    WEIGHT_DECAY = 1e-6\n    \n    # --- Hardware & Reproducibility ---\n    DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    # FIX: Set NUM_WORKERS to 0 to prevent the AssertionError in notebooks\n    NUM_WORKERS = 0 \n    SEED = 42\n\nif CFG.DEBUG:\n    CFG.EPOCHS = 1\n\nos.makedirs(CFG.PREPROCESSED_DIR, exist_ok=True)\nos.makedirs(CFG.MODEL_OUTPUT_DIR, exist_ok=True)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-20T02:51:26.034063Z","iopub.execute_input":"2025-08-20T02:51:26.034425Z","iopub.status.idle":"2025-08-20T02:51:26.137214Z","shell.execute_reply.started":"2025-08-20T02:51:26.034402Z","shell.execute_reply":"2025-08-20T02:51:26.136554Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(CFG.SEED)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-20T02:51:26.138222Z","iopub.execute_input":"2025-08-20T02:51:26.138565Z","iopub.status.idle":"2025-08-20T02:51:26.161447Z","shell.execute_reply.started":"2025-08-20T02:51:26.138530Z","shell.execute_reply":"2025-08-20T02:51:26.160707Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load Data and basic EDA","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(CFG.TRAIN_CSV_PATH)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-20T02:51:26.162520Z","iopub.execute_input":"2025-08-20T02:51:26.162826Z","iopub.status.idle":"2025-08-20T02:51:57.372013Z","shell.execute_reply.started":"2025-08-20T02:51:26.162796Z","shell.execute_reply":"2025-08-20T02:51:57.371239Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(df.shape)\nprint(df.index)\nprint(df.dtypes)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-20T02:51:57.372993Z","iopub.execute_input":"2025-08-20T02:51:57.373197Z","iopub.status.idle":"2025-08-20T02:51:57.380510Z","shell.execute_reply.started":"2025-08-20T02:51:57.373180Z","shell.execute_reply":"2025-08-20T02:51:57.379449Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.sample(10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-20T02:51:57.381542Z","iopub.execute_input":"2025-08-20T02:51:57.382290Z","iopub.status.idle":"2025-08-20T02:51:57.484149Z","shell.execute_reply.started":"2025-08-20T02:51:57.382261Z","shell.execute_reply":"2025-08-20T02:51:57.483275Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.info","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-20T02:51:57.485197Z","iopub.execute_input":"2025-08-20T02:51:57.485473Z","iopub.status.idle":"2025-08-20T02:51:57.597292Z","shell.execute_reply.started":"2025-08-20T02:51:57.485455Z","shell.execute_reply":"2025-08-20T02:51:57.596647Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(len(df['sequence_id'].unique()))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-20T02:51:57.599944Z","iopub.execute_input":"2025-08-20T02:51:57.600163Z","iopub.status.idle":"2025-08-20T02:51:57.636337Z","shell.execute_reply.started":"2025-08-20T02:51:57.600146Z","shell.execute_reply":"2025-08-20T02:51:57.635603Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(df['gesture'].unique())\nprint(df['orientation'].unique())\nprint(df['phase'].unique())\nprint(df['behavior'].unique())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-20T02:51:57.637091Z","iopub.execute_input":"2025-08-20T02:51:57.637388Z","iopub.status.idle":"2025-08-20T02:51:57.759940Z","shell.execute_reply.started":"2025-08-20T02:51:57.637350Z","shell.execute_reply":"2025-08-20T02:51:57.759260Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"def process_and_save_data(df, sequence_ids, scalers, label_encoders, cfg):\n    \"\"\"\n    Preprocesses each sequence and saves it locally as a tensor.\n    \"\"\"\n    gaf = GramianAngularField(image_size=cfg.GAF_IMG_SIZE, method='summation')\n    \n    for seq_id in tqdm(sequence_ids, desc=\"Preprocessing Sequences\"):\n        save_path = Path(cfg.PREPROCESSED_DIR) / f\"{seq_id}.pt\"\n        if save_path.exists():\n            continue\n        \n        seq_df = df[df['sequence_id'] == seq_id].copy()\n        \n        # Fill missing values with linear interpolation\n        for col_group in [cfg.IMU_COLS, cfg.THM_COLS, cfg.TOF_COLS]:\n            seq_df[col_group] = seq_df[col_group].interpolate(method='linear', limit_direction='both').fillna(0)\n        \n        # Standardization\n        seq_df[cfg.IMU_COLS] = scalers['imu'].transform(seq_df[cfg.IMU_COLS])\n        seq_df[cfg.THM_COLS] = scalers['thm'].transform(seq_df[cfg.THM_COLS])\n        seq_df[cfg.TOF_COLS] = scalers['tof'].transform(seq_df[cfg.TOF_COLS])\n        \n        # Fix sequence length (padding or truncation)\n        current_len = len(seq_df)\n        if current_len > cfg.SEQUENCE_LENGTH:\n            seq_df = seq_df.iloc[:cfg.SEQUENCE_LENGTH]\n        elif current_len < cfg.SEQUENCE_LENGTH:\n            padding_len = cfg.SEQUENCE_LENGTH - current_len\n            padding_df = pd.DataFrame(np.zeros((padding_len, len(df.columns))), columns=df.columns)\n            # Copy non-numeric data like sequence_id to maintain it\n            for col in ['sequence_id', 'subject'] + cfg.TARGET_COLS:\n                 if not seq_df[col].dropna().empty:\n                    padding_df[col] = seq_df[col].iloc[0]\n            seq_df = pd.concat([seq_df, padding_df], ignore_index=True)\n            \n        # Create 3D tensor from ToF data\n        tof_data = seq_df[cfg.TOF_COLS].values.reshape(cfg.SEQUENCE_LENGTH, 5, 8, 8)\n        tensor_3d = torch.tensor(tof_data, dtype=torch.float32).permute(1, 0, 2, 3) # [C, D, H, W]\n        \n        # Generate GAF images from IMU data\n        imu_data = seq_df[cfg.IMU_COLS].values.T\n        imu_gaf = torch.tensor(gaf.fit_transform(imu_data), dtype=torch.float32)\n        \n        # Generate spectrograms from temperature data\n        thm_data = seq_df[cfg.THM_COLS].values.T\n        thm_specs = []\n        for i in range(thm_data.shape[0]):\n            _, _, Sxx = spectrogram(thm_data[i, :], fs=20, nperseg=16, noverlap=8)\n            Sxx_resized = resize(np.log1p(Sxx), (cfg.SPECTROGRAM_IMG_SIZE, cfg.SPECTROGRAM_IMG_SIZE))\n            thm_specs.append(Sxx_resized)\n        thm_spec = torch.tensor(np.array(thm_specs), dtype=torch.float32)\n        \n        # Combine GAF images and spectrograms to create 2D tensor\n        tensor_2d_combined = torch.cat([imu_gaf, thm_spec], dim=0)\n        tensor_2d = F.interpolate(tensor_2d_combined.unsqueeze(0), size=(cfg.IMG_SIZE_2D, cfg.IMG_SIZE_2D), mode='bilinear', align_corners=False).squeeze(0)\n        \n        # Encode labels\n        labels = {}\n        for col in cfg.TARGET_COLS:\n            # Dropna and get the first valid label for the sequence\n            valid_labels = seq_df[col].dropna()\n            if not valid_labels.empty:\n                label_val = str(valid_labels.iloc[0])\n                encoded_label = label_encoders[col].transform([label_val])[0]\n                labels[col] = torch.tensor(encoded_label, dtype=torch.long)\n            else: # Handle cases with no valid labels if necessary\n                labels[col] = torch.tensor(-1, dtype=torch.long) # Or some placeholder\n        \n        torch.save({'tensor_3d': tensor_3d, 'tensor_2d': tensor_2d, 'labels': labels}, save_path)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-20T02:51:57.760850Z","iopub.execute_input":"2025-08-20T02:51:57.761037Z","iopub.status.idle":"2025-08-20T02:51:57.774710Z","shell.execute_reply.started":"2025-08-20T02:51:57.761023Z","shell.execute_reply":"2025-08-20T02:51:57.773922Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"label_encoders = {col: LabelEncoder().fit(df[col].astype(str).unique()) for col in CFG.TARGET_COLS}\nCFG.N_CLASSES = [len(label_encoders[col].classes_) for col in CFG.TARGET_COLS]\nscalers = {\n    'imu': StandardScaler().fit(df[CFG.IMU_COLS].fillna(0)),\n    'thm': StandardScaler().fit(df[CFG.THM_COLS].fillna(0)),\n    'tof': StandardScaler().fit(df[CFG.TOF_COLS].fillna(0)),\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-20T02:51:57.775549Z","iopub.execute_input":"2025-08-20T02:51:57.775853Z","iopub.status.idle":"2025-08-20T02:52:01.136098Z","shell.execute_reply.started":"2025-08-20T02:51:57.775823Z","shell.execute_reply":"2025-08-20T02:52:01.135516Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"with open(os.path.join(CFG.MODEL_OUTPUT_DIR, 'scalers.pkl'), 'wb') as f:\n    pickle.dump(scalers, f)\nwith open(os.path.join(CFG.MODEL_OUTPUT_DIR, 'label_encoders.pkl'), 'wb') as f:\n    pickle.dump(label_encoders, f)\nprint(\"Scalers and label encoders saved.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-20T02:52:01.136781Z","iopub.execute_input":"2025-08-20T02:52:01.136958Z","iopub.status.idle":"2025-08-20T02:52:01.143111Z","shell.execute_reply.started":"2025-08-20T02:52:01.136945Z","shell.execute_reply":"2025-08-20T02:52:01.142528Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"all_sequence_ids = df['sequence_id'].unique()\nprocess_and_save_data(df, all_sequence_ids, scalers, label_encoders, CFG)\ndel df\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-20T02:52:01.143996Z","iopub.execute_input":"2025-08-20T02:52:01.144204Z","iopub.status.idle":"2025-08-20T03:11:28.183729Z","shell.execute_reply.started":"2025-08-20T02:52:01.144188Z","shell.execute_reply":"2025-08-20T03:11:28.183114Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Dataset and DataLoader","metadata":{}},{"cell_type":"code","source":"class CustomDataset(Dataset):\n    def __init__(self, sequence_ids, cfg):\n        self.sequence_ids = sequence_ids\n        self.cfg = cfg\n\n    def __len__(self):\n        return len(self.sequence_ids)\n\n    def __getitem__(self, idx):\n        seq_id = self.sequence_ids[idx]\n        file_path = os.path.join(self.cfg.PREPROCESSED_DIR, f\"{seq_id}.pt\")\n        data = torch.load(file_path)\n        \n        tensor_3d = data['tensor_3d']\n        tensor_2d = data['tensor_2d']\n        labels = data['labels']\n        \n        return tensor_3d, tensor_2d, labels","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-20T03:11:28.184463Z","iopub.execute_input":"2025-08-20T03:11:28.184742Z","iopub.status.idle":"2025-08-20T03:11:28.189919Z","shell.execute_reply.started":"2025-08-20T03:11:28.184716Z","shell.execute_reply":"2025-08-20T03:11:28.189045Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model Architecture","metadata":{}},{"cell_type":"markdown","source":"The core of this solution is a custom 3D version of EfficientNet. We adapt the standard 2D EfficientNet architecture to handle 3D spatio-temporal data from the ToF sensor, which has dimensions (Length, Height, Width).\n \nThe architecture consists of several key components, each defined in its own cell for clarity.","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\n\nimg = mpimg.imread(\"/kaggle/input/multilablehybrideffecinetnetmodel/MultiLableHybridEffecinetNetModel.png\")\nplt.imshow(img)\nplt.axis(\"off\")  \nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-20T03:11:28.190752Z","iopub.execute_input":"2025-08-20T03:11:28.190997Z","iopub.status.idle":"2025-08-20T03:11:28.494174Z","shell.execute_reply.started":"2025-08-20T03:11:28.190976Z","shell.execute_reply":"2025-08-20T03:11:28.493337Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Swish Activation\nThe Swish activation function is defined as $f(x) = x \\cdot \\sigma(\\beta x)$, where $\\sigma$ is the sigmoid function. It's a smooth, non-monotonic function that often performs better than ReLU.\n \n$$\n \\text{Swish}(x) = x \\cdot \\text{Sigmoid}(\\beta x)\n$$","metadata":{}},{"cell_type":"code","source":"class Swish(nn.Module):\n    def forward(self, x):\n        return x * torch.sigmoid(x)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-20T03:11:28.495104Z","iopub.execute_input":"2025-08-20T03:11:28.495310Z","iopub.status.idle":"2025-08-20T03:11:28.499329Z","shell.execute_reply.started":"2025-08-20T03:11:28.495294Z","shell.execute_reply":"2025-08-20T03:11:28.498537Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Squeeze-and-Excitation (SE) Block\n\n\nThe SE block is a mechanism that allows the network to perform feature recalibration. It explicitly models the interdependencies between channels to adaptively adjust the weight of each feature map.\n\n1.  **Squeeze:** Global information is aggregated from each channel by using global average pooling. This reduces each channel's spatial dimensions (L, H, W) to a single value, producing a channel descriptor vector. For input $X \\in \\mathbb{R}^{C \\times L \\times H \\times W}$, the squeezed output $z \\in \\mathbb{R}^{C \\times 1 \\times 1 \\times 1}$ is calculated as:\n2.  $$\nz_c = \\frac{1}{L \\times H \\times W} \\sum_{l=1}^{L} \\sum_{h=1}^{H} \\sum_{w=1}^{W} X_{c, l, h, w}\n$$\n2.  **Excitation:** Two fully connected layers are used to learn a non-linear interaction between channels, producing a set of channel-wise weights (an attention mask).\n3.  **Scale:** The original input feature maps are scaled by the output of the excitation step.","metadata":{}},{"cell_type":"code","source":"class SEBlock3D(nn.Module):\n    def __init__(self, channels, reduction=4):\n        super().__init__()\n        self.squeeze = nn.AdaptiveAvgPool3d(1)\n        self.excitation = nn.Sequential(\n            nn.Conv3d(channels, channels // reduction, 1, bias=False), Swish(),\n            nn.Conv3d(channels // reduction, channels, 1, bias=False), nn.Sigmoid()\n        )\n    def forward(self, x):\n        return x * self.excitation(self.squeeze(x)).expand_as(x)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-20T03:11:28.499999Z","iopub.execute_input":"2025-08-20T03:11:28.500234Z","iopub.status.idle":"2025-08-20T03:11:28.511324Z","shell.execute_reply.started":"2025-08-20T03:11:28.500215Z","shell.execute_reply":"2025-08-20T03:11:28.510530Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### MBConv Block (3D)\n\nThe Mobile Inverted Bottleneck Convolution (MBConv) is the main building block of EfficientNet.\n\n1.\tExpansion: The number of channels is increased using a 1x1x1 convolution.\n\n    \n2.\tDepthwise Convolution: A 3D depthwise convolution is applied to filter features spatially and temporally. This is more efficient than a standard 3D convolution.\n\n    \n3.\tSqueeze-and-Excitation: The SE block is applied to recalibrate channel features.\n\n    \n4.\tProjection: A final 1x1x1 convolution projects the features back to a lower-dimensional space.\n\n    \n5.\tStochastic Depth & Skip Connection: A residual connection is added from the input to the output. Stochastic depth randomly drops entire blocks during training, acting as a form of regularization.","metadata":{}},{"cell_type":"code","source":"class MBConv3D(nn.Module):\n    def __init__(self, in_channels, out_channels, kernel_size=3, stride=1, expand_ratio=1, se_ratio=0.25, dropout_rate=0.0):\n        super().__init__()\n\n        #Residual Connection\n        #Preventing gradient vanishing and stabilizing learning\n        self.use_residual = (stride == 1 and in_channels == out_channels)\n        expanded_channels = in_channels * expand_ratio\n\n        layers = []\n\n\n        #Expansion\n        if expand_ratio != 1:\n            layers.extend([\n                nn.Conv3d(in_channels, expanded_channels, 1, bias=False),\n                nn.BatchNorm3d(expanded_channels),\n                Swish()\n            ])\n\n\n        #Depthwise Convolution\n        #groups=expanded_channels -> Depthwise Conv\n        #->Extracting local features of space-time for each channel\n        layers.extend([\n            nn.Conv3d(expanded_channels, expanded_channels, kernel_size, stride=stride,\n                      padding=kernel_size//2, groups=expanded_channels, bias=False),\n            nn.BatchNorm3d(expanded_channels),\n            Swish()\n        ])\n\n        #Squeeze-and-Excitation (SE Block)\n        if se_ratio > 0:\n            layers.append(SEBlock3D(expanded_channels, reduction=int(1/se_ratio)))\n\n\n        #Projection\n        layers.extend([\n            nn.Conv3d(expanded_channels, out_channels, 1, bias=False),\n            nn.BatchNorm3d(out_channels)\n        ])\n\n        self.conv = nn.Sequential(*layers)\n        self.dropout = nn.Dropout3d(dropout_rate) if dropout_rate > 0 else None\n\n    def forward(self, x):\n        residual = x\n        x = self.conv(x)\n        if self.dropout:\n            x = self.dropout(x)\n        if self.use_residual:\n            x += residual\n        return x","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-20T03:11:28.512276Z","iopub.execute_input":"2025-08-20T03:11:28.512549Z","iopub.status.idle":"2025-08-20T03:11:28.531258Z","shell.execute_reply.started":"2025-08-20T03:11:28.512527Z","shell.execute_reply":"2025-08-20T03:11:28.530481Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### EfficientNet3D\n\nThis class assembles the MBConv blocks to form the complete 3D EfficientNet backbone.\n\n\n1.\tInitial Convolution: A stem convolution to process the raw input.\n\n    \n2.\tMBConv Blocks: A series of MBConv blocks with varying configurations (channels, layers, strides) to extract features.\n\n    \n3.\tFinal Convolution: A final 1x1x1 convolution to prepare the features for the classifier.\n\n    \n4.\tGlobal Average Pooling (GAP): Reduces the spatial and temporal dimensions of the feature maps, creating a feature vector for each channel. The 3D GAP operation is:\n\n$$\ny_c = \\frac{1}{D \\cdot H \\cdot W} \\sum_{d=1}^{D} \\sum_{h=1}^{H} \\sum_{w=1}^{W} X_{c,d,h,w}\n$$","metadata":{}},{"cell_type":"code","source":"class EfficientNet3D_Branch(nn.Module):\n    def __init__(self, in_channels=5, width_mult=1.0, depth_mult=1.0, dropout_rate=0.2):\n        super().__init__()\n        init_channels = int(32 * width_mult)\n        self.initial_conv = nn.Sequential(\n            nn.Conv3d(in_channels, init_channels, kernel_size=3, stride=(1, 2, 2), padding=1, bias=False),\n            nn.BatchNorm3d(init_channels), Swish()\n        )\n\n        in_channels = init_channels\n        mbconv_configs = [\n            [1, 16, 1, (1,1,1), 3], [6, 24, 2, (2,2,2), 3], [6, 40, 2, (2,2,2), 5],\n            [6, 80, 3, (2,2,2), 3], [6, 112, 3, (1,1,1), 5], [6, 192, 4, (2,2,2), 5],\n            [6, 320, 1, (1,1,1), 3]\n        ]\n\n        self.mbconv_layers = nn.ModuleList()\n        for expand_ratio, channels, num_layers, stride, kernel_size in mbconv_configs:\n            out_channels = int(channels * width_mult)\n            layers_repeat = int(np.ceil(num_layers * depth_mult))\n            for i in range(layers_repeat):\n                layer_stride = stride if i == 0 else (1,1,1)\n                self.mbconv_layers.append(MBConv3D(in_channels, out_channels, kernel_size, layer_stride, expand_ratio, dropout_rate=dropout_rate))\n                in_channels = out_channels\n\n        self.final_channels = int(1280 * width_mult)\n        self.final_conv = nn.Sequential(\n            nn.Conv3d(in_channels, self.final_channels, 1, bias=False),\n            nn.BatchNorm3d(self.final_channels), Swish()\n        )\n        self.global_avg_pool = nn.AdaptiveAvgPool3d(1)\n\n    def forward(self, x):\n        x = self.initial_conv(x)\n        for layer in self.mbconv_layers:\n            x = layer(x)\n        x = self.final_conv(x)\n        x = self.global_avg_pool(x)\n        return x.view(x.size(0), -1)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-20T03:11:28.532139Z","iopub.execute_input":"2025-08-20T03:11:28.532532Z","iopub.status.idle":"2025-08-20T03:11:28.549209Z","shell.execute_reply.started":"2025-08-20T03:11:28.532483Z","shell.execute_reply":"2025-08-20T03:11:28.548420Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### EffecientNet2D\n\nThis uses EffecientNet trained on ImageNet.\nHowever, this time, instead of using this pre-trained model as a classifier, it is used as a 2D feature extractor.\nFurthermore, transfer learning was performed without changing the weights.\nThe final layer of EffecientNet normally performs classification using a fully connected layer + softmax, but nn.Identity() is used to output the feature vector as is.","metadata":{}},{"cell_type":"code","source":"class EfficientNet2D_Branch(nn.Module):\n    def __init__(self, in_channels=12):\n        super().__init__()\n        self.base_model = models.efficientnet_b0(weights=models.EfficientNet_B0_Weights.DEFAULT)\n        \n        original_conv = self.base_model.features[0][0]\n        self.base_model.features[0][0] = nn.Conv2d(\n            in_channels, original_conv.out_channels,\n            kernel_size=original_conv.kernel_size, stride=original_conv.stride,\n            padding=original_conv.padding, bias=False\n        )\n        self.base_model.classifier = nn.Identity()\n\n    def forward(self, x):\n        return self.base_model(x)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-20T03:11:28.550043Z","iopub.execute_input":"2025-08-20T03:11:28.550817Z","iopub.status.idle":"2025-08-20T03:11:28.566756Z","shell.execute_reply.started":"2025-08-20T03:11:28.550792Z","shell.execute_reply":"2025-08-20T03:11:28.565978Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Main Hybrud Model","metadata":{}},{"cell_type":"code","source":"class MultiTaskHybridNet(nn.Module):\n    def __init__(self, target_cols, n_classes_list,CFG):\n        super().__init__()\n        self.branch_3d = EfficientNet3D_Branch(\n            in_channels=5,\n            width_mult=CFG.MODEL_WIDTH_MULT,\n            depth_mult=CFG.MODEL_DEPTH_MULT,\n            dropout_rate=CFG.DROPOUT_RATE\n        )\n        self.branch_2d = EfficientNet2D_Branch(\n            in_channels=len(CFG.IMU_COLS) + len(CFG.THM_COLS)\n        )\n        combined_features_dim = self.branch_3d.final_channels + 1280\n        self.dropout = nn.Dropout(p=CFG.DROPOUT_RATE)\n        self.heads = nn.ModuleDict({\n            col: nn.Linear(combined_features_dim, num_classes)\n            for col, num_classes in zip(target_cols, n_classes_list)\n        })\n\n    def forward(self, x_3d, x_2d):\n        with autocast(enabled=True):\n            features_3d = self.branch_3d(x_3d)\n            features_2d = self.branch_2d(x_2d)\n            combined_features = torch.cat([features_3d, features_2d], dim=1)\n            combined_features = self.dropout(combined_features)\n            outputs = {col: head(combined_features) for col, head in self.heads.items()}\n        return outputs","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-20T03:11:28.567727Z","iopub.execute_input":"2025-08-20T03:11:28.567974Z","iopub.status.idle":"2025-08-20T03:11:28.584337Z","shell.execute_reply.started":"2025-08-20T03:11:28.567952Z","shell.execute_reply":"2025-08-20T03:11:28.583683Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_for_summary = MultiTaskHybridNet(\n    target_cols=CFG.TARGET_COLS,\n    n_classes_list=CFG.N_CLASSES,\n    CFG=CFG\n).to('cpu')\n\ninput_shape_3d = (5, CFG.SEQUENCE_LENGTH, 8, 8)\ninput_shape_2d = (len(CFG.IMU_COLS) + len(CFG.THM_COLS), CFG.IMG_SIZE_2D, CFG.IMG_SIZE_2D)\n\nprint(\"--- Summary for 3D Branch (EfficientNet3D_Branch) ---\")\nsummary(model_for_summary.branch_3d, input_size=input_shape_3d, device='cpu')\n\nprint(\"\\n\" + \"=\"*80 + \"\\n\")\n\nprint(\"--- Summary for 2D Branch (EfficientNet2D_Branch) ---\")\nsummary(model_for_summary.branch_2d, input_size=input_shape_2d, device='cpu')\n\ndel model_for_summary\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-20T03:30:34.953247Z","iopub.execute_input":"2025-08-20T03:30:34.954133Z","iopub.status.idle":"2025-08-20T03:30:35.689966Z","shell.execute_reply.started":"2025-08-20T03:30:34.954095Z","shell.execute_reply":"2025-08-20T03:30:35.689268Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training & Validation Loop","metadata":{}},{"cell_type":"markdown","source":"The training loop remains the same, executing the training and validation for the specified number of epochs and saving the best model to Google Drive.\n","metadata":{}},{"cell_type":"markdown","source":"### Training Utilities","metadata":{}},{"cell_type":"code","source":"class AverageMeter:\n    def __init__(self): self.reset()\n    def reset(self): self.val, self.avg, self.sum, self.count = 0, 0, 0, 0\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-20T03:11:30.228839Z","iopub.execute_input":"2025-08-20T03:11:30.229095Z","iopub.status.idle":"2025-08-20T03:11:30.233736Z","shell.execute_reply.started":"2025-08-20T03:11:30.229069Z","shell.execute_reply":"2025-08-20T03:11:30.232876Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Train & Validation Functions","metadata":{}},{"cell_type":"code","source":"def train_fn(loader, model, criterions, optimizer, scheduler, device):\n    model.train()\n    scaler = GradScaler()\n    loss_meter = AverageMeter()\n    \n    for x_3d, x_2d, labels in tqdm(loader, desc=\"Training\"):\n        x_3d, x_2d = x_3d.to(device), x_2d.to(device)\n        \n        optimizer.zero_grad()\n        \n        with autocast():\n            outputs = model(x_3d, x_2d)          \n            loss = 0\n            for col in CFG.TARGET_COLS:\n                loss += criterions[col](outputs[col], labels[col].to(device))\n        \n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n        \n        loss_meter.update(loss.item(), x_3d.size(0))\n        \n    scheduler.step()\n    return loss_meter.avg","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-20T03:11:30.234518Z","iopub.execute_input":"2025-08-20T03:11:30.234733Z","iopub.status.idle":"2025-08-20T03:11:30.247540Z","shell.execute_reply.started":"2025-08-20T03:11:30.234716Z","shell.execute_reply":"2025-08-20T03:11:30.246757Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def valid_fn(loader, model, criterions, device):\n    model.eval()\n    loss_meter = AverageMeter()\n    all_preds = {col: [] for col in CFG.TARGET_COLS}\n    all_trues = {col: [] for col in CFG.TARGET_COLS}\n    \n    with torch.no_grad():\n        for x_3d, x_2d, labels in tqdm(loader, desc=\"Validating\"):\n            x_3d, x_2d = x_3d.to(device), x_2d.to(device)\n            outputs = model(x_3d, x_2d) \n            loss = 0\n            for col in CFG.TARGET_COLS:\n                loss += criterions[col](outputs[col], labels[col].to(device))\n                all_preds[col].append(F.softmax(outputs[col], dim=1).cpu().numpy())\n                all_trues[col].append(labels[col].numpy())\n\n            loss_meter.update(loss.item(), x_3d.size(0))\n\n    for col in CFG.TARGET_COLS:\n        all_preds[col] = np.concatenate(all_preds[col])\n        all_trues[col] = np.concatenate(all_trues[col])\n        \n    return loss_meter.avg, all_preds, all_trues","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-20T03:11:30.248245Z","iopub.execute_input":"2025-08-20T03:11:30.248516Z","iopub.status.idle":"2025-08-20T03:11:30.262864Z","shell.execute_reply.started":"2025-08-20T03:11:30.248469Z","shell.execute_reply":"2025-08-20T03:11:30.262290Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def comprehensive_evaluation_function(trues, preds, label_encoders):\n    \"\"\"\n    Calculates and displays a comprehensive set of metrics for each target\n    in a clean, tabular format.\n    \"\"\"\n    print(\"=\" * 60)\n    print(\"Comprehensive Evaluation Metrics\")\n    print(\"=\" * 60)\n\n    # A list to store the results for each target as a dictionary\n    results_list = []\n\n    # Calculate metrics for each target\n    for col in CFG.TARGET_COLS:\n        y_true = trues[col]\n        y_pred_proba = preds[col]\n        y_pred_label = np.argmax(y_pred_proba, axis=1)\n        labels = np.arange(len(label_encoders[col].classes_))\n\n        # Accuracy\n        accuracy = accuracy_score(y_true, y_pred_label)\n\n        # Precision, Recall, F1-score\n        precision, recall, f1, _ = precision_recall_fscore_support(\n            y_true, y_pred_label, average='macro', zero_division=0, labels=labels\n        )\n\n        # Append the results for the current target to the list\n        results_list.append({\n            'Target': col,\n            'Accuracy': f\"{accuracy:.4f}\",\n            'Precision': f\"{precision:.4f}\",\n            'Recall': f\"{recall:.4f}\",\n            'F1-score': f\"{f1:.4f}\"\n        })\n\n    # Convert the list of results into a pandas DataFrame for beautiful printing\n    results_df = pd.DataFrame(results_list)\n\n    # Print the DataFrame without the index for a cleaner look\n    print(results_df.to_string(index=False))\n    print(\"=\" * 60)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-20T03:11:30.265729Z","iopub.execute_input":"2025-08-20T03:11:30.266435Z","iopub.status.idle":"2025-08-20T03:11:30.276640Z","shell.execute_reply.started":"2025-08-20T03:11:30.266407Z","shell.execute_reply":"2025-08-20T03:11:30.276017Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Main Loop","metadata":{}},{"cell_type":"code","source":"def run_training(train_ids, val_ids, cfg, label_encoders):\n    # Set up Datasets and DataLoaders\n    train_dataset = CustomDataset(train_ids, cfg)\n    val_dataset = CustomDataset(val_ids, cfg)\n    \n    train_loader = DataLoader(train_dataset, batch_size=cfg.BATCH_SIZE, shuffle=True, num_workers=cfg.NUM_WORKERS, pin_memory=True)\n    val_loader = DataLoader(val_dataset, batch_size=cfg.BATCH_SIZE, shuffle=False, num_workers=cfg.NUM_WORKERS, pin_memory=True)\n    \n    # Initialize the model\n    model = MultiTaskHybridNet(\n        target_cols=cfg.TARGET_COLS,\n        n_classes_list=cfg.N_CLASSES,\n        CFG=cfg\n    ).to(cfg.DEVICE)\n    \n    # Set up loss functions, optimizer, and scheduler\n    criterions = {col: nn.CrossEntropyLoss() for col in cfg.TARGET_COLS}\n    optimizer = optim.Adam(model.parameters(), lr=cfg.LEARNING_RATE, weight_decay=cfg.WEIGHT_DECAY)\n    scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=cfg.EPOCHS, eta_min=1e-6)\n    \n    best_loss = float('inf')\n    \n    # Dictionary to store training history for plotting\n    history = {'train_loss': [], 'valid_loss': []}\n    \n    # Start the training loop\n    for epoch in range(cfg.EPOCHS):\n        print(f\"\\nEpoch {epoch + 1}/{cfg.EPOCHS}\")\n        print(\"-\" * 15)\n        \n        # Training step\n        train_loss = train_fn(train_loader, model, criterions, optimizer, scheduler, cfg.DEVICE)\n        \n        # Validation step\n        valid_loss, preds, trues = valid_fn(val_loader, model, criterions, cfg.DEVICE)\n        \n        # Record losses for this epoch\n        history['train_loss'].append(train_loss)\n        history['valid_loss'].append(valid_loss)\n        \n        # Display evaluation metrics for the epoch\n        comprehensive_evaluation_function(trues, preds, label_encoders)\n        \n        print(f\"Summary - Train Loss: {train_loss:.4f} | Valid Loss: {valid_loss:.4f}\")\n\n        # Save the best model based on validation loss\n        if valid_loss < best_loss:\n            best_loss = valid_loss\n            print(f\"  -> Validation loss improved. Saving model to best_model.pth\")\n            save_path = os.path.join(CFG.MODEL_OUTPUT_DIR, \"best_model.pth\")\n            torch.save(model.state_dict(), save_path)\n            \n    # Return the history dictionary for later analysis\n    return history\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-20T03:11:30.277350Z","iopub.execute_input":"2025-08-20T03:11:30.277602Z","iopub.status.idle":"2025-08-20T03:11:30.296878Z","shell.execute_reply.started":"2025-08-20T03:11:30.277578Z","shell.execute_reply":"2025-08-20T03:11:30.296242Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_ids, val_ids = train_test_split(all_sequence_ids, test_size=0.2, random_state=CFG.SEED)\nrun_training(train_ids, val_ids, CFG, label_encoders)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-20T03:11:30.297720Z","iopub.execute_input":"2025-08-20T03:11:30.297996Z","iopub.status.idle":"2025-08-20T03:23:31.388244Z","shell.execute_reply.started":"2025-08-20T03:11:30.297978Z","shell.execute_reply":"2025-08-20T03:23:31.387697Z"}},"outputs":[],"execution_count":null}]}