{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport random\nimport math\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport torch\nfrom torch.utils.data import Dataset\nfrom typing import Dict, Any, Optional, List, Tuple\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.model_selection import train_test_split\n\nfrom scipy import stats\nfrom collections import Counter\nfrom tqdm.auto import tqdm\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-08T15:16:13.815285Z","iopub.execute_input":"2025-06-08T15:16:13.815640Z","iopub.status.idle":"2025-06-08T15:16:13.965072Z","shell.execute_reply.started":"2025-06-08T15:16:13.815617Z","shell.execute_reply":"2025-06-08T15:16:13.964047Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"kaggle_root = \"/kaggle/input/cmi-detect-behavior-with-sensor-data\"\ntrain_csv = \"train.csv\"\ntest_csv = \"test.csv\"\ntest_demo = \"test_demographics.csv\"\nsequence_percentile = 85","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T14:53:32.708891Z","iopub.execute_input":"2025-06-08T14:53:32.709876Z","iopub.status.idle":"2025-06-08T14:53:32.715076Z","shell.execute_reply.started":"2025-06-08T14:53:32.709841Z","shell.execute_reply":"2025-06-08T14:53:32.713924Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class GestureDataset(Dataset):\n    def __init__(self, X: np.ndarray, y: np.ndarray, augment: bool = False, augment_prob: float = 0.3):\n        self.X = X\n        self.y = y\n        self.augment = augment\n        self.augment_prob = augment_prob\n        \n    def __len__(self):\n        return len(self.X)\n    \n    def __getitem__(self, idx: int):\n        x = torch.from_numpy(self.X[idx]).float()\n        y = torch.tensor(self.y[idx], dtype=torch.long)\n        \n        if self.augment and random.random() < self.augment_prob:\n            x = self._augment(x)\n            \n        x = x.transpose(0, 1)\n        return x, y\n    \n    def _augment(self, x: torch.Tensor) -> torch.Tensor:\n        if random.random() < 0.3:\n            noise = torch.randn_like(x) * 0.003\n            x = x + noise\n            \n        if random.random() < 0.2:\n            shift = random.randint(-2, 2)\n            x = torch.roll(x, shift, dims=0)\n            \n        if random.random() < 0.2:\n            scale = random.uniform(0.98, 1.02)\n            x = x * scale\n            \n        return x","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T14:50:01.151682Z","iopub.execute_input":"2025-06-08T14:50:01.152042Z","iopub.status.idle":"2025-06-08T14:50:01.161510Z","shell.execute_reply.started":"2025-06-08T14:50:01.152018Z","shell.execute_reply":"2025-06-08T14:50:01.160081Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def preprocess_sequence(seq_df: pd.DataFrame, imu_cols: List[str]) -> np.ndarray:\n    data = seq_df[imu_cols].copy()\n    data = data.interpolate(method='linear').ffill().bfill().fillna(0)\n    \n    for col in imu_cols:\n        q1, q99 = data[col].quantile([0.01, 0.99])\n        data[col] = data[col].clip(q1, q99)\n    \n    scaler = StandardScaler()\n    data = scaler.fit_transform(data)\n    \n    return data.astype(np.float32)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T14:50:02.952326Z","iopub.execute_input":"2025-06-08T14:50:02.952673Z","iopub.status.idle":"2025-06-08T14:50:02.958708Z","shell.execute_reply.started":"2025-06-08T14:50:02.952650Z","shell.execute_reply":"2025-06-08T14:50:02.957517Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def pad_to_len(arr: np.ndarray, maxlen: int, dtype=np.float32) -> np.ndarray:\n    t, c = arr.shape\n    if t >= maxlen:\n        return arr[:maxlen].astype(dtype, copy=False)\n    out = np.zeros((maxlen, c), dtype=dtype)\n    out[:t] = arr\n    return out","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T14:50:04.449361Z","iopub.execute_input":"2025-06-08T14:50:04.449700Z","iopub.status.idle":"2025-06-08T14:50:04.456442Z","shell.execute_reply.started":"2025-06-08T14:50:04.449677Z","shell.execute_reply":"2025-06-08T14:50:04.455298Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_and_process_data(**args):\n    print(\"Loading and processing data...\")\n    \n    train_path = os.path.join(kaggle_root, train_csv)\n    df = pd.read_csv(train_path)\n    print(f\"Loaded {len(df):,} rows.\")\n    \n    le = LabelEncoder()\n    df[\"gesture\"] = le.fit_transform(df[\"gesture\"].astype(str))\n    np.save(\"gesture_classes.npy\", le.classes_)\n    \n    imu_cols = [\"acc_x\", \"acc_y\", \"acc_z\", \"rot_w\", \"rot_x\", \"rot_y\", \"rot_z\"]\n    \n    print(\"Building sequences...\")\n    seq_groups = df.groupby(\"sequence_id\")\n    X_list, y_list, lengths = [], [], []\n    \n    for seq_id, seq_df in tqdm(seq_groups, desc=\"Processing sequences\"):\n        X_seq = preprocess_sequence(seq_df, imu_cols)\n        X_list.append(X_seq)\n        lengths.append(X_seq.shape[0])\n        y_list.append(seq_df[\"gesture\"].iloc[0])\n    \n    pad_len = int(np.percentile(lengths, sequence_percentile))\n    print(f\"Padding/truncating to length: {pad_len}\")\n    print(f\"Length statistics: min={np.min(lengths)}, max={np.max(lengths)}, mean={np.mean(lengths):.1f}\")\n    np.save(\"sequence_maxlen.npy\", pad_len)\n    \n    X = np.stack([pad_to_len(arr, pad_len) for arr in X_list])\n    y = np.array(y_list, dtype=np.int64)\n    \n    print(f\"Final data shape: X={X.shape}, y={y.shape}\")\n    print(f\"Number of classes: {len(le.classes_)}\")\n    print(f\"Class distribution: {np.bincount(y)}\")\n    \n    return X, y, len(le.classes_), pad_len, le.classes_","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T14:54:50.947273Z","iopub.execute_input":"2025-06-08T14:54:50.947620Z","iopub.status.idle":"2025-06-08T14:54:50.957317Z","shell.execute_reply.started":"2025-06-08T14:54:50.947598Z","shell.execute_reply":"2025-06-08T14:54:50.956090Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def analyze_class_distribution(y, class_names=None, title=\"Class Distribution\"):\n    fig, axes = plt.subplots(2, 1, figsize=(8, 12))\n    \n    class_counts = Counter(y)\n    classes = sorted(class_counts.keys())\n    counts = [class_counts[c] for c in classes]\n    \n    if class_names is not None:\n        labels = [class_names[c] for c in classes]\n    else:\n        labels = [f'Class {c}' for c in classes]\n    \n    # Pie chart\n    colors = plt.cm.Set3(np.linspace(0, 1, len(classes)))\n    wedges, texts, autotexts = axes[0].pie(counts, labels=labels, autopct='%1.1f%%', colors=colors, startangle=90)\n    axes[0].set_title(f'{title} - Distribution', fontsize=14, fontweight='bold')\n    \n    # Bar chart\n    bars = axes[1].bar(range(len(classes)), counts, color=colors)\n    axes[1].set_title(f'{title} - Counts', fontsize=14, fontweight='bold')\n    axes[1].set_xlabel('Class')\n    axes[1].set_ylabel('Count')\n    axes[1].set_xticks(range(len(classes)))\n    axes[1].set_xticklabels(labels, rotation=45, ha='right')\n    \n    for i, (bar, count) in enumerate(zip(bars, counts)):\n        axes[1].text(bar.get_x() + bar.get_width()/2, bar.get_height() + max(counts)*0.01,\n                       f'{count}', ha='center', va='bottom', fontweight='bold')\n    \n    # Imbalance statistics\n    total = sum(counts)\n    percentages = [c/total*100 for c in counts]\n    imbalance_ratio = max(counts) / min(counts)\n    \n    plt.tight_layout()\n    plt.show()\n    \n    print(\"\\033[1;36m\" + \"=\" * 60 + \"\\033[0m\")\n    print(f\"\\033[1;37m{title.upper()} ANALYSIS\\033[0m\")\n    print(\"\\033[1;36m\" + \"=\" * 60 + \"\\033[0m\")\n    print(f\"Total samples: \\033[1;33m{total:,}\\033[0m\")\n    print(f\"Number of classes: \\033[1;33m{len(classes)}\\033[0m\")\n    print(f\"Class imbalance ratio: \\033[1;33m{imbalance_ratio:.2f}\\033[0m (max/min)\")\n    \n    if imbalance_ratio > 2:\n        print(\"\\033[1;31mHIGH CLASS IMBALANCE DETECTED!\\033[0m\")\n        print(\"\\033[0;31m   This could lead to overfitting on majority classes\\033[0m\")\n    elif imbalance_ratio > 1.5:\n        print(\"\\033[1;33mModerate class imbalance detected\\033[0m\")\n    else:\n        print(\"\\033[1;32mClasses are relatively balanced\\033[0m\")\n    \n    print(\"\\n\\033[1;37mClass breakdown:\\033[0m\")\n    for i, (cls, count, pct) in enumerate(zip(classes, counts, percentages)):\n        label = labels[i] if class_names else f'Class {cls}'\n        print(f\"  \\033[1;34m{label}:\\033[0m {count:,} samples (\\033[0;33m{pct:.1f}%\\033[0m)\")\n    \n    return {\n        'class_counts': class_counts,\n        'imbalance_ratio': imbalance_ratio,\n        'total_samples': total,\n        'num_classes': len(classes)\n    }","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T16:08:05.925597Z","iopub.execute_input":"2025-06-08T16:08:05.925967Z","iopub.status.idle":"2025-06-08T16:08:05.940886Z","shell.execute_reply.started":"2025-06-08T16:08:05.925926Z","shell.execute_reply":"2025-06-08T16:08:05.939756Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_train_val_pie_chart(total_train, total_val):\n    fig, ax = plt.subplots(1, 1, figsize=(8, 5))\n    sizes = [total_train, total_val]\n    labels = ['Train', 'Validation']\n    colors = ['#FF6B6B', '#4ECDC4']\n    \n    ax.pie(sizes, labels=labels, autopct='%1.1f%%', colors=colors, startangle=90)\n    ax.set_title('Train/Val Split', fontsize=14, fontweight='bold')\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_class_distribution_chart(y_data, classes, class_names, title, color):\n    fig, ax = plt.subplots(1, 1, figsize=(8, 6.6))\n    counts = Counter(y_data)\n    class_counts = [counts[c] for c in classes]\n    \n    if class_names is not None:\n        class_labels = [class_names[c] for c in classes]\n    else:\n        class_labels = [f'Class {c}' for c in classes]\n    \n    bars = ax.bar(range(len(classes)), class_counts, color=color, alpha=0.7)\n    ax.set_title(title, fontsize=14, fontweight='bold')\n    ax.set_xlabel('Class')\n    ax.set_ylabel('Count')\n    ax.set_xticks(range(len(classes)))\n    ax.set_xticklabels(class_labels, rotation=45, ha='right')\n    \n    for bar, count in zip(bars, class_counts):\n        ax.text(bar.get_x() + bar.get_width()/2, bar.get_height() + max(class_counts)*0.01,\n                f'{count}', ha='center', va='bottom', fontweight='bold')\n    \n    plt.tight_layout()\n    plt.show()\n    \n    return class_counts","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def calculate_stratification_quality(y_train, y_val, classes, val_split):\n    total_train = len(y_train)\n    total_val = len(y_val)\n    \n    train_counts = Counter(y_train)\n    val_counts = Counter(y_val)\n    \n    train_class_counts = [train_counts[c] for c in classes]\n    val_class_counts = [val_counts[c] for c in classes]\n    \n    train_proportions = np.array(train_class_counts) / total_train * 100\n    val_proportions = np.array(val_class_counts) / total_val * 100\n    \n    proportion_diffs = []\n    for i in range(len(classes)):\n        train_pct = train_proportions[i]\n        val_pct = val_proportions[i]\n        diff = abs(train_pct - val_pct)\n        proportion_diffs.append(diff)\n    \n    max_diff = max(proportion_diffs)\n    mean_diff = np.mean(proportion_diffs)\n    \n    return max_diff, mean_diff","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def print_split_analysis_summary(total_train, total_val, val_split, max_diff, mean_diff):\n    print(\"\\033[1;36m\" + \"=\" * 70 + \"\\033[0m\")\n    print(\"\\033[1;37mTRAIN/VALIDATION SPLIT ANALYSIS\\033[0m\")\n    print(\"\\033[1;36m\" + \"=\" * 70 + \"\\033[0m\")\n    print(f\"Train set: \\033[1;33m{total_train:,}\\033[0m samples (\\033[0;33m{(1-val_split)*100:.1f}%\\033[0m)\")\n    print(f\"Validation set: \\033[1;33m{total_val:,}\\033[0m samples (\\033[0;33m{val_split*100:.1f}%\\033[0m)\")\n    print(f\"Maximum class proportion difference: \\033[1;33m{max_diff:.2f}%\\033[0m\")\n    print(f\"Average class proportion difference: \\033[1;33m{mean_diff:.2f}%\\033[0m\")\n\n    if max_diff > 5.0:\n        print(\"\\033[1;31mWARNING: Poor stratification detected!\\033[0m\")\n        print(\"\\033[0;31m   Large differences in class proportions between train/val\\033[0m\")\n        print(\"\\033[0;31m   This could contribute to overfitting issues\\033[0m\")\n\ndef analyze_train_val_split(X, y, val_split=0.2, random_state=42, class_names=None):\n    X_train, X_val, y_train, y_val = train_test_split(\n        X, y, test_size=val_split, random_state=random_state, stratify=y\n    )\n    \n    total_train = len(y_train)\n    total_val = len(y_val)\n    \n    create_train_val_pie_chart(total_train, total_val)\n    \n    train_counts = Counter(y_train)\n    classes = sorted(train_counts.keys())\n    \n    create_class_distribution_chart(y_train, classes, class_names, \n                                  'Class Distribution - Train Set', '#FF6B6B')\n    create_class_distribution_chart(y_val, classes, class_names, \n                                  'Class Distribution - Validation Set', '#4ECDC4')\n    \n    max_diff, mean_diff = calculate_stratification_quality(y_train, y_val, classes, val_split)\n    print_split_analysis_summary(total_train, total_val, val_split, max_diff, mean_diff)\n    \n    return {\n        'train_data': (X_train, y_train),\n        'val_data': (X_val, y_val),\n        'stratification_quality': {\n            'max_diff': max_diff,\n            'mean_diff': mean_diff,\n            'quality': 'excellent' if max_diff < 1.0 else 'good' if max_diff < 2.0 else 'fair' if max_diff < 5.0 else 'poor'\n        }\n    }\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def analyze_train_val_split(X, y, val_split=0.2, random_state=42, class_names=None):\n    X_train, X_val, y_train, y_val = train_test_split(\n        X, y, test_size=val_split, random_state=random_state, stratify=y\n    )\n    \n    total_train = len(y_train)\n    total_val = len(y_val)\n    \n    create_train_val_pie_chart(total_train, total_val)\n    \n    train_counts = Counter(y_train)\n    classes = sorted(train_counts.keys())\n    \n    create_class_distribution_chart(y_train, classes, class_names, \n                                  'Class Distribution - Train Set', '#FF6B6B')\n    create_class_distribution_chart(y_val, classes, class_names, \n                                  'Class Distribution - Validation Set', '#4ECDC4')\n    \n    max_diff, mean_diff = calculate_stratification_quality(y_train, y_val, classes, val_split)\n    print_split_analysis_summary(total_train, total_val, val_split, max_diff, mean_diff)\n    \n    return {\n        'train_data': (X_train, y_train),\n        'val_data': (X_val, y_val),\n        'stratification_quality': {\n            'max_diff': max_diff,\n            'mean_diff': mean_diff,\n            'quality': 'excellent' if max_diff < 1.0 else 'good' if max_diff < 2.0 else 'fair' if max_diff < 5.0 else 'poor'\n        }\n    }","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def calculate_sequence_stats(X, y, classes):\n    sequence_stats = {}\n    \n    for cls in classes:\n        cls_mask = (y == cls)\n        cls_sequences = X[cls_mask]\n        \n        effective_lengths = []\n        for seq in cls_sequences:\n            nonzero_indices = np.where(np.any(seq != 0, axis=1))[0]\n            if len(nonzero_indices) > 0:\n                effective_lengths.append(nonzero_indices[-1] + 1)\n            else:\n                effective_lengths.append(0)\n        \n        sequence_stats[cls] = {\n            'count': len(cls_sequences),\n            'lengths': effective_lengths,\n            'mean_length': np.mean(effective_lengths),\n            'std_length': np.std(effective_lengths),\n            'min_length': np.min(effective_lengths),\n            'max_length': np.max(effective_lengths)\n        }\n    \n    return sequence_stats","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_sequence_length_boxplot(X, y, classes, class_names):\n    sequence_stats = calculate_sequence_stats(X, y, classes)\n    \n    length_data = []\n    length_labels = []\n    \n    for cls in classes:\n        lengths = sequence_stats[cls]['lengths']\n        length_data.append(lengths)\n        label = class_names[cls] if class_names else f'Class {cls}'\n        length_labels.append(label)\n    \n    fig, ax = plt.subplots(1, 1, figsize=(12, 5))\n    ax.boxplot(length_data, labels=length_labels)\n    ax.set_title('Sequence Lengths by Class', fontsize=16, fontweight='bold', pad=20)\n    ax.set_xlabel('Class', fontsize=12)\n    ax.set_ylabel('Effective Sequence Length', fontsize=12)\n    ax.tick_params(axis='x', rotation=45)\n    ax.grid(True, alpha=0.3)\n    \n    return sequence_stats","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_signal_variability_boxplot(X, y, classes, class_names):\n    class_variabilities = []\n    length_labels = []\n    \n    for cls in classes:\n        cls_mask = (y == cls)\n        cls_sequences = X[cls_mask]\n        \n        variabilities = []\n        for seq in cls_sequences:\n            nonzero_mask = np.any(seq != 0, axis=1)\n            if np.any(nonzero_mask):\n                active_seq = seq[nonzero_mask]\n                variabilities.append(np.std(active_seq.flatten()))\n        \n        class_variabilities.append(variabilities)\n        label = class_names[cls] if class_names else f'Class {cls}'\n        length_labels.append(label)\n    \n    fig, ax = plt.subplots(1, 1, figsize=(12, 5))\n    ax.boxplot(class_variabilities, labels=length_labels)\n    ax.set_title('Signal Variability by Class', fontsize=16, fontweight='bold', pad=20)\n    ax.set_xlabel('Class', fontsize=12)\n    ax.set_ylabel('Standard Deviation', fontsize=12)\n    ax.tick_params(axis='x', rotation=45)\n    ax.grid(True, alpha=0.3)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_signal_magnitude_boxplot(X, y, classes, class_names):\n    class_means = []\n    length_labels = []\n    \n    for cls in classes:\n        cls_mask = (y == cls)\n        cls_sequences = X[cls_mask]\n        \n        means = []\n        for seq in cls_sequences:\n            nonzero_mask = np.any(seq != 0, axis=1)\n            if np.any(nonzero_mask):\n                active_seq = seq[nonzero_mask]\n                means.append(np.mean(np.abs(active_seq.flatten())))\n        \n        class_means.append(means)\n        label = class_names[cls] if class_names else f'Class {cls}'\n        length_labels.append(label)\n    \n    fig, ax = plt.subplots(1, 1, figsize=(12, 5))\n    ax.boxplot(class_means, labels=length_labels)\n    ax.set_title('Signal Magnitude by Class', fontsize=16, fontweight='bold', pad=20)\n    ax.set_xlabel('Class', fontsize=12)\n    ax.set_ylabel('Mean Absolute Value', fontsize=12)\n    ax.tick_params(axis='x', rotation=45)\n    ax.grid(True, alpha=0.3)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_sample_sequence_plots(X, y, classes, class_names, sample_sequences):\n    n_classes = len(classes)\n    cols = min(3, n_classes)\n    rows = (n_classes + cols - 1) // cols\n    \n    fig, axes = plt.subplots(rows, cols, figsize=(6*cols, 5*rows))\n    \n    if n_classes == 1:\n        axes = [axes]\n    elif rows == 1:\n        axes = axes.reshape(1, -1)\n    elif cols == 1:\n        axes = axes.reshape(-1, 1)\n    \n    sample_colors = plt.cm.tab10(np.linspace(0, 1, sample_sequences))\n    \n    for i, cls in enumerate(classes):\n        row = i // cols\n        col = i % cols\n        \n        cls_mask = (y == cls)\n        cls_sequences = X[cls_mask]\n        \n        n_samples = min(sample_sequences, len(cls_sequences))\n        \n        if len(cls_sequences) > 0:\n            sample_indices = np.random.choice(len(cls_sequences), n_samples, replace=False)\n            \n            ax = axes[row, col] if rows > 1 else axes[col]\n            \n            for j, idx in enumerate(sample_indices):\n                seq = cls_sequences[idx]\n                nonzero_mask = np.any(seq != 0, axis=1)\n                if np.any(nonzero_mask):\n                    active_seq = seq[nonzero_mask, 0]\n                    color = sample_colors[j % len(sample_colors)]\n                    ax.plot(active_seq, alpha=0.8, linewidth=1.5, color=color, label=f'Sample {j+1}')\n            \n            label = class_names[cls] if class_names else f'Class {cls}'\n            ax.set_title(f'Sample Sequences - {label}', fontsize=12, fontweight='bold')\n            ax.set_xlabel('Time Step', fontsize=10)\n            ax.set_ylabel('Feature Value', fontsize=10)\n            ax.grid(True, alpha=0.3)\n            ax.legend(fontsize=8)\n        else:\n            ax = axes[row, col] if rows > 1 else axes[col]\n            ax.text(0.5, 0.5, 'No sequences\\navailable', \n                   horizontalalignment='center', verticalalignment='center',\n                   transform=ax.transAxes, fontsize=12)\n            ax.set_title(f'Class {cls} - No Data', fontsize=12, fontweight='bold')\n    \n    total_plots = rows * cols\n    for i in range(n_classes, total_plots):\n        row = i // cols\n        col = i % cols\n        ax = axes[row, col] if rows > 1 else axes[col]\n        ax.axis('off')\n    \n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def interpolate_sequence_for_profile(seq, common_length, n_features):\n    from scipy import interpolate\n    \n    if len(seq) > 1:\n        interp_seq = np.zeros((common_length, n_features))\n        for feat_idx in range(n_features):\n            x_orig = np.linspace(0, 1, len(seq))\n            x_new = np.linspace(0, 1, common_length)\n            f = interpolate.interp1d(x_orig, seq[:, feat_idx], \n                                   kind='linear', fill_value='extrapolate')\n            interp_seq[:, feat_idx] = f(x_new)\n        return interp_seq\n    return None","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_feature_specific_profiles(X, y, classes, class_names, n_features):\n    import matplotlib.pyplot as plt\n    import numpy as np\n    from scipy import interpolate\n    \n    n_classes = len(classes)\n    \n    feature_colors = [\n        '#E74C3C',\n        '#8E44AD',\n        '#3498DB',\n        '#2ECC71',\n        '#F39C12',\n        '#1ABC9C',\n        '#E67E22',\n        '#9B59B6',\n        '#34495E',\n        '#16A085',\n        '#D35400',\n        '#7F8C8D',\n        '#C0392B',\n        '#27AE60',\n        '#2980B9',\n        '#8F4A84',\n        '#B7950B',\n        '#17A2B8'\n    ]\n    \n    def interpolate_sequences(sequences, common_length=100):\n        interpolated = []\n        for seq in sequences:\n            if len(seq) > 1:\n                x_orig = np.linspace(0, 1, len(seq))\n                x_new = np.linspace(0, 1, common_length)\n                f = interpolate.interp1d(x_orig, seq, kind='linear', fill_value='extrapolate')\n                interpolated.append(f(x_new))\n        return np.array(interpolated) if interpolated else None\n    \n    all_profiles = {}\n    \n    for feat_idx in range(n_features):\n        print(f\"\\n\\033[1;37mFeature {feat_idx + 1} profiles:\\033[0m\")\n        \n        feature_color = feature_colors[feat_idx % len(feature_colors)]\n        all_profiles[feat_idx] = {}\n        \n        cols = min(3, n_classes)\n        rows = (n_classes + cols - 1) // cols\n        \n        fig, axes = plt.subplots(rows, cols, figsize=(6*cols, 5*rows))\n        \n        if n_classes == 1:\n            axes = [axes]\n        elif rows == 1:\n            axes = axes.reshape(1, -1)\n        elif cols == 1:\n            axes = axes.reshape(-1, 1)\n        \n        for i, cls in enumerate(classes):\n            row = i // cols\n            col = i % cols\n            \n            cls_mask = (y == cls)\n            cls_sequences = X[cls_mask]\n            \n            if len(cls_sequences) > 0:\n                ax = axes[row, col] if rows > 1 else axes[col]\n                \n                all_sequences = []\n                for seq in cls_sequences:\n                    nonzero_mask = np.any(seq != 0, axis=1)\n                    if np.any(nonzero_mask):\n                        active_seq = seq[nonzero_mask, feat_idx]\n                        if len(active_seq) > 0:\n                            all_sequences.append(active_seq)\n                \n                if all_sequences:\n                    max_length = max([len(seq) for seq in all_sequences])\n                    common_length = min(100, max_length)\n                    \n                    sequences_array = interpolate_sequences(all_sequences, common_length)\n                    \n                    if sequences_array is not None and len(sequences_array) > 0:\n                        mean_profile = np.mean(sequences_array, axis=0)\n                        std_profile = np.std(sequences_array, axis=0)\n                        \n                        all_profiles[feat_idx][cls] = {\n                            'mean': mean_profile,\n                            'std': std_profile,\n                            'length': common_length\n                        }\n                        \n                        x_axis = np.linspace(0, 100, common_length)\n                        \n                        ax.plot(x_axis, mean_profile, \n                               color=feature_color, linewidth=3, \n                               label=f'Feature {feat_idx + 1} Profile')\n                        \n                        ax.fill_between(x_axis, \n                                      mean_profile - std_profile,\n                                      mean_profile + std_profile,\n                                      color=feature_color, alpha=0.2,\n                                      label='± 1 Std Dev')\n                \n                label = class_names[cls] if class_names else f'Class {cls}'\n                ax.set_title(f'Feature {feat_idx + 1} Profile - {label}', fontsize=12, fontweight='bold')\n                ax.set_xlabel('Normalized Time (%)', fontsize=10)\n                ax.set_ylabel(f'Feature {feat_idx + 1} Value', fontsize=10)\n                ax.grid(True, alpha=0.3)\n                ax.legend(fontsize=8)\n            else:\n                ax = axes[row, col] if rows > 1 else axes[col]\n                ax.text(0.5, 0.5, 'No sequences\\navailable', \n                       horizontalalignment='center', verticalalignment='center',\n                       transform=ax.transAxes, fontsize=12)\n                ax.set_title(f'Class {cls} - No Data', fontsize=12, fontweight='bold')\n        \n        total_plots = rows * cols\n        for i in range(n_classes, total_plots):\n            row = i // cols\n            col = i % cols\n            ax = axes[row, col] if rows > 1 else axes[col]\n            ax.axis('off')\n        \n        plt.tight_layout()\n        plt.show()\n    \n    print(f\"\\n\\033[1;37mClass-wise feature comparison:\\033[0m\")\n    \n    cols = min(3, n_classes)\n    rows = (n_classes + cols - 1) // cols\n    \n    fig, axes = plt.subplots(rows, cols, figsize=(8*cols, 6*rows))\n    \n    if n_classes == 1:\n        axes = [axes]\n    elif rows == 1:\n        axes = axes.reshape(1, -1)\n    elif cols == 1:\n        axes = axes.reshape(-1, 1)\n    \n    for i, cls in enumerate(classes):\n        row = i // cols\n        col = i % cols\n        ax = axes[row, col] if rows > 1 else axes[col]\n        \n        has_data = False\n        \n        for feat_idx in range(n_features):\n            if feat_idx in all_profiles and cls in all_profiles[feat_idx]:\n                profile_data = all_profiles[feat_idx][cls]\n                feature_color = feature_colors[feat_idx % len(feature_colors)]\n                \n                mean_profile = profile_data['mean']\n                std_profile = profile_data['std']\n                common_length = profile_data['length']\n                x_axis = np.linspace(0, 100, common_length)\n                \n                ax.plot(x_axis, mean_profile, \n                       color=feature_color, linewidth=2.5, \n                       label=f'Feature {feat_idx + 1}')\n                \n                ax.fill_between(x_axis, \n                              mean_profile - std_profile,\n                              mean_profile + std_profile,\n                              color=feature_color, alpha=0.15)\n                \n                has_data = True\n        \n        if has_data:\n            label = class_names[cls] if class_names else f'Class {cls}'\n            ax.set_title(f'All Features - {label}', fontsize=14, fontweight='bold')\n            ax.set_xlabel('Normalized Time (%)', fontsize=11)\n            ax.set_ylabel('Feature Values', fontsize=11)\n            ax.grid(True, alpha=0.3)\n            ax.legend(fontsize=9, loc='best')\n        else:\n            ax.text(0.5, 0.5, 'No data\\navailable', \n                   horizontalalignment='center', verticalalignment='center',\n                   transform=ax.transAxes, fontsize=12)\n            label = class_names[cls] if class_names else f'Class {cls}'\n            ax.set_title(f'All Features - {label}', fontsize=14, fontweight='bold')\n    \n    total_plots = rows * cols\n    for i in range(n_classes, total_plots):\n        row = i // cols\n        col = i % cols\n        ax = axes[row, col] if rows > 1 else axes[col]\n        ax.axis('off')\n    \n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T16:40:32.061647Z","iopub.execute_input":"2025-06-08T16:40:32.062154Z","iopub.status.idle":"2025-06-08T16:40:32.114694Z","shell.execute_reply.started":"2025-06-08T16:40:32.062112Z","shell.execute_reply":"2025-06-08T16:40:32.113464Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_overall_average_profiles(X, y, classes, class_names, n_features):\n    n_classes = len(classes)\n    cols = min(3, n_classes)\n    rows = (n_classes + cols - 1) // cols\n    \n    fig, axes = plt.subplots(rows, cols, figsize=(6*cols, 5*rows))\n    \n    if n_classes == 1:\n        axes = [axes]\n    elif rows == 1:\n        axes = axes.reshape(1, -1)\n    elif cols == 1:\n        axes = axes.reshape(-1, 1)\n    \n    for i, cls in enumerate(classes):\n        row = i // cols\n        col = i % cols\n        \n        cls_mask = (y == cls)\n        cls_sequences = X[cls_mask]\n        \n        if len(cls_sequences) > 0:\n            ax = axes[row, col] if rows > 1 else axes[col]\n            \n            max_length = max([len(seq[np.any(seq != 0, axis=1)]) for seq in cls_sequences \n                             if np.any(np.any(seq != 0, axis=1))])\n            \n            if max_length > 0:\n                all_sequences = []\n                for seq in cls_sequences:\n                    nonzero_mask = np.any(seq != 0, axis=1)\n                    if np.any(nonzero_mask):\n                        active_seq = seq[nonzero_mask]\n                        if len(active_seq) > 0:\n                            all_sequences.append(active_seq)\n                \n                if all_sequences:\n                    common_length = min(100, max_length)\n                    interpolated_sequences = []\n                    \n                    for seq in all_sequences:\n                        interp_seq = interpolate_sequence_for_profile(seq, common_length, n_features)\n                        if interp_seq is not None:\n                            interpolated_sequences.append(interp_seq)\n                    \n                    if interpolated_sequences:\n                        sequences_array = np.array(interpolated_sequences)\n                        mean_profile = np.mean(sequences_array, axis=0)\n                        std_profile = np.std(sequences_array, axis=0)\n                        \n                        x_axis = np.linspace(0, 100, common_length)\n                        \n                        overall_mean = np.mean(mean_profile, axis=1)\n                        overall_std = np.mean(std_profile, axis=1)\n                        \n                        ax.plot(x_axis, overall_mean, \n                               color='blue', linewidth=3, \n                               label='Average Profile')\n                        \n                        ax.fill_between(x_axis, \n                                      overall_mean - overall_std,\n                                      overall_mean + overall_std,\n                                      color='blue', alpha=0.2,\n                                      label='± 1 Std Dev')\n            \n            label = class_names[cls] if class_names else f'Class {cls}'\n            ax.set_title(f'Overall Average Profile - {label}', fontsize=12, fontweight='bold')\n            ax.set_xlabel('Normalized Time (%)', fontsize=10)\n            ax.set_ylabel('Average Feature Value', fontsize=10)\n            ax.grid(True, alpha=0.3)\n            ax.legend(fontsize=8)\n        else:\n            ax = axes[row, col] if rows > 1 else axes[col]\n            ax.text(0.5, 0.5, 'No sequences\\navailable', \n                   horizontalalignment='center', verticalalignment='center',\n                   transform=ax.transAxes, fontsize=12)\n            ax.set_title(f'Class {cls} - No Data', fontsize=12, fontweight='bold')\n    \n    total_plots = rows * cols\n    for i in range(n_classes, total_plots):\n        row = i // cols\n        col = i % cols\n        ax = axes[row, col] if rows > 1 else axes[col]\n        ax.axis('off')\n    \n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T16:28:03.910505Z","iopub.execute_input":"2025-06-08T16:28:03.910832Z","iopub.status.idle":"2025-06-08T16:28:03.927890Z","shell.execute_reply.started":"2025-06-08T16:28:03.910810Z","shell.execute_reply":"2025-06-08T16:28:03.927016Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def interpolate_energy_profile(time_percent, energy, common_time):\n    from scipy import interpolate\n    \n    if len(time_percent) > 1:\n        f = interpolate.interp1d(time_percent, energy, \n                               kind='linear', fill_value='extrapolate')\n        return f(common_time)\n    return None","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_signal_energy_distribution(X, y, classes, class_names):\n    n_classes = len(classes)\n    cols = min(3, n_classes)\n    rows = (n_classes + cols - 1) // cols\n    \n    fig, axes = plt.subplots(rows, cols, figsize=(6*cols, 5*rows))\n    \n    if n_classes == 1:\n        axes = [axes]\n    elif rows == 1:\n        axes = axes.reshape(1, -1)\n    elif cols == 1:\n        axes = axes.reshape(-1, 1)\n    \n    for i, cls in enumerate(classes):\n        row = i // cols\n        col = i % cols\n        \n        cls_mask = (y == cls)\n        cls_sequences = X[cls_mask]\n        \n        if len(cls_sequences) > 0:\n            ax = axes[row, col] if rows > 1 else axes[col]\n            \n            energy_distributions = []\n            \n            for seq in cls_sequences:\n                nonzero_mask = np.any(seq != 0, axis=1)\n                if np.any(nonzero_mask):\n                    active_seq = seq[nonzero_mask]\n                    if len(active_seq) > 1:\n                        energy = np.sum(active_seq**2, axis=1)\n                        time_percent = np.linspace(0, 100, len(energy))\n                        energy_distributions.append((time_percent, energy))\n            \n            if energy_distributions:\n                for time_percent, energy in energy_distributions:\n                    ax.plot(time_percent, energy, alpha=0.3, color='blue', linewidth=0.5)\n                \n                common_time = np.linspace(0, 100, 50)\n                interpolated_energies = []\n                \n                for time_percent, energy in energy_distributions:\n                    interp_energy = interpolate_energy_profile(time_percent, energy, common_time)\n                    if interp_energy is not None:\n                        interpolated_energies.append(interp_energy)\n                \n                if interpolated_energies:\n                    mean_energy = np.mean(interpolated_energies, axis=0)\n                    std_energy = np.std(interpolated_energies, axis=0)\n                    \n                    ax.plot(common_time, mean_energy, color='red', linewidth=3, \n                           label='Average Energy', zorder=10)\n                    \n                    ax.fill_between(common_time, \n                                  mean_energy - std_energy,\n                                  mean_energy + std_energy,\n                                  color='red', alpha=0.2, zorder=5)\n            \n            label = class_names[cls] if class_names else f'Class {cls}'\n            ax.set_title(f'Signal Energy Distribution - {label}', fontsize=12, fontweight='bold')\n            ax.set_xlabel('Normalized Time (%)', fontsize=10)\n            ax.set_ylabel('Signal Energy', fontsize=10)\n            ax.grid(True, alpha=0.3)\n            ax.legend(fontsize=8)\n        else:\n            ax = axes[row, col] if rows > 1 else axes[col]\n            ax.text(0.5, 0.5, 'No sequences\\navailable', \n                   horizontalalignment='center', verticalalignment='center',\n                   transform=ax.transAxes, fontsize=12)\n            ax.set_title(f'Class {cls} - No Data', fontsize=12, fontweight='bold')\n    \n    total_plots = rows * cols\n    for i in range(n_classes, total_plots):\n        row = i // cols\n        col = i % cols\n        ax = axes[row, col] if rows > 1 else axes[col]\n        ax.axis('off')\n    \n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def analyze_sequence_characteristics(X, y, class_names=None, sample_sequences=5):\n    print(\"\\033[1;36m\" + \"=\" * 70 + \"\\033[0m\")\n    print(\"\\033[1;37mSEQUENCE CHARACTERISTICS ANALYSIS\\033[0m\")\n    print(\"\\033[1;36m\" + \"=\" * 70 + \"\\033[0m\")\n    \n    classes = np.unique(y)\n    n_classes = len(classes)\n    n_features = X.shape[2]\n    \n    sequence_stats = create_sequence_length_boxplot(X, y, classes, class_names)\n    create_signal_variability_boxplot(X, y, classes, class_names)\n    create_signal_magnitude_boxplot(X, y, classes, class_names)\n    \n    print(f\"\\n\\033[1;37mCreating sample sequence plots for {n_classes} classes...\\033[0m\")\n    create_sample_sequence_plots(X, y, classes, class_names, sample_sequences)\n    \n    print(f\"\\n\\033[1;37mCreating average sequence profiles for {n_classes} classes...\\033[0m\")\n    create_feature_specific_profiles(X, y, classes, class_names, n_features)\n    \n    print(f\"\\n\\033[1;37mCreating overall average profiles for {n_classes} classes...\\033[0m\")\n    create_overall_average_profiles(X, y, classes, class_names, n_features)\n    \n    print(f\"\\n\\033[1;37mCreating signal energy distribution analysis...\\033[0m\")\n    create_signal_energy_distribution(X, y, classes, class_names)\n    \n    max_length_diff = max([stats['mean_length'] for stats in sequence_stats.values()]) - \\\n                     min([stats['mean_length'] for stats in sequence_stats.values()])\n    \n    print(f\"\\n\\033[1;32mAnalysis completed successfully!\\033[0m\")\n    print(f\"Maximum difference in mean lengths: \\033[1;33m{max_length_diff:.1f}\\033[0m\")\n    \n    return {\n        'sequence_stats': sequence_stats,\n        'max_length_difference': max_length_diff\n    }","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T16:40:51.722693Z","iopub.execute_input":"2025-06-08T16:40:51.723017Z","iopub.status.idle":"2025-06-08T16:40:51.732456Z","shell.execute_reply.started":"2025-06-08T16:40:51.722999Z","shell.execute_reply":"2025-06-08T16:40:51.731050Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X, y, num_classes, pad_len, class_names = load_and_process_data(\n    kaggle_root=kaggle_root,\n    train_csv=train_csv,\n    test_csv=test_csv,\n    test_demo=test_demo,\n    sequence_percentile=sequence_percentile\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T15:22:04.801252Z","iopub.execute_input":"2025-06-08T15:22:04.802631Z","iopub.status.idle":"2025-06-08T15:24:36.806562Z","shell.execute_reply.started":"2025-06-08T15:22:04.802580Z","shell.execute_reply":"2025-06-08T15:24:36.805484Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"\\nDataset Summary:\")\nprint(f\"   • Data shape: {X.shape}\")\nprint(f\"   • Number of classes: {num_classes}\")\nprint(f\"   • Sequence length: {pad_len}\")\nprint(f\"   • Class names: {list(class_names)}\")\n\nif isinstance(class_names, np.ndarray):\n    class_names_list = class_names.tolist()\nelse:\n    class_names_list = class_names\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"1. ANALYZING CLASS DISTRIBUTION\")\nprint(\"=\"*80)\nclass_stats = analyze_class_distribution(y, class_names_list, \"Training Data\")\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"2. ANALYZING TRAIN/VALIDATION SPLIT\")\nprint(\"=\"*80)\nsplit_results = analyze_train_val_split(X, y, \n                                       val_split=0.2, \n                                       random_state=42,\n                                       class_names=class_names_list)\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"3. ANALYZING SEQUENCE CHARACTERISTICS\")\nprint(\"=\"*80)\nsequence_stats = analyze_sequence_characteristics(X, y, \n                                                class_names=class_names_list,\n                                                sample_sequences=3)\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"ANALYSIS SUMMARY\")\nprint(\"=\"*80)\n\nprint(\"Class Balance Assessment:\")\nif class_stats['imbalance_ratio'] > 2:\n    print(\"   HIGH IMBALANCE - Consider class weighting or resampling\")\n    print(f\"   Imbalance ratio: {class_stats['imbalance_ratio']:.2f}\")\nelif class_stats['imbalance_ratio'] > 1.5:\n    print(\"   MODERATE IMBALANCE - Monitor for overfitting\")\n    print(f\"   Imbalance ratio: {class_stats['imbalance_ratio']:.2f}\")\nelse:\n    print(\"   WELL BALANCED - Good for training\")\n    print(f\"   Imbalance ratio: {class_stats['imbalance_ratio']:.2f}\")\n\nprint(f\"\\nTrain/Val Stratification:\")\nstratification = split_results['stratification_quality']\nprint(f\"   Quality: {stratification['quality'].upper()}\")\nprint(f\"   Max difference: {stratification['max_diff']:.2f}%\")\n\nprint(f\"\\nSequence Length Analysis:\")\nmax_length_diff = sequence_stats['max_length_difference']\nif max_length_diff > pad_len * 0.2:\n    print(\"   SIGNIFICANT LENGTH VARIATION - May cause bias\")\n    print(f\"  Max difference: {max_length_diff:.1f} steps\")\nelse:\n    print(\"   CONSISTENT LENGTHS - Good for training\")\n    print(f\"  Max difference: {max_length_diff:.1f} steps\")\n\nprint(f\"\\nAnalysis complete! Ready for model training.\")\nprint(\"=\"*80)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-08T16:40:55.459380Z","iopub.execute_input":"2025-06-08T16:40:55.459739Z","iopub.status.idle":"2025-06-08T16:42:24.914190Z","shell.execute_reply.started":"2025-06-08T16:40:55.459715Z","shell.execute_reply":"2025-06-08T16:42:24.913145Z"}},"outputs":[],"execution_count":null}]}