{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59156,"databundleVersionId":13874099,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\"\"\"\nMABe Mouse Behavior Detection - MEMORY-EFFICIENT TEMPORAL FEATURES\nAddresses the long-range temporal dependency problem with:\n1. Hierarchical multi-scale windows (exponential spacing)\n2. Selective windowing (long windows only for important features)\n3. EMA for efficient long-term trend capture\n4. Temporal compression via smart aggregation\n\"\"\"\n\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\nimport itertools\nimport warnings\nimport json\nimport os\nimport gc\nimport lightgbm as lgb\nfrom collections import defaultdict\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom scipy.spatial import ConvexHull, distance\nfrom scipy import signal\nfrom scipy.stats import skew, kurtosis\nfrom scipy.signal import welch\nfrom scipy.cluster.hierarchy import dendrogram, linkage, fcluster\nfrom scipy.spatial.distance import pdist, squareform\n\nfrom sklearn.base import ClassifierMixin, BaseEstimator, clone\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom sklearn.decomposition import PCA\nfrom sklearn.cluster import KMeans\nfrom sklearn.metrics import silhouette_score\n\nwarnings.filterwarnings('ignore')\n\n# ==================== CONFIGURATION ====================\n\nclass CFG:\n \"\"\"Configuration for intelligent feature reduction\"\"\"\n \n # Training configuration\n PRESET = 'FAST'\n MAX_MEMORY_ROWS = 4000000\n SAMPLE_PERCENT = 0.80\n \n USE_CHUNK_MIXING = True\n CHUNK_OVERLAP = 0.30\n N_MIXED_CHUNKS = 2\n \n USE_CLASS_WEIGHTS = True\n PREDICTION_THRESHOLD = 0.27\n MIN_BOUT_LENGTH = 3\n VERBOSE = True\n \n # Feature importance and reduction\n TRACK_FEATURE_IMPORTANCE = True\n TOP_N_FEATURES = 50\n \n # Feature stability settings\n STABILITY_WEIGHT = 0.3\n MIN_STABILITY_SCORE = 0.5\n \n # Feature reduction settings\n LOW_IMPORTANCE_THRESHOLD = 0.1\n FEATURE_SIMILARITY_THRESHOLD = 0.7\n MIN_GROUP_SIZE = 2\n MAX_GROUP_SIZE = 5\n USE_PCA_REDUCTION = True\n USE_MEDOID_REDUCTION = True\n \n # Temporal window analysis\n ANALYZE_TEMPORAL_WINDOWS = True\n TEMPORAL_WINDOWS = [5, 10, 30, 60, 120, 180] # Extended for analysis\n \n # MEMORY-EFFICIENT TEMPORAL FEATURES\n # Hierarchical multi-scale windows (exponentially spaced)\n SHORT_WINDOWS = [5, 10, 15] # All features get these\n MEDIUM_WINDOWS = [30, 60] # Top 50% features get these\n LONG_WINDOWS = [90, 120, 180] # Top 20% features get these\n \n # EMA spans for efficient long-term trends (very memory efficient)\n EMA_SPANS = [10, 30, 60, 120, 180] # Extended to capture long-term\n \n # Which statistics to compute per window\n COMPUTE_MEAN = True\n COMPUTE_STD = True # Only for short/medium windows\n COMPUTE_MAX = False # Disabled to save memory\n COMPUTE_MIN = False # Disabled to save memory\n \n # Advanced features\n USE_EMA_FEATURES = True\n USE_WAVELET_FEATURES = False\n USE_PCA_FEATURES = False\n USE_LAGGED_FEATURES = True\n LAG_WINDOWS = [30, 60, 120] # Specific lag windows\n \n # Temporal compression\n USE_TEMPORAL_COMPRESSION = True # Compress temporal features with PCA\n TEMPORAL_PCA_VARIANCE = 0.95 # Keep 95% of variance\n \n @staticmethod\n def get_preset_models():\n \"\"\"Returns model configurations based on PRESET\"\"\"\n \n if CFG.PRESET == 'FAST':\n return [\n {\n 'name': 'Shallow',\n 'params': {\n 'n_estimators': 200,\n 'learning_rate': 0.08,\n 'num_leaves': 31,\n 'min_child_samples': 40,\n 'subsample': 0.8,\n 'colsample_bytree': 0.8,\n 'max_bin': 255,\n 'verbose': -1,\n 'force_col_wise': True,\n 'importance_type': 'gain'\n }\n }\n ]\n \n elif CFG.PRESET == 'BALANCED':\n return [\n {\n 'name': 'Shallow',\n 'params': {\n 'n_estimators': 225,\n 'learning_rate': 0.07,\n 'num_leaves': 31,\n 'min_child_samples': 40,\n 'subsample': 0.8,\n 'colsample_bytree': 0.8,\n 'max_bin': 255,\n 'verbose': -1,\n 'force_col_wise': True,\n 'importance_type': 'gain'\n }\n },\n {\n 'name': 'Deep',\n 'params': {\n 'n_estimators': 150,\n 'learning_rate': 0.1,\n 'num_leaves': 63,\n 'max_depth': 8,\n 'min_child_samples': 20,\n 'subsample': 0.7,\n 'colsample_bytree': 0.9,\n 'reg_alpha': 0.1,\n 'reg_lambda': 0.1,\n 'max_bin': 255,\n 'verbose': -1,\n 'force_col_wise': True,\n 'importance_type': 'gain'\n }\n }\n ]\n \n else:\n raise ValueError(f\"Unknown PRESET: {CFG.PRESET}\")\n\n# ==================== FEATURE IMPORTANCE TRACKER ====================\n\nclass FeatureImportanceTracker:\n \"\"\"Track and analyze feature importance and stability across models\"\"\"\n \n def __init__(self):\n self.importance_dict = defaultdict(lambda: defaultdict(list))\n self.global_importance = {}\n self.feature_stability = {}\n \n def add_importance(self, action, feature_names, importance_values):\n \"\"\"Add feature importance for a specific action\"\"\"\n for fname, imp_val in zip(feature_names, importance_values):\n self.importance_dict[action][fname].append(imp_val)\n \n def compute_global_importance(self):\n \"\"\"Compute global feature importance across all actions\"\"\"\n global_importance = defaultdict(list)\n \n for action in self.importance_dict.keys():\n for fname, imp_list in self.importance_dict[action].items():\n global_importance[fname].extend(imp_list)\n \n self.global_importance = {\n fname: np.mean(imp_list) \n for fname, imp_list in global_importance.items()\n }\n \n return self.global_importance\n \n def compute_feature_stability(self):\n \"\"\"Compute stability scores for features (consistency across actions)\"\"\"\n feature_stability = {}\n \n for fname in self.global_importance.keys():\n action_importances = []\n for action in self.importance_dict.keys():\n if fname in self.importance_dict[action]:\n action_importances.append(np.mean(self.importance_dict[action][fname]))\n \n if len(action_importances) > 1:\n mean_imp = np.mean(action_importances)\n std_imp = np.std(action_importances)\n cv = std_imp / (mean_imp + 1e-10)\n stability = 1.0 / (cv + 0.1)\n \n stability_score = 1.0 / (1.0 + np.exp(-0.5 * (stability - 2)))\n feature_stability[fname] = stability_score\n else:\n feature_stability[fname] = 0.5\n \n self.feature_stability = feature_stability\n return feature_stability\n \n def get_stable_important_features(self, n=50, stability_weight=0.3):\n \"\"\"Get top features based on combined importance and stability score\"\"\"\n if not self.global_importance:\n self.compute_global_importance()\n \n if not self.feature_stability:\n self.compute_feature_stability()\n \n importances = np.array(list(self.global_importance.values()))\n if len(importances) == 0:\n return []\n \n normalized_importance = (importances - importances.min()) / (importances.max() - importances.min() + 1e-10)\n \n combined_scores = {}\n for i, fname in enumerate(self.global_importance.keys()):\n importance_score = normalized_importance[i]\n stability_score = self.feature_stability.get(fname, 0.5)\n \n combined = (1 - stability_weight) * importance_score + stability_weight * stability_score\n combined_scores[fname] = combined\n \n sorted_features = sorted(combined_scores.items(), key=lambda x: x[1], reverse=True)[:n]\n \n return [f[0] for f in sorted_features]\n \n def get_top_base_features(self, n=20, exclude_temporal=True):\n \"\"\"Get top N base features (excluding temporal aggregations if specified)\"\"\"\n if not self.global_importance:\n self.compute_global_importance()\n \n if exclude_temporal:\n # Exclude features with temporal indicators\n temporal_indicators = ['_m', '_s', '_ema', '_lag', '_max', '_min', 'pca', 'medoid']\n base_features = {\n fname: imp for fname, imp in self.global_importance.items()\n if not any(ind in fname for ind in temporal_indicators)\n }\n else:\n base_features = self.global_importance\n \n sorted_features = sorted(\n base_features.items(), \n key=lambda x: x[1], \n reverse=True\n )[:n]\n \n return [f[0] for f in sorted_features]\n \n def get_low_importance_features(self, percentile_threshold=10, available_features=None):\n \"\"\"Get features below importance threshold\"\"\"\n if not self.global_importance:\n self.compute_global_importance()\n \n if len(self.global_importance) == 0:\n return []\n \n if available_features is not None:\n available_set = set(available_features)\n filtered_importance = {\n fname: imp for fname, imp in self.global_importance.items()\n if fname in available_set\n }\n else:\n filtered_importance = self.global_importance\n \n if len(filtered_importance) == 0:\n return []\n \n importances = np.array(list(filtered_importance.values()))\n threshold = np.percentile(importances, percentile_threshold)\n \n low_importance = [\n fname for fname, imp in filtered_importance.items()\n if imp <= threshold\n ]\n \n return low_importance\n \n def get_unstable_low_importance_features(self, importance_percentile=20, min_stability=0.5, available_features=None):\n \"\"\"Get features that are both low importance AND unstable\"\"\"\n if not self.global_importance:\n self.compute_global_importance()\n \n if not self.feature_stability:\n self.compute_feature_stability()\n \n if len(self.global_importance) == 0:\n return []\n \n if available_features is not None:\n available_set = set(available_features)\n filtered_importance = {\n fname: imp for fname, imp in self.global_importance.items()\n if fname in available_set\n }\n else:\n filtered_importance = self.global_importance\n \n if len(filtered_importance) == 0:\n return []\n \n importances = np.array(list(filtered_importance.values()))\n threshold = np.percentile(importances, importance_percentile)\n \n unstable_low_importance = [\n fname for fname, imp in filtered_importance.items()\n if imp <= threshold and self.feature_stability.get(fname, 1.0) < min_stability\n ]\n \n return unstable_low_importance\n \n def get_top_features(self, n=50):\n \"\"\"Get top N most important features\"\"\"\n if not self.global_importance:\n self.compute_global_importance()\n \n sorted_features = sorted(\n self.global_importance.items(), \n key=lambda x: x[1], \n reverse=True\n )[:n]\n \n return [f[0] for f in sorted_features]\n \n def plot_importance_distribution(self, save_path='feature_importance_distribution.png'):\n \"\"\"Plot distribution of feature importance\"\"\"\n if not self.global_importance:\n self.compute_global_importance()\n \n importances = list(self.global_importance.values())\n \n fig, axes = plt.subplots(1, 2, figsize=(14, 5))\n \n axes[0].hist(importances, bins=50, color='steelblue', edgecolor='black')\n axes[0].set_xlabel('Feature Importance', fontsize=12)\n axes[0].set_ylabel('Frequency', fontsize=12)\n axes[0].set_title('Feature Importance Distribution', fontsize=14, fontweight='bold')\n axes[0].axvline(np.percentile(importances, 10), color='red', linestyle='--', \n label=f'10th percentile: {np.percentile(importances, 10):.2f}')\n axes[0].legend()\n \n sorted_imp = np.sort(importances)\n cumsum = np.cumsum(sorted_imp) / np.sum(sorted_imp)\n axes[1].plot(range(len(cumsum)), cumsum, color='green', linewidth=2)\n axes[1].set_xlabel('Feature Rank', fontsize=12)\n axes[1].set_ylabel('Cumulative Importance', fontsize=12)\n axes[1].set_title('Cumulative Feature Importance', fontsize=14, fontweight='bold')\n axes[1].axhline(0.8, color='red', linestyle='--', label='80% threshold')\n axes[1].axhline(0.9, color='orange', linestyle='--', label='90% threshold')\n axes[1].legend()\n axes[1].grid(True, alpha=0.3)\n \n plt.tight_layout()\n plt.savefig(save_path, dpi=150, bbox_inches='tight')\n print(f\" Saved importance distribution: {save_path}\")\n plt.close()\n \n def plot_stability_analysis(self, save_path='feature_stability_analysis.png'):\n \"\"\"Plot feature importance vs stability\"\"\"\n if not self.global_importance:\n self.compute_global_importance()\n \n if not self.feature_stability:\n self.compute_feature_stability()\n \n features = list(self.global_importance.keys())\n importances = [self.global_importance[f] for f in features]\n stabilities = [self.feature_stability.get(f, 0.5) for f in features]\n \n fig, axes = plt.subplots(1, 2, figsize=(14, 5))\n \n scatter = axes[0].scatter(importances, stabilities, alpha=0.6, s=30, c=importances, cmap='viridis')\n axes[0].set_xlabel('Feature Importance', fontsize=12)\n axes[0].set_ylabel('Feature Stability', fontsize=12)\n axes[0].set_title('Feature Importance vs Stability', fontsize=14, fontweight='bold')\n axes[0].axhline(CFG.MIN_STABILITY_SCORE, color='red', linestyle='--', \n label=f'Min stability: {CFG.MIN_STABILITY_SCORE}')\n axes[0].axvline(np.percentile(importances, 20), color='orange', linestyle='--',\n label='20th percentile')\n axes[0].legend()\n axes[0].grid(True, alpha=0.3)\n plt.colorbar(scatter, ax=axes[0], label='Importance')\n \n axes[1].hist(stabilities, bins=30, color='coral', edgecolor='black')\n axes[1].set_xlabel('Stability Score', fontsize=12)\n axes[1].set_ylabel('Frequency', fontsize=12)\n axes[1].set_title('Feature Stability Distribution', fontsize=14, fontweight='bold')\n axes[1].axvline(CFG.MIN_STABILITY_SCORE, color='red', linestyle='--',\n label=f'Min: {CFG.MIN_STABILITY_SCORE}')\n axes[1].axvline(np.median(stabilities), color='green', linestyle='--',\n label=f'Median: {np.median(stabilities):.2f}')\n axes[1].legend()\n \n plt.tight_layout()\n plt.savefig(save_path, dpi=150, bbox_inches='tight')\n print(f\" Saved stability analysis: {save_path}\")\n plt.close()\n \n def get_summary_stats(self):\n \"\"\"Get summary statistics\"\"\"\n if not self.global_importance:\n self.compute_global_importance()\n \n if not self.feature_stability:\n self.compute_feature_stability()\n \n importances = list(self.global_importance.values())\n stabilities = list(self.feature_stability.values())\n \n return {\n 'n_features': len(importances),\n 'mean_importance': np.mean(importances),\n 'median_importance': np.median(importances),\n 'std_importance': np.std(importances),\n 'p10_importance': np.percentile(importances, 10),\n 'p25_importance': np.percentile(importances, 25),\n 'p75_importance': np.percentile(importances, 75),\n 'p90_importance': np.percentile(importances, 90),\n 'mean_stability': np.mean(stabilities) if stabilities else 0,\n 'median_stability': np.median(stabilities) if stabilities else 0,\n 'min_stability': np.min(stabilities) if stabilities else 0,\n 'max_stability': np.max(stabilities) if stabilities else 0\n }\n\nfeature_tracker = FeatureImportanceTracker()\n\n# ==================== FEATURE SIMILARITY ANALYZER ====================\n\nclass FeatureSimilarityAnalyzer:\n \"\"\"Analyze feature similarity and create groups for dimensionality reduction\"\"\"\n \n def __init__(self, similarity_threshold=0.7, min_group_size=2, max_group_size=5):\n self.similarity_threshold = similarity_threshold\n self.min_group_size = min_group_size\n self.max_group_size = max_group_size\n self.feature_groups = {}\n self.correlation_matrix = None\n self.feature_names = None\n \n def compute_feature_similarity(self, X, feature_names):\n \"\"\"Compute pairwise feature correlations\"\"\"\n X_clean = np.nan_to_num(X, nan=0.0)\n \n self.correlation_matrix = np.corrcoef(X_clean.T)\n self.feature_names = feature_names\n \n corr_df = pd.DataFrame(\n self.correlation_matrix,\n index=feature_names,\n columns=feature_names\n )\n \n return corr_df\n \n def group_similar_features(self, low_importance_features, corr_df):\n \"\"\"Group similar low-importance features with adaptive thresholds\"\"\"\n available_features = set(corr_df.index)\n low_importance_features = [f for f in low_importance_features if f in available_features]\n \n if len(low_importance_features) < self.min_group_size:\n print(f\" Warning: Only {len(low_importance_features)} low-importance features available\")\n return {}\n \n low_imp_corr = corr_df.loc[low_importance_features, low_importance_features]\n \n distance_matrix = 1 - np.abs(low_imp_corr.values)\n \n thresholds_to_try = [self.similarity_threshold, 0.6, 0.5, 0.4]\n \n for threshold in thresholds_to_try:\n try:\n if np.any(np.isnan(distance_matrix)) or np.any(np.isinf(distance_matrix)):\n continue\n \n distance_matrix_sym = (distance_matrix + distance_matrix.T) / 2\n np.fill_diagonal(distance_matrix_sym, 0)\n \n linkage_matrix = linkage(squareform(distance_matrix_sym), method='average')\n \n distance_threshold = 1 - threshold\n cluster_labels = fcluster(linkage_matrix, distance_threshold, criterion='distance')\n \n groups = defaultdict(list)\n for feature, label in zip(low_importance_features, cluster_labels):\n groups[label].append(feature)\n \n valid_groups = {\n f\"group_{i}\": features \n for i, features in groups.items()\n if self.min_group_size <= len(features) <= self.max_group_size\n }\n \n if len(valid_groups) > 0:\n self.feature_groups = valid_groups\n print(f\" Successfully grouped features with threshold {threshold}\")\n return valid_groups\n \n except Exception as e:\n continue\n \n print(f\" Warning: Could not create valid groups\")\n return {}\n \n def plot_feature_groups(self, save_path='feature_groups_dendrogram.png'):\n \"\"\"Plot dendrogram of feature groups\"\"\"\n if not self.feature_groups:\n return\n \n all_grouped_features = []\n for features in self.feature_groups.values():\n all_grouped_features.extend(features)\n \n if len(all_grouped_features) < 2:\n return\n \n try:\n if self.correlation_matrix is not None and self.feature_names is not None:\n feature_to_idx = {f: i for i, f in enumerate(self.feature_names)}\n indices = [feature_to_idx[f] for f in all_grouped_features if f in feature_to_idx]\n \n if len(indices) < 2:\n return\n \n distance_matrix = 1 - np.abs(self.correlation_matrix[np.ix_(indices, indices)])\n distance_matrix = (distance_matrix + distance_matrix.T) / 2\n np.fill_diagonal(distance_matrix, 0)\n \n linkage_matrix = linkage(squareform(distance_matrix), method='average')\n \n plt.figure(figsize=(12, 6))\n dendrogram(linkage_matrix, labels=all_grouped_features, \n leaf_rotation=90, leaf_font_size=8)\n plt.title('Feature Similarity Dendrogram', fontsize=14, fontweight='bold')\n plt.xlabel('Features', fontsize=12)\n plt.ylabel('Distance (1 - |correlation|)', fontsize=12)\n plt.tight_layout()\n plt.savefig(save_path, dpi=150, bbox_inches='tight')\n print(f\" Saved dendrogram: {save_path}\")\n plt.close()\n except Exception as e:\n print(f\" Warning: Dendrogram plotting failed: {e}\")\n\n# ==================== DIMENSIONALITY REDUCTION ====================\n\nclass FeatureReducer:\n \"\"\"Reduce dimensionality of feature groups\"\"\"\n \n def __init__(self, use_pca=True, use_medoid=True):\n self.use_pca = use_pca\n self.use_medoid = use_medoid\n self.reducers = {}\n \n def reduce_feature_group(self, X, feature_group, group_name):\n \"\"\"Reduce a group of features\"\"\"\n X_group = X[feature_group].values\n \n X_group_clean = np.nan_to_num(X_group, nan=0.0)\n \n reduced_features = {}\n \n if self.use_pca and X_group_clean.shape[1] > 1:\n try:\n pca = PCA(n_components=1)\n X_pca = pca.fit_transform(X_group_clean)\n reduced_features[f'{group_name}_pca'] = X_pca.flatten()\n self.reducers[f'{group_name}_pca'] = pca\n except:\n pass\n \n if self.use_medoid and X_group_clean.shape[1] > 1:\n try:\n distances = squareform(pdist(X_group_clean.T, metric='euclidean'))\n \n medoid_idx = np.argmin(distances.sum(axis=1))\n medoid_feature = feature_group[medoid_idx]\n \n reduced_features[f'{group_name}_medoid'] = X_group[:, medoid_idx]\n self.reducers[f'{group_name}_medoid'] = {'type': 'medoid', \n 'feature': medoid_feature,\n 'index': medoid_idx}\n except:\n pass\n \n return reduced_features\n \n def transform_test_data(self, X_test, feature_group, group_name):\n \"\"\"Transform test data using fitted reducers\"\"\"\n X_group = X_test[feature_group].values\n X_group_clean = np.nan_to_num(X_group, nan=0.0)\n \n reduced_features = {}\n \n pca_key = f'{group_name}_pca'\n if pca_key in self.reducers:\n try:\n pca = self.reducers[pca_key]\n X_pca = pca.transform(X_group_clean)\n reduced_features[pca_key] = X_pca.flatten()\n except:\n pass\n \n medoid_key = f'{group_name}_medoid'\n if medoid_key in self.reducers:\n try:\n medoid_info = self.reducers[medoid_key]\n if medoid_info['type'] == 'medoid':\n medoid_idx = medoid_info['index']\n reduced_features[medoid_key] = X_group[:, medoid_idx]\n except:\n pass\n \n return reduced_features\n\n# ==================== TEMPORAL WINDOW ANALYZER ====================\n\nclass TemporalWindowAnalyzer:\n \"\"\"Analyze optimal temporal windows using elbow method\"\"\"\n \n def __init__(self, windows=[3, 5, 10, 15, 30, 45, 60, 90, 120, 150, 180]):\n self.windows = windows\n self.performance_by_window = defaultdict(list)\n \n def evaluate_window_performance(self, series, window, metric='variance'):\n \"\"\"Evaluate how well a window captures temporal patterns\"\"\"\n if len(series.dropna()) < window * 2:\n return np.nan\n \n rolling_stats = series.rolling(window=window, min_periods=max(1, window//2))\n \n if metric == 'variance':\n rolling_mean = rolling_stats.mean()\n score = rolling_mean.var() / (series.var() + 1e-10)\n elif metric == 'stability':\n rolling_std = rolling_stats.std()\n score = 1.0 / (rolling_std.mean() + 1e-10)\n else:\n score = rolling_stats.mean().std()\n \n return score\n \n def analyze_windows(self, X, feature_cols):\n \"\"\"Analyze multiple features across different windows\"\"\"\n window_scores = defaultdict(list)\n \n sample_features = feature_cols[:min(10, len(feature_cols))]\n \n for window in self.windows:\n for feature in sample_features:\n if feature in X.columns:\n series = X[feature]\n score = self.evaluate_window_performance(series, window)\n if not np.isnan(score):\n window_scores[window].append(score)\n \n avg_scores = {\n window: np.mean(scores) if scores else 0\n for window, scores in window_scores.items()\n }\n \n return avg_scores\n \n def find_optimal_windows(self, window_scores, n_windows=5):\n \"\"\"Find optimal windows using elbow detection\"\"\"\n windows = sorted(window_scores.keys())\n scores = [window_scores[w] for w in windows]\n \n if len(scores) < 3:\n return windows[:n_windows]\n \n # Calculate second derivative (curvature)\n first_deriv = np.diff(scores)\n second_deriv = np.diff(first_deriv)\n \n # Find elbow points (where second derivative changes sign or is closest to 0)\n optimal_indices = []\n \n # Always include shortest window\n optimal_indices.append(0)\n \n # Find points with high curvature change\n if len(second_deriv) > 0:\n abs_second_deriv = np.abs(second_deriv)\n # Normalize\n abs_second_deriv = abs_second_deriv / (abs_second_deriv.max() + 1e-10)\n \n # Find peaks in curvature\n for i in range(1, len(abs_second_deriv) - 1):\n if abs_second_deriv[i] > 0.5: # Significant curvature\n optimal_indices.append(i + 1) # Adjust for derivative offset\n \n # Always include longest window if it performs well\n if scores[-1] > np.mean(scores):\n optimal_indices.append(len(windows) - 1)\n \n # Remove duplicates and sort\n optimal_indices = sorted(list(set(optimal_indices)))\n \n # Select n_windows evenly spaced from optimal\n if len(optimal_indices) > n_windows:\n step = len(optimal_indices) / n_windows\n optimal_indices = [optimal_indices[int(i * step)] for i in range(n_windows)]\n \n optimal_windows = [windows[i] for i in optimal_indices if i < len(windows)]\n \n return optimal_windows\n \n def plot_elbow(self, window_scores, optimal_windows=None, save_path='temporal_window_elbow.png'):\n \"\"\"Plot elbow curve for temporal windows\"\"\"\n windows = sorted(window_scores.keys())\n scores = [window_scores[w] for w in windows]\n \n fig, axes = plt.subplots(1, 2, figsize=(14, 5))\n \n axes[0].plot(windows, scores, 'o-', linewidth=2, markersize=8, color='steelblue', label='Performance')\n \n if optimal_windows:\n optimal_scores = [window_scores[w] for w in optimal_windows if w in window_scores]\n axes[0].scatter(optimal_windows, optimal_scores, color='red', s=100, zorder=5, label='Optimal Windows')\n \n axes[0].set_xlabel('Window Size (frames)', fontsize=12)\n axes[0].set_ylabel('Performance Score', fontsize=12)\n axes[0].set_title('Temporal Window Performance', fontsize=14, fontweight='bold')\n axes[0].legend()\n axes[0].grid(True, alpha=0.3)\n \n if len(scores) > 2:\n first_deriv = np.diff(scores)\n second_deriv = np.diff(first_deriv)\n \n axes[1].plot(windows[1:-1], second_deriv, 'o-', linewidth=2, \n markersize=8, color='green', label='Curvature')\n axes[1].set_xlabel('Window Size (frames)', fontsize=12)\n axes[1].set_ylabel('Second Derivative', fontsize=12)\n axes[1].set_title('Rate of Change (Elbow Detection)', fontsize=14, fontweight='bold')\n axes[1].grid(True, alpha=0.3)\n axes[1].axhline(0, color='red', linestyle='--', alpha=0.5)\n axes[1].legend()\n \n plt.tight_layout()\n plt.savefig(save_path, dpi=150, bbox_inches='tight')\n print(f\" Saved elbow plot: {save_path}\")\n plt.close()\n \n return window_scores\n\n# ==================== TEMPORAL FEATURE COMPRESSION ====================\n\nclass TemporalFeatureCompressor:\n \"\"\"Compress temporal features using PCA to reduce dimensionality\"\"\"\n \n def __init__(self, variance_threshold=0.95):\n self.variance_threshold = variance_threshold\n self.pca_models = {}\n self.temporal_feature_groups = {}\n \n def identify_temporal_features(self, feature_names):\n \"\"\"Group features by their base feature\"\"\"\n temporal_indicators = ['_m', '_s', '_ema', '_lag']\n \n # Group temporal features by base\n groups = defaultdict(list)\n base_features = []\n \n for fname in feature_names:\n is_temporal = any(ind in fname for ind in temporal_indicators)\n \n if is_temporal:\n # Extract base feature name\n base = fname\n for ind in temporal_indicators:\n if ind in fname:\n base = fname.split(ind)[0]\n break\n groups[base].append(fname)\n else:\n base_features.append(fname)\n \n # Only keep groups with multiple features\n self.temporal_feature_groups = {\n base: features for base, features in groups.items()\n if len(features) > 2 # At least 3 temporal variations\n }\n \n return self.temporal_feature_groups, base_features\n \n def fit_compress(self, X):\n \"\"\"Fit PCA models and compress temporal features\"\"\"\n feature_names = X.columns.tolist()\n temporal_groups, base_features = self.identify_temporal_features(feature_names)\n \n if len(temporal_groups) == 0:\n print(\" No temporal feature groups found for compression\")\n return X, {}\n \n print(f\" Found {len(temporal_groups)} temporal feature groups for compression\")\n \n X_compressed = X[base_features].copy()\n \n for base, temp_features in temporal_groups.items():\n X_temp = X[temp_features].values\n X_temp_clean = np.nan_to_num(X_temp, nan=0.0)\n \n # Determine number of components\n n_components = min(\n max(1, int(len(temp_features) * self.variance_threshold)),\n len(temp_features) - 1\n )\n \n if n_components >= len(temp_features):\n # Just keep all features\n for feat in temp_features:\n X_compressed[feat] = X[feat]\n continue\n \n try:\n pca = PCA(n_components=n_components)\n X_pca = pca.fit_transform(X_temp_clean)\n \n # Store PCA model\n self.pca_models[base] = pca\n \n # Add compressed features\n for i in range(n_components):\n X_compressed[f'{base}_temporal_pc{i+1}'] = X_pca[:, i]\n \n explained_var = pca.explained_variance_ratio_.sum()\n print(f\" Compressed {base}: {len(temp_features)} → {n_components} features ({explained_var:.2%} variance)\")\n \n except:\n # If PCA fails, just keep original\n for feat in temp_features:\n X_compressed[feat] = X[feat]\n \n return X_compressed, self.pca_models\n \n def transform(self, X_test):\n \"\"\"Transform test data using fitted PCA models\"\"\"\n feature_names = X_test.columns.tolist()\n temporal_groups, base_features = self.identify_temporal_features(feature_names)\n \n X_compressed = X_test[base_features].copy()\n \n for base, temp_features in temporal_groups.items():\n if base in self.pca_models:\n X_temp = X_test[temp_features].values\n X_temp_clean = np.nan_to_num(X_temp, nan=0.0)\n \n try:\n pca = self.pca_models[base]\n X_pca = pca.transform(X_temp_clean)\n \n for i in range(X_pca.shape[1]):\n X_compressed[f'{base}_temporal_pc{i+1}'] = X_pca[:, i]\n except:\n # If transform fails, keep original\n for feat in temp_features:\n if feat in X_test.columns:\n X_compressed[feat] = X_test[feat]\n else:\n # No PCA model, keep original\n for feat in temp_features:\n if feat in X_test.columns:\n X_compressed[feat] = X_test[feat]\n \n return X_compressed\n\n# ==================== HELPER FUNCTIONS ====================\n\ndef compute_ema(series, span):\n \"\"\"Compute exponential moving average\"\"\"\n return series.ewm(span=span, adjust=False).mean()\n\ndef compute_centroid_features(mouse_data, available_body_parts):\n \"\"\"Compute centroid-based geometric features\"\"\"\n x_coords = []\n y_coords = []\n \n for bp in available_body_parts:\n if bp in mouse_data.columns.get_level_values(0):\n x_coords.append(mouse_data[bp]['x'])\n y_coords.append(mouse_data[bp]['y'])\n \n if len(x_coords) == 0:\n return None, None, None, None, None\n \n x_stack = pd.concat(x_coords, axis=1)\n y_stack = pd.concat(y_coords, axis=1)\n \n centroid_x = x_stack.mean(axis=1, skipna=True)\n centroid_y = y_stack.mean(axis=1, skipna=True)\n \n distances = []\n for i in range(len(x_coords)):\n dist = np.sqrt((x_coords[i] - centroid_x)**2 + (y_coords[i] - centroid_y)**2)\n distances.append(dist)\n \n if len(distances) == 0:\n return centroid_x, centroid_y, None, None, None\n \n dist_stack = pd.concat(distances, axis=1)\n \n max_radius = dist_stack.max(axis=1, skipna=True)\n min_radius = dist_stack.min(axis=1, skipna=True)\n avg_radius = dist_stack.mean(axis=1, skipna=True)\n \n return centroid_x, centroid_y, max_radius, min_radius, avg_radius\n\ndef add_smart_temporal_features(X, base_feature_name, series, important_features=None):\n \"\"\"\n Add temporal features smartly based on feature importance\n - All features get short windows\n - Important features get medium and long windows\n - Use EMA for memory-efficient long-term trends\n \"\"\"\n is_important = (important_features is not None and base_feature_name in important_features)\n is_very_important = False\n \n if important_features is not None and len(important_features) > 0:\n # Top 20% are \"very important\"\n top_20_pct = int(len(important_features) * 0.2)\n is_very_important = (important_features.index(base_feature_name) < top_20_pct if base_feature_name in important_features else False)\n \n # SHORT WINDOWS: All features\n for w in CFG.SHORT_WINDOWS:\n if CFG.COMPUTE_MEAN:\n X[f'{base_feature_name}_m{w}'] = series.rolling(w, min_periods=1).mean()\n if CFG.COMPUTE_STD:\n X[f'{base_feature_name}_s{w}'] = series.rolling(w, min_periods=1).std()\n \n # MEDIUM WINDOWS: Important features only\n if is_important or is_very_important:\n for w in CFG.MEDIUM_WINDOWS:\n if CFG.COMPUTE_MEAN:\n X[f'{base_feature_name}_m{w}'] = series.rolling(w, min_periods=max(1, w//4)).mean()\n \n # LONG WINDOWS: Very important features only\n if is_very_important:\n for w in CFG.LONG_WINDOWS:\n if CFG.COMPUTE_MEAN:\n X[f'{base_feature_name}_m{w}'] = series.rolling(w, min_periods=max(1, w//4)).mean()\n \n # EMA: Memory-efficient long-term trends (all important features)\n if CFG.USE_EMA_FEATURES and (is_important or is_very_important):\n for span in CFG.EMA_SPANS:\n X[f'{base_feature_name}_ema{span}'] = compute_ema(series, span)\n \n # LAGS: For very important features\n if CFG.USE_LAGGED_FEATURES and is_very_important:\n for lag in CFG.LAG_WINDOWS:\n X[f'{base_feature_name}_lag{lag}'] = series.shift(lag)\n\n# ==================== HELPER CLASSES ====================\n\nclass SmartChunker:\n \"\"\"Creates diverse training chunks\"\"\"\n \n def __init__(self, total_rows, sample_percent, use_mixing=True, overlap=0.3, n_mixed=2):\n self.total_rows = total_rows\n self.sample_percent = sample_percent\n self.use_mixing = use_mixing\n self.overlap = overlap\n self.n_mixed = n_mixed\n \n def create_chunks(self):\n \"\"\"Generate indices for different data chunks\"\"\"\n chunks = []\n sample_size = int(self.total_rows * self.sample_percent)\n \n if not self.use_mixing:\n indices = np.random.choice(self.total_rows, sample_size, replace=False)\n chunks.append({\n 'name': 'Main',\n 'indices': sorted(indices),\n 'description': f'{len(indices):,} rows'\n })\n return chunks\n \n overlap_size = int(sample_size * self.overlap)\n \n chunk1_end = sample_size\n chunk1_indices = np.arange(0, min(chunk1_end, self.total_rows))\n chunks.append({\n 'name': 'Chunk1',\n 'indices': chunk1_indices,\n 'description': f'First portion ({len(chunk1_indices):,} rows)'\n })\n \n chunk2_start = sample_size - overlap_size\n if self.total_rows - chunk2_start >= sample_size * 0.5:\n chunk2_indices = np.arange(max(chunk2_start, 0), self.total_rows)\n if len(chunk2_indices) > sample_size:\n chunk2_indices = np.random.choice(chunk2_indices, sample_size, replace=False)\n chunk2_indices = sorted(chunk2_indices)\n chunks.append({\n 'name': 'Chunk2',\n 'indices': chunk2_indices,\n 'description': f'Last portion ({len(chunk2_indices):,} rows)'\n })\n \n if self.n_mixed > 0 and len(chunks) >= 2:\n for i in range(self.n_mixed):\n ratio1 = np.random.uniform(0.3, 0.7)\n mix_size = int(sample_size * ratio1)\n \n sample1 = np.random.choice(chunks[0]['indices'], \n size=min(mix_size, len(chunks[0]['indices'])), \n replace=False)\n sample2 = np.random.choice(chunks[1]['indices'], \n size=min(sample_size - len(sample1), len(chunks[1]['indices'])), \n replace=False)\n \n mixed_indices = np.concatenate([sample1, sample2])\n mixed_indices = sorted(np.unique(mixed_indices))\n \n chunks.append({\n 'name': f'Mixed{i+1}',\n 'indices': mixed_indices,\n 'description': f'Mixed ({len(mixed_indices):,} rows)'\n })\n \n return chunks\n\nclass StratifiedSubsetClassifier(ClassifierMixin, BaseEstimator):\n \"\"\"Classifier with stratified sampling and class weights\"\"\"\n \n def __init__(self, estimator, indices=None, use_class_weights=True):\n self.estimator = estimator\n self.indices = indices\n self.use_class_weights = use_class_weights\n\n def fit(self, X, y):\n X = np.array(X, copy=False)\n y = np.array(y, copy=False)\n \n if self.indices is not None:\n valid_indices = self.indices[self.indices < len(X)]\n if len(valid_indices) == 0:\n valid_indices = np.arange(len(X))\n X_subset = X[valid_indices]\n y_subset = y[valid_indices]\n else:\n X_subset = X\n y_subset = y\n \n sample_weights = None\n if self.use_class_weights:\n classes = np.unique(y_subset)\n if len(classes) > 1:\n class_weights = compute_class_weight('balanced', classes=classes, y=y_subset)\n weight_dict = {classes[i]: class_weights[i] for i in range(len(classes))}\n sample_weights = np.array([weight_dict[yi] for yi in y_subset])\n \n if sample_weights is not None:\n self.estimator.fit(X_subset, y_subset, sample_weight=sample_weights)\n else:\n self.estimator.fit(X_subset, y_subset)\n \n self.classes_ = self.estimator.classes_\n return self\n\n def predict_proba(self, X):\n X = np.array(X, copy=False)\n if len(self.classes_) == 1:\n return np.full((len(X), 1), 1.0)\n return self.estimator.predict_proba(X)\n\n# ==================== DATA LOADING ====================\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"MABe MOUSE BEHAVIOR - MEMORY-EFFICIENT TEMPORAL FEATURES\")\nprint(\"=\"*80)\nprint(\"\\nLoading data...\")\ntrain = pd.read_csv('/kaggle/input/MABe-mouse-behavior-detection/train.csv')\ntest = pd.read_csv('/kaggle/input/MABe-mouse-behavior-detection/test.csv')\nbody_parts_tracked_list = list(np.unique(train.body_parts_tracked))\n\nprint(f\"Loaded: {len(train)} train videos, {len(test)} test videos\")\n\ndrop_body_parts = ['headpiece_bottombackleft', 'headpiece_bottombackright', \n 'headpiece_bottomfrontleft', 'headpiece_bottomfrontright',\n 'headpiece_topbackleft', 'headpiece_topbackright', \n 'headpiece_topfrontleft', 'headpiece_topfrontright',\n 'spine_1', 'spine_2', 'tail_middle_1', 'tail_middle_2', 'tail_midpoint']\n\n# ==================== DATA GENERATION ====================\n\ndef generate_mouse_data(dataset, traintest, traintest_directory=None, \n generate_single=True, generate_pair=True):\n \"\"\"Generate mouse feature data\"\"\"\n if traintest_directory is None:\n traintest_directory = f\"/kaggle/input/MABe-mouse-behavior-detection/{traintest}_tracking\"\n \n for _, row in dataset.iterrows():\n lab_id = row['lab_id']\n if lab_id.startswith('MABe22'): \n continue\n video_id = row['video_id']\n if type(row['behaviors_labeled']) != str: \n continue\n\n path = f\"{traintest_directory}/{lab_id}/{video_id}.parquet\"\n try:\n vid = pd.read_parquet(path)\n except:\n continue\n \n if len(np.unique(vid.bodypart)) > 5:\n vid = vid.query(\"~ bodypart.isin(@drop_body_parts)\")\n \n pvid = vid.pivot(columns=['mouse_id', 'bodypart'], index='video_frame', values=['x', 'y'])\n del vid\n \n pvid = pvid.reorder_levels([1, 2, 0], axis=1).T.sort_index().T\n pvid /= row['pix_per_cm_approx']\n\n vid_behaviors = json.loads(row['behaviors_labeled'])\n vid_behaviors = sorted(list({b.replace(\"'\", \"\") for b in vid_behaviors}))\n vid_behaviors = [b.split(',') for b in vid_behaviors]\n vid_behaviors = pd.DataFrame(vid_behaviors, columns=['agent', 'target', 'action'])\n \n arena_info = {\n 'width': row['arena_width_cm'],\n 'height': row['arena_height_cm'],\n 'shape': row['arena_shape']\n }\n \n if traintest == 'train':\n try:\n annot = pd.read_parquet(path.replace('train_tracking', 'train_annotation'))\n except:\n continue\n\n if generate_single:\n vid_behaviors_subset = vid_behaviors.query(\"target == 'self'\")\n for mouse_id_str in np.unique(vid_behaviors_subset.agent):\n try:\n mouse_id = int(mouse_id_str[-1])\n vid_agent_actions = np.unique(vid_behaviors_subset.query(\"agent == @mouse_id_str\").action)\n single_mouse = pvid.loc[:, mouse_id]\n \n single_mouse_meta = pd.DataFrame({\n 'video_id': video_id,\n 'agent_id': mouse_id_str,\n 'target_id': 'self',\n 'video_frame': single_mouse.index\n })\n \n if traintest == 'train':\n single_mouse_label = pd.DataFrame(0.0, columns=vid_agent_actions, \n index=single_mouse.index)\n annot_subset = annot.query(\"(agent_id == @mouse_id) & (target_id == @mouse_id)\")\n for i in range(len(annot_subset)):\n annot_row = annot_subset.iloc[i]\n single_mouse_label.loc[annot_row['start_frame']:annot_row['stop_frame'], \n annot_row.action] = 1.0\n yield 'single', single_mouse, single_mouse_meta, single_mouse_label, arena_info\n else:\n yield 'single', single_mouse, single_mouse_meta, vid_agent_actions, arena_info\n except:\n pass\n\n if generate_pair:\n vid_behaviors_subset = vid_behaviors.query(\"target != 'self'\")\n if len(vid_behaviors_subset) > 0:\n for agent, target in itertools.permutations(\n np.unique(pvid.columns.get_level_values('mouse_id')), 2):\n agent_str = f\"mouse{agent}\"\n target_str = f\"mouse{target}\"\n vid_agent_actions = np.unique(\n vid_behaviors_subset.query(\n \"(agent == @agent_str) & (target == @target_str)\"\n ).action\n )\n \n mouse_pair = pd.concat([pvid[agent], pvid[target]], axis=1, keys=['A', 'B'])\n mouse_pair_meta = pd.DataFrame({\n 'video_id': video_id,\n 'agent_id': agent_str,\n 'target_id': target_str,\n 'video_frame': mouse_pair.index\n })\n \n if traintest == 'train':\n mouse_pair_label = pd.DataFrame(0.0, columns=vid_agent_actions, \n index=mouse_pair.index)\n annot_subset = annot.query(\"(agent_id == @agent) & (target_id == @target)\")\n for i in range(len(annot_subset)):\n annot_row = annot_subset.iloc[i]\n mouse_pair_label.loc[annot_row['start_frame']:annot_row['stop_frame'], \n annot_row.action] = 1.0\n yield 'pair', mouse_pair, mouse_pair_meta, mouse_pair_label, arena_info\n else:\n yield 'pair', mouse_pair, mouse_pair_meta, vid_agent_actions, arena_info\n\n# ==================== FEATURE ENGINEERING ====================\n\ndef transform_single(single_mouse, body_parts_tracked, arena_info, important_features=None):\n \"\"\"Transform single mouse data with memory-efficient temporal features\"\"\"\n available_body_parts = single_mouse.columns.get_level_values(0)\n \n X = pd.DataFrame(index=single_mouse.index)\n \n # Base distances between body parts\n for p1, p2 in itertools.combinations(body_parts_tracked, 2):\n col_name = f\"{p1}+{p2}\"\n if p1 in available_body_parts and p2 in available_body_parts:\n X[col_name] = np.square(single_mouse[p1] - single_mouse[p2]).sum(axis=1, skipna=False)\n else:\n X[col_name] = np.nan\n \n # Centroid features\n centroid_x, centroid_y, max_radius, min_radius, avg_radius = compute_centroid_features(\n single_mouse, available_body_parts\n )\n \n if max_radius is not None:\n X['max_radius'] = max_radius\n X['min_radius'] = min_radius\n X['avg_radius'] = avg_radius\n X['radius_range'] = max_radius - min_radius\n X['radius_ratio'] = (max_radius + 0.001) / (min_radius + 0.001)\n \n # Smart temporal features for radius\n add_smart_temporal_features(X, 'max_radius', max_radius, important_features)\n add_smart_temporal_features(X, 'avg_radius', avg_radius, important_features)\n \n X['max_rad_change'] = max_radius.diff().abs()\n X['compactness_change'] = (max_radius - min_radius).diff().abs()\n \n # Body center features\n if 'body_center' in body_parts_tracked and 'body_center' in available_body_parts:\n cx = single_mouse['body_center']['x']\n cy = single_mouse['body_center']['y']\n \n # Smart temporal features for position\n add_smart_temporal_features(X, 'cx', cx, important_features)\n add_smart_temporal_features(X, 'cy', cy, important_features)\n \n # Velocity and acceleration\n dx = cx.diff()\n dy = cy.diff()\n speed = np.sqrt(dx**2 + dy**2)\n \n X['speed'] = speed\n \n # Smart temporal features for speed\n add_smart_temporal_features(X, 'speed', speed, important_features)\n \n acceleration = speed.diff()\n X['acceleration'] = acceleration\n add_smart_temporal_features(X, 'acc', acceleration, important_features)\n \n jerk = acceleration.diff()\n X['jerk'] = jerk\n X['jerk_m10'] = jerk.rolling(10, min_periods=1).mean()\n \n # Angular features\n angle = np.arctan2(dy, dx)\n angular_velocity = angle.diff()\n X['ang_vel'] = angular_velocity\n X['ang_vel_m5'] = angular_velocity.rolling(5, min_periods=1).mean()\n \n # Activity patterns\n X['freezing'] = (speed < 0.5).astype(float)\n X['high_activity'] = (speed > 5.0).astype(float)\n \n # Nose features\n if 'nose' in body_parts_tracked and 'nose' in available_body_parts:\n nose_x = single_mouse['nose']['x']\n nose_y = single_mouse['nose']['y']\n nose_speed = np.sqrt(nose_x.diff()**2 + nose_y.diff()**2)\n \n X['nose_speed'] = nose_speed\n add_smart_temporal_features(X, 'nose_speed', nose_speed, important_features)\n X['nose_accel'] = nose_speed.diff()\n \n # Body shape\n if all(p in body_parts_tracked for p in ['nose', 'tail_base']):\n if all(p in available_body_parts for p in ['nose', 'tail_base']):\n body_length = np.sqrt(np.square(single_mouse['nose'] - single_mouse['tail_base']).sum(axis=1, skipna=False))\n X['body_len'] = body_length\n add_smart_temporal_features(X, 'body_len', body_length, important_features)\n \n dx_body = single_mouse['nose']['x'] - single_mouse['tail_base']['x']\n dy_body = single_mouse['nose']['y'] - single_mouse['tail_base']['y']\n body_angle = np.arctan2(dy_body, dx_body)\n X['body_angle'] = body_angle\n X['body_angle_change'] = body_angle.diff().abs()\n \n # Ear features\n if all(p in body_parts_tracked for p in ['ear_left', 'ear_right']):\n if all(p in available_body_parts for p in ['ear_left', 'ear_right']):\n ear_dist = np.sqrt(np.square(single_mouse['ear_left'] - single_mouse['ear_right']).sum(axis=1, skipna=False))\n X['ear_spread'] = ear_dist\n X['ear_spread_m10'] = ear_dist.rolling(10, min_periods=1).mean()\n \n return X\n\ndef transform_pair(mouse_pair, body_parts_tracked, arena_info, important_features=None):\n \"\"\"Transform mouse pair data with memory-efficient temporal features\"\"\"\n avail_A = mouse_pair['A'].columns.get_level_values(0)\n avail_B = mouse_pair['B'].columns.get_level_values(0)\n \n X = pd.DataFrame(index=mouse_pair.index)\n \n # Base inter-mouse distances\n for p1, p2 in itertools.product(body_parts_tracked, repeat=2):\n col_name = f\"12+{p1}+{p2}\"\n if p1 in avail_A and p2 in avail_B:\n X[col_name] = np.square(mouse_pair['A'][p1] - mouse_pair['B'][p2]).sum(axis=1, skipna=False)\n else:\n X[col_name] = np.nan\n \n # Centroid features for both mice\n centroid_A_x, centroid_A_y, max_rad_A, min_rad_A, avg_rad_A = compute_centroid_features(\n mouse_pair['A'], avail_A\n )\n centroid_B_x, centroid_B_y, max_rad_B, min_rad_B, avg_rad_B = compute_centroid_features(\n mouse_pair['B'], avail_B\n )\n \n if max_rad_A is not None:\n X['max_rad_A'] = max_rad_A\n X['avg_rad_A'] = avg_rad_A\n add_smart_temporal_features(X, 'max_rad_A', max_rad_A, important_features)\n add_smart_temporal_features(X, 'avg_rad_A', avg_rad_A, important_features)\n \n if max_rad_B is not None:\n X['max_rad_B'] = max_rad_B\n X['avg_rad_B'] = avg_rad_B\n add_smart_temporal_features(X, 'max_rad_B', max_rad_B, important_features)\n add_smart_temporal_features(X, 'avg_rad_B', avg_rad_B, important_features)\n \n if max_rad_A is not None and max_rad_B is not None:\n X['rad_diff'] = avg_rad_A - avg_rad_B\n X['rad_ratio'] = (avg_rad_A + 0.001) / (avg_rad_B + 0.001)\n X['both_compact'] = ((max_rad_A - min_rad_A < 3) & (max_rad_B - min_rad_B < 3)).astype(float)\n \n # Centroid distance\n if centroid_A_x is not None and centroid_B_x is not None:\n centroid_dist = np.sqrt((centroid_A_x - centroid_B_x)**2 + (centroid_A_y - centroid_B_y)**2)\n X['centroid_dist'] = centroid_dist\n add_smart_temporal_features(X, 'centroid_dist', centroid_dist, important_features)\n \n # Body center interactions\n if 'body_center' in body_parts_tracked:\n if 'body_center' in avail_A and 'body_center' in avail_B:\n dx_center = mouse_pair['A']['body_center']['x'] - mouse_pair['B']['body_center']['x']\n dy_center = mouse_pair['A']['body_center']['y'] - mouse_pair['B']['body_center']['y']\n cd = np.sqrt(dx_center**2 + dy_center**2)\n \n # Proximity categories\n X['v_cls'] = (cd < 5.0).astype(float)\n X['cls'] = ((cd >= 5.0) & (cd < 15.0)).astype(float)\n X['far'] = (cd >= 30.0).astype(float)\n \n # Smart temporal features for distance\n add_smart_temporal_features(X, 'cd', cd, important_features)\n \n cd_change = cd.diff()\n X['approaching'] = (cd_change < -0.1).astype(float)\n X['retreating'] = (cd_change > 0.1).astype(float)\n \n add_smart_temporal_features(X, 'cd_change', cd_change, important_features)\n \n cd_accel = cd_change.diff()\n X['cd_accel'] = cd_accel\n \n # Individual velocities\n v_A_x = mouse_pair['A']['body_center']['x'].diff()\n v_A_y = mouse_pair['A']['body_center']['y'].diff()\n v_B_x = mouse_pair['B']['body_center']['x'].diff()\n v_B_y = mouse_pair['B']['body_center']['y'].diff()\n \n speed_A = np.sqrt(v_A_x**2 + v_A_y**2)\n speed_B = np.sqrt(v_B_x**2 + v_B_y**2)\n \n X['speed_A'] = speed_A\n X['speed_B'] = speed_B\n \n add_smart_temporal_features(X, 'speed_A', speed_A, important_features)\n add_smart_temporal_features(X, 'speed_B', speed_B, important_features)\n \n X['speed_ratio'] = (speed_A + 0.001) / (speed_B + 0.001)\n X['speed_diff'] = speed_A - speed_B\n \n # Synchronized movement\n dot_product = v_A_x * v_B_x + v_A_y * v_B_y\n X['sync_move'] = dot_product / (speed_A * speed_B + 0.001)\n X['sync_move_m10'] = X['sync_move'].rolling(10, min_periods=1).mean()\n \n # Nose-to-nose\n if 'nose' in body_parts_tracked:\n if 'nose' in avail_A and 'nose' in avail_B:\n nose_dist = np.sqrt(np.square(mouse_pair['A']['nose'] - mouse_pair['B']['nose']).sum(axis=1, skipna=False))\n X['nose_dist'] = nose_dist\n add_smart_temporal_features(X, 'nose_dist', nose_dist, important_features)\n X['nose_close'] = (nose_dist < 3.0).astype(float)\n \n # Relative orientations\n if all(p in body_parts_tracked for p in ['nose', 'tail_base', 'body_center']):\n if all(p in avail_A for p in ['nose', 'tail_base']) and all(p in avail_B for p in ['nose', 'tail_base']):\n angle_A = np.arctan2(mouse_pair['A']['nose']['y'] - mouse_pair['A']['tail_base']['y'],\n mouse_pair['A']['nose']['x'] - mouse_pair['A']['tail_base']['x'])\n angle_B = np.arctan2(mouse_pair['B']['nose']['y'] - mouse_pair['B']['tail_base']['y'],\n mouse_pair['B']['nose']['x'] - mouse_pair['B']['tail_base']['x'])\n \n rel_angle = angle_A - angle_B\n X['rel_orientation'] = np.cos(rel_angle)\n X['perpendicular'] = np.abs(np.sin(rel_angle))\n \n if 'body_center' in avail_A and 'body_center' in avail_B:\n dx_center = mouse_pair['B']['body_center']['x'] - mouse_pair['A']['body_center']['x']\n dy_center = mouse_pair['B']['body_center']['y'] - mouse_pair['A']['body_center']['y']\n angle_to_B = np.arctan2(dy_center, dx_center)\n \n facing_A_to_B = np.cos(angle_A - angle_to_B)\n facing_B_to_A = np.cos(angle_B - angle_to_B + np.pi)\n \n X['A_facing_B'] = (facing_A_to_B > 0.7).astype(float)\n X['B_facing_A'] = (facing_B_to_A > 0.7).astype(float)\n X['mutual_facing'] = ((facing_A_to_B > 0.7) & (facing_B_to_A > 0.7)).astype(float)\n \n return X\n\n# ==================== PREDICTION ====================\n\naction_thresholds = defaultdict(lambda: 0.27)\n\ndef predict_multiclass_adaptive(pred, meta, action_thresholds):\n \"\"\"Adaptive thresholding with temporal smoothing\"\"\"\n pred_smoothed = pred.rolling(window=5, min_periods=1, center=True).mean()\n ama = np.argmax(pred_smoothed, axis=1)\n max_probs = pred_smoothed.max(axis=1)\n \n threshold_mask = np.zeros(len(pred_smoothed), dtype=bool)\n for i, action in enumerate(pred_smoothed.columns):\n action_mask = (ama == i)\n threshold = action_thresholds.get(action, CFG.PREDICTION_THRESHOLD)\n threshold_mask |= (action_mask & (max_probs >= threshold))\n \n ama = np.where(threshold_mask, ama, -1)\n ama = pd.Series(ama, index=meta.video_frame)\n \n changes_mask = (ama != ama.shift(1)).values\n ama_changes = ama[changes_mask]\n meta_changes = meta[changes_mask]\n mask = ama_changes.values >= 0\n mask[-1] = False\n \n submission_part = pd.DataFrame({\n 'video_id': meta_changes['video_id'][mask].values,\n 'agent_id': meta_changes['agent_id'][mask].values,\n 'target_id': meta_changes['target_id'][mask].values,\n 'action': pred.columns[ama_changes[mask].values],\n 'start_frame': ama_changes.index[mask],\n 'stop_frame': ama_changes.index[1:][mask[:-1]]\n })\n \n stop_video_id = meta_changes['video_id'][1:][mask[:-1]].values\n stop_agent_id = meta_changes['agent_id'][1:][mask[:-1]].values\n stop_target_id = meta_changes['target_id'][1:][mask[:-1]].values\n \n for i in range(len(submission_part)):\n video_id = submission_part.video_id.iloc[i]\n if i < len(stop_video_id):\n if (stop_video_id[i] != video_id or stop_agent_id[i] != submission_part.agent_id.iloc[i] or \n stop_target_id[i] != submission_part.target_id.iloc[i]):\n new_stop_frame = meta.query(\"(video_id == @video_id)\").video_frame.max() + 1\n submission_part.iat[i, submission_part.columns.get_loc('stop_frame')] = new_stop_frame\n else:\n new_stop_frame = meta.query(\"(video_id == @video_id)\").video_frame.max() + 1\n submission_part.iat[i, submission_part.columns.get_loc('stop_frame')] = new_stop_frame\n \n duration = submission_part.stop_frame - submission_part.start_frame\n submission_part = submission_part[duration >= CFG.MIN_BOUT_LENGTH].reset_index(drop=True)\n \n return submission_part\n\ndef robustify(submission, dataset):\n \"\"\"Ensure all videos have predictions\"\"\"\n submission = submission[submission.start_frame < submission.stop_frame]\n \n group_list = []\n for _, group in submission.groupby(['video_id', 'agent_id', 'target_id']):\n group = group.sort_values('start_frame')\n mask = np.ones(len(group), dtype=bool)\n last_stop = 0\n for i, (_, row) in enumerate(group.iterrows()):\n if row['start_frame'] < last_stop:\n mask[i] = False\n else:\n last_stop = row['stop_frame']\n group_list.append(group[mask])\n submission = pd.concat(group_list) if group_list else submission\n\n for _, row in dataset.iterrows():\n video_id = row['video_id']\n if (submission.video_id == video_id).any():\n continue\n\n vid_behaviors = eval(row['behaviors_labeled'])\n vid_behaviors = sorted(list({b.replace(\"'\", \"\") for b in vid_behaviors}))\n vid_behaviors = [b.split(',') for b in vid_behaviors]\n vid_behaviors = pd.DataFrame(vid_behaviors, columns=['agent', 'target', 'action'])\n \n for (agent, target), actions in vid_behaviors.groupby(['agent', 'target']):\n for _, action_row in actions.iterrows():\n new_row = pd.DataFrame({\n 'video_id': [video_id],\n 'agent_id': [agent],\n 'target_id': [target],\n 'action': [action_row['action']],\n 'start_frame': [0],\n 'stop_frame': [100]\n })\n submission = pd.concat([submission, new_row], ignore_index=True)\n break\n break\n\n return submission.reset_index(drop=True)\n\n# ==================== ENSEMBLE TRAINING WITH FEATURE REDUCTION ====================\n\ndef train_with_intelligent_reduction(body_parts_tracked_str, switch_tr, X_tr, label, meta):\n \"\"\"Train with intelligent feature reduction considering stability\"\"\"\n \n sample_idx = None\n if len(X_tr) > CFG.MAX_MEMORY_ROWS:\n if CFG.VERBOSE:\n print(f\" Sampling to {CFG.MAX_MEMORY_ROWS:,} rows\")\n sample_idx = np.random.choice(len(X_tr), CFG.MAX_MEMORY_ROWS, replace=False)\n X_tr = X_tr.iloc[sample_idx]\n label = label.iloc[sample_idx]\n \n print(f\" Training shape: {X_tr.shape}\")\n \n os.makedirs('analysis_plots', exist_ok=True)\n \n print(\" Step 1: Training initial models for feature importance...\")\n \n chunker = SmartChunker(\n total_rows=len(X_tr),\n sample_percent=CFG.SAMPLE_PERCENT,\n use_mixing=CFG.USE_CHUNK_MIXING,\n overlap=CFG.CHUNK_OVERLAP,\n n_mixed=CFG.N_MIXED_CHUNKS\n )\n chunks = chunker.create_chunks()\n \n model_configs = CFG.get_preset_models()\n \n actions_trained = 0\n max_actions_for_importance = 5\n \n for action in label.columns:\n if actions_trained >= max_actions_for_importance:\n break\n \n action_mask = ~label[action].isna().values\n y_action = label[action][action_mask].values.astype(int)\n\n if not (y_action == 0).all() and y_action.sum() >= 5:\n for model_config in model_configs:\n chunk = chunks[0]\n \n model = make_pipeline(\n SimpleImputer(),\n StratifiedSubsetClassifier(\n lgb.LGBMClassifier(**model_config['params']),\n indices=chunk['indices'],\n use_class_weights=CFG.USE_CLASS_WEIGHTS\n )\n )\n \n model.fit(X_tr.iloc[action_mask], y_action)\n \n if CFG.TRACK_FEATURE_IMPORTANCE:\n try:\n lgbm_model = model.named_steps['stratifiedsubsetclassifier'].estimator\n feature_importance = lgbm_model.feature_importances_\n feature_names = X_tr.columns.tolist()\n feature_tracker.add_importance(action, feature_names, feature_importance)\n except:\n pass\n \n actions_trained += 1\n gc.collect()\n \n print(\" Step 2: Analyzing feature importance and stability...\")\n \n global_importance = feature_tracker.compute_global_importance()\n feature_stability = feature_tracker.compute_feature_stability()\n \n if len(global_importance) > 0:\n feature_tracker.plot_importance_distribution(\n save_path='analysis_plots/feature_importance_distribution.png'\n )\n \n feature_tracker.plot_stability_analysis(\n save_path='analysis_plots/feature_stability_analysis.png'\n )\n \n stats = feature_tracker.get_summary_stats()\n print(f\" Features: {stats['n_features']}\")\n print(f\" Mean importance: {stats['mean_importance']:.4f}\")\n print(f\" Mean stability: {stats['mean_stability']:.4f}\")\n print(f\" P10 importance: {stats['p10_importance']:.4f}, P90: {stats['p90_importance']:.4f}\")\n \n stable_important_features = feature_tracker.get_stable_important_features(\n n=CFG.TOP_N_FEATURES,\n stability_weight=CFG.STABILITY_WEIGHT\n )\n print(f\" Top stable+important features: {len(stable_important_features)}\")\n \n # Get top base features (excluding temporal aggregations)\n top_base_features = feature_tracker.get_top_base_features(n=20, exclude_temporal=True)\n print(f\" Top base features: {len(top_base_features)}\")\n print(f\" Base features: {', '.join(top_base_features[:10])}\")\n \n available_features = X_tr.columns.tolist()\n unstable_low_importance = feature_tracker.get_unstable_low_importance_features(\n importance_percentile=CFG.LOW_IMPORTANCE_THRESHOLD * 100,\n min_stability=CFG.MIN_STABILITY_SCORE,\n available_features=available_features\n )\n print(f\" Unstable low-importance features: {len(unstable_low_importance)}\")\n \n print(\" Step 3: Grouping similar low-importance unstable features...\")\n \n if len(unstable_low_importance) >= CFG.MIN_GROUP_SIZE:\n similarity_analyzer = FeatureSimilarityAnalyzer(\n similarity_threshold=CFG.FEATURE_SIMILARITY_THRESHOLD,\n min_group_size=CFG.MIN_GROUP_SIZE,\n max_group_size=CFG.MAX_GROUP_SIZE\n )\n \n X_sample = X_tr.sample(n=min(5000, len(X_tr)), random_state=42)\n corr_df = similarity_analyzer.compute_feature_similarity(\n X_sample.values, X_sample.columns.tolist()\n )\n \n feature_groups = similarity_analyzer.group_similar_features(\n unstable_low_importance, corr_df\n )\n \n print(f\" Created {len(feature_groups)} feature groups\")\n \n if len(feature_groups) > 0:\n similarity_analyzer.plot_feature_groups(\n save_path='analysis_plots/feature_similarity_dendrogram.png'\n )\n \n print(\" Step 4: Reducing feature dimensionality...\")\n \n feature_reducer = FeatureReducer(\n use_pca=CFG.USE_PCA_REDUCTION,\n use_medoid=CFG.USE_MEDOID_REDUCTION\n )\n \n X_reduced = X_tr.copy()\n \n features_to_remove = []\n for group_name, feature_group in feature_groups.items():\n reduced_feats = feature_reducer.reduce_feature_group(\n X_tr, feature_group, group_name\n )\n \n for feat_name, feat_values in reduced_feats.items():\n X_reduced[feat_name] = feat_values\n \n features_to_remove.extend(feature_group)\n \n X_reduced = X_reduced.drop(columns=features_to_remove, errors='ignore')\n \n print(f\" Original features: {X_tr.shape[1]}\")\n print(f\" Reduced features: {X_reduced.shape[1]}\")\n print(f\" Reduction: {X_tr.shape[1] - X_reduced.shape[1]} features\")\n \n # Optional: Temporal compression\n if CFG.USE_TEMPORAL_COMPRESSION:\n print(\" Step 4b: Compressing temporal features...\")\n compressor = TemporalFeatureCompressor(variance_threshold=CFG.TEMPORAL_PCA_VARIANCE)\n X_reduced, pca_models = compressor.fit_compress(X_reduced)\n print(f\" After temporal compression: {X_reduced.shape[1]} features\")\n \n return X_reduced, feature_reducer, feature_groups, label, compressor\n \n return X_reduced, feature_reducer, feature_groups, label, None\n \n if CFG.ANALYZE_TEMPORAL_WINDOWS:\n print(\" Step 5: Analyzing temporal windows...\")\n \n temporal_analyzer = TemporalWindowAnalyzer(windows=CFG.TEMPORAL_WINDOWS)\n \n sample_cols = [col for col in X_tr.columns if any(pattern in col for pattern in ['_m', '_s', 'rad', 'speed', 'cd'])]\n sample_cols = sample_cols[:min(10, len(sample_cols))]\n \n if len(sample_cols) > 0:\n window_scores = temporal_analyzer.analyze_windows(X_tr, sample_cols)\n \n # Find optimal windows\n optimal_windows = temporal_analyzer.find_optimal_windows(window_scores, n_windows=5)\n print(f\" Optimal windows: {optimal_windows}\")\n \n temporal_analyzer.plot_elbow(\n window_scores,\n optimal_windows=optimal_windows,\n save_path='analysis_plots/temporal_window_elbow.png'\n )\n \n if len(window_scores) > 0:\n sorted_windows = sorted(window_scores.items(), key=lambda x: x[1], reverse=True)\n print(f\" Top 3 windows by performance: {sorted_windows[:3]}\")\n \n return X_tr, None, {}, label, None\n\ndef submit_ensemble_with_reduction(body_parts_tracked_str, switch_tr, X_tr, label, meta, train_feature_cols):\n \"\"\"Train ensemble with feature reduction and make predictions\"\"\"\n \n reduction_result = train_with_intelligent_reduction(\n body_parts_tracked_str, switch_tr, X_tr, label, meta\n )\n \n X_tr_reduced, feature_reducer, feature_groups, label_reduced, temporal_compressor = reduction_result\n \n train_feature_cols = X_tr_reduced.columns.tolist()\n \n print(f\" Final training with {len(train_feature_cols)} features...\")\n \n # Get important features for test-time feature generation\n important_base_features = feature_tracker.get_top_base_features(n=20, exclude_temporal=True)\n \n chunker = SmartChunker(\n total_rows=len(X_tr_reduced),\n sample_percent=CFG.SAMPLE_PERCENT,\n use_mixing=CFG.USE_CHUNK_MIXING,\n overlap=CFG.CHUNK_OVERLAP,\n n_mixed=CFG.N_MIXED_CHUNKS\n )\n chunks = chunker.create_chunks()\n \n model_configs = CFG.get_preset_models()\n \n models = []\n for i, model_config in enumerate(model_configs):\n chunk_idx = i % len(chunks)\n chunk = chunks[chunk_idx]\n \n model = make_pipeline(\n SimpleImputer(),\n StratifiedSubsetClassifier(\n lgb.LGBMClassifier(**model_config['params']),\n indices=chunk['indices'],\n use_class_weights=CFG.USE_CLASS_WEIGHTS\n )\n )\n models.append({\n 'name': f\"{model_config['name']}_{chunk['name']}\",\n 'model': model\n })\n \n model_list = []\n for action in label_reduced.columns:\n action_mask = ~label_reduced[action].isna().values\n y_action = label_reduced[action][action_mask].values.astype(int)\n\n if not (y_action == 0).all() and y_action.sum() >= 5:\n trained_models = []\n for model_info in models:\n m_clone = clone(model_info['model'])\n m_clone.fit(X_tr_reduced.iloc[action_mask], y_action)\n trained_models.append(m_clone)\n \n model_list.append((action, trained_models))\n gc.collect()\n \n del X_tr_reduced\n gc.collect()\n\n body_parts_tracked = json.loads(body_parts_tracked_str)\n if len(body_parts_tracked) > 5:\n body_parts_tracked = [b for b in body_parts_tracked if b not in drop_body_parts]\n \n test_subset = test[test.body_parts_tracked == body_parts_tracked_str]\n generator = generate_mouse_data(test_subset, 'test',\n generate_single=(switch_tr == 'single'), \n generate_pair=(switch_tr == 'pair'))\n \n if CFG.VERBOSE: \n print(f\" Predicting on {len(test_subset)} test videos...\")\n \n submission_list = []\n \n for switch_te, data_te, meta_te, actions_te, arena_info_te in generator:\n try:\n # Generate test features with important features for smart temporal windowing\n if switch_te == 'single':\n X_te = transform_single(data_te, body_parts_tracked, arena_info_te, important_base_features)\n else:\n X_te = transform_pair(data_te, body_parts_tracked, arena_info_te, important_base_features)\n \n # Apply feature reduction to test data\n if feature_reducer and feature_groups:\n for group_name, feature_group in feature_groups.items():\n available_features = [f for f in feature_group if f in X_te.columns]\n if len(available_features) > 0:\n reduced_feats = feature_reducer.transform_test_data(\n X_te, available_features, group_name\n )\n for feat_name, feat_values in reduced_feats.items():\n X_te[feat_name] = feat_values\n \n features_to_remove = []\n for feature_group in feature_groups.values():\n features_to_remove.extend(feature_group)\n X_te = X_te.drop(columns=features_to_remove, errors='ignore')\n \n # Apply temporal compression if used\n if temporal_compressor:\n X_te = temporal_compressor.transform(X_te)\n \n # Align with training features\n X_te = X_te.reindex(columns=train_feature_cols, fill_value=0)\n \n del data_te\n gc.collect()\n \n pred = pd.DataFrame(index=meta_te.video_frame)\n for action, trained_models in model_list:\n if action in actions_te:\n probs = [m.predict_proba(X_te)[:, 1] for m in trained_models]\n pred[action] = np.mean(probs, axis=0)\n \n del X_te\n gc.collect()\n \n if pred.shape[1] != 0:\n sub_part = predict_multiclass_adaptive(pred, meta_te, action_thresholds)\n submission_list.append(sub_part)\n except Exception as e:\n if CFG.VERBOSE: \n print(f' ERROR: {str(e)[:100]}')\n gc.collect()\n \n return submission_list\n\n# ==================== MAIN TRAINING LOOP ====================\n\nsubmission_list = []\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"STARTING MEMORY-EFFICIENT TRAINING WITH SMART TEMPORAL FEATURES\")\nprint(\"=\"*80)\nprint(f\"Configuration: {CFG.PRESET}\")\nprint(f\"Feature Reduction: Enabled\")\nprint(f\"Max Memory Rows: {CFG.MAX_MEMORY_ROWS:,}\")\nprint(f\"Smart Temporal Windows:\")\nprint(f\" - Short (all features): {CFG.SHORT_WINDOWS}\")\nprint(f\" - Medium (top 50%): {CFG.MEDIUM_WINDOWS}\")\nprint(f\" - Long (top 20%): {CFG.LONG_WINDOWS}\")\nprint(f\" - EMA spans: {CFG.EMA_SPANS}\")\nprint(f\"Temporal Compression: {CFG.USE_TEMPORAL_COMPRESSION}\")\nprint(\"=\"*80)\n\nfor section in range(1, min(3, len(body_parts_tracked_list))):\n body_parts_tracked_str = body_parts_tracked_list[section]\n \n try:\n body_parts_tracked = json.loads(body_parts_tracked_str)\n print(f\"\\n{'='*80}\")\n print(f\"SECTION {section}/{min(2, len(body_parts_tracked_list)-1)}: {len(body_parts_tracked)} body parts\")\n print(f\"{'='*80}\")\n \n if len(body_parts_tracked) > 5:\n body_parts_tracked = [b for b in body_parts_tracked if b not in drop_body_parts]\n \n train_subset = train[train.body_parts_tracked == body_parts_tracked_str]\n \n single_list, single_label_list, single_meta_list = [], [], []\n \n for switch, data, meta, label, arena_info in generate_mouse_data(train_subset, 'train', \n generate_single=True, generate_pair=False):\n single_list.append(data)\n single_meta_list.append(meta)\n single_label_list.append(label)\n \n if sum(len(x) for x in single_list) > CFG.MAX_MEMORY_ROWS:\n break\n \n if len(single_list) > 0:\n single_mouse = pd.concat(single_list)\n single_label = pd.concat(single_label_list)\n single_meta = pd.concat(single_meta_list)\n del single_list, single_label_list, single_meta_list\n gc.collect()\n \n # Initial transform without important features\n X_tr = transform_single(single_mouse, body_parts_tracked, arena_info, important_features=None)\n train_feature_cols = X_tr.columns.tolist()\n del single_mouse\n \n print(f\"\\nSINGLE behaviors\")\n submissions = submit_ensemble_with_reduction(body_parts_tracked_str, 'single', \n X_tr, single_label, single_meta, train_feature_cols)\n submission_list.extend(submissions)\n \n gc.collect()\n \n pair_list, pair_label_list, pair_meta_list = [], [], []\n \n for switch, data, meta, label, arena_info in generate_mouse_data(train_subset, 'train', \n generate_single=False, generate_pair=True):\n pair_list.append(data)\n pair_meta_list.append(meta)\n pair_label_list.append(label)\n \n if sum(len(x) for x in pair_list) > CFG.MAX_MEMORY_ROWS:\n break\n \n if len(pair_list) > 0:\n mouse_pair = pd.concat(pair_list)\n pair_label = pd.concat(pair_label_list)\n pair_meta = pd.concat(pair_meta_list)\n del pair_list, pair_label_list, pair_meta_list\n gc.collect()\n \n # Initial transform without important features\n X_tr = transform_pair(mouse_pair, body_parts_tracked, arena_info, important_features=None)\n train_feature_cols = X_tr.columns.tolist()\n del mouse_pair\n \n print(f\"\\nPAIR behaviors\")\n submissions = submit_ensemble_with_reduction(body_parts_tracked_str, 'pair', \n X_tr, pair_label, pair_meta, train_feature_cols)\n submission_list.extend(submissions)\n \n except Exception as e:\n print(f'\\nERROR in section {section}: {str(e)}')\n import traceback\n traceback.print_exc()\n \n gc.collect()\n\n# ==================== FINAL ANALYSIS ====================\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"FINAL FEATURE IMPORTANCE AND TEMPORAL ANALYSIS\")\nprint(\"=\"*80)\n\nif len(feature_tracker.global_importance) > 0:\n stable_important_features = feature_tracker.get_stable_important_features(\n n=CFG.TOP_N_FEATURES,\n stability_weight=CFG.STABILITY_WEIGHT\n )\n \n print(f\"\\nTop {min(20, len(stable_important_features))} Most Stable & Important Features:\")\n print(\"-\" * 80)\n for i, feature in enumerate(stable_important_features[:20], 1):\n importance = feature_tracker.global_importance[feature]\n stability = feature_tracker.feature_stability.get(feature, 0.5)\n combined = (1 - CFG.STABILITY_WEIGHT) * importance + CFG.STABILITY_WEIGHT * stability\n print(f\"{i:2d}. {feature:35s} | Imp: {importance:8.2f} | Stab: {stability:.3f} | Combined: {combined:.3f}\")\n \n # Get top base features (excluding temporal)\n top_base = feature_tracker.get_top_base_features(n=20, exclude_temporal=True)\n print(f\"\\nTop {min(20, len(top_base))} Base Features (no temporal aggregations):\")\n print(\"-\" * 80)\n for i, feature in enumerate(top_base[:20], 1):\n importance = feature_tracker.global_importance[feature]\n print(f\"{i:2d}. {feature:35s} | Importance: {importance:8.2f}\")\n \n stats = feature_tracker.get_summary_stats()\n print(f\"\\nFeature Statistics:\")\n print(\"-\" * 80)\n print(f\"Total features: {stats['n_features']}\")\n print(f\"Mean importance: {stats['mean_importance']:.4f}\")\n print(f\"Mean stability: {stats['mean_stability']:.4f}\")\n print(f\"Median stability: {stats['median_stability']:.4f}\")\n print(f\"Stability range: [{stats['min_stability']:.3f}, {stats['max_stability']:.3f}]\")\n print(f\"10th percentile importance: {stats['p10_importance']:.4f}\")\n print(f\"90th percentile importance: {stats['p90_importance']:.4f}\")\n \n print(\"\\n\" + \"=\"*80)\n print(\"MEMORY-EFFICIENT TEMPORAL STRATEGY SUMMARY\")\n print(\"=\"*80)\n print(f\"Short windows (all features): {CFG.SHORT_WINDOWS}\")\n print(f\"Medium windows (top 50%): {CFG.MEDIUM_WINDOWS}\")\n print(f\"Long windows (top 20%): {CFG.LONG_WINDOWS}\")\n print(f\"EMA for long-term trends: {CFG.EMA_SPANS}\")\n print(f\"Temporal compression: {CFG.USE_TEMPORAL_COMPRESSION}\")\n print(\"\\nThis strategy captures 180+ frame dependencies while reducing\")\n print(\"temporal feature count by ~60-70% compared to naive approach.\")\n\n# ==================== FINALIZE SUBMISSION ====================\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"FINALIZING SUBMISSION\")\nprint(\"=\"*80)\n\nif len(submission_list) > 0:\n submission = pd.concat(submission_list)\nelse:\n submission = pd.DataFrame({\n 'video_id': [438887472],\n 'agent_id': ['mouse1'],\n 'target_id': ['self'],\n 'action': ['rear'],\n 'start_frame': [278],\n 'stop_frame': [500]\n })\n\nsubmission_robust = robustify(submission, test)\nsubmission_robust.index.name = 'row_id'\n\noutput_filename = 'submission.csv'\nsubmission_robust.to_csv(output_filename)\n\nprint(f\"\\nSUCCESS!\")\nprint(f\"Output file: {output_filename}\")\nprint(f\"Total predictions: {len(submission_robust):,}\")\nprint(f\"Analysis plots saved to: analysis_plots/\")\nprint(\" - feature_importance_distribution.png\")\nprint(\" - feature_stability_analysis.png\")\nprint(\" - feature_similarity_dendrogram.png (if applicable)\")\nprint(\" - temporal_window_elbow.png\")\nprint(\"=\"*80)","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}