{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n\nimport numpy as np\nimport pandas as pd\nimport os\nimport signal\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import GroupKFold\nfrom lightgbm import LGBMClassifier, early_stopping, log_evaluation\nfrom tqdm import tqdm\nimport joblib\nfrom scipy import stats\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Constants for default values and column names\nDEFAULT_VALUE = -1\nACC_COLS = ['acc_x', 'acc_y', 'acc_z']\nROT_COLS = ['rot_w', 'rot_x', 'rot_y', 'rot_z']\nMAX_RUNTIME = 60 * 60  # 1 hour timeout\n\nclass TimeoutException(Exception):\n    \"\"\"Custom exception for timeout handling\"\"\"\n    pass\n\ndef timeout_handler(signum, frame):\n    \"\"\"Handler function for alarm signal\"\"\"\n    raise TimeoutException(\"Function timed out\")\n\ndef set_timeout(seconds):\n    \"\"\"Decorator to set timeout for functions\"\"\"\n    def decorator(func):\n        def wrapper(*args, **kwargs):\n            signal.signal(signal.SIGALRM, timeout_handler)\n            signal.alarm(seconds)\n            try:\n                result = func(*args, **kwargs)\n            finally:\n                signal.alarm(0)\n            return result\n        return wrapper\n    return decorator\n\ndef calculate_motion_features(df):\n    \"\"\"Enhanced motion feature calculation with robust error handling\"\"\"\n    features = {\n        'acc_mean': DEFAULT_VALUE,\n        'acc_std': DEFAULT_VALUE,\n        'acc_max': DEFAULT_VALUE,\n        'acc_min': DEFAULT_VALUE,\n        'acc_range': DEFAULT_VALUE,\n        'acc_rms': DEFAULT_VALUE,\n        'acc_correlation_xy': DEFAULT_VALUE,\n        'acc_correlation_xz': DEFAULT_VALUE,\n        'acc_correlation_yz': DEFAULT_VALUE,\n        'quat_change_rate': DEFAULT_VALUE,\n        'quat_entropy': DEFAULT_VALUE\n    }\n    \n    if len(df) == 0:\n        return features\n    \n    try:\n        # Check for required columns\n        available_cols = set(df.columns)\n        missing_acc = set(ACC_COLS) - available_cols\n        if missing_acc:\n            print(f\"Warning: Missing acceleration columns: {missing_acc}\")\n            return features\n        \n        # Acceleration features\n        acc_data = df[ACC_COLS].values\n        if len(acc_data) > 0:\n            features.update({\n                'acc_mean': np.nanmean(acc_data),\n                'acc_std': np.nanstd(acc_data),\n                'acc_max': np.nanmax(acc_data),\n                'acc_min': np.nanmin(acc_data),\n                'acc_range': np.ptp(acc_data) if len(acc_data) > 0 else DEFAULT_VALUE,\n                'acc_rms': np.sqrt(np.nanmean(acc_data**2))\n            })\n        \n        # Correlation features (need at least 2 samples)\n        if len(acc_data) > 1:\n            with warnings.catch_warnings():\n                warnings.simplefilter(\"ignore\", category=RuntimeWarning)\n                corr_matrix = np.corrcoef(acc_data, rowvar=False)\n                if not np.isnan(corr_matrix).any():\n                    features.update({\n                        'acc_correlation_xy': corr_matrix[0, 1],\n                        'acc_correlation_xz': corr_matrix[0, 2],\n                        'acc_correlation_yz': corr_matrix[1, 2]\n                    })\n        \n        # Quaternion features\n        if all(col in available_cols for col in ROT_COLS):\n            quat_data = df[ROT_COLS].values\n            if len(quat_data) > 1:\n                features.update({\n                    'quat_change_rate': np.nanmean(np.abs(np.diff(quat_data, axis=0))),\n                    'quat_entropy': stats.entropy(np.abs(quat_data).flatten()) if len(quat_data) > 0 else DEFAULT_VALUE\n                })\n            \n    except Exception as e:\n        print(f\"Error calculating motion features: {str(e)[:200]}\")\n    \n    return features\n\ndef calculate_thermal_features(df):\n    \"\"\"Enhanced thermal feature calculation with robust error handling\"\"\"\n    features = {}\n    thm_cols = sorted([c for c in df.columns if c.startswith('thm_')])\n    \n    if not thm_cols:\n        return features\n    \n    try:\n        thm_data = df[thm_cols].values\n        valid_thm = thm_data[~np.isnan(thm_data) & (thm_data != DEFAULT_VALUE)]\n        \n        if len(valid_thm) > 0:\n            features.update({\n                'thm_mean': np.nanmean(valid_thm),\n                'thm_std': np.nanstd(valid_thm),\n                'thm_gradient': np.nanmean(np.abs(np.diff(thm_data, axis=1))) if len(thm_data) > 1 else DEFAULT_VALUE,\n                'thm_max_diff': np.nanmax(thm_data) - np.nanmin(thm_data) if len(thm_data) > 0 else DEFAULT_VALUE\n            })\n            \n            for col in thm_cols:\n                sensor_data = df[col].values\n                valid_data = sensor_data[~np.isnan(sensor_data) & (sensor_data != DEFAULT_VALUE)]\n                if len(valid_data) > 0:\n                    features.update({\n                        f'{col}_mean': np.nanmean(valid_data),\n                        f'{col}_std': np.nanstd(valid_data),\n                        f'{col}_trend': np.polyfit(range(len(valid_data)), valid_data, 1)[0] if len(valid_data) > 1 else DEFAULT_VALUE\n                    })\n                    \n    except Exception as e:\n        print(f\"Error calculating thermal features: {str(e)[:200]}\")\n    \n    return features\n\ndef calculate_tof_features(df):\n    \"\"\"Enhanced ToF feature calculation with robust error handling\"\"\"\n    features = {}\n    tof_cols = sorted([c for c in df.columns if c.startswith('tof_')])\n    \n    if not tof_cols or len(df) == 0:\n        return features\n    \n    try:\n        # Initialize array with default values\n        tof_data = np.full((len(df), 5, 64), DEFAULT_VALUE, dtype=np.float32)\n        \n        for col in tof_cols:\n            try:\n                parts = col.split('_')\n                if len(parts) != 3:\n                    continue\n                    \n                sensor_num = int(parts[1]) - 1  # Convert to 0-based index\n                pixel_num = int(parts[2][1:])   # Extract pixel number\n                \n                if 0 <= sensor_num < 5 and 0 <= pixel_num < 64:\n                    tof_data[:, sensor_num, pixel_num] = df[col].values\n            except (ValueError, IndexError):\n                continue\n        \n        for i in range(5):\n            sensor_data = tof_data[:, i, :]\n            valid_data = sensor_data[~np.isnan(sensor_data) & (sensor_data != DEFAULT_VALUE)]\n            \n            if len(valid_data) > 0:\n                try:\n                    grid_data = sensor_data.reshape(-1, 8, 8)\n                    valid_grid = grid_data[~np.isnan(grid_data) & (grid_data != DEFAULT_VALUE)]\n                    \n                    features.update({\n                        f'tof_{i+1}_mean': np.nanmean(valid_data),\n                        f'tof_{i+1}_std': np.nanstd(valid_data),\n                        f'tof_{i+1}_valid_ratio': np.mean(~np.isnan(sensor_data) & (sensor_data != DEFAULT_VALUE)),\n                        f'tof_{i+1}_spatial_var': np.nanvar(valid_grid) if len(valid_grid) > 0 else DEFAULT_VALUE\n                    })\n                except Exception:\n                    continue\n                \n    except Exception as e:\n        print(f\"Error calculating ToF features: {str(e)[:200]}\")\n    \n    return features\n\n@set_timeout(300)  # 5 minutes timeout per sequence\ndef extract_sequence_features(sequence_df):\n    \"\"\"Comprehensive feature extraction with timeout protection\"\"\"\n    features = {}\n    \n    try:\n        # Handle test data which may not have 'behavior' column\n        if 'behavior' in sequence_df.columns:\n            phases = {\n                'transition': sequence_df[sequence_df['behavior'] == 'Transition'].copy(),\n                'pause': sequence_df[sequence_df['behavior'] == 'Pause'].copy(),\n                'gesture': sequence_df[sequence_df['behavior'] == 'Gesture'].copy()\n            }\n        else:\n            phases = {'gesture': sequence_df.copy()}\n        \n        for phase_name, phase_df in phases.items():\n            features[f'{phase_name}_duration'] = len(phase_df)\n            motion_features = calculate_motion_features(phase_df)\n            features.update({f'{phase_name}_{k}': v for k,v in motion_features.items()})\n        \n        if len(phases.get('gesture', pd.DataFrame())) > 0:\n            thermal_features = calculate_thermal_features(phases['gesture'])\n            features.update(thermal_features)\n            \n            tof_features = calculate_tof_features(phases['gesture'])\n            features.update(tof_features)\n        \n        if hasattr(extract_sequence_features, 'demo_df') and 'subject' in sequence_df.columns:\n            subject = sequence_df['subject'].iloc[0]\n            if subject in extract_sequence_features.demo_df.index:\n                subj_features = extract_sequence_features.demo_df.loc[subject]\n                features.update({\n                    'age': subj_features.get('age', DEFAULT_VALUE),\n                    'height': subj_features.get('height_cm', DEFAULT_VALUE),\n                    'arm_length': subj_features.get('shoulder_to_wrist_cm', DEFAULT_VALUE)\n                })\n            \n    except TimeoutException:\n        print(\"Feature extraction timed out for sequence\")\n    except Exception as e:\n        print(f\"Error extracting sequence features: {str(e)[:200]}\")\n    \n    return features\n\ndef load_data(train_path, demo_path, test_path):\n    \"\"\"Load and preprocess data with robust error handling\"\"\"\n    try:\n        print(f\"Attempting to load data from:\\n- {train_path}\\n- {demo_path}\\n- {test_path}\")\n        \n        if not all(os.path.exists(p) for p in [train_path, demo_path, test_path]):\n            raise FileNotFoundError(\"One or more data files not found\")\n        \n        train_df = pd.read_csv(train_path)\n        demo_df = pd.read_csv(demo_path).set_index('subject')\n        test_df = pd.read_csv(test_path)\n        \n        print(\"Data loaded successfully\")\n        print(f\"Train shape: {train_df.shape}, Test shape: {test_df.shape}\")\n        print(f\"Train columns: {train_df.columns.tolist()}\")\n        \n        return train_df, demo_df, test_df\n    except Exception as e:\n        print(f\"Error loading data: {str(e)[:500]}\")\n        raise\n\ndef train_model(X_train, y_train, groups, n_splits=5):\n    \"\"\"Train LGBM model with GroupKFold cross-validation\"\"\"\n    models = []\n    feature_names = list(X_train.columns)\n    oof_preds = np.zeros((len(X_train), len(np.unique(y_train))))\n    \n    for fold, (train_idx, val_idx) in enumerate(GroupKFold(n_splits=n_splits).split(X_train, y_train, groups)):\n        print(f\"\\nFold {fold + 1}\")\n        X_tr, y_tr = X_train.iloc[train_idx], y_train[train_idx]\n        X_val, y_val = X_train.iloc[val_idx], y_train[val_idx]\n        \n        model = LGBMClassifier(\n            n_estimators=1000,\n            learning_rate=0.05,\n            num_leaves=31,\n            max_depth=6,\n            min_child_samples=20,\n            subsample=0.8,\n            colsample_bytree=0.8,\n            reg_alpha=0.1,\n            reg_lambda=0.1,\n            objective='multiclass',\n            n_jobs=-1,\n            random_state=42 + fold\n        )\n        \n        try:\n            model.fit(\n                X_tr, y_tr,\n                eval_set=[(X_val, y_val)],\n                eval_metric='multi_logloss',\n                callbacks=[\n                    early_stopping(50, verbose=False),\n                    log_evaluation(100)\n                ],\n                feature_name=feature_names\n            )\n            models.append(model)\n            \n            # Store out-of-fold predictions\n            oof_preds[val_idx] = model.predict_proba(X_val)\n            \n            print(f\"Fold {fold + 1} completed with {model.best_iteration_} iterations\")\n        except Exception as e:\n            print(f\"Error training fold {fold + 1}: {str(e)[:200]}\")\n            # Create a simple model as fallback\n            fallback_model = LGBMClassifier(n_estimators=100, random_state=42)\n            fallback_model.fit(X_tr, y_tr)\n            models.append(fallback_model)\n    \n    return models, oof_preds\n\ndef make_predictions(models, test_df, le, X_train_columns):\n    \"\"\"Generate predictions with robust error handling\"\"\"\n    predictions = []\n    default_probs = np.ones(len(le.classes_))/len(le.classes_)\n    \n    for seq_id in tqdm(test_df['sequence_id'].unique(), desc=\"Predicting\"):\n        try:\n            seq_df = test_df[test_df['sequence_id'] == seq_id].copy()\n            features = extract_sequence_features(seq_df)\n            features_df = pd.DataFrame([features])\n            \n            # Ensure we have all expected columns\n            missing_cols = set(X_train_columns) - set(features_df.columns)\n            for col in missing_cols:\n                features_df[col] = DEFAULT_VALUE\n            \n            # Ensure correct column order and handle NaN\n            features_df = features_df[X_train_columns].fillna(DEFAULT_VALUE)\n            \n            # Generate predictions from all models\n            pred_probs = np.zeros((len(models), len(le.classes_)))\n            for i, model in enumerate(models):\n                try:\n                    pred_probs[i] = model.predict_proba(features_df)[0]\n                except Exception:\n                    pred_probs[i] = default_probs\n            \n            avg_probs = np.mean(pred_probs, axis=0)\n            pred_class = le.inverse_transform([np.argmax(avg_probs)])[0]\n            \n            predictions.append({'sequence_id': seq_id, 'gesture': pred_class})\n        except Exception as e:\n            print(f\"Error predicting sequence {seq_id}: {str(e)[:200]}\")\n            # Fallback prediction\n            predictions.append({'sequence_id': seq_id, 'gesture': le.classes_[0]})\n    \n    return pd.DataFrame(predictions)\n\ndef validate_submission(submission, test_df):\n    \"\"\"Validate submission format meets competition requirements\"\"\"\n    required_columns = {'sequence_id', 'gesture'}\n    \n    if not isinstance(submission, pd.DataFrame):\n        raise ValueError(\"Submission must be a pandas DataFrame\")\n    \n    if not required_columns.issubset(submission.columns):\n        missing = required_columns - set(submission.columns)\n        raise ValueError(f\"Submission missing required columns: {missing}\")\n    \n    if len(submission) != len(test_df['sequence_id'].unique()):\n        raise ValueError(f\"Submission has {len(submission)} rows but expected {len(test_df['sequence_id'].unique())}\")\n    \n    if submission.isnull().any().any():\n        raise ValueError(\"Submission contains null values\")\n    \n    if not all(submission['sequence_id'].isin(test_df['sequence_id'].unique())):\n        raise ValueError(\"Submission contains invalid sequence_ids\")\n    \n    print(\"Submission validation passed\")\n\n@set_timeout(MAX_RUNTIME)\ndef train_and_predict():\n    \"\"\"Main pipeline function with timeout protection\"\"\"\n    try:\n        # Initialize timeout handler\n        signal.signal(signal.SIGALRM, timeout_handler)\n        signal.alarm(MAX_RUNTIME)\n        \n        # Data paths - robust to different environments\n        DATA_DIR = '/kaggle/input'\n        COMPETITION_DIR = None\n        \n        # Try to find competition data directory\n        for dirname in os.listdir(DATA_DIR):\n            if 'cmi-detect-behavior' in dirname.lower():\n                COMPETITION_DIR = os.path.join(DATA_DIR, dirname)\n                break\n        \n        if COMPETITION_DIR is None:\n            raise FileNotFoundError(\"Could not find competition data directory\")\n        \n        train_path = os.path.join(COMPETITION_DIR, 'train.csv')\n        demo_path = os.path.join(COMPETITION_DIR, 'train_demographics.csv')\n        test_path = os.path.join(COMPETITION_DIR, 'test.csv')\n        \n        print(\"Loading data...\")\n        train_df, demo_df, test_df = load_data(train_path, demo_path, test_path)\n        \n        # Store demographics data for feature extraction\n        extract_sequence_features.demo_df = demo_df\n        \n        # Encode target variable\n        le = LabelEncoder()\n        train_df['gesture_encoded'] = le.fit_transform(train_df['gesture'])\n        \n        print(\"\\nExtracting training features...\")\n        X_train = []\n        y_train = []\n        \n        for seq_id in tqdm(train_df['sequence_id'].unique(), desc=\"Processing sequences\"):\n            try:\n                seq_df = train_df[train_df['sequence_id'] == seq_id]\n                features = extract_sequence_features(seq_df)\n                X_train.append(features)\n                y_train.append(seq_df['gesture_encoded'].iloc[0])\n            except Exception as e:\n                print(f\"Error processing sequence {seq_id}: {str(e)[:200]}\")\n                continue\n        \n        X_train = pd.DataFrame(X_train).fillna(DEFAULT_VALUE)\n        y_train = np.array(y_train)\n        \n        # Store feature names for prediction time\n        X_train_columns = list(X_train.columns)\n        \n        # Get groups for GroupKFold\n        groups = train_df.groupby('sequence_id')['subject'].first().values\n        \n        print(\"\\nTraining model...\")\n        models, oof_preds = train_model(X_train, y_train, groups)\n        \n        # Save models\n        for i, model in enumerate(models):\n            joblib.dump(model, f'lgbm_model_fold{i}.pkl')\n        \n        print(\"\\nGenerating predictions...\")\n        submission = make_predictions(models, test_df, le, X_train_columns)\n        \n        # Validate submission before saving\n        validate_submission(submission, test_df)\n        \n        # Save in required parquet format\n        submission.to_parquet('submission.parquet', index=False)\n        print(\"\\nSubmission saved as submission.parquet!\")\n        \n        # Also save as CSV for reference\n        submission.to_csv('submission.csv', index=False)\n        \n        return submission\n        \n    except TimeoutException:\n        print(f\"Notebook exceeded maximum runtime of {MAX_RUNTIME} seconds\")\n        # Try to save a fallback submission if possible\n        if 'submission' in locals():\n            submission.to_parquet('submission.parquet', index=False)\n            return submission\n        else:\n            # Create minimal submission if we failed early\n            seq_ids = test_df['sequence_id'].unique()\n            fallback = pd.DataFrame({'sequence_id': seq_ids, 'gesture': 'Gesture1'})\n            fallback.to_parquet('submission.parquet', index=False)\n            return fallback\n    except Exception as e:\n        print(f\"Error in main pipeline: {str(e)[:500]}\")\n        raise\n    finally:\n        signal.alarm(0)  # Disable alarm\n\nif __name__ == \"__main__\":\n    submission = train_and_predict()\n    print(\"\\nSubmission head:\")\n    print(submission.head())\n    print(\"\\nSubmission tail:\")\n    print(submission.tail())","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-06T06:38:49.266009Z","iopub.execute_input":"2025-06-06T06:38:49.266575Z","iopub.status.idle":"2025-06-06T06:45:00.418392Z","shell.execute_reply.started":"2025-06-06T06:38:49.266544Z","shell.execute_reply":"2025-06-06T06:45:00.417921Z"}},"outputs":[],"execution_count":null}]}