{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":114135,"databundleVersionId":13695648,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-09-14T18:37:20.684224Z","iopub.execute_input":"2025-09-14T18:37:20.684463Z","iopub.status.idle":"2025-09-14T18:37:22.737036Z","shell.execute_reply.started":"2025-09-14T18:37:20.684436Z","shell.execute_reply":"2025-09-14T18:37:22.736249Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#!/usr/bin/env python3\n\n# Install required packages with specific versions to avoid conflicts\n!pip install pandas numpy scikit-learn==1.3.2 imbalanced-learn psutil lightgbm catboost xgboost -q\n\nimport pandas as pd\nimport numpy as np\nimport gc\nimport psutil\nimport os\nfrom abc import ABC, abstractmethod\nfrom dataclasses import dataclass, field\nfrom typing import Dict, List, Any, Optional, Tuple, Type\nimport hashlib\nimport json\nimport itertools\nimport random\nfrom collections import defaultdict, Counter\nfrom datetime import datetime\n\n# Core imports\nfrom sklearn.preprocessing import StandardScaler, RobustScaler, MinMaxScaler, QuantileTransformer\nfrom sklearn.decomposition import PCA, TruncatedSVD, FastICA, IncrementalPCA\nfrom sklearn.feature_selection import SelectKBest, mutual_info_classif, f_classif, VarianceThreshold, RFE\nfrom sklearn.ensemble import (\n    RandomForestClassifier, ExtraTreesClassifier, GradientBoostingClassifier,\n    VotingClassifier, StackingClassifier, HistGradientBoostingClassifier, IsolationForest\n)\nfrom sklearn.linear_model import LogisticRegression, RidgeClassifier, SGDClassifier\nfrom sklearn.svm import LinearSVC\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.model_selection import cross_val_score, StratifiedKFold, TimeSeriesSplit\nfrom sklearn.metrics import f1_score, make_scorer, precision_score, recall_score, confusion_matrix\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.cluster import MiniBatchKMeans\nfrom sklearn.calibration import CalibratedClassifierCV\nfrom sklearn.isotonic import IsotonicRegression\nfrom imblearn.over_sampling import SMOTE, RandomOverSampler, ADASYN\nfrom imblearn.under_sampling import RandomUnderSampler\nfrom imblearn.ensemble import BalancedRandomForestClassifier, BalancedBaggingClassifier\n\n# Advanced model imports\nimport lightgbm as lgb\nfrom catboost import CatBoostClassifier\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Enhanced configuration\nSKIP_HIGH_MEMORY_OPERATIONS = True\nMEMORY_THRESHOLD_MB = 12000\nMAX_CONFIGURATIONS_TO_TEST = 15\nEXPLORATION_RATE = 0.3  \nSUBMISSION_DIR = '/kaggle/working/submissions'\nENSEMBLE_TOP_N = 5  \n\n# CRITICAL IMPROVEMENT: Threshold for detecting reversals\nREVERSAL_CONFIDENCE_THRESHOLD = {\n    'H': 0.20,  # Need 20% confidence to predict High reversal\n    'L': 0.20,  # Need 20% confidence to predict Low reversal\n}\n\nif not os.path.exists(SUBMISSION_DIR):\n    os.makedirs(SUBMISSION_DIR)\n\n# ========== ENHANCED TIME SERIES FEATURE ENGINEERING ==========\n\ndef create_advanced_time_series_features(X, y=None, ticker_ids=None, timestamps=None, \n                                        lookback_periods=[5, 10, 20, 50], \n                                        max_features_to_process=50):\n    \"\"\"\n    Create comprehensive time series features for reversal detection\n    Including pattern tracking, momentum indicators, and regime detection\n    \"\"\"\n    \n    n_samples, n_features = X.shape\n    feature_list = []\n    \n    print(f\"  Creating time series features for {n_samples} samples with {n_features} features...\")\n    \n    # Convert timestamps to datetime if provided\n    if timestamps is not None:\n        dates = pd.to_datetime(timestamps)\n        # Extract temporal features\n        time_features = np.column_stack([\n            dates.dayofweek.values.astype(np.float32),  # Day of week pattern\n            dates.day.values.astype(np.float32),         # Day of month\n            dates.month.values.astype(np.float32),       # Seasonality\n            (dates.dayofyear.values % 91).astype(np.float32),  # Quarter position\n        ])\n        feature_list.append(time_features)\n    \n    if ticker_ids is not None:\n        unique_tickers = np.unique(ticker_ids)\n        print(f\"  Processing {len(unique_tickers)} tickers...\")\n        \n        all_ticker_features = []\n        \n        for ticker_idx, ticker in enumerate(unique_tickers):\n            ticker_mask = ticker_ids == ticker\n            ticker_indices = np.where(ticker_mask)[0]\n            ticker_data = X[ticker_mask]\n            n_ticker_samples = len(ticker_data)\n            \n            if n_ticker_samples < 20:\n                continue\n            \n            # Process only the most important features to save memory\n            num_features_to_process = min(max_features_to_process, n_features)\n            \n            # Calculate feature importance using variance\n            feature_variances = np.var(ticker_data[:, :num_features_to_process], axis=0)\n            important_features = np.argsort(feature_variances)[-num_features_to_process:]\n            \n            ticker_feature_arrays = []\n            \n            for feat_idx in important_features[:20]:  # Limit to top 20 features per ticker\n                series = ticker_data[:, feat_idx].astype(np.float32)\n                \n                # === ROLLING STATISTICS ===\n                for window in [5, 10, 20]:  # Reduced lookback periods\n                    if window < n_ticker_samples:\n                        # Rolling mean\n                        roll_mean = pd.Series(series).rolling(window, min_periods=1).mean().values.astype(np.float32)\n                        \n                        # Rolling std (volatility)\n                        roll_std = pd.Series(series).rolling(window, min_periods=1).std().fillna(0).values.astype(np.float32)\n                        \n                        # Distance from rolling mean (z-score)\n                        z_score = np.where(roll_std != 0, \n                                         (series - roll_mean) / (roll_std + 1e-10), \n                                         0).astype(np.float32)\n                        \n                        # Rolling min/max\n                        roll_min = pd.Series(series).rolling(window, min_periods=1).min().values.astype(np.float32)\n                        roll_max = pd.Series(series).rolling(window, min_periods=1).max().values.astype(np.float32)\n                        \n                        # Position in range (0 to 1)\n                        range_val = roll_max - roll_min\n                        range_pos = np.where(range_val != 0, \n                                           (series - roll_min) / (range_val + 1e-10), \n                                           0.5).astype(np.float32)\n                        \n                        ticker_feature_arrays.extend([z_score, range_pos])\n                \n                # === MOMENTUM INDICATORS ===\n                # Rate of change\n                for lag in [1, 5]:\n                    if lag < n_ticker_samples:\n                        roc = np.zeros(n_ticker_samples, dtype=np.float32)\n                        roc[lag:] = (series[lag:] - series[:-lag]) / (np.abs(series[:-lag]) + 1e-10)\n                        ticker_feature_arrays.append(roc)\n                \n                # === PATTERN DETECTION ===\n                # Simple peak and trough detection\n                peaks = np.zeros(n_ticker_samples, dtype=np.float32)\n                troughs = np.zeros(n_ticker_samples, dtype=np.float32)\n                \n                for i in range(2, n_ticker_samples - 2):\n                    # Peak: higher than neighbors\n                    if (series[i] > series[i-1] and series[i] > series[i+1] and \n                        series[i] > series[i-2] and series[i] > series[i+2]):\n                        peaks[i] = 1.0\n                    # Trough: lower than neighbors\n                    elif (series[i] < series[i-1] and series[i] < series[i+1] and \n                          series[i] < series[i-2] and series[i] < series[i+2]):\n                        troughs[i] = 1.0\n                \n                # Distance from last peak/trough\n                dist_from_peak = np.full(n_ticker_samples, fill_value=n_ticker_samples, dtype=np.float32)\n                dist_from_trough = np.full(n_ticker_samples, fill_value=n_ticker_samples, dtype=np.float32)\n                \n                peak_indices = np.where(peaks == 1)[0]\n                trough_indices = np.where(troughs == 1)[0]\n                \n                for i in range(n_ticker_samples):\n                    if len(peak_indices) > 0:\n                        past_peaks = peak_indices[peak_indices <= i]\n                        if len(past_peaks) > 0:\n                            dist_from_peak[i] = i - past_peaks[-1]\n                    \n                    if len(trough_indices) > 0:\n                        past_troughs = trough_indices[trough_indices <= i]\n                        if len(past_troughs) > 0:\n                            dist_from_trough[i] = i - past_troughs[-1]\n                \n                ticker_feature_arrays.extend([peaks, troughs, \n                                            dist_from_peak / n_ticker_samples,  # Normalize\n                                            dist_from_trough / n_ticker_samples])\n                \n                # === CONSECUTIVE MOVEMENTS ===\n                consecutive_ups = np.zeros(n_ticker_samples, dtype=np.float32)\n                consecutive_downs = np.zeros(n_ticker_samples, dtype=np.float32)\n                \n                up_count = 0\n                down_count = 0\n                for i in range(1, n_ticker_samples):\n                    if series[i] > series[i-1]:\n                        up_count += 1\n                        down_count = 0\n                    else:\n                        down_count += 1\n                        up_count = 0\n                    consecutive_ups[i] = up_count\n                    consecutive_downs[i] = down_count\n                \n                ticker_feature_arrays.extend([consecutive_ups / 10,  # Normalize\n                                             consecutive_downs / 10])\n            \n            # Stack all features for this ticker\n            if ticker_feature_arrays:\n                ticker_feature_matrix = np.column_stack(ticker_feature_arrays)\n                \n                # Map back to full dataset\n                full_ticker_features = np.zeros((n_samples, ticker_feature_matrix.shape[1]), dtype=np.float32)\n                full_ticker_features[ticker_mask] = ticker_feature_matrix\n                all_ticker_features.append(full_ticker_features)\n        \n        # Combine all ticker features\n        if all_ticker_features:\n            combined_ticker_features = np.hstack(all_ticker_features)\n            feature_list.append(combined_ticker_features)\n            print(f\"  Added {combined_ticker_features.shape[1]} ticker-based time series features\")\n    \n    # Global time series features (if no ticker info or as supplement)\n    else:\n        print(\"  Creating global time series features...\")\n        global_features = []\n        \n        for feat_idx in range(min(20, n_features)):  # Process first 20 features\n            series = X[:, feat_idx].astype(np.float32)\n            \n            # Simple differences\n            diff_1 = np.diff(series, prepend=series[0])\n            diff_2 = np.diff(diff_1, prepend=diff_1[0])\n            \n            # Rolling statistics\n            roll_mean_5 = pd.Series(series).rolling(5, min_periods=1).mean().values.astype(np.float32)\n            roll_std_5 = pd.Series(series).rolling(5, min_periods=1).std().fillna(0).values.astype(np.float32)\n            \n            global_features.extend([\n                diff_1.reshape(-1, 1),\n                diff_2.reshape(-1, 1),\n                (series - roll_mean_5).reshape(-1, 1),\n                roll_std_5.reshape(-1, 1)\n            ])\n        \n        if global_features:\n            feature_list.append(np.hstack(global_features))\n    \n    # Combine all features with original\n    if feature_list:\n        new_features = np.hstack(feature_list)\n        X_enhanced = np.hstack([X, new_features])\n        print(f\"  Final shape: {X_enhanced.shape} (added {new_features.shape[1]} features)\")\n        return X_enhanced\n    else:\n        return X\n\n# Memory Manager Class\nclass MemoryManager:\n    @staticmethod\n    def get_memory_usage():\n        process = psutil.Process()\n        return process.memory_info().rss / 1024 / 1024\n    \n    @staticmethod\n    def get_available_memory():\n        return psutil.virtual_memory().available / 1024 / 1024\n    \n    @staticmethod\n    def get_total_memory():\n        return psutil.virtual_memory().total / 1024 / 1024\n    \n    @staticmethod\n    def estimate_array_memory(shape, dtype=np.float64):\n        bytes_per_element = np.dtype(dtype).itemsize\n        total_bytes = np.prod(shape) * bytes_per_element\n        return total_bytes / 1024 / 1024\n    \n    @staticmethod\n    def clean_memory():\n        gc.collect()\n    \n    @staticmethod\n    def check_memory_for_operation(estimated_mb, safety_factor=2.0):\n        available = MemoryManager.get_available_memory()\n        required = estimated_mb * safety_factor\n        return available > required, available, required\n\n# Data classes\n@dataclass\nclass ProcessingData:\n    X: np.ndarray\n    y: np.ndarray\n    feature_names: Optional[List[str]] = None\n    metadata: Dict[str, Any] = field(default_factory=dict)\n    \n    def copy(self):\n        return ProcessingData(\n            X=self.X.copy(),\n            y=self.y.copy() if self.y is not None else None,\n            feature_names=self.feature_names.copy() if self.feature_names else None,\n            metadata=self.metadata.copy()\n        )\n    \n    def get_memory_usage(self):\n        x_memory = self.X.nbytes / 1024 / 1024\n        y_memory = self.y.nbytes / 1024 / 1024 if self.y is not None else 0\n        return x_memory + y_memory\n\n@dataclass\nclass StepConfig:\n    step_type: str\n    method: str\n    params: Dict[str, Any] = field(default_factory=dict)\n    enabled: bool = True\n    memory_limit_mb: Optional[float] = None\n    \n    def to_dict(self):\n        return {\n            'type': self.step_type,\n            'method': self.method,\n            'params': self.params,\n            'enabled': self.enabled,\n            'memory_limit_mb': self.memory_limit_mb\n        }\n    \n    def __hash__(self):\n        return hash((self.step_type, self.method, json.dumps(self.params, sort_keys=True)))\n\n@dataclass\nclass PipelineConfig:\n    steps: List[StepConfig]\n    config_id: Optional[str] = None\n    skip_high_memory: bool = True\n    memory_threshold_mb: float = 1000\n    model_type: str = 'logistic'\n    model_params: Dict[str, Any] = field(default_factory=dict)\n    \n    def generate_id(self):\n        config_str = json.dumps([s.to_dict() for s in self.steps] + [self.model_type, self.model_params], sort_keys=True)\n        self.config_id = hashlib.md5(config_str.encode()).hexdigest()[:12]\n        return self.config_id\n    \n    def __str__(self):\n        return ' -> '.join([f\"{s.step_type}:{s.method}\" for s in self.steps if s.enabled])\n\n# Base preprocessing classes\nclass BasePreprocessingStep(ABC):\n    STEP_TYPE = None\n    METHOD_NAME = None\n    \n    def __init__(self, config: StepConfig):\n        self.config = config\n        self.is_fitted = False\n        self._fitted_params = {}\n    \n    def estimate_memory_requirement(self, data: ProcessingData) -> float:\n        return data.get_memory_usage()\n    \n    @abstractmethod\n    def fit(self, data: ProcessingData) -> 'BasePreprocessingStep':\n        pass\n    \n    @abstractmethod\n    def transform(self, data: ProcessingData) -> ProcessingData:\n        pass\n    \n    def fit_transform(self, data: ProcessingData) -> ProcessingData:\n        self.fit(data)\n        return self.transform(data)\n\nclass StepRegistry:\n    _registry = defaultdict(dict)\n    \n    @classmethod\n    def register(cls, step_class: Type[BasePreprocessingStep]):\n        if step_class.STEP_TYPE and step_class.METHOD_NAME:\n            cls._registry[step_class.STEP_TYPE][step_class.METHOD_NAME] = step_class\n    \n    @classmethod\n    def get_step_class(cls, step_type: str, method: str) -> Optional[Type[BasePreprocessingStep]]:\n        return cls._registry.get(step_type, {}).get(method)\n    \n    @classmethod\n    def get_available_steps(cls) -> Dict[str, List[str]]:\n        return {step_type: list(methods.keys()) \n                for step_type, methods in cls._registry.items()}\n    \n    @classmethod\n    def create_step(cls, config: StepConfig) -> Optional[BasePreprocessingStep]:\n        step_class = cls.get_step_class(config.step_type, config.method)\n        if step_class:\n            return step_class(config)\n        return None\n\ndef register_step(cls):\n    StepRegistry.register(cls)\n    return cls\n\n# Preprocessing step implementations\n@register_step\nclass DowncastStep(BasePreprocessingStep):\n    STEP_TYPE = \"downcast\"\n    METHOD_NAME = \"auto\"\n    \n    def fit(self, data: ProcessingData) -> 'DowncastStep':\n        self.is_fitted = True\n        return self\n    \n    def transform(self, data: ProcessingData) -> ProcessingData:\n        result = data.copy()\n        original_memory = result.get_memory_usage()\n        \n        # Convert to float32 for memory efficiency\n        if result.X.dtype != np.float32:\n            result.X = result.X.astype(np.float32)\n        \n        new_memory = result.get_memory_usage()\n        reduction_pct = (1 - new_memory/original_memory) * 100 if original_memory > 0 else 0\n        result.metadata['memory_reduction'] = f\"{reduction_pct:.1f}%\"\n        return result\n\n@register_step\nclass ImputationStep(BasePreprocessingStep):\n    STEP_TYPE = \"impute\"\n    METHOD_NAME = \"simple\"\n    \n    def fit(self, data: ProcessingData) -> 'ImputationStep':\n        strategy = self.config.params.get('strategy', 'median')\n        fill_value = self.config.params.get('fill_value', 0)\n        X_clean = data.X.copy()\n        X_clean = np.where(np.isinf(X_clean), np.nan, X_clean)\n        \n        if strategy == 'constant':\n            self._fitted_params['imputer'] = SimpleImputer(strategy='constant', fill_value=fill_value)\n        else:\n            self._fitted_params['imputer'] = SimpleImputer(strategy=strategy)\n        \n        self._fitted_params['imputer'].fit(X_clean)\n        self.is_fitted = True\n        return self\n    \n    def transform(self, data: ProcessingData) -> ProcessingData:\n        if not self.is_fitted:\n            raise ValueError(\"Must fit before transform\")\n        \n        result = data.copy()\n        result.X = np.where(np.isinf(result.X), np.nan, result.X)\n        result.X = self._fitted_params['imputer'].transform(result.X)\n        result.X = np.nan_to_num(result.X, nan=0.0, posinf=0.0, neginf=0.0)\n        return result\n\n@register_step\nclass StandardScalerStep(BasePreprocessingStep):\n    STEP_TYPE = \"scaling\"\n    METHOD_NAME = \"standard\"\n    \n    def fit(self, data: ProcessingData) -> 'StandardScalerStep':\n        X_float = data.X.astype(np.float32)\n        self._fitted_params['scaler'] = StandardScaler()\n        self._fitted_params['scaler'].fit(X_float)\n        self.is_fitted = True\n        return self\n    \n    def transform(self, data: ProcessingData) -> ProcessingData:\n        if not self.is_fitted:\n            raise ValueError(\"Must fit before transform\")\n        \n        result = data.copy()\n        result.X = result.X.astype(np.float32)\n        result.X = self._fitted_params['scaler'].transform(result.X)\n        result.X = np.nan_to_num(result.X, nan=0.0, posinf=3.0, neginf=-3.0)\n        return result\n\n@register_step\nclass RobustScalerStep(BasePreprocessingStep):\n    STEP_TYPE = \"scaling\"\n    METHOD_NAME = \"robust\"\n    \n    def fit(self, data: ProcessingData) -> 'RobustScalerStep':\n        X_float = data.X.astype(np.float32)\n        self._fitted_params['scaler'] = RobustScaler()\n        self._fitted_params['scaler'].fit(X_float)\n        self.is_fitted = True\n        return self\n    \n    def transform(self, data: ProcessingData) -> ProcessingData:\n        if not self.is_fitted:\n            raise ValueError(\"Must fit before transform\")\n        \n        result = data.copy()\n        result.X = result.X.astype(np.float32)\n        result.X = self._fitted_params['scaler'].transform(result.X)\n        result.X = np.nan_to_num(result.X, nan=0.0, posinf=3.0, neginf=-3.0)\n        return result\n\n@register_step\nclass VarianceThresholdReduction(BasePreprocessingStep):\n    STEP_TYPE = \"dim_reduce\"\n    METHOD_NAME = \"variance_threshold\"\n    \n    def estimate_memory_requirement(self, data: ProcessingData) -> float:\n        return MemoryManager.estimate_array_memory((data.X.shape[1],), dtype=np.float32)\n    \n    def fit(self, data: ProcessingData) -> 'VarianceThresholdReduction':\n        threshold = self.config.params.get('threshold', 0.01)\n        chunk_size = 1000\n        variances = np.zeros(data.X.shape[1], dtype=np.float32)\n        \n        for i in range(0, data.X.shape[1], chunk_size):\n            end_idx = min(i + chunk_size, data.X.shape[1])\n            variances[i:end_idx] = np.var(data.X[:, i:end_idx].astype(np.float32), axis=0)\n        \n        self._fitted_params['keep_indices'] = np.where(variances > threshold)[0]\n        \n        # Ensure we keep at least some features\n        if len(self._fitted_params['keep_indices']) < 100:\n            # Keep top 100 by variance\n            top_indices = np.argsort(variances)[-min(100, len(variances)):]\n            self._fitted_params['keep_indices'] = top_indices\n        \n        self.is_fitted = True\n        return self\n    \n    def transform(self, data: ProcessingData) -> ProcessingData:\n        if not self.is_fitted:\n            raise ValueError(\"Must fit before transform\")\n        \n        result = data.copy()\n        result.X = result.X[:, self._fitted_params['keep_indices']]\n        if result.feature_names:\n            result.feature_names = [result.feature_names[i] for i in self._fitted_params['keep_indices']]\n        return result\n\n@register_step\nclass SelectKBestReduction(BasePreprocessingStep):\n    STEP_TYPE = \"dim_reduce\"\n    METHOD_NAME = \"select_k_best\"\n    \n    def estimate_memory_requirement(self, data: ProcessingData) -> float:\n        return data.get_memory_usage() * 0.1\n    \n    def fit(self, data: ProcessingData) -> 'SelectKBestReduction':\n        k = min(self.config.params.get('k', 500), data.X.shape[1])\n        score_func = self.config.params.get('score_func', 'f_classif')\n        \n        if score_func == 'mutual_info':\n            func = mutual_info_classif\n        else:\n            func = f_classif\n        \n        self._fitted_params['selector'] = SelectKBest(score_func=func, k=k)\n        X_float = data.X.astype(np.float32)\n        self._fitted_params['selector'].fit(X_float, data.y)\n        self.is_fitted = True\n        return self\n    \n    def transform(self, data: ProcessingData) -> ProcessingData:\n        if not self.is_fitted:\n            raise ValueError(\"Must fit before transform\")\n        \n        result = data.copy()\n        result.X = result.X.astype(np.float32)\n        result.X = self._fitted_params['selector'].transform(result.X)\n        if result.feature_names:\n            mask = self._fitted_params['selector'].get_support()\n            result.feature_names = [name for name, keep in zip(result.feature_names, mask) if keep]\n        return result\n\n# Pipeline class\nclass MemoryAwarePipeline:\n    def __init__(self, config: PipelineConfig):\n        self.config = config\n        self.steps = []\n        self.mandatory_steps = []\n        self._initialize_steps()\n    \n    def _initialize_steps(self):\n        self.mandatory_steps = [\n            DowncastStep(StepConfig('downcast', 'auto')),\n            ImputationStep(StepConfig('impute', 'simple', {'strategy': 'median'}))\n        ]\n        \n        for step_config in self.config.steps:\n            if step_config.enabled:\n                step = StepRegistry.create_step(step_config)\n                if step:\n                    self.steps.append(step)\n    \n    def fit(self, X: np.ndarray, y: np.ndarray, feature_names: Optional[List[str]] = None):\n        data = ProcessingData(X=X, y=y, feature_names=feature_names)\n        \n        for step in self.mandatory_steps:\n            step.fit(data)\n            data = step.transform(data)\n            MemoryManager.clean_memory()\n        \n        for step in self.steps:\n            if self.config.skip_high_memory:\n                estimated_mem = step.estimate_memory_requirement(data)\n                can_proceed, available, required = MemoryManager.check_memory_for_operation(estimated_mem)\n                \n                if not can_proceed or estimated_mem > self.config.memory_threshold_mb:\n                    continue\n            \n            try:\n                step.fit(data)\n                data = step.transform(data)\n                MemoryManager.clean_memory()\n            except Exception as e:\n                print(f\"    Warning: Step failed - {e}\")\n                continue\n        \n        self._final_shape = data.X.shape\n        return self\n    \n    def transform(self, X: np.ndarray, y: Optional[np.ndarray] = None, feature_names: Optional[List[str]] = None):\n        data = ProcessingData(X=X, y=y if y is not None else np.zeros(X.shape[0]), feature_names=feature_names)\n        \n        for step in self.mandatory_steps:\n            data = step.transform(data)\n        \n        for step in self.steps:\n            try:\n                data = step.transform(data)\n            except:\n                continue\n        \n        if y is None:\n            return data.X\n        return data.X, data.y\n    \n    def fit_transform(self, X: np.ndarray, y: np.ndarray, feature_names: Optional[List[str]] = None):\n        self.fit(X, y, feature_names)\n        return self.transform(X, y, feature_names)\n\n# Grid Search Class\nclass IntelligentGridSearch:\n    def __init__(self, X_train, y_train, X_test=None):\n        self.X_train = X_train\n        self.y_train = y_train\n        self.X_test = X_test\n        self.exploration_rate = EXPLORATION_RATE\n        self.memory_threshold_mb = MEMORY_THRESHOLD_MB\n        \n        self.results = []\n        self.config_performance = {}\n        self.step_performance = defaultdict(lambda: defaultdict(list))\n        self.all_predictions = {}\n        \n        self.best_config = None\n        self.best_score = -np.inf\n        self.best_model = None\n        self.best_pipeline = None\n        \n        # Track class distribution\n        unique, counts = np.unique(y_train, return_counts=True)\n        self.class_distribution = dict(zip(unique, counts))\n        print(f\"Training class distribution: {self.class_distribution}\")\n    \n    def generate_memory_efficient_configs(self, n_configs=10):\n        configs = []\n        available_steps = StepRegistry.get_available_steps()\n        \n        # Step orderings\n        step_orderings = [\n            ['dim_reduce', 'scaling'],  \n            ['scaling', 'dim_reduce'],\n            ['dim_reduce'],\n            ['scaling'],\n            [],  # No preprocessing\n        ]\n        \n        # Model configurations with adjusted weights\n        model_configs = [\n            ('logistic', {'C': 0.01, 'solver': 'liblinear', 'weight_factor': 10.0}),\n            ('logistic', {'C': 0.1, 'solver': 'liblinear', 'weight_factor': 15.0}),\n            ('logistic', {'C': 1.0, 'solver': 'saga', 'weight_factor': 20.0}),\n            \n            ('random_forest', {'n_estimators': 100, 'max_depth': 5, 'min_samples_split': 50, 'weight_factor': 10.0}),\n            ('random_forest', {'n_estimators': 200, 'max_depth': 8, 'min_samples_split': 30, 'weight_factor': 15.0}),\n            \n            ('lightgbm', {'num_leaves': 15, 'learning_rate': 0.03, 'min_child_samples': 50, 'weight_factor': 10.0}),\n            ('lightgbm', {'num_leaves': 31, 'learning_rate': 0.05, 'min_child_samples': 30, 'weight_factor': 15.0}),\n            \n            ('hist_gradient', {'max_iter': 100, 'max_depth': 5, 'min_samples_leaf': 50, 'weight_factor': 10.0}),\n            \n            ('isolation_forest', {'contamination': 0.06}),\n        ]\n        \n        for _ in range(n_configs):\n            ordering_choice = random.choice(step_orderings)\n            model_type, model_params = random.choice(model_configs)\n            \n            steps = []\n            for step_type in ordering_choice:\n                if step_type == 'dim_reduce' and 'dim_reduce' in available_steps:\n                    method = random.choice(['variance_threshold', 'select_k_best'])\n                    if method == 'variance_threshold':\n                        params = {'threshold': random.choice([0.0001, 0.001, 0.01])}\n                    else:\n                        params = {'k': random.choice([500, 1000, 2000]), 'score_func': 'f_classif'}\n                    steps.append(StepConfig('dim_reduce', method, params, True, self.memory_threshold_mb))\n                    \n                elif step_type == 'scaling' and 'scaling' in available_steps:\n                    method = random.choice(['standard', 'robust'])\n                    steps.append(StepConfig('scaling', method, {}, True, self.memory_threshold_mb))\n            \n            config = PipelineConfig(\n                steps=steps,\n                skip_high_memory=True,\n                memory_threshold_mb=self.memory_threshold_mb,\n                model_type=model_type,\n                model_params=model_params\n            )\n            config.generate_id()\n            configs.append(config)\n        \n        return configs\n    \n    def create_model(self, model_type, model_params, class_weight_dict):\n        \"\"\"Create model with class weight handling\"\"\"\n        \n        weight_factor = model_params.get('weight_factor', 10.0)\n        \n        # Adjust weights\n        adjusted_weights = {}\n        max_class = max(self.class_distribution, key=self.class_distribution.get)\n        \n        for class_idx in class_weight_dict:\n            if class_idx == max_class:\n                adjusted_weights[class_idx] = 1.0\n            else:\n                adjusted_weights[class_idx] = weight_factor\n        \n        print(f\"  Adjusted weights: {adjusted_weights}\")\n        \n        if model_type == 'isolation_forest':\n            contamination = model_params.get('contamination', 0.06)\n            return IsolationForest(contamination=contamination, random_state=42)\n            \n        elif model_type == 'lightgbm':\n            params = {\n                'objective': 'multiclass',\n                'num_class': 3,\n                'boosting_type': 'gbdt',\n                'num_leaves': model_params.get('num_leaves', 31),\n                'learning_rate': model_params.get('learning_rate', 0.05),\n                'feature_fraction': 0.8,\n                'bagging_fraction': 0.7,\n                'bagging_freq': 5,\n                'n_estimators': model_params.get('n_estimators', 100),\n                'class_weight': adjusted_weights,\n                'random_state': 42,\n                'verbose': -1,\n                'min_child_samples': model_params.get('min_child_samples', 30),\n                'reg_alpha': 0.1,\n                'reg_lambda': 0.1,\n            }\n            return lgb.LGBMClassifier(**params)\n            \n        elif model_type == 'random_forest':\n            params = {\n                'n_estimators': model_params.get('n_estimators', 100),\n                'max_depth': model_params.get('max_depth', 10),\n                'min_samples_split': model_params.get('min_samples_split', 30),\n                'min_samples_leaf': model_params.get('min_samples_leaf', 15),\n                'class_weight': adjusted_weights,\n                'random_state': 42,\n                'n_jobs': -1\n            }\n            return RandomForestClassifier(**params)\n            \n        elif model_type == 'hist_gradient':\n            params = {\n                'max_iter': model_params.get('max_iter', 100),\n                'max_depth': model_params.get('max_depth', 5),\n                'learning_rate': model_params.get('learning_rate', 0.1),\n                'random_state': 42,\n                'min_samples_leaf': model_params.get('min_samples_leaf', 30),\n            }\n            return HistGradientBoostingClassifier(**params)\n            \n        else:  # logistic\n            params = {\n                'C': model_params.get('C', 1.0),\n                'max_iter': 1000,\n                'class_weight': adjusted_weights,\n                'solver': model_params.get('solver', 'liblinear'),\n                'random_state': 42\n            }\n            return LogisticRegression(**params)\n    \n    def apply_probability_threshold(self, probabilities, thresholds=None):\n        \"\"\"Apply probability thresholds to convert predictions\"\"\"\n        if thresholds is None:\n            thresholds = REVERSAL_CONFIDENCE_THRESHOLD\n        \n        n_samples = probabilities.shape[0]\n        predictions = []\n        \n        for i in range(n_samples):\n            probs = probabilities[i]\n            \n            # Map: 0=H, 1=L, 2=None\n            if len(probs) == 3:\n                if probs[0] > thresholds.get('H', 0.20):  # H class\n                    predictions.append(0)\n                elif probs[1] > thresholds.get('L', 0.20):  # L class\n                    predictions.append(1)\n                else:\n                    predictions.append(2)  # None class\n            else:\n                predictions.append(2)  # Default to None\n        \n        return np.array(predictions)\n    \n    def evaluate_config(self, config: PipelineConfig, config_index: int):\n        try:\n            pipeline = MemoryAwarePipeline(config)\n            X_processed, y_processed = pipeline.fit_transform(self.X_train, self.y_train)\n            \n            unique_classes = np.unique(y_processed)\n            class_weights = compute_class_weight('balanced', classes=unique_classes, y=y_processed)\n            class_weight_dict = {c: w for c, w in zip(unique_classes, class_weights)}\n            \n            # Create model\n            model = self.create_model(config.model_type, config.model_params, class_weight_dict)\n            \n            # Cross-validation\n            cv = StratifiedKFold(n_splits=3, shuffle=True, random_state=42)\n            \n            if config.model_type == 'hist_gradient':\n                # Manual CV with sample weights\n                sample_weights = np.array([class_weight_dict.get(y, 1.0) for y in y_processed])\n                scores = []\n                for train_idx, val_idx in cv.split(X_processed, y_processed):\n                    X_tr, X_val = X_processed[train_idx], X_processed[val_idx]\n                    y_tr, y_val = y_processed[train_idx], y_processed[val_idx]\n                    w_tr = sample_weights[train_idx]\n                    \n                    model.fit(X_tr, y_tr, sample_weight=w_tr)\n                    y_pred = model.predict(X_val)\n                    score = f1_score(y_val, y_pred, average='macro', zero_division=0)\n                    scores.append(score)\n                cv_scores = np.array(scores)\n            else:\n                cv_scores = cross_val_score(model, X_processed, y_processed, cv=cv, scoring='f1_macro')\n            \n            score = cv_scores.mean()\n            \n            # Fit final model\n            if config.model_type == 'hist_gradient':\n                sample_weights = np.array([class_weight_dict.get(y, 1.0) for y in y_processed])\n                model.fit(X_processed, y_processed, sample_weight=sample_weights)\n            else:\n                model.fit(X_processed, y_processed)\n            \n            # Generate test predictions\n            if self.X_test is not None:\n                X_test_transformed = pipeline.transform(self.X_test)\n                \n                if config.model_type == 'isolation_forest':\n                    anomaly_scores = model.decision_function(X_test_transformed)\n                    # Lower scores are anomalies\n                    threshold_high = np.percentile(anomaly_scores, 3)  # Bottom 3% as H\n                    threshold_low = np.percentile(anomaly_scores, 6)   # Next 3% as L\n                    \n                    predictions = np.ones(len(anomaly_scores), dtype=int) * 2  # Default None\n                    predictions[anomaly_scores < threshold_high] = 0  # H\n                    predictions[(anomaly_scores >= threshold_high) & (anomaly_scores < threshold_low)] = 1  # L\n                    \n                    probabilities = np.zeros((len(predictions), 3))\n                    for i, pred in enumerate(predictions):\n                        probabilities[i, pred] = 1.0\n                else:\n                    if hasattr(model, 'predict_proba'):\n                        probabilities = model.predict_proba(X_test_transformed)\n                        predictions = self.apply_probability_threshold(probabilities)\n                    else:\n                        predictions = model.predict(X_test_transformed)\n                        probabilities = np.zeros((len(predictions), 3))\n                        for i, pred in enumerate(predictions):\n                            probabilities[i, int(pred)] = 1.0\n                \n                predictions = np.array(predictions).astype(int)\n                \n                self.all_predictions[config.config_id] = {\n                    'predictions': predictions,\n                    'probabilities': probabilities,\n                    'score': score,\n                    'config': config,\n                    'model': model,\n                    'pipeline': pipeline,\n                    'config_index': config_index,\n                    'model_type': config.model_type,\n                    'cv_scores': cv_scores\n                }\n            \n            MemoryManager.clean_memory()\n            \n            result = {\n                'config': config,\n                'score': score,\n                'model': model,\n                'pipeline': pipeline,\n                'shape': X_processed.shape,\n                'memory_used': MemoryManager.get_memory_usage(),\n                'model_type': config.model_type,\n                'cv_scores': cv_scores\n            }\n            \n            for step in config.steps:\n                if step.enabled:\n                    self.step_performance[step.step_type][step.method].append(score)\n            \n            return result\n            \n        except Exception as e:\n            print(f\"  Error in config evaluation: {str(e)}\")\n            MemoryManager.clean_memory()\n            return None\n    \n    def generate_intelligent_ensembles(self, test_ids, class_names):\n        \"\"\"Generate ensemble predictions\"\"\"\n        \n        if len(self.all_predictions) < 2:\n            return {}\n        \n        sorted_results = sorted(self.all_predictions.items(), key=lambda x: x[1]['score'], reverse=True)\n        \n        ensembles = {}\n        \n        # Conservative voting ensemble\n        if len(sorted_results) >= 3:\n            top_n = min(5, len(sorted_results))\n            top_n_preds = [sorted_results[i][1]['predictions'] for i in range(top_n)]\n            \n            ensemble_conservative = []\n            for i in range(len(top_n_preds[0])):\n                votes = [int(top_n_preds[j][i]) for j in range(top_n)]\n                vote_counts = Counter(votes)\n                \n                # Require strong agreement for minority classes\n                if vote_counts.get(0, 0) >= top_n * 0.6:  # H\n                    ensemble_conservative.append(0)\n                elif vote_counts.get(1, 0) >= top_n * 0.6:  # L\n                    ensemble_conservative.append(1)\n                else:\n                    ensemble_conservative.append(2)  # None\n            \n            ensembles['conservative_voting'] = [class_names[int(p)] for p in ensemble_conservative]\n        \n        return ensembles\n    \n    def run_search(self, n_iterations=None, test_ids=None, class_names=None):\n        if n_iterations is None:\n            n_iterations = MAX_CONFIGURATIONS_TO_TEST\n        \n        initial_configs = self.generate_memory_efficient_configs(min(50, n_iterations * 2))\n        \n        for i in range(n_iterations):\n            if i < len(initial_configs):\n                config = initial_configs[i]\n                strategy = \"initial\"\n            else:\n                config = self.generate_memory_efficient_configs(1)[0]\n                strategy = \"explore\"\n            \n            if config.config_id in self.config_performance:\n                continue\n            \n            print(f\"\\nConfig {i+1}/{n_iterations} [{strategy}] - Model: {config.model_type}\")\n            print(f\"  Pipeline: {str(config)}\")\n            \n            result = self.evaluate_config(config, i+1)\n            \n            if result:\n                self.results.append(result)\n                self.config_performance[config.config_id] = result['score']\n                \n                print(f\"  Score: {result['score']:.4f} (std: {result['cv_scores'].std():.4f}), Shape: {result['shape']}\")\n                \n                # Print prediction distribution\n                if config.config_id in self.all_predictions:\n                    pred_counts = Counter(self.all_predictions[config.config_id]['predictions'])\n                    total_preds = sum(pred_counts.values())\n                    print(f\"  Predictions: H={pred_counts[0]} ({pred_counts[0]/total_preds:.1%}), \" + \n                          f\"L={pred_counts[1]} ({pred_counts[1]/total_preds:.1%}), \" +\n                          f\"None={pred_counts[2]} ({pred_counts[2]/total_preds:.1%})\")\n                \n                if result['score'] > self.best_score:\n                    self.best_score = result['score']\n                    self.best_config = config\n                    self.best_model = result['model']\n                    self.best_pipeline = result['pipeline']\n                    print(f\"  *** New best score: {self.best_score:.4f} ***\")\n                \n                if config.config_id in self.all_predictions and test_ids is not None and class_names is not None:\n                    pred_data = self.all_predictions[config.config_id]\n                    filename = f\"config_{i+1:03d}_{pred_data['model_type']}_score_{result['score']:.4f}.csv\"\n                    self.save_submission(pred_data['predictions'], filename, test_ids, class_names)\n        \n        if test_ids is not None and class_names is not None and len(self.all_predictions) > 0:\n            print(\"\\nGenerating ensemble submissions...\")\n            ensembles = self.generate_intelligent_ensembles(test_ids, class_names)\n            \n            for ensemble_name, predictions in ensembles.items():\n                submission = pd.DataFrame({\n                    'id': test_ids,\n                    'class_label': predictions\n                })\n                filepath = os.path.join(SUBMISSION_DIR, f'ensemble_{ensemble_name}.csv')\n                submission.to_csv(filepath, index=False)\n                print(f\"  Saved ensemble: {ensemble_name}\")\n                \n                dist = Counter(predictions)\n                print(f\"    Distribution: {dist}\")\n        \n        return self.best_config, self.best_model, self.best_pipeline\n    \n    def save_submission(self, predictions, filename, test_ids, class_names):\n        predictions = np.array(predictions).astype(int)\n        \n        submission = pd.DataFrame({\n            'id': test_ids,\n            'class_label': [class_names[int(p)] for p in predictions]\n        })\n        filepath = os.path.join(SUBMISSION_DIR, filename)\n        submission.to_csv(filepath, index=False)\n        return filepath\n\ndef main():\n    print(\"Loading data...\")\n    train = pd.read_csv('/kaggle/input/detecting-reversal-points-in-us-equities/competition_data/train.csv')\n    test = pd.read_csv('/kaggle/input/detecting-reversal-points-in-us-equities/competition_data/test.csv')\n    \n    # Map targets\n    target_mapping = {'HH': 'H', 'LH': 'H', 'HL': 'L', 'LL': 'L'}\n    train['class_label_mapped'] = train['class_label'].map(target_mapping).fillna('None')\n    \n    # Extract ticker and time information\n    ticker_ids = train['ticker_id'].values if 'ticker_id' in train.columns else None\n    timestamps = train['t'].values if 't' in train.columns else None\n    \n    test_ticker_ids = test['ticker_id'].values if 'ticker_id' in test.columns else None\n    test_timestamps = test['t'].values if 't' in test.columns else None\n    \n    metadata_cols = ['ticker_id', 't', 'class_label', 'class_label_mapped', 'train_id', 'id']\n    feature_cols = [col for col in train.columns if col not in metadata_cols]\n    \n    X_train = train[feature_cols].values.astype(np.float32)\n    y_train = pd.factorize(train['class_label_mapped'])[0]\n    X_test = test[feature_cols].values.astype(np.float32)\n    \n    # Apply enhanced time series feature engineering\n    print(\"Creating advanced time series features...\")\n    X_train = create_advanced_time_series_features(X_train, y_train, ticker_ids, timestamps)\n    X_test = create_advanced_time_series_features(X_test, None, test_ticker_ids, test_timestamps)\n    \n    class_names = ['H', 'L', 'None']\n    test_ids = test['id'].values if 'id' in test.columns else range(len(test))\n    \n    print(f\"\\nFinal data shapes - Train: {X_train.shape}, Test: {X_test.shape}\")\n    print(f\"Class distribution: {Counter(y_train)}\")\n    \n    # Calculate expected distribution\n    train_dist = Counter(y_train)\n    total = sum(train_dist.values())\n    print(f\"Class percentages: H={train_dist[0]/total:.1%}, L={train_dist[1]/total:.1%}, None={train_dist[2]/total:.1%}\")\n    \n    # Run grid search\n    search = IntelligentGridSearch(X_train, y_train, X_test)\n    best_config, best_model, best_pipeline = search.run_search(\n        n_iterations=MAX_CONFIGURATIONS_TO_TEST,\n        test_ids=test_ids,\n        class_names=class_names\n    )\n    \n    if best_model is not None:\n        X_test_transformed = best_pipeline.transform(X_test)\n        \n        # Generate final predictions\n        if hasattr(best_model, 'predict_proba'):\n            probabilities = best_model.predict_proba(X_test_transformed)\n            predictions = search.apply_probability_threshold(probabilities)\n        else:\n            predictions = best_model.predict(X_test_transformed)\n        \n        predictions = np.array(predictions).astype(int)\n        final_predictions = [class_names[int(p)] for p in predictions]\n        \n        submission = pd.DataFrame({\n            'id': test_ids,\n            'class_label': final_predictions\n        })\n        \n        submission.to_csv('submission.csv', index=False)\n        submission.to_csv(os.path.join(SUBMISSION_DIR, 'FINAL_BEST_submission.csv'), index=False)\n        \n        print(f\"\\n{'='*60}\")\n        print(f\"Best model type: {best_config.model_type}\")\n        print(f\"Best score: {search.best_score:.4f}\")\n        print(f\"Final submission distribution: {submission['class_label'].value_counts()}\")\n        \n        # Check distribution\n        dist_pct = submission['class_label'].value_counts(normalize=True)\n        print(f\"Percentages: H={dist_pct.get('H', 0):.1%}, L={dist_pct.get('L', 0):.1%}, None={dist_pct.get('None', 0):.1%}\")\n        \n        if dist_pct.get('None', 0) < 0.8:\n            print(\"WARNING: 'None' class may be under-represented.\")\n        elif dist_pct.get('None', 0) > 0.98:\n            print(\"WARNING: Model may be too conservative - consider adjusting thresholds.\")\n        \n        print(f\"{'='*60}\")\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-14T18:37:22.738558Z","iopub.execute_input":"2025-09-14T18:37:22.738920Z","execution_failed":"2025-09-14T19:16:58.818Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# #!/usr/bin/env python3\n\n# # Install required packages\n# !pip install pandas numpy scikit-learn imbalanced-learn -q\n\n# import pandas as pd\n# import numpy as np\n# from dataclasses import dataclass, field\n# from typing import Dict, List, Any, Optional, Tuple\n# from abc import ABC, abstractmethod\n# import hashlib\n# import json\n# import itertools\n# from sklearn.preprocessing import (\n#     StandardScaler, RobustScaler, MinMaxScaler, QuantileTransformer,\n#     PowerTransformer, Normalizer, MaxAbsScaler, KBinsDiscretizer\n# )\n# from sklearn.decomposition import PCA, TruncatedSVD, FastICA, NMF, FactorAnalysis\n# from sklearn.feature_selection import (\n#     VarianceThreshold, SelectKBest, SelectPercentile, \n#     f_classif, mutual_info_classif, chi2, RFE\n# )\n# from sklearn.linear_model import LogisticRegression, RidgeClassifier\n# from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier\n# from sklearn.model_selection import cross_val_score, StratifiedKFold\n# from sklearn.metrics import f1_score\n# from sklearn.utils.class_weight import compute_class_weight\n# from sklearn.cluster import KMeans\n# from imblearn.over_sampling import SMOTE, RandomOverSampler, ADASYN, BorderlineSMOTE\n# from imblearn.under_sampling import RandomUnderSampler, TomekLinks, NearMiss\n# from imblearn.combine import SMOTETomek, SMOTEENN\n# import warnings\n# warnings.filterwarnings('ignore')\n\n# # ============================================================================\n# # ENHANCED DATA TYPE HANDLER\n# # ============================================================================\n\n# class DataTypeHandler:\n#     \"\"\"Advanced handler for mixed data types\"\"\"\n    \n#     @staticmethod\n#     def identify_feature_types(X, threshold=3):\n#         \"\"\"Identify boolean, categorical, and continuous features\"\"\"\n#         n_samples, n_features = X.shape\n#         boolean_mask = np.zeros(n_features, dtype=bool)\n#         categorical_mask = np.zeros(n_features, dtype=bool)\n        \n#         for i in range(n_features):\n#             col = X[:, i]\n#             try:\n#                 col_numeric = pd.to_numeric(col, errors='coerce')\n#                 unique_vals = np.unique(col_numeric[~pd.isna(col_numeric)])\n                \n#                 if len(unique_vals) <= 2:\n#                     boolean_mask[i] = True\n#                 elif len(unique_vals) <= threshold:\n#                     categorical_mask[i] = True\n#             except:\n#                 pass\n                \n#         return boolean_mask, categorical_mask\n    \n#     @staticmethod\n#     def safe_variance_threshold(X, threshold=0.01):\n#         \"\"\"Apply variance threshold safely\"\"\"\n#         n_features = X.shape[1]\n#         keep_features = []\n        \n#         for i in range(n_features):\n#             col = X[:, i].astype(float)\n#             if np.var(col) > threshold:\n#                 keep_features.append(i)\n                \n#         return np.array(keep_features)\n\n# # ============================================================================\n# # EXPANDED PREPROCESSING COMPONENTS\n# # ============================================================================\n\n# @dataclass\n# class ProcessingInput:\n#     X: np.ndarray\n#     y: np.ndarray\n#     boolean_mask: Optional[np.ndarray] = None\n#     categorical_mask: Optional[np.ndarray] = None\n#     metadata: Dict[str, Any] = field(default_factory=dict)\n\n# @dataclass\n# class ProcessingOutput:\n#     X: np.ndarray\n#     y: np.ndarray\n#     metadata: Dict[str, Any] = field(default_factory=dict)\n#     transformation_log: List[str] = field(default_factory=list)\n\n# @dataclass\n# class StepConfig:\n#     enabled: bool = True\n#     method: str = \"default\"\n#     params: Dict[str, Any] = field(default_factory=dict)\n\n# @dataclass\n# class PipelineConfig:\n#     config_id: str = \"\"\n#     outlier_removal: StepConfig = field(default_factory=lambda: StepConfig(enabled=False))\n#     scaling: StepConfig = field(default_factory=lambda: StepConfig(enabled=True, method=\"standard\"))\n#     feature_selection: StepConfig = field(default_factory=lambda: StepConfig(enabled=False))\n#     dimensionality_reduction: StepConfig = field(default_factory=lambda: StepConfig(enabled=False))\n#     balancing: StepConfig = field(default_factory=lambda: StepConfig(enabled=False))\n    \n#     def to_dict(self):\n#         return {\n#             'outlier_removal': {'enabled': self.outlier_removal.enabled, 'method': self.outlier_removal.method},\n#             'scaling': {'enabled': self.scaling.enabled, 'method': self.scaling.method},\n#             'feature_selection': {'enabled': self.feature_selection.enabled, 'method': self.feature_selection.method},\n#             'dimensionality_reduction': {'enabled': self.dimensionality_reduction.enabled, 'method': self.dimensionality_reduction.method},\n#             'balancing': {'enabled': self.balancing.enabled, 'method': self.balancing.method}\n#         }\n    \n#     def generate_id(self):\n#         config_str = json.dumps(self.to_dict(), sort_keys=True)\n#         self.config_id = hashlib.md5(config_str.encode()).hexdigest()[:8]\n#         return self.config_id\n\n# class ExpandedOutlierRemovalStep:\n#     \"\"\"Expanded outlier removal with multiple strategies\"\"\"\n    \n#     def __init__(self, config: StepConfig):\n#         self.config = config\n#         self.bounds = {}\n#         self.boolean_mask = None\n#         self.is_fitted = False\n        \n#     def fit(self, data: ProcessingInput):\n#         if not self.config.enabled:\n#             return self\n        \n#         self.boolean_mask = data.boolean_mask\n#         continuous_indices = np.where(~self.boolean_mask)[0] if self.boolean_mask is not None else np.arange(data.X.shape[1])\n        \n#         method = self.config.method\n        \n#         if method == \"iqr\":\n#             multiplier = self.config.params.get('multiplier', 1.5)\n#             for i in continuous_indices:\n#                 col = data.X[:, i].astype(float)\n#                 Q1 = np.percentile(col[~np.isnan(col)], 25)\n#                 Q3 = np.percentile(col[~np.isnan(col)], 75)\n#                 IQR = Q3 - Q1\n#                 self.bounds[i] = (Q1 - multiplier * IQR, Q3 + multiplier * IQR)\n                \n#         elif method == \"zscore\":\n#             threshold = self.config.params.get('threshold', 3)\n#             for i in continuous_indices:\n#                 col = data.X[:, i].astype(float)\n#                 mean = np.nanmean(col)\n#                 std = np.nanstd(col)\n#                 self.bounds[i] = (mean - threshold * std, mean + threshold * std)\n                \n#         elif method == \"mad\":  # Median Absolute Deviation\n#             threshold = self.config.params.get('threshold', 3)\n#             for i in continuous_indices:\n#                 col = data.X[:, i].astype(float)\n#                 median = np.nanmedian(col)\n#                 mad = np.nanmedian(np.abs(col - median))\n#                 self.bounds[i] = (median - threshold * mad, median + threshold * mad)\n                \n#         elif method == \"percentile\":\n#             lower_percentile = self.config.params.get('lower', 1)\n#             upper_percentile = self.config.params.get('upper', 99)\n#             for i in continuous_indices:\n#                 col = data.X[:, i].astype(float)\n#                 self.bounds[i] = (\n#                     np.percentile(col[~np.isnan(col)], lower_percentile),\n#                     np.percentile(col[~np.isnan(col)], upper_percentile)\n#                 )\n                \n#         elif method == \"isolation_forest\":\n#             # Simple isolation using distance from mean\n#             contamination = self.config.params.get('contamination', 0.1)\n#             for i in continuous_indices:\n#                 col = data.X[:, i].astype(float)\n#                 threshold = np.percentile(np.abs(col - np.nanmean(col)), 100 * (1 - contamination))\n#                 mean = np.nanmean(col)\n#                 self.bounds[i] = (mean - threshold, mean + threshold)\n        \n#         self.is_fitted = True\n#         return self\n    \n#     def transform(self, data: ProcessingInput):\n#         if not self.config.enabled:\n#             return ProcessingOutput(X=data.X, y=data.y, metadata=data.metadata,\n#                                   transformation_log=[\"outlier_removal: skipped\"])\n        \n#         X_transformed = data.X.copy()\n#         for i, (lower, upper) in self.bounds.items():\n#             X_transformed[:, i] = np.clip(X_transformed[:, i].astype(float), lower, upper)\n        \n#         return ProcessingOutput(\n#             X=X_transformed, y=data.y, metadata=data.metadata,\n#             transformation_log=[f\"outlier_removal: {self.config.method}\"]\n#         )\n\n# class ExpandedScalingStep:\n#     \"\"\"Expanded scaling with binning and transformation strategies\"\"\"\n    \n#     def __init__(self, config: StepConfig):\n#         self.config = config\n#         self.scaler = None\n#         self.boolean_mask = None\n#         self.continuous_indices = None\n#         self.is_fitted = False\n        \n#     def fit(self, data: ProcessingInput):\n#         if not self.config.enabled:\n#             return self\n        \n#         self.boolean_mask = data.boolean_mask\n#         self.continuous_indices = np.where(~self.boolean_mask)[0] if self.boolean_mask is not None else np.arange(data.X.shape[1])\n        \n#         if len(self.continuous_indices) > 0:\n#             continuous_data = data.X[:, self.continuous_indices].astype(float)\n#             method = self.config.method\n            \n#             if method == \"standard\":\n#                 self.scaler = StandardScaler()\n#             elif method == \"robust\":\n#                 self.scaler = RobustScaler()\n#             elif method == \"minmax\":\n#                 self.scaler = MinMaxScaler()\n#             elif method == \"maxabs\":\n#                 self.scaler = MaxAbsScaler()\n#             elif method == \"quantile_uniform\":\n#                 n_quantiles = min(self.config.params.get('n_quantiles', 1000), len(data.X))\n#                 self.scaler = QuantileTransformer(output_distribution='uniform', n_quantiles=n_quantiles, random_state=42)\n#             elif method == \"quantile_normal\":\n#                 n_quantiles = min(self.config.params.get('n_quantiles', 1000), len(data.X))\n#                 self.scaler = QuantileTransformer(output_distribution='normal', n_quantiles=n_quantiles, random_state=42)\n#             elif method == \"power_yeo\":\n#                 self.scaler = PowerTransformer(method='yeo-johnson', standardize=True)\n#             elif method == \"power_box\":\n#                 # Box-Cox requires positive values\n#                 self.scaler = PowerTransformer(method='box-cox', standardize=True)\n#             elif method == \"normalizer\":\n#                 norm = self.config.params.get('norm', 'l2')\n#                 self.scaler = Normalizer(norm=norm)\n#             elif method == \"binning_uniform\":\n#                 n_bins = self.config.params.get('n_bins', 10)\n#                 self.scaler = KBinsDiscretizer(n_bins=n_bins, encode='ordinal', strategy='uniform')\n#             elif method == \"binning_quantile\":\n#                 n_bins = self.config.params.get('n_bins', 10)\n#                 self.scaler = KBinsDiscretizer(n_bins=n_bins, encode='ordinal', strategy='quantile')\n#             elif method == \"binning_kmeans\":\n#                 n_bins = self.config.params.get('n_bins', 10)\n#                 self.scaler = KBinsDiscretizer(n_bins=n_bins, encode='ordinal', strategy='kmeans')\n#             else:\n#                 self.scaler = StandardScaler()\n            \n#             try:\n#                 self.scaler.fit(continuous_data)\n#             except:\n#                 # Fallback to standard scaler if method fails\n#                 self.scaler = StandardScaler()\n#                 self.scaler.fit(continuous_data)\n        \n#         self.is_fitted = True\n#         return self\n    \n#     def transform(self, data: ProcessingInput):\n#         if not self.config.enabled:\n#             return ProcessingOutput(X=data.X, y=data.y, metadata=data.metadata,\n#                                   transformation_log=[\"scaling: skipped\"])\n        \n#         X_transformed = data.X.copy().astype(float)\n        \n#         if self.scaler is not None and len(self.continuous_indices) > 0:\n#             try:\n#                 X_transformed[:, self.continuous_indices] = self.scaler.transform(\n#                     X_transformed[:, self.continuous_indices]\n#                 )\n#             except:\n#                 pass\n        \n#         return ProcessingOutput(\n#             X=X_transformed, y=data.y, metadata=data.metadata,\n#             transformation_log=[f\"scaling: {self.config.method}\"]\n#         )\n\n# class ExpandedFeatureSelectionStep:\n#     \"\"\"Expanded feature selection with multiple strategies\"\"\"\n    \n#     def __init__(self, config: StepConfig):\n#         self.config = config\n#         self.selected_features = None\n#         self.selector = None\n#         self.is_fitted = False\n        \n#     def fit(self, data: ProcessingInput):\n#         if not self.config.enabled:\n#             return self\n        \n#         X_float = data.X.astype(float)\n#         method = self.config.method\n        \n#         if method == \"variance\":\n#             threshold = self.config.params.get('threshold', 0.01)\n#             self.selected_features = DataTypeHandler.safe_variance_threshold(X_float, threshold)\n            \n#         elif method == \"kbest_f\":\n#             k = min(self.config.params.get('k', 100), data.X.shape[1])\n#             selector = SelectKBest(score_func=f_classif, k=k)\n#             selector.fit(X_float, data.y)\n#             self.selected_features = np.where(selector.get_support())[0]\n            \n#         elif method == \"kbest_mutual\":\n#             k = min(self.config.params.get('k', 100), data.X.shape[1])\n#             selector = SelectKBest(score_func=mutual_info_classif, k=k)\n#             selector.fit(X_float, data.y)\n#             self.selected_features = np.where(selector.get_support())[0]\n            \n#         elif method == \"kbest_chi2\":\n#             # Chi2 requires non-negative features\n#             k = min(self.config.params.get('k', 100), data.X.shape[1])\n#             X_nonneg = X_float - X_float.min() + 0.001\n#             selector = SelectKBest(score_func=chi2, k=k)\n#             selector.fit(X_nonneg, data.y)\n#             self.selected_features = np.where(selector.get_support())[0]\n            \n#         elif method == \"percentile\":\n#             percentile = self.config.params.get('percentile', 50)\n#             selector = SelectPercentile(score_func=f_classif, percentile=percentile)\n#             selector.fit(X_float, data.y)\n#             self.selected_features = np.where(selector.get_support())[0]\n            \n#         elif method == \"rfe\":\n#             n_features = self.config.params.get('n_features', min(100, data.X.shape[1] // 2))\n#             estimator = LogisticRegression(max_iter=100, random_state=42)\n#             selector = RFE(estimator, n_features_to_select=n_features, step=0.1)\n#             selector.fit(X_float, data.y)\n#             self.selected_features = np.where(selector.get_support())[0]\n            \n#         elif method == \"tree_based\":\n#             # Use tree importance for selection\n#             clf = ExtraTreesClassifier(n_estimators=50, random_state=42)\n#             clf.fit(X_float, data.y)\n#             importances = clf.feature_importances_\n#             threshold = np.percentile(importances, self.config.params.get('percentile', 50))\n#             self.selected_features = np.where(importances >= threshold)[0]\n            \n#         elif method == \"correlation\":\n#             threshold = self.config.params.get('threshold', 0.95)\n#             try:\n#                 corr_matrix = np.corrcoef(X_float.T)\n#                 corr_matrix = np.nan_to_num(corr_matrix, nan=0.0)\n#                 upper = np.triu(np.abs(corr_matrix), k=1)\n#                 to_drop = np.unique(np.where(upper > threshold)[1])\n#                 self.selected_features = np.setdiff1d(np.arange(data.X.shape[1]), to_drop)\n#             except:\n#                 self.selected_features = np.arange(data.X.shape[1])\n        \n#         # Ensure minimum features\n#         if self.selected_features is None or len(self.selected_features) < 10:\n#             self.selected_features = np.arange(min(100, data.X.shape[1]))\n        \n#         self.is_fitted = True\n#         return self\n    \n#     def transform(self, data: ProcessingInput):\n#         if not self.config.enabled or self.selected_features is None:\n#             return ProcessingOutput(X=data.X, y=data.y, metadata=data.metadata,\n#                                   transformation_log=[\"feature_selection: skipped\"])\n        \n#         X_transformed = data.X[:, self.selected_features]\n        \n#         return ProcessingOutput(\n#             X=X_transformed, y=data.y, metadata=data.metadata,\n#             transformation_log=[f\"feature_selection: {self.config.method} ({len(self.selected_features)} features)\"]\n#         )\n\n# class ExpandedDimensionalityReductionStep:\n#     \"\"\"Expanded dimensionality reduction with multiple strategies\"\"\"\n    \n#     def __init__(self, config: StepConfig):\n#         self.config = config\n#         self.reducer = None\n#         self.is_fitted = False\n        \n#     def fit(self, data: ProcessingInput):\n#         if not self.config.enabled:\n#             return self\n        \n#         n_components = min(\n#             self.config.params.get('n_components', 100),\n#             min(data.X.shape) - 1\n#         )\n        \n#         if n_components < 2:\n#             return self\n        \n#         X_float = data.X.astype(float)\n#         method = self.config.method\n        \n#         try:\n#             if method == \"pca\":\n#                 self.reducer = PCA(n_components=n_components, random_state=42)\n#             elif method == \"svd\":\n#                 self.reducer = TruncatedSVD(n_components=n_components, random_state=42)\n#             elif method == \"ica\":\n#                 self.reducer = FastICA(n_components=n_components, random_state=42, max_iter=500)\n#             elif method == \"nmf\":\n#                 # NMF requires non-negative features\n#                 X_nonneg = X_float - X_float.min() + 0.001\n#                 self.reducer = NMF(n_components=n_components, random_state=42, max_iter=500)\n#                 self.reducer.fit(X_nonneg)\n#                 return self\n#             elif method == \"factor\":\n#                 self.reducer = FactorAnalysis(n_components=n_components, random_state=42)\n#             elif method == \"kernel_pca\":\n#                 from sklearn.decomposition import KernelPCA\n#                 kernel = self.config.params.get('kernel', 'rbf')\n#                 self.reducer = KernelPCA(n_components=n_components, kernel=kernel, random_state=42)\n#             else:\n#                 self.reducer = PCA(n_components=n_components, random_state=42)\n            \n#             if self.reducer and method != \"nmf\":\n#                 self.reducer.fit(X_float)\n#         except:\n#             self.reducer = None\n        \n#         self.is_fitted = True\n#         return self\n    \n#     def transform(self, data: ProcessingInput):\n#         if not self.config.enabled or self.reducer is None:\n#             return ProcessingOutput(X=data.X, y=data.y, metadata=data.metadata,\n#                                   transformation_log=[\"dimensionality_reduction: skipped\"])\n        \n#         try:\n#             X_float = data.X.astype(float)\n#             if self.config.method == \"nmf\":\n#                 X_float = X_float - X_float.min() + 0.001\n#             X_transformed = self.reducer.transform(X_float)\n#         except:\n#             X_transformed = data.X\n        \n#         return ProcessingOutput(\n#             X=X_transformed, y=data.y, metadata=data.metadata,\n#             transformation_log=[f\"dimensionality_reduction: {self.config.method}\"]\n#         )\n\n# class ExpandedBalancingStep:\n#     \"\"\"Expanded balancing with multiple strategies including hybrid methods\"\"\"\n    \n#     def __init__(self, config: StepConfig):\n#         self.config = config\n#         self.balancer = None\n#         self.is_fitted = False\n        \n#     def fit(self, data: ProcessingInput):\n#         if not self.config.enabled:\n#             return self\n        \n#         unique, counts = np.unique(data.y, return_counts=True)\n#         min_samples = counts.min()\n#         method = self.config.method\n        \n#         try:\n#             if method == \"smote\":\n#                 k_neighbors = min(self.config.params.get('k_neighbors', 5), min_samples - 1)\n#                 if k_neighbors > 0:\n#                     self.balancer = SMOTE(k_neighbors=k_neighbors, random_state=42)\n#                 else:\n#                     self.balancer = RandomOverSampler(random_state=42)\n                    \n#             elif method == \"borderline_smote\":\n#                 k_neighbors = min(self.config.params.get('k_neighbors', 5), min_samples - 1)\n#                 if k_neighbors > 0:\n#                     self.balancer = BorderlineSMOTE(k_neighbors=k_neighbors, random_state=42)\n#                 else:\n#                     self.balancer = RandomOverSampler(random_state=42)\n                    \n#             elif method == \"adasyn\":\n#                 n_neighbors = min(self.config.params.get('n_neighbors', 5), min_samples - 1)\n#                 if n_neighbors > 0:\n#                     self.balancer = ADASYN(n_neighbors=n_neighbors, random_state=42)\n#                 else:\n#                     self.balancer = RandomOverSampler(random_state=42)\n                    \n#             elif method == \"oversample\":\n#                 self.balancer = RandomOverSampler(random_state=42)\n                \n#             elif method == \"undersample\":\n#                 self.balancer = RandomUnderSampler(random_state=42)\n                \n#             elif method == \"tomek\":\n#                 self.balancer = TomekLinks()\n                \n#             elif method == \"nearmiss\":\n#                 self.balancer = NearMiss(version=1)\n                \n#             elif method == \"smote_tomek\":\n#                 k_neighbors = min(self.config.params.get('k_neighbors', 5), min_samples - 1)\n#                 if k_neighbors > 0:\n#                     self.balancer = SMOTETomek(smote=SMOTE(k_neighbors=k_neighbors), random_state=42)\n#                 else:\n#                     self.balancer = RandomOverSampler(random_state=42)\n                    \n#             elif method == \"smote_enn\":\n#                 k_neighbors = min(self.config.params.get('k_neighbors', 5), min_samples - 1)\n#                 if k_neighbors > 0:\n#                     self.balancer = SMOTEENN(smote=SMOTE(k_neighbors=k_neighbors), random_state=42)\n#                 else:\n#                     self.balancer = RandomOverSampler(random_state=42)\n#             else:\n#                 self.balancer = RandomOverSampler(random_state=42)\n#         except:\n#             self.balancer = None\n        \n#         self.is_fitted = True\n#         return self\n    \n#     def transform(self, data: ProcessingInput):\n#         if not self.config.enabled or self.balancer is None:\n#             return ProcessingOutput(X=data.X, y=data.y, metadata=data.metadata,\n#                                   transformation_log=[\"balancing: skipped\"])\n        \n#         try:\n#             X_float = data.X.astype(float)\n#             X_balanced, y_balanced = self.balancer.fit_resample(X_float, data.y)\n#             return ProcessingOutput(\n#                 X=X_balanced, y=y_balanced, metadata=data.metadata,\n#                 transformation_log=[f\"balancing: {self.config.method}\"]\n#             )\n#         except Exception as e:\n#             return ProcessingOutput(X=data.X, y=data.y, metadata=data.metadata,\n#                                   transformation_log=[f\"balancing: failed\"])\n\n# class ComprehensivePipeline:\n#     \"\"\"Pipeline that handles all preprocessing steps\"\"\"\n    \n#     def __init__(self, config: PipelineConfig, boolean_mask=None):\n#         self.config = config\n#         self.boolean_mask = boolean_mask\n#         self.steps = self._initialize_steps()\n        \n#     def _initialize_steps(self):\n#         return [\n#             ExpandedOutlierRemovalStep(self.config.outlier_removal),\n#             ExpandedScalingStep(self.config.scaling),\n#             ExpandedFeatureSelectionStep(self.config.feature_selection),\n#             ExpandedDimensionalityReductionStep(self.config.dimensionality_reduction),\n#             ExpandedBalancingStep(self.config.balancing)\n#         ]\n    \n#     def fit(self, X: np.ndarray, y: np.ndarray):\n#         data = ProcessingInput(X=X, y=y, boolean_mask=self.boolean_mask)\n#         for step in self.steps:\n#             step.fit(data)\n#             output = step.transform(data)\n#             if isinstance(step, ExpandedFeatureSelectionStep) and step.selected_features is not None:\n#                 if self.boolean_mask is not None:\n#                     self.boolean_mask = self.boolean_mask[step.selected_features]\n#             data = ProcessingInput(X=output.X, y=output.y, boolean_mask=self.boolean_mask)\n#         return self\n    \n#     def transform(self, X: np.ndarray, y: Optional[np.ndarray] = None):\n#         data = ProcessingInput(X=X, y=y if y is not None else np.zeros(X.shape[0]), boolean_mask=self.boolean_mask)\n        \n#         for i, step in enumerate(self.steps[:-1]):\n#             output = step.transform(data)\n#             data = ProcessingInput(X=output.X, y=output.y, boolean_mask=self.boolean_mask)\n        \n#         if y is not None and self.config.balancing.enabled:\n#             output = self.steps[-1].transform(data)\n#             return output.X, output.y\n#         else:\n#             return data.X, y\n    \n#     def fit_transform(self, X: np.ndarray, y: np.ndarray):\n#         self.fit(X, y)\n#         return self.transform(X, y)\n\n# class GridSearchOrchestrator:\n#     \"\"\"Orchestrates comprehensive grid search across all preprocessing combinations\"\"\"\n    \n#     def __init__(self, X_train, y_train, X_test, boolean_mask):\n#         self.X_train = X_train\n#         self.y_train = y_train\n#         self.X_test = X_test\n#         self.boolean_mask = boolean_mask\n#         self.results = []\n#         self.best_config = None\n#         self.best_score = -np.inf\n#         self.best_model = None\n#         self.best_pipeline = None\n        \n#     def generate_all_configurations(self):\n#         \"\"\"Generate all possible configurations\"\"\"\n        \n#         outlier_options = [\n#             StepConfig(enabled=False),\n#             StepConfig(enabled=True, method=\"iqr\", params={'multiplier': 1.5}),\n#             StepConfig(enabled=True, method=\"zscore\", params={'threshold': 3}),\n#             StepConfig(enabled=True, method=\"mad\", params={'threshold': 3}),\n#             StepConfig(enabled=True, method=\"percentile\", params={'lower': 1, 'upper': 99})\n#         ]\n        \n#         scaling_options = [\n#             StepConfig(enabled=False),\n#             StepConfig(enabled=True, method=\"standard\"),\n#             StepConfig(enabled=True, method=\"robust\"),\n#             StepConfig(enabled=True, method=\"minmax\"),\n#             StepConfig(enabled=True, method=\"quantile_uniform\", params={'n_quantiles': 1000}),\n#             StepConfig(enabled=True, method=\"normalizer\", params={'norm': 'l2'}),\n#             StepConfig(enabled=True, method=\"binning_quantile\", params={'n_bins': 10})\n#         ]\n        \n#         feature_selection_options = [\n#             StepConfig(enabled=False),\n#             StepConfig(enabled=True, method=\"variance\", params={'threshold': 0.01}),\n#             StepConfig(enabled=True, method=\"kbest_f\", params={'k': 500}),\n#             StepConfig(enabled=True, method=\"kbest_mutual\", params={'k': 500}),\n#             StepConfig(enabled=True, method=\"percentile\", params={'percentile': 50})\n#         ]\n        \n#         dim_reduction_options = [\n#             StepConfig(enabled=False),\n#             StepConfig(enabled=True, method=\"pca\", params={'n_components': 100}),\n#             StepConfig(enabled=True, method=\"svd\", params={'n_components': 100}),\n#             StepConfig(enabled=True, method=\"ica\", params={'n_components': 50})\n#         ]\n        \n#         balancing_options = [\n#             StepConfig(enabled=False),\n#             StepConfig(enabled=True, method=\"oversample\"),\n#             StepConfig(enabled=True, method=\"smote\", params={'k_neighbors': 5}),\n#             StepConfig(enabled=True, method=\"adasyn\", params={'n_neighbors': 5}),\n#             StepConfig(enabled=True, method=\"smote_tomek\", params={'k_neighbors': 5})\n#         ]\n        \n#         # Generate all combinations\n#         all_combinations = list(itertools.product(\n#             outlier_options,\n#             scaling_options,\n#             feature_selection_options,\n#             dim_reduction_options,\n#             balancing_options\n#         ))\n        \n#         configs = []\n#         for combo in all_combinations:\n#             config = PipelineConfig(\n#                 outlier_removal=combo[0],\n#                 scaling=combo[1],\n#                 feature_selection=combo[2],\n#                 dimensionality_reduction=combo[3],\n#                 balancing=combo[4]\n#             )\n#             config.generate_id()\n#             configs.append(config)\n        \n#         return configs\n    \n#     def evaluate_configuration(self, config):\n#         \"\"\"Evaluate a single configuration\"\"\"\n#         try:\n#             pipeline = ComprehensivePipeline(config, self.boolean_mask.copy())\n#             X_processed, y_processed = pipeline.fit_transform(self.X_train, self.y_train)\n            \n#             # Calculate class weights\n#             unique_classes = np.unique(y_processed)\n#             class_weights = compute_class_weight('balanced', classes=unique_classes, y=y_processed)\n#             class_weight_dict = {c: w for c, w in zip(unique_classes, class_weights)}\n            \n#             # Quick evaluation with logistic regression\n#             model = LogisticRegression(max_iter=1000, class_weight=class_weight_dict, random_state=42)\n#             cv = StratifiedKFold(n_splits=3, shuffle=True, random_state=42)\n#             cv_scores = cross_val_score(model, X_processed, y_processed, cv=cv, scoring='f1_macro')\n#             cv_mean = cv_scores.mean()\n            \n#             # Fit model for potential final use\n#             model.fit(X_processed, y_processed)\n            \n#             return {\n#                 'config': config,\n#                 'cv_score': cv_mean,\n#                 'model': model,\n#                 'pipeline': pipeline,\n#                 'shape': X_processed.shape\n#             }\n#         except:\n#             return None\n    \n#     def run_grid_search(self, max_configs=10):\n#         \"\"\"Run grid search with specified number of configurations\"\"\"\n        \n#         all_configs = self.generate_all_configurations()\n#         total_configs = len(all_configs)\n        \n#         print(f\"\\nTotal configuration space: {total_configs} combinations\")\n#         print(f\"Testing {min(max_configs, total_configs)} configurations\")\n#         print(f\"Initial data shape: {self.X_train.shape}\")\n#         print(f\"Class distribution: {dict(zip(*np.unique(self.y_train, return_counts=True)))}\\n\")\n        \n#         # Sample configurations if necessary\n#         if max_configs < total_configs:\n#             # Take strategic sample: some from beginning, middle, end, and random\n#             indices = []\n#             indices.extend(range(0, min(max_configs//4, total_configs)))  # First quarter\n#             indices.extend(range(total_configs//2, min(total_configs//2 + max_configs//4, total_configs)))  # Middle quarter\n#             indices.extend(range(max(0, total_configs - max_configs//4), total_configs))  # Last quarter\n#             remaining = max_configs - len(indices)\n#             if remaining > 0:\n#                 random_indices = np.random.choice(total_configs, size=remaining, replace=False)\n#                 indices.extend(random_indices)\n#             configs_to_test = [all_configs[i] for i in set(indices[:max_configs])]\n#         else:\n#             configs_to_test = all_configs\n        \n#         for i, config in enumerate(configs_to_test):\n#             print(f\"Testing configuration {i+1}/{len(configs_to_test)} (ID: {config.config_id})\", end=\" \")\n            \n#             result = self.evaluate_configuration(config)\n            \n#             if result:\n#                 print(f\"CV F1: {result['cv_score']:.4f}, Shape: {result['shape']}\")\n                \n#                 if result['cv_score'] > self.best_score:\n#                     self.best_score = result['cv_score']\n#                     self.best_config = result['config']\n#                     self.best_model = result['model']\n#                     self.best_pipeline = result['pipeline']\n                \n#                 self.results.append(result)\n#             else:\n#                 print(\"Failed\")\n        \n#         self._print_summary()\n#         return self.best_config, self.best_model, self.best_pipeline\n    \n#     def _print_summary(self):\n#         \"\"\"Print summary of grid search results\"\"\"\n#         print(\"\\n\" + \"=\"*80)\n#         print(\"GRID SEARCH SUMMARY\")\n#         print(\"=\"*80)\n        \n#         if self.results:\n#             print(f\"Configurations tested: {len(self.results)}\")\n#             print(f\"Best CV F1 Score: {self.best_score:.4f}\")\n#             print(f\"Best Configuration ID: {self.best_config.config_id}\")\n            \n#             # Show top 5 configurations\n#             sorted_results = sorted(self.results, key=lambda x: x['cv_score'], reverse=True)[:5]\n#             print(\"\\nTop 5 Configurations:\")\n#             for i, result in enumerate(sorted_results):\n#                 config_desc = []\n#                 if result['config'].outlier_removal.enabled:\n#                     config_desc.append(f\"outlier:{result['config'].outlier_removal.method}\")\n#                 if result['config'].scaling.enabled:\n#                     config_desc.append(f\"scale:{result['config'].scaling.method}\")\n#                 if result['config'].feature_selection.enabled:\n#                     config_desc.append(f\"feature:{result['config'].feature_selection.method}\")\n#                 if result['config'].dimensionality_reduction.enabled:\n#                     config_desc.append(f\"dim:{result['config'].dimensionality_reduction.method}\")\n#                 if result['config'].balancing.enabled:\n#                     config_desc.append(f\"balance:{result['config'].balancing.method}\")\n                \n#                 print(f\"  {i+1}. CV F1: {result['cv_score']:.4f} | {' | '.join(config_desc)}\")\n\n# def main():\n#     \"\"\"Main execution function\"\"\"\n#     print(\"Loading data...\")\n#     train = pd.read_csv('/kaggle/input/detecting-reversal-points-in-us-equities/competition_data/train.csv')\n#     test = pd.read_csv('/kaggle/input/detecting-reversal-points-in-us-equities/competition_data/test.csv')\n    \n#     # Map targets\n#     target_mapping = {'HH': 'H', 'LH': 'H', 'HL': 'L', 'LL': 'L'}\n#     train['class_label_mapped'] = train['class_label'].map(target_mapping).fillna('None')\n    \n#     # Identify columns\n#     metadata_cols = ['ticker_id', 't', 'class_label', 'class_label_mapped', 'train_id', 'id']\n#     feature_cols = [col for col in train.columns if col not in metadata_cols]\n    \n#     # Prepare data\n#     X_train = train[feature_cols].values\n#     y_train = pd.factorize(train['class_label_mapped'])[0]\n#     X_test = test[feature_cols].values\n    \n#     # Clean data\n#     X_train = np.nan_to_num(X_train, nan=0.0, posinf=0.0, neginf=0.0)\n#     X_test = np.nan_to_num(X_test, nan=0.0, posinf=0.0, neginf=0.0)\n    \n#     # Identify feature types\n#     print(\"Identifying feature types...\")\n#     boolean_mask, categorical_mask = DataTypeHandler.identify_feature_types(X_train)\n#     print(f\"Boolean features: {boolean_mask.sum()}, Categorical: {categorical_mask.sum()}, Continuous: {(~boolean_mask & ~categorical_mask).sum()}\")\n    \n#     # Run grid search\n#     orchestrator = GridSearchOrchestrator(X_train, y_train, X_test, boolean_mask)\n    \n#     # Set number of configurations to test\n#     NUM_CONFIGS_TO_TEST = 10  # Adjust this value as needed\n    \n#     best_config, best_model, best_pipeline = orchestrator.run_grid_search(max_configs=NUM_CONFIGS_TO_TEST)\n    \n#     # Generate predictions\n#     if best_model is not None:\n#         print(\"\\nGenerating final predictions...\")\n#         X_test_final, _ = best_pipeline.transform(X_test)\n#         predictions = best_model.predict(X_test_final)\n        \n#         class_names = ['H', 'L', 'None']\n#         final_predictions = [class_names[p] for p in predictions]\n        \n#         submission = pd.DataFrame({\n#             'id': test['id'] if 'id' in test.columns else range(len(test)),\n#             'class_label': final_predictions\n#         })\n        \n#         submission.to_csv('submission.csv', index=False)\n        \n#         print(f\"\\nSubmission saved: {submission.shape}\")\n#         print(f\"Prediction distribution: {submission['class_label'].value_counts().to_dict()}\")\n#         print(\"Prediction percentages:\")\n#         for class_name, pct in submission['class_label'].value_counts(normalize=True).items():\n#             print(f\"  {class_name}: {pct:.1%}\")\n\n# if __name__ == \"__main__\":\n#     main()","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"execution":{"execution_failed":"2025-09-14T19:16:58.819Z"}},"outputs":[],"execution_count":null}]}