{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":114135,"databundleVersionId":13695648,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-09-14T18:37:20.684224Z","iopub.execute_input":"2025-09-14T18:37:20.684463Z","iopub.status.idle":"2025-09-14T18:37:22.737036Z","shell.execute_reply.started":"2025-09-14T18:37:20.684436Z","shell.execute_reply":"2025-09-14T18:37:22.736249Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom abc import ABC, abstractmethod\nfrom dataclasses import dataclass, field\nfrom typing import Dict, List, Any, Optional, ClassVar\nfrom enum import Enum\nimport hashlib\nimport json\nimport random\nimport warnings\nimport time\nfrom datetime import datetime, timedelta\nwarnings.filterwarnings('ignore')\n\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler, RobustScaler, MinMaxScaler\nfrom sklearn.ensemble import (RandomForestClassifier, ExtraTreesClassifier, \n                             VotingClassifier, StackingClassifier, AdaBoostClassifier,\n                             GradientBoostingClassifier, HistGradientBoostingClassifier)\nfrom sklearn.linear_model import LogisticRegression, RidgeClassifier\nfrom sklearn.neural_network import MLPClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import f1_score\nfrom sklearn.feature_selection import SelectKBest, f_classif, mutual_info_classif\nfrom sklearn.decomposition import PCA, FastICA, TruncatedSVD\nfrom sklearn.cluster import AgglomerativeClustering\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis\nimport lightgbm as lgb\nimport xgboost as xgb\nfrom scipy import stats\nimport catboost as cb\n\ntry:\n    from sklearn_extra.cluster import KMedoids\nexcept ImportError:\n    print(\"Installing sklearn-extra for KMedoids...\")\n    import subprocess\n    import sys\n    subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"scikit-learn-extra\"])\n    from sklearn_extra.cluster import KMedoids\n\n# ============================================================================\n# CORE FRAMEWORK (Keep existing structure)\n# ============================================================================\n\nclass StepType(Enum):\n    \"\"\"Enumeration of preprocessing step types\"\"\"\n    IMPUTATION = \"imputation\"\n    SCALING = \"scaling\"\n    UNIVARIATE_OUTLIER = \"univariate_outlier\"\n    TIME_SERIES_FEATURES = \"time_series_features\"\n    PATTERN_FEATURES = \"pattern_features\"\n    FEATURE_ENGINEERING = \"feature_engineering\"\n    FEATURE_REDUCTION = \"feature_reduction\"\n    MODELING = \"modeling\"\n\n@dataclass\nclass StepConfig:\n    \"\"\"Configuration for a single preprocessing step\"\"\"\n    step_type: StepType\n    method: str\n    params: Dict[str, Any] = field(default_factory=dict)\n    \n    def to_dict(self):\n        return {\n            'type': self.step_type.value,\n            'method': self.method,\n            'params': self.params\n        }\n    \n    def __hash__(self):\n        return hash((self.step_type.value, self.method, json.dumps(self.params, sort_keys=True)))\n\nclass BaseStep(ABC):\n    \"\"\"Abstract base class for all preprocessing steps\"\"\"\n    \n    CONFIGURATIONS: ClassVar[List[Dict[str, Any]]] = []\n    STEP_TYPE: ClassVar[StepType] = None\n    ALLOW_SKIP: ClassVar[bool] = False\n    \n    def __init__(self, config: StepConfig):\n        self.config = config\n        self.is_fitted = False\n        self._state = {}\n        self._cache = {}\n    \n    @classmethod\n    def get_configurations(cls) -> List[StepConfig]:\n        \"\"\"Get all possible configurations for this step\"\"\"\n        configs = []\n        for config_dict in cls.CONFIGURATIONS:\n            configs.append(StepConfig(\n                step_type=cls.STEP_TYPE,\n                method=config_dict['method'],\n                params=config_dict.get('params', {})\n            ))\n        \n        if cls.ALLOW_SKIP:\n            configs.append(None)\n            \n        return configs\n    \n    @abstractmethod\n    def fit(self, X: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> 'BaseStep':\n        pass\n    \n    @abstractmethod\n    def transform(self, X: np.ndarray, **kwargs) -> np.ndarray:\n        pass\n    \n    def fit_transform(self, X: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> np.ndarray:\n        self.fit(X, y, **kwargs)\n        return self.transform(X, **kwargs)\n    \n    @property\n    def name(self) -> str:\n        params_str = \"_\".join([f\"{k}{v}\" for k, v in self.config.params.items()])\n        if params_str:\n            return f\"{self.config.step_type.value}_{self.config.method}_{params_str}\"\n        return f\"{self.config.step_type.value}_{self.config.method}\"\n\n# [Keep existing Imputation, Scaling, UnivariateOutlier steps as before]\n\nclass ImputationStep(BaseStep):\n    \"\"\"Handles null, inf, and -inf values\"\"\"\n    \n    STEP_TYPE = StepType.IMPUTATION\n    ALLOW_SKIP = False\n    \n    CONFIGURATIONS = [\n        {'method': 'median'},\n        {'method': 'mean'},\n        {'method': 'zero'},\n    ]\n    \n    def fit(self, X: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> 'ImputationStep':\n        self._state['fill_values'] = np.zeros(X.shape[1])\n        \n        for i in range(X.shape[1]):\n            col = X[:, i]\n            finite_mask = np.isfinite(col)\n            if np.any(finite_mask):\n                if self.config.method == \"median\":\n                    self._state['fill_values'][i] = np.median(col[finite_mask])\n                elif self.config.method == \"mean\":\n                    self._state['fill_values'][i] = np.mean(col[finite_mask])\n                elif self.config.method == \"zero\":\n                    self._state['fill_values'][i] = 0\n            else:\n                self._state['fill_values'][i] = 0\n        \n        self.is_fitted = True\n        return self\n    \n    def transform(self, X: np.ndarray, **kwargs) -> np.ndarray:\n        if not self.is_fitted:\n            raise ValueError(\"Must fit before transform\")\n        \n        X_transformed = X.copy()\n        X_transformed = np.where(np.isinf(X_transformed), np.nan, X_transformed)\n        \n        for i in range(X.shape[1]):\n            mask = np.isnan(X_transformed[:, i])\n            X_transformed[mask, i] = self._state['fill_values'][i]\n        \n        return np.nan_to_num(X_transformed, nan=0.0, posinf=0.0, neginf=0.0)\n\nclass ScalingStep(BaseStep):\n    \"\"\"Scales features\"\"\"\n    \n    STEP_TYPE = StepType.SCALING\n    ALLOW_SKIP = True\n    \n    CONFIGURATIONS = [\n        {'method': 'standard'},\n        {'method': 'robust'},\n        {'method': 'minmax'},\n    ]\n    \n    def fit(self, X: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> 'ScalingStep':\n        if self.config.method == \"standard\":\n            self._state['scaler'] = StandardScaler()\n        elif self.config.method == \"robust\":\n            self._state['scaler'] = RobustScaler()\n        elif self.config.method == \"minmax\":\n            self._state['scaler'] = MinMaxScaler()\n        \n        self._state['scaler'].fit(X)\n        self.is_fitted = True\n        return self\n    \n    def transform(self, X: np.ndarray, **kwargs) -> np.ndarray:\n        if not self.is_fitted:\n            raise ValueError(\"Must fit before transform\")\n        \n        X_transformed = self._state['scaler'].transform(X)\n        return np.nan_to_num(X_transformed, nan=0.0, posinf=3.0, neginf=-3.0)\n\nclass UnivariateOutlierStep(BaseStep):\n    \"\"\"Handles outliers on a per-feature basis\"\"\"\n    \n    STEP_TYPE = StepType.UNIVARIATE_OUTLIER\n    ALLOW_SKIP = True\n    \n    CONFIGURATIONS = [\n        {'method': 'iqr', 'params': {'multiplier': 1.5}},\n        {'method': 'percentile', 'params': {'lower': 5, 'upper': 95}},\n    ]\n    \n    def fit(self, X: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> 'UnivariateOutlierStep':\n        self._state['bounds'] = []\n        \n        if self.config.method == \"iqr\":\n            multiplier = self.config.params.get('multiplier', 1.5)\n            \n            for i in range(X.shape[1]):\n                col = X[:, i]\n                Q1 = np.percentile(col, 25)\n                Q3 = np.percentile(col, 75)\n                IQR = Q3 - Q1\n                lower = Q1 - multiplier * IQR\n                upper = Q3 + multiplier * IQR\n                self._state['bounds'].append((lower, upper))\n                \n        elif self.config.method == \"percentile\":\n            lower_pct = self.config.params.get('lower', 5)\n            upper_pct = self.config.params.get('upper', 95)\n            \n            for i in range(X.shape[1]):\n                col = X[:, i]\n                lower = np.percentile(col, lower_pct)\n                upper = np.percentile(col, upper_pct)\n                self._state['bounds'].append((lower, upper))\n        \n        self.is_fitted = True\n        return self\n    \n    def transform(self, X: np.ndarray, **kwargs) -> np.ndarray:\n        if not self.is_fitted:\n            raise ValueError(\"Must fit before transform\")\n        \n        X_transformed = X.copy()\n        \n        for i, (lower, upper) in enumerate(self._state['bounds']):\n            X_transformed[:, i] = np.clip(X_transformed[:, i], lower, upper)\n        \n        return X_transformed\n\n# ============================================================================\n# ENHANCED TIME SERIES & PATTERN FEATURES\n# ============================================================================\n\nclass TimeSeriesFeaturesStep(BaseStep):\n    \"\"\"Enhanced time series features for reversal detection\"\"\"\n    \n    STEP_TYPE = StepType.TIME_SERIES_FEATURES\n    ALLOW_SKIP = True\n    \n    CONFIGURATIONS = [\n        {'method': 'basic_lags', 'params': {'lags': [1, 2, 3]}},\n        {'method': 'rolling_stats', 'params': {'windows': [5, 10, 20]}},\n        {'method': 'momentum', 'params': {'periods': [5, 10, 15]}},\n        {'method': 'acceleration', 'params': {'periods': [3, 5]}},\n    ]\n    \n    def fit(self, X: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> 'TimeSeriesFeaturesStep':\n        variances = np.var(X, axis=0)\n        n_top = min(15, X.shape[1])\n        self._state['top_features'] = np.argsort(variances)[-n_top:]\n        \n        self.is_fitted = True\n        return self\n    \n    def transform(self, X: np.ndarray, **kwargs) -> np.ndarray:\n        if not self.is_fitted:\n            raise ValueError(\"Must fit before transform\")\n        \n        features_list = [X]\n        X_selected = X[:, self._state['top_features']]\n        \n        if self.config.method == \"basic_lags\":\n            lags = self.config.params.get('lags', [1, 2, 3])\n            new_features = []\n            for lag in lags:\n                for j in range(min(5, X_selected.shape[1])):\n                    lagged = np.zeros(X.shape[0])\n                    if lag < X.shape[0]:\n                        lagged[lag:] = X_selected[:-lag, j]\n                    new_features.append(lagged.reshape(-1, 1))\n            if new_features:\n                features_list.append(np.hstack(new_features))\n        \n        elif self.config.method == \"rolling_stats\":\n            windows = self.config.params.get('windows', [5, 10, 20])\n            new_features = []\n            for window in windows:\n                for j in range(min(3, X_selected.shape[1])):\n                    rolling_mean = np.zeros(X.shape[0])\n                    rolling_std = np.zeros(X.shape[0])\n                    for i in range(window, X.shape[0]):\n                        window_data = X_selected[i-window:i, j]\n                        rolling_mean[i] = np.mean(window_data)\n                        rolling_std[i] = np.std(window_data)\n                    new_features.append(rolling_mean.reshape(-1, 1))\n                    new_features.append(rolling_std.reshape(-1, 1))\n            if new_features:\n                features_list.append(np.hstack(new_features))\n                \n        elif self.config.method == \"momentum\":\n            periods = self.config.params.get('periods', [5, 10, 15])\n            new_features = []\n            for period in periods:\n                for j in range(min(5, X_selected.shape[1])):\n                    momentum = np.zeros(X.shape[0])\n                    if period < X.shape[0]:\n                        momentum[period:] = X_selected[period:, j] - X_selected[:-period, j]\n                    new_features.append(momentum.reshape(-1, 1))\n            if new_features:\n                features_list.append(np.hstack(new_features))\n                \n        elif self.config.method == \"acceleration\":\n            periods = self.config.params.get('periods', [3, 5])\n            new_features = []\n            for period in periods:\n                for j in range(min(3, X_selected.shape[1])):\n                    accel = np.zeros(X.shape[0])\n                    if period*2 < X.shape[0]:\n                        momentum1 = X_selected[period:, j] - X_selected[:-period, j]\n                        momentum2 = X_selected[:-period, j] - X_selected[:-period*2, j][:len(momentum1)]\n                        accel[period*2:] = momentum1[period:] - momentum2\n                    new_features.append(accel.reshape(-1, 1))\n            if new_features:\n                features_list.append(np.hstack(new_features))\n        \n        return np.hstack(features_list)\n\nclass PatternFeaturesStep(BaseStep):\n    \"\"\"Enhanced pattern features for reversal detection\"\"\"\n    \n    STEP_TYPE = StepType.PATTERN_FEATURES\n    ALLOW_SKIP = True\n    \n    CONFIGURATIONS = [\n        {'method': 'extrema'},\n        {'method': 'reversal_strength'},\n        {'method': 'zigzag'},\n    ]\n    \n    def fit(self, X: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> 'PatternFeaturesStep':\n        variances = np.var(X, axis=0)\n        n_top = min(15, X.shape[1])\n        self._state['pattern_features'] = np.argsort(variances)[-n_top:]\n        \n        self.is_fitted = True\n        return self\n    \n    def transform(self, X: np.ndarray, **kwargs) -> np.ndarray:\n        if not self.is_fitted:\n            raise ValueError(\"Must fit before transform\")\n        \n        features_list = [X]\n        \n        if self.config.method == \"extrema\":\n            new_features = []\n            for idx in self._state['pattern_features'][:5]:\n                for window in [3, 5, 7]:\n                    local_max = np.zeros(X.shape[0])\n                    local_min = np.zeros(X.shape[0])\n                    for i in range(window, X.shape[0] - window):\n                        if X[i, idx] == np.max(X[i-window:i+window+1, idx]):\n                            local_max[i] = 1\n                        if X[i, idx] == np.min(X[i-window:i+window+1, idx]):\n                            local_min[i] = 1\n                    new_features.append(local_max.reshape(-1, 1))\n                    new_features.append(local_min.reshape(-1, 1))\n            if new_features:\n                features_list.append(np.hstack(new_features))\n        \n        elif self.config.method == \"reversal_strength\":\n            new_features = []\n            for idx in self._state['pattern_features'][:5]:\n                reversal_score = np.zeros(X.shape[0])\n                window = 10\n                for i in range(window*2, X.shape[0]):\n                    left_trend = X[i-window, idx] - X[i-window*2, idx]\n                    right_trend = X[i, idx] - X[i-window, idx]\n                    if left_trend * right_trend < 0:  # Opposite signs = potential reversal\n                        reversal_score[i] = abs(left_trend - right_trend)\n                new_features.append(reversal_score.reshape(-1, 1))\n            if new_features:\n                features_list.append(np.hstack(new_features))\n                \n        elif self.config.method == \"zigzag\":\n            new_features = []\n            for idx in self._state['pattern_features'][:5]:\n                zigzag = np.zeros(X.shape[0])\n                threshold = np.std(X[:, idx]) * 0.1\n                last_extreme = X[0, idx]\n                last_extreme_idx = 0\n                is_rising = True\n                \n                for i in range(1, X.shape[0]):\n                    if is_rising:\n                        if X[i, idx] >= last_extreme:\n                            last_extreme = X[i, idx]\n                            last_extreme_idx = i\n                        elif X[i, idx] < last_extreme - threshold:\n                            zigzag[last_extreme_idx] = 1  # Mark as high\n                            last_extreme = X[i, idx]\n                            last_extreme_idx = i\n                            is_rising = False\n                    else:\n                        if X[i, idx] <= last_extreme:\n                            last_extreme = X[i, idx]\n                            last_extreme_idx = i\n                        elif X[i, idx] > last_extreme + threshold:\n                            zigzag[last_extreme_idx] = -1  # Mark as low\n                            last_extreme = X[i, idx]\n                            last_extreme_idx = i\n                            is_rising = True\n                \n                new_features.append(zigzag.reshape(-1, 1))\n            if new_features:\n                features_list.append(np.hstack(new_features))\n        \n        return np.hstack(features_list)\n\nclass FeatureEngineeringStep(BaseStep):\n    \"\"\"Feature engineering\"\"\"\n    \n    STEP_TYPE = StepType.FEATURE_ENGINEERING\n    ALLOW_SKIP = True\n    \n    CONFIGURATIONS = [\n        {'method': 'polynomial'},\n        {'method': 'interactions'},\n        {'method': 'ratios'},\n    ]\n    \n    def fit(self, X: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> 'FeatureEngineeringStep':\n        variances = np.var(X, axis=0)\n        self._state['top_features'] = np.argsort(variances)[-15:]\n        \n        self.is_fitted = True\n        return self\n    \n    def transform(self, X: np.ndarray, **kwargs) -> np.ndarray:\n        if not self.is_fitted:\n            raise ValueError(\"Must fit before transform\")\n        \n        features = [X]\n        \n        if self.config.method == \"polynomial\":\n            for idx in self._state['top_features'][:7]:\n                col_squared = X[:, idx] ** 2\n                features.append(col_squared.reshape(-1, 1))\n        \n        elif self.config.method == \"interactions\":\n            top_indices = self._state['top_features'][:7]\n            for i in range(len(top_indices)):\n                for j in range(i+1, min(i+3, len(top_indices))):\n                    interaction = X[:, top_indices[i]] * X[:, top_indices[j]]\n                    features.append(interaction.reshape(-1, 1))\n                    \n        elif self.config.method == \"ratios\":\n            top_indices = self._state['top_features'][:7]\n            for i in range(len(top_indices)):\n                for j in range(i+1, min(i+3, len(top_indices))):\n                    ratio = X[:, top_indices[i]] / (X[:, top_indices[j]] + 1e-10)\n                    ratio = np.clip(ratio, -100, 100)\n                    features.append(ratio.reshape(-1, 1))\n        \n        return np.hstack(features)\n\n# ============================================================================\n# ENHANCED FEATURE REDUCTION WITH MULTIPLE MEDOID CONFIGURATIONS\n# ============================================================================\n\nclass FeatureReductionStep(BaseStep):\n    \"\"\"Enhanced feature reduction with extensive MEDOID grid\"\"\"\n    \n    STEP_TYPE = StepType.FEATURE_REDUCTION\n    ALLOW_SKIP = False\n    \n    CONFIGURATIONS = [\n        # Traditional methods\n        {'method': 'variance', 'params': {'n_features': 100}},\n        {'method': 'variance', 'params': {'n_features': 200}},\n        {'method': 'variance', 'params': {'n_features': 500}},\n        {'method': 'statistical', 'params': {'n_features': 150}},\n        {'method': 'statistical', 'params': {'n_features': 300}},\n        {'method': 'tree_importance', 'params': {'n_features': 200}},\n        {'method': 'tree_importance', 'params': {'n_features': 400}},\n        \n        # PCA variations\n        {'method': 'pca', 'params': {'n_components': 50}},\n        {'method': 'pca', 'params': {'n_components': 100}},\n        {'method': 'pca', 'params': {'n_components': 200}},\n        \n        # EXTENSIVE MEDOIDS GRID\n        {'method': 'medoids', 'params': {'n_clusters': 50}},\n        {'method': 'medoids', 'params': {'n_clusters': 75}},\n        {'method': 'medoids', 'params': {'n_clusters': 100}},\n        {'method': 'medoids', 'params': {'n_clusters': 150}},\n        {'method': 'medoids', 'params': {'n_clusters': 200}},\n        {'method': 'medoids', 'params': {'n_clusters': 250}},\n        {'method': 'medoids', 'params': {'n_clusters': 300}},\n        {'method': 'medoids', 'params': {'n_clusters': 400}},\n        \n        # Other methods\n        {'method': 'lda', 'params': {'n_components': 2}},\n        {'method': 'mutual_info', 'params': {'n_features': 300}},\n    ]\n    \n    def fit(self, X: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> 'FeatureReductionStep':\n        n_features = X.shape[1]\n        \n        if 'n_features' in self.config.params:\n            target_features = self.config.params['n_features']\n        elif 'n_components' in self.config.params:\n            target_features = self.config.params['n_components']\n        elif 'n_clusters' in self.config.params:\n            target_features = self.config.params['n_clusters']\n        else:\n            target_features = min(500, n_features)\n        \n        target_features = min(target_features, n_features)\n        \n        if self.config.method == \"variance\":\n            variances = np.var(X, axis=0)\n            self._state['selected_features'] = np.argsort(variances)[-target_features:]\n        \n        elif self.config.method == \"statistical\":\n            if y is not None and target_features < n_features:\n                selector = SelectKBest(f_classif, k=target_features)\n                selector.fit(X, y)\n                self._state['selected_features'] = np.where(selector.get_support())[0]\n            else:\n                variances = np.var(X, axis=0)\n                self._state['selected_features'] = np.argsort(variances)[-target_features:]\n                \n        elif self.config.method == \"mutual_info\":\n            if y is not None and target_features < n_features:\n                # Subsample for speed\n                subset_size = min(1000, X.shape[0])\n                subset_idx = np.random.choice(X.shape[0], subset_size, replace=False)\n                selector = SelectKBest(mutual_info_classif, k=target_features)\n                selector.fit(X[subset_idx], y[subset_idx])\n                self._state['selected_features'] = np.where(selector.get_support())[0]\n            else:\n                variances = np.var(X, axis=0)\n                self._state['selected_features'] = np.argsort(variances)[-target_features:]\n        \n        elif self.config.method == \"tree_importance\":\n            if y is not None:\n                clf = ExtraTreesClassifier(n_estimators=50, random_state=42, n_jobs=-1, max_depth=7)\n                clf.fit(X, y)\n                importances = clf.feature_importances_\n                self._state['selected_features'] = np.argsort(importances)[-target_features:]\n            else:\n                variances = np.var(X, axis=0)\n                self._state['selected_features'] = np.argsort(variances)[-target_features:]\n        \n        elif self.config.method == \"pca\":\n            n_components = min(target_features, min(X.shape))\n            self._state['pca'] = PCA(n_components=n_components, random_state=42)\n            self._state['pca'].fit(X)\n            \n        elif self.config.method == \"lda\":\n            if y is not None:\n                n_components = min(target_features, len(np.unique(y)) - 1)\n                self._state['lda'] = LinearDiscriminantAnalysis(n_components=n_components)\n                self._state['lda'].fit(X, y)\n            else:\n                # Fallback to PCA\n                n_components = min(target_features, min(X.shape))\n                self._state['pca'] = PCA(n_components=n_components, random_state=42)\n                self._state['pca'].fit(X)\n        \n        elif self.config.method == \"medoids\":\n            # Enhanced MEDOIDS implementation\n            n_clusters = self.config.params.get('n_clusters', 100)\n            n_clusters = min(n_clusters, n_features // 2)\n            \n            # Pre-select features by variance\n            variances = np.var(X, axis=0)\n            n_candidates = min(2000, n_features)\n            top_features = np.argsort(variances)[-n_candidates:]\n            \n            if n_clusters < 10:\n                self._state['selected_features'] = top_features[:target_features]\n            else:\n                try:\n                    # Sample for correlation computation\n                    sample_size = min(800, X.shape[0])\n                    sample_idx = np.random.choice(X.shape[0], sample_size, replace=False)\n                    X_sampled = X[sample_idx][:, top_features]\n                    \n                    # Compute distance matrix\n                    corr_matrix = np.abs(np.corrcoef(X_sampled.T))\n                    np.fill_diagonal(corr_matrix, 1.0)  # Ensure diagonal is 1\n                    distance_matrix = 1 - corr_matrix\n                    distance_matrix = np.clip(distance_matrix, 0, 2)  # Ensure valid distances\n                    \n                    # K-Medoids clustering with better parameters\n                    kmedoids = KMedoids(\n                        n_clusters=n_clusters, \n                        metric='precomputed',\n                        init='k-medoids++',\n                        random_state=42, \n                        max_iter=100\n                    )\n                    cluster_labels = kmedoids.fit_predict(distance_matrix)\n                    \n                    # Select best features from each cluster\n                    selected = []\n                    for cluster_id in range(n_clusters):\n                        cluster_mask = cluster_labels == cluster_id\n                        cluster_features = top_features[cluster_mask]\n                        \n                        if len(cluster_features) > 0:\n                            # Select feature with highest variance and importance\n                            cluster_vars = variances[cluster_features]\n                            \n                            # If we have labels, also consider correlation with target\n                            if y is not None:\n                                correlations = np.array([\n                                    abs(np.corrcoef(X[:, f], y)[0, 1]) \n                                    for f in cluster_features\n                                ])\n                                scores = cluster_vars * correlations\n                            else:\n                                scores = cluster_vars\n                            \n                            best_idx = cluster_features[np.argmax(scores)]\n                            selected.append(best_idx)\n                    \n                    # Ensure we have enough features\n                    selected = list(set(selected))  # Remove duplicates\n                    if len(selected) < target_features:\n                        remaining = top_features[~np.isin(top_features, selected)]\n                        additional_needed = target_features - len(selected)\n                        if len(remaining) >= additional_needed:\n                            additional = remaining[-additional_needed:]\n                        else:\n                            additional = remaining\n                        selected.extend(additional)\n                    \n                    self._state['selected_features'] = np.array(selected[:target_features])\n                    \n                except Exception as e:\n                    print(f\"  MEDOIDS failed ({str(e)[:50]}), using variance fallback\")\n                    self._state['selected_features'] = top_features[:target_features]\n        \n        self.is_fitted = True\n        return self\n    \n    def transform(self, X: np.ndarray, **kwargs) -> np.ndarray:\n        if not self.is_fitted:\n            raise ValueError(\"Must fit before transform\")\n        \n        if 'selected_features' in self._state:\n            return X[:, self._state['selected_features']]\n        elif 'pca' in self._state:\n            return self._state['pca'].transform(X)\n        elif 'lda' in self._state:\n            return self._state['lda'].transform(X)\n        else:\n            return X\n\n# ============================================================================\n# ENHANCED MODELING WITH MLPs AND ADVANCED ENSEMBLES\n# ============================================================================\n\nclass ModelingStep(BaseStep):\n    \"\"\"Enhanced modeling with neural networks and advanced ensembles\"\"\"\n    \n    STEP_TYPE = StepType.MODELING\n    ALLOW_SKIP = False\n    \n    CONFIGURATIONS = [\n        # Tree-based models\n        {'method': 'lightgbm', 'params': {\n            'num_leaves': 20, 'learning_rate': 0.05, 'n_estimators': 200,\n            'class_weight': {0: 15, 1: 15, 2: 1}, 'max_depth': 7,\n            'min_child_samples': 30, 'reg_alpha': 0.5, 'reg_lambda': 0.5\n        }},\n        {'method': 'lightgbm_v2', 'params': {\n            'num_leaves': 31, 'learning_rate': 0.03, 'n_estimators': 300,\n            'class_weight': {0: 20, 1: 20, 2: 1}, 'max_depth': 10\n        }},\n        {'method': 'xgboost', 'params': {\n            'max_depth': 6, 'learning_rate': 0.05, 'n_estimators': 250,\n            'gamma': 0.1, 'subsample': 0.8, 'colsample_bytree': 0.8\n        }},\n        {'method': 'catboost', 'params': {\n            'depth': 6, 'learning_rate': 0.03, 'iterations': 200,\n            'class_weights': {0: 15, 1: 15, 2: 1}, 'l2_leaf_reg': 3\n        }},\n        {'method': 'random_forest', 'params': {\n            'n_estimators': 300, 'max_depth': 12, 'min_samples_leaf': 10,\n            'class_weight': {0: 15, 1: 15, 2: 1}, 'max_features': 'sqrt'\n        }},\n        {'method': 'extra_trees', 'params': {\n            'n_estimators': 400, 'max_depth': 15, 'min_samples_leaf': 8,\n            'class_weight': {0: 12, 1: 12, 2: 1}\n        }},\n        {'method': 'hist_gradient', 'params': {\n            'max_iter': 200, 'learning_rate': 0.05, 'max_depth': 8,\n            'class_weight': {0: 15, 1: 15, 2: 1}\n        }},\n        \n        # Neural Networks (MLPs)\n        {'method': 'mlp_small', 'params': {\n            'hidden_layers': (100, 50), 'activation': 'relu',\n            'learning_rate': 0.001, 'max_iter': 500\n        }},\n        {'method': 'mlp_medium', 'params': {\n            'hidden_layers': (200, 100, 50), 'activation': 'relu',\n            'learning_rate': 0.001, 'max_iter': 500, 'early_stopping': True\n        }},\n        {'method': 'mlp_large', 'params': {\n            'hidden_layers': (512, 256, 128, 64), 'activation': 'relu',\n            'learning_rate': 0.0005, 'max_iter': 1000, 'early_stopping': True\n        }},\n        {'method': 'mlp_deep', 'params': {\n            'hidden_layers': (256, 128, 64, 32, 16), 'activation': 'tanh',\n            'learning_rate': 0.001, 'max_iter': 800\n        }},\n        \n        # Advanced Ensembles\n        {'method': 'voting_ensemble', 'params': {}},\n        {'method': 'stacking_ensemble', 'params': {}},\n        {'method': 'mixed_ensemble', 'params': {}},\n    ]\n    \n    def __init__(self, config: StepConfig):\n        super().__init__(config)\n        self.model = None\n        self.cv_score = None\n        self.train_score = None\n        self.overfitting_gap = 0\n        \n    def fit(self, X: np.ndarray, y: np.ndarray, **kwargs) -> 'ModelingStep':\n        model_params = self.config.params.copy()\n        \n        # Create the appropriate model\n        if 'lightgbm' in self.config.method:\n            self.model = lgb.LGBMClassifier(\n                objective='multiclass',\n                num_leaves=model_params.get('num_leaves', 31),\n                learning_rate=model_params.get('learning_rate', 0.05),\n                n_estimators=model_params.get('n_estimators', 200),\n                max_depth=model_params.get('max_depth', -1),\n                class_weight=model_params.get('class_weight'),\n                min_child_samples=model_params.get('min_child_samples', 20),\n                reg_alpha=model_params.get('reg_alpha', 0),\n                reg_lambda=model_params.get('reg_lambda', 0),\n                random_state=42,\n                verbose=-1,\n                n_jobs=-1\n            )\n            \n        elif self.config.method == \"xgboost\":\n            self.model = xgb.XGBClassifier(\n                max_depth=model_params.get('max_depth', 6),\n                learning_rate=model_params.get('learning_rate', 0.03),\n                n_estimators=model_params.get('n_estimators', 200),\n                gamma=model_params.get('gamma', 0),\n                subsample=model_params.get('subsample', 1.0),\n                colsample_bytree=model_params.get('colsample_bytree', 1.0),\n                objective='multi:softprob',\n                random_state=42,\n                use_label_encoder=False,\n                eval_metric='logloss',\n                n_jobs=-1\n            )\n            \n        elif self.config.method == \"catboost\":\n            self.model = cb.CatBoostClassifier(\n                depth=model_params.get('depth', 6),\n                learning_rate=model_params.get('learning_rate', 0.03),\n                iterations=model_params.get('iterations', 150),\n                class_weights=model_params.get('class_weights'),\n                l2_leaf_reg=model_params.get('l2_leaf_reg', 3),\n                random_seed=42,\n                verbose=False,\n                loss_function='MultiClass'\n            )\n            \n        elif self.config.method == \"random_forest\":\n            self.model = RandomForestClassifier(\n                n_estimators=model_params.get('n_estimators', 200),\n                max_depth=model_params.get('max_depth', 10),\n                min_samples_leaf=model_params.get('min_samples_leaf', 10),\n                max_features=model_params.get('max_features', 'sqrt'),\n                class_weight=model_params.get('class_weight'),\n                random_state=42,\n                n_jobs=-1\n            )\n            \n        elif self.config.method == \"extra_trees\":\n            self.model = ExtraTreesClassifier(\n                n_estimators=model_params.get('n_estimators', 250),\n                max_depth=model_params.get('max_depth', 12),\n                min_samples_leaf=model_params.get('min_samples_leaf', 10),\n                class_weight=model_params.get('class_weight'),\n                random_state=42,\n                n_jobs=-1\n            )\n            \n        elif self.config.method == \"hist_gradient\":\n            # Map class weights for HistGradientBoostingClassifier\n            class_weight = model_params.get('class_weight', {0: 1, 1: 1, 2: 1})\n            sample_weights = np.ones(len(y))\n            for class_idx, weight in class_weight.items():\n                sample_weights[y == class_idx] = weight\n            \n            self.model = HistGradientBoostingClassifier(\n                max_iter=model_params.get('max_iter', 100),\n                learning_rate=model_params.get('learning_rate', 0.1),\n                max_depth=model_params.get('max_depth', None),\n                random_state=42\n            )\n            self._sample_weights = sample_weights\n            \n        elif 'mlp' in self.config.method:\n            hidden_layers = model_params.get('hidden_layers', (100,))\n            \n            self.model = MLPClassifier(\n                hidden_layer_sizes=hidden_layers,\n                activation=model_params.get('activation', 'relu'),\n                solver='adam',\n                learning_rate_init=model_params.get('learning_rate', 0.001),\n                max_iter=model_params.get('max_iter', 500),\n                early_stopping=model_params.get('early_stopping', False),\n                validation_fraction=0.1 if model_params.get('early_stopping', False) else 0.1,\n                random_state=42\n            )\n            \n        elif self.config.method == \"voting_ensemble\":\n            estimators = [\n                ('lgb', lgb.LGBMClassifier(\n                    num_leaves=25, n_estimators=150,\n                    class_weight={0: 15, 1: 15, 2: 1},\n                    random_state=42, verbose=-1\n                )),\n                ('rf', RandomForestClassifier(\n                    n_estimators=150, max_depth=10,\n                    class_weight={0: 15, 1: 15, 2: 1},\n                    random_state=42, n_jobs=-1\n                )),\n                ('mlp', MLPClassifier(\n                    hidden_layer_sizes=(128, 64),\n                    max_iter=500, random_state=42\n                ))\n            ]\n            self.model = VotingClassifier(estimators=estimators, voting='soft', n_jobs=-1)\n            \n        elif self.config.method == \"stacking_ensemble\":\n            base_estimators = [\n                ('lgb', lgb.LGBMClassifier(\n                    num_leaves=20, n_estimators=100,\n                    class_weight={0: 15, 1: 15, 2: 1},\n                    random_state=42, verbose=-1\n                )),\n                ('rf', RandomForestClassifier(\n                    n_estimators=100, max_depth=8,\n                    class_weight={0: 15, 1: 15, 2: 1},\n                    random_state=42, n_jobs=-1\n                )),\n            ]\n            final_estimator = LogisticRegression(\n                class_weight={0: 15, 1: 15, 2: 1},\n                random_state=42\n            )\n            self.model = StackingClassifier(\n                estimators=base_estimators,\n                final_estimator=final_estimator,\n                cv=3,\n                n_jobs=-1\n            )\n            \n        elif self.config.method == \"mixed_ensemble\":\n            # A more complex ensemble mixing different model types\n            estimators = [\n                ('lgb', lgb.LGBMClassifier(\n                    num_leaves=31, n_estimators=150,\n                    class_weight={0: 12, 1: 12, 2: 1},\n                    random_state=42, verbose=-1\n                )),\n                ('xgb', xgb.XGBClassifier(\n                    max_depth=6, n_estimators=150,\n                    random_state=42, use_label_encoder=False,\n                    eval_metric='logloss'\n                )),\n                ('mlp', MLPClassifier(\n                    hidden_layer_sizes=(200, 100),\n                    max_iter=500, random_state=42\n                )),\n                ('et', ExtraTreesClassifier(\n                    n_estimators=150, max_depth=10,\n                    class_weight={0: 12, 1: 12, 2: 1},\n                    random_state=42, n_jobs=-1\n                ))\n            ]\n            self.model = VotingClassifier(estimators=estimators, voting='soft', n_jobs=-1)\n        \n        # Cross-validation\n        cv = StratifiedKFold(n_splits=3, shuffle=True, random_state=42)\n        cv_scores = []\n        train_scores = []\n        \n        for train_idx, val_idx in cv.split(X, y):\n            X_tr, X_val = X[train_idx], X[val_idx]\n            y_tr, y_val = y[train_idx], y[val_idx]\n            \n            # Clone and fit model\n            if hasattr(self.model, 'get_params'):\n                model_copy = self.model.__class__(**self.model.get_params())\n            else:\n                # For complex ensembles, recreate\n                model_copy = self._create_model_copy()\n            \n            if self.config.method == \"hist_gradient\" and hasattr(self, '_sample_weights'):\n                model_copy.fit(X_tr, y_tr, sample_weight=self._sample_weights[train_idx])\n            else:\n                model_copy.fit(X_tr, y_tr)\n            \n            # Evaluate\n            train_pred = model_copy.predict(X_tr)\n            val_pred = model_copy.predict(X_val)\n            \n            train_scores.append(f1_score(y_tr, train_pred, average='macro'))\n            cv_scores.append(f1_score(y_val, val_pred, average='macro'))\n        \n        self.cv_score = np.mean(cv_scores)\n        self.train_score = np.mean(train_scores)\n        self.overfitting_gap = self.train_score - self.cv_score\n        \n        # Fit final model\n        if self.config.method == \"hist_gradient\" and hasattr(self, '_sample_weights'):\n            self.model.fit(X, y, sample_weight=self._sample_weights)\n        else:\n            self.model.fit(X, y)\n        \n        self.is_fitted = True\n        return self\n    \n    def _create_model_copy(self):\n        \"\"\"Helper to recreate complex models\"\"\"\n        if self.config.method == \"voting_ensemble\":\n            return VotingClassifier(\n                estimators=[\n                    ('lgb', lgb.LGBMClassifier(num_leaves=25, n_estimators=150,\n                                              class_weight={0: 15, 1: 15, 2: 1},\n                                              random_state=42, verbose=-1)),\n                    ('rf', RandomForestClassifier(n_estimators=150, max_depth=10,\n                                                 class_weight={0: 15, 1: 15, 2: 1},\n                                                 random_state=42, n_jobs=-1)),\n                    ('mlp', MLPClassifier(hidden_layer_sizes=(128, 64),\n                                        max_iter=500, random_state=42))\n                ],\n                voting='soft', n_jobs=-1\n            )\n        elif self.config.method == \"mixed_ensemble\":\n            return VotingClassifier(\n                estimators=[\n                    ('lgb', lgb.LGBMClassifier(num_leaves=31, n_estimators=150,\n                                              class_weight={0: 12, 1: 12, 2: 1},\n                                              random_state=42, verbose=-1)),\n                    ('xgb', xgb.XGBClassifier(max_depth=6, n_estimators=150,\n                                             random_state=42, use_label_encoder=False,\n                                             eval_metric='logloss')),\n                    ('mlp', MLPClassifier(hidden_layer_sizes=(200, 100),\n                                        max_iter=500, random_state=42)),\n                    ('et', ExtraTreesClassifier(n_estimators=150, max_depth=10,\n                                               class_weight={0: 12, 1: 12, 2: 1},\n                                               random_state=42, n_jobs=-1))\n                ],\n                voting='soft', n_jobs=-1\n            )\n        else:\n            return self.model.__class__(**self.model.get_params())\n    \n    def transform(self, X: np.ndarray, **kwargs) -> np.ndarray:\n        if not self.is_fitted:\n            raise ValueError(\"Must fit before transform\")\n        return self.model.predict(X)\n    \n    def predict_proba(self, X: np.ndarray) -> np.ndarray:\n        if not self.is_fitted:\n            raise ValueError(\"Must fit before transform\")\n        return self.model.predict_proba(X)\n\n# [Keep Pipeline, SmartGridSearchEngine, apply_dynamic_thresholds, and main() similar to before]\n# But update SmartGridSearchEngine to handle the new models\n\nclass Pipeline:\n    \"\"\"Pipeline with caching\"\"\"\n    \n    def __init__(self, steps: List[BaseStep]):\n        self.steps = steps\n        self.train_score = None\n        self.cv_score = None\n        self.overfitting_gap = None\n    \n    def fit(self, X: np.ndarray, y: np.ndarray, df: Optional[pd.DataFrame] = None) -> 'Pipeline':\n        X_current = X.copy()\n        \n        for step in self.steps:\n            if isinstance(step, ModelingStep):\n                step.fit(X_current, y)\n                self.train_score = step.train_score\n                self.cv_score = step.cv_score\n                self.overfitting_gap = step.overfitting_gap\n            else:\n                step.fit(X_current, y, df=df)\n                X_current = step.transform(X_current, df=df)\n        \n        return self\n    \n    def transform(self, X: np.ndarray, df: Optional[pd.DataFrame] = None) -> np.ndarray:\n        X_current = X.copy()\n        \n        for step in self.steps:\n            if not isinstance(step, ModelingStep):\n                X_current = step.transform(X_current, df=df)\n        \n        return X_current\n    \n    def predict_proba(self, X: np.ndarray, df: Optional[pd.DataFrame] = None) -> np.ndarray:\n        X_transformed = self.transform(X, df=df)\n        \n        for step in self.steps:\n            if isinstance(step, ModelingStep):\n                return step.predict_proba(X_transformed)\n        \n        raise ValueError(\"No modeling step in pipeline\")\n    \n    def get_scores(self) -> Dict[str, float]:\n        return {\n            'train_score': self.train_score or 0,\n            'cv_score': self.cv_score or 0,\n            'overfitting_gap': self.overfitting_gap or 0\n        }\n\nclass SmartGridSearchEngine:\n    \"\"\"Enhanced grid search with focus on diverse models\"\"\"\n    \n    def __init__(self, X_train, y_train, X_test=None, train_df=None, test_df=None, max_runtime_minutes=90):\n        self.X_train = X_train\n        self.y_train = y_train\n        self.X_test = X_test\n        self.train_df = train_df\n        self.test_df = test_df\n        self.results = []\n        self.best_pipeline = None\n        self.best_score = -np.inf\n        self.max_runtime = timedelta(minutes=max_runtime_minutes)\n        self.start_time = None\n        \n        # Register step classes\n        self.step_classes = {\n            StepType.IMPUTATION: ImputationStep,\n            StepType.SCALING: ScalingStep,\n            StepType.UNIVARIATE_OUTLIER: UnivariateOutlierStep,\n            StepType.TIME_SERIES_FEATURES: TimeSeriesFeaturesStep,\n            StepType.PATTERN_FEATURES: PatternFeaturesStep,\n            StepType.FEATURE_ENGINEERING: FeatureEngineeringStep,\n            StepType.FEATURE_REDUCTION: FeatureReductionStep,\n            StepType.MODELING: ModelingStep,\n        }\n    \n    def sample_strategic_configurations(self, step_order, n_samples=50):\n        \"\"\"Sample configurations with focus on diversity\"\"\"\n        \n        # Prioritize different reduction methods and models\n        reduction_priority = [\n            'medoids_150', 'medoids_200', 'medoids_250', 'medoids_300',\n            'pca_100', 'pca_200', \n            'variance_200', 'statistical_300', \n            'tree_importance_200', 'mutual_info_300'\n        ]\n        \n        model_priority = [\n            'mlp_medium', 'mlp_large', 'mlp_deep',\n            'lightgbm', 'lightgbm_v2',\n            'xgboost', 'catboost',\n            'mixed_ensemble', 'stacking_ensemble',\n            'random_forest', 'extra_trees'\n        ]\n        \n        pipelines = []\n        \n        # Create combinations\n        for reduction_spec in reduction_priority:\n            for model_spec in model_priority:\n                if len(pipelines) >= n_samples:\n                    break\n                    \n                config = []\n                \n                for step_type in step_order:\n                    step_class = self.step_classes.get(step_type)\n                    if not step_class:\n                        continue\n                    \n                    if step_type == StepType.FEATURE_REDUCTION:\n                        # Parse reduction spec\n                        if 'medoids' in reduction_spec:\n                            n_clusters = int(reduction_spec.split('_')[1])\n                            matching = [c for c in step_class.CONFIGURATIONS \n                                      if c['method'] == 'medoids' and \n                                      c.get('params', {}).get('n_clusters') == n_clusters]\n                        elif 'pca' in reduction_spec:\n                            n_components = int(reduction_spec.split('_')[1])\n                            matching = [c for c in step_class.CONFIGURATIONS \n                                      if c['method'] == 'pca' and \n                                      c.get('params', {}).get('n_components') == n_components]\n                        else:\n                            method = reduction_spec.split('_')[0]\n                            matching = [c for c in step_class.CONFIGURATIONS \n                                      if c['method'] == method]\n                        \n                        if matching:\n                            chosen = matching[0]\n                        else:\n                            continue\n                            \n                    elif step_type == StepType.MODELING:\n                        matching = [c for c in step_class.CONFIGURATIONS \n                                  if c['method'] == model_spec or \n                                  (model_spec in c['method'])]\n                        if matching:\n                            chosen = matching[0]\n                        else:\n                            continue\n                    else:\n                        # Random selection for other steps\n                        configs = step_class.get_configurations()\n                        valid = [c for c in configs if c is not None]\n                        if valid:\n                            chosen_config = random.choice(valid)\n                            chosen = chosen_config.to_dict()\n                        else:\n                            continue\n                    \n                    config.append(StepConfig(\n                        step_type=step_type,\n                        method=chosen['method'],\n                        params=chosen.get('params', {})\n                    ))\n                \n                if self._is_valid_pipeline(config):\n                    pipelines.append(config)\n        \n        return pipelines[:n_samples]\n    \n    def _is_valid_pipeline(self, steps):\n        \"\"\"Check if pipeline has required steps\"\"\"\n        has_imputation = any(s.step_type == StepType.IMPUTATION for s in steps)\n        has_modeling = any(s.step_type == StepType.MODELING for s in steps)\n        has_reduction = any(s.step_type == StepType.FEATURE_REDUCTION for s in steps)\n        return has_imputation and has_modeling and has_reduction\n    \n    def evaluate_pipeline(self, config_list):\n        \"\"\"Evaluate a single pipeline configuration\"\"\"\n        try:\n            eval_start = time.time()\n            \n            steps = []\n            for config in config_list:\n                step_class = self.step_classes.get(config.step_type)\n                if step_class:\n                    steps.append(step_class(config))\n            \n            pipeline = Pipeline(steps)\n            pipeline.fit(self.X_train, self.y_train, df=self.train_df)\n            \n            scores = pipeline.get_scores()\n            X_transformed = pipeline.transform(self.X_train, df=self.train_df)\n            \n            eval_time = time.time() - eval_start\n            \n            return {\n                'pipeline': pipeline,\n                'cv_score': scores['cv_score'],\n                'train_score': scores['train_score'],\n                'overfitting_gap': scores['overfitting_gap'],\n                'feature_shape': X_transformed.shape,\n                'description': ' → '.join([s.name for s in steps]),\n                'eval_time': eval_time\n            }\n            \n        except Exception as e:\n            print(f\"  ✗ Error: {str(e)[:100]}\")\n            return None\n    \n    def run(self, step_order, max_iterations=50):\n        \"\"\"Run grid search\"\"\"\n        self.start_time = datetime.now()\n        \n        configs = self.sample_strategic_configurations(step_order, max_iterations)\n        \n        print(f\"Testing {len(configs)} unique pipeline configurations\")\n        print(f\"Initial data shape: {self.X_train.shape}\")\n        print(\"=\"*60)\n        \n        for i, config_list in enumerate(configs):\n            if (datetime.now() - self.start_time) > self.max_runtime:\n                print(f\"\\n⏰ Time limit reached after {i} configurations\")\n                break\n            \n            # Short description for monitoring\n            reduction_step = [s for s in config_list if s.step_type == StepType.FEATURE_REDUCTION][0]\n            model_step = [s for s in config_list if s.step_type == StepType.MODELING][0]\n            \n            print(f\"\\n[{i+1}/{len(configs)}] Reduction: {reduction_step.method} | Model: {model_step.method}\")\n            \n            result = self.evaluate_pipeline(config_list)\n            \n            if result:\n                self.results.append(result)\n                \n                overfitting_warning = \"⚠️ \" if result['overfitting_gap'] > 0.2 else \"\"\n                print(f\"  ✓ CV: {result['cv_score']:.4f} | Train: {result['train_score']:.4f} | \"\n                      f\"Gap: {result['overfitting_gap']:.4f} {overfitting_warning}| \"\n                      f\"Shape: {result['feature_shape']} | Time: {result['eval_time']:.1f}s\")\n                \n                if result['cv_score'] > self.best_score:\n                    self.best_score = result['cv_score']\n                    self.best_pipeline = result['pipeline']\n                    print(f\"  ★ New best CV score!\")\n        \n        total_time = (datetime.now() - self.start_time).total_seconds() / 60\n        print(f\"\\nTotal runtime: {total_time:.1f} minutes\")\n        \n        return self.best_pipeline\n\ndef apply_dynamic_thresholds(probabilities, target_h_pct=0.03, target_l_pct=0.029):\n    \"\"\"Apply dynamic thresholds for class balancing\"\"\"\n    n_samples = len(probabilities)\n    \n    target_h_count = max(1, int(n_samples * target_h_pct))\n    target_l_count = max(1, int(n_samples * target_l_pct))\n    \n    h_probs = probabilities[:, 0]\n    l_probs = probabilities[:, 1]\n    \n    predictions = np.full(n_samples, 2)  # Default to None\n    \n    # Select top H predictions\n    h_indices = np.argsort(h_probs)[-target_h_count:]\n    predictions[h_indices] = 0\n    \n    # Select top L predictions (excluding already assigned H)\n    l_candidates = [(i, l_probs[i]) for i in range(n_samples) if predictions[i] == 2]\n    l_candidates.sort(key=lambda x: x[1], reverse=True)\n    \n    for idx, _ in l_candidates[:target_l_count]:\n        predictions[idx] = 1\n    \n    return predictions\n\ndef main():\n    \"\"\"Main execution with enhanced models\"\"\"\n    \n    print(\"Loading competition data...\")\n    \n    # Load data\n    train = pd.read_csv('/kaggle/input/detecting-reversal-points-in-us-equities/competition_data/train.csv')\n    test = pd.read_csv('/kaggle/input/detecting-reversal-points-in-us-equities/competition_data/test.csv')\n    \n    print(f\"Train shape: {train.shape}\")\n    print(f\"Test shape: {test.shape}\")\n    \n    # Map targets\n    target_mapping = {'HH': 'H', 'LH': 'H', 'HL': 'L', 'LL': 'L'}\n    train['class_label_mapped'] = train['class_label'].map(target_mapping).fillna('None')\n    \n    # Label encoding\n    label_encoder = LabelEncoder()\n    label_encoder.classes_ = np.array(['H', 'L', 'None'])\n    y_train = label_encoder.transform(train['class_label_mapped'])\n    \n    # Verify encoding\n    print(\"\\nLabel encoding verification:\")\n    unique_vals, counts = np.unique(y_train, return_counts=True)\n    for val, count in zip(unique_vals, counts):\n        original_label = label_encoder.inverse_transform([val])[0]\n        print(f\"  {original_label} -> {val}: {count} samples ({count/len(y_train)*100:.1f}%)\")\n    \n    # Extract features\n    metadata_cols = ['ticker_id', 't', 'class_label', 'class_label_mapped', 'train_id', 'id']\n    feature_cols = [col for col in train.columns if col not in metadata_cols]\n    \n    X_train = train[feature_cols].values.astype(np.float32)\n    X_test = test[feature_cols].values.astype(np.float32)\n    \n    test_ids = test['id'].values if 'id' in test.columns else range(len(test))\n    \n    print(f\"\\nNumber of features: {len(feature_cols)}\")\n    \n    # Keep metadata\n    train_metadata = train[['ticker_id', 't']] if 'ticker_id' in train.columns else None\n    test_metadata = test[['ticker_id', 't']] if 'ticker_id' in test.columns else None\n    \n    # Define step order\n    step_order = [\n        StepType.IMPUTATION,\n        StepType.SCALING,\n        StepType.UNIVARIATE_OUTLIER,\n        StepType.PATTERN_FEATURES,\n        StepType.TIME_SERIES_FEATURES,\n        StepType.FEATURE_ENGINEERING,\n        StepType.FEATURE_REDUCTION,\n        StepType.MODELING\n    ]\n    \n    # Run grid search\n    print(\"\\n\" + \"=\"*60)\n    print(\"ENHANCED GRID SEARCH WITH NEURAL NETWORKS\")\n    print(\"=\"*60)\n    \n    search = SmartGridSearchEngine(\n        X_train, y_train, X_test,\n        train_df=train_metadata,\n        test_df=test_metadata,\n        max_runtime_minutes=90\n    )\n    \n    best_pipeline = search.run(step_order, max_iterations=40)\n    \n    # Generate predictions\n    if best_pipeline and X_test is not None:\n        print(\"\\n\" + \"=\"*60)\n        print(\"Generating predictions...\")\n        print(\"=\"*60)\n        \n        probabilities = best_pipeline.predict_proba(X_test, df=test_metadata)\n        predictions = apply_dynamic_thresholds(probabilities)\n        \n        predicted_labels = label_encoder.inverse_transform(predictions)\n        \n        submission = pd.DataFrame({\n            'id': test_ids,\n            'class_label': predicted_labels\n        })\n        \n        print(\"\\nSubmission distribution:\")\n        print(submission['class_label'].value_counts())\n        print(\"\\nSubmission percentages:\")\n        print(submission['class_label'].value_counts(normalize=True))\n        \n        submission.to_csv('submission.csv', index=False)\n        print(\"\\n✓ Submission saved to submission.csv\")\n        \n        # Summary\n        print(\"\\n\" + \"=\"*60)\n        print(\"FINAL RESULTS SUMMARY\")\n        print(\"=\"*60)\n        print(f\"Best CV score: {search.best_score:.4f}\")\n        \n        # Top configurations\n        sorted_results = sorted(search.results, key=lambda x: x['cv_score'], reverse=True)\n        \n        print(\"\\nTop 10 configurations:\")\n        for i, result in enumerate(sorted_results[:10]):\n            gap_indicator = \"🔴\" if result['overfitting_gap'] > 0.3 else \"🟡\" if result['overfitting_gap'] > 0.15 else \"🟢\"\n            \n            # Extract model and reduction method\n            parts = result['description'].split(' → ')\n            reduction = [p for p in parts if 'feature_reduction' in p]\n            model = [p for p in parts if 'modeling' in p]\n            \n            print(f\"\\n{i+1}. CV: {result['cv_score']:.4f} | Train: {result['train_score']:.4f} | \"\n                  f\"Gap: {result['overfitting_gap']:.4f} {gap_indicator}\")\n            \n            if reduction:\n                print(f\"   Reduction: {reduction[0].split('_')[2]}\")\n            if model:\n                print(f\"   Model: {model[0].replace('modeling_', '')}\")\n    \n    print(\"\\nDone!\")\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-14T18:37:22.738558Z","iopub.execute_input":"2025-09-14T18:37:22.73892Z","execution_failed":"2025-09-14T19:16:58.818Z"}},"outputs":[],"execution_count":null}]}