{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":114135,"databundleVersionId":13695648,"sourceType":"competition"}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install pandas numpy scikit-learn==1.5.2 imbalanced-learn psutil umap-learn scikit-learn-extra xgboost lightgbm -q","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\nimport psutil\nimport os\nimport sys\nfrom abc import ABC, abstractmethod\nfrom dataclasses import dataclass, field\nfrom typing import Dict, List, Any, Optional, Tuple, Type\nimport hashlib\nimport json\nimport itertools\nimport random\nfrom collections import defaultdict, Counter\nfrom datetime import datetime\nimport pickle\nimport time\n\nfrom sklearn.preprocessing import StandardScaler, RobustScaler, MinMaxScaler, QuantileTransformer, PowerTransformer, Normalizer\nfrom sklearn.decomposition import PCA, TruncatedSVD, FastICA, SparsePCA, KernelPCA, IncrementalPCA, DictionaryLearning, NMF, FactorAnalysis\nfrom sklearn.manifold import Isomap, LocallyLinearEmbedding, SpectralEmbedding, MDS\nfrom sklearn.feature_selection import SelectKBest, mutual_info_classif, f_classif, chi2, VarianceThreshold, SelectPercentile, RFE\nfrom sklearn.ensemble import IsolationForest, RandomForestClassifier, ExtraTreesClassifier\nfrom sklearn.covariance import EllipticEnvelope\nfrom sklearn.neighbors import LocalOutlierFactor\nfrom sklearn.linear_model import LogisticRegression, RidgeClassifier, SGDClassifier\nfrom sklearn.model_selection import cross_val_score, StratifiedKFold\nfrom sklearn.metrics import f1_score, make_scorer, classification_report, confusion_matrix, balanced_accuracy_score, matthews_corrcoef\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom sklearn.impute import SimpleImputer, KNNImputer\nfrom sklearn.random_projection import GaussianRandomProjection, SparseRandomProjection\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis\nfrom sklearn_extra.cluster import KMedoids\nfrom imblearn.over_sampling import SMOTE, RandomOverSampler, ADASYN, BorderlineSMOTE, KMeansSMOTE, SVMSMOTE\nfrom imblearn.under_sampling import RandomUnderSampler, TomekLinks, EditedNearestNeighbours, RepeatedEditedNearestNeighbours, AllKNN, CondensedNearestNeighbour, OneSidedSelection, NeighbourhoodCleaningRule, NearMiss, InstanceHardnessThreshold\nfrom imblearn.combine import SMOTETomek, SMOTEENN\nimport umap\nimport xgboost as xgb\nimport lightgbm as lgb\nfrom scipy.stats import pearsonr, spearmanr\nfrom scipy.spatial.distance import cdist\nfrom sklearn.preprocessing import PolynomialFeatures\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nprint(f\"[{datetime.now()}] Starting imports complete\")\n\nSKIP_HIGH_MEMORY_OPERATIONS = False\nMEMORY_THRESHOLD_MB = 2000\nMAX_CONFIGURATIONS_TO_TEST = 8\nEXPLORATION_RATE = 0.3\nBASE_DIR = '/kaggle/working'\nSUBMISSION_DIR = f'{BASE_DIR}/submissions'\nRESULTS_DIR = f'{BASE_DIR}/results'\nCONFIGS_DIR = f'{BASE_DIR}/configs'\nMIN_CLASS_RATIO = 0.02\nMAX_SINGLE_CLASS_RATIO = 0.98\n\nfor dir_path in [SUBMISSION_DIR, RESULTS_DIR, CONFIGS_DIR]:\n    if not os.path.exists(dir_path):\n        os.makedirs(dir_path)\n        print(f\"[{datetime.now()}] Created directory: {dir_path}\")\n\nclass MemoryManager:\n    @staticmethod\n    def get_memory_usage():\n        process = psutil.Process()\n        return process.memory_info().rss / 1024 / 1024\n    \n    @staticmethod\n    def get_available_memory():\n        return psutil.virtual_memory().available / 1024 / 1024\n    \n    @staticmethod\n    def get_total_memory():\n        return psutil.virtual_memory().total / 1024 / 1024\n    \n    @staticmethod\n    def estimate_array_memory(shape, dtype=np.float64):\n        bytes_per_element = np.dtype(dtype).itemsize\n        total_bytes = np.prod(shape) * bytes_per_element\n        return total_bytes / 1024 / 1024\n    \n    @staticmethod\n    def clean_memory():\n        gc.collect()\n    \n    @staticmethod\n    def check_memory_for_operation(estimated_mb, safety_factor=1.5):\n        available = MemoryManager.get_available_memory()\n        required = estimated_mb * safety_factor\n        return available > required, available, required\n\n@dataclass\nclass ProcessingData:\n    X: np.ndarray\n    y: np.ndarray\n    feature_names: Optional[List[str]] = None\n    metadata: Dict[str, Any] = field(default_factory=dict)\n    \n    def copy(self):\n        return ProcessingData(\n            X=self.X.copy(),\n            y=self.y.copy() if self.y is not None else None,\n            feature_names=self.feature_names.copy() if self.feature_names else None,\n            metadata=self.metadata.copy()\n        )\n    \n    def get_memory_usage(self):\n        x_memory = self.X.nbytes / 1024 / 1024\n        y_memory = self.y.nbytes / 1024 / 1024 if self.y is not None else 0\n        return x_memory + y_memory\n\n@dataclass\nclass StepConfig:\n    step_type: str\n    method: str\n    params: Dict[str, Any] = field(default_factory=dict)\n    enabled: bool = True\n    memory_limit_mb: Optional[float] = None\n    \n    def to_dict(self):\n        return {\n            'type': self.step_type,\n            'method': self.method,\n            'params': self.params,\n            'enabled': self.enabled,\n            'memory_limit_mb': self.memory_limit_mb\n        }\n    \n    def __hash__(self):\n        return hash((self.step_type, self.method, json.dumps(self.params, sort_keys=True)))\n\n@dataclass\nclass PipelineConfig:\n    steps: List[StepConfig]\n    config_id: Optional[str] = None\n    skip_high_memory: bool = False\n    memory_threshold_mb: float = 2000\n    \n    def generate_id(self):\n        config_str = json.dumps([s.to_dict() for s in self.steps], sort_keys=True)\n        self.config_id = hashlib.md5(config_str.encode()).hexdigest()[:12]\n        return self.config_id\n    \n    def __str__(self):\n        return ' -> '.join([f\"{s.step_type}:{s.method}\" for s in self.steps if s.enabled])\n\nclass BasePreprocessingStep(ABC):\n    STEP_TYPE = None\n    METHOD_NAME = None\n    \n    def __init__(self, config: StepConfig):\n        self.config = config\n        self.is_fitted = False\n        self._fitted_params = {}\n    \n    def estimate_memory_requirement(self, data: ProcessingData) -> float:\n        return data.get_memory_usage()\n    \n    @abstractmethod\n    def fit(self, data: ProcessingData) -> 'BasePreprocessingStep':\n        pass\n    \n    @abstractmethod\n    def transform(self, data: ProcessingData) -> ProcessingData:\n        pass\n    \n    def fit_transform(self, data: ProcessingData) -> ProcessingData:\n        self.fit(data)\n        return self.transform(data)\n\nclass StepRegistry:\n    _registry = defaultdict(dict)\n    \n    @classmethod\n    def register(cls, step_class: Type[BasePreprocessingStep]):\n        if step_class.STEP_TYPE and step_class.METHOD_NAME:\n            cls._registry[step_class.STEP_TYPE][step_class.METHOD_NAME] = step_class\n    \n    @classmethod\n    def get_step_class(cls, step_type: str, method: str) -> Optional[Type[BasePreprocessingStep]]:\n        return cls._registry.get(step_type, {}).get(method)\n    \n    @classmethod\n    def get_available_steps(cls) -> Dict[str, List[str]]:\n        return {step_type: list(methods.keys()) \n                for step_type, methods in cls._registry.items()}\n    \n    @classmethod\n    def create_step(cls, config: StepConfig) -> Optional[BasePreprocessingStep]:\n        step_class = cls.get_step_class(config.step_type, config.method)\n        if step_class:\n            return step_class(config)\n        return None\n\ndef register_step(cls):\n    StepRegistry.register(cls)\n    return cls\n\n@register_step\nclass DowncastStep(BasePreprocessingStep):\n    STEP_TYPE = \"downcast\"\n    METHOD_NAME = \"auto\"\n    \n    def fit(self, data: ProcessingData) -> 'DowncastStep':\n        self.is_fitted = True\n        return self\n    \n    def transform(self, data: ProcessingData) -> ProcessingData:\n        result = data.copy()\n        original_memory = result.get_memory_usage()\n        \n        for i in range(result.X.shape[1]):\n            col = result.X[:, i]\n            try:\n                col_numeric = col.astype(float)\n                if np.all(np.equal(np.mod(col_numeric, 1), 0)) or np.all(np.isnan(col_numeric) | np.equal(np.mod(col_numeric, 1), 0)):\n                    col_int = col_numeric.astype(np.float32)\n                    min_val = np.nanmin(col_int) if not np.all(np.isnan(col_int)) else 0\n                    max_val = np.nanmax(col_int) if not np.all(np.isnan(col_int)) else 0\n                    \n                    result.X[:, i] = col_int.astype(np.float32)\n                else:\n                    col_float = col_numeric.astype(np.float32)\n                    if np.allclose(col_numeric[~np.isnan(col_numeric)], \n                                  col_float[~np.isnan(col_float)], \n                                  rtol=1e-6, equal_nan=True):\n                        result.X[:, i] = col_float\n                    else:\n                        result.X[:, i] = col_numeric\n            except:\n                pass\n        \n        if result.X.dtype == np.float64:\n            result.X = result.X.astype(np.float32)\n        \n        new_memory = result.get_memory_usage()\n        reduction_pct = (1 - new_memory/original_memory) * 100\n        result.metadata['memory_reduction'] = f\"{reduction_pct:.1f}%\"\n        return result\n\n@register_step\nclass ImputationStep(BasePreprocessingStep):\n    STEP_TYPE = \"impute\"\n    METHOD_NAME = \"simple\"\n    \n    def fit(self, data: ProcessingData) -> 'ImputationStep':\n        strategy = self.config.params.get('strategy', 'median')\n        fill_value = self.config.params.get('fill_value', 0)\n        X_clean = data.X.copy()\n        X_clean = np.where(np.isinf(X_clean), np.nan, X_clean)\n        \n        if strategy == 'constant':\n            self._fitted_params['imputer'] = SimpleImputer(strategy='constant', fill_value=fill_value)\n        else:\n            self._fitted_params['imputer'] = SimpleImputer(strategy=strategy)\n        \n        self._fitted_params['imputer'].fit(X_clean)\n        self.is_fitted = True\n        return self\n    \n    def transform(self, data: ProcessingData) -> ProcessingData:\n        if not self.is_fitted:\n            raise ValueError(\"Must fit before transform\")\n        \n        result = data.copy()\n        result.X = np.where(np.isinf(result.X), np.nan, result.X)\n        result.X = self._fitted_params['imputer'].transform(result.X)\n        result.X = np.nan_to_num(result.X, nan=0.0, posinf=0.0, neginf=0.0)\n        return result\n\n@register_step\nclass KNNImputationStep(BasePreprocessingStep):\n    STEP_TYPE = \"impute\"\n    METHOD_NAME = \"knn\"\n    \n    def fit(self, data: ProcessingData) -> 'KNNImputationStep':\n        n_neighbors = self.config.params.get('n_neighbors', 5)\n        X_clean = data.X.copy()\n        X_clean = np.where(np.isinf(X_clean), np.nan, X_clean)\n        \n        self._fitted_params['imputer'] = KNNImputer(n_neighbors=n_neighbors)\n        self._fitted_params['imputer'].fit(X_clean)\n        self.is_fitted = True\n        return self\n    \n    def transform(self, data: ProcessingData) -> ProcessingData:\n        if not self.is_fitted:\n            raise ValueError(\"Must fit before transform\")\n        \n        result = data.copy()\n        result.X = np.where(np.isinf(result.X), np.nan, result.X)\n        result.X = self._fitted_params['imputer'].transform(result.X)\n        result.X = np.nan_to_num(result.X, nan=0.0, posinf=0.0, neginf=0.0)\n        return result\n\n@register_step\nclass CausationFeatureEngineering(BasePreprocessingStep):\n    STEP_TYPE = \"feature_eng\"\n    METHOD_NAME = \"causation\"\n    \n    def fit(self, data: ProcessingData) -> 'CausationFeatureEngineering':\n        X_float = data.X.astype(np.float32)\n        n_samples, n_features = X_float.shape\n        \n        self._fitted_params['feature_means'] = np.nanmean(X_float, axis=0)\n        self._fitted_params['feature_stds'] = np.nanstd(X_float, axis=0)\n        \n        max_pairs = min(200, n_features)\n        sample_size = min(500, n_samples)\n        sample_indices = np.random.choice(n_samples, sample_size, replace=False)\n        X_sample = X_float[sample_indices]\n        \n        correlations = []\n        for i in range(min(max_pairs, n_features)):\n            for j in range(i+1, min(i+10, n_features)):\n                col1 = X_sample[:, i]\n                col2 = X_sample[:, j]\n                if np.std(col1) > 1e-10 and np.std(col2) > 1e-10:\n                    try:\n                        corr = np.abs(np.corrcoef(col1[~np.isnan(col1) & ~np.isnan(col2)], \n                                                  col2[~np.isnan(col1) & ~np.isnan(col2)])[0, 1])\n                        if not np.isnan(corr):\n                            correlations.append((i, j, corr))\n                    except:\n                        pass\n        \n        correlations.sort(key=lambda x: x[2], reverse=True)\n        self._fitted_params['top_correlations'] = correlations[:50]\n        \n        high_var_features = []\n        for i in range(n_features):\n            var = np.nanvar(X_float[:, i])\n            if var > 0.1:\n                high_var_features.append(i)\n        self._fitted_params['high_var_features'] = high_var_features[:100]\n        \n        self.is_fitted = True\n        return self\n    \n    def transform(self, data: ProcessingData) -> ProcessingData:\n        if not self.is_fitted:\n            raise ValueError(\"Must fit before transform\")\n        \n        result = data.copy()\n        X_float = result.X.astype(np.float32)\n        new_features = []\n        \n        for i, j, corr in self._fitted_params['top_correlations']:\n            if i < X_float.shape[1] and j < X_float.shape[1]:\n                interaction = X_float[:, i] * X_float[:, j]\n                new_features.append(interaction)\n                \n                ratio = np.divide(X_float[:, i], X_float[:, j] + 1e-10)\n                ratio = np.clip(ratio, -1e6, 1e6)\n                new_features.append(ratio)\n                \n                diff = X_float[:, i] - X_float[:, j]\n                new_features.append(diff)\n        \n        for feat_idx in self._fitted_params['high_var_features'][:20]:\n            if feat_idx < X_float.shape[1]:\n                squared = X_float[:, feat_idx] ** 2\n                squared = np.clip(squared, -1e10, 1e10)\n                new_features.append(squared)\n                \n                log_feat = np.log1p(np.abs(X_float[:, feat_idx]))\n                new_features.append(log_feat)\n        \n        for i in range(min(10, X_float.shape[1])):\n            rolling_mean = np.convolve(X_float[:, i], np.ones(3)/3, mode='same')\n            new_features.append(rolling_mean)\n        \n        if new_features:\n            new_features = np.column_stack(new_features)\n            new_features = np.nan_to_num(new_features, nan=0.0, posinf=1e6, neginf=-1e6)\n            result.X = np.hstack([X_float, new_features.astype(np.float32)])\n            \n            if result.feature_names:\n                new_names = [f\"causal_feat_{i}\" for i in range(new_features.shape[1])]\n                result.feature_names = result.feature_names + new_names\n        \n        return result\n\n@register_step\nclass ZScoreOutlier(BasePreprocessingStep):\n    STEP_TYPE = \"outlier\"\n    METHOD_NAME = \"zscore\"\n    \n    def fit(self, data: ProcessingData) -> 'ZScoreOutlier':\n        threshold = self.config.params.get('threshold', 3)\n        X_float = data.X.astype(np.float32)\n        self._fitted_params['means'] = np.nanmean(X_float, axis=0)\n        self._fitted_params['stds'] = np.nanstd(X_float, axis=0)\n        self._fitted_params['threshold'] = threshold\n        self.is_fitted = True\n        return self\n    \n    def transform(self, data: ProcessingData) -> ProcessingData:\n        if not self.is_fitted:\n            raise ValueError(\"Must fit before transform\")\n        \n        result = data.copy()\n        result.X = result.X.astype(np.float32)\n        \n        with np.errstate(divide='ignore', invalid='ignore'):\n            z_scores = np.abs((result.X - self._fitted_params['means']) / \n                            (self._fitted_params['stds'] + 1e-10))\n        \n        z_scores = np.nan_to_num(z_scores, nan=0.0, posinf=self._fitted_params['threshold'] + 1)\n        mask = z_scores > self._fitted_params['threshold']\n        \n        for i in range(result.X.shape[1]):\n            col_mask = mask[:, i]\n            if col_mask.any() and self._fitted_params['stds'][i] > 1e-10:\n                lower = self._fitted_params['means'][i] - self._fitted_params['threshold'] * self._fitted_params['stds'][i]\n                upper = self._fitted_params['means'][i] + self._fitted_params['threshold'] * self._fitted_params['stds'][i]\n                result.X[:, i] = np.clip(result.X[:, i], lower, upper)\n        \n        return result\n\n@register_step\nclass IQROutlier(BasePreprocessingStep):\n    STEP_TYPE = \"outlier\"\n    METHOD_NAME = \"iqr\"\n    \n    def fit(self, data: ProcessingData) -> 'IQROutlier':\n        multiplier = self.config.params.get('multiplier', 1.5)\n        X_float = data.X.astype(np.float32)\n        self._fitted_params['q1'] = np.nanpercentile(X_float, 25, axis=0)\n        self._fitted_params['q3'] = np.nanpercentile(X_float, 75, axis=0)\n        iqr = self._fitted_params['q3'] - self._fitted_params['q1']\n        self._fitted_params['lower'] = self._fitted_params['q1'] - multiplier * iqr\n        self._fitted_params['upper'] = self._fitted_params['q3'] + multiplier * iqr\n        self.is_fitted = True\n        return self\n    \n    def transform(self, data: ProcessingData) -> ProcessingData:\n        if not self.is_fitted:\n            raise ValueError(\"Must fit before transform\")\n        \n        result = data.copy()\n        result.X = result.X.astype(np.float32)\n        \n        for i in range(result.X.shape[1]):\n            result.X[:, i] = np.clip(result.X[:, i], \n                                    self._fitted_params['lower'][i],\n                                    self._fitted_params['upper'][i])\n        return result\n\n@register_step\nclass IsolationForestOutlier(BasePreprocessingStep):\n    STEP_TYPE = \"outlier\"\n    METHOD_NAME = \"isolation_forest\"\n    \n    def fit(self, data: ProcessingData) -> 'IsolationForestOutlier':\n        contamination = self.config.params.get('contamination', 0.1)\n        X_float = data.X.astype(np.float32)\n        \n        self._fitted_params['detector'] = IsolationForest(\n            contamination=contamination,\n            random_state=42,\n            n_jobs=-1\n        )\n        self._fitted_params['detector'].fit(X_float)\n        self._fitted_params['outlier_mask'] = self._fitted_params['detector'].predict(X_float) == -1\n        self.is_fitted = True\n        return self\n    \n    def transform(self, data: ProcessingData) -> ProcessingData:\n        if not self.is_fitted:\n            raise ValueError(\"Must fit before transform\")\n        \n        result = data.copy()\n        for i in range(result.X.shape[1]):\n            col_median = np.median(result.X[~self._fitted_params['outlier_mask'], i])\n            result.X[self._fitted_params['outlier_mask'], i] = col_median\n        return result\n\n@register_step\nclass StandardScalerStep(BasePreprocessingStep):\n    STEP_TYPE = \"scaling\"\n    METHOD_NAME = \"standard\"\n    \n    def fit(self, data: ProcessingData) -> 'StandardScalerStep':\n        X_float = data.X.astype(np.float32)\n        self._fitted_params['scaler'] = StandardScaler()\n        self._fitted_params['scaler'].fit(X_float)\n        self.is_fitted = True\n        return self\n    \n    def transform(self, data: ProcessingData) -> ProcessingData:\n        if not self.is_fitted:\n            raise ValueError(\"Must fit before transform\")\n        \n        result = data.copy()\n        result.X = result.X.astype(np.float32)\n        result.X = self._fitted_params['scaler'].transform(result.X)\n        result.X = np.nan_to_num(result.X, nan=0.0, posinf=3.0, neginf=-3.0)\n        return result\n\n@register_step\nclass RobustScalerStep(BasePreprocessingStep):\n    STEP_TYPE = \"scaling\"\n    METHOD_NAME = \"robust\"\n    \n    def fit(self, data: ProcessingData) -> 'RobustScalerStep':\n        X_float = data.X.astype(np.float32)\n        self._fitted_params['scaler'] = RobustScaler()\n        self._fitted_params['scaler'].fit(X_float)\n        self.is_fitted = True\n        return self\n    \n    def transform(self, data: ProcessingData) -> ProcessingData:\n        if not self.is_fitted:\n            raise ValueError(\"Must fit before transform\")\n        \n        result = data.copy()\n        result.X = result.X.astype(np.float32)\n        result.X = self._fitted_params['scaler'].transform(result.X)\n        result.X = np.nan_to_num(result.X, nan=0.0, posinf=3.0, neginf=-3.0)\n        return result\n\n@register_step\nclass MinMaxScalerStep(BasePreprocessingStep):\n    STEP_TYPE = \"scaling\"\n    METHOD_NAME = \"minmax\"\n    \n    def fit(self, data: ProcessingData) -> 'MinMaxScalerStep':\n        X_float = data.X.astype(np.float32)\n        self._fitted_params['scaler'] = MinMaxScaler()\n        self._fitted_params['scaler'].fit(X_float)\n        self.is_fitted = True\n        return self\n    \n    def transform(self, data: ProcessingData) -> ProcessingData:\n        if not self.is_fitted:\n            raise ValueError(\"Must fit before transform\")\n        \n        result = data.copy()\n        result.X = result.X.astype(np.float32)\n        result.X = self._fitted_params['scaler'].transform(result.X)\n        return result\n\n@register_step\nclass QuantileScalerStep(BasePreprocessingStep):\n    STEP_TYPE = \"scaling\"\n    METHOD_NAME = \"quantile\"\n    \n    def fit(self, data: ProcessingData) -> 'QuantileScalerStep':\n        X_float = data.X.astype(np.float32)\n        n_quantiles = min(self.config.params.get('n_quantiles', 1000), len(X_float))\n        output_distribution = self.config.params.get('output_distribution', 'uniform')\n        \n        self._fitted_params['scaler'] = QuantileTransformer(\n            n_quantiles=n_quantiles,\n            output_distribution=output_distribution,\n            random_state=42\n        )\n        self._fitted_params['scaler'].fit(X_float)\n        self.is_fitted = True\n        return self\n    \n    def transform(self, data: ProcessingData) -> ProcessingData:\n        if not self.is_fitted:\n            raise ValueError(\"Must fit before transform\")\n        \n        result = data.copy()\n        result.X = result.X.astype(np.float32)\n        result.X = self._fitted_params['scaler'].transform(result.X)\n        return result\n\n@register_step\nclass PCAReduction(BasePreprocessingStep):\n    STEP_TYPE = \"dim_reduce\"\n    METHOD_NAME = \"pca\"\n    \n    def estimate_memory_requirement(self, data: ProcessingData) -> float:\n        n_components = self.config.params.get('n_components', 100)\n        return MemoryManager.estimate_array_memory((data.X.shape[1], data.X.shape[1]), dtype=np.float32)\n    \n    def fit(self, data: ProcessingData) -> 'PCAReduction':\n        n_components = min(self.config.params.get('n_components', 100), min(data.X.shape) - 1)\n        \n        if n_components < 2:\n            self._fitted_params['reducer'] = None\n        else:\n            if data.X.shape[0] * data.X.shape[1] > 1e7:\n                self._fitted_params['reducer'] = IncrementalPCA(n_components=n_components, batch_size=min(1000, data.X.shape[0]))\n            else:\n                self._fitted_params['reducer'] = PCA(n_components=n_components, random_state=42)\n            \n            X_float = data.X.astype(np.float32)\n            self._fitted_params['reducer'].fit(X_float)\n        \n        self.is_fitted = True\n        return self\n    \n    def transform(self, data: ProcessingData) -> ProcessingData:\n        if not self.is_fitted or self._fitted_params['reducer'] is None:\n            return data.copy()\n        \n        result = data.copy()\n        result.X = result.X.astype(np.float32)\n        result.X = self._fitted_params['reducer'].transform(result.X)\n        result.feature_names = [f\"PC{i+1}\" for i in range(result.X.shape[1])]\n        MemoryManager.clean_memory()\n        return result\n\n@register_step\nclass TruncatedSVDReduction(BasePreprocessingStep):\n    STEP_TYPE = \"dim_reduce\"\n    METHOD_NAME = \"svd\"\n    \n    def fit(self, data: ProcessingData) -> 'TruncatedSVDReduction':\n        n_components = min(self.config.params.get('n_components', 100), min(data.X.shape) - 1)\n        \n        if n_components < 2:\n            self._fitted_params['reducer'] = None\n        else:\n            self._fitted_params['reducer'] = TruncatedSVD(n_components=n_components, random_state=42)\n            X_float = data.X.astype(np.float32)\n            self._fitted_params['reducer'].fit(X_float)\n        \n        self.is_fitted = True\n        return self\n    \n    def transform(self, data: ProcessingData) -> ProcessingData:\n        if not self.is_fitted or self._fitted_params['reducer'] is None:\n            return data.copy()\n        \n        result = data.copy()\n        result.X = result.X.astype(np.float32)\n        result.X = self._fitted_params['reducer'].transform(result.X)\n        result.feature_names = [f\"SVD{i+1}\" for i in range(result.X.shape[1])]\n        return result\n\n@register_step\nclass LDAReduction(BasePreprocessingStep):\n    STEP_TYPE = \"dim_reduce\"\n    METHOD_NAME = \"lda\"\n    \n    def fit(self, data: ProcessingData) -> 'LDAReduction':\n        n_classes = len(np.unique(data.y))\n        max_components = n_classes - 1\n        \n        n_components_param = self.config.params.get('n_components', None)\n        \n        if n_components_param is not None:\n            n_components = min(n_components_param, max_components)\n        else:\n            n_components = max_components\n        \n        if n_components < 1:\n            self._fitted_params['reducer'] = None\n        else:\n            self._fitted_params['reducer'] = LinearDiscriminantAnalysis(n_components=n_components)\n            X_float = data.X.astype(np.float32)\n            self._fitted_params['reducer'].fit(X_float, data.y)\n        \n        self.is_fitted = True\n        return self\n    \n    def transform(self, data: ProcessingData) -> ProcessingData:\n        if not self.is_fitted or self._fitted_params.get('reducer') is None:\n            return data.copy()\n        \n        result = data.copy()\n        result.X = result.X.astype(np.float32)\n        result.X = self._fitted_params['reducer'].transform(result.X)\n        result.feature_names = [f\"LDA{i+1}\" for i in range(result.X.shape[1])]\n        return result\n\n@register_step\nclass VarianceThresholdReduction(BasePreprocessingStep):\n    STEP_TYPE = \"dim_reduce\"\n    METHOD_NAME = \"variance_threshold\"\n    \n    def estimate_memory_requirement(self, data: ProcessingData) -> float:\n        return MemoryManager.estimate_array_memory((data.X.shape[1],), dtype=np.float32)\n    \n    def fit(self, data: ProcessingData) -> 'VarianceThresholdReduction':\n        threshold = self.config.params.get('threshold', 0.01)\n        chunk_size = 1000\n        variances = np.zeros(data.X.shape[1], dtype=np.float32)\n        \n        for i in range(0, data.X.shape[1], chunk_size):\n            end_idx = min(i + chunk_size, data.X.shape[1])\n            variances[i:end_idx] = np.var(data.X[:, i:end_idx].astype(np.float32), axis=0)\n        \n        self._fitted_params['keep_indices'] = np.where(variances > threshold)[0]\n        self.is_fitted = True\n        return self\n    \n    def transform(self, data: ProcessingData) -> ProcessingData:\n        if not self.is_fitted:\n            raise ValueError(\"Must fit before transform\")\n        \n        result = data.copy()\n        result.X = result.X[:, self._fitted_params['keep_indices']]\n        if result.feature_names:\n            result.feature_names = [result.feature_names[i] for i in self._fitted_params['keep_indices']]\n        return result\n\n@register_step\nclass SelectKBestReduction(BasePreprocessingStep):\n    STEP_TYPE = \"dim_reduce\"\n    METHOD_NAME = \"select_k_best\"\n    \n    def estimate_memory_requirement(self, data: ProcessingData) -> float:\n        return data.get_memory_usage() * 0.1\n    \n    def fit(self, data: ProcessingData) -> 'SelectKBestReduction':\n        k = min(self.config.params.get('k', 500), data.X.shape[1])\n        score_func = self.config.params.get('score_func', 'f_classif')\n        \n        if score_func == 'mutual_info':\n            func = mutual_info_classif\n        elif score_func == 'chi2':\n            func = chi2\n        else:\n            func = f_classif\n        \n        self._fitted_params['selector'] = SelectKBest(score_func=func, k=k)\n        X_float = data.X.astype(np.float32)\n        \n        if score_func == 'chi2':\n            X_float = X_float - X_float.min() + 0.001\n        \n        self._fitted_params['selector'].fit(X_float, data.y)\n        self.is_fitted = True\n        return self\n    \n    def transform(self, data: ProcessingData) -> ProcessingData:\n        if not self.is_fitted:\n            raise ValueError(\"Must fit before transform\")\n        \n        result = data.copy()\n        result.X = result.X.astype(np.float32)\n        \n        if self.config.params.get('score_func', 'f_classif') == 'chi2':\n            result.X = result.X - result.X.min() + 0.001\n        \n        result.X = self._fitted_params['selector'].transform(result.X)\n        if result.feature_names:\n            mask = self._fitted_params['selector'].get_support()\n            result.feature_names = [name for name, keep in zip(result.feature_names, mask) if keep]\n        return result\n\n@register_step\nclass SMOTEBalance(BasePreprocessingStep):\n    STEP_TYPE = \"balance\"\n    METHOD_NAME = \"smote\"\n    \n    def fit(self, data: ProcessingData) -> 'SMOTEBalance':\n        unique, counts = np.unique(data.y, return_counts=True)\n        min_samples = counts.min()\n        k_neighbors = min(self.config.params.get('k_neighbors', 5), min_samples - 1)\n        \n        if k_neighbors > 0:\n            self._fitted_params['balancer'] = SMOTE(k_neighbors=k_neighbors, random_state=42)\n        else:\n            self._fitted_params['balancer'] = RandomOverSampler(random_state=42)\n        \n        self.is_fitted = True\n        return self\n    \n    def transform(self, data: ProcessingData) -> ProcessingData:\n        if not self.is_fitted:\n            return data.copy()\n        \n        result = data.copy()\n        result.X = result.X.astype(np.float32)\n        \n        try:\n            result.X, result.y = self._fitted_params['balancer'].fit_resample(result.X, result.y)\n        except:\n            return data.copy()\n        \n        return result\n\n@register_step\nclass HybridBalance(BasePreprocessingStep):\n    STEP_TYPE = \"balance\"\n    METHOD_NAME = \"hybrid\"\n    \n    def fit(self, data: ProcessingData) -> 'HybridBalance':\n        unique, counts = np.unique(data.y, return_counts=True)\n        majority_class = unique[np.argmax(counts)]\n        target_count = int(np.median(counts))\n        \n        self._fitted_params['undersample'] = RandomUnderSampler(\n            sampling_strategy={majority_class: min(target_count * 2, counts[np.argmax(counts)])},\n            random_state=42\n        )\n        \n        min_samples = counts.min()\n        k_neighbors = min(5, min_samples - 1)\n        if k_neighbors > 0:\n            self._fitted_params['oversample'] = SMOTE(\n                sampling_strategy='not majority',\n                k_neighbors=k_neighbors,\n                random_state=42\n            )\n        else:\n            self._fitted_params['oversample'] = RandomOverSampler(random_state=42)\n        \n        self.is_fitted = True\n        return self\n    \n    def transform(self, data: ProcessingData) -> ProcessingData:\n        if not self.is_fitted:\n            return data.copy()\n        \n        result = data.copy()\n        result.X = result.X.astype(np.float32)\n        \n        try:\n            result.X, result.y = self._fitted_params['undersample'].fit_resample(result.X, result.y)\n            result.X, result.y = self._fitted_params['oversample'].fit_resample(result.X, result.y)\n        except:\n            return data.copy()\n        \n        return result\n\n@register_step\nclass RandomOversampleBalance(BasePreprocessingStep):\n    STEP_TYPE = \"balance\"\n    METHOD_NAME = \"random_oversample\"\n    \n    def fit(self, data: ProcessingData) -> 'RandomOversampleBalance':\n        self._fitted_params['balancer'] = RandomOverSampler(random_state=42)\n        self.is_fitted = True\n        return self\n    \n    def transform(self, data: ProcessingData) -> ProcessingData:\n        if not self.is_fitted:\n            return data.copy()\n        \n        result = data.copy()\n        result.X = result.X.astype(np.float32)\n        \n        try:\n            result.X, result.y = self._fitted_params['balancer'].fit_resample(result.X, result.y)\n        except:\n            return data.copy()\n        \n        return result\n\nclass MemoryAwarePipeline:\n    def __init__(self, config: PipelineConfig):\n        self.config = config\n        self.steps = []\n        self.mandatory_steps = []\n        self._initialize_steps()\n        self.execution_times = {}\n    \n    def _initialize_steps(self):\n        self.mandatory_steps = [\n            DowncastStep(StepConfig('downcast', 'auto')),\n            ImputationStep(StepConfig('impute', 'simple', {'strategy': 'median'}))\n        ]\n        \n        for step_config in self.config.steps:\n            if step_config.enabled:\n                step = StepRegistry.create_step(step_config)\n                if step:\n                    self.steps.append(step)\n    \n    def fit(self, X: np.ndarray, y: np.ndarray, feature_names: Optional[List[str]] = None):\n        print(f\"[{datetime.now()}] Pipeline fit started with shape {X.shape}\")\n        data = ProcessingData(X=X, y=y, feature_names=feature_names)\n        \n        for idx, step in enumerate(self.mandatory_steps):\n            step_name = f\"{step.config.step_type}:{step.config.method}\"\n            print(f\"[{datetime.now()}] Fitting mandatory step {idx+1}/{len(self.mandatory_steps)}: {step_name}\")\n            start_time = time.time()\n            step.fit(data)\n            data = step.transform(data)\n            self.execution_times[f\"mandatory_{step_name}\"] = time.time() - start_time\n            MemoryManager.clean_memory()\n            print(f\"[{datetime.now()}] Completed mandatory step {step_name}, shape: {data.X.shape}\")\n        \n        for idx, step in enumerate(self.steps):\n            step_name = f\"{step.config.step_type}:{step.config.method}\"\n            print(f\"[{datetime.now()}] Processing optional step {idx+1}/{len(self.steps)}: {step_name}\")\n            \n            if self.config.skip_high_memory:\n                estimated_mem = step.estimate_memory_requirement(data)\n                can_proceed, available, required = MemoryManager.check_memory_for_operation(estimated_mem)\n                \n                if not can_proceed or estimated_mem > self.config.memory_threshold_mb:\n                    print(f\"[{datetime.now()}] Skipping {step_name} due to memory constraints\")\n                    continue\n            \n            try:\n                print(f\"[{datetime.now()}] Fitting {step_name}\")\n                start_time = time.time()\n                step.fit(data)\n                data = step.transform(data)\n                self.execution_times[f\"optional_{step_name}\"] = time.time() - start_time\n                MemoryManager.clean_memory()\n                print(f\"[{datetime.now()}] Completed {step_name}, shape: {data.X.shape}\")\n            except Exception as e:\n                print(f\"[{datetime.now()}] Error in {step_name}: {str(e)}\")\n                continue\n        \n        self._final_shape = data.X.shape\n        print(f\"[{datetime.now()}] Pipeline fit completed with final shape: {self._final_shape}\")\n        return self\n    \n    def transform(self, X: np.ndarray, y: Optional[np.ndarray] = None, feature_names: Optional[List[str]] = None):\n        print(f\"[{datetime.now()}] Pipeline transform started with shape {X.shape}\")\n        data = ProcessingData(X=X, y=y if y is not None else np.zeros(X.shape[0]), feature_names=feature_names)\n        \n        for step in self.mandatory_steps:\n            data = step.transform(data)\n        \n        for step in self.steps:\n            try:\n                data = step.transform(data)\n            except:\n                continue\n        \n        print(f\"[{datetime.now()}] Pipeline transform completed with shape {data.X.shape}\")\n        \n        if y is None:\n            return data.X\n        return data.X, data.y\n    \n    def fit_transform(self, X: np.ndarray, y: np.ndarray, feature_names: Optional[List[str]] = None):\n        self.fit(X, y, feature_names)\n        return self.transform(X, y, feature_names)\n\nclass PredictionValidator:\n    @staticmethod\n    def is_valid_prediction(predictions, class_names):\n        unique_preds, counts = np.unique(predictions, return_counts=True)\n        total_preds = len(predictions)\n        \n        if len(unique_preds) == 1:\n            print(f\"[{datetime.now()}] Invalid: All predictions are single class\")\n            return False\n        \n        for pred, count in zip(unique_preds, counts):\n            ratio = count / total_preds\n            if ratio > MAX_SINGLE_CLASS_RATIO:\n                print(f\"[{datetime.now()}] Invalid: Class {pred} exceeds {MAX_SINGLE_CLASS_RATIO*100}% threshold ({ratio*100:.1f}%)\")\n                return False\n        \n        minority_classes = 0\n        for pred, count in zip(unique_preds, counts):\n            ratio = count / total_preds\n            if ratio < MIN_CLASS_RATIO:\n                minority_classes += 1\n        \n        if minority_classes == len(unique_preds) - 1:\n            print(f\"[{datetime.now()}] Warning: Highly imbalanced predictions detected\")\n        \n        return True\n\nclass ResultsManager:\n    @staticmethod\n    def save_config_details(config: PipelineConfig, config_index: int):\n        config_dir = os.path.join(CONFIGS_DIR, f\"config_{config_index:03d}_{config.config_id}\")\n        os.makedirs(config_dir, exist_ok=True)\n        \n        config_details = {\n            'config_id': config.config_id,\n            'config_index': config_index,\n            'timestamp': datetime.now().isoformat(),\n            'steps': [step.to_dict() for step in config.steps],\n            'skip_high_memory': config.skip_high_memory,\n            'memory_threshold_mb': config.memory_threshold_mb,\n            'pipeline_string': str(config)\n        }\n        \n        with open(os.path.join(config_dir, 'config.json'), 'w') as f:\n            json.dump(config_details, f, indent=2)\n        \n        return config_dir\n    \n    @staticmethod\n    def save_evaluation_results(config_dir: str, results: Dict[str, Any]):\n        eval_file = os.path.join(config_dir, 'evaluation_results.json')\n        \n        json_results = {}\n        for key, value in results.items():\n            if isinstance(value, np.ndarray):\n                json_results[key] = value.tolist()\n            elif isinstance(value, (np.float32, np.float64)):\n                json_results[key] = float(value)\n            elif isinstance(value, (np.int32, np.int64)):\n                json_results[key] = int(value)\n            elif isinstance(value, dict):\n                json_results[key] = ResultsManager._convert_dict_for_json(value)\n            else:\n                json_results[key] = value\n        \n        with open(eval_file, 'w') as f:\n            json.dump(json_results, f, indent=2)\n    \n    @staticmethod\n    def _convert_dict_for_json(d):\n        result = {}\n        for k, v in d.items():\n            if isinstance(k, (np.int32, np.int64)):\n                k = str(int(k))\n            elif isinstance(k, (np.float32, np.float64)):\n                k = str(float(k))\n            \n            if isinstance(v, np.ndarray):\n                result[k] = v.tolist()\n            elif isinstance(v, (np.float32, np.float64)):\n                result[k] = float(v)\n            elif isinstance(v, (np.int32, np.int64)):\n                result[k] = int(v)\n            elif isinstance(v, dict):\n                result[k] = ResultsManager._convert_dict_for_json(v)\n            else:\n                result[k] = v\n        return result\n    \n    @staticmethod\n    def save_model_predictions(config_dir: str, model_name: str, predictions: np.ndarray, \n                             probabilities: np.ndarray, test_ids: np.ndarray, class_names: List[str]):\n        model_dir = os.path.join(config_dir, model_name)\n        os.makedirs(model_dir, exist_ok=True)\n        \n        submission = pd.DataFrame({\n            'id': test_ids,\n            'class_label': [class_names[p] for p in predictions]\n        })\n        submission.to_csv(os.path.join(model_dir, 'submission.csv'), index=False)\n        \n        proba_df = pd.DataFrame(probabilities, columns=class_names)\n        proba_df['id'] = test_ids\n        proba_df.to_csv(os.path.join(model_dir, 'probabilities.csv'), index=False)\n        \n        return os.path.join(model_dir, 'submission.csv')\n\nclass EnsembleGenerator:\n    @staticmethod\n    def create_voting_ensemble(predictions_list, weights=None):\n        if weights is None:\n            weights = [1] * len(predictions_list)\n        \n        ensemble_predictions = []\n        for i in range(len(predictions_list[0])):\n            votes = []\n            for pred, weight in zip(predictions_list, weights):\n                votes.extend([pred[i]] * weight)\n            most_common = Counter(votes).most_common(1)[0][0]\n            ensemble_predictions.append(most_common)\n        return ensemble_predictions\n    \n    @staticmethod\n    def create_probability_ensemble(proba_list, weights=None):\n        if weights is None:\n            weights = np.ones(len(proba_list)) / len(proba_list)\n        else:\n            weights = np.array(weights) / np.sum(weights)\n        \n        weighted_proba = np.zeros_like(proba_list[0])\n        for proba, weight in zip(proba_list, weights):\n            weighted_proba += proba * weight\n        \n        return np.argmax(weighted_proba, axis=1)\n\nclass IntelligentGridSearch:\n    def __init__(self, X_train, y_train, X_test=None):\n        self.X_train = X_train\n        self.y_train = y_train\n        self.X_test = X_test\n        self.exploration_rate = EXPLORATION_RATE\n        self.memory_threshold_mb = MEMORY_THRESHOLD_MB\n        \n        self.results = []\n        self.config_performance = {}\n        self.step_performance = defaultdict(lambda: defaultdict(list))\n        self.all_predictions = {}\n        self.valid_predictions = {}\n        \n        self.best_config = None\n        self.best_score = -np.inf\n        self.best_model = None\n        self.best_pipeline = None\n        self.best_model_type = None\n        \n        self.start_time = time.time()\n        \n        print(f\"[{datetime.now()}] IntelligentGridSearch initialized\")\n        print(f\"[{datetime.now()}] Training data shape: {X_train.shape}\")\n        print(f\"[{datetime.now()}] Test data shape: {X_test.shape if X_test is not None else 'None'}\")\n        print(f\"[{datetime.now()}] Memory threshold: {MEMORY_THRESHOLD_MB}MB\")\n        print(f\"[{datetime.now()}] Available memory: {MemoryManager.get_available_memory():.1f}MB\")\n    \n    def generate_comprehensive_configs(self, n_configs=50):\n        configs = []\n        available_steps = StepRegistry.get_available_steps()\n        \n        step_orderings = [\n            ['feature_eng', 'outlier', 'scaling', 'dim_reduce', 'balance'],\n            ['outlier', 'feature_eng', 'scaling', 'dim_reduce', 'balance'],\n            ['outlier', 'scaling', 'feature_eng', 'dim_reduce', 'balance'],\n            ['scaling', 'outlier', 'dim_reduce', 'balance'],\n            ['dim_reduce', 'feature_eng', 'scaling', 'balance'],\n            ['dim_reduce', 'outlier', 'scaling', 'balance'],\n            ['feature_eng', 'dim_reduce', 'scaling'],\n            ['scaling', 'dim_reduce'],\n            ['dim_reduce', 'balance'],\n            ['feature_eng', 'scaling', 'balance'],\n        ]\n        \n        outlier_methods = ['zscore', 'iqr', 'isolation_forest', None]\n        scaling_methods = ['standard', 'robust', 'minmax', 'quantile', None]\n        dim_reduce_methods = ['pca', 'svd', 'lda', 'variance_threshold', 'select_k_best', None]\n        balance_methods = ['smote', 'hybrid', 'random_oversample', None]\n        feature_eng_methods = ['causation', None]\n        \n        for _ in range(n_configs):\n            ordering = random.choice(step_orderings)\n            steps = []\n            \n            for step_type in ordering:\n                if step_type == 'feature_eng':\n                    method = random.choice([m for m in feature_eng_methods if m])\n                    if method and random.random() > 0.5:\n                        steps.append(StepConfig(\n                            step_type='feature_eng',\n                            method=method,\n                            params={},\n                            enabled=True\n                        ))\n                \n                elif step_type == 'outlier':\n                    method = random.choice([m for m in outlier_methods if m])\n                    if method:\n                        params = self._generate_params_for_method('outlier', method)\n                        steps.append(StepConfig(\n                            step_type='outlier',\n                            method=method,\n                            params=params,\n                            enabled=True\n                        ))\n                \n                elif step_type == 'scaling':\n                    method = random.choice([m for m in scaling_methods if m])\n                    if method:\n                        params = self._generate_params_for_method('scaling', method)\n                        steps.append(StepConfig(\n                            step_type='scaling',\n                            method=method,\n                            params=params,\n                            enabled=True\n                        ))\n                \n                elif step_type == 'dim_reduce':\n                    method = random.choice([m for m in dim_reduce_methods if m])\n                    if method:\n                        params = self._generate_params_for_method('dim_reduce', method)\n                        steps.append(StepConfig(\n                            step_type='dim_reduce',\n                            method=method,\n                            params=params,\n                            enabled=True\n                        ))\n                \n                elif step_type == 'balance':\n                    method = random.choice([m for m in balance_methods if m])\n                    if method and random.random() > 0.3:\n                        params = self._generate_params_for_method('balance', method)\n                        steps.append(StepConfig(\n                            step_type='balance',\n                            method=method,\n                            params=params,\n                            enabled=True\n                        ))\n            \n            if steps:\n                config = PipelineConfig(\n                    steps=steps,\n                    skip_high_memory=False,\n                    memory_threshold_mb=self.memory_threshold_mb\n                )\n                config.generate_id()\n                configs.append(config)\n        \n        unique_configs = {}\n        for config in configs:\n            if config.config_id not in unique_configs:\n                unique_configs[config.config_id] = config\n        \n        return list(unique_configs.values())[:n_configs]\n    \n    def _generate_params_for_method(self, step_type, method):\n        params = {}\n        \n        if step_type == 'outlier':\n            if method == 'zscore':\n                params['threshold'] = random.choice([2.0, 2.5, 3.0, 3.5, 4.0])\n            elif method == 'iqr':\n                params['multiplier'] = random.choice([1.0, 1.5, 2.0, 2.5, 3.0])\n            elif method == 'isolation_forest':\n                params['contamination'] = random.choice([0.05, 0.1, 0.15, 0.2])\n        \n        elif step_type == 'scaling':\n            if method == 'quantile':\n                params['n_quantiles'] = random.choice([100, 500, 1000])\n                params['output_distribution'] = random.choice(['uniform', 'normal'])\n        \n        elif step_type == 'dim_reduce':\n            if method == 'pca':\n                params['n_components'] = random.choice([10, 20, 30, 50, 100, 150, 200])\n            elif method == 'svd':\n                params['n_components'] = random.choice([10, 20, 30, 50, 100, 150])\n            elif method == 'lda':\n                params['n_components'] = None\n            elif method == 'select_k_best':\n                params['k'] = random.choice([50, 100, 200, 500, 1000, 2000, 5000])\n                params['score_func'] = random.choice(['f_classif', 'mutual_info', 'chi2'])\n            elif method == 'variance_threshold':\n                params['threshold'] = random.choice([0.0, 0.0001, 0.001, 0.01, 0.05])\n        \n        elif step_type == 'balance':\n            if method in ['smote', 'hybrid']:\n                params['k_neighbors'] = random.choice([3, 5, 7, 9])\n        \n        return params\n    \n    def evaluate_config(self, config: PipelineConfig, config_index: int):\n        print(f\"\\n[{datetime.now()}] ========== Evaluating Config {config_index} ==========\")\n        print(f\"[{datetime.now()}] Config ID: {config.config_id}\")\n        print(f\"[{datetime.now()}] Pipeline: {str(config)}\")\n        \n        config_dir = ResultsManager.save_config_details(config, config_index)\n        \n        try:\n            print(f\"[{datetime.now()}] Creating pipeline...\")\n            pipeline = MemoryAwarePipeline(config)\n            \n            print(f\"[{datetime.now()}] Running fit_transform...\")\n            X_processed, y_processed = pipeline.fit_transform(self.X_train, self.y_train)\n            print(f\"[{datetime.now()}] Processed shape: {X_processed.shape}\")\n            \n            print(f\"[{datetime.now()}] Computing class weights...\")\n            unique_classes = np.unique(y_processed)\n            class_weights = compute_class_weight('balanced', classes=unique_classes, y=y_processed)\n            class_weight_dict = {int(c): float(w) for c, w in zip(unique_classes, class_weights)}\n            print(f\"[{datetime.now()}] Classes: {unique_classes}, Weights: {class_weight_dict}\")\n            \n            cv = StratifiedKFold(n_splits=3, shuffle=True, random_state=42)\n            f1_scorer = make_scorer(f1_score, average='macro')\n            \n            models_to_evaluate = []\n            \n            unique_y, counts_y = np.unique(y_processed, return_counts=True)\n            class_distribution = {int(k): int(v) for k, v in zip(unique_y, counts_y)}\n            \n            evaluation_results = {\n                'config_id': config.config_id,\n                'config_index': config_index,\n                'pipeline': str(config),\n                'shape_before': list(self.X_train.shape),\n                'shape_after': list(X_processed.shape),\n                'class_distribution': class_distribution,\n                'execution_times': pipeline.execution_times,\n                'models': {}\n            }\n            \n            print(f\"[{datetime.now()}] Training LogisticRegression...\")\n            lr_start = time.time()\n            lr_model = LogisticRegression(\n                max_iter=1000,\n                class_weight=class_weight_dict,\n                random_state=42,\n                solver='saga',\n                n_jobs=-1\n            )\n            lr_scores = cross_val_score(lr_model, X_processed, y_processed, cv=cv, scoring='f1_macro', n_jobs=-1)\n            lr_score = lr_scores.mean()\n            lr_model.fit(X_processed, y_processed)\n            lr_time = time.time() - lr_start\n            \n            print(f\"[{datetime.now()}] LogisticRegression CV F1: {lr_score:.4f} (time: {lr_time:.2f}s)\")\n            models_to_evaluate.append(('logistic', lr_model, lr_score))\n            evaluation_results['models']['logistic'] = {\n                'cv_scores': lr_scores.tolist(),\n                'mean_score': float(lr_score),\n                'training_time': float(lr_time)\n            }\n            \n            print(f\"[{datetime.now()}] Training XGBoost...\")\n            xgb_start = time.time()\n            scale_pos_weight = class_weight_dict.get(0, 1.0)\n            xgb_model = xgb.XGBClassifier(\n                n_estimators=200,\n                max_depth=8,\n                learning_rate=0.1,\n                objective='multi:softprob',\n                scale_pos_weight=scale_pos_weight,\n                random_state=42,\n                n_jobs=-1,\n                eval_metric='mlogloss',\n                use_label_encoder=False,\n                subsample=0.8,\n                colsample_bytree=0.8\n            )\n            xgb_scores = cross_val_score(xgb_model, X_processed, y_processed, cv=cv, scoring='f1_macro', n_jobs=-1)\n            xgb_score = xgb_scores.mean()\n            xgb_model.fit(X_processed, y_processed)\n            xgb_time = time.time() - xgb_start\n            \n            print(f\"[{datetime.now()}] XGBoost CV F1: {xgb_score:.4f} (time: {xgb_time:.2f}s)\")\n            models_to_evaluate.append(('xgboost', xgb_model, xgb_score))\n            evaluation_results['models']['xgboost'] = {\n                'cv_scores': xgb_scores.tolist(),\n                'mean_score': float(xgb_score),\n                'training_time': float(xgb_time)\n            }\n            \n            print(f\"[{datetime.now()}] Training LightGBM...\")\n            lgb_start = time.time()\n            lgb_model = lgb.LGBMClassifier(\n                n_estimators=200,\n                max_depth=8,\n                learning_rate=0.1,\n                objective='multiclass',\n                num_class=len(unique_classes),\n                class_weight='balanced',\n                random_state=42,\n                n_jobs=-1,\n                verbosity=-1,\n                subsample=0.8,\n                colsample_bytree=0.8\n            )\n            lgb_scores = cross_val_score(lgb_model, X_processed, y_processed, cv=cv, scoring='f1_macro', n_jobs=-1)\n            lgb_score = lgb_scores.mean()\n            lgb_model.fit(X_processed, y_processed)\n            lgb_time = time.time() - lgb_start\n            \n            print(f\"[{datetime.now()}] LightGBM CV F1: {lgb_score:.4f} (time: {lgb_time:.2f}s)\")\n            models_to_evaluate.append(('lightgbm', lgb_model, lgb_score))\n            evaluation_results['models']['lightgbm'] = {\n                'cv_scores': lgb_scores.tolist(),\n                'mean_score': float(lgb_score),\n                'training_time': float(lgb_time)\n            }\n            \n            ResultsManager.save_evaluation_results(config_dir, evaluation_results)\n            \n            if self.X_test is not None:\n                print(f\"[{datetime.now()}] Transforming test data...\")\n                X_test_transformed = pipeline.transform(self.X_test)\n                print(f\"[{datetime.now()}] Test shape after transform: {X_test_transformed.shape}\")\n                \n                for model_name, model, score in models_to_evaluate:\n                    print(f\"[{datetime.now()}] Making predictions for {model_name}...\")\n                    predictions = model.predict(X_test_transformed)\n                    probabilities = model.predict_proba(X_test_transformed)\n                    \n                    model_key = f\"{config.config_id}_{model_name}\"\n                    \n                    class_names_temp = ['None', 'L', 'H']\n                    if PredictionValidator.is_valid_prediction(predictions, class_names_temp):\n                        self.valid_predictions[model_key] = {\n                            'predictions': predictions,\n                            'probabilities': probabilities,\n                            'score': score,\n                            'config': config,\n                            'model': model,\n                            'model_type': model_name,\n                            'pipeline': pipeline,\n                            'config_index': config_index,\n                            'config_dir': config_dir\n                        }\n                        \n                        if score > self.best_score:\n                            print(f\"[{datetime.now()}] NEW BEST VALID SCORE: {score:.4f} ({model_name})\")\n                            self.best_score = score\n                            self.best_config = config\n                            self.best_model = model\n                            self.best_pipeline = pipeline\n                            self.best_model_type = model_name\n                    else:\n                        print(f\"[{datetime.now()}] Skipping invalid predictions for {model_name}\")\n                    \n                    self.all_predictions[model_key] = {\n                        'predictions': predictions,\n                        'probabilities': probabilities,\n                        'score': score,\n                        'config': config,\n                        'model': model,\n                        'model_type': model_name,\n                        'pipeline': pipeline,\n                        'config_index': config_index,\n                        'config_dir': config_dir\n                    }\n            \n            MemoryManager.clean_memory()\n            \n            result = {\n                'config': config,\n                'scores': {'logistic': lr_score, 'xgboost': xgb_score, 'lightgbm': lgb_score},\n                'best_score': max(lr_score, xgb_score, lgb_score),\n                'models': models_to_evaluate,\n                'pipeline': pipeline,\n                'shape': X_processed.shape,\n                'memory_used': MemoryManager.get_memory_usage(),\n                'config_dir': config_dir\n            }\n            \n            for step in config.steps:\n                if step.enabled:\n                    self.step_performance[step.step_type][step.method].append(result['best_score'])\n            \n            print(f\"[{datetime.now()}] Config evaluation completed successfully\")\n            print(f\"[{datetime.now()}] Best score for this config: {result['best_score']:.4f}\")\n            print(f\"[{datetime.now()}] Memory usage: {result['memory_used']:.1f}MB\")\n            print(f\"[{datetime.now()}] Time elapsed: {(time.time() - self.start_time)/60:.1f} minutes\")\n            \n            return result\n            \n        except Exception as e:\n            print(f\"[{datetime.now()}] ERROR evaluating config: {str(e)}\")\n            import traceback\n            traceback.print_exc()\n            \n            error_details = {\n                'config_id': config.config_id,\n                'config_index': config_index,\n                'error': str(e),\n                'traceback': traceback.format_exc()\n            }\n            \n            with open(os.path.join(config_dir, 'error.json'), 'w') as f:\n                json.dump(error_details, f, indent=2)\n            \n            MemoryManager.clean_memory()\n            return None\n    \n    def run_search(self, n_iterations=None, test_ids=None, class_names=None):\n        if n_iterations is None:\n            n_iterations = MAX_CONFIGURATIONS_TO_TEST\n        \n        print(f\"\\n[{datetime.now()}] Starting grid search with {n_iterations} iterations\")\n        print(f\"[{datetime.now()}] Target runtime: >30 minutes\")\n        \n        configs = self.generate_comprehensive_configs(n_iterations * 2)\n        print(f\"[{datetime.now()}] Generated {len(configs)} unique configurations\")\n        \n        for i in range(min(n_iterations, len(configs))):\n            config = configs[i]\n            \n            print(f\"\\n[{datetime.now()}] Iteration {i+1}/{n_iterations}\")\n            print(f\"[{datetime.now()}] Time elapsed: {(time.time() - self.start_time)/60:.1f} minutes\")\n            \n            if config.config_id in self.config_performance:\n                print(f\"[{datetime.now()}] Config already evaluated, skipping\")\n                continue\n            \n            result = self.evaluate_config(config, i+1)\n            \n            if result:\n                self.results.append(result)\n                self.config_performance[config.config_id] = result['best_score']\n                \n                if test_ids is not None and class_names is not None:\n                    for model_key, pred_data in self.valid_predictions.items():\n                        if pred_data['config'].config_id == config.config_id:\n                            ResultsManager.save_model_predictions(\n                                pred_data['config_dir'],\n                                pred_data['model_type'],\n                                pred_data['predictions'],\n                                pred_data['probabilities'],\n                                test_ids,\n                                class_names\n                            )\n        \n        self.generate_final_summary(test_ids, class_names)\n        \n        print(f\"\\n[{datetime.now()}] Grid search completed\")\n        print(f\"[{datetime.now()}] Total time: {(time.time() - self.start_time)/60:.1f} minutes\")\n        print(f\"[{datetime.now()}] Best valid score: {self.best_score:.4f} (Model: {self.best_model_type})\")\n        print(f\"[{datetime.now()}] Total valid predictions: {len(self.valid_predictions)}\")\n        \n        return self.best_config, self.best_model, self.best_pipeline\n    \n    def generate_final_summary(self, test_ids, class_names):\n        summary_file = os.path.join(RESULTS_DIR, 'search_summary.json')\n        \n        summary = {\n            'total_time_minutes': (time.time() - self.start_time) / 60,\n            'configurations_tested': len(self.results),\n            'valid_predictions_count': len(self.valid_predictions),\n            'best_score': float(self.best_score) if self.best_score is not None else None,\n            'best_model_type': self.best_model_type,\n            'best_config_id': self.best_config.config_id if self.best_config else None,\n            'best_pipeline': str(self.best_config) if self.best_config else None,\n            'top_configurations': []\n        }\n        \n        sorted_results = sorted(self.results, key=lambda x: x['best_score'], reverse=True)[:10]\n        for rank, result in enumerate(sorted_results, 1):\n            summary['top_configurations'].append({\n                'rank': rank,\n                'config_id': result['config'].config_id,\n                'pipeline': str(result['config']),\n                'scores': {k: float(v) for k, v in result['scores'].items()},\n                'best_score': float(result['best_score']),\n                'shape': list(result['shape'])\n            })\n        \n        with open(summary_file, 'w') as f:\n            json.dump(summary, f, indent=2)\n        \n        print(f\"[{datetime.now()}] Summary saved to {summary_file}\")\n\ndef main():\n    print(f\"\\n[{datetime.now()}] ========== STARTING MAIN EXECUTION ==========\")\n    print(f\"[{datetime.now()}] Loading data...\")\n    \n    train = pd.read_csv('/kaggle/input/detecting-reversal-points-in-us-equities/competition_data/train.csv')\n    test = pd.read_csv('/kaggle/input/detecting-reversal-points-in-us-equities/competition_data/test.csv')\n    \n    print(f\"[{datetime.now()}] Train shape: {train.shape}\")\n    print(f\"[{datetime.now()}] Test shape: {test.shape}\")\n    \n    print(f\"[{datetime.now()}] Processing target labels...\")\n    target_mapping = {'HH': 'H', 'LH': 'H', 'HL': 'L', 'LL': 'L'}\n    train['class_label_mapped'] = train['class_label'].map(target_mapping).fillna('None')\n    \n    print(f\"[{datetime.now()}] Target distribution:\")\n    print(train['class_label_mapped'].value_counts())\n    \n    print(f\"[{datetime.now()}] Separating features and metadata...\")\n    metadata_cols = ['ticker_id', 't', 'class_label', 'class_label_mapped', 'train_id', 'id']\n    feature_cols = [col for col in train.columns if col not in metadata_cols]\n    print(f\"[{datetime.now()}] Number of features: {len(feature_cols)}\")\n    \n    print(f\"[{datetime.now()}] Creating arrays...\")\n    X_train = train[feature_cols].values.astype(np.float32)\n    y_train = pd.factorize(train['class_label_mapped'])[0]\n    X_test = test[feature_cols].values.astype(np.float32)\n    \n    print(f\"[{datetime.now()}] X_train shape: {X_train.shape}, dtype: {X_train.dtype}\")\n    print(f\"[{datetime.now()}] y_train shape: {y_train.shape}, unique values: {np.unique(y_train)}\")\n    print(f\"[{datetime.now()}] X_test shape: {X_test.shape}, dtype: {X_test.dtype}\")\n    \n    _, class_names = pd.factorize(train['class_label_mapped'])\n    class_names = list(class_names)\n    test_ids = test['id'].values if 'id' in test.columns else range(len(test))\n    \n    print(f\"[{datetime.now()}] Class names: {class_names}\")\n    print(f\"[{datetime.now()}] Initializing IntelligentGridSearch...\")\n    \n    search = IntelligentGridSearch(X_train, y_train, X_test)\n    \n    print(f\"[{datetime.now()}] Starting search...\")\n    best_config, best_model, best_pipeline = search.run_search(\n        n_iterations=MAX_CONFIGURATIONS_TO_TEST,\n        test_ids=test_ids,\n        class_names=class_names\n    )\n    \n    if best_model is not None:\n        print(f\"\\n[{datetime.now()}] Making final predictions...\")\n        X_test_transformed = best_pipeline.transform(X_test)\n        predictions = best_model.predict(X_test_transformed)\n        \n        final_predictions = [class_names[p] for p in predictions]\n        \n        if PredictionValidator.is_valid_prediction(predictions, class_names):\n            submission = pd.DataFrame({\n                'id': test_ids,\n                'class_label': final_predictions\n            })\n            \n            print(f\"[{datetime.now()}] Saving final submissions...\")\n            submission.to_csv('submission.csv', index=False)\n            final_path = os.path.join(SUBMISSION_DIR, f'FINAL_BEST_{search.best_model_type}_submission.csv')\n            submission.to_csv(final_path, index=False)\n            \n            print(f\"[{datetime.now()}] Final submission saved to submission.csv\")\n            print(f\"[{datetime.now()}] Best model type: {search.best_model_type}\")\n            print(f\"[{datetime.now()}] Prediction distribution:\")\n            print(pd.Series(final_predictions).value_counts())\n        else:\n            print(f\"[{datetime.now()}] ERROR: Final predictions are invalid - generating fallback predictions\")\n            \n            valid_configs = sorted(search.valid_predictions.items(), key=lambda x: x[1]['score'], reverse=True)\n            \n            if valid_configs:\n                best_valid = valid_configs[0][1]\n                X_test_transformed = best_valid['pipeline'].transform(X_test)\n                predictions = best_valid['model'].predict(X_test_transformed)\n                final_predictions = [class_names[p] for p in predictions]\n                \n                submission = pd.DataFrame({\n                    'id': test_ids,\n                    'class_label': final_predictions\n                })\n                \n                submission.to_csv('submission.csv', index=False)\n                print(f\"[{datetime.now()}] Used fallback valid model: {best_valid['model_type']}\")\n                print(f\"[{datetime.now()}] Fallback score: {best_valid['score']:.4f}\")\n            else:\n                print(f\"[{datetime.now()}] ERROR: No valid predictions found in entire search\")\n    else:\n        print(f\"[{datetime.now()}] ERROR: No successful model found\")\n    \n    print(f\"\\n[{datetime.now()}] ========== EXECUTION COMPLETED ==========\")\n    print(f\"[{datetime.now()}] Total runtime: {(time.time() - search.start_time)/60:.1f} minutes\")\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# !pip install pandas numpy scikit-learn==1.5.2 imbalanced-learn psutil umap-learn scikit-learn-extra xgboost lightgbm -q\n\n# import pandas as pd\n# import numpy as np\n# import gc\n# import psutil\n# import os\n# import sys\n# from abc import ABC, abstractmethod\n# from dataclasses import dataclass, field\n# from typing import Dict, List, Any, Optional, Tuple, Type\n# import hashlib\n# import json\n# import itertools\n# import random\n# from collections import defaultdict, Counter\n# from datetime import datetime\n# import pickle\n# import time\n\n# from sklearn.preprocessing import StandardScaler, RobustScaler, MinMaxScaler, QuantileTransformer, PowerTransformer, Normalizer\n# from sklearn.decomposition import PCA, TruncatedSVD, FastICA, SparsePCA, KernelPCA, IncrementalPCA, DictionaryLearning, NMF, FactorAnalysis\n# from sklearn.manifold import Isomap, LocallyLinearEmbedding, SpectralEmbedding, MDS\n# from sklearn.feature_selection import SelectKBest, mutual_info_classif, f_classif, chi2, VarianceThreshold, SelectPercentile, RFE\n# from sklearn.ensemble import IsolationForest, RandomForestClassifier, ExtraTreesClassifier\n# from sklearn.covariance import EllipticEnvelope\n# from sklearn.neighbors import LocalOutlierFactor\n# from sklearn.linear_model import LogisticRegression, RidgeClassifier, SGDClassifier\n# from sklearn.model_selection import cross_val_score, StratifiedKFold\n# from sklearn.metrics import f1_score, make_scorer, classification_report, confusion_matrix, balanced_accuracy_score, matthews_corrcoef\n# from sklearn.utils.class_weight import compute_class_weight\n# from sklearn.impute import SimpleImputer, KNNImputer\n# from sklearn.random_projection import GaussianRandomProjection, SparseRandomProjection\n# from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n# from sklearn_extra.cluster import KMedoids\n# from imblearn.over_sampling import SMOTE, RandomOverSampler, ADASYN, BorderlineSMOTE, KMeansSMOTE, SVMSMOTE\n# from imblearn.under_sampling import RandomUnderSampler, TomekLinks, EditedNearestNeighbours, RepeatedEditedNearestNeighbours, AllKNN, CondensedNearestNeighbour, OneSidedSelection, NeighbourhoodCleaningRule, NearMiss, InstanceHardnessThreshold\n# from imblearn.combine import SMOTETomek, SMOTEENN\n# import umap\n# import xgboost as xgb\n# import lightgbm as lgb\n\n# import warnings\n# warnings.filterwarnings('ignore')\n\n# print(f\"[{datetime.now()}] Starting imports complete\")\n\n# # Configuration Parameters\n# SKIP_HIGH_MEMORY_OPERATIONS = False  # Allow high memory operations\n# MEMORY_THRESHOLD_MB = 2000  # Increased memory threshold\n# MAX_CONFIGURATIONS_TO_TEST = 5  # Test more configurations\n# EXPLORATION_RATE = 0.3\n# BASE_DIR = '/kaggle/working'\n# SUBMISSION_DIR = f'{BASE_DIR}/submissions'\n# RESULTS_DIR = f'{BASE_DIR}/results'\n# CONFIGS_DIR = f'{BASE_DIR}/configs'\n\n# # Create all necessary directories\n# for dir_path in [SUBMISSION_DIR, RESULTS_DIR, CONFIGS_DIR]:\n#     if not os.path.exists(dir_path):\n#         os.makedirs(dir_path)\n#         print(f\"[{datetime.now()}] Created directory: {dir_path}\")\n\n# class MemoryManager:\n#     @staticmethod\n#     def get_memory_usage():\n#         process = psutil.Process()\n#         return process.memory_info().rss / 1024 / 1024\n    \n#     @staticmethod\n#     def get_available_memory():\n#         return psutil.virtual_memory().available / 1024 / 1024\n    \n#     @staticmethod\n#     def get_total_memory():\n#         return psutil.virtual_memory().total / 1024 / 1024\n    \n#     @staticmethod\n#     def estimate_array_memory(shape, dtype=np.float64):\n#         bytes_per_element = np.dtype(dtype).itemsize\n#         total_bytes = np.prod(shape) * bytes_per_element\n#         return total_bytes / 1024 / 1024\n    \n#     @staticmethod\n#     def clean_memory():\n#         gc.collect()\n    \n#     @staticmethod\n#     def check_memory_for_operation(estimated_mb, safety_factor=1.5):\n#         available = MemoryManager.get_available_memory()\n#         required = estimated_mb * safety_factor\n#         return available > required, available, required\n\n# @dataclass\n# class ProcessingData:\n#     X: np.ndarray\n#     y: np.ndarray\n#     feature_names: Optional[List[str]] = None\n#     metadata: Dict[str, Any] = field(default_factory=dict)\n    \n#     def copy(self):\n#         return ProcessingData(\n#             X=self.X.copy(),\n#             y=self.y.copy() if self.y is not None else None,\n#             feature_names=self.feature_names.copy() if self.feature_names else None,\n#             metadata=self.metadata.copy()\n#         )\n    \n#     def get_memory_usage(self):\n#         x_memory = self.X.nbytes / 1024 / 1024\n#         y_memory = self.y.nbytes / 1024 / 1024 if self.y is not None else 0\n#         return x_memory + y_memory\n\n# @dataclass\n# class StepConfig:\n#     step_type: str\n#     method: str\n#     params: Dict[str, Any] = field(default_factory=dict)\n#     enabled: bool = True\n#     memory_limit_mb: Optional[float] = None\n    \n#     def to_dict(self):\n#         return {\n#             'type': self.step_type,\n#             'method': self.method,\n#             'params': self.params,\n#             'enabled': self.enabled,\n#             'memory_limit_mb': self.memory_limit_mb\n#         }\n    \n#     def __hash__(self):\n#         return hash((self.step_type, self.method, json.dumps(self.params, sort_keys=True)))\n\n# @dataclass\n# class PipelineConfig:\n#     steps: List[StepConfig]\n#     config_id: Optional[str] = None\n#     skip_high_memory: bool = False\n#     memory_threshold_mb: float = 2000\n    \n#     def generate_id(self):\n#         config_str = json.dumps([s.to_dict() for s in self.steps], sort_keys=True)\n#         self.config_id = hashlib.md5(config_str.encode()).hexdigest()[:12]\n#         return self.config_id\n    \n#     def __str__(self):\n#         return ' -> '.join([f\"{s.step_type}:{s.method}\" for s in self.steps if s.enabled])\n\n# class BasePreprocessingStep(ABC):\n#     STEP_TYPE = None\n#     METHOD_NAME = None\n    \n#     def __init__(self, config: StepConfig):\n#         self.config = config\n#         self.is_fitted = False\n#         self._fitted_params = {}\n    \n#     def estimate_memory_requirement(self, data: ProcessingData) -> float:\n#         return data.get_memory_usage()\n    \n#     @abstractmethod\n#     def fit(self, data: ProcessingData) -> 'BasePreprocessingStep':\n#         pass\n    \n#     @abstractmethod\n#     def transform(self, data: ProcessingData) -> ProcessingData:\n#         pass\n    \n#     def fit_transform(self, data: ProcessingData) -> ProcessingData:\n#         self.fit(data)\n#         return self.transform(data)\n\n# class StepRegistry:\n#     _registry = defaultdict(dict)\n    \n#     @classmethod\n#     def register(cls, step_class: Type[BasePreprocessingStep]):\n#         if step_class.STEP_TYPE and step_class.METHOD_NAME:\n#             cls._registry[step_class.STEP_TYPE][step_class.METHOD_NAME] = step_class\n    \n#     @classmethod\n#     def get_step_class(cls, step_type: str, method: str) -> Optional[Type[BasePreprocessingStep]]:\n#         return cls._registry.get(step_type, {}).get(method)\n    \n#     @classmethod\n#     def get_available_steps(cls) -> Dict[str, List[str]]:\n#         return {step_type: list(methods.keys()) \n#                 for step_type, methods in cls._registry.items()}\n    \n#     @classmethod\n#     def create_step(cls, config: StepConfig) -> Optional[BasePreprocessingStep]:\n#         step_class = cls.get_step_class(config.step_type, config.method)\n#         if step_class:\n#             return step_class(config)\n#         return None\n\n# def register_step(cls):\n#     StepRegistry.register(cls)\n#     return cls\n\n# # Preprocessing Steps\n# @register_step\n# class DowncastStep(BasePreprocessingStep):\n#     STEP_TYPE = \"downcast\"\n#     METHOD_NAME = \"auto\"\n    \n#     def fit(self, data: ProcessingData) -> 'DowncastStep':\n#         self.is_fitted = True\n#         return self\n    \n#     def transform(self, data: ProcessingData) -> ProcessingData:\n#         result = data.copy()\n#         original_memory = result.get_memory_usage()\n        \n#         for i in range(result.X.shape[1]):\n#             col = result.X[:, i]\n#             try:\n#                 col_numeric = col.astype(float)\n#                 if np.all(np.equal(np.mod(col_numeric, 1), 0)) or np.all(np.isnan(col_numeric) | np.equal(np.mod(col_numeric, 1), 0)):\n#                     col_int = col_numeric.astype(np.float32)\n#                     min_val = np.nanmin(col_int) if not np.all(np.isnan(col_int)) else 0\n#                     max_val = np.nanmax(col_int) if not np.all(np.isnan(col_int)) else 0\n                    \n#                     result.X[:, i] = col_int.astype(np.float32)\n#                 else:\n#                     col_float = col_numeric.astype(np.float32)\n#                     if np.allclose(col_numeric[~np.isnan(col_numeric)], \n#                                   col_float[~np.isnan(col_float)], \n#                                   rtol=1e-6, equal_nan=True):\n#                         result.X[:, i] = col_float\n#                     else:\n#                         result.X[:, i] = col_numeric\n#             except:\n#                 pass\n        \n#         if result.X.dtype == np.float64:\n#             result.X = result.X.astype(np.float32)\n        \n#         new_memory = result.get_memory_usage()\n#         reduction_pct = (1 - new_memory/original_memory) * 100\n#         result.metadata['memory_reduction'] = f\"{reduction_pct:.1f}%\"\n#         return result\n\n# @register_step\n# class ImputationStep(BasePreprocessingStep):\n#     STEP_TYPE = \"impute\"\n#     METHOD_NAME = \"simple\"\n    \n#     def fit(self, data: ProcessingData) -> 'ImputationStep':\n#         strategy = self.config.params.get('strategy', 'median')\n#         fill_value = self.config.params.get('fill_value', 0)\n#         X_clean = data.X.copy()\n#         X_clean = np.where(np.isinf(X_clean), np.nan, X_clean)\n        \n#         if strategy == 'constant':\n#             self._fitted_params['imputer'] = SimpleImputer(strategy='constant', fill_value=fill_value)\n#         else:\n#             self._fitted_params['imputer'] = SimpleImputer(strategy=strategy)\n        \n#         self._fitted_params['imputer'].fit(X_clean)\n#         self.is_fitted = True\n#         return self\n    \n#     def transform(self, data: ProcessingData) -> ProcessingData:\n#         if not self.is_fitted:\n#             raise ValueError(\"Must fit before transform\")\n        \n#         result = data.copy()\n#         result.X = np.where(np.isinf(result.X), np.nan, result.X)\n#         result.X = self._fitted_params['imputer'].transform(result.X)\n#         result.X = np.nan_to_num(result.X, nan=0.0, posinf=0.0, neginf=0.0)\n#         return result\n\n# @register_step\n# class KNNImputationStep(BasePreprocessingStep):\n#     STEP_TYPE = \"impute\"\n#     METHOD_NAME = \"knn\"\n    \n#     def fit(self, data: ProcessingData) -> 'KNNImputationStep':\n#         n_neighbors = self.config.params.get('n_neighbors', 5)\n#         X_clean = data.X.copy()\n#         X_clean = np.where(np.isinf(X_clean), np.nan, X_clean)\n        \n#         self._fitted_params['imputer'] = KNNImputer(n_neighbors=n_neighbors)\n#         self._fitted_params['imputer'].fit(X_clean)\n#         self.is_fitted = True\n#         return self\n    \n#     def transform(self, data: ProcessingData) -> ProcessingData:\n#         if not self.is_fitted:\n#             raise ValueError(\"Must fit before transform\")\n        \n#         result = data.copy()\n#         result.X = np.where(np.isinf(result.X), np.nan, result.X)\n#         result.X = self._fitted_params['imputer'].transform(result.X)\n#         result.X = np.nan_to_num(result.X, nan=0.0, posinf=0.0, neginf=0.0)\n#         return result\n\n# @register_step\n# class ZScoreOutlier(BasePreprocessingStep):\n#     STEP_TYPE = \"outlier\"\n#     METHOD_NAME = \"zscore\"\n    \n#     def fit(self, data: ProcessingData) -> 'ZScoreOutlier':\n#         threshold = self.config.params.get('threshold', 3)\n#         X_float = data.X.astype(np.float32)\n#         self._fitted_params['means'] = np.nanmean(X_float, axis=0)\n#         self._fitted_params['stds'] = np.nanstd(X_float, axis=0)\n#         self._fitted_params['threshold'] = threshold\n#         self.is_fitted = True\n#         return self\n    \n#     def transform(self, data: ProcessingData) -> ProcessingData:\n#         if not self.is_fitted:\n#             raise ValueError(\"Must fit before transform\")\n        \n#         result = data.copy()\n#         result.X = result.X.astype(np.float32)\n        \n#         with np.errstate(divide='ignore', invalid='ignore'):\n#             z_scores = np.abs((result.X - self._fitted_params['means']) / \n#                             (self._fitted_params['stds'] + 1e-10))\n        \n#         z_scores = np.nan_to_num(z_scores, nan=0.0, posinf=self._fitted_params['threshold'] + 1)\n#         mask = z_scores > self._fitted_params['threshold']\n        \n#         for i in range(result.X.shape[1]):\n#             col_mask = mask[:, i]\n#             if col_mask.any() and self._fitted_params['stds'][i] > 1e-10:\n#                 lower = self._fitted_params['means'][i] - self._fitted_params['threshold'] * self._fitted_params['stds'][i]\n#                 upper = self._fitted_params['means'][i] + self._fitted_params['threshold'] * self._fitted_params['stds'][i]\n#                 result.X[:, i] = np.clip(result.X[:, i], lower, upper)\n        \n#         return result\n\n# @register_step\n# class IQROutlier(BasePreprocessingStep):\n#     STEP_TYPE = \"outlier\"\n#     METHOD_NAME = \"iqr\"\n    \n#     def fit(self, data: ProcessingData) -> 'IQROutlier':\n#         multiplier = self.config.params.get('multiplier', 1.5)\n#         X_float = data.X.astype(np.float32)\n#         self._fitted_params['q1'] = np.nanpercentile(X_float, 25, axis=0)\n#         self._fitted_params['q3'] = np.nanpercentile(X_float, 75, axis=0)\n#         iqr = self._fitted_params['q3'] - self._fitted_params['q1']\n#         self._fitted_params['lower'] = self._fitted_params['q1'] - multiplier * iqr\n#         self._fitted_params['upper'] = self._fitted_params['q3'] + multiplier * iqr\n#         self.is_fitted = True\n#         return self\n    \n#     def transform(self, data: ProcessingData) -> ProcessingData:\n#         if not self.is_fitted:\n#             raise ValueError(\"Must fit before transform\")\n        \n#         result = data.copy()\n#         result.X = result.X.astype(np.float32)\n        \n#         for i in range(result.X.shape[1]):\n#             result.X[:, i] = np.clip(result.X[:, i], \n#                                     self._fitted_params['lower'][i],\n#                                     self._fitted_params['upper'][i])\n#         return result\n\n# @register_step\n# class IsolationForestOutlier(BasePreprocessingStep):\n#     STEP_TYPE = \"outlier\"\n#     METHOD_NAME = \"isolation_forest\"\n    \n#     def fit(self, data: ProcessingData) -> 'IsolationForestOutlier':\n#         contamination = self.config.params.get('contamination', 0.1)\n#         X_float = data.X.astype(np.float32)\n        \n#         self._fitted_params['detector'] = IsolationForest(\n#             contamination=contamination,\n#             random_state=42,\n#             n_jobs=-1\n#         )\n#         self._fitted_params['detector'].fit(X_float)\n#         self._fitted_params['outlier_mask'] = self._fitted_params['detector'].predict(X_float) == -1\n#         self.is_fitted = True\n#         return self\n    \n#     def transform(self, data: ProcessingData) -> ProcessingData:\n#         if not self.is_fitted:\n#             raise ValueError(\"Must fit before transform\")\n        \n#         result = data.copy()\n#         # Simply mark outliers by setting them to median values\n#         for i in range(result.X.shape[1]):\n#             col_median = np.median(result.X[~self._fitted_params['outlier_mask'], i])\n#             result.X[self._fitted_params['outlier_mask'], i] = col_median\n#         return result\n\n# # Scaling Steps\n# @register_step\n# class StandardScalerStep(BasePreprocessingStep):\n#     STEP_TYPE = \"scaling\"\n#     METHOD_NAME = \"standard\"\n    \n#     def fit(self, data: ProcessingData) -> 'StandardScalerStep':\n#         X_float = data.X.astype(np.float32)\n#         self._fitted_params['scaler'] = StandardScaler()\n#         self._fitted_params['scaler'].fit(X_float)\n#         self.is_fitted = True\n#         return self\n    \n#     def transform(self, data: ProcessingData) -> ProcessingData:\n#         if not self.is_fitted:\n#             raise ValueError(\"Must fit before transform\")\n        \n#         result = data.copy()\n#         result.X = result.X.astype(np.float32)\n#         result.X = self._fitted_params['scaler'].transform(result.X)\n#         result.X = np.nan_to_num(result.X, nan=0.0, posinf=3.0, neginf=-3.0)\n#         return result\n\n# @register_step\n# class RobustScalerStep(BasePreprocessingStep):\n#     STEP_TYPE = \"scaling\"\n#     METHOD_NAME = \"robust\"\n    \n#     def fit(self, data: ProcessingData) -> 'RobustScalerStep':\n#         X_float = data.X.astype(np.float32)\n#         self._fitted_params['scaler'] = RobustScaler()\n#         self._fitted_params['scaler'].fit(X_float)\n#         self.is_fitted = True\n#         return self\n    \n#     def transform(self, data: ProcessingData) -> ProcessingData:\n#         if not self.is_fitted:\n#             raise ValueError(\"Must fit before transform\")\n        \n#         result = data.copy()\n#         result.X = result.X.astype(np.float32)\n#         result.X = self._fitted_params['scaler'].transform(result.X)\n#         result.X = np.nan_to_num(result.X, nan=0.0, posinf=3.0, neginf=-3.0)\n#         return result\n\n# @register_step\n# class MinMaxScalerStep(BasePreprocessingStep):\n#     STEP_TYPE = \"scaling\"\n#     METHOD_NAME = \"minmax\"\n    \n#     def fit(self, data: ProcessingData) -> 'MinMaxScalerStep':\n#         X_float = data.X.astype(np.float32)\n#         self._fitted_params['scaler'] = MinMaxScaler()\n#         self._fitted_params['scaler'].fit(X_float)\n#         self.is_fitted = True\n#         return self\n    \n#     def transform(self, data: ProcessingData) -> ProcessingData:\n#         if not self.is_fitted:\n#             raise ValueError(\"Must fit before transform\")\n        \n#         result = data.copy()\n#         result.X = result.X.astype(np.float32)\n#         result.X = self._fitted_params['scaler'].transform(result.X)\n#         return result\n\n# @register_step\n# class QuantileScalerStep(BasePreprocessingStep):\n#     STEP_TYPE = \"scaling\"\n#     METHOD_NAME = \"quantile\"\n    \n#     def fit(self, data: ProcessingData) -> 'QuantileScalerStep':\n#         X_float = data.X.astype(np.float32)\n#         n_quantiles = min(self.config.params.get('n_quantiles', 1000), len(X_float))\n#         output_distribution = self.config.params.get('output_distribution', 'uniform')\n        \n#         self._fitted_params['scaler'] = QuantileTransformer(\n#             n_quantiles=n_quantiles,\n#             output_distribution=output_distribution,\n#             random_state=42\n#         )\n#         self._fitted_params['scaler'].fit(X_float)\n#         self.is_fitted = True\n#         return self\n    \n#     def transform(self, data: ProcessingData) -> ProcessingData:\n#         if not self.is_fitted:\n#             raise ValueError(\"Must fit before transform\")\n        \n#         result = data.copy()\n#         result.X = result.X.astype(np.float32)\n#         result.X = self._fitted_params['scaler'].transform(result.X)\n#         return result\n\n# # Dimensionality Reduction Steps\n# @register_step\n# class PCAReduction(BasePreprocessingStep):\n#     STEP_TYPE = \"dim_reduce\"\n#     METHOD_NAME = \"pca\"\n    \n#     def estimate_memory_requirement(self, data: ProcessingData) -> float:\n#         n_components = self.config.params.get('n_components', 100)\n#         return MemoryManager.estimate_array_memory((data.X.shape[1], data.X.shape[1]), dtype=np.float32)\n    \n#     def fit(self, data: ProcessingData) -> 'PCAReduction':\n#         n_components = min(self.config.params.get('n_components', 100), min(data.X.shape) - 1)\n        \n#         if n_components < 2:\n#             self._fitted_params['reducer'] = None\n#         else:\n#             if data.X.shape[0] * data.X.shape[1] > 1e7:\n#                 self._fitted_params['reducer'] = IncrementalPCA(n_components=n_components, batch_size=min(1000, data.X.shape[0]))\n#             else:\n#                 self._fitted_params['reducer'] = PCA(n_components=n_components, random_state=42)\n            \n#             X_float = data.X.astype(np.float32)\n#             self._fitted_params['reducer'].fit(X_float)\n        \n#         self.is_fitted = True\n#         return self\n    \n#     def transform(self, data: ProcessingData) -> ProcessingData:\n#         if not self.is_fitted or self._fitted_params['reducer'] is None:\n#             return data.copy()\n        \n#         result = data.copy()\n#         result.X = result.X.astype(np.float32)\n#         result.X = self._fitted_params['reducer'].transform(result.X)\n#         result.feature_names = [f\"PC{i+1}\" for i in range(result.X.shape[1])]\n#         MemoryManager.clean_memory()\n#         return result\n\n# @register_step\n# class TruncatedSVDReduction(BasePreprocessingStep):\n#     STEP_TYPE = \"dim_reduce\"\n#     METHOD_NAME = \"svd\"\n    \n#     def fit(self, data: ProcessingData) -> 'TruncatedSVDReduction':\n#         n_components = min(self.config.params.get('n_components', 100), min(data.X.shape) - 1)\n        \n#         if n_components < 2:\n#             self._fitted_params['reducer'] = None\n#         else:\n#             self._fitted_params['reducer'] = TruncatedSVD(n_components=n_components, random_state=42)\n#             X_float = data.X.astype(np.float32)\n#             self._fitted_params['reducer'].fit(X_float)\n        \n#         self.is_fitted = True\n#         return self\n    \n#     def transform(self, data: ProcessingData) -> ProcessingData:\n#         if not self.is_fitted or self._fitted_params['reducer'] is None:\n#             return data.copy()\n        \n#         result = data.copy()\n#         result.X = result.X.astype(np.float32)\n#         result.X = self._fitted_params['reducer'].transform(result.X)\n#         result.feature_names = [f\"SVD{i+1}\" for i in range(result.X.shape[1])]\n#         return result\n\n# @register_step\n# class LDAReduction(BasePreprocessingStep):\n#     STEP_TYPE = \"dim_reduce\"\n#     METHOD_NAME = \"lda\"\n    \n#     def fit(self, data: ProcessingData) -> 'LDAReduction':\n#         n_components = min(self.config.params.get('n_components', None), len(np.unique(data.y)) - 1)\n        \n#         self._fitted_params['reducer'] = LinearDiscriminantAnalysis(n_components=n_components)\n#         X_float = data.X.astype(np.float32)\n#         self._fitted_params['reducer'].fit(X_float, data.y)\n        \n#         self.is_fitted = True\n#         return self\n    \n#     def transform(self, data: ProcessingData) -> ProcessingData:\n#         if not self.is_fitted:\n#             return data.copy()\n        \n#         result = data.copy()\n#         result.X = result.X.astype(np.float32)\n#         result.X = self._fitted_params['reducer'].transform(result.X)\n#         result.feature_names = [f\"LDA{i+1}\" for i in range(result.X.shape[1])]\n#         return result\n\n# @register_step\n# class VarianceThresholdReduction(BasePreprocessingStep):\n#     STEP_TYPE = \"dim_reduce\"\n#     METHOD_NAME = \"variance_threshold\"\n    \n#     def estimate_memory_requirement(self, data: ProcessingData) -> float:\n#         return MemoryManager.estimate_array_memory((data.X.shape[1],), dtype=np.float32)\n    \n#     def fit(self, data: ProcessingData) -> 'VarianceThresholdReduction':\n#         threshold = self.config.params.get('threshold', 0.01)\n#         chunk_size = 1000\n#         variances = np.zeros(data.X.shape[1], dtype=np.float32)\n        \n#         for i in range(0, data.X.shape[1], chunk_size):\n#             end_idx = min(i + chunk_size, data.X.shape[1])\n#             variances[i:end_idx] = np.var(data.X[:, i:end_idx].astype(np.float32), axis=0)\n        \n#         self._fitted_params['keep_indices'] = np.where(variances > threshold)[0]\n#         self.is_fitted = True\n#         return self\n    \n#     def transform(self, data: ProcessingData) -> ProcessingData:\n#         if not self.is_fitted:\n#             raise ValueError(\"Must fit before transform\")\n        \n#         result = data.copy()\n#         result.X = result.X[:, self._fitted_params['keep_indices']]\n#         if result.feature_names:\n#             result.feature_names = [result.feature_names[i] for i in self._fitted_params['keep_indices']]\n#         return result\n\n# @register_step\n# class SelectKBestReduction(BasePreprocessingStep):\n#     STEP_TYPE = \"dim_reduce\"\n#     METHOD_NAME = \"select_k_best\"\n    \n#     def estimate_memory_requirement(self, data: ProcessingData) -> float:\n#         return data.get_memory_usage() * 0.1\n    \n#     def fit(self, data: ProcessingData) -> 'SelectKBestReduction':\n#         k = min(self.config.params.get('k', 500), data.X.shape[1])\n#         score_func = self.config.params.get('score_func', 'f_classif')\n        \n#         if score_func == 'mutual_info':\n#             func = mutual_info_classif\n#         elif score_func == 'chi2':\n#             func = chi2\n#         else:\n#             func = f_classif\n        \n#         self._fitted_params['selector'] = SelectKBest(score_func=func, k=k)\n#         X_float = data.X.astype(np.float32)\n        \n#         # For chi2, make sure data is non-negative\n#         if score_func == 'chi2':\n#             X_float = X_float - X_float.min() + 0.001\n        \n#         self._fitted_params['selector'].fit(X_float, data.y)\n#         self.is_fitted = True\n#         return self\n    \n#     def transform(self, data: ProcessingData) -> ProcessingData:\n#         if not self.is_fitted:\n#             raise ValueError(\"Must fit before transform\")\n        \n#         result = data.copy()\n#         result.X = result.X.astype(np.float32)\n        \n#         # For chi2, make sure data is non-negative\n#         if self.config.params.get('score_func', 'f_classif') == 'chi2':\n#             result.X = result.X - result.X.min() + 0.001\n        \n#         result.X = self._fitted_params['selector'].transform(result.X)\n#         if result.feature_names:\n#             mask = self._fitted_params['selector'].get_support()\n#             result.feature_names = [name for name, keep in zip(result.feature_names, mask) if keep]\n#         return result\n\n# # Balancing Steps\n# @register_step\n# class SMOTEBalance(BasePreprocessingStep):\n#     STEP_TYPE = \"balance\"\n#     METHOD_NAME = \"smote\"\n    \n#     def fit(self, data: ProcessingData) -> 'SMOTEBalance':\n#         unique, counts = np.unique(data.y, return_counts=True)\n#         min_samples = counts.min()\n#         k_neighbors = min(self.config.params.get('k_neighbors', 5), min_samples - 1)\n        \n#         if k_neighbors > 0:\n#             self._fitted_params['balancer'] = SMOTE(k_neighbors=k_neighbors, random_state=42)\n#         else:\n#             self._fitted_params['balancer'] = RandomOverSampler(random_state=42)\n        \n#         self.is_fitted = True\n#         return self\n    \n#     def transform(self, data: ProcessingData) -> ProcessingData:\n#         if not self.is_fitted:\n#             return data.copy()\n        \n#         result = data.copy()\n#         result.X = result.X.astype(np.float32)\n        \n#         try:\n#             result.X, result.y = self._fitted_params['balancer'].fit_resample(result.X, result.y)\n#         except:\n#             return data.copy()\n        \n#         return result\n\n# @register_step\n# class HybridBalance(BasePreprocessingStep):\n#     STEP_TYPE = \"balance\"\n#     METHOD_NAME = \"hybrid\"\n    \n#     def fit(self, data: ProcessingData) -> 'HybridBalance':\n#         unique, counts = np.unique(data.y, return_counts=True)\n#         majority_class = unique[np.argmax(counts)]\n#         target_count = int(np.median(counts))\n        \n#         self._fitted_params['undersample'] = RandomUnderSampler(\n#             sampling_strategy={majority_class: min(target_count * 2, counts[np.argmax(counts)])},\n#             random_state=42\n#         )\n        \n#         min_samples = counts.min()\n#         k_neighbors = min(5, min_samples - 1)\n#         if k_neighbors > 0:\n#             self._fitted_params['oversample'] = SMOTE(\n#                 sampling_strategy='not majority',\n#                 k_neighbors=k_neighbors,\n#                 random_state=42\n#             )\n#         else:\n#             self._fitted_params['oversample'] = RandomOverSampler(random_state=42)\n        \n#         self.is_fitted = True\n#         return self\n    \n#     def transform(self, data: ProcessingData) -> ProcessingData:\n#         if not self.is_fitted:\n#             return data.copy()\n        \n#         result = data.copy()\n#         result.X = result.X.astype(np.float32)\n        \n#         try:\n#             result.X, result.y = self._fitted_params['undersample'].fit_resample(result.X, result.y)\n#             result.X, result.y = self._fitted_params['oversample'].fit_resample(result.X, result.y)\n#         except:\n#             return data.copy()\n        \n#         return result\n\n# @register_step\n# class RandomOversampleBalance(BasePreprocessingStep):\n#     STEP_TYPE = \"balance\"\n#     METHOD_NAME = \"random_oversample\"\n    \n#     def fit(self, data: ProcessingData) -> 'RandomOversampleBalance':\n#         self._fitted_params['balancer'] = RandomOverSampler(random_state=42)\n#         self.is_fitted = True\n#         return self\n    \n#     def transform(self, data: ProcessingData) -> ProcessingData:\n#         if not self.is_fitted:\n#             return data.copy()\n        \n#         result = data.copy()\n#         result.X = result.X.astype(np.float32)\n        \n#         try:\n#             result.X, result.y = self._fitted_params['balancer'].fit_resample(result.X, result.y)\n#         except:\n#             return data.copy()\n        \n#         return result\n\n# # Memory-Aware Pipeline\n# class MemoryAwarePipeline:\n#     def __init__(self, config: PipelineConfig):\n#         self.config = config\n#         self.steps = []\n#         self.mandatory_steps = []\n#         self._initialize_steps()\n#         self.execution_times = {}\n    \n#     def _initialize_steps(self):\n#         self.mandatory_steps = [\n#             DowncastStep(StepConfig('downcast', 'auto')),\n#             ImputationStep(StepConfig('impute', 'simple', {'strategy': 'median'}))\n#         ]\n        \n#         for step_config in self.config.steps:\n#             if step_config.enabled:\n#                 step = StepRegistry.create_step(step_config)\n#                 if step:\n#                     self.steps.append(step)\n    \n#     def fit(self, X: np.ndarray, y: np.ndarray, feature_names: Optional[List[str]] = None):\n#         print(f\"[{datetime.now()}] Pipeline fit started with shape {X.shape}\")\n#         data = ProcessingData(X=X, y=y, feature_names=feature_names)\n        \n#         for idx, step in enumerate(self.mandatory_steps):\n#             step_name = f\"{step.config.step_type}:{step.config.method}\"\n#             print(f\"[{datetime.now()}] Fitting mandatory step {idx+1}/{len(self.mandatory_steps)}: {step_name}\")\n#             start_time = time.time()\n#             step.fit(data)\n#             data = step.transform(data)\n#             self.execution_times[f\"mandatory_{step_name}\"] = time.time() - start_time\n#             MemoryManager.clean_memory()\n#             print(f\"[{datetime.now()}] Completed mandatory step {step_name}, shape: {data.X.shape}\")\n        \n#         for idx, step in enumerate(self.steps):\n#             step_name = f\"{step.config.step_type}:{step.config.method}\"\n#             print(f\"[{datetime.now()}] Processing optional step {idx+1}/{len(self.steps)}: {step_name}\")\n            \n#             if self.config.skip_high_memory:\n#                 estimated_mem = step.estimate_memory_requirement(data)\n#                 can_proceed, available, required = MemoryManager.check_memory_for_operation(estimated_mem)\n                \n#                 if not can_proceed or estimated_mem > self.config.memory_threshold_mb:\n#                     print(f\"[{datetime.now()}] Skipping {step_name} due to memory constraints\")\n#                     continue\n            \n#             try:\n#                 print(f\"[{datetime.now()}] Fitting {step_name}\")\n#                 start_time = time.time()\n#                 step.fit(data)\n#                 data = step.transform(data)\n#                 self.execution_times[f\"optional_{step_name}\"] = time.time() - start_time\n#                 MemoryManager.clean_memory()\n#                 print(f\"[{datetime.now()}] Completed {step_name}, shape: {data.X.shape}\")\n#             except Exception as e:\n#                 print(f\"[{datetime.now()}] Error in {step_name}: {str(e)}\")\n#                 continue\n        \n#         self._final_shape = data.X.shape\n#         print(f\"[{datetime.now()}] Pipeline fit completed with final shape: {self._final_shape}\")\n#         return self\n    \n#     def transform(self, X: np.ndarray, y: Optional[np.ndarray] = None, feature_names: Optional[List[str]] = None):\n#         print(f\"[{datetime.now()}] Pipeline transform started with shape {X.shape}\")\n#         data = ProcessingData(X=X, y=y if y is not None else np.zeros(X.shape[0]), feature_names=feature_names)\n        \n#         for step in self.mandatory_steps:\n#             data = step.transform(data)\n        \n#         for step in self.steps:\n#             try:\n#                 data = step.transform(data)\n#             except:\n#                 continue\n        \n#         print(f\"[{datetime.now()}] Pipeline transform completed with shape {data.X.shape}\")\n        \n#         if y is None:\n#             return data.X\n#         return data.X, data.y\n    \n#     def fit_transform(self, X: np.ndarray, y: np.ndarray, feature_names: Optional[List[str]] = None):\n#         self.fit(X, y, feature_names)\n#         return self.transform(X, y, feature_names)\n\n# # Results Manager\n# class ResultsManager:\n#     @staticmethod\n#     def save_config_details(config: PipelineConfig, config_index: int):\n#         config_dir = os.path.join(CONFIGS_DIR, f\"config_{config_index:03d}_{config.config_id}\")\n#         os.makedirs(config_dir, exist_ok=True)\n        \n#         # Save config as JSON\n#         config_details = {\n#             'config_id': config.config_id,\n#             'config_index': config_index,\n#             'timestamp': datetime.now().isoformat(),\n#             'steps': [step.to_dict() for step in config.steps],\n#             'skip_high_memory': config.skip_high_memory,\n#             'memory_threshold_mb': config.memory_threshold_mb,\n#             'pipeline_string': str(config)\n#         }\n        \n#         with open(os.path.join(config_dir, 'config.json'), 'w') as f:\n#             json.dump(config_details, f, indent=2)\n        \n#         return config_dir\n    \n#     @staticmethod\n#     def save_evaluation_results(config_dir: str, results: Dict[str, Any]):\n#         eval_file = os.path.join(config_dir, 'evaluation_results.json')\n        \n#         # Convert numpy arrays to lists for JSON serialization\n#         json_results = {}\n#         for key, value in results.items():\n#             if isinstance(value, np.ndarray):\n#                 json_results[key] = value.tolist()\n#             elif isinstance(value, (np.float32, np.float64)):\n#                 json_results[key] = float(value)\n#             elif isinstance(value, (np.int32, np.int64)):\n#                 json_results[key] = int(value)\n#             else:\n#                 json_results[key] = value\n        \n#         with open(eval_file, 'w') as f:\n#             json.dump(json_results, f, indent=2)\n    \n#     @staticmethod\n#     def save_model_predictions(config_dir: str, model_name: str, predictions: np.ndarray, \n#                              probabilities: np.ndarray, test_ids: np.ndarray, class_names: List[str]):\n#         model_dir = os.path.join(config_dir, model_name)\n#         os.makedirs(model_dir, exist_ok=True)\n        \n#         # Save predictions\n#         submission = pd.DataFrame({\n#             'id': test_ids,\n#             'class_label': [class_names[p] for p in predictions]\n#         })\n#         submission.to_csv(os.path.join(model_dir, 'submission.csv'), index=False)\n        \n#         # Save probabilities\n#         proba_df = pd.DataFrame(probabilities, columns=class_names)\n#         proba_df['id'] = test_ids\n#         proba_df.to_csv(os.path.join(model_dir, 'probabilities.csv'), index=False)\n        \n#         return os.path.join(model_dir, 'submission.csv')\n\n# # Ensemble Generator\n# class EnsembleGenerator:\n#     @staticmethod\n#     def create_voting_ensemble(predictions_list, weights=None):\n#         if weights is None:\n#             weights = [1] * len(predictions_list)\n        \n#         ensemble_predictions = []\n#         for i in range(len(predictions_list[0])):\n#             votes = []\n#             for pred, weight in zip(predictions_list, weights):\n#                 votes.extend([pred[i]] * weight)\n#             most_common = Counter(votes).most_common(1)[0][0]\n#             ensemble_predictions.append(most_common)\n#         return ensemble_predictions\n    \n#     @staticmethod\n#     def create_probability_ensemble(proba_list, weights=None):\n#         if weights is None:\n#             weights = np.ones(len(proba_list)) / len(proba_list)\n#         else:\n#             weights = np.array(weights) / np.sum(weights)\n        \n#         weighted_proba = np.zeros_like(proba_list[0])\n#         for proba, weight in zip(proba_list, weights):\n#             weighted_proba += proba * weight\n        \n#         return np.argmax(weighted_proba, axis=1)\n\n# # Main Grid Search Class\n# class IntelligentGridSearch:\n#     def __init__(self, X_train, y_train, X_test=None):\n#         self.X_train = X_train\n#         self.y_train = y_train\n#         self.X_test = X_test\n#         self.exploration_rate = EXPLORATION_RATE\n#         self.memory_threshold_mb = MEMORY_THRESHOLD_MB\n        \n#         self.results = []\n#         self.config_performance = {}\n#         self.step_performance = defaultdict(lambda: defaultdict(list))\n#         self.all_predictions = {}\n        \n#         self.best_config = None\n#         self.best_score = -np.inf\n#         self.best_model = None\n#         self.best_pipeline = None\n#         self.best_model_type = None\n        \n#         self.start_time = time.time()\n        \n#         print(f\"[{datetime.now()}] IntelligentGridSearch initialized\")\n#         print(f\"[{datetime.now()}] Training data shape: {X_train.shape}\")\n#         print(f\"[{datetime.now()}] Test data shape: {X_test.shape if X_test is not None else 'None'}\")\n#         print(f\"[{datetime.now()}] Memory threshold: {MEMORY_THRESHOLD_MB}MB\")\n#         print(f\"[{datetime.now()}] Available memory: {MemoryManager.get_available_memory():.1f}MB\")\n    \n#     def generate_comprehensive_configs(self, n_configs=50):\n#         configs = []\n#         available_steps = StepRegistry.get_available_steps()\n        \n#         # Define comprehensive step orderings\n#         step_orderings = [\n#             ['outlier', 'scaling', 'dim_reduce', 'balance'],\n#             ['scaling', 'outlier', 'dim_reduce', 'balance'],\n#             ['dim_reduce', 'scaling', 'balance'],\n#             ['dim_reduce', 'outlier', 'scaling', 'balance'],\n#             ['outlier', 'dim_reduce', 'scaling'],\n#             ['scaling', 'dim_reduce'],\n#             ['dim_reduce', 'scaling'],\n#             ['dim_reduce', 'balance'],\n#             ['scaling', 'balance'],\n#         ]\n        \n#         # Define comprehensive method options\n#         outlier_methods = ['zscore', 'iqr', 'isolation_forest', None]\n#         scaling_methods = ['standard', 'robust', 'minmax', 'quantile', None]\n#         dim_reduce_methods = ['pca', 'svd', 'lda', 'variance_threshold', 'select_k_best', None]\n#         balance_methods = ['smote', 'hybrid', 'random_oversample', None]\n        \n#         # Generate diverse configurations\n#         for _ in range(n_configs):\n#             ordering = random.choice(step_orderings)\n#             steps = []\n            \n#             for step_type in ordering:\n#                 if step_type == 'outlier':\n#                     method = random.choice([m for m in outlier_methods if m])\n#                     if method:\n#                         params = self._generate_params_for_method('outlier', method)\n#                         steps.append(StepConfig(\n#                             step_type='outlier',\n#                             method=method,\n#                             params=params,\n#                             enabled=True\n#                         ))\n                \n#                 elif step_type == 'scaling':\n#                     method = random.choice([m for m in scaling_methods if m])\n#                     if method:\n#                         params = self._generate_params_for_method('scaling', method)\n#                         steps.append(StepConfig(\n#                             step_type='scaling',\n#                             method=method,\n#                             params=params,\n#                             enabled=True\n#                         ))\n                \n#                 elif step_type == 'dim_reduce':\n#                     method = random.choice([m for m in dim_reduce_methods if m])\n#                     if method:\n#                         params = self._generate_params_for_method('dim_reduce', method)\n#                         steps.append(StepConfig(\n#                             step_type='dim_reduce',\n#                             method=method,\n#                             params=params,\n#                             enabled=True\n#                         ))\n                \n#                 elif step_type == 'balance':\n#                     method = random.choice([m for m in balance_methods if m])\n#                     if method and random.random() > 0.3:  # 70% chance to include balancing\n#                         params = self._generate_params_for_method('balance', method)\n#                         steps.append(StepConfig(\n#                             step_type='balance',\n#                             method=method,\n#                             params=params,\n#                             enabled=True\n#                         ))\n            \n#             if steps:  # Only add config if it has at least one step\n#                 config = PipelineConfig(\n#                     steps=steps,\n#                     skip_high_memory=False,\n#                     memory_threshold_mb=self.memory_threshold_mb\n#                 )\n#                 config.generate_id()\n#                 configs.append(config)\n        \n#         # Ensure uniqueness\n#         unique_configs = {}\n#         for config in configs:\n#             if config.config_id not in unique_configs:\n#                 unique_configs[config.config_id] = config\n        \n#         return list(unique_configs.values())[:n_configs]\n    \n#     def _generate_params_for_method(self, step_type, method):\n#         params = {}\n        \n#         if step_type == 'outlier':\n#             if method == 'zscore':\n#                 params['threshold'] = random.choice([2.0, 2.5, 3.0, 3.5, 4.0])\n#             elif method == 'iqr':\n#                 params['multiplier'] = random.choice([1.0, 1.5, 2.0, 2.5, 3.0])\n#             elif method == 'isolation_forest':\n#                 params['contamination'] = random.choice([0.05, 0.1, 0.15, 0.2])\n        \n#         elif step_type == 'scaling':\n#             if method == 'quantile':\n#                 params['n_quantiles'] = random.choice([100, 500, 1000])\n#                 params['output_distribution'] = random.choice(['uniform', 'normal'])\n        \n#         elif step_type == 'dim_reduce':\n#             if method == 'pca':\n#                 params['n_components'] = random.choice([10, 20, 30, 50, 100, 150, 200])\n#             elif method == 'svd':\n#                 params['n_components'] = random.choice([10, 20, 30, 50, 100, 150])\n#             elif method == 'lda':\n#                 params['n_components'] = None  # Will be set based on number of classes\n#             elif method == 'select_k_best':\n#                 params['k'] = random.choice([50, 100, 200, 500, 1000, 2000, 5000])\n#                 params['score_func'] = random.choice(['f_classif', 'mutual_info', 'chi2'])\n#             elif method == 'variance_threshold':\n#                 params['threshold'] = random.choice([0.0, 0.0001, 0.001, 0.01, 0.05])\n        \n#         elif step_type == 'balance':\n#             if method in ['smote', 'hybrid']:\n#                 params['k_neighbors'] = random.choice([3, 5, 7, 9])\n        \n#         return params\n    \n#     def evaluate_config(self, config: PipelineConfig, config_index: int):\n#         print(f\"\\n[{datetime.now()}] ========== Evaluating Config {config_index} ==========\")\n#         print(f\"[{datetime.now()}] Config ID: {config.config_id}\")\n#         print(f\"[{datetime.now()}] Pipeline: {str(config)}\")\n        \n#         # Save config details\n#         config_dir = ResultsManager.save_config_details(config, config_index)\n        \n#         try:\n#             print(f\"[{datetime.now()}] Creating pipeline...\")\n#             pipeline = MemoryAwarePipeline(config)\n            \n#             print(f\"[{datetime.now()}] Running fit_transform...\")\n#             X_processed, y_processed = pipeline.fit_transform(self.X_train, self.y_train)\n#             print(f\"[{datetime.now()}] Processed shape: {X_processed.shape}\")\n            \n#             print(f\"[{datetime.now()}] Computing class weights...\")\n#             unique_classes = np.unique(y_processed)\n#             class_weights = compute_class_weight('balanced', classes=unique_classes, y=y_processed)\n#             class_weight_dict = {c: w for c, w in zip(unique_classes, class_weights)}\n#             print(f\"[{datetime.now()}] Classes: {unique_classes}, Weights: {class_weight_dict}\")\n            \n#             # Prepare for cross-validation\n#             cv = StratifiedKFold(n_splits=3, shuffle=True, random_state=42)\n#             f1_scorer = make_scorer(f1_score, average='macro')\n            \n#             models_to_evaluate = []\n#             evaluation_results = {\n#                 'config_id': config.config_id,\n#                 'config_index': config_index,\n#                 'pipeline': str(config),\n#                 'shape_before': self.X_train.shape,\n#                 'shape_after': X_processed.shape,\n#                 'class_distribution': dict(zip(*np.unique(y_processed, return_counts=True))),\n#                 'execution_times': pipeline.execution_times,\n#                 'models': {}\n#             }\n            \n#             # Model 1: Logistic Regression\n#             print(f\"[{datetime.now()}] Training LogisticRegression...\")\n#             lr_start = time.time()\n#             lr_model = LogisticRegression(\n#                 max_iter=1000,\n#                 class_weight=class_weight_dict,\n#                 random_state=42,\n#                 solver='saga',\n#                 n_jobs=-1\n#             )\n#             lr_scores = cross_val_score(lr_model, X_processed, y_processed, cv=cv, scoring='f1_macro', n_jobs=-1)\n#             lr_score = lr_scores.mean()\n#             lr_model.fit(X_processed, y_processed)\n#             lr_time = time.time() - lr_start\n            \n#             print(f\"[{datetime.now()}] LogisticRegression CV F1: {lr_score:.4f} (time: {lr_time:.2f}s)\")\n#             models_to_evaluate.append(('logistic', lr_model, lr_score))\n#             evaluation_results['models']['logistic'] = {\n#                 'cv_scores': lr_scores.tolist(),\n#                 'mean_score': lr_score,\n#                 'training_time': lr_time\n#             }\n            \n#             # Model 2: XGBoost\n#             print(f\"[{datetime.now()}] Training XGBoost...\")\n#             xgb_start = time.time()\n#             scale_pos_weight = class_weight_dict.get(0, 1.0)\n#             xgb_model = xgb.XGBClassifier(\n#                 n_estimators=200,\n#                 max_depth=8,\n#                 learning_rate=0.1,\n#                 objective='multi:softprob',\n#                 scale_pos_weight=scale_pos_weight,\n#                 random_state=42,\n#                 n_jobs=-1,\n#                 eval_metric='mlogloss',\n#                 use_label_encoder=False,\n#                 subsample=0.8,\n#                 colsample_bytree=0.8\n#             )\n#             xgb_scores = cross_val_score(xgb_model, X_processed, y_processed, cv=cv, scoring='f1_macro', n_jobs=-1)\n#             xgb_score = xgb_scores.mean()\n#             xgb_model.fit(X_processed, y_processed)\n#             xgb_time = time.time() - xgb_start\n            \n#             print(f\"[{datetime.now()}] XGBoost CV F1: {xgb_score:.4f} (time: {xgb_time:.2f}s)\")\n#             models_to_evaluate.append(('xgboost', xgb_model, xgb_score))\n#             evaluation_results['models']['xgboost'] = {\n#                 'cv_scores': xgb_scores.tolist(),\n#                 'mean_score': xgb_score,\n#                 'training_time': xgb_time\n#             }\n            \n#             # Model 3: LightGBM\n#             print(f\"[{datetime.now()}] Training LightGBM...\")\n#             lgb_start = time.time()\n#             lgb_model = lgb.LGBMClassifier(\n#                 n_estimators=200,\n#                 max_depth=8,\n#                 learning_rate=0.1,\n#                 objective='multiclass',\n#                 num_class=len(unique_classes),\n#                 class_weight='balanced',\n#                 random_state=42,\n#                 n_jobs=-1,\n#                 verbosity=-1,\n#                 subsample=0.8,\n#                 colsample_bytree=0.8\n#             )\n#             lgb_scores = cross_val_score(lgb_model, X_processed, y_processed, cv=cv, scoring='f1_macro', n_jobs=-1)\n#             lgb_score = lgb_scores.mean()\n#             lgb_model.fit(X_processed, y_processed)\n#             lgb_time = time.time() - lgb_start\n            \n#             print(f\"[{datetime.now()}] LightGBM CV F1: {lgb_score:.4f} (time: {lgb_time:.2f}s)\")\n#             models_to_evaluate.append(('lightgbm', lgb_model, lgb_score))\n#             evaluation_results['models']['lightgbm'] = {\n#                 'cv_scores': lgb_scores.tolist(),\n#                 'mean_score': lgb_score,\n#                 'training_time': lgb_time\n#             }\n            \n#             # Save evaluation results\n#             ResultsManager.save_evaluation_results(config_dir, evaluation_results)\n            \n#             # Generate predictions for all models\n#             if self.X_test is not None:\n#                 print(f\"[{datetime.now()}] Transforming test data...\")\n#                 X_test_transformed = pipeline.transform(self.X_test)\n#                 print(f\"[{datetime.now()}] Test shape after transform: {X_test_transformed.shape}\")\n                \n#                 for model_name, model, score in models_to_evaluate:\n#                     print(f\"[{datetime.now()}] Making predictions for {model_name}...\")\n#                     predictions = model.predict(X_test_transformed)\n#                     probabilities = model.predict_proba(X_test_transformed)\n                    \n#                     model_key = f\"{config.config_id}_{model_name}\"\n#                     self.all_predictions[model_key] = {\n#                         'predictions': predictions,\n#                         'probabilities': probabilities,\n#                         'score': score,\n#                         'config': config,\n#                         'model': model,\n#                         'model_type': model_name,\n#                         'pipeline': pipeline,\n#                         'config_index': config_index,\n#                         'config_dir': config_dir\n#                     }\n                    \n#                     # Update best model if needed\n#                     if score > self.best_score:\n#                         print(f\"[{datetime.now()}] NEW BEST SCORE: {score:.4f} ({model_name})\")\n#                         self.best_score = score\n#                         self.best_config = config\n#                         self.best_model = model\n#                         self.best_pipeline = pipeline\n#                         self.best_model_type = model_name\n            \n#             MemoryManager.clean_memory()\n            \n#             result = {\n#                 'config': config,\n#                 'scores': {'logistic': lr_score, 'xgboost': xgb_score, 'lightgbm': lgb_score},\n#                 'best_score': max(lr_score, xgb_score, lgb_score),\n#                 'models': models_to_evaluate,\n#                 'pipeline': pipeline,\n#                 'shape': X_processed.shape,\n#                 'memory_used': MemoryManager.get_memory_usage(),\n#                 'config_dir': config_dir\n#             }\n            \n#             # Update step performance\n#             for step in config.steps:\n#                 if step.enabled:\n#                     self.step_performance[step.step_type][step.method].append(result['best_score'])\n            \n#             print(f\"[{datetime.now()}] Config evaluation completed successfully\")\n#             print(f\"[{datetime.now()}] Best score for this config: {result['best_score']:.4f}\")\n#             print(f\"[{datetime.now()}] Memory usage: {result['memory_used']:.1f}MB\")\n#             print(f\"[{datetime.now()}] Time elapsed: {(time.time() - self.start_time)/60:.1f} minutes\")\n            \n#             return result\n            \n#         except Exception as e:\n#             print(f\"[{datetime.now()}] ERROR evaluating config: {str(e)}\")\n#             import traceback\n#             traceback.print_exc()\n            \n#             # Save error details\n#             error_details = {\n#                 'config_id': config.config_id,\n#                 'config_index': config_index,\n#                 'error': str(e),\n#                 'traceback': traceback.format_exc()\n#             }\n            \n#             with open(os.path.join(config_dir, 'error.json'), 'w') as f:\n#                 json.dump(error_details, f, indent=2)\n            \n#             MemoryManager.clean_memory()\n#             return None\n    \n#     def run_search(self, n_iterations=None, test_ids=None, class_names=None):\n#         if n_iterations is None:\n#             n_iterations = MAX_CONFIGURATIONS_TO_TEST\n        \n#         print(f\"\\n[{datetime.now()}] Starting grid search with {n_iterations} iterations\")\n#         print(f\"[{datetime.now()}] Target runtime: >30 minutes\")\n        \n#         # Generate comprehensive configurations\n#         configs = self.generate_comprehensive_configs(n_iterations * 2)  # Generate extra configs\n#         print(f\"[{datetime.now()}] Generated {len(configs)} unique configurations\")\n        \n#         for i in range(min(n_iterations, len(configs))):\n#             config = configs[i]\n            \n#             print(f\"\\n[{datetime.now()}] Iteration {i+1}/{n_iterations}\")\n#             print(f\"[{datetime.now()}] Time elapsed: {(time.time() - self.start_time)/60:.1f} minutes\")\n            \n#             if config.config_id in self.config_performance:\n#                 print(f\"[{datetime.now()}] Config already evaluated, skipping\")\n#                 continue\n            \n#             result = self.evaluate_config(config, i+1)\n            \n#             if result:\n#                 self.results.append(result)\n#                 self.config_performance[config.config_id] = result['best_score']\n                \n#                 # Save predictions for all three models\n#                 if test_ids is not None and class_names is not None:\n#                     for model_key, pred_data in self.all_predictions.items():\n#                         if pred_data['config'].config_id == config.config_id:\n#                             # Save to config directory\n#                             ResultsManager.save_model_predictions(\n#                                 pred_data['config_dir'],\n#                                 pred_data['model_type'],\n#                                 pred_data['predictions'],\n#                                 pred_data['probabilities'],\n#                                 test_ids,\n#                                 class_names\n#                             )\n        \n#         # Generate final summary\n#         self.generate_final_summary(test_ids, class_names)\n        \n#         print(f\"\\n[{datetime.now()}] Grid search completed\")\n#         print(f\"[{datetime.now()}] Total time: {(time.time() - self.start_time)/60:.1f} minutes\")\n#         print(f\"[{datetime.now()}] Best score: {self.best_score:.4f} (Model: {self.best_model_type})\")\n        \n#         return self.best_config, self.best_model, self.best_pipeline\n    \n#     def generate_final_summary(self, test_ids, class_names):\n#         summary_file = os.path.join(RESULTS_DIR, 'search_summary.json')\n        \n#         summary = {\n#             'total_time_minutes': (time.time() - self.start_time) / 60,\n#             'configurations_tested': len(self.results),\n#             'best_score': self.best_score,\n#             'best_model_type': self.best_model_type,\n#             'best_config_id': self.best_config.config_id if self.best_config else None,\n#             'best_pipeline': str(self.best_config) if self.best_config else None,\n#             'top_configurations': []\n#         }\n        \n#         # Get top 10 configurations\n#         sorted_results = sorted(self.results, key=lambda x: x['best_score'], reverse=True)[:10]\n#         for rank, result in enumerate(sorted_results, 1):\n#             summary['top_configurations'].append({\n#                 'rank': rank,\n#                 'config_id': result['config'].config_id,\n#                 'pipeline': str(result['config']),\n#                 'scores': result['scores'],\n#                 'best_score': result['best_score'],\n#                 'shape': result['shape']\n#             })\n        \n#         with open(summary_file, 'w') as f:\n#             json.dump(summary, f, indent=2)\n        \n#         print(f\"[{datetime.now()}] Summary saved to {summary_file}\")\n\n# def main():\n#     print(f\"\\n[{datetime.now()}] ========== STARTING MAIN EXECUTION ==========\")\n#     print(f\"[{datetime.now()}] Loading data...\")\n    \n#     train = pd.read_csv('/kaggle/input/detecting-reversal-points-in-us-equities/competition_data/train.csv')\n#     test = pd.read_csv('/kaggle/input/detecting-reversal-points-in-us-equities/competition_data/test.csv')\n    \n#     print(f\"[{datetime.now()}] Train shape: {train.shape}\")\n#     print(f\"[{datetime.now()}] Test shape: {test.shape}\")\n    \n#     print(f\"[{datetime.now()}] Processing target labels...\")\n#     target_mapping = {'HH': 'H', 'LH': 'H', 'HL': 'L', 'LL': 'L'}\n#     train['class_label_mapped'] = train['class_label'].map(target_mapping).fillna('None')\n    \n#     print(f\"[{datetime.now()}] Target distribution:\")\n#     print(train['class_label_mapped'].value_counts())\n    \n#     print(f\"[{datetime.now()}] Separating features and metadata...\")\n#     metadata_cols = ['ticker_id', 't', 'class_label', 'class_label_mapped', 'train_id', 'id']\n#     feature_cols = [col for col in train.columns if col not in metadata_cols]\n#     print(f\"[{datetime.now()}] Number of features: {len(feature_cols)}\")\n    \n#     print(f\"[{datetime.now()}] Creating arrays...\")\n#     X_train = train[feature_cols].values.astype(np.float32)\n#     y_train = pd.factorize(train['class_label_mapped'])[0]\n#     X_test = test[feature_cols].values.astype(np.float32)\n    \n#     print(f\"[{datetime.now()}] X_train shape: {X_train.shape}, dtype: {X_train.dtype}\")\n#     print(f\"[{datetime.now()}] y_train shape: {y_train.shape}, unique values: {np.unique(y_train)}\")\n#     print(f\"[{datetime.now()}] X_test shape: {X_test.shape}, dtype: {X_test.dtype}\")\n    \n#     # Fix class names order based on factorize\n#     _, class_names = pd.factorize(train['class_label_mapped'])\n#     class_names = list(class_names)\n#     test_ids = test['id'].values if 'id' in test.columns else range(len(test))\n    \n#     print(f\"[{datetime.now()}] Class names: {class_names}\")\n#     print(f\"[{datetime.now()}] Initializing IntelligentGridSearch...\")\n    \n#     search = IntelligentGridSearch(X_train, y_train, X_test)\n    \n#     print(f\"[{datetime.now()}] Starting search...\")\n#     best_config, best_model, best_pipeline = search.run_search(\n#         n_iterations=MAX_CONFIGURATIONS_TO_TEST,\n#         test_ids=test_ids,\n#         class_names=class_names\n#     )\n    \n#     if best_model is not None:\n#         print(f\"\\n[{datetime.now()}] Making final predictions...\")\n#         X_test_transformed = best_pipeline.transform(X_test)\n#         predictions = best_model.predict(X_test_transformed)\n        \n#         final_predictions = [class_names[p] for p in predictions]\n        \n#         submission = pd.DataFrame({\n#             'id': test_ids,\n#             'class_label': final_predictions\n#         })\n        \n#         print(f\"[{datetime.now()}] Saving final submissions...\")\n#         submission.to_csv('submission.csv', index=False)\n#         final_path = os.path.join(SUBMISSION_DIR, f'FINAL_BEST_{search.best_model_type}_submission.csv')\n#         submission.to_csv(final_path, index=False)\n        \n#         print(f\"[{datetime.now()}] Final submission saved to submission.csv\")\n#         print(f\"[{datetime.now()}] Best model type: {search.best_model_type}\")\n#         print(f\"[{datetime.now()}] Prediction distribution:\")\n#         print(pd.Series(final_predictions).value_counts())\n#     else:\n#         print(f\"[{datetime.now()}] ERROR: No successful model found\")\n    \n#     print(f\"\\n[{datetime.now()}] ========== EXECUTION COMPLETED ==========\")\n#     print(f\"[{datetime.now()}] Total runtime: {(time.time() - search.start_time)/60:.1f} minutes\")\n\n# if __name__ == \"__main__\":\n#     main()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}