{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.13"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":108426,"databundleVersionId":13143440,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":134.274237,"end_time":"2025-07-28T14:48:33.155057","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-07-28T14:46:18.88082","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#!/usr/bin/env python3\n\"\"\"\nAdvanced Insurance Premium Prediction Pipeline\n- Memory-efficient processing\n- Chunk-based operations\n- Genetic algorithm for feature selection\n- Advanced ensemble methods\n\"\"\"\n\nimport gc\nimport os\nimport sys\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport numpy as np\nimport pandas as pd\nfrom datetime import datetime\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Memory optimization\nfrom pandas.api.types import CategoricalDtype\nimport psutil\n\n# ML imports\nfrom sklearn.model_selection import train_test_split, KFold, cross_val_score\nfrom sklearn.preprocessing import StandardScaler, RobustScaler, LabelEncoder\nfrom sklearn.impute import SimpleImputer, KNNImputer\nfrom sklearn.metrics import mean_squared_log_error, mean_squared_error\nfrom sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor, VotingRegressor\nfrom sklearn.linear_model import Ridge, Lasso, ElasticNet, HuberRegressor\nimport xgboost as xgb\nimport lightgbm as lgb\nfrom catboost import CatBoostRegressor\n\n# Advanced optimization\nimport optuna\nfrom deap import base, creator, tools, algorithms\nimport joblib\nfrom joblib import Parallel, delayed\n\n# Set random seed\nRANDOM_STATE = 42\nnp.random.seed(RANDOM_STATE)\n\nprint(\"=\"*100)\nprint(\"ADVANCED INSURANCE PREMIUM PREDICTION PIPELINE\")\nprint(\"=\"*100)\nprint(f\"Started at: {datetime.now()}\")\nprint(f\"Available RAM: {psutil.virtual_memory().available / 1024**3:.2f} GB\")\nprint(\"=\"*100)\n\n# ========================\n# MEMORY OPTIMIZATION\n# ========================\n\ndef optimize_dtypes(df, verbose=True):\n    \"\"\"Optimize dataframe dtypes to reduce memory usage\"\"\"\n    if verbose:\n        start_mem = df.memory_usage().sum() / 1024**2\n        print(f\"Memory usage before optimization: {start_mem:.2f} MB\")\n    \n    # Optimize numeric columns\n    for col in df.select_dtypes(include=['int']).columns:\n        col_min = df[col].min()\n        col_max = df[col].max()\n        \n        if col_min >= 0:\n            if col_max < 255:\n                df[col] = df[col].astype(np.uint8)\n            elif col_max < 65535:\n                df[col] = df[col].astype(np.uint16)\n            elif col_max < 4294967295:\n                df[col] = df[col].astype(np.uint32)\n        else:\n            if col_min > np.iinfo(np.int8).min and col_max < np.iinfo(np.int8).max:\n                df[col] = df[col].astype(np.int8)\n            elif col_min > np.iinfo(np.int16).min and col_max < np.iinfo(np.int16).max:\n                df[col] = df[col].astype(np.int16)\n            elif col_min > np.iinfo(np.int32).min and col_max < np.iinfo(np.int32).max:\n                df[col] = df[col].astype(np.int32)\n    \n    # Optimize float columns\n    for col in df.select_dtypes(include=['float']).columns:\n        df[col] = pd.to_numeric(df[col], downcast='float')\n    \n    # Convert object columns with low cardinality to category\n    for col in df.select_dtypes(include=['object']).columns:\n        if df[col].nunique() / len(df) < 0.5:\n            df[col] = df[col].astype('category')\n    \n    if verbose:\n        end_mem = df.memory_usage().sum() / 1024**2\n        print(f\"Memory usage after optimization: {end_mem:.2f} MB\")\n        print(f\"Memory reduction: {100 * (start_mem - end_mem) / start_mem:.1f}%\")\n    \n    return df\n\n# ========================\n# DATA LOADING\n# ========================\n\nprint(\"\\n📊 LOADING DATA WITH MEMORY OPTIMIZATION...\")\n\n# Load data in chunks for memory efficiency\ndef load_data_optimized(filepath, chunksize=50000):\n    \"\"\"Load data in chunks and optimize dtypes\"\"\"\n    chunks = []\n    \n    for chunk in pd.read_csv(filepath, chunksize=chunksize):\n        chunk = optimize_dtypes(chunk, verbose=False)\n        chunks.append(chunk)\n    \n    df = pd.concat(chunks, ignore_index=True)\n    return df\n\n# Load data\ntrain_df = load_data_optimized('/kaggle/input/big-oai-final-course-1/train.csv')\ntest_df = load_data_optimized('/kaggle/input/big-oai-final-course-1/test.csv')\n\nprint(f\"✅ Train data: {train_df.shape[0]:,} rows × {train_df.shape[1]} columns\")\nprint(f\"✅ Test data: {test_df.shape[0]:,} rows × {test_df.shape[1]} columns\")\n\n# Save test IDs\ntest_ids = test_df['id'].values\n\n# ========================\n# FEATURE ENGINEERING\n# ========================\n\nclass AdvancedFeatureEngineer:\n    \"\"\"Advanced feature engineering with memory efficiency\"\"\"\n    \n    def __init__(self):\n        self.date_features = []\n        self.encoders = {}\n        self.scalers = {}\n        self.imputers = {}\n    \n    def process_dates(self, df):\n        \"\"\"Extract advanced date features\"\"\"\n        df = df.copy()\n        \n        if 'Policy Start Date' in df.columns:\n            df['Policy Start Date'] = pd.to_datetime(df['Policy Start Date'])\n            \n            # Basic features\n            df['year'] = df['Policy Start Date'].dt.year.astype(np.int16)\n            df['month'] = df['Policy Start Date'].dt.month.astype(np.int8)\n            df['day'] = df['Policy Start Date'].dt.day.astype(np.int8)\n            df['dayofweek'] = df['Policy Start Date'].dt.dayofweek.astype(np.int8)\n            df['quarter'] = df['Policy Start Date'].dt.quarter.astype(np.int8)\n            df['weekofyear'] = df['Policy Start Date'].dt.isocalendar().week.astype(np.int8)\n            \n            # Advanced features\n            df['is_weekend'] = (df['dayofweek'] >= 5).astype(np.int8)\n            df['is_month_start'] = (df['day'] <= 7).astype(np.int8)\n            df['is_month_end'] = (df['day'] >= 24).astype(np.int8)\n            df['is_quarter_start'] = ((df['month'] - 1) % 3 == 0) & (df['day'] <= 7)\n            df['is_quarter_start'] = df['is_quarter_start'].astype(np.int8)\n            \n            # Cyclic encoding\n            df['month_sin'] = np.sin(2 * np.pi * df['month'] / 12).astype(np.float32)\n            df['month_cos'] = np.cos(2 * np.pi * df['month'] / 12).astype(np.float32)\n            df['day_sin'] = np.sin(2 * np.pi * df['day'] / 31).astype(np.float32)\n            df['day_cos'] = np.cos(2 * np.pi * df['day'] / 31).astype(np.float32)\n            \n            # Days since start\n            min_date = df['Policy Start Date'].min()\n            df['days_since_start'] = (df['Policy Start Date'] - min_date).dt.days.astype(np.int16)\n            \n            df.drop('Policy Start Date', axis=1, inplace=True)\n            \n        return df\n    \n    def create_interaction_features(self, df):\n        \"\"\"Create advanced interaction features\"\"\"\n        df = df.copy()\n        \n        # Age-based risk features\n        if 'Age' in df.columns:\n            df['age_risk'] = np.where(df['Age'] < 25, 2,\n                            np.where(df['Age'] > 65, 2,\n                            np.where(df['Age'] > 55, 1, 0))).astype(np.int8)\n            \n            # Age groups with better granularity\n            df['age_group'] = pd.cut(df['Age'], \n                                    bins=[0, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 100],\n                                    labels=['<20', '20-25', '25-30', '30-35', '35-40', \n                                           '40-45', '45-50', '50-55', '55-60', '60-65', '65+'])\n        \n        # Income features\n        if 'Annual Income' in df.columns:\n            # Log transform for skewed distribution\n            df['income_log'] = np.log1p(df['Annual Income']).astype(np.float32)\n            \n            # Income per dependent\n            if 'Number of Dependents' in df.columns:\n                df['income_per_dependent'] = df['Annual Income'] / (df['Number of Dependents'] + 1)\n                df['income_per_dependent'] = df['income_per_dependent'].astype(np.float32)\n        \n        # Health risk score\n        if 'Health Score' in df.columns and 'Smoking Status' in df.columns:\n            smoking_mult = df['Smoking Status'].map({'Yes': 1.5, 'No': 1.0})\n            df['health_risk_adjusted'] = df['Health Score'] * smoking_mult\n            df['health_risk_adjusted'] = df['health_risk_adjusted'].astype(np.float32)\n        \n        # Vehicle features\n        if 'Vehicle Age' in df.columns:\n            df['vehicle_risk'] = np.where(df['Vehicle Age'] > 15, 3,\n                                 np.where(df['Vehicle Age'] > 10, 2,\n                                 np.where(df['Vehicle Age'] > 5, 1, 0))).astype(np.int8)\n        \n        # Credit score categories\n        if 'Credit Score' in df.columns:\n            df['credit_category'] = pd.cut(df['Credit Score'],\n                                          bins=[0, 580, 670, 740, 800, 1000],\n                                          labels=['Poor', 'Fair', 'Good', 'VeryGood', 'Excellent'])\n        \n        # Policy features\n        if 'Insurance Duration' in df.columns:\n            df['is_new_customer'] = (df['Insurance Duration'] <= 1).astype(np.int8)\n            df['is_loyal_customer'] = (df['Insurance Duration'] >= 5).astype(np.int8)\n        \n        # Exercise and health interaction\n        if 'Exercise Frequency' in df.columns and 'Health Score' in df.columns:\n            exercise_map = {'Never': 0, 'Monthly': 1, 'Weekly': 2, 'Daily': 3}\n            exercise_score = df['Exercise Frequency'].map(exercise_map)\n            df['fitness_score'] = exercise_score * df['Health Score'] / 3\n            df['fitness_score'] = df['fitness_score'].astype(np.float32)\n        \n        return df\n    \n    def engineer_features(self, df):\n        \"\"\"Main feature engineering pipeline\"\"\"\n        print(\"🔧 Engineering features...\")\n        \n        # Process dates\n        df = self.process_dates(df)\n        \n        # Create interactions\n        df = self.create_interaction_features(df)\n        \n        # Optimize memory\n        df = optimize_dtypes(df, verbose=False)\n        \n        return df\n\n# ========================\n# PREPROCESSING PIPELINE\n# ========================\n\nclass MemoryEfficientPreprocessor:\n    \"\"\"Memory-efficient preprocessing with advanced imputation\"\"\"\n    \n    def __init__(self):\n        self.numeric_features = []\n        self.categorical_features = []\n        self.target_encoder_features = []\n        self.label_encoders = {}\n        self.target_encoders = {}\n        self.scalers = {}\n        self.imputers = {}\n        \n    def identify_features(self, df, target_col=None):\n        \"\"\"Identify feature types\"\"\"\n        if target_col and target_col in df.columns:\n            df = df.drop(columns=[target_col])\n        \n        self.numeric_features = df.select_dtypes(include=['int8', 'int16', 'int32', 'int64', \n                                                          'uint8', 'uint16', 'uint32', 'uint64',\n                                                          'float16', 'float32', 'float64']).columns.tolist()\n        \n        self.categorical_features = df.select_dtypes(include=['object', 'category']).columns.tolist()\n        \n        # Remove ID columns\n        for col in ['id', 'Id', 'ID']:\n            if col in self.numeric_features:\n                self.numeric_features.remove(col)\n            if col in self.categorical_features:\n                self.categorical_features.remove(col)\n        \n        # High cardinality features for target encoding\n        self.target_encoder_features = [col for col in self.categorical_features \n                                       if df[col].nunique() > 10]\n        \n        print(f\"📊 Numeric features: {len(self.numeric_features)}\")\n        print(f\"📊 Categorical features: {len(self.categorical_features)}\")\n        print(f\"📊 Target encoding features: {len(self.target_encoder_features)}\")\n    \n    def impute_missing(self, df, strategy='smart'):\n        \"\"\"Advanced missing value imputation\"\"\"\n        df = df.copy()\n        \n        # Numeric imputation\n        for col in self.numeric_features:\n            if col in df.columns and df[col].isnull().any():\n                if strategy == 'smart':\n                    # Use different strategies based on missing percentage\n                    missing_pct = df[col].isnull().sum() / len(df)\n                    \n                    if missing_pct < 0.05:\n                        # KNN imputation for low missing percentage\n                        imputer = KNNImputer(n_neighbors=5)\n                        df[col] = imputer.fit_transform(df[[col]])\n                    else:\n                        # Median for higher missing percentage\n                        df[col].fillna(df[col].median(), inplace=True)\n                else:\n                    df[col].fillna(df[col].median(), inplace=True)\n        \n        # Categorical imputation\n        for col in self.categorical_features:\n            if col in df.columns:\n                df[col].fillna('Unknown', inplace=True)\n        \n        return df\n    \n    def encode_categoricals(self, df, y=None, is_train=True):\n        \"\"\"Encode categorical variables with multiple strategies\"\"\"\n        df = df.copy()\n        \n        # Label encoding for ordinal features\n        ordinal_features = ['Education Level', 'Exercise Frequency']\n        ordinal_mappings = {\n            'Education Level': {'High School': 0, \"Bachelor's\": 1, \"Master's\": 2, 'PhD': 3},\n            'Exercise Frequency': {'Never': 0, 'Monthly': 1, 'Weekly': 2, 'Daily': 3}\n        }\n        \n        for col in ordinal_features:\n            if col in df.columns:\n                if col in ordinal_mappings:\n                    df[col] = df[col].map(ordinal_mappings[col])\n                    df[col].fillna(0, inplace=True)\n        \n        # One-hot encoding for low cardinality\n        low_card_features = [col for col in self.categorical_features \n                           if col not in self.target_encoder_features \n                           and col not in ordinal_features\n                           and col in df.columns]\n        \n        if low_card_features:\n            df = pd.get_dummies(df, columns=low_card_features, drop_first=True, dtype=np.int8)\n        \n        # Target encoding for high cardinality\n        if y is not None and self.target_encoder_features:\n            for col in self.target_encoder_features:\n                if col in df.columns:\n                    if is_train:\n                        # Calculate target encoding with smoothing\n                        smooth_factor = 10\n                        mean_target = y.mean()\n                        \n                        encoding_dict = {}\n                        for cat in df[col].unique():\n                            mask = df[col] == cat\n                            cat_count = mask.sum()\n                            cat_mean = y[mask].mean() if mask.sum() > 0 else mean_target\n                            \n                            # Smoothing\n                            smooth_mean = (cat_count * cat_mean + smooth_factor * mean_target) / (cat_count + smooth_factor)\n                            encoding_dict[cat] = smooth_mean\n                        \n                        self.target_encoders[col] = encoding_dict\n                        df[col + '_target_enc'] = df[col].map(encoding_dict).fillna(mean_target)\n                    else:\n                        # Use saved encodings for test set\n                        if col in self.target_encoders:\n                            mean_target = np.mean(list(self.target_encoders[col].values()))\n                            df[col + '_target_enc'] = df[col].map(self.target_encoders[col]).fillna(mean_target)\n                    \n                    df.drop(columns=[col], inplace=True)\n        \n        return df\n    \n    def scale_features(self, df, is_train=True):\n        \"\"\"Scale numeric features\"\"\"\n        df = df.copy()\n        \n        # Get numeric columns in the current dataframe\n        numeric_cols = [col for col in self.numeric_features if col in df.columns]\n        \n        if numeric_cols:\n            if is_train:\n                self.scalers['robust'] = RobustScaler()\n                df[numeric_cols] = self.scalers['robust'].fit_transform(df[numeric_cols])\n            else:\n                if 'robust' in self.scalers:\n                    df[numeric_cols] = self.scalers['robust'].transform(df[numeric_cols])\n        \n        return df\n    \n    def preprocess(self, df, y=None, is_train=True):\n        \"\"\"Main preprocessing pipeline\"\"\"\n        # Impute missing values\n        df = self.impute_missing(df)\n        \n        # Encode categoricals\n        df = self.encode_categoricals(df, y, is_train)\n        \n        # Scale features\n        df = self.scale_features(df, is_train)\n        \n        # Final memory optimization\n        df = optimize_dtypes(df, verbose=False)\n        \n        return df\n\n# ========================\n# GENETIC ALGORITHM FOR FEATURE SELECTION\n# ========================\n\nclass GeneticFeatureSelector:\n    \"\"\"Genetic algorithm for optimal feature selection\"\"\"\n    \n    def __init__(self, n_features, n_generations=20, population_size=50):\n        self.n_features = n_features\n        self.n_generations = n_generations\n        self.population_size = population_size\n        self.best_features = None\n        self.best_score = None\n        \n    def evaluate_features(self, individual, X, y, cv=3):\n        \"\"\"Evaluate a feature subset using cross-validation\"\"\"\n        if sum(individual) == 0:\n            return (1e10,)  # Return high error for empty feature set\n        \n        # Select features\n        feature_mask = np.array(individual, dtype=bool)\n        X_subset = X[:, feature_mask]\n        \n        # Use simple model for speed\n        model = lgb.LGBMRegressor(\n            n_estimators=50,\n            num_leaves=31,\n            learning_rate=0.1,\n            verbosity=-1,\n            random_state=RANDOM_STATE\n        )\n        \n        # Cross-validation\n        scores = cross_val_score(model, X_subset, y, cv=cv, \n                               scoring='neg_mean_squared_error', n_jobs=-1)\n        \n        return (-scores.mean(),)\n    \n    def run_genetic_algorithm(self, X, y):\n        \"\"\"Run genetic algorithm for feature selection\"\"\"\n        print(\"🧬 Running genetic algorithm for feature selection...\")\n        \n        # Create fitness and individual classes\n        creator.create(\"FitnessMin\", base.Fitness, weights=(-1.0,))\n        creator.create(\"Individual\", list, fitness=creator.FitnessMin)\n        \n        # Initialize toolbox\n        toolbox = base.Toolbox()\n        toolbox.register(\"attr_bool\", np.random.randint, 0, 2)\n        toolbox.register(\"individual\", tools.initRepeat, creator.Individual,\n                        toolbox.attr_bool, n=self.n_features)\n        toolbox.register(\"population\", tools.initRepeat, list, toolbox.individual)\n        \n        # Register genetic operators\n        toolbox.register(\"evaluate\", self.evaluate_features, X=X, y=y)\n        toolbox.register(\"mate\", tools.cxTwoPoint)\n        toolbox.register(\"mutate\", tools.mutFlipBit, indpb=0.05)\n        toolbox.register(\"select\", tools.selTournament, tournsize=3)\n        \n        # Create initial population\n        population = toolbox.population(n=self.population_size)\n        \n        # Run evolution\n        for gen in range(self.n_generations):\n            # Evaluate fitness\n            fitnesses = Parallel(n_jobs=-1)(\n                delayed(self.evaluate_features)(ind, X, y) for ind in population\n            )\n            for ind, fit in zip(population, fitnesses):\n                ind.fitness.values = fit\n            \n            # Select next generation\n            offspring = toolbox.select(population, len(population))\n            offspring = list(map(toolbox.clone, offspring))\n            \n            # Apply crossover and mutation\n            for child1, child2 in zip(offspring[::2], offspring[1::2]):\n                if np.random.random() < 0.5:\n                    toolbox.mate(child1, child2)\n                    del child1.fitness.values\n                    del child2.fitness.values\n            \n            for mutant in offspring:\n                if np.random.random() < 0.2:\n                    toolbox.mutate(mutant)\n                    del mutant.fitness.values\n            \n            # Evaluate offspring\n            invalid_ind = [ind for ind in offspring if not ind.fitness.valid]\n            fitnesses = Parallel(n_jobs=-1)(\n                delayed(self.evaluate_features)(ind, X, y) for ind in invalid_ind\n            )\n            for ind, fit in zip(invalid_ind, fitnesses):\n                ind.fitness.values = fit\n            \n            population[:] = offspring\n            \n            # Track best\n            fits = [ind.fitness.values[0] for ind in population]\n            best_idx = np.argmin(fits)\n            \n            if gen % 5 == 0:\n                print(f\"   Generation {gen}: Best fitness = {fits[best_idx]:.4f}\")\n        \n        # Get best solution\n        best_ind = tools.selBest(population, 1)[0]\n        self.best_features = np.array(best_ind, dtype=bool)\n        self.best_score = best_ind.fitness.values[0]\n        \n        print(f\"✅ Selected {sum(self.best_features)} features out of {self.n_features}\")\n        \n        return self.best_features\n\n# ========================\n# ADVANCED ENSEMBLE MODEL\n# ========================\n\nclass AdvancedEnsembleModel:\n    \"\"\"Advanced ensemble with stacking and blending\"\"\"\n    \n    def __init__(self):\n        self.base_models = {}\n        self.meta_model = None\n        self.weights = None\n        self.use_genetic_features = True\n        self.feature_selector = None\n        self.selected_features = None\n        \n    def initialize_models(self):\n        \"\"\"Initialize diverse base models\"\"\"\n        self.base_models = {\n            'lgb': lgb.LGBMRegressor(\n                n_estimators=1000,\n                num_leaves=31,\n                learning_rate=0.05,\n                feature_fraction=0.8,\n                bagging_fraction=0.8,\n                bagging_freq=5,\n                verbosity=-1,\n                random_state=RANDOM_STATE\n            ),\n            'xgb': xgb.XGBRegressor(\n                n_estimators=1000,\n                max_depth=6,\n                learning_rate=0.05,\n                subsample=0.8,\n                colsample_bytree=0.8,\n                random_state=RANDOM_STATE\n            ),\n            'cat': CatBoostRegressor(\n                iterations=1000,\n                learning_rate=0.05,\n                depth=6,\n                verbose=False,\n                random_state=RANDOM_STATE\n            ),\n            'rf': RandomForestRegressor(\n                n_estimators=200,\n                max_depth=15,\n                min_samples_split=20,\n                min_samples_leaf=10,\n                random_state=RANDOM_STATE,\n                n_jobs=-1\n            ),\n            'gb': GradientBoostingRegressor(\n                n_estimators=200,\n                max_depth=6,\n                learning_rate=0.1,\n                subsample=0.8,\n                random_state=RANDOM_STATE\n            ),\n            'huber': HuberRegressor(\n                max_iter=200,\n                alpha=1.0,\n                epsilon=1.35\n            )\n        }\n        \n        # Meta model for stacking\n        self.meta_model = Ridge(alpha=1.0)\n    \n    def train_with_cv(self, X_train, y_train, n_folds=5):\n        \"\"\"Train models with cross-validation\"\"\"\n        print(\"🎯 Training ensemble models...\")\n        \n        # Genetic feature selection (optional)\n        if self.use_genetic_features and X_train.shape[1] > 50:\n            self.feature_selector = GeneticFeatureSelector(\n                n_features=X_train.shape[1],\n                n_generations=15,\n                population_size=30\n            )\n            self.selected_features = self.feature_selector.run_genetic_algorithm(X_train, y_train)\n            X_train = X_train[:, self.selected_features]\n        \n        # Initialize models\n        self.initialize_models()\n        \n        # Prepare for stacking\n        n_samples = X_train.shape[0]\n        n_models = len(self.base_models)\n        \n        # OOF predictions for stacking\n        oof_predictions = np.zeros((n_samples, n_models))\n        \n        # K-Fold cross-validation\n        kf = KFold(n_splits=n_folds, shuffle=True, random_state=RANDOM_STATE)\n        \n        for fold_idx, (train_idx, val_idx) in enumerate(kf.split(X_train)):\n            print(f\"\\n  Fold {fold_idx + 1}/{n_folds}\")\n            \n            X_fold_train, X_fold_val = X_train[train_idx], X_train[val_idx]\n            y_fold_train, y_fold_val = y_train[train_idx], y_train[val_idx]\n            \n            # Train each base model\n            for model_idx, (name, model) in enumerate(self.base_models.items()):\n                print(f\"    Training {name}...\", end='')\n                \n                # Clone model to avoid fitting on same instance\n                model_clone = model.__class__(**model.get_params())\n                \n                # Fit model\n                if name in ['lgb', 'xgb', 'cat']:\n                    model_clone.fit(\n                        X_fold_train, y_fold_train,\n                        eval_set=[(X_fold_val, y_fold_val)],\n                        verbose=False\n                    )\n                else:\n                    model_clone.fit(X_fold_train, y_fold_train)\n                \n                # Store OOF predictions\n                oof_predictions[val_idx, model_idx] = model_clone.predict(X_fold_val)\n                \n                # Calculate fold score\n                fold_score = np.sqrt(mean_squared_error(y_fold_val, oof_predictions[val_idx, model_idx]))\n                print(f\" RMSE: {fold_score:.4f}\")\n        \n        # Train meta model on OOF predictions\n        print(\"\\n  Training meta model...\")\n        self.meta_model.fit(oof_predictions, y_train)\n        \n        # Calculate final OOF score\n        final_predictions = self.meta_model.predict(oof_predictions)\n        final_score = np.sqrt(mean_squared_error(y_train, final_predictions))\n        print(f\"\\n✅ Ensemble OOF RMSE: {final_score:.4f}\")\n        \n        # Retrain all models on full data\n        print(\"\\n  Retraining on full dataset...\")\n        for name, model in self.base_models.items():\n            if name in ['lgb', 'xgb', 'cat']:\n                model.fit(X_train, y_train, verbose=False)\n            else:\n                model.fit(X_train, y_train)\n        \n        return self\n    \n    def predict(self, X_test):\n        \"\"\"Make ensemble predictions\"\"\"\n        # Apply feature selection if used\n        if self.selected_features is not None:\n            X_test = X_test[:, self.selected_features]\n        \n        # Get base model predictions\n        base_predictions = np.zeros((X_test.shape[0], len(self.base_models)))\n        \n        for idx, (name, model) in enumerate(self.base_models.items()):\n            base_predictions[:, idx] = model.predict(X_test)\n        \n        # Meta model prediction\n        final_predictions = self.meta_model.predict(base_predictions)\n        \n        # Ensure non-negative predictions\n        final_predictions = np.maximum(final_predictions, 0)\n        \n        return final_predictions\n\n# ========================\n# MAIN PIPELINE\n# ========================\n\ndef main():\n    \"\"\"Main execution pipeline\"\"\"\n    \n    try:\n        # Feature engineering\n        print(\"\\n🔧 FEATURE ENGINEERING\")\n        print(\"=\"*50)\n        \n        feature_engineer = AdvancedFeatureEngineer()\n        train_df_fe = feature_engineer.engineer_features(train_df)\n        test_df_fe = feature_engineer.engineer_features(test_df)\n        \n        # Separate features and target\n        target_column = 'Premium Amount'\n        y_train = train_df_fe[target_column].values\n        X_train_df = train_df_fe.drop(columns=[target_column, 'id'])\n        X_test_df = test_df_fe.drop(columns=['id'])\n        \n        print(f\"✅ Features engineered: {X_train_df.shape[1]} features\")\n        \n        # Preprocessing\n        print(\"\\n🔄 PREPROCESSING\")\n        print(\"=\"*50)\n        \n        preprocessor = MemoryEfficientPreprocessor()\n        preprocessor.identify_features(X_train_df, target_column)\n        \n        X_train_processed = preprocessor.preprocess(X_train_df, y_train, is_train=True)\n        X_test_processed = preprocessor.preprocess(X_test_df, is_train=False)\n        \n        # Ensure same columns\n        train_cols = set(X_train_processed.columns)\n        test_cols = set(X_test_processed.columns)\n        \n        # Add missing columns to test\n        for col in train_cols - test_cols:\n            X_test_processed[col] = 0\n        \n        # Remove extra columns from test\n        X_test_processed = X_test_processed[X_train_processed.columns]\n        \n        print(f\"✅ Final shape - Train: {X_train_processed.shape}, Test: {X_test_processed.shape}\")\n        \n        # Convert to numpy arrays for modeling\n        X_train_final = X_train_processed.values\n        X_test_final = X_test_processed.values\n        \n        # Free memory\n        del train_df_fe, test_df_fe, X_train_df, X_test_df, X_train_processed, X_test_processed\n        gc.collect()\n        \n        # Model training\n        print(\"\\n🚀 MODEL TRAINING\")\n        print(\"=\"*50)\n        \n        ensemble = AdvancedEnsembleModel()\n        ensemble.train_with_cv(X_train_final, y_train, n_folds=5)\n        \n        # Make predictions\n        print(\"\\n📊 GENERATING PREDICTIONS\")\n        print(\"=\"*50)\n        \n        predictions = ensemble.predict(X_test_final)\n        \n        # Post-processing predictions\n        predictions = np.round(predictions, 2)\n        predictions = np.clip(predictions, 0, 5000)  # Reasonable bounds\n        \n        print(f\"✅ Predictions generated\")\n        print(f\"   Mean: ${predictions.mean():.2f}\")\n        print(f\"   Std: ${predictions.std():.2f}\")\n        print(f\"   Min: ${predictions.min():.2f}\")\n        print(f\"   Max: ${predictions.max():.2f}\")\n        \n        # Create submission\n        submission_df = pd.DataFrame({\n            'id': test_ids,\n            'Premium Amount': predictions\n        })\n        \n        # Save submission\n        submission_df.to_csv('submission_advanced.csv', index=False)\n        print(\"\\n✅ Submission saved to 'submission_advanced.csv'\")\n        \n        # Plot prediction distribution\n        plt.figure(figsize=(12, 5))\n        \n        plt.subplot(1, 2, 1)\n        plt.hist(y_train, bins=50, alpha=0.5, label='Training', color='blue')\n        plt.hist(predictions, bins=50, alpha=0.5, label='Predictions', color='red')\n        plt.xlabel('Premium Amount')\n        plt.ylabel('Frequency')\n        plt.title('Distribution Comparison')\n        plt.legend()\n        \n        plt.subplot(1, 2, 2)\n        plt.scatter(range(len(predictions[:1000])), predictions[:1000], alpha=0.5)\n        plt.xlabel('Sample Index')\n        plt.ylabel('Predicted Premium')\n        plt.title('First 1000 Predictions')\n        \n        plt.tight_layout()\n        plt.savefig('predictions_analysis.png')\n        plt.show()\n        \n        print(\"\\n\" + \"=\"*100)\n        print(\"🎉 PIPELINE COMPLETED SUCCESSFULLY!\")\n        print(f\"Finished at: {datetime.now()}\")\n        print(\"=\"*100)\n        \n    except Exception as e:\n        print(f\"\\n❌ Error occurred: {str(e)}\")\n        import traceback\n        traceback.print_exc()\n        \n    finally:\n        # Clean up memory\n        gc.collect()\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}