{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":94147,"databundleVersionId":11390004,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import h5py\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport itertools\n\nfrom sklearn.ensemble import VotingRegressor, ExtraTreesRegressor\nfrom sklearn.svm import SVR\nfrom sklearn.multioutput import MultiOutputRegressor\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error\n\nclass CellTypeEnsemblePipeline:\n    \"\"\"\n    A pipeline for cell type prediction that loads data from an H5 file,\n    prepares features and targets, trains a multi-output ensemble regressor \n    (combining SVR and ExtraTreesRegressor) with cross-validation (including OOF predictions),\n    and performs finetuning via a simple grid search.\n    \n    Reproducibility is ensured using a fixed random_state.\n    \"\"\"\n    \n    def __init__(self, h5_file_path, random_state=42):\n        self.h5_file_path = h5_file_path\n        self.train_spot_tables = {}\n        self.cell_type_columns = None\n        self.model = None\n        self.random_state = random_state\n\n    def load_train_data(self):\n        \"\"\"\n        Loads training data from the H5 file and converts each slide to a pandas DataFrame.\n        \"\"\"\n        with h5py.File(self.h5_file_path, \"r\") as f:\n            train_spots = f[\"spots/Train\"]\n            for slide_name in train_spots.keys():\n                spot_array = np.array(train_spots[slide_name])\n                df = pd.DataFrame(spot_array)\n                self.train_spot_tables[slide_name] = df\n        print(\"Training data loaded successfully.\")\n\n    def prepare_training_set(self, slide_id='S_1'):\n        \"\"\"\n        Prepares training features and targets from a given slide.\n        Assumes that the first two columns are 'x' and 'y', and the remaining columns \n        are continuous cell type abundances.\n        \"\"\"\n        if slide_id not in self.train_spot_tables:\n            raise ValueError(f\"Slide {slide_id} not found in training data.\")\n        df = self.train_spot_tables[slide_id]\n        feature_cols = ['x', 'y']\n        target_cols = [col for col in df.columns if col not in feature_cols]\n        self.cell_type_columns = target_cols\n        X = df[feature_cols].values.astype(float)\n        y = df[target_cols].values.astype(float)\n        return X, y\n\n    def load_test_data(self, slide_id):\n        \"\"\"\n        Loads test data for a given slide and returns a pandas DataFrame.\n        \"\"\"\n        with h5py.File(self.h5_file_path, \"r\") as f:\n            test_spots = f[\"spots/Test\"]\n            if slide_id not in test_spots:\n                raise ValueError(f\"Slide {slide_id} not found in test data.\")\n            spot_array = np.array(test_spots[slide_id])\n            df = pd.DataFrame(spot_array)\n        print(f\"Test data for slide {slide_id} loaded successfully.\")\n        return df\n\n    def build_model(self, et_params=None):\n        \"\"\"\n        Builds a multi-output ensemble regressor using a VotingRegressor that combines\n        an SVR and an ExtraTreesRegressor.\n        \n        Parameters:\n            et_params (dict): ExtraTreesRegressor parameters. If not provided, defaults are used.\n        \"\"\"\n        et_params = et_params if et_params is not None else {}\n        # Ensure reproducibility in ExtraTreesRegressor\n        et_params.setdefault('random_state', self.random_state)\n        base_estimator = VotingRegressor(\n            estimators=[\n                ('svr', SVR()),\n                ('etr', ExtraTreesRegressor(**et_params))\n            ]\n        )\n        self.model = MultiOutputRegressor(base_estimator)\n        print(\"Model built successfully.\")\n        return self.model\n\n    def train(self, X, y, et_params=None):\n        \"\"\"\n        Trains the multi-output ensemble regressor on the full training set.\n        \n        Parameters:\n            X (np.array): Training features.\n            y (np.array): Training targets.\n            et_params (dict): ExtraTreesRegressor parameters.\n        \"\"\"\n        if self.model is None:\n            self.build_model(et_params=et_params)\n        self.model.fit(X, y)\n        print(\"Model training complete on full training set.\")\n        return self.model\n\n    def predict(self, X):\n        \"\"\"\n        Makes predictions on the provided feature array.\n        \"\"\"\n        if self.model is None:\n            raise ValueError(\"Model has not been trained.\")\n        predictions = self.model.predict(X)\n        return predictions\n\n    def create_submission(self, test_df, predictions, submission_filename=\"submission.csv\"):\n        \"\"\"\n        Creates a submission CSV file with spot IDs and predicted cell type abundances.\n        \"\"\"\n        pred_df = pd.DataFrame(predictions, columns=self.cell_type_columns, index=test_df.index)\n        pred_df.insert(0, 'ID', test_df.index)\n        pred_df.to_csv(submission_filename, index=False)\n        print(f\"Submission file '{submission_filename}' created!\")\n\n    def cross_validate(self, X, y, n_splits=5, callback=None, et_params=None):\n        \"\"\"\n        Performs K-Fold cross-validation with out-of-fold (OOF) predictions.\n        \n        Parameters:\n            X (np.array): Feature array.\n            y (np.array): Target array.\n            n_splits (int): Number of folds.\n            callback (function): Optional callback called after each fold with arguments:\n                fold_index, fold_mse, and validation indices.\n            et_params (dict): Parameters for ExtraTreesRegressor.\n        \n        Returns:\n            mse_scores (list): MSE scores for each fold.\n            oof_preds (np.array): Out-of-fold predictions.\n        \"\"\"\n        kf = KFold(n_splits=n_splits, shuffle=True, random_state=self.random_state)\n        n_samples = X.shape[0]\n        n_targets = y.shape[1]\n        oof_preds = np.zeros((n_samples, n_targets))\n        mse_scores = []\n        \n        for fold, (train_idx, val_idx) in enumerate(kf.split(X)):\n            X_train_fold = X[train_idx]\n            y_train_fold = y[train_idx]\n            X_val_fold = X[val_idx]\n            y_val_fold = y[val_idx]\n            \n            # Build and train a new model for this fold.\n            model = self.build_model(et_params=et_params)\n            model.fit(X_train_fold, y_train_fold)\n            y_pred = model.predict(X_val_fold)\n            \n            fold_mse = mean_squared_error(y_val_fold, y_pred)\n            mse_scores.append(fold_mse)\n            oof_preds[val_idx, :] = y_pred\n            \n            print(f\"Fold {fold+1} MSE: {fold_mse:.4f}\")\n            if callback is not None:\n                callback(fold_index=fold+1, fold_mse=fold_mse, val_indices=val_idx)\n                \n        return mse_scores, oof_preds\n\n    def finetune(self, X, y, param_grid, n_splits=5):\n        \"\"\"\n        Performs a simple grid search to finetune ExtraTreesRegressor hyperparameters.\n        \n        Parameters:\n            X (np.array): Feature array.\n            y (np.array): Target array.\n            param_grid (dict): Dictionary where keys are parameter names and values are lists of possible values.\n            n_splits (int): Number of folds for CV.\n        \n        Returns:\n            best_params (dict): Best parameter combination found.\n            best_score (float): Corresponding average CV MSE.\n        \"\"\"\n        best_params = None\n        best_score = float('inf')\n        param_names = list(param_grid.keys())\n        param_values = list(param_grid.values())\n        \n        for param_comb in itertools.product(*param_values):\n            params = dict(zip(param_names, param_comb))\n            print(f\"Testing params: {params}\")\n            mse_scores, _ = self.cross_validate(X, y, n_splits=n_splits, et_params=params)\n            avg_score = np.mean(mse_scores)\n            print(f\"Avg CV MSE for params {params}: {avg_score:.4f}\")\n            if avg_score < best_score:\n                best_score = avg_score\n                best_params = params\n                \n        print(f\"Best params: {best_params}, with Avg CV MSE: {best_score:.4f}\")\n        return best_params, best_score\n\n# Example callback function.\ndef fold_callback(fold_index, fold_mse, val_indices):\n    print(f\"Callback: Fold {fold_index} completed. MSE: {fold_mse:.4f}, Validation samples: {len(val_indices)}\")\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Example usage.\nif __name__ == \"__main__\":\n    # Path to the dataset file.\n    h5_file_path = \"/kaggle/input/el-hackathon-2025/elucidata_ai_challenge_data.h5\"\n    \n    # Initialize pipeline with reproducibility.\n    pipeline = CellTypeEnsemblePipeline(h5_file_path, random_state=42)\n    \n    # Load training data and prepare the training set from slide 'S_1'.\n    pipeline.load_train_data()\n    X_train, y_train = pipeline.prepare_training_set(slide_id='S_1')\n    \n    # Perform K-Fold cross-validation with OOF predictions and a callback.\n    mse_scores, oof_predictions = pipeline.cross_validate(\n        X_train, y_train, n_splits=5, callback=fold_callback\n    )\n    print(\"Average CV MSE:\", np.mean(mse_scores))\n    \n    # Finetune hyperparameters for the ExtraTreesRegressor.\n    param_grid = {\n        'n_estimators': [50, 100],\n        'max_depth': [None, 10],\n        'min_samples_split': [2, 5]\n    }\n    best_params, best_score = pipeline.finetune(X_train, y_train, param_grid, n_splits=3)\n    \n    # Train final model on the full training set using the best hyperparameters.\n    pipeline.train(X_train, y_train, et_params=best_params)\n    \n    # Load test data for slide 'S_7' and make predictions.\n    test_df = pipeline.load_test_data(slide_id='S_7')\n    X_test = test_df[['x', 'y']].values.astype(float)\n    predictions = pipeline.predict(X_test)\n    \n    # Create the submission file.\n    pipeline.create_submission(test_df, predictions, submission_filename=\"submission.csv\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}