{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":94147,"databundleVersionId":11390004,"sourceType":"competition"}],"dockerImageVersionId":30919,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import h5py\nimport numpy as np\nimport cudf\nimport cupy as cp\nimport pandas as pd  # Used only for CSV export if desired\nimport matplotlib.pyplot as plt\n\nfrom cuml.preprocessing import StandardScaler\nfrom cuml.ensemble import RandomForestRegressor\n\nclass GPUMultiOutputRegressor:\n    \"\"\"\n    A simple multi-output regressor that fits one cuML RandomForestRegressor per target.\n    \"\"\"\n    def __init__(self, estimator, n_estimators=100, random_state=42):\n        self.estimator = estimator\n        self.n_estimators = n_estimators\n        self.random_state = random_state\n        self.models_ = {}  # Dictionary to store one model per target\n\n    def fit(self, X, Y):\n        # Y is expected to be a cuDF DataFrame with each column a target variable.\n        for col in Y.columns:\n            # Initialize a new estimator for each target\n            model = self.estimator(n_estimators=self.n_estimators, random_state=self.random_state)\n            # Fit the model on the current target column\n            model.fit(X, Y[col])\n            self.models_[col] = model\n        return self\n\n    def predict(self, X):\n        # Predict each target separately and combine into a cuDF DataFrame.\n        preds = {}\n        for col, model in self.models_.items():\n            preds[col] = model.predict(X)\n        # Construct a cuDF DataFrame from the dictionary.\n        return cudf.DataFrame(preds)\n\nclass CellTypeGPUPipeline:\n    \"\"\"\n    A GPU-accelerated pipeline for cell type prediction using RAPIDS.\n    \n    This version includes feature engineering:\n      - It augments the original features (\"x\" and \"y\") with:\n          * x2: x^2\n          * y2: y^2\n          * xy: x * y\n          * dist: sqrt(x^2 + y^2)\n    \"\"\"\n    def __init__(self, h5_file_path):\n        self.h5_file_path = h5_file_path\n        self.train_spot_tables = {}\n        self.cell_type_columns = None\n        self.scaler = StandardScaler()  # cuML StandardScaler\n        self.model = None  # To be set after training\n\n    def load_train_data(self):\n        \"\"\"\n        Loads training spot data from the H5 file and converts each slide to a cuDF DataFrame.\n        \"\"\"\n        with h5py.File(self.h5_file_path, \"r\") as f:\n            train_spots = f[\"spots/Train\"]\n            for slide_name in train_spots.keys():\n                # Read the structured array and convert first to pandas then to cuDF\n                spot_array = np.array(train_spots[slide_name])\n                pdf = pd.DataFrame(spot_array)\n                df = cudf.DataFrame.from_pandas(pdf)\n                self.train_spot_tables[slide_name] = df\n        print(\"Training data loaded (GPU version) successfully.\")\n\n    def engineer_features(self, X):\n        \"\"\"\n        Applies feature engineering on the cuDF DataFrame X containing 'x' and 'y' columns.\n        \n        Adds:\n            - x2: x^2\n            - y2: y^2\n            - xy: x * y\n            - dist: sqrt(x^2 + y^2)\n        \"\"\"\n        X = X.copy()\n        X['x2'] = X['x'] ** 2\n        X['y2'] = X['y'] ** 2\n        X['xy'] = X['x'] * X['y']\n        # Compute the Euclidean distance using exponentiation\n        X['dist'] = (X['x'] ** 2 + X['y'] ** 2) ** 0.5\n        return X\n\n    def prepare_training_set(self, slide_id='S_1'):\n        \"\"\"\n        Prepares training features and targets from a given slide.\n        \n        Assumes that the first two columns are 'x' and 'y', and the remaining columns are cell type abundances.\n        Applies feature engineering to augment the original features.\n        \"\"\"\n        if slide_id not in self.train_spot_tables:\n            raise ValueError(f\"Slide {slide_id} not found in training data.\")\n        df = self.train_spot_tables[slide_id]\n        # Use original coordinate columns.\n        feature_cols = ['x', 'y']\n        target_cols = [col for col in df.columns if col not in feature_cols]\n        self.cell_type_columns = target_cols\n        # Convert features and targets to float32 for GPU ops.\n        X = df[feature_cols].astype('float32')\n        y = df[target_cols].astype('float32')\n        # Apply feature engineering\n        X = self.engineer_features(X)\n        return X, y\n\n    def load_test_data(self, slide_id):\n        \"\"\"\n        Loads test spot data for a given slide and returns a cuDF DataFrame.\n        \"\"\"\n        with h5py.File(self.h5_file_path, \"r\") as f:\n            test_spots = f[\"spots/Test\"]\n            if slide_id not in test_spots:\n                raise ValueError(f\"Slide {slide_id} not found in test data.\")\n            spot_array = np.array(test_spots[slide_id])\n            pdf = pd.DataFrame(spot_array)\n            test_df = cudf.DataFrame.from_pandas(pdf)\n        print(f\"Test data for slide {slide_id} loaded (GPU version) successfully.\")\n        return test_df\n\n    def prepare_test_set(self, test_df):\n        \"\"\"\n        Prepares test features by selecting coordinate columns and applying feature engineering.\n        \"\"\"\n        X = test_df[['x', 'y']].astype('float32')\n        X = self.engineer_features(X)\n        return X\n\n    def train(self, X_train, y_train):\n        \"\"\"\n        Scales training features and trains a multi-output model on GPU.\n        \"\"\"\n        # Fit the scaler and transform X_train on GPU.\n        X_train_scaled = self.scaler.fit_transform(X_train)\n        # Initialize and train a multi-output regressor using cuML's RandomForestRegressor.\n        gpu_multioutput = GPUMultiOutputRegressor(RandomForestRegressor)\n        gpu_multioutput.fit(X_train_scaled, y_train)\n        self.model = gpu_multioutput\n        print(\"GPU model training complete.\")\n        return self.model\n\n    def predict(self, X_test):\n        \"\"\"\n        Scales test features and predicts cell type abundances.\n        \"\"\"\n        X_test_scaled = self.scaler.transform(X_test)\n        predictions = self.model.predict(X_test_scaled)\n        return predictions\n\n    def create_submission(self, test_df, predictions, submission_filename=\"submission.csv\"):\n        \"\"\"\n        Creates a submission CSV file. Converts GPU DataFrames to pandas before export.\n        \"\"\"\n        pred_df = predictions\n        pred_df.insert(0, 'ID', test_df.index)\n        submission_pdf = pred_df.to_pandas()\n        submission_pdf.to_csv(submission_filename, index=False)\n        print(f\"Submission file '{submission_filename}' created!\")\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# Example usage:\nif __name__ == \"__main__\":\n    # Specify the path to your dataset file.\n    h5_file_path = \"/kaggle/input/el-hackathon-2025/elucidata_ai_challenge_data.h5\"\n    \n    # Initialize the pipeline.\n    pipeline = CellTypeGPUPipeline(h5_file_path)\n    \n    # Load training data and prepare the training set from slide 'S_1' with feature engineering.\n    pipeline.load_train_data()\n    X_train, y_train = pipeline.prepare_training_set(slide_id='S_1')\n    \n    # Train the final model on the full training set.\n    pipeline.train(X_train, y_train)\n    \n    # Load test data for slide 'S_7', prepare it with feature engineering, and make predictions.\n    test_df = pipeline.load_test_data(slide_id='S_7')\n    X_test = pipeline.prepare_test_set(test_df)\n    predictions = pipeline.predict(X_test)\n    \n    # Create and save the submission CSV file.\n    pipeline.create_submission(test_df, predictions, submission_filename=\"submission.csv\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}