{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":94147,"databundleVersionId":11390004,"sourceType":"competition"}],"dockerImageVersionId":30918,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Credit goes to this notebook and author --> https://www.kaggle.com/code/muhammadqasimshabbir/kaggle-starter-notebook","metadata":{}},{"cell_type":"code","source":"import h5py\nimport cudf\nimport cupy as cp\nimport numpy as np\nfrom cuml.model_selection import train_test_split\nfrom cuml.linear_model import LinearRegression\n\nclass ElucidataGPUPipeline:\n    def __init__(self, h5_file_path, test_slide='S_7', submission_path='submission.csv', test_size=0.025, random_state=23):\n        self.h5_file_path = h5_file_path\n        self.test_slide = test_slide\n        self.submission_path = submission_path\n        self.test_size = test_size\n        self.random_state = random_state\n        self.train_df = None\n        self.X = None\n        self.y = None\n        self.X_train = None\n        self.X_valid = None\n        self.y_train = None\n        self.y_valid = None\n        self.models = {}\n        self.test_df = None\n        self.predictions = None\n\n    def load_train_data(self):\n        print(\"Loading training data on GPU...\")\n        with h5py.File(self.h5_file_path, \"r\") as f:\n            train_spots = f[\"spots/Train\"]\n            # Convert each slide's data to a cuDF DataFrame\n            train_spot_tables = {\n                slide_name: cudf.DataFrame(np.array(train_spots[slide_name]))\n                for slide_name in train_spots.keys()\n            }\n        # Concatenate all training data into one cuDF DataFrame\n        self.train_df = cudf.concat(list(train_spot_tables.values()), ignore_index=True)\n        print(f\"Training data loaded. Shape: {self.train_df.shape}\")\n\n    def prepare_data(self):\n        print(\"Preparing training data on GPU...\")\n        # Use spatial coordinates as features and the remaining columns as target labels.\n        self.X = self.train_df[['x', 'y']]\n        self.y = self.train_df.drop(columns=['x', 'y'])\n        # Use cuML's train_test_split to create GPU DataFrames for training/validation\n        self.X_train, self.X_valid, self.y_train, self.y_valid = train_test_split(\n            self.X, self.y, test_size=self.test_size, random_state=self.random_state\n        )\n        print(\"Data split into training and validation sets.\")\n\n    def define_models(self):\n        print(\"Defining GPU models...\")\n        # Since RAPIDS does not provide a GPU RANSACRegressor, we use cuML's LinearRegression.\n        self.models = {\n            \"LinearRegression\": LinearRegression()\n        }\n        print(\"Models defined:\", list(self.models.keys()))\n\n    def train_models(self):\n        print(\"Training models on GPU...\")\n        for name, model in self.models.items():\n            print(f\"Training {name}...\")\n            model.fit(self.X_train, self.y_train)\n        print(\"Model training complete.\")\n\n    def validate_models(self):\n        print(\"Validating models on the validation set...\")\n        preds_valid = {}\n        for name, model in self.models.items():\n            print(f\"Predicting with {name} on validation data...\")\n            preds_valid[name] = model.predict(self.X_valid)\n        return preds_valid\n\n    def load_test_data(self):\n        print(\"Loading test data on GPU...\")\n        with h5py.File(self.h5_file_path, \"r\") as f:\n            test_spots = f[\"spots/Test\"]\n            # Convert the selected test slide into a cuDF DataFrame\n            self.test_df = cudf.DataFrame(np.array(test_spots[self.test_slide]))\n        print(f\"Test data loaded. Shape: {self.test_df.shape}\")\n\n    def predict_test(self):\n        print(\"Predicting on test data on GPU...\")\n        X_test = self.test_df[['x', 'y']]\n        # Determine the number of target columns from training labels\n        num_targets = self.y.shape[1]\n        # Create a Cupy array to accumulate predictions\n        test_preds = cp.zeros((X_test.shape[0], num_targets))\n        for name, model in self.models.items():\n            print(f\"Predicting with {name}...\")\n            preds = model.predict(X_test)\n            # Ensure predictions are Cupy arrays\n            if isinstance(preds, cudf.DataFrame):\n                preds = preds.to_cupy()\n            elif isinstance(preds, np.ndarray):\n                preds = cp.asarray(preds)\n            test_preds += preds\n        test_preds /= len(self.models)\n        # Convert predictions back to a cuDF DataFrame\n        self.predictions = cudf.DataFrame(cp.asnumpy(test_preds), columns=self.y.columns)\n        print(\"Test predictions complete.\")\n\n    def create_submission(self):\n        print(\"Creating submission file...\")\n        submission_df = self.predictions.copy()\n        # Convert the test DataFrame index to a CPU pandas Index for CSV output\n        submission_df.insert(0, 'ID', self.test_df.index.to_pandas())\n        submission_df.to_csv(self.submission_path, index=False)\n        print(f\"Submission file '{self.submission_path}' created!\")\n\n    def run_pipeline(self):\n        self.load_train_data()\n        self.prepare_data()\n        self.define_models()\n        self.train_models()\n        _ = self.validate_models()  # Optionally use validation predictions\n        self.load_test_data()\n        self.predict_test()\n        self.create_submission()\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# Example usage:\nif __name__ == \"__main__\":\n    h5_file_path = \"/kaggle/input/el-hackathon-2025/elucidata_ai_challenge_data.h5\"\n    pipeline = ElucidataGPUPipeline(h5_file_path)\n    pipeline.run_pipeline()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}