{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":94147,"databundleVersionId":11390004,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import h5py\nimport numpy as np\nimport cudf\nimport cupy as cp\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport optuna\n\nfrom cuml.preprocessing import MinMaxScaler\nfrom cuml.ensemble import RandomForestRegressor\nfrom sklearn.multioutput import MultiOutputRegressor\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.isotonic import IsotonicRegression\n\n# --------------------------------------\n# EDA & Data Preparation Pipeline Class\n# --------------------------------------\nclass CellTypeEDAPipeline:\n    \"\"\"\n    A unified pipeline for cell type prediction that performs EDA, hyperparameter optimization with Optuna,\n    model calibration, and submission generation.\n    \n    Steps:\n      1. Load training data from an H5 file.\n      2. Prepare training features and targets (assumes first two columns 'x' and 'y' are coordinates,\n         and remaining columns are continuous targets).\n      3. Perform basic EDA (descriptive statistics and histograms).\n    \"\"\"\n    def __init__(self, h5_file_path):\n        self.h5_file_path = h5_file_path\n        self.train_spot_tables = {}\n        self.cell_type_columns = None\n\n    def load_train_data(self):\n        with h5py.File(self.h5_file_path, \"r\") as f:\n            train_spots = f[\"spots/Train\"]\n            for slide_name in train_spots.keys():\n                spot_array = np.array(train_spots[slide_name])\n                df = pd.DataFrame(spot_array)\n                self.train_spot_tables[slide_name] = df\n        print(\"Training data loaded successfully.\")\n\n    def prepare_training_set(self, slide_id='S_1'):\n        if slide_id not in self.train_spot_tables:\n            raise ValueError(f\"Slide {slide_id} not found in training data.\")\n        df = self.train_spot_tables[slide_id]\n        feature_cols = ['x', 'y']\n        target_cols = [col for col in df.columns if col not in feature_cols]\n        self.cell_type_columns = target_cols\n        X = df[feature_cols].astype('float32')\n        y = df[target_cols].astype('float32')\n        return X, y\n\n    def perform_eda(self, slide_id='S_1'):\n        if slide_id not in self.train_spot_tables:\n            raise ValueError(f\"Slide {slide_id} not found in training data.\")\n        df = self.train_spot_tables[slide_id]\n        print(\"Descriptive statistics for slide\", slide_id)\n        print(df.describe())\n        df[['x', 'y']].hist(bins=30, figsize=(12,5))\n        plt.suptitle(\"Histograms of 'x' and 'y' features\")\n        plt.show()\n\n# --------------------------\n# Helper functions for test data and submission\n# --------------------------\ndef load_test_data(h5_file_path, slide_id):\n    with h5py.File(h5_file_path, \"r\") as f:\n        test_spots = f[\"spots/Test\"]\n        if slide_id not in test_spots:\n            raise ValueError(f\"Slide {slide_id} not found in test data.\")\n        spot_array = np.array(test_spots[slide_id])\n        df = pd.DataFrame(spot_array)\n    print(f\"Test data for slide {slide_id} loaded successfully.\")\n    return df\n\ndef create_submission(test_df, predictions, submission_filename, cell_type_columns):\n    pred_df = pd.DataFrame(predictions, columns=cell_type_columns, index=test_df.index)\n    pred_df.insert(0, 'ID', test_df.index)\n    pred_df.to_csv(submission_filename, index=False)\n    print(f\"Submission file '{submission_filename}' created!\")\n\ndef calibrate_model(best_model, X_train_np, y_train_np):\n    \"\"\"\n    Calibrates the regression predictions using isotonic regression for each target.\n    Returns a list of IsotonicRegression calibrators.\n    \"\"\"\n    y_train_pred = best_model.predict(X_train_np)\n    n_targets = y_train_np.shape[1]\n    calibrators = []\n    for i in range(n_targets):\n        iso = IsotonicRegression(out_of_bounds='clip')\n        iso.fit(y_train_pred[:, i], y_train_np[:, i])\n        calibrators.append(iso)\n    return calibrators\n\ndef apply_calibration(predictions, calibrators):\n    n_targets = predictions.shape[1]\n    calibrated_preds = np.zeros_like(predictions)\n    for i in range(n_targets):\n        calibrated_preds[:, i] = calibrators[i].predict(predictions[:, i])\n    return calibrated_preds\n\n# --------------------------\n# Optuna Hyperparameter Optimization\n# --------------------------\ndef objective(trial):\n    # Suggest hyperparameters.\n    n_estimators = trial.suggest_categorical('n_estimators', [50, 100, 200])\n    # Remove None option; use only integer values.\n    max_depth = trial.suggest_categorical('max_depth', [10, 20])\n    min_samples_split = trial.suggest_int('min_samples_split', 2, 10)\n    \n    # Create pipeline with suggested hyperparameters.\n    pipe = Pipeline([\n        ('scaler', MinMaxScaler()),\n        ('model', MultiOutputRegressor(\n            RandomForestRegressor(random_state=42,\n                                  n_estimators=n_estimators,\n                                  max_depth=max_depth,\n                                  min_samples_split=min_samples_split)\n        ))\n    ])\n    \n    # Evaluate using 5-fold CV.\n    cv = KFold(n_splits=5, shuffle=True, random_state=42)\n    mse_scores = []\n    for train_idx, val_idx in cv.split(X_train_np):\n        X_tr, X_val = X_train_np[train_idx], X_train_np[val_idx]\n        y_tr, y_val = y_train_np[train_idx], y_train_np[val_idx]\n        pipe.fit(X_tr, y_tr)\n        y_pred = pipe.predict(X_val)\n        mse = mean_squared_error(y_val, y_pred)\n        mse_scores.append(mse)\n    return np.mean(mse_scores)\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --------------------------\n# Main Code\n# --------------------------\nif __name__ == \"__main__\":\n    # Specify the path to the dataset.\n    h5_file_path = \"/kaggle/input/el-hackathon-2025/elucidata_ai_challenge_data.h5\"\n    \n    # Initialize EDA pipeline and load data.\n    eda_pipeline = CellTypeEDAPipeline(h5_file_path)\n    eda_pipeline.load_train_data()\n    eda_pipeline.perform_eda(slide_id='S_1')\n    X_train, y_train = eda_pipeline.prepare_training_set(slide_id='S_1')\n    \n    # Convert training data to numpy arrays.\n    if hasattr(X_train, 'to_pandas'):\n        X_train_np = X_train.to_pandas().values\n    else:\n        X_train_np = X_train.values\n    if hasattr(y_train, 'to_pandas'):\n        y_train_np = y_train.to_pandas().values\n    else:\n        y_train_np = y_train.values\n    \n    # Optimize hyperparameters with Optuna.\n    study = optuna.create_study(direction='minimize')\n    study.optimize(objective, n_trials=20)\n    print(\"Best trial:\")\n    best_trial = study.best_trial\n    print(\"  MSE:\", best_trial.value)\n    print(\"  Params:\", best_trial.params)\n    \n    # Build final pipeline with best hyperparameters.\n    best_pipe = Pipeline([\n        ('scaler', MinMaxScaler()),\n        ('model', MultiOutputRegressor(\n            RandomForestRegressor(\n                random_state=42,\n                n_estimators=best_trial.params['n_estimators'],\n                max_depth=best_trial.params['max_depth'],\n                min_samples_split=best_trial.params['min_samples_split']\n            )\n        ))\n    ])\n    best_pipe.fit(X_train_np, y_train_np)\n    \n    # Calibrate model predictions on training set.\n    calibrators = calibrate_model(best_pipe, X_train_np, y_train_np)\n    \n    # Load test data.\n    test_df = load_test_data(h5_file_path, slide_id='S_7')\n    # Prepare test features: select 'x' and 'y' and convert to numpy.\n    X_test = test_df[['x', 'y']].astype('float32')\n    X_test_np = X_test.values\n    \n    # Predict on test data using the best model.\n    test_predictions = best_pipe.predict(X_test_np)\n    \n    # Apply calibration to test predictions.\n    calibrated_predictions = apply_calibration(test_predictions, calibrators)\n    \n    # Create the submission CSV file with calibrated predictions.\n    create_submission(test_df, calibrated_predictions, submission_filename=\"submission.csv\",\n                      cell_type_columns=eda_pipeline.cell_type_columns)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}