{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":116062,"databundleVersionId":14084779,"sourceType":"competition"},{"sourceId":5549164,"sourceType":"datasetVersion","datasetId":3197305},{"sourceId":5792099,"sourceType":"datasetVersion","datasetId":3327296},{"sourceId":6247561,"sourceType":"datasetVersion","datasetId":3590060}],"dockerImageVersionId":31154,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### IMPORTING LIBRARIES","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\nimport time\nimport matplotlib.pyplot as plt\nimport os\n\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers, models\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping\nfrom sklearn.model_selection import train_test_split\nfrom keras import regularizers","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T20:14:02.801109Z","iopub.execute_input":"2025-11-04T20:14:02.801409Z","iopub.status.idle":"2025-11-04T20:14:16.728749Z","shell.execute_reply.started":"2025-11-04T20:14:02.801387Z","shell.execute_reply":"2025-11-04T20:14:16.728128Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"TensorFlow version: {tf.__version__}\")\nprint(f\"GPU Available: {tf.config.list_physical_devices('GPU')}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T20:14:16.729977Z","iopub.execute_input":"2025-11-04T20:14:16.730566Z","iopub.status.idle":"2025-11-04T20:14:17.236361Z","shell.execute_reply.started":"2025-11-04T20:14:16.730538Z","shell.execute_reply":"2025-11-04T20:14:17.235331Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### SETUP & CONFIGURATION","metadata":{}},{"cell_type":"code","source":"class config:\n    MAIN_DIR = \"/kaggle/input/cafa-6-protein-function-prediction\"\n    \n    num_labels = 500\n    n_epochs = 20  \n    batch_size = 64 #128 \n    lr = 0.0005 #5e-4  \n    \n  \n    weight_decay = 1e-5\n    \n    \n    use_mixed_precision = True\n    \n    device = '/GPU:0' if tf.config.list_physical_devices('GPU') else '/CPU:0'\n    \n    \nprint(f\"Using device: {config.device}\")\n\n\nembeds_map = {\n    \"T5\": \"t5embeds\",\n    \"ProtBERT\": \"protbert-embeddings-for-cafa5\",\n    \"EMS2\": \"cafa-5-ems-2-embeddings-numpy\"\n}\nembeds_dim = {\n    \"T5\": 1024,\n    \"ProtBERT\": 1024,\n    \"EMS2\": 1280\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T20:14:17.237357Z","iopub.execute_input":"2025-11-04T20:14:17.237653Z","iopub.status.idle":"2025-11-04T20:14:17.269602Z","shell.execute_reply.started":"2025-11-04T20:14:17.237629Z","shell.execute_reply":"2025-11-04T20:14:17.268690Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Loading the Data","metadata":{}},{"cell_type":"code","source":"def load_protein_data(datatype, embeddings_source):\n \n    base_path = f\"/kaggle/input/{embeds_map[embeddings_source]}/\"\n    \n  \n    embeds_path = os.path.join(base_path, f\"{datatype}_embeddings.npy\")\n    ids_path = os.path.join(base_path, f\"{datatype}_ids.npy\")\n    \n\n    if embeddings_source == \"T5\":\n        embeds_path = os.path.join(base_path, f\"{datatype}_embeds.npy\")\n\n    embeds = np.load(embeds_path)\n    ids = np.load(ids_path)\n    \n    if datatype == \"train\":\n        labels_path = f\"/kaggle/input/train-targets-top{config.num_labels}/train_targets_top{config.num_labels}.npy\"\n        labels = np.load(labels_path)\n        return embeds, labels, ids\n    else:\n        return embeds, ids","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T20:14:17.270475Z","iopub.execute_input":"2025-11-04T20:14:17.270746Z","iopub.status.idle":"2025-11-04T20:14:17.283905Z","shell.execute_reply.started":"2025-11-04T20:14:17.270722Z","shell.execute_reply":"2025-11-04T20:14:17.283200Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### MODEL ARCHITECTURE: 1D CNN ","metadata":{}},{"cell_type":"markdown","source":"we are building a 1D Convolutional Neural Network (CNN) for multi-label classification. Staring with input layer that reshapes the data so that is it fitted as per the NN requirements then we are applying 32 filters to get the baic features then 3 more Conv1D are applied to get the advances features. We are using GlobalAveragePooling layer so that the features are reduces to a compact form. Followed by dense layer and drop out to reduce overfitting. ","metadata":{}},{"cell_type":"code","source":"def build_cnn_model(input_dim, num_classes):\n\n    inputs = layers.Input(shape=(input_dim,))\n    x = layers.Reshape((input_dim, 1))(inputs)\n    \n    \n    x = layers.Conv1D(64, 7, padding='same')(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.Activation('relu')(x)\n    x = layers.SpatialDropout1D(0.1)(x)\n    \n\n    residual = layers.Conv1D(128, 1, strides=2, padding='same')(x)\n    residual = layers.BatchNormalization()(residual)\n    \n    x = layers.Conv1D(128, 5, padding='same')(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.Activation('relu')(x)\n    x = layers.SpatialDropout1D(0.1)(x)\n    x = layers.MaxPooling1D(2)(x)\n    x = layers.Add()([x, residual])\n    x = layers.Activation('relu')(x)\n    \n  \n    residual = layers.Conv1D(256, 1, strides=2, padding='same')(x)\n    residual = layers.BatchNormalization()(residual)\n    \n    x = layers.Conv1D(256, 3, padding='same')(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.Activation('relu')(x)\n    x = layers.SpatialDropout1D(0.2)(x)\n    x = layers.MaxPooling1D(2)(x)\n    x = layers.Add()([x, residual])\n    x = layers.Activation('relu')(x)\n    \n  \n    residual = layers.Conv1D(512, 1, padding='same')(x)\n    residual = layers.BatchNormalization()(residual)\n    \n    x = layers.Conv1D(512, 3, padding='same')(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.Activation('relu')(x)\n    x = layers.SpatialDropout1D(0.2)(x)\n    x = layers.Add()([x, residual])\n    x = layers.Activation('relu')(x)\n    \n\n    x = layers.GlobalAveragePooling1D()(x)\n    \n\n    x = layers.Dense(256, kernel_regularizer=regularizers.l2(0.001))(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.Activation('relu')(x)\n    x = layers.Dropout(0.5)(x)\n    \n    x = layers.Dense(128, kernel_regularizer=regularizers.l2(0.001))(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.Activation('relu')(x)\n    x = layers.Dropout(0.4)(x)\n    \n \n    \n    outputs = layers.Dense(num_classes, activation='sigmoid')(x)\n    \n    model = models.Model(inputs=inputs, outputs=outputs)\n    \n    return model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T20:14:17.286176Z","iopub.execute_input":"2025-11-04T20:14:17.286472Z","iopub.status.idle":"2025-11-04T20:14:17.300440Z","shell.execute_reply.started":"2025-11-04T20:14:17.286454Z","shell.execute_reply":"2025-11-04T20:14:17.299636Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### CUSTOM METRICS","metadata":{}},{"cell_type":"markdown","source":"F1-score is calculatedby tracking true positives, false positives, and false negatives during training. Predictions are converted to binary values using a defined threshold (default 0.5). From these values, precision and recall are computed, and the F1-score is derived.  The metric is reset after each epoch in order to track the metric correctly. The metric has configuration methods to ensure it is fully serializable and can be saved and loaded with the model. Keras will automatically serialize and deserialize this metric when the model is trained and reloaded, because the class is decorated with the @keras.utils.register_keras_serializable decorator.","metadata":{}},{"cell_type":"code","source":"@keras.utils.register_keras_serializable(package=\"Custom\", name=\"MultilabelF1Score\")\nclass MultilabelF1Score(keras.metrics.Metric):\n    \"\"\"\n    F1 Score metric with LOWER threshold for focal loss compatibility\n    \"\"\"\n    \n    def __init__(self, num_labels=500, threshold=0.1, name='f1_score', **kwargs):\n        # CRITICAL: Changed default threshold from 0.5 to 0.1\n        super().__init__(name=name, **kwargs)\n        self.num_labels = num_labels\n        self.threshold = threshold\n        self.true_positives = self.add_weight(name='tp', initializer='zeros')\n        self.false_positives = self.add_weight(name='fp', initializer='zeros')\n        self.false_negatives = self.add_weight(name='fn', initializer='zeros')\n    \n    def update_state(self, y_true, y_pred, sample_weight=None):\n        y_pred = tf.cast(y_pred > self.threshold, tf.float32)\n        y_true = tf.cast(y_true, tf.float32)\n        \n        tp = tf.reduce_sum(y_true * y_pred)\n        fp = tf.reduce_sum((1 - y_true) * y_pred)\n        fn = tf.reduce_sum(y_true * (1 - y_pred))\n        \n        self.true_positives.assign_add(tp)\n        self.false_positives.assign_add(fp)\n        self.false_negatives.assign_add(fn)\n    \n    def result(self):\n        precision = self.true_positives / (self.true_positives + self.false_positives + tf.keras.backend.epsilon())\n        recall = self.true_positives / (self.true_positives + self.false_negatives + tf.keras.backend.epsilon())\n        f1 = 2 * (precision * recall) / (precision + recall + tf.keras.backend.epsilon())\n        return f1\n    \n    def reset_state(self):\n        self.true_positives.assign(0)\n        self.false_positives.assign(0)\n        self.false_negatives.assign(0)\n    \n    def get_config(self):\n        config = super().get_config()\n        config.update({\n            'num_labels': self.num_labels,\n            'threshold': self.threshold\n        })\n        return config\n    \n    @classmethod\n    def from_config(cls, config):\n        return cls(**config)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T20:14:17.301105Z","iopub.execute_input":"2025-11-04T20:14:17.301377Z","iopub.status.idle":"2025-11-04T20:14:17.313717Z","shell.execute_reply.started":"2025-11-04T20:14:17.301357Z","shell.execute_reply":"2025-11-04T20:14:17.312966Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### THRESHOLD FINDING FUNCTION","metadata":{}},{"cell_type":"markdown","source":"A function is created for the optimal prediction threshold that gives the highest F1-score","metadata":{}},{"cell_type":"code","source":"def find_best_threshold(model, X_val, y_val, thresholds=np.arange(0.05, 0.55, 0.05)):\n    \n    predictions = model.predict(X_val, batch_size=config.batch_size, verbose=0)\n    \n    best_f1 = 0\n    best_thresh = 0.5\n    threshold_scores = []\n    \n    for thresh in thresholds:\n        y_pred_binary = (predictions > thresh).astype(np.float32)\n        \n        tp = np.sum(y_val * y_pred_binary)\n        fp = np.sum((1 - y_val) * y_pred_binary)\n        fn = np.sum(y_val * (1 - y_pred_binary))\n        \n        precision = tp / (tp + fp + 1e-7)\n        recall = tp / (tp + fn + 1e-7)\n        f1 = 2 * (precision * recall) / (precision + recall + 1e-7)\n        \n        threshold_scores.append({\n            'threshold': thresh,\n            'f1': f1,\n            'precision': precision,\n            'recall': recall\n        })\n        \n        if f1 > best_f1:\n            best_f1 = f1\n            best_thresh = thresh\n    \n  \n    df_scores = pd.DataFrame(threshold_scores)\n    print(\"\\nThreshold Analysis:\")\n    print(df_scores.to_string())\n    \n    return best_f1, best_thresh","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T20:14:17.314592Z","iopub.execute_input":"2025-11-04T20:14:17.314854Z","iopub.status.idle":"2025-11-04T20:14:17.332137Z","shell.execute_reply.started":"2025-11-04T20:14:17.314830Z","shell.execute_reply":"2025-11-04T20:14:17.331347Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def augment_embeddings(embeddings, labels, augment_factor=0.2):\n    \n    n_augment = int(len(embeddings) * augment_factor)\n    \n   \n    indices = np.random.choice(len(embeddings), n_augment, replace=True)\n    \n    augmented_embeddings = embeddings[indices].copy()\n    augmented_labels = labels[indices].copy()\n    \n \n    noise = np.random.normal(0, 0.01, augmented_embeddings.shape)\n    augmented_embeddings += noise\n    \n   \n    X_combined = np.vstack([embeddings, augmented_embeddings])\n    y_combined = np.vstack([labels, augmented_labels])\n    \n    return X_combined, y_combined","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T20:14:17.332975Z","iopub.execute_input":"2025-11-04T20:14:17.333264Z","iopub.status.idle":"2025-11-04T20:14:17.350404Z","shell.execute_reply.started":"2025-11-04T20:14:17.333237Z","shell.execute_reply":"2025-11-04T20:14:17.349620Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### TRAINING FUNCTION","metadata":{}},{"cell_type":"markdown","source":"The function trains a convolutional neural network using the specified embedding source. It monitors performance on a validation set, calculates F1-score across multiple thresholds, and saves the model with the best F1 automatically. Finally, it returns the best model and optimal threshold for predictions.","metadata":{}},{"cell_type":"code","source":"def focal_loss(gamma=2.0, alpha=0.25):\n    def loss(y_true, y_pred):\n        y_pred = tf.clip_by_value(y_pred, 1e-7, 1 - 1e-7)\n        pt = tf.where(tf.equal(y_true, 1), y_pred, 1 - y_pred)\n        alpha_factor = tf.where(tf.equal(y_true, 1), alpha, 1 - alpha)\n        focal_weight = alpha_factor * tf.pow(1 - pt, gamma)\n        loss = -focal_weight * tf.math.log(pt)\n        return tf.reduce_mean(loss)\n    return loss\n\n\n\ndef train_model(embeddings_source, model_type=\"convolutional\", train_size=0.9, use_augmentation=True):\n    print(\"Loading training data...\")\n    X_train_full, y_train_full, ids = load_protein_data(\"train\", embeddings_source)\n    \n\n    X_train, X_val, y_train, y_val = train_test_split(\n        X_train_full, y_train_full, \n        train_size=train_size, \n        random_state=42\n    )\n    \n  \n    if use_augmentation:\n        print(\"Applying data augmentation...\")\n        X_train, y_train = augment_embeddings(X_train, y_train, augment_factor=0.2)\n        print(f\"Training samples after augmentation: {len(X_train)}\")\n    \n    print(f\"Training samples: {len(X_train)}, Validation samples: {len(X_val)}\")\n    \n  \n    model = build_cnn_model(\n        input_dim=embeds_dim[embeddings_source], \n        num_classes=config.num_labels\n    )\n    \n    model.compile(\n        optimizer=keras.optimizers.Adam(learning_rate=0.0005),\n        loss=focal_loss(gamma=2.0, alpha=0.25),\n        metrics=[\n            MultilabelF1Score(num_labels=config.num_labels, threshold=0.1),\n            keras.metrics.AUC(name='auc', multi_label=True)  # Add this\n        ]\n    )  \n\n    checkpoint = ModelCheckpoint(\n        'best_model.keras',\n        monitor='val_auc',  \n        save_best_only=True,\n        mode='max',\n        verbose=1\n    )\n    \n    early_stop = EarlyStopping(\n        monitor='val_auc',  \n        patience=5,\n        mode='max',\n        restore_best_weights=True,\n        verbose=1\n    )\n  \n    history = model.fit(\n        X_train, y_train,\n        batch_size=config.batch_size,\n        epochs=config.n_epochs,\n        validation_data=(X_val, y_val),\n        callbacks=[checkpoint, early_stop],\n        verbose=1\n    )\n    \n   \n    \n    \n  \n    best_val_f1, best_threshold = find_best_threshold(model, X_val, y_val)\n    print(f\"\\nBest Validation F1: {best_val_f1:.4f} at threshold {best_threshold:.2f}\")\n    \n    return model, best_threshold, history\n\n\nems2_model, best_threshold, history = train_model(\n    embeddings_source=\"EMS2\", \n    model_type=\"convolutional\",\n    use_augmentation=True  \n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T20:14:17.351224Z","iopub.execute_input":"2025-11-04T20:14:17.351506Z","iopub.status.idle":"2025-11-04T20:24:58.349810Z","shell.execute_reply.started":"2025-11-04T20:14:17.351478Z","shell.execute_reply":"2025-11-04T20:24:58.348996Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def predict_with_tta(model, X_test, threshold, n_tta=5):\n    \n    print(f\"Generating predictions with {n_tta} TTA iterations...\")\n    all_predictions = []\n    \n    for i in range(n_tta):\n        if i == 0:\n           \n            preds = model.predict(X_test, batch_size=config.batch_size, verbose=0)\n        else:\n           \n            X_noisy = X_test + np.random.normal(0, 0.005, X_test.shape)\n            preds = model.predict(X_noisy, batch_size=config.batch_size, verbose=0)\n        \n        all_predictions.append(preds)\n        print(f\"TTA iteration {i+1}/{n_tta} complete\")\n    \n    \n    final_predictions = np.mean(all_predictions, axis=0)\n    print(\"TTA averaging complete\")\n    \n    return final_predictions","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T20:24:58.350736Z","iopub.execute_input":"2025-11-04T20:24:58.351423Z","iopub.status.idle":"2025-11-04T20:24:58.357113Z","shell.execute_reply.started":"2025-11-04T20:24:58.351395Z","shell.execute_reply":"2025-11-04T20:24:58.356391Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def predict_ensemble(models, embeddings_source, thresholds, use_tta=False):\n    \n    print(\"\\n=== ENSEMBLE PREDICTION ===\")\n    print(f\"Number of models in ensemble: {len(models)}\")\n    \n    \n    print(\"\\nLoading test data...\")\n    X_test, test_ids = load_protein_data(\"test\", embeddings_source)\n    \n \n    labels_df = pd.read_csv(os.path.join(config.MAIN_DIR, \"Train/train_terms.tsv\"), sep=\"\\t\")\n    top_terms = labels_df.groupby(\"term\")[\"EntryID\"].count().sort_values(ascending=False)\n    labels_names = top_terms.head(config.num_labels).index.values\n    \n    \n    all_predictions = []\n    for idx, (model, threshold) in enumerate(zip(models, thresholds)):\n        print(f\"\\nModel {idx+1}/{len(models)} - Threshold: {threshold:.2f}\")\n        \n        if use_tta:\n            predictions = predict_with_tta(model, X_test, threshold, n_tta=3)\n        else:\n            predictions = model.predict(X_test, batch_size=config.batch_size, verbose=1)\n        \n        all_predictions.append(predictions)\n    \n  \n    print(\"\\nAveraging ensemble predictions...\")\n    ensemble_predictions = np.mean(all_predictions, axis=0)\n    \n    \n    ensemble_threshold = np.mean(thresholds)\n    print(f\"Using ensemble threshold: {ensemble_threshold:.2f}\")\n    \n   \n    results = []\n    for i, protein_id in enumerate(tqdm(test_ids, desc=\"Processing ensemble predictions\")):\n        protein_probs = ensemble_predictions[i]\n        go_indices = np.where(protein_probs > ensemble_threshold)[0]\n        for idx in go_indices:\n            results.append({\n                \"Id\": protein_id,\n                \"GO term\": labels_names[idx],\n                \"Confidence\": float(protein_probs[idx])\n            })\n    \n    submission_df = pd.DataFrame(results)\n    print(f\"ENSEMBLE PREDICTIONS COMPLETE. Generated {len(submission_df)} predictions.\")\n    \n    return submission_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T20:24:58.357945Z","iopub.execute_input":"2025-11-04T20:24:58.358212Z","iopub.status.idle":"2025-11-04T20:24:58.373922Z","shell.execute_reply.started":"2025-11-04T20:24:58.358189Z","shell.execute_reply":"2025-11-04T20:24:58.373129Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ndef predict_in_batches(model, embeddings_source, threshold, batch_size=5000):\n\n    print(\"\\n=== BATCH PREDICTION (Memory-Efficient) ===\")\n    \n   \n    print(\"Loading test data...\")\n    X_test, test_ids = load_protein_data(\"test\", embeddings_source)\n    \n\n    labels_df = pd.read_csv(os.path.join(config.MAIN_DIR, \"Train/train_terms.tsv\"), sep=\"\\t\")\n    top_terms = labels_df.groupby(\"term\")[\"EntryID\"].count().sort_values(ascending=False)\n    labels_names = top_terms.head(config.num_labels).index.values\n    \n    results = []\n    n_batches = (len(X_test) + batch_size - 1) // batch_size\n    \n    print(f\"Processing {len(X_test)} samples in {n_batches} batches...\")\n    \n    for i in tqdm(range(n_batches), desc=\"Batch prediction\"):\n        start_idx = i * batch_size\n        end_idx = min((i + 1) * batch_size, len(X_test))\n        \n        batch_X = X_test[start_idx:end_idx]\n        batch_ids = test_ids[start_idx:end_idx]\n        \n    \n        predictions = model.predict(batch_X, batch_size=128, verbose=0)\n        \n        \n        for j, protein_id in enumerate(batch_ids):\n            protein_probs = predictions[j]\n            go_indices = np.where(protein_probs > threshold)[0]\n            \n            for idx in go_indices:\n                results.append({\n                    \"Id\": protein_id,\n                    \"GO term\": labels_names[idx],\n                    \"Confidence\": float(protein_probs[idx])\n                })\n        \n     \n        if i % 10 == 0:\n            del predictions\n            import gc\n            gc.collect()\n            tf.keras.backend.clear_session()\n    \n    submission_df = pd.DataFrame(results)\n    print(f\"BATCH PREDICTIONS COMPLETE. Generated {len(submission_df)} predictions.\")\n    \n    return submission_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T20:24:58.374869Z","iopub.execute_input":"2025-11-04T20:24:58.375358Z","iopub.status.idle":"2025-11-04T20:24:58.391911Z","shell.execute_reply.started":"2025-11-04T20:24:58.375331Z","shell.execute_reply":"2025-11-04T20:24:58.391239Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ndef train_with_kfold(embeddings_source, n_folds=5, use_augmentation=True):\n  \n    from sklearn.model_selection import KFold\n    \n    print(\"\\n\" + \"=\"*70)\n    print(f\"TRAINING WITH {n_folds}-FOLD CROSS-VALIDATION\")\n    print(\"=\"*70)\n    \n   \n    print(\"\\nLoading training data...\")\n    X_full, y_full, ids = load_protein_data(\"train\", embeddings_source)\n    print(f\"Total samples: {len(X_full)}\")\n    \n    kfold = KFold(n_splits=n_folds, shuffle=True, random_state=42)\n    \n    models = []\n    thresholds = []\n    histories = []\n    fold_scores = []\n    \n    for fold, (train_idx, val_idx) in enumerate(kfold.split(X_full)):\n        print(\"\\n\" + \"=\"*70)\n        print(f\"FOLD {fold + 1}/{n_folds}\")\n        print(\"=\"*70)\n        \n      \n        X_train, X_val = X_full[train_idx], X_full[val_idx]\n        y_train, y_val = y_full[train_idx], y_full[val_idx]\n        \n      \n        if use_augmentation:\n            print(\"Applying data augmentation...\")\n            X_train, y_train = augment_embeddings(X_train, y_train, augment_factor=0.2)\n            print(f\"Augmented training samples: {len(X_train)}\")\n        \n        print(f\"Training: {len(X_train)}, Validation: {len(X_val)}\")\n        \n      \n        model = build_cnn_model(\n            input_dim=embeds_dim[embeddings_source],\n            num_classes=config.num_labels\n        )\n        \n        model.compile(\n            optimizer=keras.optimizers.Adam(learning_rate=0.0005),\n            loss=focal_loss(gamma=2.0, alpha=0.25),\n            metrics=[\n                MultilabelF1Score(num_labels=config.num_labels, threshold=0.1),\n                keras.metrics.AUC(name='auc', multi_label=True)  \n            ]\n        )  \n    \n        checkpoint = ModelCheckpoint(\n            'best_model.keras',\n            monitor='val_auc',  \n            save_best_only=True,\n            mode='max',\n            verbose=1\n        )\n        \n        early_stop = EarlyStopping(\n            monitor='val_auc',  \n            patience=5,\n            mode='max',\n            restore_best_weights=True,\n            verbose=1\n        )\n        \n       \n        history = model.fit(\n            X_train, y_train,\n            batch_size=config.batch_size,\n            epochs=config.n_epochs,\n            validation_data=(X_val, y_val),\n            callbacks=[checkpoint, early_stop],\n            verbose=1\n        )\n        \n\n        \n        \n       \n        best_f1, best_thresh = find_best_threshold(model, X_val, y_val)\n        \n        print(f\"\\nFold {fold+1} Results:\")\n        print(f\"  Best F1: {best_f1:.4f}\")\n        print(f\"  Best Threshold: {best_thresh:.2f}\")\n        \n        models.append(model)\n        thresholds.append(best_thresh)\n        histories.append(history)\n        fold_scores.append(best_f1)\n        \n     \n        tf.keras.backend.clear_session()\n    \n   \n    print(\"\\n\" + \"=\"*70)\n    print(\"CROSS-VALIDATION SUMMARY\")\n    print(\"=\"*70)\n    for fold, (score, thresh) in enumerate(zip(fold_scores, thresholds)):\n        print(f\"Fold {fold+1}: F1={score:.4f}, Threshold={thresh:.2f}\")\n    print(f\"\\nMean F1: {np.mean(fold_scores):.4f} ± {np.std(fold_scores):.4f}\")\n    print(f\"Mean Threshold: {np.mean(thresholds):.2f} ± {np.std(thresholds):.4f}\")\n    print(\"=\"*70)\n    \n    return models, thresholds, histories","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T20:24:58.392758Z","iopub.execute_input":"2025-11-04T20:24:58.393120Z","iopub.status.idle":"2025-11-04T20:24:58.409418Z","shell.execute_reply.started":"2025-11-04T20:24:58.393100Z","shell.execute_reply":"2025-11-04T20:24:58.408539Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_single_fold_for_ensemble(embeddings_source, fold_num, train_size=0.9, seed=None):\n   \n    if seed is None:\n        seed = fold_num * 42\n    \n    print(f\"\\n=== Training Ensemble Model {fold_num} (seed={seed}) ===\")\n    \n   \n    X_train_full, y_train_full, ids = load_protein_data(\"train\", embeddings_source)\n    \n\n    X_train, X_val, y_train, y_val = train_test_split(\n        X_train_full, y_train_full,\n        train_size=train_size,\n        random_state=seed\n    )\n    \n   \n    print(\"Applying data augmentation...\")\n    X_train, y_train = augment_embeddings(X_train, y_train, augment_factor=0.2)\n    print(f\"Training samples: {len(X_train)}, Validation: {len(X_val)}\")\n    \n  \n    model = build_cnn_model(\n        input_dim=embeds_dim[embeddings_source],\n        num_classes=config.num_labels\n    )\n    \n    model.compile(\n        optimizer=keras.optimizers.Adam(learning_rate=0.0005),\n        loss=focal_loss(gamma=2.0, alpha=0.25),\n        metrics=[\n            MultilabelF1Score(num_labels=config.num_labels, threshold=0.1),\n            keras.metrics.AUC(name='auc', multi_label=True)  # Add this\n        ]\n    )  \n\n    checkpoint = ModelCheckpoint(\n        'best_model.keras',\n        monitor='val_auc',  \n        save_best_only=True,\n        mode='max',\n        verbose=1\n    )\n    \n    early_stop = EarlyStopping(\n        monitor='val_auc',  \n        patience=5,\n        mode='max',\n        restore_best_weights=True,\n        verbose=1\n    )\n    \n    history = model.fit(\n        X_train, y_train,\n        batch_size=config.batch_size,\n        epochs=config.n_epochs,\n        validation_data=(X_val, y_val),\n        callbacks=[checkpoint, early_stop],\n        verbose=1\n    )\n    \n  \n    model = keras.models.load_model(f'ensemble_model_{fold_num}.keras')\n    best_f1, best_thresh = find_best_threshold(model, X_val, y_val)\n    \n    print(f\"Model {fold_num} - F1: {best_f1:.4f}, Threshold: {best_thresh:.2f}\")\n    \n    tf.keras.backend.clear_session()\n    \n    return model, best_thresh","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T20:24:58.411968Z","iopub.execute_input":"2025-11-04T20:24:58.412634Z","iopub.status.idle":"2025-11-04T20:24:58.425629Z","shell.execute_reply.started":"2025-11-04T20:24:58.412607Z","shell.execute_reply":"2025-11-04T20:24:58.424844Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### GENERATING PREDICTIONS ","metadata":{}},{"cell_type":"code","source":"\nprint(\"Starting K-Fold Cross-Validation Training...\")\nkfold_models, kfold_thresholds, kfold_histories = train_with_kfold(\n    embeddings_source=\"EMS2\",\n    n_folds=5,\n    use_augmentation=True\n)\n\n\nsubmission_df = predict_ensemble(\n    models=kfold_models,\n    embeddings_source=\"EMS2\",\n    thresholds=kfold_thresholds,\n    use_tta=True  \n)\n\nprint(\"K-FOLD ENSEMBLE PREDICTION COMPLETE!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T20:24:58.426353Z","iopub.execute_input":"2025-11-04T20:24:58.426601Z","iopub.status.idle":"2025-11-04T21:21:18.069647Z","shell.execute_reply.started":"2025-11-04T20:24:58.426583Z","shell.execute_reply":"2025-11-04T21:21:18.068921Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### SUBMISSION FILE GENERATION ","metadata":{}},{"cell_type":"code","source":"print(\"\\nMerging submission files...\")\n\n\nsubmission2 = pd.read_csv('/kaggle/input/blast-quick-sprof-zero-pred/submission.tsv',\n                          sep='\\t', header=None, names=['Id', 'GO term', 'Confidence2'])\n\n\nsubs = pd.merge(submission_df, submission2, on=['Id', 'GO term'], how='outer')\n\n\nsubs['Confidence_combined'] = subs['Confidence2'].fillna(subs['Confidence'])\n\n\nfinal_submission = subs[['Id', 'GO term', 'Confidence_combined']]\nfinal_submission.to_csv('submission.tsv', sep='\\t', header=False, index=False)\n\nprint(\"Submission file 'submission.tsv' created successfully!\")\nprint(f\"It contains {len(final_submission)} predictions in total.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T21:21:18.070449Z","iopub.execute_input":"2025-11-04T21:21:18.070687Z","iopub.status.idle":"2025-11-04T21:22:07.072687Z","shell.execute_reply.started":"2025-11-04T21:21:18.070670Z","shell.execute_reply":"2025-11-04T21:22:07.071832Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Acknowledgement: - [https://www.kaggle.com/code/momerer/cafa-6-protein-function-prediction-with-1d-cnn](https://www.kaggle.com/code/momerer/cafa-6-protein-function-prediction-with-1d-cnn)","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}