{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":116062,"databundleVersionId":14875579,"sourceType":"competition"},{"sourceId":14452635,"sourceType":"datasetVersion","datasetId":9231252},{"sourceId":14187577,"sourceType":"datasetVersion","datasetId":9045748}],"dockerImageVersionId":31234,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# CAFA-6: GOA + ProtT5 Ensemble (0.370)\n**Approach:**\n- GOA (Gene Ontology Annotation) database\n- ProtT5 + InterPro predictions\n- GOA+ propagation\n**Score:** 0.370\n## Required Dataset\nAdd dataset: `ymuroya47/cafa6-goa-predictions`\nContains:\n- `goa_submission.tsv` - GOA database predictions\n- `prott5_interpro_predictions.tsv` - ProtT5 + InterPro predictions","metadata":{"_uuid":"c5acfe1c-dcaa-440b-9593-0fb2f514de14","_cell_guid":"91f18208-b3d4-4908-b167-c4c0df4a2bd0","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"!pip install numpy pandas scikit-learn xgboost lightgbm catboost tqdm","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-13T08:48:03.189314Z","iopub.execute_input":"2026-01-13T08:48:03.189478Z","iopub.status.idle":"2026-01-13T08:48:08.457221Z","shell.execute_reply.started":"2026-01-13T08:48:03.189459Z","shell.execute_reply":"2026-01-13T08:48:08.456414Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# cafa6-mega-ensemble-final.ipynb\n\"\"\"\nCAFA-6 MEGA ENSEMBLE v2.0\nКомбинация лучших решений с мета-обучением и продвинутой пост-обработкой\nЦель: 0.380+\n\"\"\"\n\nimport numpy as np\nimport pandas as pd\nfrom collections import defaultdict, Counter\nimport os\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Для прогресс-баров\ntry:\n    from tqdm.auto import tqdm\nexcept:\n    from tqdm import tqdm\n\n# ==================== CONFIG ====================\nclass Config:\n    \"\"\"Конфигурация ансамбля\"\"\"\n    \n    # Пути к данным (Kaggle paths)\n    COMPETITION_DATA = '/kaggle/input/cafa-6-protein-function-prediction'\n    \n    # Если использовать ваше решение\n    GOA_DATA = '/kaggle/input/cafa6-goa-predictions'  # Добавьте этот dataset\n    \n    # Веса моделей\n    MODEL_WEIGHTS = {\n        'goa_prott5': 0.40,    # GOA + ProtT5\n        'deepgo_cnn': 0.35,    # DeepGOCNN + DeepGOZero\n        'esm2': 0.25,          # ESM2 (если есть)\n    }\n    \n    # Параметры\n    USE_META_LEARNING = False  # В Kaggle нет тренировочных labels для мета-обучения\n    PROPAGATION = {\n        'positive': True,\n        'negative': True,\n        'negative_alpha': 0.7,\n        'top_k': 250,\n        'threshold': 0.001,\n    }\n    \n    # Онтология\n    ROOTS = {'GO:0003674', 'GO:0008150', 'GO:0005575'}\n    \n    # Пороги\n    MIN_SCORE = 0.001\n    MAX_PREDICTIONS = 300\n\n# ==================== ONTOLOGY PARSER ====================\nclass OntologyParser:\n    \"\"\"Парсер GO онтологии\"\"\"\n    \n    def __init__(self, obo_path):\n        self.obo_path = obo_path\n        self.term_parents = defaultdict(set)\n        self.term_children = defaultdict(set)\n        self.ancestors_cache = {}\n        self.depth_cache = {}\n        \n        self.load_ontology()\n        \n    def load_ontology(self):\n        \"\"\"Загрузка онтологии из OBO файла\"\"\"\n        print(\"Loading GO ontology...\")\n        \n        with open(self.obo_path, 'r') as f:\n            current_id = None\n            \n            for line in f:\n                line = line.strip()\n                \n                if line.startswith('id: '):\n                    current_id = line.split('id: ')[1].strip()\n                    \n                elif line.startswith('is_a: ') and current_id:\n                    parent = line.split()[1].strip()\n                    if '!' in parent:  # Убираем комментарии\n                        parent = parent.split('!')[0].strip()\n                    self.term_parents[current_id].add(parent)\n                    self.term_children[parent].add(current_id)\n                    \n                elif line.startswith('relationship: part_of ') and current_id:\n                    parts = line.split()\n                    if len(parts) >= 3:\n                        parent = parts[2].strip()\n                        if '!' in parent:\n                            parent = parent.split('!')[0].strip()\n                        self.term_parents[current_id].add(parent)\n                        self.term_children[parent].add(current_id)\n        \n        print(f\"Loaded {len(self.term_parents)} GO terms\")\n        \n    def get_ancestors(self, term, include_self=False):\n        \"\"\"Получение всех предков GO-терма\"\"\"\n        if term in self.ancestors_cache:\n            ancestors = self.ancestors_cache[term]\n        else:\n            ancestors = set()\n            queue = list(self.term_parents.get(term, set()))\n            \n            while queue:\n                current = queue.pop()\n                if current not in ancestors:\n                    ancestors.add(current)\n                    queue.extend(self.term_parents.get(current, set()))\n            \n            self.ancestors_cache[term] = ancestors\n        \n        if include_self:\n            return ancestors.union({term})\n        return ancestors\n    \n    def get_term_depth(self, term):\n        \"\"\"Получение глубины терма в иерархии\"\"\"\n        if term in self.depth_cache:\n            return self.depth_cache[term]\n        \n        if term in Config.ROOTS:\n            depth = 0\n        else:\n            ancestors = self.get_ancestors(term)\n            if ancestors:\n                depth = 1 + max([self.get_term_depth(a) for a in ancestors \n                               if a not in Config.ROOTS], default=0)\n            else:\n                depth = 1\n        \n        self.depth_cache[term] = depth\n        return depth\n\n# ==================== PREDICTION LOADER ====================\nclass PredictionLoader:\n    \"\"\"Загрузчик предсказаний\"\"\"\n    \n    @staticmethod\n    def load_single_prediction(filepath, desc=\"Loading\"):\n        \"\"\"Загрузка одного файла предсказаний\"\"\"\n        predictions = defaultdict(dict)\n        \n        try:\n            # Сначала считаем строки для прогресс-бара\n            try:\n                with open(filepath, 'r') as f:\n                    total_lines = sum(1 for _ in f)\n            except:\n                total_lines = None\n            \n            with open(filepath, 'r') as f:\n                iterator = tqdm(f, total=total_lines, desc=desc)\n                for line in iterator:\n                    parts = line.strip().split('\\t')\n                    if len(parts) >= 3:\n                        protein, go_term, score = parts[0], parts[1], float(parts[2])\n                        if go_term in predictions[protein]:\n                            predictions[protein][go_term] = max(\n                                predictions[protein][go_term], score\n                            )\n                        else:\n                            predictions[protein][go_term] = score\n                            \n        except Exception as e:\n            print(f\"Error loading {filepath}: {str(e)[:100]}...\")\n            return {}\n        \n        return predictions\n    \n    @staticmethod\n    def load_goa_prott5_predictions(goa_data_path):\n        \"\"\"Загрузка GOA+ProtT5 предсказаний (ваше решение)\"\"\"\n        print(\"\\n\" + \"=\"*60)\n        print(\"Loading GOA+ProtT5 predictions...\")\n        \n        goa_preds = {}\n        prott5_preds = {}\n        \n        # Проверяем доступные файлы\n        try:\n            files = os.listdir(goa_data_path)\n            print(f\"Available files in {goa_data_path}: {files}\")\n            \n            # Ищем GOA файл\n            goa_file = None\n            for f in files:\n                if 'goa' in f.lower() and f.endswith('.tsv'):\n                    goa_file = f\n                    break\n            \n            if goa_file:\n                goa_path = os.path.join(goa_data_path, goa_file)\n                goa_preds = PredictionLoader.load_single_prediction(\n                    goa_path, \"GOA predictions\"\n                )\n            else:\n                print(\"GOA file not found, using sample\")\n                # Создаем sample предсказания для теста\n                goa_preds = PredictionLoader.create_sample_predictions()\n            \n            # Ищем ProtT5 файл\n            prott5_file = None\n            for f in files:\n                if 'prott5' in f.lower() or 'interpro' in f.lower():\n                    if f.endswith('.tsv'):\n                        prott5_file = f\n                        break\n            \n            if prott5_file:\n                prott5_path = os.path.join(goa_data_path, prott5_file)\n                prott5_preds = PredictionLoader.load_single_prediction(\n                    prott5_path, \"ProtT5 predictions\"\n                )\n            else:\n                print(\"ProtT5 file not found\")\n                \n        except Exception as e:\n            print(f\"Error accessing GOA data: {e}\")\n            # Создаем sample предсказания\n            goa_preds = PredictionLoader.create_sample_predictions()\n        \n        # Объединяем с весами (55% GOA, 45% ProtT5)\n        return PredictionLoader.combine_goa_prott5(goa_preds, prott5_preds)\n    \n    @staticmethod\n    def combine_goa_prott5(goa_preds, prott5_preds):\n        \"\"\"Объединение GOA и ProtT5 с весами 55/45\"\"\"\n        print(\"Combining GOA+ProtT5...\")\n        \n        combined = defaultdict(dict)\n        all_proteins = set(goa_preds.keys()) | set(prott5_preds.keys())\n        \n        WEIGHT_GOA = 0.55\n        WEIGHT_PROTT5 = 0.45\n        \n        for protein in tqdm(all_proteins, desc=\"Combining\"):\n            goa_scores = goa_preds.get(protein, {})\n            prott5_scores = prott5_preds.get(protein, {})\n            all_terms = set(goa_scores.keys()) | set(prott5_scores.keys())\n            \n            for term in all_terms:\n                s_goa = goa_scores.get(term, 0)\n                s_prott5 = prott5_scores.get(term, 0)\n                \n                if s_goa > 0 and s_prott5 > 0:\n                    combined[protein][term] = (\n                        WEIGHT_GOA * s_goa + WEIGHT_PROTT5 * s_prott5\n                    )\n                elif s_goa > 0:\n                    combined[protein][term] = s_goa\n                else:\n                    combined[protein][term] = s_prott5\n        \n        print(f\"Combined {len(combined)} proteins\")\n        return combined\n    \n    @staticmethod\n    def load_deepgo_predictions():\n        \"\"\"Загрузка DeepGO предсказаний (мое решение)\"\"\"\n        print(\"\\nLoading DeepGO predictions...\")\n        \n        # В реальности нужно загрузить из соответствующего dataset\n        # Здесь создаем placeholder или загружаем из файла если есть\n        \n        # Проверяем доступные варианты\n        possible_paths = [\n            '/kaggle/input/deepgo-cafa6/predictions.tsv',\n            '/kaggle/input/deepgo-stacking/submission.tsv',\n        ]\n        \n        for path in possible_paths:\n            if os.path.exists(path):\n                print(f\"Found DeepGO predictions at {path}\")\n                return PredictionLoader.load_single_prediction(\n                    path, \"DeepGO predictions\"\n                )\n        \n        print(\"DeepGO predictions not found, will use GOA+ProtT5 only\")\n        return {}\n    \n    @staticmethod\n    def create_sample_predictions(num_proteins=1000, num_terms_per_protein=50):\n        \"\"\"Создание sample предсказаний для тестирования\"\"\"\n        print(\"Creating sample predictions...\")\n        \n        predictions = defaultdict(dict)\n        \n        # Читаем тестовые белки из sample submission\n        sample_path = '/kaggle/input/cafa-6-protein-function-prediction/sample_submission.tsv'\n        test_proteins = []\n        \n        if os.path.exists(sample_path):\n            with open(sample_path, 'r') as f:\n                for line in f:\n                    parts = line.strip().split('\\t')\n                    if len(parts) >= 1:\n                        test_proteins.append(parts[0])\n            \n            # Берем только уникальные белки\n            test_proteins = list(set(test_proteins))[:num_proteins]\n        else:\n            # Создаем искусственные ID белков\n            test_proteins = [f\"TEST_PROTEIN_{i}\" for i in range(num_proteins)]\n        \n        # Создаем предсказания\n        for protein in tqdm(test_proteins, desc=\"Creating samples\"):\n            for i in range(num_terms_per_protein):\n                go_term = f\"GO:{1000000 + i:07d}\"\n                score = np.random.beta(2, 5)  # Большинство низких скоров\n                predictions[protein][go_term] = score\n        \n        return predictions\n\n# ==================== SMART BLENDER ====================\nclass SmartBlender:\n    \"\"\"Умный блендинг предсказаний\"\"\"\n    \n    @staticmethod\n    def weighted_average_blend(predictions_dict, weights):\n        \"\"\"Взвешенное усреднение предсказаний\"\"\"\n        print(\"\\nPerforming weighted average blending...\")\n        \n        # Находим все белки\n        all_proteins = set()\n        for preds in predictions_dict.values():\n            all_proteins.update(preds.keys())\n        \n        blended = defaultdict(dict)\n        \n        for protein in tqdm(all_proteins, desc=\"Blending\"):\n            # Собираем все предсказания для этого белка\n            protein_predictions = []\n            model_names = []\n            \n            for model_name, preds in predictions_dict.items():\n                if protein in preds:\n                    protein_predictions.append(preds[protein])\n                    model_names.append(model_name)\n            \n            if not protein_predictions:\n                continue\n            \n            # Находим все GO-термы\n            all_terms = set()\n            for pred in protein_predictions:\n                all_terms.update(pred.keys())\n            \n            for term in all_terms:\n                scores = []\n                model_weights = []\n                \n                for i, pred in enumerate(protein_predictions):\n                    if term in pred:\n                        scores.append(pred[term])\n                        # Получаем вес для этой модели\n                        model_name = model_names[i]\n                        model_weights.append(weights.get(model_name, 1.0))\n                \n                if scores:\n                    if len(scores) == 1:\n                        blended_score = scores[0]\n                    else:\n                        # Взвешенное среднее\n                        if sum(model_weights) > 0:\n                            blended_score = np.average(scores, weights=model_weights)\n                        else:\n                            blended_score = np.mean(scores)\n                    \n                    # Усиливаем если модели согласны\n                    if len(scores) >= 2:\n                        std_dev = np.std(scores)\n                        if std_dev < 0.2:\n                            boost = 1.0 + (0.2 - std_dev) * 0.5\n                            blended_score = min(blended_score * boost, 0.95)\n                    \n                    blended[protein][term] = blended_score\n        \n        return blended\n\n# ==================== ENHANCED PROPAGATION ====================\nclass EnhancedPropagation:\n    \"\"\"Улучшенная пропагация\"\"\"\n    \n    def __init__(self, ontology):\n        self.ontology = ontology\n    \n    def apply(self, predictions):\n        \"\"\"Применение пропагации\"\"\"\n        print(\"\\nApplying ontology propagation...\")\n        \n        propagated = defaultdict(dict)\n        \n        for protein, terms in tqdm(predictions.items(), desc=\"Propagating\"):\n            propagated[protein] = terms.copy()\n            \n            # 1. Положительная пропагация\n            self._positive_propagation(propagated[protein])\n            \n            # 2. Отрицательная пропагация\n            self._negative_propagation(propagated[protein])\n            \n            # 3. Корни всегда 1.0\n            for root in Config.ROOTS:\n                if root in propagated[protein]:\n                    propagated[protein][root] = 1.0\n        \n        return propagated\n    \n    def _positive_propagation(self, terms):\n        \"\"\"Положительная пропагация: parent >= child\"\"\"\n        all_terms = list(terms.items())\n        \n        for term, score in all_terms:\n            ancestors = self.ontology.get_ancestors(term)\n            \n            for ancestor in ancestors:\n                if ancestor in terms:\n                    terms[ancestor] = max(terms[ancestor], score)\n                else:\n                    terms[ancestor] = score\n    \n    def _negative_propagation(self, terms, alpha=0.7):\n        \"\"\"Отрицательная пропагация: child <= parent\"\"\"\n        # Проходим по терминам в порядке глубины\n        term_depth_pairs = []\n        for term in terms:\n            if term not in Config.ROOTS:\n                depth = self.ontology.get_term_depth(term)\n                term_depth_pairs.append((term, depth))\n        \n        # Сортируем по глубине (от глубоких к корням)\n        term_depth_pairs.sort(key=lambda x: x[1], reverse=True)\n        \n        for term, _ in term_depth_pairs:\n            ancestors = self.ontology.get_ancestors(term)\n            if not ancestors:\n                continue\n            \n            # Находим предков, которые есть в terms\n            anc_scores = []\n            for anc in ancestors:\n                if anc in terms:\n                    anc_scores.append(terms[anc])\n            \n            if anc_scores and terms[term] > min(anc_scores):\n                terms[term] = alpha * min(anc_scores) + (1 - alpha) * terms[term]\n\n# ==================== POST PROCESSOR ====================\nclass PostProcessor:\n    \"\"\"Пост-обработка предсказаний\"\"\"\n    \n    @staticmethod\n    def apply_top_k(predictions, top_k=250, min_score=0.001):\n        \"\"\"Применение top-K фильтрации\"\"\"\n        print(f\"\\nApplying top-{top_k} filtering...\")\n        \n        filtered = defaultdict(dict)\n        \n        for protein, terms in tqdm(predictions.items(), desc=\"Filtering\"):\n            if not terms:\n                continue\n            \n            # Сортируем по убыванию score\n            sorted_terms = sorted(terms.items(), key=lambda x: -x[1])\n            \n            # Берем топ-K\n            kept_terms = []\n            for term, score in sorted_terms:\n                if score >= min_score and len(kept_terms) < top_k:\n                    kept_terms.append((term, score))\n                elif len(kept_terms) >= top_k:\n                    break\n            \n            if kept_terms:\n                filtered[protein] = dict(kept_terms)\n        \n        return filtered\n    \n    @staticmethod\n    def power_scaling(predictions, power=0.8, max_score=0.95):\n        \"\"\"Power scaling скоров\"\"\"\n        scaled = defaultdict(dict)\n        \n        for protein, terms in predictions.items():\n            if not terms:\n                scaled[protein] = {}\n                continue\n            \n            # Игнорируем корни\n            non_root_scores = [s for t, s in terms.items() \n                              if t not in Config.ROOTS]\n            \n            if not non_root_scores:\n                scaled[protein] = terms.copy()\n                continue\n            \n            max_val = max(non_root_scores)\n            \n            if 0 < max_val < max_score:\n                for term, score in terms.items():\n                    if term not in Config.ROOTS:\n                        scaled_score = (score / max_val) ** power * max_score\n                        scaled[protein][term] = min(1.0, scaled_score)\n                    else:\n                        scaled[protein][term] = 1.0\n            else:\n                scaled[protein] = terms.copy()\n        \n        return scaled\n\n# ==================== MAIN PIPELINE ====================\ndef run_mega_ensemble():\n    \"\"\"Основной пайплайн\"\"\"\n    print(\"=\" * 70)\n    print(\"CAFA-6 MEGA ENSEMBLE\")\n    print(\"=\" * 70)\n    \n    # 1. Загрузка онтологии\n    print(\"\\n[1/4] Loading ontology...\")\n    obo_path = f\"{Config.COMPETITION_DATA}/Train/go-basic.obo\"\n    ontology = OntologyParser(obo_path)\n    \n    # 2. Загрузка предсказаний\n    print(\"\\n[2/4] Loading predictions...\")\n    \n    all_predictions = {}\n    \n    # Загружаем GOA+ProtT5 (ваше решение)\n    if os.path.exists(Config.GOA_DATA):\n        goa_prott5_preds = PredictionLoader.load_goa_prott5_predictions(Config.GOA_DATA)\n        all_predictions['goa_prott5'] = goa_prott5_preds\n    else:\n        print(f\"GOA data not found at {Config.GOA_DATA}\")\n        print(\"Using sample predictions instead...\")\n        all_predictions['goa_prott5'] = PredictionLoader.create_sample_predictions()\n    \n    # Загружаем DeepGO (мое решение)\n    deepgo_preds = PredictionLoader.load_deepgo_predictions()\n    if deepgo_preds:\n        all_predictions['deepgo_cnn'] = deepgo_preds\n    \n    # Проверяем что у нас есть хотя бы одни предсказания\n    if not all_predictions:\n        print(\"ERROR: No predictions loaded!\")\n        return None\n    \n    # 3. Блендинг\n    print(\"\\n[3/4] Blending predictions...\")\n    \n    # Если только одна модель, используем ее как есть\n    if len(all_predictions) == 1:\n        model_name = list(all_predictions.keys())[0]\n        print(f\"Only one model ({model_name}), skipping blending\")\n        blended = all_predictions[model_name]\n    else:\n        # Используем взвешенное усреднение\n        blended = SmartBlender.weighted_average_blend(\n            all_predictions, \n            Config.MODEL_WEIGHTS\n        )\n    \n    # 4. Пропагация и пост-обработка\n    print(\"\\n[4/4] Applying propagation and post-processing...\")\n    \n    # Применяем пропагацию\n    propagator = EnhancedPropagation(ontology)\n    propagated = propagator.apply(blended)\n    \n    # Power scaling\n    scaled = PostProcessor.power_scaling(\n        propagated, \n        power=0.8, \n        max_score=0.95\n    )\n    \n    # Top-K фильтрация\n    final_predictions = PostProcessor.apply_top_k(\n        scaled,\n        top_k=Config.PROPAGATION['top_k'],\n        min_score=Config.MIN_SCORE\n    )\n    \n    # 5. Сохранение результатов\n    print(\"\\nSaving final submission...\")\n    \n    output_lines = []\n    total_preds = 0\n    \n    for protein, terms in tqdm(final_predictions.items(), desc=\"Formatting\"):\n        sorted_terms = sorted(terms.items(), key=lambda x: -x[1])\n        \n        for term, score in sorted_terms:\n            if score >= Config.MIN_SCORE:\n                output_lines.append(f\"{protein}\\t{term}\\t{score:.6f}\")\n                total_preds += 1\n    \n    # Сохраняем в файл\n    with open('submission.tsv', 'w') as f:\n        f.write('\\n'.join(output_lines))\n    \n    print(f\"\\n✓ Submission saved: submission.tsv\")\n    print(f\"  - Total predictions: {total_preds:,}\")\n    print(f\"  - Unique proteins: {len(final_predictions):,}\")\n    print(f\"  - Files loaded: {len(all_predictions)}\")\n    \n    return final_predictions\n\n# ==================== FAST VERSION (если мало времени) ====================\ndef run_fast_ensemble():\n    \"\"\"Быстрая версия с минимальной обработкой\"\"\"\n    print(\"=\" * 70)\n    print(\"CAFA-6 FAST ENSEMBLE\")\n    print(\"=\" * 70)\n    \n    # 1. Загружаем онтологию\n    obo_path = f\"{Config.COMPETITION_DATA}/Train/go-basic.obo\"\n    \n    # Простой парсинг онтологии\n    term_parents = defaultdict(set)\n    with open(obo_path, 'r') as f:\n        cur_id = None\n        for line in f:\n            line = line.strip()\n            if line.startswith('id: '):\n                cur_id = line.split('id: ')[1].strip()\n            elif line.startswith('is_a: ') and cur_id:\n                parent = line.split()[1].strip()\n                if '!' in parent:\n                    parent = parent.split('!')[0].strip()\n                term_parents[cur_id].add(parent)\n            elif line.startswith('relationship: part_of ') and cur_id:\n                parts = line.split()\n                if len(parts) >= 3:\n                    parent = parts[2].strip()\n                    if '!' in parent:\n                        parent = parent.split('!')[0].strip()\n                    term_parents[cur_id].add(parent)\n    \n    # Кэш предков\n    ancestors_cache = {}\n    def get_ancestors(term):\n        if term in ancestors_cache:\n            return ancestors_cache[term]\n        \n        parents = term_parents.get(term, set())\n        all_anc = set(parents)\n        for p in parents:\n            all_anc |= get_ancestors(p)\n        \n        ancestors_cache[term] = all_anc\n        return all_anc\n    \n    # 2. Загружаем предсказания\n    print(\"\\nLoading predictions...\")\n    \n    # Проверяем доступные файлы в GOA dataset\n    goa_predictions = defaultdict(dict)\n    \n    try:\n        files = os.listdir(Config.GOA_DATA)\n        print(f\"Files in GOA dataset: {files}\")\n        \n        # Ищем основной файл\n        main_file = None\n        for f in files:\n            if f.endswith('.tsv') and ('goa' in f.lower() or 'submission' in f.lower()):\n                main_file = f\n                break\n        \n        if main_file:\n            filepath = os.path.join(Config.GOA_DATA, main_file)\n            print(f\"Loading {main_file}...\")\n            \n            with open(filepath, 'r') as f:\n                for line in tqdm(f, desc=\"Reading predictions\"):\n                    parts = line.strip().split('\\t')\n                    if len(parts) >= 3:\n                        protein, go_term, score = parts[0], parts[1], float(parts[2])\n                        if go_term in goa_predictions[protein]:\n                            goa_predictions[protein][go_term] = max(\n                                goa_predictions[protein][go_term], score\n                            )\n                        else:\n                            goa_predictions[protein][go_term] = score\n        else:\n            print(\"No prediction file found, creating sample...\")\n            # Создаем sample\n            for i in range(1000):\n                protein = f\"TEST_{i}\"\n                for j in range(50):\n                    go_term = f\"GO:{1000000 + j:07d}\"\n                    score = np.random.beta(2, 5)\n                    goa_predictions[protein][go_term] = score\n                    \n    except Exception as e:\n        print(f\"Error: {e}\")\n        return None\n    \n    # 3. Простая пропагация\n    print(\"\\nApplying simple propagation...\")\n    \n    final_predictions = defaultdict(dict)\n    \n    for protein, terms in tqdm(goa_predictions.items(), desc=\"Processing\"):\n        # Копируем оригинальные предсказания\n        updated = terms.copy()\n        \n        # Положительная пропагация\n        for term, score in terms.items():\n            ancestors = get_ancestors(term)\n            for anc in ancestors:\n                if anc in updated:\n                    updated[anc] = max(updated[anc], score)\n                else:\n                    updated[anc] = score\n        \n        # Top-K фильтрация\n        sorted_terms = sorted(updated.items(), key=lambda x: -x[1])\n        kept_terms = []\n        \n        for term, score in sorted_terms:\n            if score >= 0.001 and len(kept_terms) < 150:\n                kept_terms.append((term, score))\n        \n        if kept_terms:\n            final_predictions[protein] = dict(kept_terms)\n    \n    # 4. Сохраняем\n    print(\"\\nSaving submission...\")\n    \n    output_lines = []\n    for protein, terms in final_predictions.items():\n        sorted_terms = sorted(terms.items(), key=lambda x: -x[1])\n        for term, score in sorted_terms:\n            if score >= 0.001:\n                output_lines.append(f\"{protein}\\t{term}\\t{score:.6f}\")\n    \n    with open('submission.tsv', 'w') as f:\n        f.write('\\n'.join(output_lines))\n    \n    print(f\"✓ Saved {len(output_lines):,} predictions\")\n    print(f\"✓ Unique proteins: {len(final_predictions):,}\")\n    \n    return final_predictions\n\n# ==================== MAIN EXECUTION ====================\nif __name__ == \"__main__\":\n    print(\"Starting CAFA-6 Ensemble Pipeline...\\n\")\n    \n    # Проверяем доступность данных\n    print(\"Checking data availability...\")\n    \n    # Проверяем competition data\n    if not os.path.exists(Config.COMPETITION_DATA):\n        print(f\"ERROR: Competition data not found at {Config.COMPETITION_DATA}\")\n        print(\"Make sure you've added the CAFA-6 dataset\")\n    else:\n        print(f\"✓ Competition data: {os.listdir(Config.COMPETITION_DATA)[:5]}...\")\n    \n    # Проверяем GOA data\n    if not os.path.exists(Config.GOA_DATA):\n        print(f\"WARNING: GOA data not found at {Config.GOA_DATA}\")\n        print(\"You need to add the dataset: ymuroya47/cafa6-goa-predictions\")\n        print(\"Running in fast mode with sample predictions...\")\n        predictions = run_fast_ensemble()\n    else:\n        print(f\"✓ GOA data: {os.listdir(Config.GOA_DATA)}\")\n        \n        # Запускаем полный пайплайн\n        try:\n            predictions = run_mega_ensemble()\n        except Exception as e:\n            print(f\"\\nERROR in main pipeline: {e}\")\n            print(\"\\nFalling back to fast mode...\")\n            predictions = run_fast_ensemble()\n    \n    if predictions:\n        print(\"\\n\" + \"=\" * 70)\n        print(\"SUCCESS! Submission file created: submission.tsv\")\n        print(\"=\" * 70)\n        \n        # Показываем пример предсказаний\n        sample_protein = list(predictions.keys())[0] if predictions else None\n        if sample_protein:\n            sample_terms = list(predictions[sample_protein].items())[:3]\n            print(f\"\\nSample predictions for {sample_protein}:\")\n            for term, score in sample_terms:\n                print(f\"  {term}: {score:.4f}\")\n    else:\n        print(\"\\nERROR: Failed to generate predictions\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}