{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpuV5e8","dataSources":[{"sourceId":116062,"databundleVersionId":14084779,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom collections import Counter\nimport torch\nfrom transformers import AutoTokenizer, AutoModel\nfrom sklearn.svm import LinearSVC\nfrom sklearn.calibration import CalibratedClassifierCV\nfrom sklearn.multiclass import OneVsRestClassifier\nfrom sklearn.preprocessing import MultiLabelBinarizer\nfrom sklearn.neighbors import NearestNeighbors","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-26T10:05:53.640377Z","iopub.execute_input":"2025-11-26T10:05:53.640656Z","iopub.status.idle":"2025-11-26T10:05:53.64513Z","shell.execute_reply.started":"2025-11-26T10:05:53.640635Z","shell.execute_reply":"2025-11-26T10:05:53.644369Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"EMB_DIR = \"/kaggle/working/embeddings\"\nimport os\nos.makedirs(EMB_DIR, exist_ok=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-26T10:05:53.646242Z","iopub.execute_input":"2025-11-26T10:05:53.646457Z","iopub.status.idle":"2025-11-26T10:05:53.65761Z","shell.execute_reply.started":"2025-11-26T10:05:53.646442Z","shell.execute_reply":"2025-11-26T10:05:53.65685Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install fair-esm\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-26T10:05:53.658241Z","iopub.execute_input":"2025-11-26T10:05:53.658475Z","iopub.status.idle":"2025-11-26T10:05:56.863004Z","shell.execute_reply.started":"2025-11-26T10:05:53.658454Z","shell.execute_reply":"2025-11-26T10:05:56.862255Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 1. Đọc dữ liệu","metadata":{}},{"cell_type":"markdown","source":"1.1. Đọc bảng train_terms (GO labels)\n\nFile này chứa nhãn của từng protein.  \nBảng gồm 3 cột: \n- EntryID: ID của protein\n- term: GO term dạng GO:0xxxxxx\n- aspect: mỗi GO thuộc một trong ba nhóm (F-chức năng phân tử/P-quá trình sinh học/C-vị trí trong tế bào).\n","metadata":{},"attachments":{}},{"cell_type":"code","source":"terms = pd.read_csv(\"/kaggle/input/cafa-6-protein-function-prediction/Train/train_terms.tsv\", sep=\"\\t\")\nterms.head()\nterms.info()\nterms","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-26T10:05:56.864903Z","iopub.execute_input":"2025-11-26T10:05:56.865202Z","iopub.status.idle":"2025-11-26T10:05:57.164904Z","shell.execute_reply.started":"2025-11-26T10:05:56.865178Z","shell.execute_reply":"2025-11-26T10:05:57.16411Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"1.2. Đọc bảng train_taxonomy (tax_id)\n\nFile này chứa thông tin loài của từng protein.\nBảng gồm 2 cột:\n- EntryID: ID của protein\n- tax_id: mã số loài (VD: 9606 là người, 10090 là chuột, 7227 là ruồi giấm)","metadata":{}},{"cell_type":"code","source":"tax = pd.read_csv(\"/kaggle/input/cafa-6-protein-function-prediction/Train/train_taxonomy.tsv\", sep=\"\\t\")\ntest_tax = pd.read_csv(\"/kaggle/input/cafa-6-protein-function-prediction/Test/testsuperset-taxon-list.tsv\", sep=\"\\t\")\ntax.columns = [\"EntryID\", \"tax_id\"]     \ntax.head()\ntax","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-26T10:05:57.165746Z","iopub.execute_input":"2025-11-26T10:05:57.165998Z","iopub.status.idle":"2025-11-26T10:05:57.216306Z","shell.execute_reply.started":"2025-11-26T10:05:57.165971Z","shell.execute_reply":"2025-11-26T10:05:57.215731Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"1.3. Đọc file FASTA \n\nFile này chứa trình tự amino axit của protein.\n- EntryID: có dạng UniProt|EntryID|ProteinName_Organism Description...\n- sequence: chuỗi gồm các amino axit (A, C, D, E, …).\n","metadata":{}},{"cell_type":"code","source":"def read_fasta(path):\n    entries = []\n    with open(path) as f:\n        entry_id = None\n        seq_parts = []\n        for line in f:\n            line = line.strip()\n            if not line:\n                continue\n            if line.startswith(\">\"):\n                if entry_id is not None:\n                    entries.append((entry_id, \"\".join(seq_parts)))\n                entry_id = line[1:]\n                seq_parts = []\n            else:\n                seq_parts.append(line)\n        if entry_id is not None:\n            entries.append((entry_id, \"\".join(seq_parts)))\n    return pd.DataFrame(entries, columns=[\"EntryID\", \"sequence\"])\nseq_df_test = read_fasta(\"/kaggle/input/cafa-6-protein-function-prediction/Test/testsuperset.fasta\")\nseq_df = read_fasta(\"/kaggle/input/cafa-6-protein-function-prediction/Train/train_sequences.fasta\")\nseq_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-26T10:05:57.216934Z","iopub.execute_input":"2025-11-26T10:05:57.217111Z","iopub.status.idle":"2025-11-26T10:05:58.696099Z","shell.execute_reply.started":"2025-11-26T10:05:57.217097Z","shell.execute_reply":"2025-11-26T10:05:58.695343Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2. Khảo sát dữ liệu","metadata":{}},{"cell_type":"markdown","source":"2.1. Tổng quan dữ liệu train\n\nSố lượng protein, \nsố lượng nhãn GO term được gán, \nsố lượng nhãn GO term khác nhau, \nsố lượng GO term phân theo từng loại P/C/F","metadata":{}},{"cell_type":"code","source":"n_protein = seq_df[\"EntryID\"].nunique()\nn_terms_pairs = len(terms)\nn_go_terms = terms[\"term\"].nunique()\naspect_counts = terms[\"aspect\"].value_counts()\n\nn_protein, n_terms_pairs, n_go_terms, aspect_counts\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-26T10:05:58.69695Z","iopub.execute_input":"2025-11-26T10:05:58.697446Z","iopub.status.idle":"2025-11-26T10:05:58.79544Z","shell.execute_reply.started":"2025-11-26T10:05:58.697422Z","shell.execute_reply":"2025-11-26T10:05:58.794837Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"2.2.Thống kê số GO terms/protein","metadata":{}},{"cell_type":"code","source":"labels_per_protein = terms.groupby(\"EntryID\")[\"term\"].nunique()\nlabels_per_protein.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-26T10:05:58.796138Z","iopub.execute_input":"2025-11-26T10:05:58.796366Z","iopub.status.idle":"2025-11-26T10:05:58.969475Z","shell.execute_reply.started":"2025-11-26T10:05:58.796348Z","shell.execute_reply":"2025-11-26T10:05:58.968783Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.hist(labels_per_protein, bins=30)\nplt.xlabel(\"Số GO term / protein\")\nplt.ylabel(\"Số protein\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-26T10:05:58.97243Z","iopub.execute_input":"2025-11-26T10:05:58.972645Z","iopub.status.idle":"2025-11-26T10:05:59.11775Z","shell.execute_reply.started":"2025-11-26T10:05:58.972628Z","shell.execute_reply":"2025-11-26T10:05:59.117175Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"2.3. Phân phối số nhãn mỗi protein","metadata":{}},{"cell_type":"code","source":"seq_df[\"seq_len\"] = seq_df[\"sequence\"].str.len()\nseq_df[\"seq_len\"].describe()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-26T10:05:59.118372Z","iopub.execute_input":"2025-11-26T10:05:59.118583Z","iopub.status.idle":"2025-11-26T10:05:59.164574Z","shell.execute_reply.started":"2025-11-26T10:05:59.118567Z","shell.execute_reply":"2025-11-26T10:05:59.16389Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(8,5))\nplt.hist(seq_df[\"seq_len\"], bins=50)\nplt.xlabel(\"Độ dài chuỗi amino acid\")\nplt.ylabel(\"Số protein\")\nplt.title(\"Phân bố độ dài protein\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-26T10:05:59.165354Z","iopub.execute_input":"2025-11-26T10:05:59.165884Z","iopub.status.idle":"2025-11-26T10:05:59.349607Z","shell.execute_reply.started":"2025-11-26T10:05:59.165859Z","shell.execute_reply":"2025-11-26T10:05:59.348909Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"2.4. Thống kê về loài\n\nSố lượng loài, số lượng protein không có tax_id, top loài phổ biến nhất","metadata":{}},{"cell_type":"code","source":"n_species = tax[\"tax_id\"].nunique()\nmissing_tax = tax[\"tax_id\"].isna().sum()\n\nn_species, missing_tax,\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-26T10:05:59.350385Z","iopub.execute_input":"2025-11-26T10:05:59.35102Z","iopub.status.idle":"2025-11-26T10:05:59.356691Z","shell.execute_reply.started":"2025-11-26T10:05:59.350997Z","shell.execute_reply":"2025-11-26T10:05:59.356114Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tax[\"tax_id\"].value_counts().head(20).plot(kind=\"bar\", figsize=(10,4))\nplt.title(\"Top 20 loài phổ biến nhất\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-26T10:05:59.35735Z","iopub.execute_input":"2025-11-26T10:05:59.357537Z","iopub.status.idle":"2025-11-26T10:05:59.557445Z","shell.execute_reply.started":"2025-11-26T10:05:59.357523Z","shell.execute_reply":"2025-11-26T10:05:59.556877Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"2.5. Mức độ phân bố GO term\n\nSố lượng GO term xuất hiện <= 5 lần, >=100 lần và biểu đồ phân bố ","metadata":{}},{"cell_type":"code","source":"term_counts = terms[\"term\"].value_counts()\n\nrare_terms = (term_counts <= 5).sum()\ncommon_terms = (term_counts >= 100).sum()\n\nrare_terms, common_terms\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-26T10:05:59.558098Z","iopub.execute_input":"2025-11-26T10:05:59.558361Z","iopub.status.idle":"2025-11-26T10:05:59.608116Z","shell.execute_reply.started":"2025-11-26T10:05:59.558337Z","shell.execute_reply":"2025-11-26T10:05:59.607439Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"freqs = term_counts.values\nplt.figure(figsize=(8,5))\nplt.plot(freqs)\nplt.yscale(\"log\")\nplt.xlabel(\"GO term (sắp theo thứ tự từ phổ biến đến hiếm)\")\nplt.ylabel(\"Tần suất xuất hiện (log scale)\")\nplt.title(\"Biểu đồ Zipf: Phân bố tần suất GO term\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-26T10:05:59.60906Z","iopub.execute_input":"2025-11-26T10:05:59.609304Z","iopub.status.idle":"2025-11-26T10:05:59.975485Z","shell.execute_reply.started":"2025-11-26T10:05:59.609278Z","shell.execute_reply":"2025-11-26T10:05:59.974827Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(8,5))\nplt.hist(term_counts.values, bins=50, log=True)\nplt.xscale(\"log\")\nplt.xlabel(\"Tần suất xuất hiện (log)\")\nplt.ylabel(\"Số lượng GO term (log)\")\nplt.title(\"Histogram tần suất GO term (log-log)\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-26T10:05:59.976179Z","iopub.execute_input":"2025-11-26T10:05:59.976462Z","iopub.status.idle":"2025-11-26T10:06:00.500506Z","shell.execute_reply.started":"2025-11-26T10:05:59.976436Z","shell.execute_reply":"2025-11-26T10:06:00.49989Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"2.6. Kết luận\n\n- Dataset gồm 82k protein, 537k cặp (EntryID, GO), 26k GO khác nhau.\n\n- Phân bố GO theo 3 nhóm: P nhiều nhất → F ít nhất → mất cân bằng nặng.\n\n- Protein có trung bình 6.5 GO, nhưng có protein lên tới 233 GO → multi-label rất phức tạp.\n\n- Độ dài chuỗi protein rất đa dạng, có chuỗi rất ngắn hoặc rất dài.\n\n- Taxonomy cho thấy một số loài chiếm tỉ lệ lớn (đặc biệt người – 9606).\n\n- Phân bố GO term cực kỳ skew: nhiều GO hiếm, rất ít GO phổ biến.","metadata":{}},{"cell_type":"markdown","source":"## 3. Xử lý dữ liệu","metadata":{}},{"cell_type":"markdown","source":"3.1. Làm sạch dữ liệu sequences (FASTA) \n\nTách lấy EntryID, Xóa sequence rỗng / bất thường, Xử lý ký tự bất hợp lệ","metadata":{}},{"cell_type":"code","source":"def get_id(header):\n    parts = header.split(\"|\")\n    return parts[1] if len(parts) > 1 else header.replace(\">\", \"\")\nseq_df_test[\"EntryID\"] = seq_df_test[\"EntryID\"].apply(get_id)\nseq_df[\"EntryID\"] = seq_df[\"EntryID\"].apply(get_id)\nseq_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-26T10:06:00.501213Z","iopub.execute_input":"2025-11-26T10:06:00.501451Z","iopub.status.idle":"2025-11-26T10:06:00.605333Z","shell.execute_reply.started":"2025-11-26T10:06:00.501432Z","shell.execute_reply":"2025-11-26T10:06:00.604513Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"seq_df = seq_df[seq_df[\"sequence\"].str.len() > 0]\nseq_df_test = seq_df_test[seq_df_test[\"sequence\"].str.len() > 0]\nseq_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-26T10:06:00.606259Z","iopub.execute_input":"2025-11-26T10:06:00.606534Z","iopub.status.idle":"2025-11-26T10:06:00.786138Z","shell.execute_reply.started":"2025-11-26T10:06:00.606509Z","shell.execute_reply":"2025-11-26T10:06:00.785537Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"allowed = set(\"ACDEFGHIKLMNPQRSTVWY\")\n\ndef clean(seq):\n    return \"\".join([aa if aa in allowed else \"X\" for aa in seq])\n\nseq_df[\"sequence\"] = seq_df[\"sequence\"].apply(clean)\nseq_df_test[\"sequence\"] = seq_df_test[\"sequence\"].apply(clean)\nseq_df\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-26T10:06:00.786808Z","iopub.execute_input":"2025-11-26T10:06:00.787013Z","iopub.status.idle":"2025-11-26T10:06:07.01999Z","shell.execute_reply.started":"2025-11-26T10:06:00.786996Z","shell.execute_reply":"2025-11-26T10:06:07.019332Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Có 200 chuỗi aa chứa các kí tự không nằm trong 20 aa chuẩn, các kí tự đó sẽ được thay bằng kí tự 'X'","metadata":{}},{"cell_type":"markdown","source":"3.2. Làm sạch bảng GO terms\n\nXóa duplicated labels, Gom nhãn cho mỗi protein","metadata":{},"attachments":{}},{"cell_type":"code","source":"labels_grouped = terms.groupby(\"EntryID\")[\"term\"].apply(list)\nlabels_grouped","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-26T10:06:07.020695Z","iopub.execute_input":"2025-11-26T10:06:07.020899Z","iopub.status.idle":"2025-11-26T10:06:08.232826Z","shell.execute_reply.started":"2025-11-26T10:06:07.020884Z","shell.execute_reply":"2025-11-26T10:06:08.232188Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"3.3. Merge 3 bảng","metadata":{}},{"cell_type":"code","source":"train = seq_df.merge(terms, on=\"EntryID\", how=\"left\")\ntrain = train.merge(tax, on=\"EntryID\", how=\"left\")\n\ntrain","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-26T10:06:08.233533Z","iopub.execute_input":"2025-11-26T10:06:08.233743Z","iopub.status.idle":"2025-11-26T10:06:08.616457Z","shell.execute_reply.started":"2025-11-26T10:06:08.233727Z","shell.execute_reply":"2025-11-26T10:06:08.615842Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_grouped = (\n    train\n    .groupby(\"EntryID\")\n    .agg({\n        \"sequence\": \"first\",      # chuỗi protein\n        \"tax_id\": \"first\",        # có thể dùng làm feature sau\n        \"term\": list              # list GO term\n    })\n    .reset_index()\n)\ntrain_grouped","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-26T10:06:08.617145Z","iopub.execute_input":"2025-11-26T10:06:08.617363Z","iopub.status.idle":"2025-11-26T10:06:09.857434Z","shell.execute_reply.started":"2025-11-26T10:06:08.617346Z","shell.execute_reply":"2025-11-26T10:06:09.856654Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"seq_df_test","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-26T10:06:09.858268Z","iopub.execute_input":"2025-11-26T10:06:09.858526Z","iopub.status.idle":"2025-11-26T10:06:09.867835Z","shell.execute_reply.started":"2025-11-26T10:06:09.858505Z","shell.execute_reply":"2025-11-26T10:06:09.867027Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\n# Assuming your dataframe is called 'df'\nseq_df_test_copy = seq_df_test\nseq_df_test_copy[['EntryID', 'Number']] = seq_df_test_copy['EntryID'].str.split(' ', expand=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-26T10:06:09.871721Z","iopub.execute_input":"2025-11-26T10:06:09.872Z","iopub.status.idle":"2025-11-26T10:06:10.62079Z","shell.execute_reply.started":"2025-11-26T10:06:09.871982Z","shell.execute_reply":"2025-11-26T10:06:10.620179Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"del seq_df_test_copy['Number']\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-26T10:06:10.621544Z","iopub.execute_input":"2025-11-26T10:06:10.62202Z","iopub.status.idle":"2025-11-26T10:06:10.629749Z","shell.execute_reply.started":"2025-11-26T10:06:10.621996Z","shell.execute_reply":"2025-11-26T10:06:10.628997Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"seq_df_test_copy","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-26T10:06:10.63056Z","iopub.execute_input":"2025-11-26T10:06:10.630801Z","iopub.status.idle":"2025-11-26T10:06:10.645266Z","shell.execute_reply.started":"2025-11-26T10:06:10.630776Z","shell.execute_reply":"2025-11-26T10:06:10.644526Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install transformers accelerate sentencepiece\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-26T10:06:10.645983Z","iopub.execute_input":"2025-11-26T10:06:10.6467Z","iopub.status.idle":"2025-11-26T10:06:14.07221Z","shell.execute_reply.started":"2025-11-26T10:06:10.646682Z","shell.execute_reply":"2025-11-26T10:06:14.071275Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from transformers import AutoTokenizer, AutoModel\nimport torch\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\ntokenizer = AutoTokenizer.from_pretrained(\"Rostlab/prot_bert\", do_lower_case=False)\nmodel = AutoModel.from_pretrained(\"Rostlab/prot_bert\").to(device)\nmodel.eval()\nmodel.half()\n","metadata":{"trusted":true,"execution":{"execution_failed":"2025-11-26T11:33:33.788Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def prepare_sequence(seq):\n    return \" \".join(list(seq))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-26T10:06:17.999976Z","iopub.execute_input":"2025-11-26T10:06:18.000188Z","iopub.status.idle":"2025-11-26T10:06:18.003887Z","shell.execute_reply.started":"2025-11-26T10:06:18.000165Z","shell.execute_reply":"2025-11-26T10:06:18.003217Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def embed_sequence(seq):\n    seq = prepare_sequence(seq)\n\n    encoded = tokenizer(seq,\n                        return_tensors='pt',\n                        padding=True,\n                        truncation=True,\n                        max_length=1024)  # ProtBert supports up to 1k tokens\n\n    encoded = {k: v.to(device) for k, v in encoded.items()}\n\n    with torch.no_grad():\n        output = model(**encoded)\n\n    # CLS embedding = output.last_hidden_state[0,0,:]\n    embedding = output.last_hidden_state[0, 0, :].cpu().numpy()\n    return embedding\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-26T10:06:18.004585Z","iopub.execute_input":"2025-11-26T10:06:18.004771Z","iopub.status.idle":"2025-11-26T10:06:18.015866Z","shell.execute_reply.started":"2025-11-26T10:06:18.004754Z","shell.execute_reply":"2025-11-26T10:06:18.015264Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def embed_batch(seqs, batch_size=16):\n    embeddings = []\n    for i in range(0, len(seqs), batch_size):\n        batch = seqs[i:i+batch_size]\n        prepared = [prepare_sequence(s) for s in batch]\n        encoded = tokenizer(prepared, return_tensors='pt', padding=True,\n                            truncation=True, max_length=1024)\n        encoded = {k: v.to(device) for k, v in encoded.items()}\n        with torch.no_grad():\n            outputs = model(**encoded)\n        cls_emb = outputs.last_hidden_state[:, 0, :].cpu().numpy()\n        embeddings.append(cls_emb)\n    return np.concatenate(embeddings, axis=0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-26T10:06:18.016603Z","iopub.execute_input":"2025-11-26T10:06:18.016779Z","iopub.status.idle":"2025-11-26T10:06:18.026979Z","shell.execute_reply.started":"2025-11-26T10:06:18.016765Z","shell.execute_reply":"2025-11-26T10:06:18.026276Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train_grouped and seq_df_test_copy already loaded\ntrain_sequences = train_grouped[\"sequence\"].tolist()\ntest_sequences = seq_df_test_copy[\"sequence\"].tolist()\nsave_path = \"/kaggle/working/embeddings\"\nprint(\"Embedding training sequences...\")\ntrain_embeds = embed_batch(train_sequences, batch_size=64)\nprint(\"Embedding test sequences...\")\ntest_embeds = embed_batch(test_sequences, batch_size=64)\n\nnp.save(os.path.join(save_path, \"train_embeds.npy\"), train_embeds)\nnp.save(os.path.join(save_path, \"test_embeds.npy\"), test_embeds)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-26T10:06:18.027713Z","iopub.execute_input":"2025-11-26T10:06:18.027939Z","execution_failed":"2025-11-26T11:33:33.788Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"all_terms = train_grouped[\"term\"].explode()\nterm_counts = all_terms.value_counts()\n\ncommon_terms = term_counts[term_counts >= 100].index\nrare_terms   = term_counts[term_counts < 100].index\n\nprint(f\"Common labels: {len(common_terms)}, Rare labels: {len(rare_terms)}\")\n\n# Keep only common terms for classifier\ntrain_common_terms = train_grouped[\"term\"].apply(lambda terms: [t for t in terms if t in common_terms])\n\n# MultiLabelBinarizer on common labels\nmlb = MultiLabelBinarizer()\ny_train_bin = mlb.fit_transform(train_common_terms)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-11-26T11:33:33.788Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.svm import LinearSVC\nfrom sklearn.calibration import CalibratedClassifierCV\nfrom sklearn.multiclass import OneVsRestClassifier\n\nbase = LinearSVC()\nclf = OneVsRestClassifier(\n    LogisticRegression(\n        solver=\"saga\",      # better for large sparse matrices\n        penalty=\"l2\",\n        max_iter=40,\n        n_jobs=-1\n    )\n)\nprint(\"Training classifier on common labels...\")\nclf.fit(train_embeds, y_train_bin)\n","metadata":{"trusted":true,"execution":{"execution_failed":"2025-11-26T11:33:33.788Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Fitting nearest neighbors for rare labels...\")\nnbrs = NearestNeighbors(n_neighbors=20, metric='cosine').fit(train_embeds)\n\ndist, idx = nbrs.kneighbors(test_embeds)\n\nrare_predictions = []\nfor neighbors in idx:\n    neighbor_terms = train_grouped.iloc[neighbors][\"term\"].explode()\n    rare_only = [t for t in neighbor_terms if t in rare_terms]\n    rare_predictions.append(rare_only)\n","metadata":{"trusted":true,"execution":{"execution_failed":"2025-11-26T11:33:33.788Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_test_pred_scores = clf.predict_proba(test_embeds)\nentry_ids = seq_df_test_copy[\"EntryID\"].tolist()\ncommon_terms_list = mlb.classes_\n","metadata":{"trusted":true,"execution":{"execution_failed":"2025-11-26T11:33:33.788Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_terms = []\nfinal_scores = []\n\nthreshold = 0.3   # you can tune\n\nfor i in range(len(entry_ids)):\n    # common label predictions above threshold\n    common_pred_terms = [term for j, term in enumerate(common_terms_list) if y_test_pred_scores[i,j] > threshold]\n    # merge with rare predictions\n    combined = set(common_pred_terms) | set(rare_predictions[i])\n    final_terms.append(list(combined))\n    \n    # build combined score: common score if exists, else 0.05 for rare\n    scores = []\n    for t in combined:\n        if t in common_pred_terms:\n            j = np.where(common_terms_list == t)[0][0]\n            scores.append(float(y_test_pred_scores[i,j]))\n        else:\n            scores.append(0.05)  # assign small probability to rare terms\n    final_scores.append(scores)\n","metadata":{"trusted":true,"execution":{"execution_failed":"2025-11-26T11:33:33.788Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rows = []\nfor i, entry in enumerate(entry_ids):\n    for t, s in zip(final_terms[i], final_scores[i]):\n        rows.append([entry, t, s])\n\nsubmission = pd.DataFrame(rows, columns=[\"EntryID\", \"term\", \"score\"])\n","metadata":{"trusted":true,"execution":{"execution_failed":"2025-11-26T11:33:33.788Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index=False)\nprint(\"Submission saved to submission.csv\")\nsubmission.head()","metadata":{"trusted":true,"execution":{"execution_failed":"2025-11-26T11:33:33.788Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}