{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.4"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":248118764,"sourceType":"kernelVersion"},{"sourceId":166368,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":141565,"modelId":164048}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"! uv pip uninstall --system 'tensorflow'\n! uv pip install --system --no-index --find-links='/kaggle/input/latest-mdc-whls/whls' 'pymupdf' 'vllm' 'triton' 'logits-processor-zoo' 'numpy<2'\n! mkdir -p /tmp/src","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%writefile /tmp/src/helpers.py\nimport logging, os, kagglehub, inspect\nfrom pathlib import Path\nimport polars as pl\n\nIS_KAGGLE_ENV = sum(['KAGGLE' in k for k in os.environ]) > 0\nIS_KAGGLE_SUBMISSION = bool(os.getenv(\"KAGGLE_IS_COMPETITION_RERUN\"))\nCOMP_DIR = Path(('/kaggle/input/make-data-count-finding-data-references' if IS_KAGGLE_SUBMISSION else kagglehub.competition_download('make-data-count-finding-data-references')))\nPDF_DIR = COMP_DIR / ('test' if IS_KAGGLE_SUBMISSION else 'train') / 'PDF'\nWORKING_DIR = Path(('/kaggle/working/' if IS_KAGGLE_ENV else '.working/'))\n\nDOI_LINK = 'https://doi.org/'\n\nDEFAULT_LOG_LEVEL = os.getenv(\"LOG_LEVEL\", \"DEBUG\").upper() if not IS_KAGGLE_SUBMISSION else \"WARNING\"\nLOG_FILE_PATH = os.getenv(\"LOG_FILE\", \"logs/project.log\")\nLOG_DIR = Path(LOG_FILE_PATH).parent\n\nLOG_DIR.mkdir(parents=True, exist_ok=True)\n\nLOG_FORMAT = \"%(levelname)s %(asctime)s  [%(filename)s:%(lineno)d - %(funcName)s()] %(message)s\"\nLOG_DATEFMT = \"%Y-%m-%d %H:%M:%S\"\n\ndef get_logger(name=None):\n    if name is None:\n        frame = inspect.currentframe()\n        if frame is None or frame.f_back is None:\n            name = \"__main__\"\n        else:\n            name = frame.f_back.f_globals.get(\"__name__\", \"__main__\")\n\n    logger = logging.getLogger(name)\n\n    if not logger.handlers:\n        logger.setLevel(DEFAULT_LOG_LEVEL)\n        formatter = logging.Formatter(fmt=LOG_FORMAT, datefmt=LOG_DATEFMT)\n        ch = logging.StreamHandler()\n        ch.setLevel(DEFAULT_LOG_LEVEL)\n        ch.setFormatter(formatter)\n        fh = logging.FileHandler(LOG_FILE_PATH)\n        fh.setLevel(DEFAULT_LOG_LEVEL)\n        fh.setFormatter(formatter)\n        logger.addHandler(ch)\n        logger.addHandler(fh)\n        logger.propagate = False\n    return logger\n\ndef is_doi_link(name: str) -> pl.Expr:\n    return pl.col(name).str.starts_with(DOI_LINK)\n\ndef string_normalization(name: str) -> pl.Expr:\n    return pl.col(name).str.normalize(\"NFKC\").str.replace_all(r\"[^\\p{Ascii}]\", '').str.replace_all(r\"https?://zenodo\\.org/record/(\\d+)\", r\" 10.5281/zenodo.$1 \")\n\ndef get_df(parse_dir: str):\n    records = []\n    txt_files = list(Path(parse_dir).glob('*.txt'))\n    for txt_file in txt_files:\n        id_ = txt_file.stem\n        with open(txt_file, 'r') as f:\n            text = f.read()\n        records.append({'article_id': id_, 'text': text})\n    return pl.DataFrame(records).with_columns(string_normalization('text').alias('text'))\n\ndef assume_type(df: pl.DataFrame) -> pl.DataFrame:\n    return (\n        df.with_columns(pl.when(is_doi_link('dataset_id').or_(pl.col('dataset_id').str.starts_with('SAMN'))).then(pl.lit('Primary')).otherwise(pl.lit('Secondary')).alias('type'))\n    )\n\ndef score(df, gt, on, tag='all'):\n    hits = gt.join(df, on=on)\n    tp = hits.height\n    fp = df.height - tp\n    fn = gt.height - tp\n    f1 = 2 * tp / (2 * tp + fp + fn) if (2 * tp + fp + fn) != 0 else 0.0\n    return f\"{tag} - f1: {f1:.4f} [{tp}/{fp}/{fn}]\"\n\ndef evaluate(df, on=['article_id', 'dataset_id']):\n    gt = pl.read_csv(COMP_DIR/'train_labels.csv').filter(pl.col('type')!='Missing')\n    return (\n        score(df, gt, on),\n        score(df.filter(is_doi_link('dataset_id')), gt.filter(is_doi_link('dataset_id')), on, 'doi'),\n        score(df.filter(~is_doi_link('dataset_id')), gt.filter(~is_doi_link('dataset_id')), on, 'acc'),\n    )","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%writefile /tmp/src/parse.py\nimport argparse\nfrom pathlib import Path\nimport pymupdf\nfrom helpers import get_logger, PDF_DIR\n\nl = get_logger()\n\ndef pdf_to_txt(output_dir: Path):\n    output_dir.mkdir(parents=True, exist_ok=True)\n    pdf_files = list(PDF_DIR.glob(\"*.pdf\")) + list(PDF_DIR.glob(\"*.PDF\"))\n    existing_txt_files = {f.stem for f in output_dir.glob(\"*.txt\")}\n    for pdf_file in pdf_files:\n        txt_file = output_dir / f\"{pdf_file.stem}.txt\"\n        if pdf_file.stem in existing_txt_files:\n            continue\n        try:\n            text = \"\"\n            with pymupdf.open(pdf_file) as doc:\n                for page in doc:\n                    text += page.get_text()\n            txt_file.write_text(text, encoding='utf-8')\n        except Exception:\n            pass\n\ndef main():\n    parser = argparse.ArgumentParser()\n    parser.add_argument('output_dir', type=Path, help='Directory to save text files')\n    args = parser.parse_args()\n    pdf_to_txt(args.output_dir)\n\nif __name__ == \"__main__\":\n    main()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%writefile /tmp/src/check_parse.py\nimport polars as pl\nfrom pathlib import Path\nfrom helpers import *\n\nl=get_logger()\n\ndef gt_dataset_id_normalization(name:str) -> pl.Expr:\n    return (\n        pl.when(is_doi_link(name))\n        .then(pl.col(name).str.split(DOI_LINK).list.last())\n        .otherwise(name)\n        .str.to_lowercase()\n    )\n\ndef main():\n    if IS_KAGGLE_SUBMISSION:\n        l.debug('skipping check_parse for submission')\n        return\n    df = (\n        get_df('/tmp/train_parse')\n        .with_columns(pl.col('text').str.replace_all('\\s+', '').str.to_lowercase().alias('text'))\n    )\n\n    gt = (\n        pl.read_csv(COMP_DIR/'train_labels.csv')\n        .filter(pl.col('article_id').is_in(df['article_id']))\n        .filter(pl.col('type')!='Missing')\n        .with_columns(gt_dataset_id_normalization('dataset_id').alias('norm_id'))\n    )\n\n    l.info(f\"pymupdf misses: {gt.join(df, on='article_id').with_columns(hit=pl.col('text').str.contains(pl.col('norm_id'), literal=True)).filter(~pl.col('hit')).height} dataset_ids\")\n\nif __name__=='__main__': main()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%writefile /tmp/src/getid.py\nimport re\nimport polars as pl\nfrom typing import Optional, Tuple\n\nfrom helpers import *\n\nCOMPILED_PATTERNS = {\n    'ref_header_patterns': [re.compile(r'\\b(R\\s*E\\s*F\\s*E\\s*R\\s*E\\s*N\\s*C\\s*E\\s*S|BIBLIOGRAPHY|LITERATURE CITED|WORKS CITED|CITED WORKS|ACKNOWLEDGEMENTS)\\b[:\\s]*', re.IGNORECASE)],    \n    'citation_pattern': re.compile(r'^\\s*(\\[\\d+\\]|\\(\\d+\\)|\\d+\\.|\\d+\\)|\\d+(?=\\s|$))\\s*'),\n    'first_citation_patterns': [\n        re.compile(r'^\\s*\\[1\\]\\s*'),\n        re.compile(r'^\\s*\\(1\\)\\s*'),\n        re.compile(r'^\\s*1\\.\\s*'),\n        re.compile(r'^\\s*1\\)\\s*'),\n        re.compile(r'^\\s*1(?=\\s|$)'),\n    ],\n}\n\nl = get_logger()\n\ndef find_last_reference_header(text: str, header_patterns: list[re.Pattern]) -> Optional[int]:\n    last_match_idx = None\n    for pattern in header_patterns:\n        matches = list(pattern.finditer(text))\n        if matches:\n            last_match_idx = matches[-1].start()\n    return last_match_idx\n\ndef find_last_first_citation(text: str) -> Optional[int]:\n    lines = text.splitlines()\n    last_match_line = None\n    for line_num, line in enumerate(lines):\n        line = line.strip()\n        for pattern in COMPILED_PATTERNS['first_citation_patterns']:\n            if pattern.match(line):\n                next_lines = lines[line_num:line_num+3]\n                if any(COMPILED_PATTERNS['citation_pattern'].match(l.strip()) for l in next_lines[1:]):\n                    last_match_line = line_num\n                break\n    return last_match_line\n\ndef find_reference_start(text: str) -> Optional[int]:\n    lines = text.splitlines()\n    last_first_citation = find_last_first_citation(text)\n    if last_first_citation is not None:\n        return last_first_citation\n    start_search_idx = int(len(lines) * 0.5)\n    for i in range(start_search_idx, len(lines)):\n        line = lines[i].strip()\n        if COMPILED_PATTERNS['citation_pattern'].match(line):\n            next_lines = lines[i:i+3]\n            if sum(1 for l in next_lines if COMPILED_PATTERNS['citation_pattern'].match(l.strip())) >= 2:\n                for j in range(i, max(-1, i-10), -1):\n                    if not COMPILED_PATTERNS['citation_pattern'].match(lines[j].strip()):\n                        return j + 1\n                return max(0, i-10)\n    return None\n\ndef split_text_and_references(text: str) -> Tuple[str, str]:\n    header_idx = find_last_reference_header(text, COMPILED_PATTERNS['ref_header_patterns'])\n    if header_idx is not None:\n        header_idx2 = find_last_reference_header(text[:header_idx].strip(), COMPILED_PATTERNS['ref_header_patterns'])\n        if header_idx2 is not None:\n            header_idx3 = find_last_reference_header(text[:header_idx2].strip(), COMPILED_PATTERNS['ref_header_patterns'])\n            if header_idx3 is not None:\n                return text[:header_idx3].strip(), text[header_idx3:].strip()\n            return text[:header_idx2].strip(), text[header_idx2:].strip()\n        return text[:header_idx].strip(), text[header_idx:].strip()\n    ref_start_line = find_reference_start(text)\n    if ref_start_line is not None:\n        lines = text.splitlines()\n        body = '\\n'.join(lines[:ref_start_line])\n        refs = '\\n'.join(lines[ref_start_line:])\n        return body.strip(), refs.strip()\n    return text.strip(), ''\n\ndef get_splits(df: pl.DataFrame) -> pl.DataFrame:\n    bodies, refs = [], []\n    for raw_text in df['text']:\n        main, ref = split_text_and_references(raw_text)\n        bodies.append(main)\n        refs.append(ref)\n    return df.with_columns(pl.Series('body', bodies), pl.Series('ref', refs))\n\ndef tidy_extraction(df) -> pl.DataFrame:\n    bad_ids = [f'{DOI_LINK}{e}' for e in ['10.5061/dryad', '10.5281/zenodo', '10.6073/pasta']]\n\n    doi_df = (\n        df.with_columns(pl.col('body').str.extract_all(r'10\\s*\\.\\s*\\d{4,9}\\s*/\\s*\\S+').alias('match'))\n          .explode('match')\n          .drop_nulls('match')\n          .with_columns(\n              pl.col('match').str.replace_all(r'\\s+', '')\n                             .str.replace(r'[^A-Za-z0-9]+$', '')\n                             .str.to_lowercase()\n                             .alias('dataset_id')\n          )\n          .group_by('article_id', 'dataset_id')\n          .agg('match')\n          .with_columns((DOI_LINK + pl.col('dataset_id')).alias('dataset_id'))\n    )\n\n\n\n    REGEX_IDS = (\n        r\"(?i)\\b(?:\"\n        r\"CHEMBL\\d+|\"\n        r\"E-GEOD-\\d+|E-PROT-\\d+|E-MTAB-\\d+|E-MEXP-\\d+|EMPIAR-\\d+|\"\n        r\"ENSBTAG\\d+|ENSOARG\\d+|\"\n        r\"EPI_ISL_\\d{5,}|EPI\\d{6,7}|\"\n        r\"HPA\\d+|CP\\d{6}|IPR\\d{6}|PF\\d{5}|BX\\d{6}|KX\\d{6}|K0\\d{4}|CAB\\d{6}|\"\n        r\"NC_\\d{6}\\.\\d{1}|NM_\\d{9}|\"\n        r\"PRJNA\\d+|PRJDB\\d+|PXD\\d+|SAMN\\d+|\"\n        r\"GSE\\d+|GSM\\d+|GPL\\d+|\"\n        r\"CVCL_[A-Z0-9]{4}|\"\n        r\"PDB\\s?[1-9][A-Z0-9]{3}|HMDB\\d+|\"\n        r\"dryad\\.[^\\s\\\"<>]+|pasta\\/[^\\s\\\"<>]+|\"\n        r\"(?:SR[PX]|STH|ERR|DRR|DRX|DRP|ERP|ERX)\\d+|\"\n        r\"[1-5]\\.(?:10|20|30|40|50|60|70|80|90)\\.\\d{2,4}\\.\\d{2,4}\"\n        r\")\"\n    )\n\n\n\n\n\n    \n    acc_df = (\n        df.with_columns(\n            pl.col('text').str.extract_all(REGEX_IDS).alias('match')\n        )\n        .explode('match')\n        .drop_nulls('match')\n        .with_columns(\n            pl.col('match').str.replace_all(r'\\s+', '')\n                           .str.replace(r'[^A-Za-z0-9]+$', '')\n                           .str.replace(r'(?i)^PDB', '')\n                           .alias('dataset_id')\n        )\n        .group_by('article_id', 'dataset_id')\n        .agg('match')\n        .with_columns(\n            pl.when(pl.col('dataset_id').str.starts_with('dryad.'))\n              .then(f'{DOI_LINK}10.5061/' + pl.col('dataset_id'))\n              .otherwise('dataset_id')\n              .alias('dataset_id')\n        )\n        .with_columns(\n            pl.when(pl.col('dataset_id').str.starts_with('pasta/'))\n              .then(f'{DOI_LINK}10.6073/' + pl.col('dataset_id'))\n              .otherwise('dataset_id')\n              .alias('dataset_id')\n        )\n    )\n\n    df = pl.concat([doi_df, acc_df])\n\n    df = (\n        df.unique(['article_id', 'dataset_id'])  # CHANGED\n          .filter(~pl.col('article_id').str.replace('_','/').str.contains(pl.col('dataset_id').str.split(DOI_LINK).list.last().str.escape_regex()))\n          .filter(~pl.col('dataset_id').str.contains(pl.col('article_id').str.replace('_','/').str.escape_regex()))\n          .filter(~pl.col('dataset_id').str.contains('figshare', literal=True))\n          .filter(~pl.col('dataset_id').is_in(bad_ids))\n          .filter(\n              pl.when(is_doi_link('dataset_id') &\n                      (pl.col('dataset_id').str.split('/').list.last().str.len_chars() < 5))\n               .then(False)\n               .otherwise(True)\n          )\n          .with_columns(pl.col('match').list.unique())\n    )\n    return df\n\ndef get_context_window(text: str, substring: str, window: int = 100) -> str:\n    idx = text.find(substring)\n    if idx == -1:\n        raise ValueError\n    start = max(idx - window, 0)\n    end = min(idx + len(substring) + window, len(text))\n    return text[start:end]\n\ndef get_window_df(text_df, ids_df):\n    df = ids_df.join(text_df, on='article_id')\n    windows = []\n    for text, match_ids in df.select('text', 'match').rows():\n        windows.append(get_context_window(text, match_ids[0]))\n    return df.with_columns(pl.Series('window', windows)).select('article_id', 'dataset_id', 'window')\n\ndef main():\n    text_df = get_df('/tmp/train_parse')\n    df = get_splits(text_df)\n    df = tidy_extraction(df)\n    df = get_window_df(text_df, df)\n    df.write_parquet('/tmp/extracted.parquet')\n    df = assume_type(df)\n    df.select(['article_id', 'dataset_id', 'type']).with_row_index(name='row_id').write_csv('/kaggle/working/submission.csv')\n    if not IS_KAGGLE_SUBMISSION:\n        results = evaluate(df)\n        for r in results: l.info(r)\n        results = evaluate(df, on=['article_id', 'dataset_id', 'type'])\n        for r in results: l.info(r)\n\nif __name__=='__main__': main()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"%%writefile /tmp/src/llm_validate.py\nimport polars as pl\nimport os\n\nfrom helpers import *\n\nl = get_logger()\n\nSYS_PROMPT_CLASSIFY_DOI = \"\"\"\n1. Priority Rules (highest → lowest)\n1.1 Always classify as A (Data) if:\nDOI prefix matches a known data repository:\n\nDryad: 10.5061\n\nZenodo: 10.5281\n\nFigshare: 10.6084\n\nMendeley Data: 10.24433/, 10.17632\n\nDataverse: 10.7910/DVN\n\nOpenNeuro: 10.18112/openneuro.\n\nPANGAEA: 10.1594/PANGAEA.\n\nNeotoma Paleoecology: 10.21233\n\nICPSR: 10.3886\n\nNOAA NCEI: 10.7289\n\nUK Data Service: 10.5255\n\nEMPIAR: 10.6019\n\nNon-DOI dataset accession prefixes:\n\nNCBI SRA / ENA: SRP, SRA, ERP, ERX\n\nBioProject: PRJNA, PRJEB, PRJDB\n\nProteomeXchange / PRIDE: PXD\n\nArrayExpress / EMBL-EBI: E-MTAB, E-\n\nMetaboLights: MTBLS\n\nGEO Series: GSE\n\nGenBank: MN, NC_, CP, MT (context needed)\n\nEMDB: EMD-\n\nEMPIAR: EMPIAR-\n\n1.2 Context keywords trigger A (Data)\nEven if the prefix is not listed above, classify as A if the context clearly indicates dataset storage.\nKeywords (case-insensitive, include plural forms):\n\ndataset, data set\n\ndata repository, data archive, data portal\n\ndeposited in, uploaded to, archived at\n\navailable at, stored on, hosted by\n\naccessible via, retrieved from, provided by\n\nsupplementary dataset, supporting dataset\n\nexperimental data, raw data\n\npublic repository\n\n2. Classify as B (Literature) if:\nDOI prefix belongs to a publisher (e.g., 10.1038, 10.1007, 10.1126, 10.1016, 10.1101, 10.1021, 10.1145, 10.1177, 10.1093, 10.1080, 10.1111, etc.).\n\nContext indicates a journal article, book, conference paper, preprint, protocol, or method paper, without any repository/data storage signal.\n\nMentions only “supplementary material” or “supplementary information” without a repository.\n\n3. Ambiguous cases\nNo repository prefix and no clear context → default to B.\n\nRare accession formats → rely on context keywords.\n\n4. Output\nOnly output:\n\nA → data repository / dataset\n\nB → literature / non-data resource\n\nFew-shot examples\n\n“Raw images are stored on Figshare (DOI 10.6084/m9.figshare.1234567).” → A\n\n“Sequence reads available under BioProject accession PRJNA765432.” → A\n\n“As described in Nature Methods (DOI 10.1038/s41592-020-0793-2).” → B\n\n“See Supplementary Data at Zenodo (10.5281/zenodo.987654).” → A\n\n“Method details published in J. Proteome Res. DOI: 10.1021/acs.jproteome.0c00845.” → B\n\n“Data uploaded to Dryad (10.5061/dryad.x1y2z3).” → A\n\n“Referenced paper: DOI 10.1101/2020.01.01.123456 (bioRxiv preprint).” → B\n\n“Metabolomics data in MetaboLights MTBLS1234.” → A\n\n“The MRI scans are deposited at OpenNeuro (DOI 10.18112/openneuro.ds000001.v1.0.0).” → A\n\n“Protein structure described in Science (DOI 10.1126/science.abc1234).” → B\n\"\"\".strip()\n\ndef build_df():\n    df = pl.read_parquet('/tmp/extracted.parquet')\n    df.filter(~is_doi_link('dataset_id')).select('article_id', 'dataset_id').write_csv('/tmp/accid_sub.csv')\n    return df.filter(is_doi_link('dataset_id'))\n\ndef build_prompt(tokenizer, df):\n    prompts = []\n    for doi, text in df.select('dataset_id', 'window').rows():\n        messages = [{'role':'system','content': SYS_PROMPT_CLASSIFY_DOI}, {'role':'user', 'content': text}]\n        prompts.append(tokenizer.apply_chat_template(messages, add_generation_prompt=True, tokenize=False))\n    return df.with_columns(pl.Series('prompt', prompts))\n\nif __name__=='__main__':\n    os.environ[\"VLLM_USE_V1\"] = \"0\"\n    import vllm\n    from logits_processor_zoo.vllm import MultipleChoiceLogitsProcessor\n    model_path = \"/kaggle/input/qwen2.5/transformers/32b-instruct-awq/1\"\n    llm = vllm.LLM(model_path, quantization='awq', tensor_parallel_size=2, gpu_memory_utilization=0.9, trust_remote_code=True, dtype=\"half\", enforce_eager=True, max_model_len=2048, disable_log_stats=True, disable_custom_all_reduce=True, enable_prefix_caching=True, task='generate')\n    tokenizer = llm.get_tokenizer()\n    df = build_df()\n    df = build_prompt(tokenizer, df)\n    prompts = df['prompt'].to_list()\n    mclp = MultipleChoiceLogitsProcessor(tokenizer, choices=[\"A\", \"B\"])\n    outputs = llm.generate(prompts, vllm.SamplingParams(seed=777, temperature=0, skip_special_tokens=True, max_tokens=1, logits_processors=[mclp], logprobs=len(mclp.choices)), use_tqdm=True)\n    logprobs = [{lp.decoded_token: lp.logprob for lp in list(lps)} for lps in [output.outputs[0].logprobs[0].values() for output in outputs]]\n    choices = [max(d, key=d.get) for d in logprobs]\n    types = {'A': True, 'B': False}\n    choices = [types[c] for c in choices]\n    df = df.with_columns(pl.Series('type', choices))\n    df.filter(pl.col('type')).select('article_id', 'dataset_id').write_csv('/tmp/doi_sub.csv')\n    df = pl.concat([pl.read_csv('/tmp/doi_sub.csv'), pl.read_csv('/tmp/accid_sub.csv')])\n    df = assume_type(df)\n    df.select(['article_id', 'dataset_id', 'type']).with_row_index(name='row_id').write_csv('/kaggle/working/submission.csv')\n    if not IS_KAGGLE_SUBMISSION:\n        results = evaluate(df)\n        for r in results: l.info(r) \n        results = evaluate(df, on=['article_id', 'dataset_id', 'type'])\n        for r in results: l.info(r)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%writefile /tmp/src/post_filter.py\nimport polars as pl\nfrom helpers import *\n\n\"\"\"\nFourth essence: Post-filter to cut FP DOIs that look like literature.\n- Read /kaggle/working/submission.csv (output of llm_validate.py)\n- Join with /tmp/extracted.parquet to get context window\n- Drop DOI rows that (1) start with typical publisher prefixes AND (2) have no data-ish words nearby\n- Keep accessions untouched\n\"\"\"\n\nl = get_logger()\n\nPAPER_PREFIXES = [\n    \"10.5061\",\"10.5281\",\"10.17632\",\"10.1594\",\"10.15468\",\"10.17882\",\"10.7937\",\"10.7910\",\"10.6073\",\n    \"10.3886\",\"10.3334\",\"10.4121\",\"10.5066\",\"10.5067\",\"10.18150\",\"10.25377\",\"10.25387\",\"10.23642\",\"10.24381\",\"10.22033\"\n]\n\nCONTEXT_RE = r\"(?i)\\b(data(?:set)?|repository|archive|deposited|available|supplementary|raw(?:\\s+data)?|uploaded|hosted|stored|accession)\\b\"\n\ndef is_paper_prefix(col: str = \"dataset_id\") -> pl.Expr:\n    expr = pl.lit(False)\n    for p in PAPER_PREFIXES:\n        expr = expr | pl.col(col).str.starts_with(f\"{DOI_LINK}{p}\")\n    return expr\n\ndef main():\n    sub = pl.read_csv(\"/kaggle/working/submission.csv\")\n\n    # Normalize columns: drop row_id if present so concat widths match\n    if \"row_id\" in sub.columns:\n        sub = sub.drop(\"row_id\")\n\n    # Context windows\n    win = pl.read_parquet(\"/tmp/extracted.parquet\").select(\"article_id\", \"dataset_id\", \"window\")\n\n    # DOI & ACC split\n    doi_rows = sub.filter(is_doi_link(\"dataset_id\")).join(win, on=[\"article_id\", \"dataset_id\"], how=\"left\")\n    acc_rows = sub.filter(~is_doi_link(\"dataset_id\"))\n\n    keep_mask = (\n        (~is_paper_prefix(\"dataset_id\"))  # not a known paper prefix\n        | doi_rows[\"window\"].fill_null(\"\").str.contains(CONTEXT_RE)\n    )\n\n    kept_doi = doi_rows.filter(keep_mask).select(\"article_id\", \"dataset_id\", \"type\")\n    final = pl.concat([kept_doi, acc_rows.select(\"article_id\", \"dataset_id\", \"type\")])\n\n    # Re-eval & save\n    if not IS_KAGGLE_SUBMISSION:\n        for r in evaluate(final): l.info(r)\n        for r in evaluate(final, on=[\"article_id\", \"dataset_id\", \"type\"]): l.info(r)\n\n    final.with_row_index(\"row_id\").write_csv(\"/kaggle/working/submission.csv\")\n\nif __name__ == \"__main__\":\n    main()","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import shutil, os\n\nsrc = \"/tmp/extracted.parquet\"\ndst = \"/kaggle/working/extracted.parquet\"\n\nif os.path.exists(src):\n    shutil.copy(src, dst)\n    print(f\"✅ {src} → {dst} 복사 완료\")\nelse:\n    print(f\"❌ {src} 파일이 없습니다. 전처리 단계가 제대로 실행됐는지 확인하세요.\")\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%cd /tmp\n!LOG_LEVEL=INFO python src/parse.py /tmp/train_parse\n! python src/check_parse.py\n! python src/getid.py\n! python src/llm_validate.py\n! python src/post_filter.py\n! grep \"f1:\" /tmp/logs/project.log","metadata":{},"outputs":[],"execution_count":null}]}