{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391},{"sourceType":"datasetVersion","sourceId":15238910,"datasetId":9590186,"databundleVersionId":16136019},{"sourceType":"datasetVersion","sourceId":15173197,"datasetId":9624741,"databundleVersionId":16065546},{"sourceType":"datasetVersion","sourceId":15330540,"datasetId":9716209,"databundleVersionId":16238011},{"sourceType":"datasetVersion","sourceId":15336287,"datasetId":9809499,"databundleVersionId":16244468},{"sourceType":"datasetVersion","sourceId":15307198,"datasetId":9134804,"databundleVersionId":16211939},{"sourceType":"datasetVersion","sourceId":15336931,"datasetId":9734672,"databundleVersionId":16245197},{"sourceType":"kernelVersion","sourceId":287927434},{"sourceType":"kernelVersion","sourceId":303289943},{"sourceType":"kernelVersion","sourceId":303938621}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":60.143552,"end_time":"2025-12-23T04:06:08.6786","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-12-23T04:05:08.535048","version":"2.6.0"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"103bf2b4252f410db9eb748b5bbae882":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"ProgressView","bar_style":"success","description":"","description_allow_html":false,"layout":"IPY_MODEL_d04f750d278a4b0e822ce98d79032d5d","max":1,"min":0,"orientation":"horizontal","style":"IPY_MODEL_dc971fcb0bda4dfe8c54fe8829d7ed74","tabbable":null,"tooltip":null,"value":1}},"37533f95aff84f87a75a908681cf6b42":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"6a16c4a07c5b4304b651de8f5176d3b4":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_d07b8fe9e25d4a77a5b90757dc08d967","IPY_MODEL_103bf2b4252f410db9eb748b5bbae882","IPY_MODEL_afcc9676077d4d719fa6ab91ca696bf9"],"layout":"IPY_MODEL_9b3c469a638f4b89a2856fbe0e0b147e","tabbable":null,"tooltip":null}},"93b267e006e24e1ea38bb45523593938":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"9b3c469a638f4b89a2856fbe0e0b147e":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"afcc9676077d4d719fa6ab91ca696bf9":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_37533f95aff84f87a75a908681cf6b42","placeholder":"​","style":"IPY_MODEL_fdf7619b2f574beabb3c89675f5b02aa","tabbable":null,"tooltip":null,"value":" 1/1 [00:08&lt;00:00,  8.50s/it]"}},"d04f750d278a4b0e822ce98d79032d5d":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"d05eb3b22c674da19dccbc09121579fe":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"d07b8fe9e25d4a77a5b90757dc08d967":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_d05eb3b22c674da19dccbc09121579fe","placeholder":"​","style":"IPY_MODEL_93b267e006e24e1ea38bb45523593938","tabbable":null,"tooltip":null,"value":"100%"}},"dc971fcb0bda4dfe8c54fe8829d7ed74":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"fdf7619b2f574beabb3c89675f5b02aa":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}}},"version_major":2,"version_minor":0}}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport pandas as pd\nimport nbformat\nfrom nbconvert import PythonExporter\nfrom pathlib import Path\n\nimport math\nimport sacrebleu\nfrom tqdm.auto import tqdm\n\nonline = os.getenv('KAGGLE_KERNEL_RUN_TYPE') is not None\nsubmit = os.getenv('KAGGLE_IS_COMPETITION_RERUN')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# single_model_codes = [\n#     # --- Group 0 (Total: 452s) ---\n#     (\"/kaggle/input/datasets/wscmisx/dpc-single-model-infer-codes/dpc-t5-infer-v161-38.6-pretrain_noise_hf.ipynb\", 0),\n#     (\"/kaggle/input/datasets/wscmisx/dpc-single-model-infer-codes/dpc-t5-infer-vNone-None-base_fold0.ipynb\", 0),\n#     (\"/kaggle/input/datasets/wscmisx/dpc-single-model-infer-codes/dpc-t5-infer-v191-38.8-base.ipynb\", 0),\n#     (\"/kaggle/input/datasets/wscmisx/dpc-single-model-infer-codes/dpc-t5-infer-v196-39.2-add_all_base.ipynb\", 0),\n#     (\"/kaggle/input/datasets/wscmisx/dpc-single-model-infer-codes/dpc-t5-infer-vNone-None-base_fold2.ipynb\", 0),\n\n#     # --- Group 1 (Total: 453s) ---\n#     (\"/kaggle/input/datasets/wscmisx/dpc-single-model-infer-codes/dpc-t5-infer-v186-38.9-published_texts_aux.ipynb\", 1),\n#     (\"/kaggle/input/datasets/wscmisx/dpc-single-model-infer-codes/dpc-t5-infer-vNone-None-base_fold1.ipynb\", 1),\n#     (\"/kaggle/input/datasets/wscmisx/dpc-single-model-infer-codes/dpc-t5-infer-v146-38.7-large.ipynb\", 1),\n#     (\"/kaggle/input/datasets/wscmisx/dpc-single-model-infer-codes/dpc-t5-infer-v201-39.3-all_clean1_base.ipynb\", 1),\n#     (\"/kaggle/input/datasets/wscmisx/dpc-single-model-infer-codes/dpc-t5-infer-vNone-None-base_fold3.ipynb\", 1),\n# ]\n\n\nsingle_model_codes = [\n    # --- Group 0 (Total: 718s) ---\n    (\"/kaggle/input/datasets/wscmisx/dpc-single-model-infer-codes/dpc-t5-infer-v158-37.9-XL.ipynb\", 0),\n    (\"/kaggle/input/datasets/wscmisx/dpc-single-model-infer-codes/dpc-t5-infer-v146-38.7-large.ipynb\", 0),\n    (\"/kaggle/input/datasets/wscmisx/dpc-single-model-infer-codes/dpc-t5-infer-v149-37.8-name_aug.ipynb\", 0),\n    (\"/kaggle/input/datasets/wscmisx/dpc-single-model-infer-codes/dpc-t5-infer-v186-38.9-published_texts_aux.ipynb\", 0),\n    (\"/kaggle/input/datasets/wscmisx/dpc-single-model-infer-codes/dpc-t5-infer-v206-38.4-v6_data.ipynb\", 0),\n\n    # --- Group 1 (Total: 724s) ---\n    (\"/kaggle/input/datasets/wscmisx/dpc-single-model-infer-codes/dpc-t5-infer-v161-38.6-pretrain_noise_hf.ipynb\", 1),\n    (\"/kaggle/input/datasets/wscmisx/dpc-single-model-infer-codes/dpc-t5-infer-vNone-None-base_fold2.ipynb\", 1),\n    (\"/kaggle/input/datasets/wscmisx/dpc-single-model-infer-codes/dpc-t5-infer-v196-39.2-add_all_base.ipynb\", 1),\n    (\"/kaggle/input/datasets/wscmisx/dpc-single-model-infer-codes/dpc-t5-infer-v201-39.3-all_clean1_base.ipynb\", 1),\n    (\"/kaggle/input/datasets/wscmisx/dpc-single-model-infer-codes/dpc-t5-infer-vNone-None-base_fold3.ipynb\", 1),\n    (\"/kaggle/input/datasets/wscmisx/dpc-single-model-infer-codes/dpc-t5-infer-v204-38.3-v2_data.ipynb\", 1),\n    (\"/kaggle/input/datasets/wscmisx/dpc-single-model-infer-codes/dpc-t5-infer-vNone-None-base_fold0.ipynb\", 1),\n    (\"/kaggle/input/datasets/wscmisx/dpc-single-model-infer-codes/dpc-t5-infer-v204-38.2-v4_data.ipynb\", 1),\n    (\"/kaggle/input/datasets/wscmisx/dpc-single-model-infer-codes/dpc-t5-infer-vNone-None-base_fold1.ipynb\", 1),\n    (\"/kaggle/input/datasets/wscmisx/dpc-single-model-infer-codes/dpc-t5-infer-v191-38.8-base.ipynb\", 1),\n]\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from pathlib import Path\nimport nbformat\n\ntarget_directory = Path(\"./\")\ntarget_directory.mkdir(parents=True, exist_ok=True)\n\nSTRIP_ONLY_FIRST_LINE_CELL_MAGICS = {\n    \"time\", \"timeit\",\n}\n\nSKIP_ENTIRE_CELL_MAGICS = {\n    \"bash\", \"sh\", \"script\",\n}\n\ndef clean_code_cell_source(source: str) -> str | None:\n    lines = source.splitlines()\n\n    if not lines:\n        return \"\"\n\n    first = lines[0].lstrip()\n\n    if first.startswith(\"%%\"):\n        magic_name = first[2:].split(None, 1)[0].lower()\n\n        # %%time / %%timeit\n        if magic_name in STRIP_ONLY_FIRST_LINE_CELL_MAGICS:\n            lines = lines[1:]\n\n        # %%bash\n        elif magic_name in SKIP_ENTIRE_CELL_MAGICS:\n            return None\n\n        else:\n            return None\n\n    cleaned_lines = []\n    for line in lines:\n        stripped = line.lstrip()\n        \n        if stripped.startswith(\"%\") and not stripped.startswith(\"%%\"):\n            continue\n\n        cleaned_lines.append(line)\n\n    return \"\\n\".join(cleaned_lines).rstrip()\n\n\nfor path, _ in single_model_codes:\n    ipynb_path = Path(path)\n    nb = nbformat.read(ipynb_path, as_version=4)\n\n    py_blocks = []\n\n    for cell in nb.cells:\n        if cell.cell_type != \"code\":\n            continue\n\n        cleaned = clean_code_cell_source(cell.source)\n        if cleaned is None:\n            continue\n\n        if cleaned.strip():\n            py_blocks.append(cleaned)\n\n    py_code = \"\\n\\n\\n\".join(py_blocks) + \"\\n\"\n    \n    target_csv_name = f'submission_{ipynb_path.stem}.csv'\n    py_code = py_code.replace('\"submission.csv\"', f'\"{target_csv_name}\"')\n    py_code = py_code.replace(\"'submission.csv'\", f'\"{target_csv_name}\"')\n    py_path = target_directory / f\"{ipynb_path.stem}.py\"\n    py_path.write_text(py_code, encoding=\"utf-8\")\n\n    py_path = target_directory / f\"{ipynb_path.stem}.py\"\n    py_path.write_text(py_code, encoding=\"utf-8\")\n\n    print(\"Saved:\", py_path)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"group_0 = [item for item in single_model_codes if item[1] == 0]\ngroup_1 = [item for item in single_model_codes if item[1] == 1]\n\n# if not submit: \n#     python_run_cmd = '  python {code}.py 2>&1 | tee \"{code}.log.txt\"\\n'\n# else:\n#     python_run_cmd = '  python {code}.py &> \"{code}.log.txt\"\\n'\n\npython_run_cmd = '  python {code}.py &> \"{code}.log.txt\"\\n'\n\nwith open('run_t5_infer.sh', 'w') as f:\n    f.write(\"#!/bin/bash\\n\\n\")\n    f.write('BLUE_BRIGHT=\"\\\\033[94m\"\\n')\n    f.write('RED_BRIGHT=\"\\\\033[91m\"\\n')\n    f.write('RESET=\"\\\\033[0m\"\\n\\n')\n    \n    if not submit:\n        f.write('echo \"timestamp,index,utilization.gpu,memory.used\" > \"gpu_monitor.csv\"\\n')\n        f.write('(\\n')\n        f.write('  while true; do\\n')\n        f.write('    nvidia-smi --id=0,1 --query-gpu=timestamp,index,utilization.gpu,memory.used --format=csv,noheader,nounits >> \"gpu_monitor.csv\"\\n')\n        f.write('    sleep 2\\n')\n        f.write('  done\\n')\n        f.write(') &\\n')\n        f.write('MONITOR_PID=$!\\n')\n        f.write('trap \\'kill $MONITOR_PID 2>/dev/null\\' EXIT\\n\\n')\n        \n    f.write('> \"group_0_times.txt\"\\n')\n    f.write('> \"group_1_times.txt\"\\n\\n')\n\n    # GPU 0\n    f.write(\"(\\n\")\n    f.write(\"  export CUDA_VISIBLE_DEVICES=0\\n\")\n    for path, _ in group_0:\n        code = Path(path).stem\n        f.write(\"  start_time=$SECONDS\\n\")\n        f.write(python_run_cmd.format(code=code))\n        f.write(f'  echo \"{code}: $((SECONDS - start_time))s\" >> \"group_0_times.txt\"\\n')\n    f.write(\") &\\n\")\n    if not submit: f.write(\"PID0=$!\\n\\n\")\n    \n    # GPU 1\n    f.write(\"(\\n\")\n    f.write(\"  export CUDA_VISIBLE_DEVICES=1\\n\")\n    for path, _ in group_1:\n        code = Path(path).stem\n        f.write(\"  start_time=$SECONDS\\n\")\n        f.write(python_run_cmd.format(code=code))\n        f.write(f'  echo \"{code}: $((SECONDS - start_time))s\" >> \"group_1_times.txt\"\\n')\n    f.write(\") &\\n\")\n    if not submit: f.write(\"PID1=$!\\n\\n\")\n\n    if submit:\n        f.write(\"wait\\n\")\n    else:\n        f.write(\"wait $PID0 $PID1\\n\")\n        f.write(\"kill $MONITOR_PID 2>/dev/null\\n\")\n        f.write(\"wait $MONITOR_PID 2>/dev/null\\n\\n\")\n        \n    # logging\n    f.write('echo \"===== Group 0 =====\"\\n')\n    f.write('while IFS= read -r line; do echo -e \"${BLUE_BRIGHT}${line}${RESET}\"; done < \"group_0_times.txt\"\\n')\n    f.write('echo \"===== Group 1 =====\"\\n')\n    f.write('while IFS= read -r line; do echo -e \"${RED_BRIGHT}${line}${RESET}\"; done < \"group_1_times.txt\"\\n')\n    f.write('echo \"All inference tasks completed.\"\\n')\n\nprint(\"Bash script created with parallel execution strategy.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!cat run_t5_infer.sh","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!bash run_t5_infer.sh","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import math\nimport pandas as pd\nfrom tqdm.auto import tqdm\nimport sacrebleu\n\ndef sentence_level_score(hyp: str, ref: str) -> float:\n    if not hyp.strip() or not ref.strip():\n        return 0.0\n    bleu_score = sacrebleu.sentence_bleu(hyp, [ref]).score\n    chrf_score = sacrebleu.sentence_chrf(hyp, [ref], word_order=2).score\n    return math.sqrt(bleu_score * chrf_score)\n\ndef perform_mbr(dfs):\n    candidate_cols = [col for col in dfs[0].columns if col != 'id']\n    num_rows = len(dfs[0])\n    \n    mbr_results = []\n\n    for i in tqdm(range(num_rows), desc=\"Processing MBR\"):\n        row_id = dfs[0].loc[i, 'id']\n\n        candidates = []\n        for df in dfs:\n            for col in candidate_cols:\n                val = df.loc[i, col]\n                candidates.append(str(val).strip() if pd.notna(val) else \"\")\n                \n        unique_candidates = list(set([c for c in candidates if c]))\n        \n        best_score = -1.0\n        best_candidate = \"\"\n\n        n_unique = len(unique_candidates)\n        if n_unique == 0:\n            mbr_results.append({'id': row_id, 'translation': \"\"})\n            continue\n        if n_unique == 1:\n            mbr_results.append({'id': row_id, 'translation': unique_candidates[0]})\n            continue\n        \n        for hyp in unique_candidates:\n            total_score = 0.0\n\n            for ref in unique_candidates:\n                if hyp != ref: \n                    total_score += sentence_level_score(hyp, ref)\n\n            expected_score = total_score / max(1, n_unique - 1)\n\n            if expected_score > best_score:\n                best_score = expected_score\n                best_candidate = hyp\n\n        mbr_results.append({\n            'id': row_id,\n            'translation': best_candidate\n        })\n        \n    return pd.DataFrame(mbr_results)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"all_subs = []\nfor path, _ in single_model_codes:\n    code = Path(path).stem\n    sub = pd.read_csv(f'./submission_{code}.csv')\n    all_subs.append(sub)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = perform_mbr(all_subs)\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(\"MBR Ensemble Submission file saved successfully!\")\nsubmission.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if not submit:\n    from pathlib import Path\n    import textwrap\n    import matplotlib.dates as mdates\n    import matplotlib.patheffects as pe\n    import matplotlib.pyplot as plt\n    import numpy as np\n    import pandas as pd\n    from matplotlib.lines import Line2D\n    from matplotlib.patches import Patch\n\n    CSV_PATH, GROUP0_PATH, GROUP1_PATH = \"gpu_monitor.csv\", \"group_0_times.txt\", \"group_1_times.txt\"\n\n    def load_monitor_data(path):\n        df = pd.read_csv(path)\n        df.columns = [c.strip() for c in df.columns]\n        df[\"timestamp\"] = pd.to_datetime(df[\"timestamp\"])\n        for c in [\"index\", \"utilization.gpu\", \"memory.used\"]:\n            df[c] = df[c].astype(str).str.strip().astype(float if c != \"index\" else int)\n        df = df.sort_values([\"index\", \"timestamp\"]).reset_index(drop=True)\n        t0 = df[\"timestamp\"].min()\n        df[\"elapsed_s\"] = (df[\"timestamp\"] - t0).dt.total_seconds()\n        df[\"elapsed_min\"] = df[\"elapsed_s\"] / 60\n        df[\"memory_gib\"] = df[\"memory.used\"] / 1024\n        return df\n\n    def parse_task_times(path):\n        tasks = []\n        for line in Path(path).read_text(encoding=\"utf-8\").splitlines():\n            line = line.strip()\n            if line and \":\" in line:\n                name, sec = line.rsplit(\":\", 1)\n                tasks.append({\"name\": name.replace(\"dpc-t5-infer-\", \"\").strip(), \"duration_s\": float(sec.strip().rstrip(\"s\"))})\n        return tasks\n\n    def build_segments(tasks, start):\n        segs, cur = [], start\n        for t in tasks:\n            end = cur + pd.Timedelta(seconds=t[\"duration_s\"])\n            segs.append({\"name\": t[\"name\"], \"start\": cur, \"end\": end, \"duration_s\": t[\"duration_s\"]})\n            cur = end\n        return segs\n\n    def short_label(name, width=20): return name if len(name) <= width else textwrap.shorten(name, width=width, placeholder=\"…\")\n    def nice_memory_upper(x): return max(1.0, np.ceil((x + 0.5) / 2) * 2)\n\n    def plot_one_gpu(ax, gpu_df, segments, gpu_id, palette, monitor_start, monitor_end):\n        util_c, mem_c = \"#2F6BFF\", \"#16A085\"\n        plot_df = gpu_df.copy()\n        plot_df[\"util_smooth\"] = plot_df[\"utilization.gpu\"].rolling(7, center=True, min_periods=1).mean()\n        plot_df[\"mem_smooth\"] = plot_df[\"memory_gib\"].rolling(7, center=True, min_periods=1).mean()\n\n        for i, seg in enumerate(segments):\n            ax.axvspan(seg[\"start\"], seg[\"end\"], color=palette[i % len(palette)], alpha=0.11, lw=0, zorder=0)\n\n        ax.fill_between(plot_df[\"timestamp\"], plot_df[\"util_smooth\"].to_numpy(), color=util_c, alpha=0.12, zorder=1)\n        ax.plot(plot_df[\"timestamp\"], plot_df[\"util_smooth\"], color=util_c, lw=2.6, zorder=3)\n        ax2 = ax.twinx()\n        ax2.plot(plot_df[\"timestamp\"], plot_df[\"mem_smooth\"], color=mem_c, lw=2.4, zorder=4)\n        ax2.fill_between(plot_df[\"timestamp\"], plot_df[\"mem_smooth\"].to_numpy(), color=mem_c, alpha=0.08, zorder=1)\n\n        ax.set(xlim=(monitor_start, monitor_end), ylim=(0, 105))\n        ax2.set_ylim(0, nice_memory_upper(plot_df[\"memory_gib\"].max()))\n        ax.set_ylabel(\"Utilization (%)\", color=util_c, fontsize=11, fontweight=\"bold\")\n        ax2.set_ylabel(\"Memory Used (GiB)\", color=mem_c, fontsize=11, fontweight=\"bold\")\n        ax.tick_params(axis=\"y\", labelcolor=util_c)\n        ax2.tick_params(axis=\"y\", labelcolor=mem_c)\n        ax.grid(True, axis=\"y\", color=\"#D7DDE8\", alpha=0.7, linestyle=\"--\", linewidth=0.8)\n        ax.grid(False, axis=\"x\")\n        ax.set_axisbelow(True)\n        for s in list(ax.spines.values()) + list(ax2.spines.values()): s.set_visible(False)\n\n        util_mean, util_peak, mem_peak = plot_df[\"utilization.gpu\"].mean(), plot_df[\"utilization.gpu\"].max(), plot_df[\"memory_gib\"].max()\n        ax.text(0.01, 1.12, f\"GPU {gpu_id}\", transform=ax.transAxes, fontsize=16, fontweight=\"bold\", color=\"#1E293B\", va=\"center\")\n        badge = dict(boxstyle=\"round,pad=0.38,rounding_size=0.20\", fc=\"white\", ec=\"#E5EAF3\", lw=1.0)\n        for x, txt in [(0.17, f\"Avg Util {util_mean:.1f}%\"), (0.38, f\"Peak Util {util_peak:.0f}%\"), (0.60, f\"Peak Mem {mem_peak:.1f} GiB\")]:\n            ax.text(x, 1.12, txt, transform=ax.transAxes, fontsize=10.5, color=\"#334155\", va=\"center\", bbox=badge)\n\n        for i, seg in enumerate(segments):\n            mid, y, color = seg[\"start\"] + (seg[\"end\"] - seg[\"start\"]) / 2, (100.5 if i % 2 == 0 else 93.5), palette[i % len(palette)]\n            txt = ax.text(mid, y, f\"{short_label(seg['name'], 24)}\\n{int(seg['duration_s'])}s\", ha=\"center\", va=\"center\", fontsize=9.2, color=\"#0F172A\", zorder=5,\n                          bbox=dict(boxstyle=\"round,pad=0.28,rounding_size=0.18\", fc=\"white\", ec=color, lw=1.1, alpha=0.97))\n            txt.set_path_effects([pe.withStroke(linewidth=3.2, foreground=\"white\", alpha=0.75)])\n\n        if segments and segments[-1][\"end\"] < monitor_end - pd.Timedelta(seconds=2):\n            ax.axvspan(segments[-1][\"end\"], monitor_end, color=\"#94A3B8\", alpha=0.08, lw=0, zorder=0)\n            ax.text(segments[-1][\"end\"] + (monitor_end - segments[-1][\"end\"]) / 2, 8.5, \"Idle tail\", ha=\"center\", va=\"center\", fontsize=9, color=\"#475569\",\n                    bbox=dict(boxstyle=\"round,pad=0.22\", fc=\"white\", ec=\"#CBD5E1\", lw=0.8, alpha=0.92))\n        return ax2\n\n    plt.rcParams.update({\"font.size\": 10, \"axes.facecolor\": \"white\", \"figure.facecolor\": \"#F7F9FC\", \"savefig.facecolor\": \"#F7F9FC\", \"font.family\": \"DejaVu Sans\"})\n\n    df = load_monitor_data(CSV_PATH)\n    t0, t1 = df[\"timestamp\"].min(), df[\"timestamp\"].max()\n    tasks_map = {0: parse_task_times(GROUP0_PATH), 1: parse_task_times(GROUP1_PATH)}\n    seg_map = {gpu: build_segments(tasks, t0) for gpu, tasks in tasks_map.items()}\n    palettes = {0: [\"#84A9FF\", \"#A7F3D0\", \"#FDE68A\", \"#F9A8D4\"], 1: [\"#C4B5FD\", \"#93C5FD\", \"#FDBA74\", \"#86EFAC\"]}\n\n    fig, axes = plt.subplots(2, 1, figsize=(18, 10), sharex=True)\n    fig.subplots_adjust(top=0.86, left=0.07, right=0.93, bottom=0.10, hspace=0.28)\n\n    for gpu_id, ax in enumerate(axes):\n        plot_one_gpu(ax, df[df[\"index\"] == gpu_id].copy(), seg_map[gpu_id], gpu_id, palettes[gpu_id], t0, t1)\n\n    fig.text(0.07, 0.94, \"Dual-GPU Runtime Dashboard\", fontsize=23, fontweight=\"bold\", color=\"#0F172A\")\n    fig.legend(handles=[\n        Line2D([0], [0], color=\"#2F6BFF\", lw=2.8, label=\"GPU utilization\"),\n        Line2D([0], [0], color=\"#16A085\", lw=2.8, label=\"Memory used\"),\n        Patch(facecolor=\"#94A3B8\", alpha=0.12, edgecolor=\"none\", label=\"Task / idle background\"),\n    ], loc=\"upper right\", bbox_to_anchor=(0.93, 0.945), ncol=3, frameon=False, fontsize=10.5)\n\n    axes[-1].xaxis.set_major_formatter(mdates.DateFormatter(\"%H:%M:%S\"))\n    axes[-1].xaxis.set_major_locator(mdates.AutoDateLocator(minticks=8, maxticks=12))\n    axes[-1].set_xlabel(\"Timestamp\", fontsize=12, fontweight=\"bold\", color=\"#334155\")\n\n    for ax in axes:\n        ax.tick_params(axis=\"x\", labelsize=10)\n        ax.tick_params(axis=\"y\", labelsize=10)\n        pos = ax.get_position()\n        fig.patches.append(plt.Rectangle((pos.x0 - 0.01, pos.y0 - 0.02), pos.width + 0.02, pos.height + 0.06,\n                                         transform=fig.transFigure, facecolor=\"white\", edgecolor=\"#E8EDF5\",\n                                         linewidth=1.2, zorder=-1, joinstyle=\"round\"))\n\n    plt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}