{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"},{"sourceId":14641693,"sourceType":"datasetVersion","datasetId":9353239},{"sourceId":701498,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":532280,"modelId":546046},{"sourceId":728404,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":554661,"modelId":567224},{"sourceId":726890,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":553350,"modelId":565899}],"dockerImageVersionId":31259,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-29T15:02:58.837846Z","iopub.execute_input":"2026-01-29T15:02:58.838165Z","iopub.status.idle":"2026-01-29T15:03:00.320447Z","shell.execute_reply.started":"2026-01-29T15:02:58.838138Z","shell.execute_reply":"2026-01-29T15:03:00.319816Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# val_test_proxy_sampling_logic.py\n\nimport pandas as pd\nfrom pathlib import Path\n\nDATA_DIR = Path(\"/kaggle/input/deep-past-initiative-machine-translation\")\n\nTRAIN_PATH = DATA_DIR / \"train.csv\"\nPUB_PATH = DATA_DIR / \"published_texts.csv\"\nOUT_PATH = Path(\"/kaggle/working/val_test_proxy.csv\")\n\n\ndef build_val_test_proxy(\n    train_path: Path = TRAIN_PATH,\n    pub_path: Path = PUB_PATH,\n    out_path: Path = OUT_PATH,\n    n_long_train: int = 50,\n    n_test_like: int = 50,\n    min_chars_long: int = 400,\n    random_state: int = 42,\n) -> pd.DataFrame:\n    \"\"\"\n    Build a proxy validation set that approximates the hidden Kaggle test distribution.\n\n    The output mixes:\n      - n_long_train gold examples (long tablets) from train.csv\n      - n_test_like unlabeled examples from published_texts.csv\n\n    Columns are preserved from the source files where possible. For unlabeled\n    published_texts rows, a `translation` column is created and set to None.\n\n    Parameters\n    ----------\n    train_path : Path\n        Path to train.csv (gold transliteration/translation pairs).\n    pub_path : Path\n        Path to published_texts.csv (unlabeled Akkadian texts).\n    out_path : Path\n        Destination path for val_test_proxy.csv.\n    n_long_train : int\n        Number of long gold examples to sample from train.csv.\n    n_test_like : int\n        Number of unlabeled examples to sample from published_texts.csv.\n    min_chars_long : int\n        Minimum character length of `transliteration` to be considered “long”.\n    random_state : int\n        Random seed for reproducible sampling.\n\n    Returns\n    -------\n    pd.DataFrame\n        Combined proxy validation dataframe.\n    \"\"\"\n    # Load base datasets\n    train_df = pd.read_csv(train_path)\n    pub_df = pd.read_csv(pub_path)\n\n    if \"transliteration\" not in train_df.columns:\n        raise ValueError(\n            f\"`transliteration` column missing from train_df. \"\n            f\"Available columns: {list(train_df.columns)}\"\n        )\n    if \"transliteration\" not in pub_df.columns:\n        raise ValueError(\n            f\"`transliteration` column missing from pub_df. \"\n            f\"Available columns: {list(pub_df.columns)}\"\n        )\n\n    # 1) Long gold tablets from train.csv\n    long_mask = train_df[\"transliteration\"].astype(str).str.len() > min_chars_long\n    long_candidates = train_df[long_mask]\n\n    if len(long_candidates) < n_long_train:\n        raise ValueError(\n            f\"Not enough long examples: requested {n_long_train}, \"\n            f\"found {len(long_candidates)} with len>={min_chars_long}\"\n        )\n\n    long_samples = long_candidates.sample(\n        n=n_long_train, random_state=random_state\n    ).copy()\n    long_samples[\"source_split\"] = \"train_long_gold\"\n\n    # 2) Test-like samples from published_texts.csv (unlabeled)\n    test_like = pub_df.sample(\n        n=n_test_like, random_state=random_state\n    ).copy()\n    # Ensure translation column exists but is empty\n    if \"translation\" not in test_like.columns:\n        test_like[\"translation\"] = None\n    else:\n        test_like[\"translation\"] = None\n\n    test_like[\"source_split\"] = \"published_test_like\"\n\n    # 3) Combine and save\n    proxy_df = pd.concat([long_samples, test_like], ignore_index=True)\n\n    proxy_df.to_csv(out_path, index=False)\n    print(f\"[val_test_proxy] Saved {len(proxy_df)} rows to {out_path}\")\n\n    return proxy_df\n\n\nif __name__ == \"__main__\":\n    build_val_test_proxy()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-29T15:03:25.250299Z","iopub.execute_input":"2026-01-29T15:03:25.250577Z","iopub.status.idle":"2026-01-29T15:03:25.569368Z","shell.execute_reply.started":"2026-01-29T15:03:25.250558Z","shell.execute_reply":"2026-01-29T15:03:25.568734Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}