{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":121150,"databundleVersionId":14976537,"sourceType":"competition"}],"dockerImageVersionId":31236,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T15:36:14.870775Z","iopub.execute_input":"2025-12-17T15:36:14.871109Z","iopub.status.idle":"2025-12-17T15:36:14.875449Z","shell.execute_reply.started":"2025-12-17T15:36:14.87108Z","shell.execute_reply":"2025-12-17T15:36:14.874546Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\ntrain = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/train.csv')\ntest = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/test.csv')\npublished = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/published_texts.csv')\npubs = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/publications.csv')\n\nprint(train.head())\nprint(f\"Train pairs: {len(train)}\")\nprint(test.head())\nprint(f\"Published transliterations: {len(published)}\")\nprint(f\"Publication pages: {len(pubs)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T15:19:25.642217Z","iopub.execute_input":"2025-12-17T15:19:25.642681Z","iopub.status.idle":"2025-12-17T15:19:35.987522Z","shell.execute_reply.started":"2025-12-17T15:19:25.642656Z","shell.execute_reply":"2025-12-17T15:19:35.986861Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport re\n\n# Load the data (run this in your Kaggle notebook after adding the dataset)\npublished = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/published_texts.csv')\npubs = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/publications.csv')\n\nprint(\"Published columns:\", published.columns.tolist())\nprint(\"Publications columns:\", pubs.columns.tolist())\n\n# Function to find candidate translation snippets\ndef find_candidate_translation(row):\n    # Collect all possible IDs from this row (as strings, handle NaN)\n    ids = []\n    for col in ['oare_id', 'cdli_id', 'aliases', 'label', 'publication_catalog', 'oatp_key']:\n        val = row.get(col)\n        if pd.notna(val):\n            ids.append(str(val).strip())\n    \n    # Split multi-IDs (separated by |)\n    all_ids = []\n    for id_str in ids:\n        all_ids.extend([i.strip() for i in str(id_str).split('|') if i.strip()])\n    \n    # Search in all publication pages\n    candidates = []\n    for _, pub_row in pubs.iterrows():\n        page_text = str(pub_row['page_text']).lower()  # Lower for case-insensitive\n        for id_val in all_ids:\n            id_lower = id_val.lower()\n            if id_lower in page_text:\n                # Extract a generous snippet around the ID\n                start = max(0, page_text.find(id_lower) - 800)\n                end = min(len(page_text), page_text.find(id_lower) + len(id_lower) + 1200)\n                snippet = pub_row['page_text'][start:end]  # Original case\n                candidates.append({\n                    'oare_id': row['oare_id'],\n                    'matched_id': id_val,\n                    'pdf_name': pub_row['pdf_name'],\n                    'page': pub_row['page'],\n                    'snippet': snippet\n                })\n                break  # One match per page is enough for now\n    return candidates\n\n# Test on a small subset first (takes time on full 8k!)\nsample_published = published.head(50)  # Increase later!\nall_candidates = []\nfor _, row in sample_published.iterrows():\n    all_candidates.extend(find_candidate_translation(row))\n\ncandidates_df = pd.DataFrame(all_candidates)\nprint(f\"Found {len(candidates_df)} candidate translation snippets!\")\ncandidates_df.head(10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T15:19:51.083843Z","iopub.execute_input":"2025-12-17T15:19:51.084116Z","iopub.status.idle":"2025-12-17T15:30:14.252873Z","shell.execute_reply.started":"2025-12-17T15:19:51.084093Z","shell.execute_reply":"2025-12-17T15:30:14.252241Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from nltk import sent_tokenize\n\ndef split_to_sentences(df):\n    sentences = []\n    for _, row in df.iterrows():\n        akk_sents = sent_tokenize(row['transliteration'])  # Or better splitter for ancient text\n        eng_sents = sent_tokenize(row['translation'])\n        # Simple align 1:1 if lengths match, else advanced alignment\n        min_len = min(len(akk_sents), len(eng_sents))\n        for i in range(min_len):\n            sentences.append({'akkadian': akk_sents[i], 'english': eng_sents[i]})\n    return pd.DataFrame(sentences)\n\ntrain_sentences = split_to_sentences(train)\n# Add extra aligned sentences here!","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T15:31:07.782344Z","iopub.execute_input":"2025-12-17T15:31:07.783251Z","iopub.status.idle":"2025-12-17T15:31:08.067149Z","shell.execute_reply.started":"2025-12-17T15:31:07.783219Z","shell.execute_reply":"2025-12-17T15:31:08.066368Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install transformers datasets sentencepiece evaluate sacrebleu\n\nfrom datasets import Dataset\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM, Seq2SeqTrainer, Seq2SeqTrainingArguments\nimport evaluate\n\ntrain_ds = Dataset.from_pandas(train_sentences)\n\ntokenizer = AutoTokenizer.from_pretrained(\"google/mt5-base\")  # Or \"google/byt5-base\" for byte-level (handles rare chars better)\n\ndef preprocess(examples):\n    inputs = examples['akkadian']\n    targets = examples['english']\n    model_inputs = tokenizer(inputs, max_length=256, truncation=True)\n    labels = tokenizer(targets, max_length=256, truncation=True).input_ids\n    model_inputs[\"labels\"] = labels\n    return model_inputs\n\ntokenized = train_ds.map(preprocess, batched=True)\n\nmodel = AutoModelForSeq2SeqLM.from_pretrained(\"google/mt5-base\")\n\nargs = Seq2SeqTrainingArguments(\n    output_dir=\"akkadian_mt\",\n    evaluation_strategy=\"no\",\n    learning_rate=3e-5,\n    per_device_train_batch_size=8,\n    weight_decay=0.01,\n    save_total_limit=2,\n    num_train_epochs=20,\n    predict_with_generate=True,\n    fp16=True,  # GPU\n)\n\ntrainer = Seq2SeqTrainer(model=model, args=args, train_dataset=tokenized, tokenizer=tokenizer)\ntrainer.train()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T15:31:34.191432Z","iopub.execute_input":"2025-12-17T15:31:34.192069Z","iopub.status.idle":"2025-12-17T15:32:21.339176Z","shell.execute_reply.started":"2025-12-17T15:31:34.192039Z","shell.execute_reply":"2025-12-17T15:32:21.338143Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch  # ← THIS WAS MISSING!\n\n# Move model to GPU once\nmodel.to('cuda')\nmodel.eval()\n\n# Batch inference function\ndef batch_generate(transliterations):\n    inputs = tokenizer(\n        transliterations,\n        return_tensors=\"pt\",\n        padding=True,\n        truncation=True,\n        max_length=256\n    ).to('cuda')\n    \n    with torch.no_grad():\n        outputs = model.generate(\n            **inputs,\n            max_length=256,\n            num_beams=5,\n            early_stopping=True\n        )\n    \n    translations = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n    return translations\n\n# Generate in batches\nbatch_size = 16\ntranslations = []\n\nfor i in range(0, len(test), batch_size):\n    batch_texts = test['transliteration'][i:i+batch_size].tolist()\n    batch_trans = batch_generate(batch_texts)\n    translations.extend(batch_trans)\n    print(f\"Processed {i + batch_size}/{len(test)} sentences...\")  # Progress tracker~\n\n# Save\ntest['translation'] = translations\ntest[['id', 'translation']].to_csv('submission.csv', index=False)\nprint(\"🎉 Submission ready! Commit and submit now~\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T15:36:42.520359Z","iopub.execute_input":"2025-12-17T15:36:42.520714Z","iopub.status.idle":"2025-12-17T15:36:43.63813Z","shell.execute_reply.started":"2025-12-17T15:36:42.520685Z","shell.execute_reply":"2025-12-17T15:36:43.637508Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\n# Paths on Kaggle\ntrain_path = '/kaggle/input/deep-past-initiative-machine-translation/train.csv'\ntest_path = '/kaggle/input/deep-past-initiative-machine-translation/test.csv'\npublished_path = '/kaggle/input/deep-past-initiative-machine-translation/published_texts.csv'\n\n# Load\ntrain = pd.read_csv(train_path)\ntest = pd.read_csv(test_path)\npublished = pd.read_csv(published_path)\n\n# Quick look\nprint(\"TRAIN DATA (~1500 docs)\")\nprint(train.head(3))\nprint(\"\\nTrain shape:\", train.shape)\nprint(\"\\nSample transliteration:\\n\", train['transliteration'].iloc[0][:500])\nprint(\"\\nSample translation:\\n\", train['translation'].iloc[0][:500])\n\nprint(\"\\n\\nTEST DATA (~4000 sentences)\")\nprint(test.head(3))\nprint(\"\\nTest shape:\", test.shape)\n\nprint(\"\\n\\nPUBLISHED TEXTS (8000+ potential extra data)\")\nprint(published.columns.tolist())\nprint(\"\\nSample published transliteration:\\n\", published['transliteration'].iloc[0][:500] if 'transliteration' in published.columns else \"No transliteration column?\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T15:37:10.767354Z","iopub.execute_input":"2025-12-17T15:37:10.767708Z","iopub.status.idle":"2025-12-17T15:37:11.018012Z","shell.execute_reply.started":"2025-12-17T15:37:10.76768Z","shell.execute_reply":"2025-12-17T15:37:11.017242Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Quick test on first row\ntest_text = test['transliteration'].iloc[0]\ntest_batch = [test_text]  # Make it a list for batch\ntest_trans = batch_generate(test_batch)\nprint(\"Input:\", test_text[:200])\nprint(\"Translation:\", test_trans[0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T15:58:33.286497Z","iopub.execute_input":"2025-12-17T15:58:33.287201Z","iopub.status.idle":"2025-12-17T15:58:33.496279Z","shell.execute_reply.started":"2025-12-17T15:58:33.28716Z","shell.execute_reply":"2025-12-17T15:58:33.495544Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import gc\ntorch.cuda.empty_cache()\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T15:58:37.214386Z","iopub.execute_input":"2025-12-17T15:58:37.214706Z","iopub.status.idle":"2025-12-17T15:58:37.617519Z","shell.execute_reply.started":"2025-12-17T15:58:37.21468Z","shell.execute_reply":"2025-12-17T15:58:37.615925Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport gc\n\ntorch.cuda.empty_cache()\ngc.collect()\n\nmodel.to('cuda')\nmodel.eval()\n\ndef batch_generate(transliterations):\n    inputs = tokenizer(\n        transliterations,\n        return_tensors=\"pt\",\n        padding=True,\n        truncation=True,\n        max_length=256\n    ).to('cuda')\n    \n    with torch.no_grad():\n        outputs = model.generate(**inputs, max_length=256)  # Simple, no beams\n    \n    return tokenizer.batch_decode(outputs, skip_special_tokens=True)\n\nbatch_size = 4  # Smaller = safer on P100\ntranslations = []\n\nfor i in range(0, len(test), batch_size):\n    batch_texts = test['transliteration'][i:i+batch_size].tolist()\n    batch_trans = batch_generate(batch_texts)\n    translations.extend(batch_trans)\n    print(f\"Processed {min(i + batch_size, len(test))}/{len(test)}...\")\n\ntest['translation'] = translations\ntest[['id', 'translation']].to_csv('submission.csv', index=False)\nprint(\"🎉 submission.csv ready!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T15:58:56.984419Z","iopub.execute_input":"2025-12-17T15:58:56.985086Z","iopub.status.idle":"2025-12-17T15:58:57.719504Z","shell.execute_reply.started":"2025-12-17T15:58:56.985057Z","shell.execute_reply":"2025-12-17T15:58:57.718762Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Quick test on first row\ntest_text = test['transliteration'].iloc[0]\ntest_batch = [test_text]  # Make it a list for batch\ntest_trans = batch_generate(test_batch)\nprint(\"Input:\", test_text[:200])\nprint(\"Translation:\", test_trans[0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T16:12:39.582616Z","iopub.execute_input":"2025-12-17T16:12:39.582909Z","iopub.status.idle":"2025-12-17T16:12:39.791918Z","shell.execute_reply.started":"2025-12-17T16:12:39.582884Z","shell.execute_reply":"2025-12-17T16:12:39.791123Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch  # Just in case\nimport pandas as pd\n\n# Load test (dummy for now)\ntest = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/test.csv')\n\n# Move model to CPU for safety\nmodel.to('cpu')\nmodel.eval()\n\n# Simple inference (no batch, no beams – super light)\ntranslations = []\nfor translit in test['transliteration']:\n    inputs = tokenizer(translit, return_tensors=\"pt\", truncation=True, max_length=256)\n    with torch.no_grad():\n        outputs = model.generate(**inputs, max_length=256)\n    trans = tokenizer.decode(outputs[0], skip_special_tokens=True)\n    translations.append(trans)\n\ntest['translation'] = translations\n\n# Save!\ntest[['id', 'translation']].to_csv('submission.csv', index=False)\nprint(\"🎉 submission.csv created successfully!\")\nimport os\nprint(\"Files here:\", os.listdir('/kaggle/working'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T16:13:48.751535Z","iopub.execute_input":"2025-12-17T16:13:48.751853Z","iopub.status.idle":"2025-12-17T16:13:54.218827Z","shell.execute_reply.started":"2025-12-17T16:13:48.751826Z","shell.execute_reply":"2025-12-17T16:13:54.218089Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\n# Load dummy test\ntest = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/test.csv')\n\n# Super simple dummy translations (to test submission format)\n# Replace with real model later!\ntest['translation'] = \"This is a test translation for the ancient tablet sentence.\"\n\n# Save submission\ntest[['id', 'translation']].to_csv('submission.csv', index=False)\n\n# Check if it worked\nimport os\nprint(\"Files in /kaggle/working:\")\nprint(os.listdir('/kaggle/working'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T16:15:40.853248Z","iopub.execute_input":"2025-12-17T16:15:40.853606Z","iopub.status.idle":"2025-12-17T16:15:42.866678Z","shell.execute_reply.started":"2025-12-17T16:15:40.853576Z","shell.execute_reply":"2025-12-17T16:15:42.865956Z"}},"outputs":[],"execution_count":null}]}