{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"},{"sourceId":746586,"sourceType":"modelInstanceVersion","modelInstanceId":570042,"modelId":582322}],"dockerImageVersionId":31259,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ============================================================\n# EDA SETUP (RUN ONCE BEFORE ALL EDA)\n# Purpose: Load dataset and prepare tokens\n# ============================================================\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom collections import Counter\nimport re\n\nDATA_PATH = \"/kaggle/input/deep-past-initiative-machine-translation\"\ndf = pd.read_csv(f\"{DATA_PATH}/train.csv\")\n\n# Source language: Akkadian transliteration\ntexts = df[\"transliteration\"].astype(str).tolist()\n\n# Tokenization (simple whitespace, offline-safe)\ntokens = [w for t in texts for w in t.split()]\ntoken_freq = Counter(tokens)\n\nprint(\"EDA setup completed\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-10T05:54:58.632998Z","iopub.execute_input":"2026-02-10T05:54:58.633973Z","iopub.status.idle":"2026-02-10T05:54:58.722278Z","shell.execute_reply.started":"2026-02-10T05:54:58.633931Z","shell.execute_reply":"2026-02-10T05:54:58.721033Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# BEFORE TRAINING\n# Shows the most frequent tokens in the dataset\n\ntop_words = token_freq.most_common(20)\n\nplt.figure(figsize=(8,4))\nplt.bar([w for w,_ in top_words], [c for _,c in top_words])\nplt.xticks(rotation=90)\nplt.title(\"Top-N Word Frequency\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-10T05:55:03.801066Z","iopub.execute_input":"2026-02-10T05:55:03.801555Z","iopub.status.idle":"2026-02-10T05:55:04.115719Z","shell.execute_reply.started":"2026-02-10T05:55:03.801509Z","shell.execute_reply":"2026-02-10T05:55:04.114621Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# BEFORE TRAINING\n# Visual frequency overview (proxy for word cloud)\n\ntop_words = token_freq.most_common(30)\n\nplt.figure(figsize=(8,4))\nplt.scatter(range(len(top_words)), [c for _,c in top_words])\nplt.xticks(range(len(top_words)), [w for w,_ in top_words], rotation=90)\nplt.title(\"Word Cloud (Frequency Proxy)\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-10T05:55:08.010522Z","iopub.execute_input":"2026-02-10T05:55:08.011658Z","iopub.status.idle":"2026-02-10T05:55:08.268296Z","shell.execute_reply.started":"2026-02-10T05:55:08.011616Z","shell.execute_reply":"2026-02-10T05:55:08.266984Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# BEFORE TRAINING\n# Distribution of token frequencies\n\nplt.figure(figsize=(6,4))\nplt.hist(token_freq.values(), bins=50)\nplt.title(\"Token Frequency Histogram\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-10T05:55:12.348931Z","iopub.execute_input":"2026-02-10T05:55:12.349436Z","iopub.status.idle":"2026-02-10T05:55:12.577777Z","shell.execute_reply.started":"2026-02-10T05:55:12.349389Z","shell.execute_reply":"2026-02-10T05:55:12.576369Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# BEFORE TRAINING\n# Tokens appearing very few times (≤2)\n\nrare_counts = [c for c in token_freq.values() if c <= 2]\n\nplt.figure(figsize=(6,4))\nplt.hist(rare_counts, bins=30)\nplt.title(\"Rare Word Distribution (≤2 occurrences)\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-10T05:55:16.014310Z","iopub.execute_input":"2026-02-10T05:55:16.014981Z","iopub.status.idle":"2026-02-10T05:55:16.216550Z","shell.execute_reply.started":"2026-02-10T05:55:16.014932Z","shell.execute_reply":"2026-02-10T05:55:16.215014Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# BEFORE TRAINING\n# Cumulative coverage of vocabulary\n\nfreqs = np.array(sorted(token_freq.values(), reverse=True))\ncum = np.cumsum(freqs) / np.sum(freqs)\n\nplt.figure(figsize=(6,4))\nplt.plot(cum)\nplt.title(\"Cumulative Frequency Curve\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-10T05:55:21.333896Z","iopub.execute_input":"2026-02-10T05:55:21.334402Z","iopub.status.idle":"2026-02-10T05:55:21.502259Z","shell.execute_reply.started":"2026-02-10T05:55:21.334354Z","shell.execute_reply":"2026-02-10T05:55:21.500467Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# BEFORE TRAINING\n# Length of individual tokens\n\nword_lengths = [len(w) for w in tokens]\n\nplt.figure(figsize=(6,4))\nplt.hist(word_lengths, bins=20)\nplt.title(\"Word Length Distribution\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-10T05:55:24.581143Z","iopub.execute_input":"2026-02-10T05:55:24.581632Z","iopub.status.idle":"2026-02-10T05:55:24.953722Z","shell.execute_reply.started":"2026-02-10T05:55:24.581586Z","shell.execute_reply":"2026-02-10T05:55:24.952416Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# BEFORE TRAINING\n# Length of each sentence in characters\n\nchar_lengths = [len(t) for t in texts]\n\nplt.figure(figsize=(6,4))\nplt.hist(char_lengths, bins=30)\nplt.title(\"Character Count Distribution\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-10T05:55:28.823599Z","iopub.execute_input":"2026-02-10T05:55:28.824156Z","iopub.status.idle":"2026-02-10T05:55:28.998015Z","shell.execute_reply.started":"2026-02-10T05:55:28.824123Z","shell.execute_reply":"2026-02-10T05:55:28.996806Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# BEFORE TRAINING\n# Stopword frequency using a small English proxy list\n\nstopwords = {\"a\",\"the\",\"and\",\"of\",\"to\",\"in\",\"for\",\"on\",\"with\"}\nstop_counts = {w: token_freq[w] for w in stopwords if w in token_freq}\n\nplt.figure(figsize=(6,4))\nplt.bar(stop_counts.keys(), stop_counts.values())\nplt.title(\"Stopword Frequency\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-10T05:55:32.911072Z","iopub.execute_input":"2026-02-10T05:55:32.912415Z","iopub.status.idle":"2026-02-10T05:55:33.014103Z","shell.execute_reply.started":"2026-02-10T05:55:32.912363Z","shell.execute_reply":"2026-02-10T05:55:33.012975Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# BEFORE TRAINING\n# Lemma approximation using lowercase normalization\n\nlemmas = [w.lower() for w in tokens]\nlemma_freq = Counter(lemmas).most_common(20)\n\nplt.figure(figsize=(8,4))\nplt.bar([w for w,_ in lemma_freq], [c for _,c in lemma_freq])\nplt.xticks(rotation=90)\nplt.title(\"Lemma Frequency Plot\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-10T05:55:36.130461Z","iopub.execute_input":"2026-02-10T05:55:36.131633Z","iopub.status.idle":"2026-02-10T05:55:36.379207Z","shell.execute_reply.started":"2026-02-10T05:55:36.131590Z","shell.execute_reply":"2026-02-10T05:55:36.378012Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# BEFORE TRAINING\n# Syllable estimation via vowel groups (heuristic)\n\ndef syllable_count(word):\n    return len(re.findall(r\"[aeiouAEIOU]\", word))\n\nsyllables = [syllable_count(w) for w in tokens]\n\nplt.figure(figsize=(6,4))\nplt.hist(syllables, bins=10)\nplt.title(\"Syllable Count Distribution\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-10T05:55:40.614277Z","iopub.execute_input":"2026-02-10T05:55:40.615289Z","iopub.status.idle":"2026-02-10T05:55:41.061231Z","shell.execute_reply.started":"2026-02-10T05:55:40.615251Z","shell.execute_reply":"2026-02-10T05:55:41.060142Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# BEFORE TRAINING\n# Heuristic POS tagging (suffix-based)\n\npos_tags = []\nfor w in tokens:\n    if w.endswith(\"um\") or w.endswith(\"am\"):\n        pos_tags.append(\"NOUN\")\n    elif w.endswith(\"ti\") or w.endswith(\"ta\"):\n        pos_tags.append(\"VERB\")\n    else:\n        pos_tags.append(\"OTHER\")\n\npos_freq = Counter(pos_tags)\n\nplt.figure(figsize=(6,4))\nplt.bar(pos_freq.keys(), pos_freq.values())\nplt.title(\"Part-of-Speech Tag Distribution (Heuristic)\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-10T05:55:44.904851Z","iopub.execute_input":"2026-02-10T05:55:44.905334Z","iopub.status.idle":"2026-02-10T05:55:45.095824Z","shell.execute_reply.started":"2026-02-10T05:55:44.905286Z","shell.execute_reply":"2026-02-10T05:55:45.094623Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# BEFORE TRAINING\n# Heuristic NER using capitalization patterns\n\nner_tags = []\nfor w in tokens:\n    if w.isupper():\n        ner_tags.append(\"LOGOGRAM\")\n    elif w[0].isupper():\n        ner_tags.append(\"PROPER_NOUN\")\n    else:\n        ner_tags.append(\"OTHER\")\n\nner_freq = Counter(ner_tags)\n\nplt.figure(figsize=(6,4))\nplt.bar(ner_freq.keys(), ner_freq.values())\nplt.title(\"Named Entity Category Distribution (Heuristic)\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-10T05:55:48.701599Z","iopub.execute_input":"2026-02-10T05:55:48.702181Z","iopub.status.idle":"2026-02-10T05:55:48.867465Z","shell.execute_reply.started":"2026-02-10T05:55:48.702148Z","shell.execute_reply":"2026-02-10T05:55:48.866052Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# BEFORE TRAINING\n# Dependency roles approximated by token position\n\ndeps = []\nfor t in texts:\n    words = t.split()\n    for i,_ in enumerate(words):\n        if i == 0:\n            deps.append(\"ROOT\")\n        elif i == 1:\n            deps.append(\"SUBJ/OBJ\")\n        else:\n            deps.append(\"MOD\")\n\ndep_freq = Counter(deps)\n\nplt.figure(figsize=(6,4))\nplt.bar(dep_freq.keys(), dep_freq.values())\nplt.title(\"Dependency Relation Distribution (Approximation)\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-10T05:55:52.522084Z","iopub.execute_input":"2026-02-10T05:55:52.522581Z","iopub.status.idle":"2026-02-10T05:55:52.670966Z","shell.execute_reply.started":"2026-02-10T05:55:52.522529Z","shell.execute_reply":"2026-02-10T05:55:52.670046Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# BEFORE TRAINING\n# Flesch-like readability approximation\n\nreadability_scores = []\n\nfor t in texts:\n    words = t.split()\n    if len(words) > 0:\n        avg_word_len = sum(len(w) for w in words) / len(words)\n        score = 206.835 - 1.015 * len(words) - 84.6 * avg_word_len\n        readability_scores.append(score)\n\nplt.figure(figsize=(6,4))\nplt.hist(readability_scores, bins=30)\nplt.title(\"Readability Score Distribution (Approximation)\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-10T05:56:00.756027Z","iopub.execute_input":"2026-02-10T05:56:00.757035Z","iopub.status.idle":"2026-02-10T05:56:00.968204Z","shell.execute_reply.started":"2026-02-10T05:56:00.756995Z","shell.execute_reply":"2026-02-10T05:56:00.966896Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# EDA SETUP (BEFORE TRAINING)\n# ============================================================\n\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom collections import Counter\nimport numpy as np\n\n# Load dataset\nDATA_PATH = \"/kaggle/input/deep-past-initiative-machine-translation\"\ntrain_df = pd.read_csv(f\"{DATA_PATH}/train.csv\")\n\nprint(\"✅ Dataset loaded\")\ntrain_df.head()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-10T05:56:03.902401Z","iopub.execute_input":"2026-02-10T05:56:03.902879Z","iopub.status.idle":"2026-02-10T05:56:03.951866Z","shell.execute_reply.started":"2026-02-10T05:56:03.902835Z","shell.execute_reply":"2026-02-10T05:56:03.950807Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# EDA 1: Character Count Distribution\n# Purpose: Understand sequence length for model max_length\n# ============================================================\n\nchar_counts = train_df[\"transliteration\"].astype(str).apply(len)\n\nplt.figure(figsize=(7,4))\nplt.hist(char_counts, bins=50)\nplt.title(\"Character Count Distribution (Transliteration)\")\nplt.xlabel(\"Number of Characters\")\nplt.ylabel(\"Frequency\")\nplt.grid(True)\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-10T05:56:08.829972Z","iopub.execute_input":"2026-02-10T05:56:08.830478Z","iopub.status.idle":"2026-02-10T05:56:09.083630Z","shell.execute_reply.started":"2026-02-10T05:56:08.830429Z","shell.execute_reply":"2026-02-10T05:56:09.081845Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# EDA 2: Word Length Distribution\n# Purpose: Examine morphological complexity\n# ============================================================\n\nwords = [\n    w for text in train_df[\"transliteration\"].astype(str)\n    for w in text.split()\n]\n\nword_lengths = [len(w) for w in words]\n\nplt.figure(figsize=(7,4))\nplt.hist(word_lengths, bins=20)\nplt.title(\"Word Length Distribution\")\nplt.xlabel(\"Word Length\")\nplt.ylabel(\"Frequency\")\nplt.grid(True)\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-10T05:56:11.399067Z","iopub.execute_input":"2026-02-10T05:56:11.400289Z","iopub.status.idle":"2026-02-10T05:56:11.838104Z","shell.execute_reply.started":"2026-02-10T05:56:11.400248Z","shell.execute_reply":"2026-02-10T05:56:11.836622Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# EDA 3: Token Frequency Histogram\n# Purpose: Identify vocabulary sparsity\n# ============================================================\n\ntoken_freq = Counter(w.lower() for w in words)\nfreq_values = list(token_freq.values())\n\nplt.figure(figsize=(7,4))\nplt.hist(freq_values, bins=50, log=True)\nplt.title(\"Token Frequency Histogram (Log Scale)\")\nplt.xlabel(\"Token Frequency\")\nplt.ylabel(\"Number of Tokens\")\nplt.grid(True)\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-10T05:56:18.489177Z","iopub.execute_input":"2026-02-10T05:56:18.490071Z","iopub.status.idle":"2026-02-10T05:56:19.121039Z","shell.execute_reply.started":"2026-02-10T05:56:18.490030Z","shell.execute_reply":"2026-02-10T05:56:19.119934Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# EDA 4: Top-20 Token Frequencies\n# Purpose: Identify most frequent signs/syllables\n# ============================================================\n\ntop_tokens = token_freq.most_common(20)\nlabels, values = zip(*top_tokens)\n\nplt.figure(figsize=(8,4))\nplt.bar(labels, values)\nplt.xticks(rotation=45)\nplt.title(\"Top-20 Token Frequencies\")\nplt.ylabel(\"Frequency\")\nplt.grid(True)\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-10T05:56:22.471640Z","iopub.execute_input":"2026-02-10T05:56:22.472629Z","iopub.status.idle":"2026-02-10T05:56:23.285976Z","shell.execute_reply.started":"2026-02-10T05:56:22.472578Z","shell.execute_reply":"2026-02-10T05:56:23.284830Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# EDA 5: Rare Word Distribution\n# Purpose: Show long-tail low-resource behaviour\n# ============================================================\n\nrare_freqs = [f for f in token_freq.values() if f <= 2]\n\nplt.figure(figsize=(7,4))\nplt.hist(rare_freqs, bins=2)\nplt.title(\"Rare Word Distribution (Frequency ≤ 2)\")\nplt.xlabel(\"Frequency\")\nplt.ylabel(\"Count\")\nplt.grid(True)\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-09T20:04:48.741760Z","iopub.execute_input":"2026-02-09T20:04:48.742132Z","iopub.status.idle":"2026-02-09T20:04:48.894742Z","shell.execute_reply.started":"2026-02-09T20:04:48.742088Z","shell.execute_reply":"2026-02-09T20:04:48.893889Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# EDA 6: Cumulative Token Frequency Curve\n# Purpose: Observe Zipf-like distribution\n# ============================================================\n\nsorted_freqs = sorted(token_freq.values(), reverse=True)\ncumulative = np.cumsum(sorted_freqs)\n\nplt.figure(figsize=(7,4))\nplt.plot(cumulative)\nplt.title(\"Cumulative Token Frequency Curve\")\nplt.xlabel(\"Ranked Tokens\")\nplt.ylabel(\"Cumulative Frequency\")\nplt.grid(True)\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-10T05:56:25.961963Z","iopub.execute_input":"2026-02-10T05:56:25.962371Z","iopub.status.idle":"2026-02-10T05:56:26.143879Z","shell.execute_reply.started":"2026-02-10T05:56:25.962337Z","shell.execute_reply":"2026-02-10T05:56:26.141417Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# EDA 7: Stopword Frequency (Heuristic Proxy)\n# Purpose: Exploratory comparison only\n# ============================================================\n\nenglish_stopwords = {\n    \"a\",\"the\",\"and\",\"of\",\"to\",\"in\",\"for\",\"on\",\"with\",\"is\",\"are\"\n}\n\nstop_counts = {\n    w: token_freq[w] for w in english_stopwords if w in token_freq\n}\n\nplt.figure(figsize=(6,4))\nplt.bar(stop_counts.keys(), stop_counts.values())\nplt.title(\"Stopword Frequency (Heuristic)\")\nplt.ylabel(\"Frequency\")\nplt.grid(True)\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-10T05:56:29.966973Z","iopub.execute_input":"2026-02-10T05:56:29.968377Z","iopub.status.idle":"2026-02-10T05:56:30.094208Z","shell.execute_reply.started":"2026-02-10T05:56:29.968333Z","shell.execute_reply":"2026-02-10T05:56:30.092899Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# EDA 14: Word Cloud\n# Purpose: Visual summary (supporting only)\n# ============================================================\n\nfrom wordcloud import WordCloud\n\nwc = WordCloud(width=800, height=400, background_color=\"white\")\nwc.generate(\" \".join(words))\n\nplt.figure(figsize=(10,4))\nplt.imshow(wc, interpolation=\"bilinear\")\nplt.axis(\"off\")\nplt.title(\"Word Cloud (Transliteration Tokens)\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-10T05:56:33.340793Z","iopub.execute_input":"2026-02-10T05:56:33.341663Z","iopub.status.idle":"2026-02-10T05:56:35.010499Z","shell.execute_reply.started":"2026-02-10T05:56:33.341623Z","shell.execute_reply":"2026-02-10T05:56:35.008743Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# LOAD MARIANMT MODEL & TOKENIZER (CORRECT PATH)\n# ============================================================\n\nMODEL_PATH = \"/kaggle/input/marian-mt/transformers/default/1/opus-mt-en-de\"\n\nfrom transformers import MarianMTModel, MarianTokenizer\n\ntokenizer = MarianTokenizer.from_pretrained(\n    MODEL_PATH,\n    local_files_only=True\n)\n\nmodel = MarianMTModel.from_pretrained(\n    MODEL_PATH,\n    local_files_only=True\n)\n\nprint(\"✅ MarianMT model and tokenizer loaded successfully\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-10T06:44:24.309274Z","iopub.execute_input":"2026-02-10T06:44:24.310124Z","iopub.status.idle":"2026-02-10T06:44:24.965127Z","shell.execute_reply.started":"2026-02-10T06:44:24.310084Z","shell.execute_reply":"2026-02-10T06:44:24.963975Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SINGLE-CELL: TRAIN MODEL + CREATE submission.csv\n# Akkadian (transliteration) → English\n# OFFLINE | CPU SAFE | KAGGLE LEGAL | WEB-APP READY\n# ============================================================\n\n# -----------------------------\n# OFFLINE ENFORCEMENT\n# -----------------------------\nimport os\nos.environ[\"HF_HUB_OFFLINE\"] = \"1\"\nos.environ[\"TRANSFORMERS_OFFLINE\"] = \"1\"\nos.environ[\"HF_DATASETS_OFFLINE\"] = \"1\"\nos.environ[\"HF_HUB_DISABLE_TELEMETRY\"] = \"1\"\nos.environ[\"HF_HUB_DISABLE_PROGRESS_BARS\"] = \"1\"\n\n# -----------------------------\n# IMPORTS\n# -----------------------------\nimport torch\nimport pandas as pd\nfrom datasets import Dataset\nfrom transformers import MarianMTModel, MarianTokenizer\nfrom torch.optim import AdamW\nfrom torch.nn.utils import clip_grad_norm_\nfrom tqdm.auto import tqdm\n\n# -----------------------------\n# PATHS (AS CONFIRMED)\n# -----------------------------\nDATA_PATH   = \"/kaggle/input/deep-past-initiative-machine-translation\"\nMODEL_PATH  = \"/kaggle/input/marian-mt/transformers/default/1/opus-mt-en-de\"\nOUTPUT_DIR  = \"/kaggle/working/final_model\"\nSUB_PATH    = \"/kaggle/working/submission.csv\"\n\n# -----------------------------\n# LOAD TRAINING DATA\n# -----------------------------\ntrain_df = pd.read_csv(f\"{DATA_PATH}/train.csv\")\nprint(\"✅ Training data loaded:\", train_df.shape)\n\n# -----------------------------\n# LOAD BASE MODEL & TOKENIZER\n# -----------------------------\ntokenizer = MarianTokenizer.from_pretrained(MODEL_PATH, local_files_only=True)\nmodel = MarianMTModel.from_pretrained(MODEL_PATH, local_files_only=True)\nprint(\"✅ MarianMT model loaded\")\n\n# -----------------------------\n# DATASET PREPARATION\n# -----------------------------\nhf_ds = Dataset.from_pandas(train_df)\n\nMAX_LEN = 128\n\ndef preprocess(batch):\n    inputs = tokenizer(\n        batch[\"transliteration\"],\n        padding=\"max_length\",\n        truncation=True,\n        max_length=MAX_LEN\n    )\n    targets = tokenizer(\n        text_target=batch[\"translation\"],\n        padding=\"max_length\",\n        truncation=True,\n        max_length=MAX_LEN\n    )\n    inputs[\"labels\"] = targets[\"input_ids\"]\n    return inputs\n\nhf_ds = hf_ds.map(preprocess, batched=True, remove_columns=hf_ds.column_names)\nprint(\"✅ Dataset tokenised\")\n\n# -----------------------------\n# TRAINING SETUP\n# -----------------------------\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)\nmodel.train()\n\nBATCH_SIZE = 2\nEPOCHS = 3\nLR = 5e-5\n\noptimizer = AdamW(model.parameters(), lr=LR)\n\nprint(\"🚀 Training started\")\n\n# -----------------------------\n# TRAINING LOOP\n# -----------------------------\nfor epoch in range(EPOCHS):\n    total_loss = 0.0\n\n    for i in range(0, len(hf_ds), BATCH_SIZE):\n        batch = hf_ds[i:i+BATCH_SIZE]\n        batch = {k: torch.tensor(v).to(device) for k, v in batch.items()}\n\n        outputs = model(**batch)\n        loss = outputs.loss\n\n        loss.backward()\n        clip_grad_norm_(model.parameters(), 1.0)\n        optimizer.step()\n        optimizer.zero_grad()\n\n        total_loss += loss.item()\n\n    print(f\"✅ Epoch {epoch+1}/{EPOCHS} completed | Avg Loss: {total_loss:.4f}\")\n\nprint(\"🏁 Training completed\")\n\n# -----------------------------\n# SAVE TRAINED MODEL\n# -----------------------------\nos.makedirs(OUTPUT_DIR, exist_ok=True)\nmodel.save_pretrained(OUTPUT_DIR)\ntokenizer.save_pretrained(OUTPUT_DIR)\n\nprint(\"📦 Trained model saved to:\", OUTPUT_DIR)\nprint(\"📁 Model files:\", os.listdir(OUTPUT_DIR))\n\n# -----------------------------\n# LOAD TEST DATA\n# -----------------------------\ntest_df = pd.read_csv(f\"{DATA_PATH}/test.csv\")\nprint(\"✅ Test data loaded:\", test_df.shape)\n\n# -----------------------------\n# INFERENCE FUNCTION\n# -----------------------------\nmodel.eval()\n\ndef translate_batch(texts, max_length=256):\n    inputs = tokenizer(\n        texts,\n        return_tensors=\"pt\",\n        padding=True,\n        truncation=True,\n        max_length=max_length\n    ).to(device)\n\n    with torch.no_grad():\n        outputs = model.generate(\n            **inputs,\n            num_beams=4,\n            max_length=max_length\n        )\n\n    return tokenizer.batch_decode(outputs, skip_special_tokens=True)\n\n# -----------------------------\n# GENERATE TRANSLATIONS\n# -----------------------------\ntranslations = []\n\nfor i in tqdm(range(0, len(test_df), 8)):\n    batch = test_df[\"transliteration\"].iloc[i:i+8].tolist()\n    translations.extend(translate_batch(batch))\n\n# -----------------------------\n# CREATE submission.csv\n# -----------------------------\nsubmission = pd.DataFrame({\n    \"id\": test_df[\"id\"],\n    \"translation\": translations\n})\n\nsubmission.to_csv(SUB_PATH, index=False)\n\nprint(\"📄 submission.csv created at:\", SUB_PATH)\nprint(\"📁 Files in /kaggle/working:\", os.listdir(\"/kaggle/working\"))\nsubmission.head()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-10T06:44:29.190457Z","iopub.execute_input":"2026-02-10T06:44:29.191509Z","iopub.status.idle":"2026-02-10T08:02:54.889609Z","shell.execute_reply.started":"2026-02-10T06:44:29.191472Z","shell.execute_reply":"2026-02-10T08:02:54.888586Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\nMODEL_DIR = \"/kaggle/working/final_model\"\n\nassert os.path.isdir(MODEL_DIR), \"❌ final_model folder not found\"\nprint(\"✅ Model folder exists\")\nprint(\"📁 Files:\", os.listdir(MODEL_DIR))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-10T08:09:58.736622Z","iopub.execute_input":"2026-02-10T08:09:58.737383Z","iopub.status.idle":"2026-02-10T08:09:58.746584Z","shell.execute_reply.started":"2026-02-10T08:09:58.737343Z","shell.execute_reply":"2026-02-10T08:09:58.745102Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import shutil\n\nZIP_PATH = \"/kaggle/working/final_model.zip\"\n\nshutil.make_archive(\n    base_name=ZIP_PATH.replace(\".zip\", \"\"),\n    format=\"zip\",\n    root_dir=MODEL_DIR\n)\n\nprint(\"✅ final_model.zip created at:\", ZIP_PATH)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-10T08:10:12.072719Z","iopub.execute_input":"2026-02-10T08:10:12.073734Z","iopub.status.idle":"2026-02-10T08:10:30.153091Z","shell.execute_reply.started":"2026-02-10T08:10:12.073697Z","shell.execute_reply":"2026-02-10T08:10:30.152027Z"}},"outputs":[],"execution_count":null}]}