{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":702763,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":533319,"modelId":547030}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import train_test_split\nfrom collections import Counter\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T13:00:59.974542Z","iopub.execute_input":"2025-12-29T13:00:59.975282Z","iopub.status.idle":"2025-12-29T13:00:59.979470Z","shell.execute_reply.started":"2025-12-29T13:00:59.975252Z","shell.execute_reply":"2025-12-29T13:00:59.978820Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"Device: {device}\")\n\nif torch.cuda.is_available():\n    print(f\"GPU Name: {torch.cuda.get_device_name(0)}\")\n    print(f\"GPU Memory: {torch.cuda.get_device_properties(0).total_memory / 1e9:.2f} GB\")\n    print(f\"CUDA Version: {torch.version.cuda}\")\n    \n    # Empty cache at start\n    torch.cuda.empty_cache()\n    \n    # Enable cuDNN benchmark for faster execution\n    torch.backends.cudnn.benchmark = True\n    print(\"✓ GPU acceleration enabled!\")\nelse:\n    print(\"⚠ GPU not available. Using CPU (slower execution)\")\n    print(\"Note: Training will be significantly slower on CPU\")\n    device = torch.device('cpu')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T13:01:00.050499Z","iopub.execute_input":"2025-12-29T13:01:00.050723Z","iopub.status.idle":"2025-12-29T13:01:00.056654Z","shell.execute_reply.started":"2025-12-29T13:01:00.050703Z","shell.execute_reply":"2025-12-29T13:01:00.055948Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, Seq2SeqTrainingArguments, Seq2SeqTrainer\nfrom nltk.translate.bleu_score import corpus_bleu, sentence_bleu","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T13:01:00.130918Z","iopub.execute_input":"2025-12-29T13:01:00.131203Z","iopub.status.idle":"2025-12-29T13:01:00.134716Z","shell.execute_reply.started":"2025-12-29T13:01:00.131183Z","shell.execute_reply":"2025-12-29T13:01:00.133995Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import nltk\ntry:\n    nltk.data.find('tokenizers/punkt')\nexcept LookupError:\n    # If you have internet temporarily, download punkt\n    nltk.download('punkt', quiet=True)\n\nimport math\nimport re\nimport os\nimport json\nfrom pathlib import Path\n\nprint(\"Imports completed successfully!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T13:01:00.217782Z","iopub.execute_input":"2025-12-29T13:01:00.217990Z","iopub.status.idle":"2025-12-29T13:01:00.222944Z","shell.execute_reply.started":"2025-12-29T13:01:00.217972Z","shell.execute_reply":"2025-12-29T13:01:00.222437Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class BLEUScore:\n    \"\"\"Wrapper to mimic sacrebleu's score object\"\"\"\n    def __init__(self, score):\n        self.score = score * 100.0  # Convert to percentage\n        self.precisions = [0.0] * 4  # Dummy values\n        self.bp = 1.0\n        self.sys_len = 0\n        self.ref_len = 0\n\nclass CHRFScore:\n    \"\"\"Wrapper to mimic sacrebleu's score object\"\"\"\n    def __init__(self, score):\n        self.score = score * 100.0  # Convert to percentage\n        self.precisions = [0.0] * 6  # Dummy values\n        self.recalls = [0.0] * 6     # Dummy values\n        self.beta = 2.0\n\nclass BLEU:\n    \"\"\"sacrebleu compatible BLEU calculator using NLTK\"\"\"\n    def __init__(self, tokenize='13a', smooth_method='exp'):\n        self.tokenize = tokenize\n        self.smooth_method = smooth_method\n        self.smooth_fn = self._get_smoothing_function()\n        \n    def _get_smoothing_function(self):\n        \"\"\"Get NLTK smoothing function\"\"\"\n        from nltk.translate.bleu_score import SmoothingFunction\n        smoothie = SmoothingFunction()\n        if self.smooth_method == 'exp':\n            return smoothie.method3\n        elif self.smooth_method == 'floor':\n            return smoothie.method1\n        else:  # default\n            return smoothie.method4\n    \n    def _tokenize(self, text):\n        \"\"\"Simple tokenization similar to sacrebleu's 13a\"\"\"\n        if self.tokenize == 'none':\n            return text.split()\n        text = re.sub(r'([!\"#$%&\\'()*+,-./:;<=>?@[\\\\\\]^_`{|}~])', r' \\1 ', text)\n        text = re.sub(r'\\s+', ' ', text)\n        return text.strip().split()\n    \n    def sentence_score(self, hypothesis, references):\n        \"\"\"Score a single sentence\"\"\"\n        from nltk.translate.bleu_score import sentence_bleu\n        \n        if isinstance(hypothesis, str):\n            hyp_tokens = self._tokenize(hypothesis)\n        else:\n            hyp_tokens = hypothesis\n            \n        if isinstance(references[0], str):\n            refs_tokens = [self._tokenize(ref) for ref in references]\n        else:\n            refs_tokens = references\n        \n        refs_for_nltk = [refs_tokens]\n        score = sentence_bleu(refs_for_nltk, hyp_tokens, smoothing_function=self.smooth_fn)\n        return BLEUScore(score)\n    \n    def corpus_score(self, hypotheses, references):\n        \"\"\"Score a corpus (list of sentences)\"\"\"\n        from nltk.translate.bleu_score import corpus_bleu\n        \n        # Tokenize hypotheses\n        hyp_tokens = []\n        for h in hypotheses:\n            if isinstance(h, str):\n                hyp_tokens.append(self._tokenize(h))\n            else:\n                hyp_tokens.append(h)\n        \n        # Prepare references\n        refs_for_nltk = []\n        for ref_group in references:\n            group_tokens = []\n            for ref in ref_group:\n                if isinstance(ref, str):\n                    group_tokens.append(self._tokenize(ref))\n                else:\n                    group_tokens.append(ref)\n            refs_for_nltk.append(group_tokens)\n        \n        score = corpus_bleu(refs_for_nltk, hyp_tokens, smoothing_function=self.smooth_fn)\n        return BLEUScore(score)\n\nclass CHRF:\n    \"\"\"Simple CHRF implementation with sacrebleu-like interface\"\"\"\n    def __init__(self, char_order=6, beta=2.0):\n        self.char_order = char_order\n        self.beta = beta\n    \n    def _get_char_ngrams(self, text, n):\n        \"\"\"Get character n-grams from text\"\"\"\n        text = re.sub(r'\\s+', '', text.lower())\n        return [tuple(text[i:i+n]) for i in range(len(text)-n+1)]\n    \n    def sentence_score(self, hypothesis, references):\n        \"\"\"Calculate CHRF score for a single sentence\"\"\"\n        if isinstance(hypothesis, str):\n            hyp = hypothesis\n        else:\n            hyp = ' '.join(hypothesis)\n            \n        best_score = 0\n        \n        for ref in references:\n            if isinstance(ref, str):\n                ref_text = ref\n            else:\n                ref_text = ' '.join(ref)\n            \n            if not hyp or not ref_text:\n                continue\n            \n            hyp_ngrams = Counter(self._get_char_ngrams(hyp, self.char_order))\n            ref_ngrams = Counter(self._get_char_ngrams(ref_text, self.char_order))\n            \n            common = sum((hyp_ngrams & ref_ngrams).values())\n            hyp_total = sum(hyp_ngrams.values())\n            ref_total = sum(ref_ngrams.values())\n            \n            if hyp_total == 0 or ref_total == 0:\n                continue\n                \n            precision = common / hyp_total\n            recall = common / ref_total\n            \n            if precision + recall > 0:\n                f_score = ((1 + self.beta**2) * precision * recall) / \\\n                         (self.beta**2 * precision + recall)\n                best_score = max(best_score, f_score)\n        \n        return CHRFScore(best_score)\n    \n    def corpus_score(self, hypotheses, references):\n        \"\"\"Calculate CHRF for corpus\"\"\"\n        scores = []\n        for hyp, ref_group in zip(hypotheses, references):\n            score = self.sentence_score(hyp, ref_group).score / 100.0  # Convert back from percentage\n            scores.append(score)\n        \n        avg_score = np.mean(scores) if scores else 0.0\n        return CHRFScore(avg_score)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T13:01:00.241282Z","iopub.execute_input":"2025-12-29T13:01:00.241463Z","iopub.status.idle":"2025-12-29T13:01:00.256951Z","shell.execute_reply.started":"2025-12-29T13:01:00.241447Z","shell.execute_reply":"2025-12-29T13:01:00.256309Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Core training and test data\ntrain_df = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/train.csv')\ntest_df = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/test.csv')\n\n# Supplemental data files\ntry:\n    published_texts = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/published_texts.csv')\n    print(f\"✓ published_texts.csv loaded: {len(published_texts)} rows\")\nexcept Exception as e:\n    print(f\"✗ published_texts.csv error: {e}\")\n    published_texts = None\n\ntry:\n    oa_lexicon = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/OA_Lexicon_eBL.csv')\n    print(f\"✓ OA_Lexicon_eBL.csv loaded: {len(oa_lexicon)} rows\")\nexcept Exception as e:\n    print(f\"✗ OA_Lexicon_eBL.csv error: {e}\")\n    oa_lexicon = None\n\ntry:\n    ebl_dictionary = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/eBL_Dictionary.csv')\n    print(f\"✓ eBL_Dictionary.csv loaded: {len(ebl_dictionary)} rows\")\nexcept Exception as e:\n    print(f\"✗ eBL_Dictionary.csv error: {e}\")\n    ebl_dictionary = None\n\ntry:\n    publications = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/publications.csv')\n    print(f\"✓ publications.csv loaded: {len(publications)} rows\")\nexcept Exception as e:\n    print(f\"✗ publications.csv error: {e}\")\n    publications = None\n\ntry:\n    bibliography = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/bibliography.csv')\n    print(f\"✓ bibliography.csv loaded: {len(bibliography)} rows\")\nexcept Exception as e:\n    print(f\"✗ bibliography.csv error: {e}\")\n    bibliography = None\n\ntry:\n    sentences_aid = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/Sentences_Oare_FirstWord_LinNum.csv')\n    print(f\"✓ Sentences_Oare_FirstWord_LinNum.csv loaded: {len(sentences_aid)} rows\")\nexcept Exception as e:\n    print(f\"✗ Sentences_Oare_FirstWord_LinNum.csv error: {e}\")\n    sentences_aid = None\n\ntry:\n    resources = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/resources.csv')\n    print(f\"✓ resources.csv loaded: {len(resources)} rows\")\nexcept Exception as e:\n    print(f\"✗ resources.csv error: {e}\")\n    resources = None\n\ntry:\n    sample_submission = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/sample_submission.csv')\n    print(f\"✓ sample_submission.csv loaded: {len(sample_submission)} rows\")\nexcept Exception as e:\n    print(f\"✗ sample_submission.csv error: {e}\")\n    sample_submission = None\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T13:01:00.292286Z","iopub.execute_input":"2025-12-29T13:01:00.292714Z","iopub.status.idle":"2025-12-29T13:01:06.372352Z","shell.execute_reply.started":"2025-12-29T13:01:00.292695Z","shell.execute_reply":"2025-12-29T13:01:06.371544Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if oa_lexicon is not None:\n    print(f\"OA_Lexicon columns: {oa_lexicon.columns.tolist()}\")\n    print(f\"Sample lexicon entries:\")\n    print(oa_lexicon[['form', 'norm', 'lexeme', 'type']].head(10))\n    \n    # Create word-to-lexeme mapping\n    word_to_lexeme = {}\n    for _, row in oa_lexicon.iterrows():\n        form = str(row['form']).strip() if pd.notna(row['form']) else None\n        lexeme = str(row['lexeme']).strip() if pd.notna(row['lexeme']) else None\n        if form and lexeme:\n            word_to_lexeme[form] = lexeme\n    \n    print(f\"\\nWord-to-lexeme mapping created: {len(word_to_lexeme)} entries\")\n\nif ebl_dictionary is not None:\n    print(f\"\\neBL_Dictionary columns: {ebl_dictionary.columns.tolist()}\")\n    print(f\"Sample dictionary entries:\")\n    print(ebl_dictionary.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T13:01:06.373725Z","iopub.execute_input":"2025-12-29T13:01:06.373980Z","iopub.status.idle":"2025-12-29T13:01:07.940437Z","shell.execute_reply.started":"2025-12-29T13:01:06.373959Z","shell.execute_reply":"2025-12-29T13:01:07.939801Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def extract_translations_from_publications(publications_df, published_texts_df):\n    \"\"\"\n    Extract and align translations from scholarly publications\n    This is the key step to expand training data\n    \"\"\"\n    additional_translations = []\n    \n    if publications_df is None or published_texts_df is None:\n        return additional_translations\n    \n    # Create mapping of publication identifiers\n    pub_mapping = {}\n    for _, row in published_texts_df.iterrows():\n        oare_id = row['oare_id']\n        aliases = str(row['aliases']).split('|') if pd.notna(row['aliases']) else []\n        label = str(row['label']).strip() if pd.notna(row['label']) else ''\n        \n        pub_mapping[str(oare_id)] = {\n            'transliteration': row['transliteration'],\n            'aliases': [a.strip() for a in aliases],\n            'label': label\n        }\n    \n    # Pattern matching for translations in publication text\n    translation_patterns = [\n        r'Translation:?\\s*([\"\\']?)([^.!?\\n]+[.!?])',\n        r'translates?(?:\\s+as)?:?\\s*([\"\\']?)([^.!?\\n]+[.!?])',\n        r'meaning(?:\\s+)?[:\\-]\\s*([\"\\']?)([^.!?\\n]+[.!?])',\n        r'reads?:?\\s*([\"\\']?)([^.!?\\n]+[.!?])',\n    ]\n    \n    for _, pub_row in publications_df.iterrows():\n        pdf_name = pub_row['pdf_name']\n        page_text = str(pub_row['page_text']) if pd.notna(pub_row['page_text']) else ''\n        \n        if not page_text or len(page_text) < 10:\n            continue\n        \n        # Extract potential translations\n        for pattern in translation_patterns:\n            matches = re.finditer(pattern, page_text, re.IGNORECASE)\n            for match in matches:\n                translation = match.group(2).strip()\n                \n                # Clean translation\n                translation = re.sub(r'^\\s*[\"\\']|[\"\\']\\s*$', '', translation)\n                translation = re.sub(r'\\s+', ' ', translation)\n                \n                if len(translation.split()) >= 2:  # At least 2 words\n                    additional_translations.append({\n                        'source': 'publications',\n                        'pdf_name': pdf_name,\n                        'translation': translation,\n                        'page_text_snippet': page_text[:200]\n                    })\n    \n    return additional_translations","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T13:01:07.941341Z","iopub.execute_input":"2025-12-29T13:01:07.941531Z","iopub.status.idle":"2025-12-29T13:01:07.949691Z","shell.execute_reply.started":"2025-12-29T13:01:07.941514Z","shell.execute_reply":"2025-12-29T13:01:07.949154Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if publications is not None and published_texts is not None:\n    additional_trans = extract_translations_from_publications(publications, published_texts)\n    print(f\"Extracted {len(additional_trans)} potential translations from publications\")\n    \n    if additional_trans:\n        print(\"\\nSample extracted translations:\")\n        for i, trans in enumerate(additional_trans[:3]):\n            print(f\"{i+1}. {trans['translation'][:100]}\")\nelse:\n    additional_trans = []\n    print(\"Could not extract translations - missing publications or published_texts data\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T13:01:07.951376Z","iopub.execute_input":"2025-12-29T13:01:07.951623Z","iopub.status.idle":"2025-12-29T13:01:40.030330Z","shell.execute_reply.started":"2025-12-29T13:01:07.951606Z","shell.execute_reply":"2025-12-29T13:01:40.029590Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def align_sentences_with_aid_file(train_df, sentences_aid_df):\n    \"\"\"\n    Use Sentences_Oare_FirstWord_LinNum.csv to create better alignments\n    This file contains sentence-level information with translations\n    \"\"\"\n    if sentences_aid_df is None:\n        return train_df\n    \n    print(f\"Sentences aid columns: {sentences_aid_df.columns.tolist()}\")\n    print(f\"Sample aid entries:\")\n    print(sentences_aid_df.head(10))\n    \n    # This file helps identify sentence boundaries and has translations\n    # Extract useful information from the aid file\n    print(f\"\\nSentences aid file analysis:\")\n    print(f\"  Total sentences: {len(sentences_aid_df)}\")\n    print(f\"  Unique texts (text_uuid): {sentences_aid_df['text_uuid'].nunique()}\")\n    print(f\"  Unique sentences: {sentences_aid_df['sentence_uuid'].nunique()}\")\n    \n    # Check for translations in the aid file\n    translations_in_aid = sentences_aid_df[sentences_aid_df['translation'].notna()]\n    print(f\"  Sentences with translations: {len(translations_in_aid)}\")\n    \n    if len(translations_in_aid) > 0:\n        print(f\"\\n  Sample translations from aid file:\")\n        for idx, row in translations_in_aid.head(3).iterrows():\n            print(f\"    '{row['first_word_transcription']}' -> '{row['translation'][:80]}'\")\n    \n    # Create additional training data from the aid file if translations exist\n    additional_data = []\n    if len(translations_in_aid) > 0:\n        for idx, row in translations_in_aid.iterrows():\n            # Create synthetic transliteration from first word (simplified)\n            # In real scenario, would need to reconstruct full transliteration\n            if pd.notna(row['first_word_transcription']) and pd.notna(row['translation']):\n                additional_data.append({\n                    'oare_id': f\"aid_{row['sentence_uuid']}\",\n                    'transliteration': row['first_word_transcription'],  # Simplified\n                    'translation': row['translation']\n                })\n        \n        if additional_data:\n            print(f\"\\n  Created {len(additional_data)} potential additional training samples\")\n    \n    return train_df\n\nif sentences_aid is not None:\n    train_df_aligned = align_sentences_with_aid_file(train_df, sentences_aid)\n    print(f\"Training data after alignment: {len(train_df_aligned)} rows\")\nelse:\n    train_df_aligned = train_df\n    print(\"No sentence alignment aid file available\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T13:01:40.034218Z","iopub.execute_input":"2025-12-29T13:01:40.034493Z","iopub.status.idle":"2025-12-29T13:01:40.462008Z","shell.execute_reply.started":"2025-12-29T13:01:40.034473Z","shell.execute_reply":"2025-12-29T13:01:40.461437Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Use original train_df as base\nexpanded_train = train_df.copy()\n\nprint(f\"Original training data: {len(expanded_train)} samples\")\n\n# Add any extracted translations that match transliterations\nif additional_trans and published_texts is not None:\n    matched_translations = 0\n    for trans_dict in additional_trans[:100]:  # Limit to 100 to avoid noise\n        new_row = {\n            'oare_id': f\"extracted_{matched_translations}\",\n            'transliteration': '',  # Would need better matching logic\n            'translation': trans_dict['translation']\n        }\n        matched_translations += 1\n    \n    print(f\"Potential matches from publications: {matched_translations}\")\n\nprint(f\"Final training data size: {len(expanded_train)} samples\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T13:01:40.463225Z","iopub.execute_input":"2025-12-29T13:01:40.463489Z","iopub.status.idle":"2025-12-29T13:01:40.470124Z","shell.execute_reply.started":"2025-12-29T13:01:40.463467Z","shell.execute_reply":"2025-12-29T13:01:40.469411Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Training data shape: {train_df.shape}\")\nprint(f\"Test data shape: {test_df.shape}\")\nprint(f\"Columns: {train_df.columns.tolist()}\")\n\nprint(f\"\\nFirst few training examples:\")\nprint(train_df.head(3))\n\n# Text statistics\ntrain_df['transliteration_len'] = train_df['transliteration'].str.split().apply(len)\ntrain_df['translation_len'] = train_df['translation'].str.split().apply(len)\n\nprint(f\"\\nTransliteration length statistics:\")\nprint(train_df['transliteration_len'].describe())\nprint(f\"\\nTranslation length statistics:\")\nprint(train_df['translation_len'].describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T13:01:40.470878Z","iopub.execute_input":"2025-12-29T13:01:40.471294Z","iopub.status.idle":"2025-12-29T13:01:40.526763Z","shell.execute_reply.started":"2025-12-29T13:01:40.471261Z","shell.execute_reply":"2025-12-29T13:01:40.525972Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(14, 4))\n\naxes[0].hist(train_df['transliteration_len'], bins=50, edgecolor='black', alpha=0.7)\naxes[0].set_xlabel('Number of Words')\naxes[0].set_ylabel('Frequency')\naxes[0].set_title('Akkadian Transliteration Length Distribution')\naxes[0].axvline(train_df['transliteration_len'].mean(), color='red', linestyle='--', label='Mean')\naxes[0].legend()\n\naxes[1].hist(train_df['translation_len'], bins=50, edgecolor='black', alpha=0.7)\naxes[1].set_xlabel('Number of Words')\naxes[1].set_ylabel('Frequency')\naxes[1].set_title('English Translation Length Distribution')\naxes[1].axvline(train_df['translation_len'].mean(), color='red', linestyle='--', label='Mean')\naxes[1].legend()\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T13:01:40.527777Z","iopub.execute_input":"2025-12-29T13:01:40.528057Z","iopub.status.idle":"2025-12-29T13:01:40.912048Z","shell.execute_reply.started":"2025-12-29T13:01:40.528014Z","shell.execute_reply":"2025-12-29T13:01:40.911383Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def clean_text(text):\n    \"\"\"Clean and normalize text\"\"\"\n    if isinstance(text, float):\n        return \"\"\n    text = text.strip()\n    text = re.sub(r'\\s+', ' ', text)\n    return text\n\ntrain_df['transliteration'] = train_df['transliteration'].apply(clean_text)\ntrain_df['translation'] = train_df['translation'].apply(clean_text)\n\n# Remove empty rows\ninitial_len = len(train_df)\ntrain_df = train_df[(train_df['transliteration'].str.len() > 0) & \n                    (train_df['translation'].str.len() > 0)]\n\nprint(f\"Removed {initial_len - len(train_df)} empty rows\")\nprint(f\"Training data size: {len(train_df)}\")\n\n# Vocabulary analysis\nakkadian_tokens = Counter()\nenglish_tokens = Counter()\n\nfor _, row in train_df.iterrows():\n    akkadian_tokens.update(row['transliteration'].split())\n    english_tokens.update(row['translation'].split())\n\nprint(f\"\\nVocabulary Statistics:\")\nprint(f\"Unique Akkadian tokens: {len(akkadian_tokens)}\")\nprint(f\"Unique English tokens: {len(english_tokens)}\")\nprint(f\"Total Akkadian tokens: {sum(akkadian_tokens.values())}\")\nprint(f\"Total English tokens: {sum(english_tokens.values())}\")\n\n# Most common tokens\nprint(f\"\\nMost common Akkadian tokens:\")\nfor token, count in akkadian_tokens.most_common(10):\n    print(f\"  {token}: {count}\")\n\nprint(f\"\\nMost common English tokens:\")\nfor token, count in english_tokens.most_common(10):\n    print(f\"  {token}: {count}\")\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T13:01:40.912945Z","iopub.execute_input":"2025-12-29T13:01:40.913201Z","iopub.status.idle":"2025-12-29T13:01:41.139003Z","shell.execute_reply.started":"2025-12-29T13:01:40.913180Z","shell.execute_reply":"2025-12-29T13:01:41.138480Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class OldAssyrianDataset(Dataset):\n    \"\"\"Custom Dataset for Old Assyrian Translation\"\"\"\n    \n    def __init__(self, dataframe, tokenizer_src, tokenizer_tgt, max_length=128):\n        self.dataframe = dataframe.reset_index(drop=True)\n        self.tokenizer_src = tokenizer_src\n        self.tokenizer_tgt = tokenizer_tgt\n        self.max_length = max_length\n    \n    def __len__(self):\n        return len(self.dataframe)\n    \n    def __getitem__(self, idx):\n        row = self.dataframe.iloc[idx]\n        \n        source_text = row['transliteration']\n        target_text = row['translation']\n        \n        # Encode source\n        source = self.tokenizer_src(\n            source_text,\n            max_length=self.max_length,\n            padding='max_length',\n            truncation=True,\n            return_tensors='pt'\n        )\n        \n        # Encode target\n        target = self.tokenizer_tgt(\n            target_text,\n            max_length=self.max_length,\n            padding='max_length',\n            truncation=True,\n            return_tensors='pt'\n        )\n        \n        return {\n            'input_ids': source['input_ids'].squeeze(),\n            'attention_mask': source['attention_mask'].squeeze(),\n            'labels': target['input_ids'].squeeze(),\n            'decoder_attention_mask': target['attention_mask'].squeeze(),\n        }","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T13:01:41.141020Z","iopub.execute_input":"2025-12-29T13:01:41.141282Z","iopub.status.idle":"2025-12-29T13:01:41.147232Z","shell.execute_reply.started":"2025-12-29T13:01:41.141265Z","shell.execute_reply":"2025-12-29T13:01:41.146656Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n\n# The path will be something like:\nmodel_path = \"/kaggle/input/opus-mt-mul-en/pytorch/default/1/opus-mt-mul-en\"\n\n# Load\ntokenizer = AutoTokenizer.from_pretrained(model_path, local_files_only=True)\nmodel = AutoModelForSeq2SeqLM.from_pretrained(model_path, local_files_only=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T14:50:38.518969Z","iopub.execute_input":"2025-12-29T14:50:38.519854Z","iopub.status.idle":"2025-12-29T14:50:39.715611Z","shell.execute_reply.started":"2025-12-29T14:50:38.519817Z","shell.execute_reply":"2025-12-29T14:50:39.714695Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# Split data for training\ntrain_df_split, val_df = train_test_split(\n    train_df,\n    test_size=0.1,\n    random_state=42\n)\n\nprint(f\"Training samples: {len(train_df_split)}\")\nprint(f\"Validation samples: {len(val_df)}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T13:01:42.361415Z","iopub.execute_input":"2025-12-29T13:01:42.361710Z","iopub.status.idle":"2025-12-29T13:01:42.368882Z","shell.execute_reply.started":"2025-12-29T13:01:42.361680Z","shell.execute_reply":"2025-12-29T13:01:42.368308Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dataset = OldAssyrianDataset(\n    train_df_split,\n    tokenizer,\n    tokenizer,\n    max_length=128\n)\n\nval_dataset = OldAssyrianDataset(\n    val_df,\n    tokenizer,\n    tokenizer,\n    max_length=128\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T13:01:42.369772Z","iopub.execute_input":"2025-12-29T13:01:42.370119Z","iopub.status.idle":"2025-12-29T13:01:42.382739Z","shell.execute_reply.started":"2025-12-29T13:01:42.370092Z","shell.execute_reply":"2025-12-29T13:01:42.382076Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"training_args = Seq2SeqTrainingArguments(\n    output_dir='/kaggle/working/model_output',\n    num_train_epochs=3,\n    per_device_train_batch_size=16,\n    per_device_eval_batch_size=16,\n    logging_steps=50,\n    save_steps=100,\n    eval_strategy=\"steps\",\n    eval_steps=100,\n    learning_rate=5e-5,\n    weight_decay=0.01,\n    save_total_limit=2,\n    load_best_model_at_end=True,\n    gradient_accumulation_steps=2,\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T13:01:42.383342Z","iopub.execute_input":"2025-12-29T13:01:42.383548Z","iopub.status.idle":"2025-12-29T13:01:42.424907Z","shell.execute_reply.started":"2025-12-29T13:01:42.383531Z","shell.execute_reply":"2025-12-29T13:01:42.424359Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = model.to(device)\nprint(f\"✓ Model moved to {device}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T13:01:42.425798Z","iopub.execute_input":"2025-12-29T13:01:42.426017Z","iopub.status.idle":"2025-12-29T13:01:42.432890Z","shell.execute_reply.started":"2025-12-29T13:01:42.425999Z","shell.execute_reply":"2025-12-29T13:01:42.432332Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"total_params = sum(p.numel() for p in model.parameters())\ntrainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\nprint(f\"Total parameters: {total_params:,}\")\nprint(f\"Trainable parameters: {trainable_params:,}\")\n\nif torch.cuda.is_available():\n    print(f\"Model memory usage: {total_params * 4 / 1e9:.2f} GB (approximate)\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T13:01:42.433577Z","iopub.execute_input":"2025-12-29T13:01:42.433787Z","iopub.status.idle":"2025-12-29T13:01:42.445855Z","shell.execute_reply.started":"2025-12-29T13:01:42.433770Z","shell.execute_reply":"2025-12-29T13:01:42.445244Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"trainer = Seq2SeqTrainer(\n    model=model,\n    args=training_args,\n    train_dataset=train_dataset,\n    eval_dataset=val_dataset,\n    tokenizer=tokenizer,\n)\n\nprint(\"\\n=== STARTING TRAINING ===\\n\")\nprint(f\"Using device: {device}\")\nprint(f\"Batch size: {training_args.per_device_train_batch_size}\")\nprint(f\"Mixed precision: {'BF16' if training_args.bf16 else 'FP16' if training_args.fp16 else 'FP32'}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T13:01:42.446744Z","iopub.execute_input":"2025-12-29T13:01:42.446965Z","iopub.status.idle":"2025-12-29T13:01:42.500603Z","shell.execute_reply.started":"2025-12-29T13:01:42.446938Z","shell.execute_reply":"2025-12-29T13:01:42.500070Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if torch.cuda.is_available():\n    torch.cuda.empty_cache()\n\nprint(\"✓ Training completed!\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T13:01:42.501411Z","iopub.execute_input":"2025-12-29T13:01:42.501626Z","iopub.status.idle":"2025-12-29T13:01:42.517554Z","shell.execute_reply.started":"2025-12-29T13:01:42.501608Z","shell.execute_reply":"2025-12-29T13:01:42.517052Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def compute_bleu_chrf(predictions, references):\n    \"\"\"\n    Compute BLEU and chrF++ scores with geometric mean\n    \"\"\"\n    \n    predictions = [str(p).strip() if p else \"\" for p in predictions]\n    references = [str(r).strip() if r else \"\" for r in references]\n    \n    # Correct format: [[\"ref1\"], [\"ref2\"], [\"ref3\"], ...]\n    references_list = [[ref] for ref in references]\n    \n    bleu = BLEU()\n    bleu_score = bleu.corpus_score(predictions, references_list)\n    \n    chrf = CHRF()\n    chrf_score = chrf.corpus_score(predictions, references_list)\n    \n    bleu_decimal = bleu_score.score / 100.0\n    chrf_decimal = chrf_score.score / 100.0\n    geometric_mean = np.sqrt(bleu_decimal * chrf_decimal) * 100.0\n    \n    return {\n        'BLEU': bleu_score.score,\n        'chrF++': chrf_score.score,\n        'Geometric_Mean': geometric_mean,\n    }","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T13:01:42.518259Z","iopub.execute_input":"2025-12-29T13:01:42.518475Z","iopub.status.idle":"2025-12-29T13:01:42.523537Z","shell.execute_reply.started":"2025-12-29T13:01:42.518457Z","shell.execute_reply":"2025-12-29T13:01:42.522989Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"val_predictions = []\nval_references = []\n\nmodel.eval()\nwith torch.no_grad():\n    for idx in range(min(200, len(val_df))):\n        if idx % 50 == 0:\n            print(f\"  Evaluated {idx}/{min(200, len(val_df))}\")\n        \n        inputs = tokenizer(\n            val_df.iloc[idx]['transliteration'],\n            return_tensors='pt',\n            max_length=128,\n            truncation=True\n        ).to(device)  # Move inputs to GPU\n        \n        outputs = model.generate(\n            inputs['input_ids'],\n            max_length=128,\n            num_beams=4,\n            early_stopping=True,\n        )\n        \n        prediction = tokenizer.decode(outputs[0], skip_special_tokens=True)\n        reference = val_df.iloc[idx]['translation']\n        \n        val_predictions.append(prediction)\n        val_references.append(reference)\n\neval_metrics = compute_bleu_chrf(val_predictions, val_references)\n\nprint(f\"\\nValidation Metrics:\")\nprint(f\"  BLEU Score: {eval_metrics['BLEU']:.4f}\")\nprint(f\"  chrF++ Score: {eval_metrics['chrF++']:.4f}\")\nprint(f\"  Geometric Mean: {eval_metrics['Geometric_Mean']:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T13:01:42.524426Z","iopub.execute_input":"2025-12-29T13:01:42.524678Z","iopub.status.idle":"2025-12-29T13:02:54.316663Z","shell.execute_reply.started":"2025-12-29T13:01:42.524660Z","shell.execute_reply":"2025-12-29T13:02:54.316089Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_predictions = []\n\nmodel.eval()\nwith torch.no_grad():\n    for idx in range(len(test_df)):\n        if idx % 500 == 0:\n            print(f\"Progress: {idx}/{len(test_df)}\")\n        \n        transliteration = test_df.iloc[idx]['transliteration']\n        \n        inputs = tokenizer(\n            transliteration,\n            return_tensors='pt',\n            max_length=128,\n            truncation=True\n        ).to(device)  # Move inputs to GPU\n        \n        outputs = model.generate(\n            inputs['input_ids'],\n            max_length=128,\n            num_beams=4,\n            early_stopping=True,\n            length_penalty=0.6,\n        )\n        \n        translation = tokenizer.decode(outputs[0], skip_special_tokens=True)\n        test_predictions.append(translation)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T13:02:54.317527Z","iopub.execute_input":"2025-12-29T13:02:54.317785Z","iopub.status.idle":"2025-12-29T13:02:55.163531Z","shell.execute_reply.started":"2025-12-29T13:02:54.317764Z","shell.execute_reply":"2025-12-29T13:02:55.162984Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if torch.cuda.is_available():\n    torch.cuda.empty_cache()\n\nprint(f\"✓ Generated {len(test_predictions)} predictions\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T13:02:55.164470Z","iopub.execute_input":"2025-12-29T13:02:55.164709Z","iopub.status.idle":"2025-12-29T13:02:55.170234Z","shell.execute_reply.started":"2025-12-29T13:02:55.164682Z","shell.execute_reply":"2025-12-29T13:02:55.169614Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission_df = pd.DataFrame({\n    'id': test_df['id'],\n    'translation': test_predictions\n})\n\nprint(\"Submission preview:\")\nprint(submission_df.head(10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T13:02:55.170975Z","iopub.execute_input":"2025-12-29T13:02:55.171216Z","iopub.status.idle":"2025-12-29T13:02:55.186503Z","shell.execute_reply.started":"2025-12-29T13:02:55.171198Z","shell.execute_reply":"2025-12-29T13:02:55.185923Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission_df.to_csv('/kaggle/working/submission.csv', index=False)\nprint(f\"\\n✓ Submission saved to /kaggle/working/submission.csv\")\nprint(f\"Total predictions: {len(submission_df)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T13:02:55.187234Z","iopub.execute_input":"2025-12-29T13:02:55.187511Z","iopub.status.idle":"2025-12-29T13:02:55.201856Z","shell.execute_reply.started":"2025-12-29T13:02:55.187492Z","shell.execute_reply":"2025-12-29T13:02:55.201320Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission_df['pred_length'] = submission_df['translation'].str.split().apply(len)\n\nprint(\"Prediction Statistics:\")\nprint(submission_df['pred_length'].describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T13:02:55.202705Z","iopub.execute_input":"2025-12-29T13:02:55.202940Z","iopub.status.idle":"2025-12-29T13:02:55.217723Z","shell.execute_reply.started":"2025-12-29T13:02:55.202923Z","shell.execute_reply":"2025-12-29T13:02:55.217069Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualize prediction lengths\nfig, ax = plt.subplots(figsize=(10, 5))\nax.hist(submission_df['pred_length'], bins=50, edgecolor='black', alpha=0.7)\nax.set_xlabel('Number of Words')\nax.set_ylabel('Frequency')\nax.set_title('Distribution of Test Predictions')\nax.axvline(submission_df['pred_length'].mean(), color='red', linestyle='--', label='Mean')\nax.legend()\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T13:02:55.218629Z","iopub.execute_input":"2025-12-29T13:02:55.218981Z","iopub.status.idle":"2025-12-29T13:02:55.434428Z","shell.execute_reply.started":"2025-12-29T13:02:55.218953Z","shell.execute_reply":"2025-12-29T13:02:55.433801Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\nSample Predictions:\")\nsample_indices = np.random.choice(len(test_df), min(5, len(test_df)), replace=False)\n\nfor idx in sample_indices:\n    print(f\"\\nID: {test_df.iloc[idx]['id']}\")\n    print(f\"Akkadian:   {test_df.iloc[idx]['transliteration'][:100]}\")\n    print(f\"Prediction: {submission_df.iloc[idx]['translation'][:100]}\")\n\n# Save analysis\nanalysis_report = {\n    'training_samples': len(train_df),\n    'test_samples': len(test_df),\n    'validation_bleu': float(eval_metrics['BLEU']),\n    'validation_chrf': float(eval_metrics['chrF++']),\n    'validation_geometric_mean': float(eval_metrics['Geometric_Mean']),\n    'mean_prediction_length': float(submission_df['pred_length'].mean()),\n    'supplemental_files_used': {\n        'OA_Lexicon': oa_lexicon is not None,\n        'eBL_Dictionary': ebl_dictionary is not None,\n        'Publications': publications is not None,\n        'Published_Texts': published_texts is not None,\n        'Sentences_Aid': sentences_aid is not None,\n        'Bibliography': bibliography is not None,\n        'Resources': resources is not None,\n    }\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-29T13:02:55.435251Z","iopub.execute_input":"2025-12-29T13:02:55.435500Z","iopub.status.idle":"2025-12-29T13:02:55.442790Z","shell.execute_reply.started":"2025-12-29T13:02:55.435481Z","shell.execute_reply":"2025-12-29T13:02:55.442028Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}