{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"datasetVersion","sourceId":16231411,"datasetId":10407544,"databundleVersionId":17213245}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from kaggle_secrets import UserSecretsClient\nfrom huggingface_hub import login\n\nuser_secrets = UserSecretsClient()\nhf_token = user_secrets.get_secret(\"hf_read\")\nlogin(token=hf_token)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-12T21:03:01.140481Z","iopub.execute_input":"2026-05-12T21:03:01.141413Z","iopub.status.idle":"2026-05-12T21:03:02.150101Z","shell.execute_reply.started":"2026-05-12T21:03:01.141380Z","shell.execute_reply":"2026-05-12T21:03:02.149509Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from huggingface_hub import HfApi\n\n# Initialize the API\napi = HfApi()\n\n# Get account details\ntry:\n    user_info = api.whoami()\n    print(f\"✅ Successfully logged in as: {user_info['name']}\")\n    print(f\"Token type: {user_info['auth']['type']}\")\nexcept Exception as e:\n    print(\"❌ Not logged in. Please check your token and secrets.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-12T21:03:06.709437Z","iopub.execute_input":"2026-05-12T21:03:06.710255Z","iopub.status.idle":"2026-05-12T21:03:06.759593Z","shell.execute_reply.started":"2026-05-12T21:03:06.710179Z","shell.execute_reply":"2026-05-12T21:03:06.759030Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.model_selection import train_test_split\nimport os\nos.environ[\"PYTORCH_ALLOC_CONF\"] = \"expandable_segments:True\"\n\ndef split_kaggle_dataset(input_dir, filename, train_ratio=0.9, val_ratio=0.1, test_ratio=0.05, seed=42):\n    \"\"\"\n    Reads data from Kaggle input and saves 90/5/5 splits to Kaggle working directory.\n    \"\"\"\n    # 1. Define Paths\n    input_path = os.path.join(input_dir, filename)\n    output_dir = \"/kaggle/working/\"\n    \n    # 2. Load the Oracc Corpus\n    print(f\"Loading data from: {input_path}...\")\n    df = pd.read_csv(input_path) if filename.endswith('.csv') else pd.read_json(input_path)\n\n    # 3. Create the 90/5/5 Split\n    # Split 1: Separate Train from the Rest (Val + Test)\n    rest_ratio = val_ratio + test_ratio\n    train_df, rest_df = train_test_split(\n        df, test_size=rest_ratio, random_state=seed, shuffle=True\n    )\n\n    # Split 2: Divide the Rest into Validation and Test\n    relative_test_ratio = test_ratio / rest_ratio\n    val_df, test_df = train_test_split(\n        rest_df, test_size=relative_test_ratio, random_state=seed, shuffle=True\n    )\n\n    # 4. Save to /kaggle/working/\n    # This allows you to download or use them in subsequent notebook steps\n    base = os.path.splitext(filename)[0]\n    train_df.to_csv(f\"{output_dir}{base}_train.csv\", index=False)\n    val_df.to_csv(f\"{output_dir}{base}_val.csv\", index=False)\n    test_df.to_csv(f\"{output_dir}{base}_test.csv\", index=False)\n\n    print(f\"✅ Splits saved to {output_dir}\")\n    print(f\"📊 Counts - Train: {len(train_df)} | Val: {len(val_df)} | Test: {len(test_df)}\")\n\n# --- Configuration ---\nDATA_PATH = \"/kaggle/input/datasets/anthonytoffetti/clean-oracc-cdli-byt5\"\nFILE_NAME = \"master_akkadian_pretrain.csv\" \n\nsplit_kaggle_dataset(DATA_PATH, FILE_NAME)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-12T21:03:10.745526Z","iopub.execute_input":"2026-05-12T21:03:10.745785Z","iopub.status.idle":"2026-05-12T21:03:12.045058Z","shell.execute_reply.started":"2026-05-12T21:03:10.745763Z","shell.execute_reply":"2026-05-12T21:03:12.044392Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/working/master_akkadian_pretrain_train.csv')\neval_df = pd.read_csv('/kaggle/working/master_akkadian_pretrain_val.csv')\ntrain_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-12T21:03:14.841800Z","iopub.execute_input":"2026-05-12T21:03:14.842397Z","iopub.status.idle":"2026-05-12T21:03:14.912819Z","shell.execute_reply.started":"2026-05-12T21:03:14.842367Z","shell.execute_reply":"2026-05-12T21:03:14.912019Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset\nfrom transformers import AutoTokenizer\nfrom datasets import Dataset\n\ntokenizer = AutoTokenizer.from_pretrained(\"google/byt5-base\")\n\nhf_dataset_train = Dataset.from_pandas(train_df)\nhf_dataset_eval = Dataset.from_pandas(eval_df)\n\n\ndef tokenize_function(examples):\n    # 1. Tokenize the input text\n    model_inputs = tokenizer(\n        examples[\"akkadian\"], \n        max_length=1024,\n    )\n    # model_inputs[\"labels\"] = model_inputs['input_ids'].copy()\n    return model_inputs\n\n\ntokenized_train_data = hf_dataset_train.map(tokenize_function, batched=True,)\ntokenized_eval_data = hf_dataset_eval.map(tokenize_function, batched=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-12T21:03:19.468955Z","iopub.execute_input":"2026-05-12T21:03:19.469629Z","iopub.status.idle":"2026-05-12T21:04:43.210146Z","shell.execute_reply.started":"2026-05-12T21:03:19.469599Z","shell.execute_reply":"2026-05-12T21:04:43.209534Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(tokenized_train_data)\nprint(tokenized_eval_data)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-12T21:04:46.007119Z","iopub.execute_input":"2026-05-12T21:04:46.008001Z","iopub.status.idle":"2026-05-12T21:04:46.011808Z","shell.execute_reply.started":"2026-05-12T21:04:46.007970Z","shell.execute_reply":"2026-05-12T21:04:46.011140Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i in range(3):\n    sample = tokenized_train_data[i]\n    print(f\"--- Sample {i} ---\")\n    print(f\"Input IDs: {sample['input_ids'][:20]}...\") # Showing first 20 bytes\n\n    # Use the tokenizer inside the dataset to decode back to text\n    decoded_text = tokenizer.decode(sample['input_ids'], skip_special_tokens=True)\n    print(f\"Decoded Text: {decoded_text}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-12T21:04:50.230931Z","iopub.execute_input":"2026-05-12T21:04:50.231722Z","iopub.status.idle":"2026-05-12T21:04:50.610079Z","shell.execute_reply.started":"2026-05-12T21:04:50.231691Z","shell.execute_reply":"2026-05-12T21:04:50.609316Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from itertools import chain\n\n# Maximum byte sequence length for ByT5 pre-training\nmodel_max_length = 1024\n\ndef group_texts(examples):\n    # 1. Concatenate all rows for both tokenized keys\n    concatenated_examples = {\n        k: list(chain(*examples[k])) for k in [\"input_ids\", \"attention_mask\"]\n    }\n    \n    # 2. Compute total length of concatenated byte IDs\n    total_length = len(concatenated_examples[\"input_ids\"])\n    \n    # 3. Drop the remainder so everything fits into exact chunks of 1024\n    if total_length >= model_max_length:\n        total_length = (total_length // model_max_length) * model_max_length\n        \n    # 4. Split sequences into chunks of size 1024\n    result = {\n        k: [t[i : i + model_max_length] for i in range(0, total_length, model_max_length)]\n        for k, t in concatenated_examples.items()\n    }\n    \n    return result\n\n# Apply the function to chunk your dataset\n# We explicitly remove the old column names since they don't align with the chunked sequences\nchunked_train_dataset = tokenized_train_data.map(\n    group_texts,\n    batched=True,\n    remove_columns=tokenized_train_data.column_names,\n    num_proc=4\n)\n\nchunked_eval_dataset = tokenized_eval_data.map(\n    group_texts,\n    batched=True,\n    remove_columns=tokenized_eval_data.column_names,\n    num_proc=4\n)\n\n\nprint(f\"Total chunked samples for pre-training: {len(chunked_train_dataset)}\")\n\nprint(chunked_train_dataset.shape)\nprint(chunked_eval_dataset.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-12T21:04:54.534241Z","iopub.execute_input":"2026-05-12T21:04:54.534803Z","iopub.status.idle":"2026-05-12T21:04:55.865163Z","shell.execute_reply.started":"2026-05-12T21:04:54.534773Z","shell.execute_reply":"2026-05-12T21:04:55.864256Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(chunked_train_dataset)\nfor i in range(5):\n    print(len(chunked_train_dataset[i][\"input_ids\"]))\n\nfor i in range(5):\n    print(len(chunked_eval_dataset[i][\"input_ids\"]))\n\nprint(len(chunked_eval_dataset[-1][\"input_ids\"]))\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-12T21:05:00.628468Z","iopub.execute_input":"2026-05-12T21:05:00.628947Z","iopub.status.idle":"2026-05-12T21:05:00.645389Z","shell.execute_reply.started":"2026-05-12T21:05:00.628912Z","shell.execute_reply":"2026-05-12T21:05:00.644584Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(chunked_train_dataset[\"input_ids\"][0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-12T21:05:04.163230Z","iopub.execute_input":"2026-05-12T21:05:04.164107Z","iopub.status.idle":"2026-05-12T21:05:04.170818Z","shell.execute_reply.started":"2026-05-12T21:05:04.164073Z","shell.execute_reply":"2026-05-12T21:05:04.169994Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nfrom dataclasses import dataclass\nfrom transformers import PreTrainedTokenizerBase\nfrom typing import List, Dict, Any\n\n@dataclass\nclass DataCollatorForByT5SpanCorruption:\n    tokenizer: PreTrainedTokenizerBase\n    noise_density: float = 0.15\n    mean_noise_span_length: float = 20.0  # ByT5 uses 20 vs T5's 3\n\n    def __call__(self, features: List[Dict[str, Any]]) -> Dict[str, Any]:\n        input_ids_list = [torch.tensor(f[\"input_ids\"]) for f in features]\n\n        inputs, labels = self._apply_span_corruption(input_ids_list)\n\n        return {\n            \"input_ids\": inputs,\n            \"labels\": labels,\n            \"attention_mask\": (inputs != self.tokenizer.pad_token_id).long(),\n        }\n\n    def _apply_span_corruption(self, input_ids_list):\n        inputs = []\n        labels = []\n\n        for ids in input_ids_list:\n            seq_len = len(ids) # This will correctly handle 1024 or 688\n        \n            # Recalculate noise based on the actual length of THIS sequence\n            num_noise_tokens = int(np.round(seq_len * self.noise_density))\n            num_spans = max(int(np.round(num_noise_tokens / self.mean_noise_span_length)), 1)\n        \n           \n            # Create a mask for this sequence\n            mask = np.zeros(seq_len, dtype=bool)\n            # Simple heuristic: pick span starts and lengths\n            span_starts = np.random.choice(range(seq_len - 5), num_spans, replace=False)\n            for start in span_starts:\n                length = r = np.random.geometric(1/self.mean_noise_span_length)\n                mask[start : min(start + length, seq_len)] = True\n            \n            # Construct Input (replace masked spans with <extra_id_k>)\n            # Construct Labels (<extra_id_k> + masked_content + <extra_id_k+1>...)\n            curr_input = []\n            curr_label = []\n            sentinel_idx = 0\n            \n            # This logic simplifies the span grouping\n            # In ByT5, sentinel tokens start from ID 258 downwards\n            is_noise = False\n            for j in range(seq_len):\n                if mask[j]:\n                    if not is_noise: # Start of a new noise span\n                        sentinel_id = self.tokenizer.convert_tokens_to_ids(f\"<extra_id_{sentinel_idx}>\")\n                        curr_input.append(sentinel_id)\n                        curr_label.append(sentinel_id)\n                        sentinel_idx += 1\n                        is_noise = True\n                    curr_label.append(ids[j].item())\n                else:\n                    curr_input.append(ids[j].item())\n                    is_noise = False\n            \n            # Append final sentinel to label\n            final_sentinel = self.tokenizer.convert_tokens_to_ids(f\"<extra_id_{sentinel_idx}>\")\n            curr_label.append(final_sentinel)\n            \n            inputs.append(torch.tensor(curr_input))\n            labels.append(torch.tensor(curr_label))\n\n        # Pad the newly formed input/label sequences back to standard shape\n        # For T5, labels use -100 to ignore padding in loss calculation\n        return (\n            torch.nn.utils.rnn.pad_sequence(inputs, batch_first=True, padding_value=self.tokenizer.pad_token_id),\n            torch.nn.utils.rnn.pad_sequence(labels, batch_first=True, padding_value=-100)\n        )\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-12T21:10:01.576359Z","iopub.execute_input":"2026-05-12T21:10:01.576815Z","iopub.status.idle":"2026-05-12T21:10:01.588465Z","shell.execute_reply.started":"2026-05-12T21:10:01.576780Z","shell.execute_reply":"2026-05-12T21:10:01.587726Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 1. Grab specific rows from your dataset using correct indexing [0] and [1]\nsample_batch = [chunked_train_dataset[0], chunked_train_dataset[1]]\n\ndata_collator = DataCollatorForByT5SpanCorruption(\n    tokenizer=tokenizer\n)\n# 2. Run the collator on the specific samples\ncollated = data_collator(sample_batch)\n\n# 3. Print shapes to verify they match exactly\nprint(\"Input IDs shape:\", collated[\"input_ids\"].shape)\nprint(\"Attention Mask shape:\", collated[\"attention_mask\"].shape)\nprint(\"Labels shape:\", collated[\"labels\"].shape)\n\n# 4. Check for any -100 values in incorrect places\nprint(\"Any -100 in input_ids?\", (collated[\"input_ids\"] == -100).any().item())\nprint(\"Any -100 in attention_mask?\", (collated[\"attention_mask\"] == -100).any().item())\nprint(\"Any -100 in labels?\", (collated[\"labels\"] == -100).any().item())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-12T21:10:08.556720Z","iopub.execute_input":"2026-05-12T21:10:08.557416Z","iopub.status.idle":"2026-05-12T21:10:08.626059Z","shell.execute_reply.started":"2026-05-12T21:10:08.557385Z","shell.execute_reply":"2026-05-12T21:10:08.625487Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(collated[\"input_ids\"][0])\nprint(collated[\"attention_mask\"][0])\nprint(collated[\"labels\"][0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-12T21:10:17.821596Z","iopub.execute_input":"2026-05-12T21:10:17.822390Z","iopub.status.idle":"2026-05-12T21:10:17.845876Z","shell.execute_reply.started":"2026-05-12T21:10:17.822358Z","shell.execute_reply":"2026-05-12T21:10:17.845275Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from transformers import AutoModelForSeq2SeqLM, Seq2SeqTrainingArguments, Seq2SeqTrainer, EarlyStoppingCallback\nimport gc\ngc.collect()\ntorch.cuda.empty_cache()\n\nMODEL_NAME = \"google/byt5-base\"\nCSV_PATH = \"/kaggle/working/cleaned_oracc_corpus_train.csv\"\nOUTPUT_DIR = \"/kaggle/working/byt5_pretrain\" \n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n\nmodel = AutoModelForSeq2SeqLM.from_pretrained(\"google/byt5-base\")\n\nmodel.to(device)\n\ntraining_args = Seq2SeqTrainingArguments(\n    output_dir=OUTPUT_DIR,\n    num_train_epochs=50,\n    per_device_train_batch_size=8,\n    gradient_accumulation_steps=2,\n    per_device_eval_batch_size=16,\n    warmup_ratio=0.1,\n\n    eval_strategy=\"epoch\",\n    logging_strategy=\"steps\",\n    logging_steps=16,\n    logging_first_step= True,\n    save_strategy=\"epoch\",\n\n    lr_scheduler_type=\"linear\",\n\n    learning_rate=5e-4,          \n    weight_decay=0.05,\n    max_grad_norm=1.0, \n   \n    save_total_limit=2,\n    load_best_model_at_end=True,\n    predict_with_generate=True,       \n    push_to_hub=False,                \n    report_to=\"none\",\n    fp16=True,\n    optim=\"adafactor\",\n    gradient_checkpointing=True,\n    log_level=\"info\",              # Forces the internal logger to be more chatty\n    disable_tqdm=False, \n)\n\ntrainer = Seq2SeqTrainer(\n    model=model,\n    args=training_args,\n    train_dataset=chunked_train_dataset,\n    eval_dataset=chunked_eval_dataset,\n    data_collator=data_collator,\n    processing_class=tokenizer,\n    callbacks=[EarlyStoppingCallback(early_stopping_patience=3)]\n)\n\ntrainer.train()\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-12T21:14:27.964070Z","iopub.execute_input":"2026-05-12T21:14:27.964747Z","iopub.status.idle":"2026-05-12T21:14:36.058721Z","shell.execute_reply.started":"2026-05-12T21:14:27.964715Z","shell.execute_reply":"2026-05-12T21:14:36.058065Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ntrainer.save_model(OUTPUT_DIR)\ntokenizer.save_pretrained(OUTPUT_DIR)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!zip -r byt5_pretrain.zip \"/kaggle/working/byt5_pretrain\"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}