{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"},{"sourceId":258034511,"sourceType":"kernelVersion"},{"sourceId":289661793,"sourceType":"kernelVersion"}],"dockerImageVersionId":31236,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"try:\n    import unsloth\nexcept Exception as e:\n    !pip uninstall -y torchvision\n    !pip uninstall -y torch\n    !pip install unsloth==2025.11.6\n    !pip uninstall -y tensorflow","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T08:56:34.604872Z","iopub.execute_input":"2026-01-01T08:56:34.605683Z","iopub.status.idle":"2026-01-01T08:59:11.691025Z","shell.execute_reply.started":"2026-01-01T08:56:34.605652Z","shell.execute_reply":"2026-01-01T08:59:11.690279Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip --version","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-02T15:28:00.915365Z","iopub.execute_input":"2026-01-02T15:28:00.915556Z","iopub.status.idle":"2026-01-02T15:28:01.412099Z","shell.execute_reply.started":"2026-01-02T15:28:00.915536Z","shell.execute_reply":"2026-01-02T15:28:01.411283Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport torch\nimport json\nimport wandb\nimport pickle as pkl\nfrom datasets import Dataset, load_dataset, DatasetDict\nfrom sklearn.model_selection import train_test_split\nfrom unsloth import FastLanguageModel, PatchFastRL, is_bfloat16_supported","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-01T08:42:11.729728Z","iopub.execute_input":"2026-01-01T08:42:11.730134Z","iopub.status.idle":"2026-01-01T08:42:20.204120Z","shell.execute_reply.started":"2026-01-01T08:42:11.730098Z","shell.execute_reply":"2026-01-01T08:42:20.202728Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_dir = '/kaggle/input/deep-past-initiative-machine-translation'\n\ntrain_data = pd.read_csv(f'{data_dir}/train.csv')\ntest_data = pd.read_csv(f'{data_dir}/test.csv')\nlexicon_data = pd.read_csv(f'{data_dir}/OA_Lexicon_eBL.csv')\ndictionary_data = pd.read_csv(f'{data_dir}/eBL_Dictionary.csv')\npublished_texts = pd.read_csv(f'{data_dir}/published_texts.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T08:42:20.204639Z","iopub.status.idle":"2026-01-01T08:42:20.204889Z","shell.execute_reply.started":"2026-01-01T08:42:20.204770Z","shell.execute_reply":"2026-01-01T08:42:20.204789Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lora_rank = 16\nmax_length = 1024\nmodel, tokenizer = FastLanguageModel.from_pretrained(\n    model_name = \"/kaggle/input/gemma-3-4b-it-bnb-4bit/gemma-3-4b-it-bnb-4bit\",\n    load_in_4bit = True, # False for LoRA 16bit\n    # fast_inference = True, # Enable vLLM fast inference\n    # use_fast=True,\n    max_lora_rank = lora_rank,\n    max_seq_length=max_length,\n    gpu_memory_utilization = 0.7, \n)\n\ntokenizer.padding_side = \"left\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T08:42:20.205967Z","iopub.status.idle":"2026-01-01T08:42:20.206202Z","shell.execute_reply.started":"2026-01-01T08:42:20.206094Z","shell.execute_reply":"2026-01-01T08:42:20.206108Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = FastLanguageModel.get_peft_model(\n    model,\n    r=lora_rank,\n    target_modules=[\n        \"q_proj\",\"k_proj\",\"v_proj\",\"o_proj\",\n        \"gate_proj\",\"up_proj\",\"down_proj\",\n    ],\n    lora_alpha=lora_rank * 2,\n    use_gradient_checkpointing=\"unsloth\",\n    random_state=3407,\n    # lora_dropout = 0.05,\n    # adapter_name=\"grpo\",\n)\n\nmodel.config.use_cache = False","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T08:42:20.207120Z","iopub.status.idle":"2026-01-01T08:42:20.207422Z","shell.execute_reply.started":"2026-01-01T08:42:20.207273Z","shell.execute_reply":"2026-01-01T08:42:20.207295Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\ndef create_translation_dataset(df, text_col='transliteration', target_col='translation'):\n    \n    def create_sample(row):\n        source_text = str(row[text_col])\n        \n        prompt = f\"\"\"\nAs an Akkadian translation expert, please translate the following text into English following functional equivalence principles:\n\nTranslation requirements:\n1. Accurately convey the semantic content of the original text\n2. Consider cultural adaptation (convert unfamiliar concepts to target-culture understandable concepts)\n3. Maintain natural and fluent language\n\nOriginal text: \"{source_text}\"\n\nPlease only provide the translation:\n\"\"\"\n        \n        conversation = [\n            {\n                \"content\": 'You are a professional Akkadian translation expert skilled in functional equivalence translation.',\n                \"role\": \"system\"\n            },\n            {\n                \"content\": prompt,\n                \"role\": \"user\"\n            },\n            \n        ]\n\n        if target_col in row:\n            target_text = str(row[target_col])\n            assistant = {\n                \"content\": f\"{target_text}\",\n                \"role\": \"assistant\"\n            }\n            conversation.append(assistant)\n        \n        return {\n            \"conversations\": conversation,\n        }\n\n    # TODO normalize the origin text\n    samples = [create_sample(df.iloc[i]) for i in range(len(df))]\n    \n    dataset = Dataset.from_list(samples)\n    \n    return dataset\n\ntrain, test = train_test_split(train_data, random_state=42, test_size=0.15)\ndataset = create_translation_dataset(\n    train, \n    text_col='transliteration', \n    target_col='translation'\n)\n\nval_dataset = create_translation_dataset(\n    test, \n    text_col='transliteration', \n    target_col='translation'\n)\n\nprint(len(dataset), dataset[100])\nprint()\nprint(len(val_dataset), val_dataset[10])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T08:42:20.208246Z","iopub.status.idle":"2026-01-01T08:42:20.208713Z","shell.execute_reply.started":"2026-01-01T08:42:20.208589Z","shell.execute_reply":"2026-01-01T08:42:20.208605Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def formatting_prompts_func(examples):\n   convos = examples[\"conversations\"]\n   texts = [tokenizer.apply_chat_template(convo, tokenize = False, add_generation_prompt = False).removeprefix('<bos>') for convo in convos]\n   return { \"text\" : texts, }\n\ndataset = dataset.map(formatting_prompts_func, batched = True)\n\n\ndataset[100]['text']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T08:42:20.209983Z","iopub.status.idle":"2026-01-01T08:42:20.210224Z","shell.execute_reply.started":"2026-01-01T08:42:20.210118Z","shell.execute_reply":"2026-01-01T08:42:20.210132Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from trl import SFTTrainer, SFTConfig\ntrainer = SFTTrainer(\n    model = model,\n    tokenizer = tokenizer,\n    train_dataset = dataset,\n    eval_dataset = None, # Can set up evaluation!\n    args = SFTConfig(\n        dataset_text_field = \"text\",\n        per_device_train_batch_size = 2,\n        gradient_accumulation_steps = 8,\n        warmup_steps = 5,\n        num_train_epochs = 10, # Set this for full full-scale training\n        # max_steps = 10,        # Comment this out for full-scale training\n        learning_rate = 1e-4, # Reduce to 2e-5 for long training runs\n        logging_steps = 1,\n        optim = \"adamw_8bit\",\n        weight_decay = 0.01,\n        lr_scheduler_type = \"linear\",\n        seed = 3407,\n        report_to = \"none\", # Use this for WandB etc\n        dataset_num_proc=2,\n    ),\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T08:42:20.211007Z","iopub.status.idle":"2026-01-01T08:42:20.211314Z","shell.execute_reply.started":"2026-01-01T08:42:20.211160Z","shell.execute_reply":"2026-01-01T08:42:20.211184Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from unsloth.chat_templates import train_on_responses_only\ntrainer = train_on_responses_only(\n    trainer,\n    instruction_part = \"<start_of_turn>user\\n\",\n    response_part = \"<start_of_turn>model\\n\",\n)\n\n\nprint(tokenizer.decode(trainer.train_dataset[100][\"input_ids\"]))\nprint()\nprint(tokenizer.decode([tokenizer.pad_token_id if x == -100 else x for x in trainer.train_dataset[100][\"labels\"]]).replace(tokenizer.pad_token, \" \"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T08:42:20.212718Z","iopub.status.idle":"2026-01-01T08:42:20.212995Z","shell.execute_reply.started":"2026-01-01T08:42:20.212867Z","shell.execute_reply":"2026-01-01T08:42:20.212898Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"trainer_stats = trainer.train()\n\nmodel.save_pretrained(\"deep_past_gm3\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-01T08:42:20.214248Z","iopub.status.idle":"2026-01-01T08:42:20.214532Z","shell.execute_reply.started":"2026-01-01T08:42:20.214402Z","shell.execute_reply":"2026-01-01T08:42:20.214417Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}