{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":121150,"databundleVersionId":14976537,"sourceType":"competition"}],"dockerImageVersionId":31236,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **TAE: Training & Model Creation Baseline**\n**Translate Akkadian to English**","metadata":{}},{"cell_type":"markdown","source":"Setup","metadata":{}},{"cell_type":"code","source":"!pip install -q transformers datasets sentencepiece accelerate\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport json\nimport pandas as pd\nfrom datasets import Dataset\nfrom transformers import (\n    AutoTokenizer,\n    AutoModelForSeq2SeqLM,\n    DataCollatorForSeq2Seq,\n    Trainer,\n    TrainingArguments\n)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Path & Directory Design","metadata":{}},{"cell_type":"code","source":"BASE_DIR = \"/kaggle/working/artifacts\"\n\nSUBDIRS = [\n    \"tokenizer\",\n    \"translator\",\n    \"rules\",\n    \"meta\"\n]\n\nfor d in SUBDIRS:\n    os.makedirs(os.path.join(BASE_DIR, d), exist_ok=True)\n\nprint(\"Artifact dirs ready\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Load Train Data","metadata":{}},{"cell_type":"code","source":"TRAIN_PATH = \"/kaggle/input/deep-past-initiative-machine-translation/train.csv\"\n\ndf = pd.read_csv(TRAIN_PATH)\ndf = df.dropna().reset_index(drop=True)\n\nprint(df.head())\nprint(\"Train size:\", len(df))\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Baseline Preprocess","metadata":{}},{"cell_type":"code","source":"def preprocess_text(x):\n    x = x.strip()\n    return x\n\ndf[\"src\"] = df[\"transliteration\"].apply(preprocess_text)\ndf[\"tgt\"] = df[\"translation\"].apply(preprocess_text)\n\ndataset = Dataset.from_pandas(df[[\"src\", \"tgt\"]])\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Tokenizer（baseline）","metadata":{}},{"cell_type":"code","source":"MODEL_NAME = \"t5-small\"\n\ntokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)\ntokenizer.save_pretrained(f\"{BASE_DIR}/tokenizer\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Tokenization","metadata":{}},{"cell_type":"code","source":"MAX_SRC_LEN = 128\nMAX_TGT_LEN = 128\n\ndef tokenize(batch):\n    model_inputs = tokenizer(\n        batch[\"src\"],\n        max_length=MAX_SRC_LEN,\n        truncation=True\n    )\n\n    with tokenizer.as_target_tokenizer():\n        labels = tokenizer(\n            batch[\"tgt\"],\n            max_length=MAX_TGT_LEN,\n            truncation=True\n        )\n\n    model_inputs[\"labels\"] = labels[\"input_ids\"]\n    return model_inputs\n\ntokenized_ds = dataset.map(tokenize, batched=True, remove_columns=dataset.column_names)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Model","metadata":{}},{"cell_type":"code","source":"model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Training Arguments","metadata":{}},{"cell_type":"code","source":"training_args = TrainingArguments(\n    output_dir=\"./tmp\",\n    per_device_train_batch_size=16,\n    gradient_accumulation_steps=1,\n    learning_rate=3e-4,\n    num_train_epochs=3,\n    fp16=True,\n    logging_steps=100,\n    save_strategy=\"no\",\n    report_to=\"none\"\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Trainer & Train","metadata":{}},{"cell_type":"code","source":"data_collator = DataCollatorForSeq2Seq(\n    tokenizer,\n    model=model\n)\n\ntrainer = Trainer(\n    model=model,\n    args=training_args,\n    train_dataset=tokenized_ds,\n    tokenizer=tokenizer,\n    data_collator=data_collator\n)\n\ntrainer.train()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Save Model","metadata":{}},{"cell_type":"code","source":"model.save_pretrained(f\"{BASE_DIR}/translator\")\nprint(\"Model saved\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Save Meta Info","metadata":{}},{"cell_type":"code","source":"meta = {\n    \"model\": MODEL_NAME,\n    \"epochs\": training_args.num_train_epochs,\n    \"batch_size\": training_args.per_device_train_batch_size,\n    \"baseline\": True\n}\n\nwith open(f\"{BASE_DIR}/meta/training_args.json\", \"w\") as f:\n    json.dump(meta, f, indent=2)\n\nprint(\"Meta saved\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}