{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"}],"dockerImageVersionId":31234,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-04T05:48:52.651980Z","iopub.execute_input":"2026-01-04T05:48:52.652852Z","iopub.status.idle":"2026-01-04T05:48:52.656715Z","shell.execute_reply.started":"2026-01-04T05:48:52.652819Z","shell.execute_reply":"2026-01-04T05:48:52.655872Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/train.csv').head(100)\ntest = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/test.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T05:48:52.658229Z","iopub.execute_input":"2026-01-04T05:48:52.658487Z","iopub.status.idle":"2026-01-04T05:48:52.716868Z","shell.execute_reply.started":"2026-01-04T05:48:52.658462Z","shell.execute_reply":"2026-01-04T05:48:52.716025Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T05:48:52.718492Z","iopub.execute_input":"2026-01-04T05:48:52.718800Z","iopub.status.idle":"2026-01-04T05:48:52.729847Z","shell.execute_reply.started":"2026-01-04T05:48:52.718774Z","shell.execute_reply":"2026-01-04T05:48:52.729059Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/sample_submission.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T05:48:52.730857Z","iopub.execute_input":"2026-01-04T05:48:52.731151Z","iopub.status.idle":"2026-01-04T05:48:52.759642Z","shell.execute_reply.started":"2026-01-04T05:48:52.731126Z","shell.execute_reply":"2026-01-04T05:48:52.758760Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\n\ndef clean_text(text):\n    text = str(text).lower()\n    text = re.sub(r\"\\[.*?\\]\", \"\", text)   # remove [ ... ]\n    text = re.sub(r\"\\(.*?\\)\", \"\", text)   # remove ( ... )\n    text = re.sub(r\"[^a-z0-9šṣṭḫūāīē\\- ]\", \"\", text)\n    text = re.sub(r\"\\s+\", \" \", text).strip()\n    return text\n\ntrain[\"src\"] = train[\"transliteration\"].apply(clean_text)\ntrain[\"tgt\"] = train[\"translation\"].apply(clean_text)\n\ntest[\"src\"] = test[\"transliteration\"].apply(clean_text)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T05:48:52.761620Z","iopub.execute_input":"2026-01-04T05:48:52.761938Z","iopub.status.idle":"2026-01-04T05:48:52.782479Z","shell.execute_reply.started":"2026-01-04T05:48:52.761913Z","shell.execute_reply":"2026-01-04T05:48:52.781619Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T05:48:52.783523Z","iopub.execute_input":"2026-01-04T05:48:52.784067Z","iopub.status.idle":"2026-01-04T05:48:52.805301Z","shell.execute_reply.started":"2026-01-04T05:48:52.784032Z","shell.execute_reply":"2026-01-04T05:48:52.804497Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dictionary = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/eBL_Dictionary.csv\")\n\ndef clean_word(text):\n    if pd.isna(text):\n        return \"\"\n    text = text.lower()\n    text = re.sub(r\"[^a-z0-9šṣṭḫūāīē\\- ]\", \"\", text)\n    return text.strip()\n\ndictionary[\"word\"] = dictionary[\"word\"].apply(clean_word)\ndictionary[\"definition\"] = dictionary[\"definition\"].apply(clean_word)\n\ndictionary.head()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T05:48:52.806252Z","iopub.execute_input":"2026-01-04T05:48:52.806570Z","iopub.status.idle":"2026-01-04T05:48:53.012925Z","shell.execute_reply.started":"2026-01-04T05:48:52.806534Z","shell.execute_reply":"2026-01-04T05:48:53.012207Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from collections import defaultdict\n\nword2def = defaultdict(list)\n\nfor _, row in dictionary.iterrows():\n    if row[\"word\"] and row[\"definition\"]:\n        word2def[row[\"word\"]].append(row[\"definition\"])\n\nword2def = {k: v[0] for k, v in word2def.items()}\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T05:48:53.013870Z","iopub.execute_input":"2026-01-04T05:48:53.014176Z","iopub.status.idle":"2026-01-04T05:48:54.038962Z","shell.execute_reply.started":"2026-01-04T05:48:53.014143Z","shell.execute_reply":"2026-01-04T05:48:54.038295Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def dict_translate(sentence):\n    tokens = sentence.split()\n    out = []\n\n    for tok in tokens:\n        if tok in word2def:\n            out.append(word2def[tok])\n        else:\n            out.append(tok)  # fallback\n    return \" \".join(out)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T05:48:54.040068Z","iopub.execute_input":"2026-01-04T05:48:54.040591Z","iopub.status.idle":"2026-01-04T05:48:54.045647Z","shell.execute_reply.started":"2026-01-04T05:48:54.040552Z","shell.execute_reply":"2026-01-04T05:48:54.044729Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(dict_translate(\"um-ma kà-ru-um kà-ni-ia-ma\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T05:48:54.046633Z","iopub.execute_input":"2026-01-04T05:48:54.047080Z","iopub.status.idle":"2026-01-04T05:48:54.066720Z","shell.execute_reply.started":"2026-01-04T05:48:54.047038Z","shell.execute_reply":"2026-01-04T05:48:54.065909Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train[\"dict_src\"] = train[\"src\"].apply(dict_translate)\ntest[\"dict_src\"] = test[\"src\"].apply(dict_translate)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T05:48:54.069117Z","iopub.execute_input":"2026-01-04T05:48:54.069436Z","iopub.status.idle":"2026-01-04T05:48:54.083750Z","shell.execute_reply.started":"2026-01-04T05:48:54.069409Z","shell.execute_reply":"2026-01-04T05:48:54.083043Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T05:48:54.084726Z","iopub.execute_input":"2026-01-04T05:48:54.085075Z","iopub.status.idle":"2026-01-04T05:48:54.102643Z","shell.execute_reply.started":"2026-01-04T05:48:54.085039Z","shell.execute_reply":"2026-01-04T05:48:54.101937Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer\n\nsrc_vectorizer = TfidfVectorizer(\n    ngram_range=(1, 2),\n    max_features=30000\n)\n\ntgt_vectorizer = TfidfVectorizer(\n    ngram_range=(1, 2),\n    max_features=30000\n)\n\nX = src_vectorizer.fit_transform(train[\"dict_src\"])\nY = tgt_vectorizer.fit_transform(train[\"tgt\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T05:50:30.962694Z","iopub.execute_input":"2026-01-04T05:50:30.963593Z","iopub.status.idle":"2026-01-04T05:50:31.016552Z","shell.execute_reply.started":"2026-01-04T05:50:30.963562Z","shell.execute_reply":"2026-01-04T05:50:31.015779Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_test, y_train, y_test = train_test_split(X,Y.toarray(),test_size=0.2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T05:50:31.198525Z","iopub.execute_input":"2026-01-04T05:50:31.198843Z","iopub.status.idle":"2026-01-04T05:50:31.214927Z","shell.execute_reply.started":"2026-01-04T05:50:31.198817Z","shell.execute_reply":"2026-01-04T05:50:31.214233Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.linear_model import Ridge\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.multioutput import MultiOutputRegressor\n\nY_dense = Y.toarray()\nmodel = MultiOutputRegressor(\n    # Ridge(alpha=1.0),\n    GradientBoostingRegressor(),\n    # RandomForestRegressor(max_depth=3)\n)\n\nmodel.fit(X_train, y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T06:56:53.196226Z","iopub.execute_input":"2026-01-04T06:56:53.197113Z","iopub.status.idle":"2026-01-04T07:24:18.679250Z","shell.execute_reply.started":"2026-01-04T06:56:53.197075Z","shell.execute_reply":"2026-01-04T07:24:18.678437Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_test","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T06:50:24.331532Z","iopub.execute_input":"2026-01-04T06:50:24.331795Z","iopub.status.idle":"2026-01-04T06:50:24.336973Z","shell.execute_reply.started":"2026-01-04T06:50:24.331771Z","shell.execute_reply":"2026-01-04T06:50:24.336322Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred = model.predict(X_test)\ndef vector_to_text(vec, vectorizer, top_k=20):\n    feature_names = np.array(vectorizer.get_feature_names_out())\n    idx = np.argsort(vec)[-top_k:]\n    return \" \".join(feature_names[idx])\n\ntranslation = [\n    vector_to_text(y, tgt_vectorizer)\n    for y in y_pred\n]\n\nprint(translation)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T06:07:11.177876Z","iopub.execute_input":"2026-01-04T06:07:11.179049Z","iopub.status.idle":"2026-01-04T06:07:12.170367Z","shell.execute_reply.started":"2026-01-04T06:07:11.178984Z","shell.execute_reply":"2026-01-04T06:07:12.169563Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nfrom sklearn.metrics.pairwise import cosine_similarity, euclidean_distances\n\n\ndef evaluate_embeddings(y_test, y_pred):\n    \"\"\"\n    Calculates standard metrics for embedding comparison.\n    \"\"\"\n    cos_sim = np.diag(cosine_similarity(y_test, y_pred))\n    avg_cosine = np.mean(cos_sim)\n\n\n    l2_dist = np.linalg.norm(y_test - y_pred, axis=1)\n    avg_l2 = np.mean(l2_dist)\n\n    mse = np.mean((y_test - y_pred)**2)\n\n    return {\n        \"Avg Cosine Similarity\": avg_cosine,\n        \"Avg L2 Distance\": avg_l2,\n        \"Mean Squared Error\": mse\n    }\n\nresults = evaluate_embeddings(y_test, y_pred)\nfor metric, value in results.items():\n    print(f\"{metric}: {value:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T06:50:24.337829Z","iopub.execute_input":"2026-01-04T06:50:24.338142Z","iopub.status.idle":"2026-01-04T06:50:24.368080Z","shell.execute_reply.started":"2026-01-04T06:50:24.338116Z","shell.execute_reply":"2026-01-04T06:50:24.366685Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Evaluation","metadata":{}},{"cell_type":"code","source":"len(translation)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T05:50:45.025959Z","iopub.execute_input":"2026-01-04T05:50:45.026263Z","iopub.status.idle":"2026-01-04T05:50:45.032031Z","shell.execute_reply.started":"2026-01-04T05:50:45.026228Z","shell.execute_reply":"2026-01-04T05:50:45.031111Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_test = src_vectorizer.transform(test[\"dict_src\"])\nY_pred = model.predict(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T05:49:07.675218Z","iopub.execute_input":"2026-01-04T05:49:07.675525Z","iopub.status.idle":"2026-01-04T05:49:08.586905Z","shell.execute_reply.started":"2026-01-04T05:49:07.675497Z","shell.execute_reply":"2026-01-04T05:49:08.586080Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\n\ndef vector_to_text(vec, vectorizer, top_k=15):\n    feature_names = np.array(vectorizer.get_feature_names_out())\n    idx = np.argsort(vec)[-top_k:]\n    return \" \".join(feature_names[idx])\n\ntest[\"translation\"] = [\n    vector_to_text(y, tgt_vectorizer)\n    for y in Y_pred\n]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T06:56:32.294489Z","iopub.execute_input":"2026-01-04T06:56:32.295094Z","iopub.status.idle":"2026-01-04T06:56:32.319995Z","shell.execute_reply.started":"2026-01-04T06:56:32.295064Z","shell.execute_reply":"2026-01-04T06:56:32.319342Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T06:56:32.490698Z","iopub.execute_input":"2026-01-04T06:56:32.491780Z","iopub.status.idle":"2026-01-04T06:56:32.502973Z","shell.execute_reply.started":"2026-01-04T06:56:32.491746Z","shell.execute_reply":"2026-01-04T06:56:32.502268Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = test[['id','translation']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T06:56:36.024336Z","iopub.execute_input":"2026-01-04T06:56:36.025153Z","iopub.status.idle":"2026-01-04T06:56:36.029734Z","shell.execute_reply.started":"2026-01-04T06:56:36.025124Z","shell.execute_reply":"2026-01-04T06:56:36.028976Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.to_csv('submission.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T06:56:37.725468Z","iopub.execute_input":"2026-01-04T06:56:37.726264Z","iopub.status.idle":"2026-01-04T06:56:37.731505Z","shell.execute_reply.started":"2026-01-04T06:56:37.726202Z","shell.execute_reply":"2026-01-04T06:56:37.730826Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}