{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391},{"sourceType":"datasetVersion","sourceId":14410665,"datasetId":9203761,"databundleVersionId":15226414},{"sourceType":"datasetVersion","sourceId":14376272,"datasetId":9181082,"databundleVersionId":15188111},{"sourceType":"datasetVersion","sourceId":14661932,"datasetId":9366631,"databundleVersionId":15503341},{"sourceType":"datasetVersion","sourceId":14374989,"datasetId":9147887,"databundleVersionId":15186684},{"sourceType":"modelInstanceVersion","sourceId":282751,"databundleVersionId":11375937,"modelInstanceId":239470,"modelId":222398}],"dockerImageVersionId":31260,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"# =============================================================================\n# Kaggle: Translate Akkadian to English — Segment-aligned template retrieval\n# Key: when test fragment matches a longer train text, extract only the\n# corresponding portion of the translation (not the full document).\n# =============================================================================\n\nimport os\nimport re\nimport math\nimport numpy as np\nimport pandas as pd\nfrom collections import Counter\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.metrics.pairwise import linear_kernel\n\n\nDATA = os.environ.get(\"KAGGLE_INPUT\", \"/kaggle/input/deep-past-initiative-machine-translation/\")\nOUT  = os.environ.get(\"KAGGLE_WORKING\", \"/kaggle/working\")\n\nW_CHAR = float(os.environ.get(\"W_CHAR\", \"0.35\"))\nW_WORD = float(os.environ.get(\"W_WORD\", \"0.15\"))\nW_COV  = float(os.environ.get(\"W_COV\",  \"0.30\"))\nW_RARE = float(os.environ.get(\"W_RARE\", \"0.20\"))\n\nMAX_REUSE   = int(os.environ.get(\"MAX_REUSE\", \"5\"))\nREP_PENALTY = float(os.environ.get(\"REP_PENALTY\", \"0.5\"))\n\n\ndef normalize_text(text: str) -> str:\n    if pd.isna(text):\n        return \"\"\n    t = str(text).lower().strip()\n    t = re.sub(r\"(\\.{3,}|…+)\", \" <big_gap> \", t)\n    t = re.sub(r\"\\bxx+\\b\", \" <gap> \", t)\n    t = re.sub(r\"\\s+\", \" \", t).strip()\n    return t\n\n\n# ── Token IDF ────────────────────────────────────────────────────────────────\n\ndef build_idf(texts: np.ndarray) -> dict:\n    n = len(texts)\n    df: Counter = Counter()\n    for t in texts:\n        for tok in set(t.split()):\n            df[tok] += 1\n    return {tok: math.log((n + 1) / (c + 1)) + 1.0 for tok, c in df.items()}\n\n\ndef token_sets(texts: np.ndarray) -> list:\n    return [set(t.split()) for t in texts]\n\n\ndef compute_coverage(test_toks: set, train_sets: list, idf: dict) -> np.ndarray:\n    n = len(train_sets)\n    if not test_toks:\n        return np.zeros(n, dtype=np.float32)\n    total = sum(idf.get(t, 1.0) for t in test_toks)\n    if total <= 0:\n        return np.zeros(n, dtype=np.float32)\n    out = np.empty(n, dtype=np.float32)\n    for j, ts in enumerate(train_sets):\n        ov = test_toks & ts\n        out[j] = sum(idf.get(t, 1.0) for t in ov) / total if ov else 0.0\n    return out\n\n\ndef compute_rare_bonus(test_toks: set, train_sets: list, idf: dict) -> np.ndarray:\n    n = len(train_sets)\n    if not test_toks:\n        return np.zeros(n, dtype=np.float32)\n    total = sum(idf.get(t, 1.0) ** 2 for t in test_toks)\n    if total <= 0:\n        return np.zeros(n, dtype=np.float32)\n    out = np.empty(n, dtype=np.float32)\n    for j, ts in enumerate(train_sets):\n        ov = test_toks & ts\n        out[j] = sum(idf.get(t, 1.0) ** 2 for t in ov) / total if ov else 0.0\n    return out\n\n\n# ── Segment extraction (core fix) ───────────────────────────────────────────\n\ndef extract_aligned_segment(test_text: str, train_text: str, train_translation: str) -> str:\n    \"\"\"When test is a fragment of a longer train text, extract only the\n    corresponding portion of the translation using sliding-window alignment\n    + sentence-boundary snapping.\"\"\"\n    test_toks  = test_text.split()\n    train_toks = train_text.split()\n\n    if not test_toks or not train_toks:\n        return train_translation\n\n    ratio = len(test_toks) / len(train_toks)\n    if ratio > 0.75:\n        return train_translation\n\n    test_set = set(test_toks)\n\n    # Sliding window: find where test aligns within train\n    window = len(test_toks)\n    best_start, best_ov = 0, -1\n    for s in range(max(1, len(train_toks) - window + 1)):\n        ov = len(test_set & set(train_toks[s:s + window]))\n        if ov > best_ov:\n            best_ov = ov\n            best_start = s\n\n    frac_start = best_start / len(train_toks)\n    frac_end   = min(1.0, (best_start + window) / len(train_toks))\n\n    # Split translation into sentences\n    sents = re.split(r'(?<=[.!?\"\\'])\\s+', train_translation.strip())\n    if len(sents) <= 1:\n        tr_toks = train_translation.split()\n        ts = int(frac_start * len(tr_toks))\n        te = min(len(tr_toks), max(ts + 1, int(frac_end * len(tr_toks))))\n        return \" \".join(tr_toks[ts:te])\n\n    # Select sentences whose CENTER falls within the aligned range\n    total_len = sum(len(s) for s in sents)\n    cum = 0\n    selected = []\n    for i, sent in enumerate(sents):\n        s_start = cum / total_len\n        cum += len(sent)\n        s_end = cum / total_len\n        center = (s_start + s_end) / 2\n        if frac_start <= center <= frac_end:\n            selected.append(i)\n\n    if not selected:\n        return train_translation\n    return \" \".join(sents[i] for i in selected)\n\n\n# ── Selection ────────────────────────────────────────────────────────────────\n\ndef select_best_index(\n    scores: np.ndarray,\n    train_translations: np.ndarray,\n    used_counts: dict,\n    max_reuse: int,\n    rep_penalty: float,\n) -> int:\n    \"\"\"Return the train index of the best match, with anti-repetition.\"\"\"\n    if scores.size == 0:\n        return 0\n\n    # Best score per unique translation, keeping the train index\n    best_per_tr: dict = {}\n    for idx in range(len(scores)):\n        s = float(scores[idx])\n        if s <= 0:\n            continue\n        tr = train_translations[idx]\n        if tr not in best_per_tr or s > best_per_tr[tr][0]:\n            best_per_tr[tr] = (s, idx)\n\n    if not best_per_tr:\n        return int(np.argmax(scores))\n\n    candidates = sorted(best_per_tr.values(), key=lambda x: -x[0])\n\n    for s, idx in candidates:\n        tr = train_translations[idx]\n        count = used_counts.get(tr, 0)\n        if max_reuse > 0 and count >= max_reuse:\n            continue\n        return idx\n\n    # All capped: pick least-used\n    all_items = [(used_counts.get(train_translations[idx], 0), s, idx)\n                 for s, idx in candidates]\n    all_items.sort(key=lambda x: (x[0], -x[1]))\n    return all_items[0][2]\n\n\n# ── Main ─────────────────────────────────────────────────────────────────────\n\ndef main():\n    if not os.path.exists(DATA):\n        raise FileNotFoundError(f\"Input directory not found: {DATA}\")\n    os.makedirs(OUT, exist_ok=True)\n\n    train = pd.read_csv(f\"{DATA}/train.csv\")\n    test  = pd.read_csv(f\"{DATA}/test.csv\")\n\n    train[\"transliteration\"] = train[\"transliteration\"].apply(normalize_text)\n    train[\"translation\"] = train[\"translation\"].fillna(\"\").astype(str).str.strip()\n    test[\"transliteration\"]  = test[\"transliteration\"].apply(normalize_text)\n\n    train = train[\n        (train[\"transliteration\"].str.len() > 0) &\n        (train[\"translation\"].str.len() > 0)\n    ].reset_index(drop=True)\n    if len(train) == 0:\n        raise ValueError(\"Training data is empty after preprocessing.\")\n\n    train_texts = train[\"transliteration\"].to_numpy()\n    test_texts  = test[\"transliteration\"].to_numpy()\n    train_translations = train[\"translation\"].to_numpy()\n\n    # TF-IDF vectorizers\n    char_vec = TfidfVectorizer(analyzer=\"char_wb\", ngram_range=(2, 5), min_df=1, lowercase=False)\n    word_vec = TfidfVectorizer(analyzer=\"word\", ngram_range=(1, 2), min_df=1, lowercase=False)\n    X_train_char = char_vec.fit_transform(train_texts)\n    X_train_word = word_vec.fit_transform(train_texts)\n\n    # Token-level IDF + sets\n    idf = build_idf(train_texts)\n    t_sets = token_sets(train_texts)\n    print(f\"Vocab: {len(idf)} tokens, IDF [{min(idf.values()):.2f}, {max(idf.values()):.2f}]\")\n\n    # Row-by-row retrieval + segment extraction\n    preds = []\n    used_counts: dict = {}\n    n_extracted = 0\n\n    for i, test_text in enumerate(test_texts):\n        test_toks = set(test_text.split())\n\n        sim_c = linear_kernel(char_vec.transform([test_text]), X_train_char).ravel()\n        sim_w = linear_kernel(word_vec.transform([test_text]), X_train_word).ravel()\n        cov   = compute_coverage(test_toks, t_sets, idf)\n        rare  = compute_rare_bonus(test_toks, t_sets, idf)\n\n        score = (W_CHAR * sim_c) + (W_WORD * sim_w) + (W_COV * cov) + (W_RARE * rare)\n\n        best_idx = select_best_index(score, train_translations, used_counts, MAX_REUSE, REP_PENALTY)\n\n        raw_tr = train_translations[best_idx]\n        translation = extract_aligned_segment(test_text, train_texts[best_idx], raw_tr)\n\n        if translation != raw_tr:\n            n_extracted += 1\n\n        preds.append(translation)\n        used_counts[raw_tr] = used_counts.get(raw_tr, 0) + 1\n\n        if (i + 1) % 500 == 0:\n            print(f\"  {i + 1}/{len(test_texts)} rows\")\n\n    # Build submission\n    pred_df = test[[\"id\"]].copy()\n    pred_df[\"id\"] = pred_df[\"id\"].astype(str).str.strip()\n    pred_df[\"translation\"] = pd.Series(preds, index=pred_df.index).fillna(\"\").astype(str).str.strip()\n\n    if int(pred_df[\"id\"].duplicated().sum()) > 0:\n        raise ValueError(\"Duplicated ids in submission\")\n\n    sub = pred_df[[\"id\", \"translation\"]].copy()\n    output_path = f\"{OUT}/submission.csv\"\n    sub.to_csv(output_path, index=False)\n\n    # Diagnostics\n    n_empty  = int(sub[\"translation\"].str.strip().eq(\"\").sum())\n    n_unique = int(sub[\"translation\"].nunique())\n    print(f\"\\nSaved: {output_path}\")\n    print(f\"Shape: {sub.shape}, Train: {len(train)}\")\n    print(f\"Weights: char={W_CHAR} word={W_WORD} cov={W_COV} rare={W_RARE}\")\n    print(f\"Segment extractions: {n_extracted}/{len(test_texts)} (test shorter than best match)\")\n    print(f\"Empty: {n_empty}, Unique: {n_unique}/{len(sub)}\")\n\n    counts = pd.Series(preds).value_counts()\n    if counts.iloc[0] > 1:\n        print(\"Top-5 repeated:\")\n        for t, c in counts.head(5).items():\n            print(f\"  [{c}x] {str(t)[:80]}...\")\n\n    return sub\n\n\nif __name__ == \"__main__\":\n    main()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-17T01:42:10.137398Z","iopub.execute_input":"2026-02-17T01:42:10.137666Z","iopub.status.idle":"2026-02-17T01:42:11.415940Z","shell.execute_reply.started":"2026-02-17T01:42:10.137646Z","shell.execute_reply":"2026-02-17T01:42:11.414902Z"}},"outputs":[],"execution_count":null}]}