{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":125766,"databundleVersionId":14873485,"isSourceIdPinned":false},{"sourceType":"datasetVersion","sourceId":14437983,"datasetId":9222249,"databundleVersionId":15256727}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-02-19T23:00:30.410670Z","iopub.execute_input":"2026-02-19T23:00:30.411165Z","iopub.status.idle":"2026-02-19T23:00:30.419869Z","shell.execute_reply.started":"2026-02-19T23:00:30.411132Z","shell.execute_reply":"2026-02-19T23:00:30.418720Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# BEGGING QUESTION 1","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T23:00:31.833696Z","iopub.execute_input":"2026-02-19T23:00:31.834074Z","iopub.status.idle":"2026-02-19T23:00:31.839422Z","shell.execute_reply.started":"2026-02-19T23:00:31.834047Z","shell.execute_reply":"2026-02-19T23:00:31.838372Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\ntrain_path = \"/kaggle/input/bigdata2025classification/train.csv\"\ntest_path  = \"/kaggle/input/bigdata2025classification/test_without_labels.csv\"\n\ntrain_df = pd.read_csv(train_path, sep=\"|\")\ntest_df  = pd.read_csv(test_path,  sep=\"|\")\n\nprint(\"Train shape:\", train_df.shape)\nprint(\"Test shape:\", test_df.shape)\n\nprint(\"\\nTrain columns:\", train_df.columns.tolist())\nprint(\"Test columns:\", test_df.columns.tolist())\n\ntest_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T23:00:32.891864Z","iopub.execute_input":"2026-02-19T23:00:32.892211Z","iopub.status.idle":"2026-02-19T23:00:41.476393Z","shell.execute_reply.started":"2026-02-19T23:00:32.892185Z","shell.execute_reply":"2026-02-19T23:00:41.475359Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\ntrain_path = \"/kaggle/input/bigdata2025classification/train.csv\"\ntest_path  = \"/kaggle/input/bigdata2025classification/test_without_labels.csv\"\n\ntrain_df = pd.read_csv(train_path) \ntest_df  = pd.read_csv(test_path)\n\nprint(\"Train shape:\", train_df.shape)\nprint(\"Test shape:\", test_df.shape)\n\nprint(\"\\nTrain columns:\", train_df.columns.tolist())\nprint(\"Test columns:\", test_df.columns.tolist())\n\ntest_df.head(20)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T23:00:42.787165Z","iopub.execute_input":"2026-02-19T23:00:42.787552Z","iopub.status.idle":"2026-02-19T23:00:49.446328Z","shell.execute_reply.started":"2026-02-19T23:00:42.787526Z","shell.execute_reply":"2026-02-19T23:00:49.445162Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install wordcloud","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T23:00:59.258992Z","iopub.execute_input":"2026-02-19T23:00:59.260246Z","iopub.status.idle":"2026-02-19T23:01:05.418022Z","shell.execute_reply.started":"2026-02-19T23:00:59.260201Z","shell.execute_reply":"2026-02-19T23:01:05.416682Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from wordcloud import WordCloud, STOPWORDS\nimport matplotlib.pyplot as plt\n\nentertainment_df = train_df[train_df[\"Label\"] == \"Entertainment\"]\n\ntext_ent = \" \".join(\n    entertainment_df[\"Title\"].astype(str) + \" \" +\n    entertainment_df[\"Content\"].astype(str)\n)\n\n\n\nwc = WordCloud(\n    width=800,\n    height=400,\n    background_color=\"white\",\n    max_words=200,\n).generate(text_ent)\n\nplt.figure(figsize=(12,6))\nplt.imshow(wc)\nplt.axis(\"off\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T23:02:07.672724Z","iopub.execute_input":"2026-02-19T23:02:07.673104Z","iopub.status.idle":"2026-02-19T23:03:12.254923Z","shell.execute_reply.started":"2026-02-19T23:02:07.673076Z","shell.execute_reply":"2026-02-19T23:03:12.253802Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from wordcloud import WordCloud, STOPWORDS\nimport matplotlib.pyplot as plt\n\nentertainment_df = train_df[train_df[\"Label\"] == \"Entertainment\"]\n\ntext_ent = \" \".join(\n    entertainment_df[\"Title\"].astype(str) + \" \" +\n    entertainment_df[\"Content\"].astype(str)\n)\n\ncustom_stopwords = set(STOPWORDS)\ncustom_stopwords.update([\"said\"])\n\nwc = WordCloud(\n    width=800,\n    height=400,\n    background_color=\"white\",\n    max_words=200,\n    stopwords=custom_stopwords\n).generate(text_ent)\n\nplt.figure(figsize=(12,6))\nplt.imshow(wc)\nplt.axis(\"off\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T23:05:11.326652Z","iopub.execute_input":"2026-02-19T23:05:11.327021Z","iopub.status.idle":"2026-02-19T23:06:11.112474Z","shell.execute_reply.started":"2026-02-19T23:05:11.326994Z","shell.execute_reply":"2026-02-19T23:06:11.111441Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"business_df = train_df[train_df[\"Label\"] == \"Business\"]\ntext_bus = \" \".join(\n    business_df[\"Title\"].astype(str) + \" \" +\n    business_df[\"Content\"].astype(str)\n)\n\ncustom_stopwords = set(STOPWORDS)\ncustom_stopwords.update([\"said\", \"one\", \"s\", \"show\", \"say\"])\n\nwc = WordCloud(\n    width=800,\n    height=400,\n    background_color=\"white\",\n    max_words=200,\n    stopwords=custom_stopwords\n).generate(text_ent)\n\nplt.figure(figsize=(12,6))\nplt.imshow(wc)\nplt.axis(\"off\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T23:03:27.351219Z","iopub.execute_input":"2026-02-19T23:03:27.351608Z","iopub.status.idle":"2026-02-19T23:04:26.567154Z","shell.execute_reply.started":"2026-02-19T23:03:27.351582Z","shell.execute_reply":"2026-02-19T23:04:26.566134Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"technology_df = train_df[train_df[\"Label\"] == \"Technology\"]\ntext_bus = \" \".join(\n    technology_df[\"Title\"].astype(str) + \" \" +\n    technology_df[\"Content\"].astype(str)\n)\ncustom_stopwords = set(STOPWORDS)\ncustom_stopwords.add(\"said\")\nwc = WordCloud(width=800, height=400, background_color=\"white\",stopwords=custom_stopwords).generate(text_bus)\n\nplt.figure(figsize=(12,6))\nplt.imshow(wc)\nplt.axis(\"off\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T23:06:17.019695Z","iopub.execute_input":"2026-02-19T23:06:17.020885Z","iopub.status.idle":"2026-02-19T23:06:59.782886Z","shell.execute_reply.started":"2026-02-19T23:06:17.020846Z","shell.execute_reply":"2026-02-19T23:06:59.781713Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"health_df = train_df[train_df[\"Label\"] == \"Health\"]\ntext_bus = \" \".join(\n    health_df[\"Title\"].astype(str) + \" \" +\n    health_df[\"Content\"].astype(str)\n)\ncustom_stopwords = set(STOPWORDS)\ncustom_stopwords.add(\"said\")\nwc = WordCloud(width=800, height=400, background_color=\"white\",stopwords=custom_stopwords).generate(text_bus)\n\nplt.figure(figsize=(12,6))\nplt.imshow(wc)\nplt.axis(\"off\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T23:06:59.784262Z","iopub.execute_input":"2026-02-19T23:06:59.784861Z","iopub.status.idle":"2026-02-19T23:07:17.363774Z","shell.execute_reply.started":"2026-02-19T23:06:59.784827Z","shell.execute_reply":"2026-02-19T23:07:17.362754Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df[\"text\"] = (train_df[\"Title\"].fillna(\"\") + \" \" + train_df[\"Content\"].fillna(\"\")).astype(str)\ntest_df[\"text\"]  = (test_df[\"Title\"].fillna(\"\")  + \" \" + test_df[\"Content\"].fillna(\"\")).astype(str)\n\nX = train_df[\"text\"]\ny = train_df[\"Label\"]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T23:07:34.448585Z","iopub.execute_input":"2026-02-19T23:07:34.449490Z","iopub.status.idle":"2026-02-19T23:07:34.842268Z","shell.execute_reply.started":"2026-02-19T23:07:34.449459Z","shell.execute_reply":"2026-02-19T23:07:34.841283Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.pipeline import Pipeline\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.svm import LinearSVC\nfrom sklearn.model_selection import StratifiedKFold, cross_val_score\n\nsvm_bow = Pipeline([\n    (\"bow\", CountVectorizer(\n        lowercase=True,\n        stop_words=\"english\",\n        max_features=50000,  \n        ngram_range=(1, 1)    \n    )),\n    (\"clf\", LinearSVC(max_iter=5000, random_state=42))\n])\n\ncv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n\nsvm_scores = cross_val_score(\n    svm_bow,\n    X, y,\n    cv=cv,\n    scoring=\"accuracy\",\n    n_jobs=-1\n)\n\nprint(\"SVM (BoW) 5-fold accuracy:\")\nprint(\"Mean:\", svm_scores.mean())\nprint(\"Std :\", svm_scores.std())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T23:07:57.443740Z","iopub.execute_input":"2026-02-19T23:07:57.444115Z","iopub.status.idle":"2026-02-19T23:16:15.847972Z","shell.execute_reply.started":"2026-02-19T23:07:57.444088Z","shell.execute_reply":"2026-02-19T23:16:15.846811Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom collections import Counter\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.feature_extraction.text import CountVectorizer\n\nX_text = train_df[\"text\"].values\ny = train_df[\"Label\"].values\n\ndef knn_jaccard_predict(X_train_bin, y_train, X_val_bin, k=7, block_size=128):\n    \n    train_nnz = np.asarray(X_train_bin.getnnz(axis=1)).ravel()\n    val_nnz   = np.asarray(X_val_bin.getnnz(axis=1)).ravel()\n\n    preds = []\n    n_val = X_val_bin.shape[0]\n\n    for start in range(0, n_val, block_size):\n        end = min(start + block_size, n_val)\n        Xb = X_val_bin[start:end]\n\n        inter = (Xb @ X_train_bin.T).toarray().astype(np.float32)\n\n        union = (val_nnz[start:end, None] + train_nnz[None, :] - inter)\n        union[union == 0] = 1.0  \n        \n        sim = inter / union \n\n        for i in range(sim.shape[0]):\n            idx = np.argpartition(sim[i], -k)[-k:] \n            vote = Counter(y_train[idx]).most_common(1)[0][0]\n            preds.append(vote)\n\n    return np.array(preds)\n\n\nK = 7\nVEC_MAX_FEATURES = 20000\n\ncv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\nfold_acc = []\n\nfor fold, (tr_idx, va_idx) in enumerate(cv.split(X_text, y), start=1):\n    X_tr_text, X_va_text = X_text[tr_idx], X_text[va_idx]\n    y_tr, y_va = y[tr_idx], y[va_idx]\n\n    vec = CountVectorizer(\n        lowercase=True,\n        stop_words=\"english\",\n        binary=True,\n        max_features=VEC_MAX_FEATURES\n    )\n\n    X_tr = vec.fit_transform(X_tr_text)\n    X_va = vec.transform(X_va_text)\n\n    y_hat = knn_jaccard_predict(X_tr, y_tr, X_va, k=K, block_size=128)\n    acc = (y_hat == y_va).mean()\n    fold_acc.append(acc)\n\n    print(f\"Fold {fold}: accuracy={acc:.6f}\")\n\nfold_acc = np.array(fold_acc)\nprint(\"\\nKNN with Jaccard (binary BoW) 5-fold accuracy:\")\nprint(\"Mean:\", fold_acc.mean())\nprint(\"Std :\", fold_acc.std())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T23:16:25.072503Z","iopub.execute_input":"2026-02-19T23:16:25.072869Z","iopub.status.idle":"2026-02-19T23:40:16.773501Z","shell.execute_reply.started":"2026-02-19T23:16:25.072841Z","shell.execute_reply":"2026-02-19T23:40:16.772459Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.pipeline import Pipeline\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.svm import LinearSVC\nfrom sklearn.model_selection import StratifiedKFold, cross_val_score\n\ntfidf_svm = Pipeline([\n    (\"tfidf\", TfidfVectorizer(\n        lowercase=True,\n        stop_words=\"english\",\n        ngram_range=(1, 2),     \n        min_df=2,              \n        sublinear_tf=True,\n        max_features=100000\n    )),\n    (\"clf\", LinearSVC(max_iter=5000))\n])\n\ncv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n\ntfidf_scores = cross_val_score(\n    tfidf_svm,\n    X, y,\n    cv=cv,\n    scoring=\"accuracy\",\n    n_jobs=-1\n)\n\nprint(\"TF-IDF + SVM (5-fold) Accuracy:\")\nprint(\"Mean:\", tfidf_scores.mean())\nprint(\"Std :\", tfidf_scores.std())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T23:41:47.109626Z","iopub.execute_input":"2026-02-19T23:41:47.110066Z","iopub.status.idle":"2026-02-19T23:47:46.640994Z","shell.execute_reply.started":"2026-02-19T23:41:47.110040Z","shell.execute_reply":"2026-02-19T23:47:46.639932Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\nbest_model = tfidf_svm\n\nbest_model.fit(X, y)\n\ntest_predictions = best_model.predict(test_df[\"text\"])\n\nsubmission = pd.DataFrame({\n    \"Id\": test_df[\"Id\"],\n    \"Predicted\": test_predictions\n})\n\nsubmission.to_csv(\"testSet_categories.csv\", index=False)\n\nsubmission.head(10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T23:47:56.170712Z","iopub.execute_input":"2026-02-19T23:47:56.171755Z","iopub.status.idle":"2026-02-19T23:51:07.401202Z","shell.execute_reply.started":"2026-02-19T23:47:56.171705Z","shell.execute_reply":"2026-02-19T23:51:07.400103Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#BEGINNING QUESTION 2 ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T22:20:55.159765Z","iopub.execute_input":"2026-02-19T22:20:55.160342Z","iopub.status.idle":"2026-02-19T22:20:55.165877Z","shell.execute_reply.started":"2026-02-19T22:20:55.160283Z","shell.execute_reply":"2026-02-19T22:20:55.164605Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install datasketch","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T00:28:36.453027Z","iopub.execute_input":"2026-02-20T00:28:36.453537Z","iopub.status.idle":"2026-02-20T00:28:49.548428Z","shell.execute_reply.started":"2026-02-20T00:28:36.453505Z","shell.execute_reply":"2026-02-20T00:28:49.547102Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import time\nimport re\nfrom collections import Counter, defaultdict\nfrom dataclasses import dataclass\nfrom typing import List, Set, Dict, Tuple, Optional\n\nimport pandas as pd\nfrom datasketch import MinHash, MinHashLSH\nfrom tqdm import tqdm","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T00:29:02.326506Z","iopub.execute_input":"2026-02-20T00:29:02.326898Z","iopub.status.idle":"2026-02-20T00:29:04.373930Z","shell.execute_reply.started":"2026-02-20T00:29:02.326866Z","shell.execute_reply":"2026-02-20T00:29:04.372877Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import time, re #Μέτρηση Built/Query Χρόνων και Tokenization\nfrom collections import Counter\nfrom datasketch import MinHash, MinHashLSH\nfrom tqdm.auto import tqdm #Γραμμές Προόδου\nimport pandas as pd\n\nTRAIN_LIMIT = 20000 #Για Ταχύτερο Πείραμα\nTEST_LIMIT  = 2000 #Για Ταχύτερο Πείραμα\n\nEVAL_N_TEST  = 2000 #Αξιολόγηση Recall\nEVAL_M_TRAIN = 20000 # Για Brute Force Επίπεδο Αναφοράς\nWORD_NGRAM      = 1 # Unigrams\nK               = 7 #Top 7 όμοιους γείτονες \nPERMS_LIST      = [16, 32, 64] #Μεταθέσεις για MinHash\n\nLSH_R           = 3 # r = Rows Per Band (Banding Trick του LSH)\n\nMIN_CANDIDATES_FOR_LSH = 30 #Αν LSH Επιστρέψει <30 Υποψήφιους, Αναξιόπιστο, Οπότε Fallback\n\n# Στο Fallback, Brute-Force (First FALLBACK_M Train Docs)\nFALLBACK_M = 3000\n\nID_COL = \"Id\"\nTITLE_COL = \"Title\"\nCONTENT_COL = \"Content\"\nLABEL_COL = \"Label\"\n\n#Έλεγχος Για Σωστές Στήλες\n\nmissing_train = {ID_COL, TITLE_COL, CONTENT_COL, LABEL_COL} - set(train_df.columns)\nmissing_test  = {ID_COL, TITLE_COL, CONTENT_COL} - set(test_df.columns)\nif missing_train:\n    raise ValueError(f\"Train is missing columns: {missing_train}. Found: {train_df.columns.tolist()}\")\nif missing_test:\n    raise ValueError(f\"Test is missing columns: {missing_test}. Found: {test_df.columns.tolist()}\")\n\n# Αποφυγή Warnings\ntrain_use = train_df.head(TRAIN_LIMIT).copy() if TRAIN_LIMIT is not None else train_df\ntest_use  = test_df.head(TEST_LIMIT).copy() if TEST_LIMIT is not None else test_df\nprint(\"Using TRAIN:\", train_use.shape, \"| Using TEST:\", test_use.shape)\n\n# Tokenization to Shingles\n\n_token_re = re.compile(r\"[A-Za-z0-9]+\", re.UNICODE) # Regex Aλφαριθμητικά\n\ndef tokenize(text: str):\n    if not isinstance(text, str):\n        return []\n    return _token_re.findall(text.lower())\n\ndef word_ngrams(tokens, n):\n    if n <= 1:\n        return set(tokens)\n    if len(tokens) < n:\n        return set([\" \".join(tokens)]) if tokens else set()\n    return set(\" \".join(tokens[i:i+n]) for i in range(len(tokens) - n + 1))\n\ndef doc_to_shingles(title, content, n=WORD_NGRAM):\n    tokens = tokenize((title or \"\") + \" \" + (content or \"\"))\n    return word_ngrams(tokens, n)\n\ndef jaccard(a: set, b: set) -> float:\n    if not a and not b:\n        return 1.0\n    if not a or not b:\n        return 0.0\n    inter = len(a & b)\n    union = len(a) + len(b) - inter\n    return inter / union if union else 0.0\n\ndef make_minhash(shingles: set, num_perm: int) -> MinHash:\n    mh = MinHash(num_perm=num_perm)\n    for s in shingles:\n        mh.update(s.encode(\"utf-8\"))\n    return mh\n\ndef majority_vote(labels):\n    if not labels:\n        return None\n    c = Counter(labels)\n    return sorted(c.items(), key=lambda x: (-x[1], x[0]))[0][0]\n\n# IDs, Labels ΣΕ String Σε Λίστες\nprint(\"Building shingles for TRAIN...\")\ntrain_ids = train_use[ID_COL].astype(str).tolist()\ntrain_labels = train_use[LABEL_COL].astype(str).tolist()\ntrain_shingles = [\n    doc_to_shingles(t, c, WORD_NGRAM)\n    for t, c in tqdm(zip(train_use[TITLE_COL], train_use[CONTENT_COL]), total=len(train_use))\n]\ntrain_sets_by_id = {tid: sh for tid, sh in zip(train_ids, train_shingles)}\ntrain_label_by_id = {tid: lab for tid, lab in zip(train_ids, train_labels)}\n\nprint(\"Building shingles for TEST...\")\ntest_ids = test_use[ID_COL].astype(str).tolist()\ntest_shingles = [\n    doc_to_shingles(t, c, WORD_NGRAM)\n    for t, c in tqdm(zip(test_use[TITLE_COL], test_use[CONTENT_COL]), total=len(test_use))\n]\n\n# Check Μεγέθη Shingles\ntrain_sizes = [len(s) for s in train_shingles[:2000]]\ntest_sizes  = [len(s) for s in test_shingles[:2000]]\n\n# Έλεγχος Για Κενά Docs\nprint(\"\\nShingle set sizes (first up to 2000 docs):\")\nprint(\"TRAIN min/med/max:\", min(train_sizes), sorted(train_sizes)[len(train_sizes)//2], max(train_sizes),\n      \"| empty:\", sum(1 for x in train_sizes if x == 0))\nprint(\"TEST  min/med/max:\", min(test_sizes),  sorted(test_sizes)[len(test_sizes)//2],  max(test_sizes),\n      \"| empty:\", sum(1 for x in test_sizes if x == 0))\n\n# Αν Υπάρξουν Διπλά IDs\nif len(train_ids) != len(set(train_ids)):\n    raise ValueError(\"Duplicate train IDs detected (breaks dict lookups).\")\n\n# Brute Force TopK\ndef brute_force_topk_ids(test_sh, train_shingles_list, train_ids_list, k, m_train=None):\n    iterable = enumerate(train_shingles_list if m_train is None else train_shingles_list[:m_train])\n    sims = []\n    for i, tr_sh in iterable:\n        sims.append((jaccard(test_sh, tr_sh), i))\n    sims.sort(reverse=True, key=lambda x: x[0])\n    top_idx = [idx for _, idx in sims[:k]]\n    return [train_ids_list[i] for i in top_idx]\n\n# LSH TopK\ndef lsh_topk_ids_from_candidates(test_sh, cand_ids, k):\n    if not cand_ids:\n        return []\n    sims = [(jaccard(test_sh, train_sets_by_id[cid]), cid) for cid in cand_ids]\n    sims.sort(reverse=True, key=lambda x: x[0])\n    return [cid for _, cid in sims[:k]]\n\n# Επιλογή Δεικτών Για Εκτίμηση\neval_n = min(EVAL_N_TEST, len(test_shingles))\neval_indices = list(range(eval_n))\n\n# Brute-force Τiming\nt_bf0 = time.perf_counter()\nfor ei in tqdm(eval_indices, desc=\"Brute-force baseline timing\"):\n    _ = brute_force_topk_ids(test_shingles[ei], train_shingles, train_ids, K, m_train=EVAL_M_TRAIN)\nbf_query_time = time.perf_counter() - t_bf0\n\n# Για Δημιουργία Πίνακα Αποτελεσμάτων\nrows = []\n\n# Αποθήκευση Προβλέψεων Ανά num_perm\npredictions_by_perm = {}\n\nrows.append({\n    \"Type\": \"Brute-Force-Jaccard\",\n    \"BuildTime(s)\": 0.0,\n    \"QueryTime(s)\": round(bf_query_time, 3),\n    \"TotalTime(s)\": round(bf_query_time, 3),\n    \"fraction_true_topK_reported\": 1.0,\n    \"fallback_rate\": 0.0,\n    \"cand_min_med_max\": \"N/A\",\n    \"Parameters\": f\"K={K}, ngram={WORD_NGRAM}, eval_test={eval_n}, eval_train={EVAL_M_TRAIN or 'ALL'}\"\n})\n\n# Πειράματα Για Κάθε num_perm\nfor num_perm in PERMS_LIST:\n    r = LSH_R\n    b = max(2, num_perm // r)\n    # Εκτύπωση Αποτελέσματος\n    print(f\"\\n=== MinHashLSH: num_perm={num_perm}, (b,r)=({b},{r}), K={K}, ngram={WORD_NGRAM} ===\")\n    \n    # Δημιουργία LSH Δείκτη\n    t0 = time.perf_counter()\n    lsh = MinHashLSH(num_perm=num_perm, params=(b, r))\n    with lsh.insertion_session() as session:\n        for tid, sh in tqdm(zip(train_ids, train_shingles), total=len(train_ids), desc=\"LSH insert\"):\n            session.insert(tid, make_minhash(sh, num_perm))\n    build_time = time.perf_counter() - t0\n    \n    # Query Και Classify\n    t1 = time.perf_counter()\n    preds = []\n    fallback_count = 0\n    cand_sizes = []\n\n    for sh in tqdm(test_shingles, desc=\"LSH query+classify\"):\n        mh = make_minhash(sh, num_perm)\n        cand_ids = lsh.query(mh)\n        cand_sizes.append(len(cand_ids))\n        # Fallback Αν Μικρός Αριθμός Υποψηφίων\n        if len(cand_ids) < MIN_CANDIDATES_FOR_LSH:\n            fallback_count += 1\n            nn_ids = brute_force_topk_ids(sh, train_shingles, train_ids, K, m_train=FALLBACK_M)\n        else:\n            nn_ids = lsh_topk_ids_from_candidates(sh, cand_ids, K)\n\n        nn_labels = [train_label_by_id[i] for i in nn_ids]\n        preds.append(majority_vote(nn_labels))\n\n    query_time = time.perf_counter() - t1\n    total_time = build_time + query_time\n    predictions_by_perm[num_perm] = preds\n\n    # Στατιστικά Υποψηφίων Και Ρυθμός Fallback\n    cand_sorted = sorted(cand_sizes)\n    cand_min = cand_sorted[0] if cand_sorted else 0\n    cand_med = cand_sorted[len(cand_sorted)//2] if cand_sorted else 0\n    cand_max = cand_sorted[-1] if cand_sorted else 0\n    cand_summary = f\"{cand_min}/{cand_med}/{cand_max}\"\n    fallback_rate = fallback_count / len(test_shingles) if len(test_shingles) else 0.0\n\n    print(f\"Candidate sizes min/med/max: {cand_summary}\")\n    print(f\"Fallback used: {fallback_count}/{len(test_shingles)} ({fallback_rate:.2%})\")\n\n    # Recall@K Εκτίμηση vs Brute-Force\n    hits = 0\n    total = 0\n    for ei in tqdm(eval_indices, desc=\"Recall@K eval\"):\n        true_ids = set(brute_force_topk_ids(test_shingles[ei], train_shingles, train_ids, K, m_train=EVAL_M_TRAIN))\n\n        mh = make_minhash(test_shingles[ei], num_perm)\n        cand_ids = lsh.query(mh)\n       \n        approx_ids = set(lsh_topk_ids_from_candidates(test_shingles[ei], cand_ids, K))\n\n        hits += len(true_ids & approx_ids)\n        total += len(true_ids)  # should be K\n    recall_at_k = hits / total if total else 0.0\n\n    rows.append({\n        \"Type\": \"LSH-Jaccard (looser)\",\n        \"BuildTime(s)\": round(build_time, 3),\n        \"QueryTime(s)\": round(query_time, 3),\n        \"TotalTime(s)\": round(total_time, 3),\n        \"fraction_true_topK_reported\": round(recall_at_k, 4),\n        \"fallback_rate\": round(fallback_rate, 4),\n        \"cand_min_med_max\": cand_summary,\n        \"Parameters\": (\n            f\"Perm={num_perm}, (b,r)=({b},{r}), K={K}, ngram={WORD_NGRAM}, \"\n            f\"MIN_CAND={MIN_CANDIDATES_FOR_LSH}, FALLBACK_M={FALLBACK_M}, \"\n            f\"TRAIN_LIMIT={TRAIN_LIMIT}, TEST_LIMIT={TEST_LIMIT}, eval_test={eval_n}, eval_train={EVAL_M_TRAIN or 'ALL'}\"\n        )\n    })\n\nresults_df = pd.DataFrame(rows)\ndisplay(results_df)\n\nfor perm in PERMS_LIST:\n    print(f\"\\nSample predictions (perm={perm}):\")\n    print(predictions_by_perm[perm][:20])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T00:29:10.730233Z","iopub.execute_input":"2026-02-20T00:29:10.731258Z","iopub.status.idle":"2026-02-20T01:05:49.524904Z","shell.execute_reply.started":"2026-02-20T00:29:10.731230Z","shell.execute_reply":"2026-02-20T01:05:49.523829Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# END QUESTION 2","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#BEGINNING QUESTION 3 ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T23:51:17.787718Z","iopub.execute_input":"2026-02-19T23:51:17.788983Z","iopub.status.idle":"2026-02-19T23:51:17.795175Z","shell.execute_reply.started":"2026-02-19T23:51:17.788941Z","shell.execute_reply":"2026-02-19T23:51:17.794053Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport ast\n\ndtw_path = \"/kaggle/input/dtw-test/dtw_test.csv\"\n\ndf = pd.read_csv(dtw_path)\n\nprint(\"Shape:\", df.shape)\nprint(df.head())\n\ndf[\"series_a\"] = df[\"series_a\"].apply(ast.literal_eval)\ndf[\"series_b\"] = df[\"series_b\"].apply(ast.literal_eval)\n\nprint(\"\\nAfter parsing:\")\nprint(type(df[\"series_a\"].iloc[0]), type(df[\"series_a\"].iloc[0][0]))\nprint(\"Length series_a[0]:\", len(df[\"series_a\"].iloc[0]))\nprint(\"Length series_b[0]:\", len(df[\"series_b\"].iloc[0]))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T23:51:19.561765Z","iopub.execute_input":"2026-02-19T23:51:19.562126Z","iopub.status.idle":"2026-02-19T23:51:26.956045Z","shell.execute_reply.started":"2026-02-19T23:51:19.562101Z","shell.execute_reply":"2026-02-19T23:51:26.955163Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import math\n\ndef dtw_distance(a, b):\n    n, m = len(a), len(b)\n    INF = float(\"inf\")\n\n    prev = [INF] * (m + 1)\n    curr = [INF] * (m + 1)\n    prev[0] = 0.0\n\n    for i in range(1, n + 1):\n        curr[0] = INF\n        ai = a[i - 1]\n        for j in range(1, m + 1):\n            bj = b[j - 1]\n            cost = abs(ai - bj)  \n            curr[j] = cost + min(prev[j],      \n                                 curr[j - 1], \n                                 prev[j - 1])  \n        prev, curr = curr, prev\n\n    return prev[m]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T23:51:31.319513Z","iopub.execute_input":"2026-02-19T23:51:31.319865Z","iopub.status.idle":"2026-02-19T23:51:31.328017Z","shell.execute_reply.started":"2026-02-19T23:51:31.319844Z","shell.execute_reply":"2026-02-19T23:51:31.326966Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"a0 = df[\"series_a\"].iloc[0]\nb0 = df[\"series_b\"].iloc[0]\nprint(\"DTW(0):\", dtw_distance(a0, b0))\n\na1 = df[\"series_a\"].iloc[1]\nb1 = df[\"series_b\"].iloc[1]\nprint(\"DTW(1):\", dtw_distance(a1, b1))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T23:51:32.731601Z","iopub.execute_input":"2026-02-19T23:51:32.731942Z","iopub.status.idle":"2026-02-19T23:51:33.172186Z","shell.execute_reply.started":"2026-02-19T23:51:32.731918Z","shell.execute_reply":"2026-02-19T23:51:33.171381Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import time\nimport pandas as pd\n\nt0 = time.perf_counter()\n\ndistances = []\nfor _, row in df.iterrows():\n    d = dtw_distance(row[\"series_a\"], row[\"series_b\"])\n    distances.append(d)\n\ntotal_time = time.perf_counter() - t0\nprint(f\"Total DTW time for {len(df)} pairs: {total_time:.3f} sec\")\n\nout = pd.DataFrame({\n    \"id\": df[\"id\"],\n    \"DTW distance\": distances\n})\n\nout.to_csv(\"dtw.csv\", index=False)\nout.head(10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T23:51:35.442233Z","iopub.execute_input":"2026-02-19T23:51:35.443197Z","iopub.status.idle":"2026-02-19T23:54:52.778422Z","shell.execute_reply.started":"2026-02-19T23:51:35.443161Z","shell.execute_reply":"2026-02-19T23:54:52.777455Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# END QUESTION 3","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-19T23:55:02.529143Z","iopub.execute_input":"2026-02-19T23:55:02.530825Z","iopub.status.idle":"2026-02-19T23:55:02.536178Z","shell.execute_reply.started":"2026-02-19T23:55:02.530755Z","shell.execute_reply":"2026-02-19T23:55:02.534787Z"}},"outputs":[],"execution_count":null}]}