{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3539,"databundleVersionId":44369,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install iterative-stratification","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-26T13:52:08.383909Z","iopub.execute_input":"2025-08-26T13:52:08.384717Z","iopub.status.idle":"2025-08-26T13:52:12.270921Z","shell.execute_reply.started":"2025-08-26T13:52:08.384691Z","shell.execute_reply":"2025-08-26T13:52:12.269889Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport re\nimport html\nimport gc\nimport zipfile\nimport tempfile\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom scipy.sparse import hstack\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.preprocessing import MultiLabelBinarizer\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.multiclass import OneVsRestClassifier\nfrom sklearn.metrics import f1_score\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold\n\n# ------------------------\n# Safe, memory-oriented config\n# ------------------------\nRANDOM_STATE = 42\nN_SPLITS = 5\nWORD_MAX_FEATURES = 5000    # safe for Kaggle/Colab\nCHAR_MAX_FEATURES = 2000\nC_VALUES = [0.5, 1.0, 2.0]\nMAX_ITER = 50\nTHRESHOLDS = np.linspace(0.3, 0.7, 9)  # sweep range\nBATCH_SIZE_TEST = 4096  # test transform batch size\n\n# ------------------------\n# Utilities\n# ------------------------\ndef clean_text(text):\n    text = html.unescape(str(text))\n    text = re.sub(r'<code>.*?</code>', ' ', text, flags=re.DOTALL)  # drop very long code blocks\n    text = re.sub(r'<.*?>', ' ', text)\n    text = text.lower()\n    text = re.sub(r'\\s+', ' ', text).strip()\n    return text\n\ndef iter_batches(indices, batch_size):\n    for i in range(0, len(indices), batch_size):\n        yield indices[i:i+batch_size]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-26T13:52:19.735803Z","iopub.execute_input":"2025-08-26T13:52:19.736137Z","iopub.status.idle":"2025-08-26T13:52:19.744981Z","shell.execute_reply.started":"2025-08-26T13:52:19.736113Z","shell.execute_reply":"2025-08-26T13:52:19.743908Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ------------------------\n# Load and preprocess raw data\n# ------------------------\nwith zipfile.ZipFile('/kaggle/input/facebook-recruiting-iii-keyword-extraction/Train.zip', 'r') as z:\n    z.extractall('.')\ntrain_df = pd.read_csv(\"Train.csv\")\ntrain_df.dropna(subset=[\"Title\", \"Body\", \"Tags\"])\ntrain_df.reset_index(drop=True, inplace = True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-26T14:27:39.764500Z","iopub.execute_input":"2025-08-26T14:27:39.765747Z","iopub.status.idle":"2025-08-26T14:31:29.007456Z","shell.execute_reply.started":"2025-08-26T14:27:39.765633Z","shell.execute_reply":"2025-08-26T14:31:29.006154Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# get duplicates\ntrain_df_dups = train_df[train_df.duplicated(['Title', 'Body', 'Tags'])]\nprint('Total Duplicates: ', len(train_df_dups))\nprint('ratio: ', len(train_df_dups)/len(train_df))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-26T14:32:55.198477Z","iopub.execute_input":"2025-08-26T14:32:55.199387Z","iopub.status.idle":"2025-08-26T14:33:33.382382Z","shell.execute_reply.started":"2025-08-26T14:32:55.199355Z","shell.execute_reply":"2025-08-26T14:33:33.381356Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# remove duplicates\ntrain_df = train_df.drop_duplicates(['Title', 'Body', 'Tags'])\nprint('After removing dups: ', len(train_df))\nprint('ratio: ', len(train_df)/6034194)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-26T14:36:36.543961Z","iopub.execute_input":"2025-08-26T14:36:36.545811Z","iopub.status.idle":"2025-08-26T14:37:14.456596Z","shell.execute_reply.started":"2025-08-26T14:36:36.545754Z","shell.execute_reply":"2025-08-26T14:37:14.455597Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-26T14:41:16.706496Z","iopub.execute_input":"2025-08-26T14:41:16.707294Z","iopub.status.idle":"2025-08-26T14:41:16.717443Z","shell.execute_reply.started":"2025-08-26T14:41:16.707264Z","shell.execute_reply":"2025-08-26T14:41:16.716123Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.reset_index(drop = True, inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-26T14:41:08.750398Z","iopub.execute_input":"2025-08-26T14:41:08.751371Z","iopub.status.idle":"2025-08-26T14:41:08.756018Z","shell.execute_reply.started":"2025-08-26T14:41:08.751343Z","shell.execute_reply":"2025-08-26T14:41:08.754833Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# combine and clean\ntrain_df['CleanText'] = (train_df['Title'].fillna('') + ' ' + train_df['Body'].fillna('')).map(clean_text)\ntrain_df['TagList'] = train_df['Tags'].str.split()\nn_samples = len(train_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-26T14:42:35.490281Z","iopub.execute_input":"2025-08-26T14:42:35.490690Z","iopub.status.idle":"2025-08-26T14:47:10.028256Z","shell.execute_reply.started":"2025-08-26T14:42:35.490663Z","shell.execute_reply":"2025-08-26T14:47:10.027219Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.isnull().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-26T14:52:08.683473Z","iopub.execute_input":"2025-08-26T14:52:08.683852Z","iopub.status.idle":"2025-08-26T14:52:11.247013Z","shell.execute_reply.started":"2025-08-26T14:52:08.683826Z","shell.execute_reply":"2025-08-26T14:52:11.245977Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.dropna(inplace = True)\ntrain_df.reset_index(drop=True, inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-26T14:56:26.792302Z","iopub.execute_input":"2025-08-26T14:56:26.792788Z","iopub.status.idle":"2025-08-26T14:56:30.113398Z","shell.execute_reply.started":"2025-08-26T14:56:26.792759Z","shell.execute_reply":"2025-08-26T14:56:30.112322Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# label binarizer\nmlb = MultiLabelBinarizer()\nY = mlb.fit_transform(train_df['TagList'])  # keep in memory; typically smaller than feature matrices\nn_labels = Y.shape[1]\n\n# free raw text columns if you want later; keep CleanText for transforms\ndel train_df['Title'], train_df['Body'], train_df['Tags']\n\ngc.collect()\n\n# ------------------------\n# Fit TF-IDF vectorizers once on the corpus\n# Only fit, do not transform entire corpus to sparse matrix at once.\n# ------------------------\nword_vec = TfidfVectorizer(max_features=WORD_MAX_FEATURES, ngram_range=(1,3), dtype=np.float32)\nchar_vec = TfidfVectorizer(analyzer='char', ngram_range=(3,6), max_features=CHAR_MAX_FEATURES, dtype=np.float32)\n\n# fit using generator to avoid extra lists (vectorizer accepts any iterable)\nword_vec.fit(train_df['CleanText'])\nchar_vec.fit(train_df['CleanText'])\ngc.collect()\n\n# ------------------------\n# Prepare memmap file for out-of-fold probabilities\n# store as float32 on disk to avoid large in-RAM array\n# ------------------------\ntmpdir = tempfile.mkdtemp()\noof_path = os.path.join(tmpdir, 'oof_probs.dat')\noof_shape = (n_samples, n_labels)\noof_probs = np.memmap(oof_path, dtype='float32', mode='w+', shape=oof_shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-26T15:21:43.719801Z","iopub.execute_input":"2025-08-26T15:21:43.720240Z","execution_failed":"2025-08-26T15:22:28.354Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ------------------------\n# Cross validation: compute OOF probabilities for each C sequentially,\n# but keep only the best C and threshold. For memory efficiency we compute\n# fold transforms and fit per fold.\n# ------------------------\nmskf = MultilabelStratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=RANDOM_STATE)\n\nbest_score = -1.0\nbest_C = None\nbest_thr = None\n\n# To avoid keeping all fold predictions for each C in memory, create a temporary memmap per C\nfor C in C_VALUES:\n    print(f\"Evaluating C={C}\")\n    # temp memmap for this C\n    c_oof_path = os.path.join(tmpdir, f'oof_C{str(C).replace(\".\",\"p\")}.dat')\n    c_oof = np.memmap(c_oof_path, dtype='float32', mode='w+', shape=oof_shape)\n\n    # iterate folds sequentially\n    for fold, (train_idx, val_idx) in enumerate(mskf.split(np.zeros(n_samples), Y)):\n        print(\" fold\", fold)\n        # transform train split in memory-efficient way\n        X_train_word = word_vec.transform(train_df['CleanText'].iloc[train_idx])\n        X_train_char = char_vec.transform(train_df['CleanText'].iloc[train_idx])\n        X_train = hstack([X_train_word, X_train_char]).tocsr()\n        del X_train_word, X_train_char\n        gc.collect()\n\n        X_val_word = word_vec.transform(train_df['CleanText'].iloc[val_idx])\n        X_val_char = char_vec.transform(train_df['CleanText'].iloc[val_idx])\n        X_val = hstack([X_val_word, X_val_char]).tocsr()\n        del X_val_word, X_val_char\n        gc.collect()\n\n        y_train = Y[train_idx]\n        # fit One-vs-Rest logistic regression for this fold\n        clf = OneVsRestClassifier(LogisticRegression(\n            C=C, solver='saga', penalty='l2', max_iter=MAX_ITER, n_jobs=1\n        ), n_jobs=1)\n        clf.fit(X_train, y_train)\n\n        # predict probabilities for validation fold in batches if needed\n        # predict_proba on the sparse X_val is ok; but keep memory low\n        probs_val = clf.predict_proba(X_val)  # shape (n_val, n_labels)\n        # write into memmap\n        c_oof[val_idx, :] = probs_val.astype('float32')\n\n        # free fold memory\n        del X_train, X_val, y_train, probs_val, clf\n        gc.collect()\n\n    # after all folds for this C are done, sweep thresholds using c_oof\n    # load into working memory fold-by-fold only when evaluating thresholds to save peak memory.\n    # For simplicity we will read the whole memmap row by row in batches to compute scores.\n    # Build predictions for each threshold incrementally to compute f1.\n    best_local_score = -1.0\n    best_local_thr = None\n\n    for thr in THRESHOLDS:\n        # compute f1 score in streaming way to avoid building full binary matrix\n        # accumulate true positives, predicted positives, etc is complex for multilabel F1.\n        # here we will accumulate predictions in batches and compute f1 using sklearn on each batch,\n        # but we will accumulate row-wise predictions to a temp memmap to allow f1_score on full dataset.\n        pred_path = os.path.join(tmpdir, f'pred_C{str(C).replace(\".\",\"p\")}_t{str(thr).replace(\".\",\"p\")}.dat')\n        preds_mem = np.memmap(pred_path, dtype='uint8', mode='w+', shape=oof_shape)\n        # fill preds_mem in batches\n        for i in range(0, n_samples, BATCH_SIZE_TEST):\n            block = c_oof[i:i+BATCH_SIZE_TEST]\n            block_bin = (block >= thr).astype('uint8')\n            preds_mem[i:i+BATCH_SIZE_TEST, :] = block_bin\n        # compute f1 on full dataset using memmap slices to reduce peak RAM\n        # read back in chunks and accumulate for sklearn\n        # sklearn expects full arrays; to avoid huge memory we will load preds_mem fully as uint8 (smaller)\n        preds_full = np.asarray(preds_mem)  # should be smaller; still watch memory\n        score = f1_score(Y, preds_full, average='samples', zero_division=0)\n        # clean up\n        del preds_mem, preds_full\n        os.remove(pred_path)\n        gc.collect()\n        if score > best_local_score:\n            best_local_score = score\n            best_local_thr = thr\n\n    print(f\"  best thr for C={C}: {best_local_thr} score={best_local_score:.6f}\")\n\n    # update global best if improved\n    if best_local_score > best_score:\n        best_score = best_local_score\n        best_C = C\n        best_thr = best_local_thr\n\n    # copy c_oof into main oof memmap only if best so far to avoid extra writes\n    # we will store final OOF probabilities for best_C later by recomputing if needed.\n    # remove c_oof to free disk memory\n    del c_oof\n    os.remove(c_oof_path)\n    gc.collect()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Best C, threshold:\", best_C, best_thr, \"F1:\", best_score)\n\n# ------------------------\n# Final: retrain on full training set with best_C and produce test predictions\n# ------------------------\n# Fit final classifier on full training data transformed in batches if necessary\n# Transform full training data once if it fits; otherwise train using liblinear is not incremental.\n# Here we transform full training data once; for very tight RAM you can use partial training strategies.\nX_word_full = word_vec.transform(train_df['CleanText'])\nX_char_full = char_vec.transform(train_df['CleanText'])\nX_full = hstack([X_word_full, X_char_full]).tocsr()\ndel X_word_full, X_char_full\ngc.collect()\n\nfinal_clf = OneVsRestClassifier(LogisticRegression(\n    C=best_C, solver='saga', penalty='l2', max_iter=MAX_ITER, n_jobs=1\n), n_jobs=1)\nfinal_clf.fit(X_full, Y)\ngc.collect()\n\n# ------------------------\n# Test processing: transform in batches, predict probabilities in batches, apply threshold\n# ------------------------\nwith zipfile.ZipFile('/kaggle/input/facebook-recruiting-iii-keyword-extraction/Test.zip', 'r') as z:\n    z.extractall('.')\ntest_df = pd.read_csv(\"Test.csv\").reset_index(drop=True)\ntest_df['CleanText'] = (test_df['Title'].fillna('') + ' ' + test_df['Body'].fillna('')).map(clean_text)\n\nn_test = len(test_df)\npreds_rows = []\n\nfor batch_idx in range(0, n_test, BATCH_SIZE_TEST):\n    ids = list(range(batch_idx, min(batch_idx + BATCH_SIZE_TEST, n_test)))\n    Xw = word_vec.transform(test_df['CleanText'].iloc[ids])\n    Xc = char_vec.transform(test_df['CleanText'].iloc[ids])\n    Xb = hstack([Xw, Xc]).tocsr()\n    probs = final_clf.predict_proba(Xb).astype('float32')\n    bins = (probs >= best_thr).astype('int8')\n    # convert each row to tag list\n    for row in bins:\n        labels = [mlb.classes_[i] for i, v in enumerate(row) if v]\n        preds_rows.append(\" \".join(labels))\n    del Xw, Xc, Xb, probs, bins\n    gc.collect()\n\ntest_df['Tags'] = preds_rows\nsubmission = test_df[['Id', 'Tags']]\nsubmission.to_csv('submission.csv', index=False)\nprint(\"Saved submission.csv\")\n\n# ------------------------\n# Clean up\n# ------------------------\ndel X_full, final_clf, train_df, test_df, oof_probs\ngc.collect()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}