{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3539,"databundleVersionId":44369,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# import all the necessary libraries.\nimport pandas as pd\nimport numpy as np\nimport re\nimport html\nfrom tqdm.notebook import tqdm\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold\n\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.preprocessing import MultiLabelBinarizer\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.multiclass import OneVsRestClassifier\nfrom sklearn.metrics import f1_score\n\nfrom scipy.sparse import hstack\n\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-26T09:14:20.043858Z","iopub.execute_input":"2025-08-26T09:14:20.044203Z","iopub.status.idle":"2025-08-26T09:14:21.795229Z","shell.execute_reply.started":"2025-08-26T09:14:20.044149Z","shell.execute_reply":"2025-08-26T09:14:21.792497Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pd.read_csv(\"/kaggle/input/facebook-recruiting-iii-keyword-extraction/Train.zip\")\ntest_df = pd.read_csv(\"/kaggle/input/facebook-recruiting-iii-keyword-extraction/Test.zip\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-26T09:14:29.767581Z","iopub.execute_input":"2025-08-26T09:14:29.768199Z","iopub.status.idle":"2025-08-26T09:18:53.089057Z","shell.execute_reply.started":"2025-08-26T09:14:29.768141Z","shell.execute_reply":"2025-08-26T09:18:53.087951Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df['FullText'] = train_df['Title'] + ' ' + train_df['Body']\ntest_df['FullText'] = test_df['Title'] + ' ' + test_df['Body']\n\nprint(\"Combined 'FullText' column created.\")\ntrain_df[['Title', 'Body', 'FullText']].head(2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-26T09:19:08.664732Z","iopub.execute_input":"2025-08-26T09:19:08.665085Z","execution_failed":"2025-08-26T09:19:28.333Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def clean_text(text):\n    \"\"\"\n    Cleans text by:\n    - Unescaping HTML\n    - Removing HTML tags\n    - Removing code blocks (while trying to keep inline code identifiers)\n    - Converting to lowercase\n    - Collapsing extra spaces\n    \"\"\"\n    # Unescape HTML entities\n    text = html.unescape(text)\n    \n    # Remove HTML tags\n    text = re.sub(r'<.*?>', ' ', text)\n    \n    # Discard very long code blocks (simplified: remove content between <code> tags)\n    text = re.sub(r'<code>(.*?)</code>', ' ', text, flags=re.DOTALL)\n    text = text.lower()\n    \n    # Collapse extra spaces\n    text = re.sub(r'\\s+', ' ', text).strip()\n    \n    return text\n\n# Apply the cleaning function to both train and test data\ntqdm.pandas(desc=\"Cleaning Train Text\")\ntrain_df['CleanText'] = train_df['FullText'].progress_apply(clean_text)\n\ntqdm.pandas(desc=\"Cleaning Test Text\")\ntest_df['CleanText'] = test_df['FullText'].progress_apply(clean_text)\n\n\ntrain_df[['FullText', 'CleanText']].head(2)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df['TagList'] = train_df['Tags'].str.split()\ntrain_df[['Tags', 'TagList']].head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"This is usually where the memory gets filled up compeletly. Sometimes, it crashes before even loading the full dataset.","metadata":{}},{"cell_type":"code","source":"\n\nN_SPLITS = 5\n\n# First, multi-hot encoded representation of the tags for the splitter\nmlb_for_split = MultiLabelBinarizer()\ny_multihot = mlb_for_split.fit_transform(train_df['TagList'])\n\nmskf = MultilabelStratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=42)\n\n# Create a new column to store the fold number for each row\ntrain_df['fold'] = -1\nfor i, (train_index, test_index) in enumerate(mskf.split(train_df['Id'], y_multihot)):\n    train_df.loc[test_index, 'fold'] = i\n\n\ntrain_df['fold'].value_counts()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Extract text features using word-level TF-IDF (uni-grams, bi-grams, tri-grams)\n\nword_vectorizer = TfidfVectorizer(\n    analyzer='word',\n    ngram_range=(1, 3),\n    max_df=0.9,\n    min_df=5,\n    max_features=200000 # Limit features to prevent memory issues\n)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Extract character-level TF-IDF features (3–6 character n-grams)\n\nchar_vectorizer = TfidfVectorizer(\n    analyzer='char',\n    ngram_range=(3, 6),\n    max_df=0.9,\n    min_df=5,\n    max_features=200000 \n)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Combine the word and character feature matrices into a single sparse representation\n\n\nX_train_word = word_vectorizer.fit_transform(train_df['CleanText'])\nX_train_char = char_vectorizer.fit_transform(train_df['CleanText'])\n\n# Combine the features using scipy's hstack\nX_train_combined = hstack([X_train_word, X_train_char])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Convert the training tags into multi-hot vectors using a MultiLabelBinarizer\n\nmlb = MultiLabelBinarizer()\ny_train = mlb.fit_transform(train_df['TagList'])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Train One‑vs‑Rest Logistic Regression with L2 regularization\n\nlogreg = LogisticRegression(solver='saga', penalty='l2', random_state=42)\novr_classifier = OneVsRestClassifier(logreg, n_jobs=-1)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# grid search over different regularization strengths using 5-fold cross-validation\n\nC_values = [0.5, 1.0, 2.0, 4.0]\noof_predictions = {} # To store OOF predictions for each C\ncv_scores = {} # To store overall CV score for each C\n\n\nfor c_val in C_values:\n    \n    # Array to store out-of-fold predictions for this C\n    oof_preds_for_c = np.zeros(y_train.shape)\n    \n    for fold in range(N_SPLITS):\n        \n        # Get train and validation indices for the current fold\n        train_idx = train_df[train_df['fold'] != fold].index\n        val_idx = train_df[train_df['fold'] == fold].index\n        \n        X_train_fold, X_val_fold = X_train_combined[train_idx], X_train_combined[val_idx]\n        y_train_fold, y_val_fold = y_train[train_idx], y_train[val_idx]\n        \n        # Initialize and train the classifier with the current C value\n        logreg = LogisticRegression(C=c_val, solver='saga', penalty='l2', random_state=42)\n        ovr = OneVsRestClassifier(logreg, n_jobs=-1)\n        ovr.fit(X_train_fold, y_train_fold)\n        \n        # Predict probabilities on the validation fold\n        val_probs = ovr.predict_proba(X_val_fold)\n        \n        # Store the OOF predictions\n        oof_preds_for_c[val_idx] = val_probs\n        \n    oof_predictions[c_val] = oof_preds_for_c","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"best_c = None\nbest_threshold = None\nbest_f1 = -1\n\nthresholds = np.arange(0.1, 0.51, 0.05)\n\nfor c_val, oof_preds in oof_predictions.items():\n    for thresh in thresholds:\n        # Apply threshold to get binary predictions\n        oof_binary_preds = (oof_preds > thresh).astype(int)\n        \n        # Calculate mean F1 score (row-wise)\n        f1 = f1_score(y_train, oof_binary_preds, average='samples')\n        if f1 > best_f1:\n            best_f1 = f1\n            best_c = c_val\n            best_threshold = thresh","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}