{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"How about now:\n1: /kaggle/input/datasets/jillanisofttech/fake-or-real-news\n2: /kaggle/input/datasets/mrisdal/fake-news\n3: /kaggle/input/datasets/rahulogoel/isot-fake-news-dataset\n4: /kaggle/input/datasets/doanquanvietnamca/liar-dataset\n5: /kaggle/input/datasets/saurabhshahane/fake-news-classification","metadata":{"trusted":true,"editable":false},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true,"editable":false},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============================================================================\n#  FAKE NEWS DETECTION — COMPREHENSIVE EXPLORATORY DATA ANALYSIS (EDA)\n#  All 5 datasets | Exact Kaggle paths confirmed\n#  Run as a single cell in a Kaggle CPU notebook (no GPU needed)\n# =============================================================================\n\n# ── 0. IMPORTS ────────────────────────────────────────────────────────────────\nimport os\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.ticker as mticker\nimport seaborn as sns\nfrom collections import Counter\n\nwarnings.filterwarnings(\"ignore\")\npd.set_option(\"display.max_columns\", 20)\npd.set_option(\"display.max_colwidth\", 80)\npd.set_option(\"display.width\", 120)\n\nplt.rcParams.update({\n    \"figure.facecolor\":  \"white\",\n    \"axes.facecolor\":    \"white\",\n    \"axes.spines.top\":   False,\n    \"axes.spines.right\": False,\n    \"font.size\":         11,\n})\n\nPALETTE  = {\"REAL\": \"#2a78d6\", \"FAKE\": \"#e34948\"}\nSAVE_DIR = \"/kaggle/working/\"\n\ndef save_fig(name):\n    path = os.path.join(SAVE_DIR, name)\n    plt.savefig(path, dpi=150, bbox_inches=\"tight\")\n    print(f\"  💾 Saved → {path}\")\n\nprint(\"=\" * 70)\nprint(\"  FAKE NEWS EDA  —  5-Dataset Comparative Analysis\")\nprint(\"=\" * 70)\n\n\n# =============================================================================\n# SECTION 1 — CONFIRM FILE EXISTENCE\n# Verify every exact path before attempting to load.\n# =============================================================================\nprint(\"\\n\" + \"─\" * 70)\nprint(\"SECTION 1 │ File Path Confirmation\")\nprint(\"─\" * 70)\n\nCONFIRMED_PATHS = {\n    \"DS1 – WELFake\":\n        \"/kaggle/input/datasets/saurabhshahane/fake-news-classification/WELFake_Dataset.csv\",\n    \"DS2 – Fake or Real News\":\n        \"/kaggle/input/datasets/jillanisofttech/fake-or-real-news/fake_or_real_news.csv\",\n    \"DS3 – ISOT News_Dataset\":\n        \"/kaggle/input/datasets/rahulogoel/isot-fake-news-dataset/News_Dataset\",\n    \"DS4 – Getting Real (fake.csv)\":\n        \"/kaggle/input/datasets/mrisdal/fake-news/fake.csv\",\n    \"DS5 – LIAR train\":\n        \"/kaggle/input/datasets/doanquanvietnamca/liar-dataset/train.tsv\",\n    \"DS5 – LIAR valid\":\n        \"/kaggle/input/datasets/doanquanvietnamca/liar-dataset/valid.tsv\",\n    \"DS5 – LIAR test\":\n        \"/kaggle/input/datasets/doanquanvietnamca/liar-dataset/test.tsv\",\n}\n\nfor label, path in CONFIRMED_PATHS.items():\n    exists = os.path.exists(path)\n    size   = f\"{os.path.getsize(path)/1e6:.2f} MB\" if exists else \"—\"\n    status = \"✅ Found\" if exists else \"❌ NOT FOUND\"\n    print(f\"  {status}  [{size:>10}]  {label}\")\n    print(f\"            {path}\")\n\n# ISOT alternative: try with .csv extension too\nISOT_BASE = \"/kaggle/input/datasets/rahulogoel/isot-fake-news-dataset/News_Dataset\"\nISOT_PATH = ISOT_BASE if os.path.exists(ISOT_BASE) else ISOT_BASE + \".csv\"\nprint(f\"\\n  ISOT resolved path: {ISOT_PATH}  \"\n      f\"[{'✅' if os.path.exists(ISOT_PATH) else '❌'}]\")\n\n\n# =============================================================================\n# SECTION 2 — LOAD ALL DATASETS\n# =============================================================================\nprint(\"\\n\\n\" + \"─\" * 70)\nprint(\"SECTION 2 │ Dataset Loading\")\nprint(\"─\" * 70)\n\n# ── DS1: WELFake ──────────────────────────────────────────────────────────────\nprint(\"\\n▶ DS1 — WELFake\")\ntry:\n    df1 = pd.read_csv(\n        \"/kaggle/input/datasets/saurabhshahane/fake-news-classification/WELFake_Dataset.csv\"\n    )\n    # Drop unnamed index column if present\n    df1 = df1.loc[:, ~df1.columns.str.contains(\"^Unnamed\")]\n    # Normalise label: WELFake uses 0=FAKE 1=REAL (integer)\n    df1[\"label_str\"] = df1[\"label\"].map({0: \"FAKE\", 1: \"REAL\",\n                                         \"0\": \"FAKE\", \"1\": \"REAL\"})\n    print(f\"  ✅ Loaded  shape={df1.shape}\")\n    print(f\"     Columns: {list(df1.columns)}\")\nexcept Exception as e:\n    df1 = None\n    print(f\"  ❌ Failed: {e}\")\n\n# ── DS2: Fake or Real News ─────────────────────────────────────────────────────\nprint(\"\\n▶ DS2 — Fake or Real News\")\ntry:\n    df2 = pd.read_csv(\n        \"/kaggle/input/datasets/jillanisofttech/fake-or-real-news/fake_or_real_news.csv\"\n    )\n    df2 = df2.loc[:, ~df2.columns.str.contains(\"^Unnamed\")]\n    # Normalise label to uppercase\n    df2[\"label_str\"] = df2[\"label\"].astype(str).str.upper().str.strip()\n    print(f\"  ✅ Loaded  shape={df2.shape}\")\n    print(f\"     Columns: {list(df2.columns)}\")\nexcept Exception as e:\n    df2 = None\n    print(f\"  ❌ Failed: {e}\")\n\n# ── DS3: ISOT — News_Dataset (single merged file) ─────────────────────────────\nprint(\"\\n▶ DS3 — ISOT News_Dataset\")\ntry:\n    df3 = pd.read_csv(ISOT_PATH)\n    df3 = df3.loc[:, ~df3.columns.str.contains(\"^Unnamed\")]\n    print(f\"  ✅ Loaded  shape={df3.shape}\")\n    print(f\"     Columns: {list(df3.columns)}\")\n    # Detect label column — ISOT merged files use 'label' or 'class' or 'Label'\n    label_candidates = [c for c in df3.columns\n                        if c.lower() in (\"label\", \"class\", \"fake\", \"category\")]\n    print(f\"     Likely label columns: {label_candidates}\")\n    if label_candidates:\n        df3[\"label_str\"] = df3[label_candidates[0]].astype(str).str.upper().str.strip()\n    else:\n        print(\"  ⚠️  No obvious label column — inspect manually after loading.\")\n        df3[\"label_str\"] = \"UNKNOWN\"\nexcept Exception as e:\n    df3 = None\n    print(f\"  ❌ Failed: {e}\")\n\n# ── DS4: Getting Real About Fake News ─────────────────────────────────────────\nprint(\"\\n▶ DS4 — Getting Real About Fake News (fake.csv)\")\ntry:\n    df4 = pd.read_csv(\n        \"/kaggle/input/datasets/mrisdal/fake-news/fake.csv\",\n        encoding=\"latin-1\"\n    )\n    df4 = df4.loc[:, ~df4.columns.str.contains(\"^Unnamed\")]\n    # This dataset is FAKE-only; mark accordingly\n    df4[\"label_str\"] = \"FAKE\"\n    print(f\"  ✅ Loaded  shape={df4.shape}\")\n    print(f\"     Columns: {list(df4.columns)}\")\n    print(f\"  ⚠️  NOTE: This dataset contains ONLY fake/biased articles.\")\n    print(f\"     A real-news complement must be sampled before modelling.\")\nexcept Exception as e:\n    df4 = None\n    print(f\"  ❌ Failed: {e}\")\n\n# ── DS5: LIAR (train + valid + test) ─────────────────────────────────────────\nprint(\"\\n▶ DS5 — LIAR Dataset (train + valid + test)\")\nLIAR_COLS = [\n    \"id\", \"label\", \"statement\", \"subject\", \"speaker\",\n    \"speaker_job\", \"state\", \"party\",\n    \"barely_true_counts\", \"false_counts\", \"half_true_counts\",\n    \"mostly_true_counts\", \"pants_fire_counts\", \"context\"\n]\nLIAR_FAKE  = {\"pants-fire\", \"false\", \"barely-true\"}\nLIAR_REAL  = {\"half-true\", \"mostly-true\", \"true\"}\nLIAR_FILES = {\n    \"train\": \"/kaggle/input/datasets/doanquanvietnamca/liar-dataset/train.tsv\",\n    \"valid\": \"/kaggle/input/datasets/doanquanvietnamca/liar-dataset/valid.tsv\",\n    \"test\":  \"/kaggle/input/datasets/doanquanvietnamca/liar-dataset/test.tsv\",\n}\nliar_parts = {}\nfor split, path in LIAR_FILES.items():\n    try:\n        part = pd.read_csv(path, sep=\"\\t\", header=None,\n                           names=LIAR_COLS, quoting=3)\n        liar_parts[split] = part\n        print(f\"  ✅ {split:5s}  shape={part.shape}\")\n    except Exception as e:\n        print(f\"  ❌ {split}: {e}\")\n\nif liar_parts:\n    df5 = pd.concat(liar_parts.values(), keys=liar_parts.keys(),\n                    ignore_index=True)\n    df5[\"label_str\"] = df5[\"label\"].apply(\n        lambda x: \"FAKE\" if str(x).strip().lower() in LIAR_FAKE\n                  else (\"REAL\" if str(x).strip().lower() in LIAR_REAL\n                        else \"UNKNOWN\")\n    )\n    df5 = df5[df5[\"label_str\"] != \"UNKNOWN\"].copy()\n    print(f\"\\n  ✅ LIAR combined  shape={df5.shape}\")\n    print(f\"     Columns: {list(df5.columns)}\")\nelse:\n    df5 = None\n    print(\"  ❌ No LIAR splits loaded.\")\n\n\n# =============================================================================\n# SECTION 3 — SCHEMA DEEP INSPECTION\n# dtypes · null counts · unique counts · sample values\n# =============================================================================\nprint(\"\\n\\n\" + \"─\" * 70)\nprint(\"SECTION 3 │ Schema & Null Audit\")\nprint(\"─\" * 70)\n\nDATASETS = {\n    \"DS1 – WELFake\":           (df1, \"text\",      \"label_str\"),\n    \"DS2 – Fake or Real News\": (df2, \"text\",      \"label_str\"),\n    \"DS3 – ISOT\":              (df3, \"text\",      \"label_str\"),\n    \"DS4 – Getting Real\":      (df4, \"text\",      \"label_str\"),\n    \"DS5 – LIAR\":              (df5, \"statement\", \"label_str\"),\n}\n\nfor ds_name, (df, txt_col, lbl_col) in DATASETS.items():\n    print(f\"\\n{'═'*65}\")\n    print(f\"  {ds_name}\")\n    print(f\"{'═'*65}\")\n    if df is None:\n        print(\"  ⚠️  DataFrame is None — revisit Section 2.\")\n        continue\n\n    print(f\"  Shape  : {df.shape[0]:,} rows  ×  {df.shape[1]} columns\")\n    audit = pd.DataFrame({\n        \"dtype\":      df.dtypes,\n        \"null_n\":     df.isnull().sum(),\n        \"null_%\":     (df.isnull().mean() * 100).round(2),\n        \"unique\":     df.nunique(),\n        \"sample\":     df.apply(\n            lambda c: str(c.dropna().iloc[0])[:55] if len(c.dropna()) > 0 else \"—\"\n        ),\n    })\n    print(f\"\\n{audit.to_string()}\")\n    print(f\"\\n  ── First 3 rows ──\")\n    print(df.head(3).to_string(index=False))\n\n\n# =============================================================================\n# SECTION 4 — CLASS DISTRIBUTION (all 5 datasets side by side)\n# =============================================================================\nprint(\"\\n\\n\" + \"─\" * 70)\nprint(\"SECTION 4 │ Class Distribution & Imbalance Ratios\")\nprint(\"─\" * 70)\n\nfig, axes = plt.subplots(1, 5, figsize=(24, 5))\nfig.suptitle(\"Class Distribution — All 5 Datasets\", fontsize=14, y=1.02)\n\ndist_rows = []\nfor ax, (ds_name, (df, txt_col, lbl_col)) in zip(axes, DATASETS.items()):\n    short = ds_name.split(\"–\")[1].strip()\n    ax.set_title(short, fontsize=10, pad=8)\n\n    if df is None or lbl_col not in df.columns:\n        ax.text(0.5, 0.5, \"No data\", ha=\"center\", va=\"center\")\n        ax.axis(\"off\")\n        continue\n\n    counts = df[lbl_col].value_counts()\n    colors = [PALETTE.get(str(l), \"#999\") for l in counts.index]\n    bars   = ax.bar(counts.index.astype(str), counts.values,\n                    color=colors, edgecolor=\"white\", linewidth=0.8)\n\n    for bar, val in zip(bars, counts.values):\n        ax.text(bar.get_x() + bar.get_width() / 2,\n                bar.get_height() + counts.max() * 0.012,\n                f\"{val:,}\", ha=\"center\", va=\"bottom\", fontsize=9)\n\n    ax.yaxis.set_major_formatter(\n        mticker.FuncFormatter(lambda x, _: f\"{int(x):,}\"))\n    ax.set_ylabel(\"Count\")\n    ax.tick_params(axis=\"x\", labelsize=9)\n\n    if len(counts) >= 2:\n        ratio = counts.max() / counts.min()\n        ax.set_xlabel(f\"Imbalance ratio  {ratio:.2f}×\", fontsize=9)\n\n    total = counts.sum()\n    for lbl, cnt in counts.items():\n        dist_rows.append({\n            \"Dataset\":    short,\n            \"Class\":      lbl,\n            \"Count\":      int(cnt),\n            \"Pct (%)\":    round(cnt / total * 100, 2),\n        })\n\nplt.tight_layout()\nsave_fig(\"01_class_distribution.png\")\nplt.show()\n\nprint(\"\\n  Distribution table:\")\ndist_df = pd.DataFrame(dist_rows)\nprint(dist_df.to_string(index=False))\n\n\n# =============================================================================\n# SECTION 5 — TEXT LENGTH ANALYSIS  (word count + character count)\n# =============================================================================\nprint(\"\\n\\n\" + \"─\" * 70)\nprint(\"SECTION 5 │ Text Length Distribution by Class\")\nprint(\"─\" * 70)\n\nfig, axes = plt.subplots(2, 5, figsize=(28, 10))\nfig.suptitle(\"Word Count (top) & Character Count (bottom) by Class\",\n             fontsize=14, y=1.01)\n\nlen_rows = []\nfor col_i, (ds_name, (df, txt_col, lbl_col)) in enumerate(DATASETS.items()):\n    ax_w = axes[0][col_i]\n    ax_c = axes[1][col_i]\n    short = ds_name.split(\"–\")[1].strip()\n    ax_w.set_title(short, fontsize=10)\n    ax_c.set_title(short, fontsize=10)\n\n    if df is None or txt_col not in df.columns:\n        for ax in (ax_w, ax_c):\n            ax.text(0.5, 0.5, f\"'{txt_col}'\\nnot found\",\n                    ha=\"center\", va=\"center\", fontsize=9)\n            ax.axis(\"off\")\n        continue\n\n    tmp = df[[txt_col, lbl_col]].dropna().copy()\n    tmp[\"wc\"] = tmp[txt_col].astype(str).str.split().str.len()\n    tmp[\"cc\"] = tmp[txt_col].astype(str).str.len()\n\n    wc_clip = int(tmp[\"wc\"].quantile(0.98))\n    cc_clip = int(tmp[\"cc\"].quantile(0.98))\n\n    print(f\"\\n  {ds_name}  (text col='{txt_col}')\")\n    stats = tmp.groupby(lbl_col)[[\"wc\",\"cc\"]].describe().round(1)\n    print(stats.to_string())\n\n    for cls in tmp[lbl_col].unique():\n        sub   = tmp[tmp[lbl_col] == cls]\n        color = PALETTE.get(str(cls), \"#888\")\n        ax_w.hist(sub[\"wc\"].clip(upper=wc_clip), bins=50,\n                  alpha=0.65, label=str(cls), color=color, edgecolor=\"none\")\n        ax_c.hist(sub[\"cc\"].clip(upper=cc_clip), bins=50,\n                  alpha=0.65, label=str(cls), color=color, edgecolor=\"none\")\n\n        mean_wc = sub[\"wc\"].mean()\n        mean_cc = sub[\"cc\"].mean()\n        len_rows.append({\n            \"Dataset\": short, \"Class\": cls,\n            \"Mean words\": round(mean_wc, 1),\n            \"Median words\": round(sub[\"wc\"].median(), 1),\n            \"Max words\": int(sub[\"wc\"].max()),\n            \"Mean chars\": round(mean_cc, 1),\n        })\n\n    ax_w.set_xlabel(f\"Word count (clipped @{wc_clip})\", fontsize=8)\n    ax_c.set_xlabel(f\"Char count (clipped @{cc_clip})\", fontsize=8)\n    ax_w.legend(fontsize=8)\n    ax_c.legend(fontsize=8)\n\nplt.tight_layout()\nsave_fig(\"02_text_length_distribution.png\")\nplt.show()\n\nprint(\"\\n  Length summary table:\")\nprint(pd.DataFrame(len_rows).to_string(index=False))\n\n\n# =============================================================================\n# SECTION 6 — MISSING VALUE HEATMAP\n# =============================================================================\nprint(\"\\n\\n\" + \"─\" * 70)\nprint(\"SECTION 6 │ Missing Value Heatmap\")\nprint(\"─\" * 70)\n\nfig, axes = plt.subplots(1, 5, figsize=(24, 6))\nfig.suptitle(\"Missing Values — Yellow = Missing  (500-row sample each)\",\n             fontsize=13)\n\nfor ax, (ds_name, (df, txt_col, lbl_col)) in zip(axes, DATASETS.items()):\n    short = ds_name.split(\"–\")[1].strip()\n    ax.set_title(short, fontsize=9, pad=6)\n\n    if df is None or df.empty:\n        ax.text(0.5, 0.5, \"No data\", ha=\"center\", va=\"center\")\n        ax.axis(\"off\")\n        continue\n\n    sample = df.isnull().sample(min(500, len(df)), random_state=42)\n    sns.heatmap(sample, ax=ax, cbar=False, yticklabels=False,\n                cmap=\"viridis\", linewidths=0)\n    ax.set_xticklabels(ax.get_xticklabels(),\n                       rotation=45, ha=\"right\", fontsize=7)\n\nplt.tight_layout()\nsave_fig(\"03_missing_value_heatmap.png\")\nplt.show()\n\n\n# =============================================================================\n# SECTION 7 — DUPLICATE DETECTION\n# =============================================================================\nprint(\"\\n\\n\" + \"─\" * 70)\nprint(\"SECTION 7 │ Duplicate & Empty Text Detection\")\nprint(\"─\" * 70)\n\ndup_rows = []\nfor ds_name, (df, txt_col, lbl_col) in DATASETS.items():\n    if df is None:\n        print(f\"\\n  {ds_name}: ⚠️  No data\")\n        continue\n\n    total      = len(df)\n    full_dups  = df.duplicated().sum()\n    text_dups  = (df[txt_col].duplicated().sum()\n                  if txt_col in df.columns else \"N/A\")\n    null_txt   = (df[txt_col].isnull().sum()\n                  if txt_col in df.columns else \"N/A\")\n    empty_txt  = ((df[txt_col].astype(str).str.strip() == \"\").sum()\n                  if txt_col in df.columns else \"N/A\")\n    short_txt  = ((df[txt_col].astype(str).str.split().str.len() < 5).sum()\n                  if txt_col in df.columns else \"N/A\")\n\n    print(f\"\\n  {ds_name}  ({total:,} rows)\")\n    print(f\"    Full-row duplicates : {full_dups:,}  \"\n          f\"({full_dups/total*100:.2f}%)\")\n    print(f\"    Text duplicates     : \"\n          f\"{text_dups:,}  ({text_dups/total*100:.2f}%)\"\n          if isinstance(text_dups, int) else\n          f\"    Text duplicates     : {text_dups}\")\n    print(f\"    Null text cells     : {null_txt}\")\n    print(f\"    Empty text cells    : {empty_txt}\")\n    print(f\"    Texts < 5 words     : {short_txt}\")\n\n    dup_rows.append({\n        \"Dataset\":       ds_name.split(\"–\")[1].strip(),\n        \"Total rows\":    total,\n        \"Full dups\":     full_dups,\n        \"Text dups\":     text_dups,\n        \"Null text\":     null_txt,\n        \"Empty text\":    empty_txt,\n        \"Texts <5 words\": short_txt,\n    })\n\nprint(\"\\n  Duplicate summary table:\")\nprint(pd.DataFrame(dup_rows).to_string(index=False))\n\n\n# =============================================================================\n# SECTION 8 — DS3 ISOT: COLUMN DISCOVERY & LABEL INSPECTION\n# News_Dataset may be a merged file; need to understand its structure fully.\n# =============================================================================\nprint(\"\\n\\n\" + \"─\" * 70)\nprint(\"SECTION 8 │ DS3 – ISOT Column Discovery & Label Values\")\nprint(\"─\" * 70)\n\nif df3 is not None:\n    print(f\"  Shape   : {df3.shape}\")\n    print(f\"  Columns : {list(df3.columns)}\")\n    print(f\"\\n  All unique values in each non-text column:\")\n    for col in df3.columns:\n        if df3[col].dtype == object:\n            n_unique = df3[col].nunique()\n            if n_unique <= 30:\n                print(f\"    {col:20s} ({n_unique} unique): \"\n                      f\"{sorted(df3[col].dropna().unique().tolist())}\")\n            else:\n                print(f\"    {col:20s} ({n_unique} unique — too many to list)\")\n        else:\n            print(f\"    {col:20s} dtype={df3[col].dtype}  \"\n                  f\"min={df3[col].min()}  max={df3[col].max()}\")\n\n    print(f\"\\n  label_str distribution:\")\n    print(df3[\"label_str\"].value_counts().to_string())\nelse:\n    print(\"  ⚠️  DS3 not loaded — check ISOT path in Section 2.\")\n\n\n# =============================================================================\n# SECTION 9 — DS4: GETTING REAL — TYPE COLUMN & LANGUAGE ANALYSIS\n# Fake-only dataset; understand categories and language coverage.\n# =============================================================================\nprint(\"\\n\\n\" + \"─\" * 70)\nprint(\"SECTION 9 │ DS4 – Getting Real: Type & Language Analysis\")\nprint(\"─\" * 70)\n\nif df4 is not None:\n    print(f\"  Columns : {list(df4.columns)}\")\n\n    # 'type' column\n    if \"type\" in df4.columns:\n        type_counts = df4[\"type\"].value_counts()\n        print(f\"\\n  'type' distribution ({df4['type'].nunique()} categories):\")\n        print(type_counts.to_string())\n\n        fig, ax = plt.subplots(figsize=(10, 4))\n        type_counts.plot(kind=\"bar\", ax=ax,\n                         color=\"#e34948\", edgecolor=\"white\")\n        ax.set_title(\"DS4 – Article Type Distribution (Getting Real)\",\n                     fontsize=12)\n        ax.set_xlabel(\"Article type\")\n        ax.set_ylabel(\"Count\")\n        ax.tick_params(axis=\"x\", rotation=40)\n        for p in ax.patches:\n            ax.annotate(f\"{int(p.get_height()):,}\",\n                        (p.get_x() + p.get_width() / 2, p.get_height()),\n                        ha=\"center\", va=\"bottom\", fontsize=8)\n        plt.tight_layout()\n        save_fig(\"04_ds4_type_distribution.png\")\n        plt.show()\n    else:\n        print(\"  'type' column not found.\")\n\n    # 'language' column\n    if \"language\" in df4.columns:\n        lang_counts = df4[\"language\"].value_counts()\n        print(f\"\\n  Language distribution (top 15):\")\n        print(lang_counts.head(15).to_string())\n        english_pct = (lang_counts.get(\"english\", 0) /\n                       lang_counts.sum() * 100)\n        print(f\"\\n  English articles: {english_pct:.1f}% of total\")\n    else:\n        print(\"  'language' column not found.\")\n\n    # Top source URLs\n    if \"site_url\" in df4.columns:\n        print(f\"\\n  Top 10 source URLs:\")\n        print(df4[\"site_url\"].value_counts().head(10).to_string())\n\n    # Text field check for DS4\n    text_candidates = [c for c in df4.columns\n                       if c.lower() in (\"text\", \"content\", \"body\",\n                                        \"article\", \"story\")]\n    print(f\"\\n  Detected text column candidates: {text_candidates}\")\n    if not text_candidates:\n        print(\"  ⚠️  No obvious 'text' column — check column names above\")\n        print(\"     and update DATASETS dict txt_col for DS4 accordingly.\")\nelse:\n    print(\"  ⚠️  DS4 not loaded — check path in Section 2.\")\n\n\n# =============================================================================\n# SECTION 10 — DS5 LIAR: 6-CLASS → BINARY MAPPING VALIDATION\n# =============================================================================\nprint(\"\\n\\n\" + \"─\" * 70)\nprint(\"SECTION 10 │ DS5 – LIAR: 6-Class Distribution & Binary Mapping\")\nprint(\"─\" * 70)\n\nif df5 is not None:\n    raw_df5 = pd.concat(liar_parts.values(), ignore_index=True)\n\n    orig_counts = raw_df5[\"label\"].value_counts()\n    bin_counts  = df5[\"label_str\"].value_counts()\n\n    label_order  = [\"pants-fire\", \"false\", \"barely-true\",\n                    \"half-true\",  \"mostly-true\", \"true\"]\n    label_colors = [\"#e34948\"] * 3 + [\"#2a78d6\"] * 3\n\n    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 5))\n    fig.suptitle(\"DS5 – LIAR Label Distribution\", fontsize=13)\n\n    vals = [orig_counts.get(l, 0) for l in label_order]\n    ax1.bar(label_order, vals, color=label_colors, edgecolor=\"white\")\n    ax1.set_title(\"Original 6-class labels\\n(red → FAKE  |  blue → REAL)\",\n                  fontsize=10)\n    ax1.set_xlabel(\"Label\")\n    ax1.set_ylabel(\"Count\")\n    ax1.tick_params(axis=\"x\", rotation=28)\n    for i, v in enumerate(vals):\n        ax1.text(i, v + 30, f\"{v:,}\", ha=\"center\", fontsize=9)\n\n    ax2.bar(bin_counts.index, bin_counts.values,\n            color=[PALETTE.get(l, \"#999\") for l in bin_counts.index],\n            edgecolor=\"white\")\n    ax2.set_title(\"After binary mapping\", fontsize=10)\n    ax2.set_xlabel(\"Binary label\")\n    ax2.set_ylabel(\"Count\")\n    for i, (lbl, cnt) in enumerate(bin_counts.items()):\n        ax2.text(i, cnt + 30, f\"{cnt:,}\", ha=\"center\", fontsize=10)\n\n    plt.tight_layout()\n    save_fig(\"05_liar_label_mapping.png\")\n    plt.show()\n\n    ratio = bin_counts.max() / bin_counts.min()\n    print(f\"\\n  Original 6-class counts:\")\n    print(orig_counts.to_string())\n    print(f\"\\n  Binary-mapped counts:\")\n    print(bin_counts.to_string())\n    print(f\"\\n  Binary imbalance ratio : {ratio:.3f}×\")\n    print(f\"  → ADASYN + Tomek Links justified: {'YES ✅' if ratio > 1.2 else 'NO — balanced'}\")\n\n    # Per-split breakdown\n    print(f\"\\n  Per-split record counts:\")\n    for split, part in liar_parts.items():\n        part_bin = part[\"label\"].apply(\n            lambda x: \"FAKE\" if str(x).strip().lower() in LIAR_FAKE\n                      else (\"REAL\" if str(x).strip().lower() in LIAR_REAL\n                            else \"UNKNOWN\")\n        ).value_counts()\n        print(f\"    {split:6s}: {dict(part_bin)}\")\n\n    # Speaker diversity\n    if \"speaker\" in df5.columns:\n        print(f\"\\n  Top 10 speakers:\")\n        print(df5[\"speaker\"].value_counts().head(10).to_string())\n\n    # Subject diversity\n    if \"subject\" in df5.columns:\n        print(f\"\\n  Top 10 subjects:\")\n        print(df5[\"subject\"].value_counts().head(10).to_string())\nelse:\n    print(\"  ⚠️  DS5 not loaded — check paths in Section 2.\")\n\n\n# =============================================================================\n# SECTION 11 — SUBJECT / CATEGORY DISTRIBUTION (DS1, DS3, DS5)\n# =============================================================================\nprint(\"\\n\\n\" + \"─\" * 70)\nprint(\"SECTION 11 │ Subject / Category Distribution\")\nprint(\"─\" * 70)\n\nSUBJECT_MAP = {\n    \"DS1 – WELFake\":  (df1, \"subject\"),\n    \"DS3 – ISOT\":     (df3, \"subject\"),\n    \"DS5 – LIAR\":     (df5, \"subject\"),\n}\n\nfor ds_name, (df, sub_col) in SUBJECT_MAP.items():\n    if df is None:\n        print(f\"\\n  {ds_name}: no data\")\n        continue\n    if sub_col not in df.columns:\n        all_cols = list(df.columns)\n        print(f\"\\n  {ds_name}: '{sub_col}' not found. Available: {all_cols}\")\n        continue\n\n    top_subjects = df[sub_col].value_counts().head(12)\n    print(f\"\\n  {ds_name} — top 12 subjects:\")\n    print(top_subjects.to_string())\n\n    fig, ax = plt.subplots(figsize=(10, 4))\n    top_subjects.plot(kind=\"barh\", ax=ax, color=\"#2a78d6\", edgecolor=\"white\")\n    ax.invert_yaxis()\n    ax.set_title(f\"{ds_name.split('–')[1].strip()} — Subject Distribution\",\n                 fontsize=12)\n    ax.set_xlabel(\"Count\")\n    plt.tight_layout()\n    safe = ds_name.replace(\" \", \"_\").replace(\"–\", \"\").replace(\"/\", \"\")\n    save_fig(f\"06_subjects_{safe}.png\")\n    plt.show()\n\n\n# =============================================================================\n# SECTION 12 — CROSS-DATASET CONSOLIDATED SUMMARY TABLE\n# =============================================================================\nprint(\"\\n\\n\" + \"─\" * 70)\nprint(\"SECTION 12 │ Cross-Dataset Consolidated Summary\")\nprint(\"─\" * 70)\n\nsummary_rows = []\nfor ds_name, (df, txt_col, lbl_col) in DATASETS.items():\n    if df is None:\n        summary_rows.append({\n            \"Dataset\": ds_name.split(\"–\")[1].strip(),\n            \"Rows\": \"—\", \"Cols\": \"—\",\n            \"Text col\": txt_col, \"Label col\": lbl_col,\n            \"Null text %\": \"—\", \"Avg words\": \"—\",\n            \"Med words\": \"—\", \"Max words\": \"—\",\n            \"Classes\": \"—\", \"Notes\": \"Load failed\",\n        })\n        continue\n\n    null_pct  = (f\"{df[txt_col].isnull().mean()*100:.1f}%\"\n                 if txt_col in df.columns else \"—\")\n    wc_series = (df[txt_col].dropna().astype(str)\n                 .str.split().str.len()\n                 if txt_col in df.columns else pd.Series(dtype=int))\n\n    avg_w  = f\"{wc_series.mean():.0f}\"  if len(wc_series) else \"—\"\n    med_w  = f\"{wc_series.median():.0f}\" if len(wc_series) else \"—\"\n    max_w  = f\"{wc_series.max():,}\"     if len(wc_series) else \"—\"\n    classes = (str(sorted(df[lbl_col].dropna().unique().tolist()))\n               if lbl_col in df.columns else \"—\")\n\n    notes = \"\"\n    if ds_name == \"DS4 – Getting Real\":\n        notes = \"FAKE-only; needs real complement\"\n    elif ds_name == \"DS5 – LIAR\":\n        notes = \"6-class binarised; statement-level text\"\n    elif ds_name == \"DS3 – ISOT\":\n        notes = \"Single merged CSV; verify label col\"\n\n    summary_rows.append({\n        \"Dataset\":    ds_name.split(\"–\")[1].strip(),\n        \"Rows\":       f\"{len(df):,}\",\n        \"Cols\":       df.shape[1],\n        \"Text col\":   txt_col,\n        \"Label col\":  lbl_col,\n        \"Null text%\": null_pct,\n        \"Avg words\":  avg_w,\n        \"Med words\":  med_w,\n        \"Max words\":  max_w,\n        \"Classes\":    classes,\n        \"Notes\":      notes,\n    })\n\nsum_df = pd.DataFrame(summary_rows)\nprint(sum_df.to_string(index=False))\nsum_df.to_csv(os.path.join(SAVE_DIR, \"eda_summary_table.csv\"), index=False)\nprint(f\"\\n  💾 Saved → {SAVE_DIR}eda_summary_table.csv\")\n\n\n# =============================================================================\n# SECTION 13 — CORRELATION: TEXT LENGTH vs LABEL (box plots)\n# =============================================================================\nprint(\"\\n\\n\" + \"─\" * 70)\nprint(\"SECTION 13 │ Word Count vs Label — Box Plots\")\nprint(\"─\" * 70)\n\nfig, axes = plt.subplots(1, 5, figsize=(24, 6))\nfig.suptitle(\"Word Count Distribution by Class (Box Plot)\", fontsize=13)\n\nfor ax, (ds_name, (df, txt_col, lbl_col)) in zip(axes, DATASETS.items()):\n    short = ds_name.split(\"–\")[1].strip()\n    ax.set_title(short, fontsize=10)\n\n    if df is None or txt_col not in df.columns or lbl_col not in df.columns:\n        ax.text(0.5, 0.5, \"No data\", ha=\"center\", va=\"center\")\n        ax.axis(\"off\")\n        continue\n\n    tmp = df[[txt_col, lbl_col]].dropna().copy()\n    tmp[\"wc\"] = tmp[txt_col].astype(str).str.split().str.len()\n    clip_val  = int(tmp[\"wc\"].quantile(0.97))\n    tmp[\"wc\"] = tmp[\"wc\"].clip(upper=clip_val)\n\n    classes = sorted(tmp[lbl_col].unique())\n    data    = [tmp[tmp[lbl_col] == cls][\"wc\"].values for cls in classes]\n    colors  = [PALETTE.get(str(cls), \"#999\") for cls in classes]\n\n    bp = ax.boxplot(data, patch_artist=True, labels=classes,\n                    medianprops=dict(color=\"black\", linewidth=2))\n    for patch, color in zip(bp[\"boxes\"], colors):\n        patch.set_facecolor(color)\n        patch.set_alpha(0.7)\n\n    ax.set_ylabel(\"Word count\")\n    ax.set_xlabel(\"Class\")\n\nplt.tight_layout()\nsave_fig(\"07_wordcount_boxplot.png\")\nplt.show()\n\n\n# =============================================================================\n# SECTION 14 — DATA QUALITY FLAGS & PREPROCESSING CHECKLIST\n# =============================================================================\nprint(\"\\n\\n\" + \"─\" * 70)\nprint(\"SECTION 14 │ Data Quality Flags & Preprocessing Checklist\")\nprint(\"─\" * 70)\n\nchecklist = \"\"\"\n  ┌─────────────────────────────────────────────────────────────────────────┐\n  │  PREPROCESSING CHECKLIST — derived from EDA findings                    │\n  ├──────────┬──────────────────────────────────────────────────────────────┤\n  │ DS1      │ ✔ Drop 'Unnamed' index column if present                     │\n  │ WELFake  │ ✔ Map label: 0 → FAKE, 1 → REAL (integer type)               │\n  │          │ ✔ Check for articles duplicated across its 4 source datasets  │\n  │          │ ✔ Drop rows where 'text' is NaN or empty                      │\n  ├──────────┼──────────────────────────────────────────────────────────────┤\n  │ DS2      │ ✔ Drop 'Unnamed: 0' index column if present                   │\n  │ Fake/Real│ ✔ Normalise label → uppercase (FAKE / REAL)                  │\n  │          │ ✔ Check if 'title' should be prepended to 'text'              │\n  ├──────────┼──────────────────────────────────────────────────────────────┤\n  │ DS3      │ ✔ Confirm which column holds binary label (check Section 8)   │\n  │ ISOT     │ ✔ Verify label values (REAL/FAKE or 1/0 or TRUE/FALSE)        │\n  │          │ ✔ Drop 'date' column to prevent temporal leakage              │\n  │          │ ✔ Filter non-English articles if any detected                  │\n  ├──────────┼──────────────────────────────────────────────────────────────┤\n  │ DS4      │ ✔ FAKE-only → sample REAL complement from ISOT True articles  │\n  │ Getting  │ ✔ Filter: language == 'english' only                          │\n  │ Real     │ ✔ Identify correct text column (check Section 9 output)       │\n  │          │ ✔ Drop rows: text NaN, empty, or < 20 characters              │\n  │          │ ✔ Remove metadata cols: uuid, scraped_at, site_url etc.       │\n  ├──────────┼──────────────────────────────────────────────────────────────┤\n  │ DS5      │ ✔ Binarise: {pants-fire, false, barely-true} → FAKE           │\n  │ LIAR     │            {half-true, mostly-true, true}  → REAL             │\n  │          │ ✔ Text column = 'statement' (avg ~20 words — very short)      │\n  │          │ ✔ Concatenate all 3 splits (train + valid + test)             │\n  │          │ ✔ Document binary imbalance ratio in methodology              │\n  │          │ ✔ ADASYN especially important here due to short text + skew   │\n  ├──────────┼──────────────────────────────────────────────────────────────┤\n  │ ALL      │ ✔ Unify schema → columns: ['text', 'label']                   │\n  │ DATASETS │ ✔ Encode label → FAKE=0, REAL=1 (integer)                     │\n  │          │ ✔ Strip HTML tags, URLs, @mentions, special characters         │\n  │          │ ✔ Lowercase all text                                           │\n  │          │ ✔ Remove full-row duplicates                                   │\n  │          │ ✔ Remove text-level duplicates (keep first)                    │\n  │          │ ✔ Remove rows where word count < 3                             │\n  │          │ ✔ Save cleaned CSV per dataset before vectorization            │\n  │          │ ✔ Apply ADASYN → then Tomek Links on training split only       │\n  └──────────┴──────────────────────────────────────────────────────────────┘\n\"\"\"\nprint(checklist)\n\n\n# =============================================================================\n# SECTION 15 — OUTPUT FILE MANIFEST\n# =============================================================================\nprint(\"\\n\" + \"─\" * 70)\nprint(\"SECTION 15 │ Output Files Generated\")\nprint(\"─\" * 70)\n\nmanifest = [\n    (\"01_class_distribution.png\",       \"Bar charts — class balance for all 5 datasets\"),\n    (\"02_text_length_distribution.png\",  \"Histograms — word & char count by class\"),\n    (\"03_missing_value_heatmap.png\",     \"Heatmap — missing values per dataset\"),\n    (\"04_ds4_type_distribution.png\",     \"DS4 article type breakdown\"),\n    (\"05_liar_label_mapping.png\",        \"LIAR 6-class → binary mapping\"),\n    (\"06_subjects_*.png\",                \"Subject distributions for DS1 / DS3 / DS5\"),\n    (\"07_wordcount_boxplot.png\",         \"Box plots — word count by class per dataset\"),\n    (\"eda_summary_table.csv\",            \"Consolidated cross-dataset summary table\"),\n]\nfor fname, desc in manifest:\n    fpath = os.path.join(SAVE_DIR, fname)\n    exists = \"✅\" if os.path.exists(fpath) else \"📋\"\n    print(f\"  {exists}  {fname:<45} {desc}\")\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"  EDA COMPLETE\")\nprint(\"  Review all section outputs and the checklist in Section 14\")\nprint(\"  before proceeding to the preprocessing notebook.\")\nprint(\"=\" * 70)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-17T08:44:15.550209Z","iopub.execute_input":"2026-07-17T08:44:15.550527Z","iopub.status.idle":"2026-07-17T08:45:08.922742Z","shell.execute_reply.started":"2026-07-17T08:44:15.550498Z","shell.execute_reply":"2026-07-17T08:45:08.921934Z"},"editable":false},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nfor root, dirs, files in os.walk(\n    \"/kaggle/input/datasets/rahulogoel/isot-fake-news-dataset/News_Dataset\"\n):\n    for f in files:\n        full = os.path.join(root, f)\n        print(f\"{os.path.getsize(full)/1e6:.2f} MB  —  {full}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-17T08:45:12.962323Z","iopub.execute_input":"2026-07-17T08:45:12.962676Z","iopub.status.idle":"2026-07-17T08:45:12.977878Z","shell.execute_reply.started":"2026-07-17T08:45:12.962646Z","shell.execute_reply":"2026-07-17T08:45:12.977201Z"},"editable":false},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============================================================================\n#  FAKE NEWS DETECTION — PREPROCESSING PIPELINE\n#  Addresses all critical findings from EDA analysis\n#  Run as a single cell in a Kaggle CPU notebook\n#  Output: 5 cleaned, schema-unified CSVs ready for vectorization\n# =============================================================================\n\n# ── 0. IMPORTS ────────────────────────────────────────────────────────────────\nimport os\nimport re\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.ticker as mticker\nimport seaborn as sns\n\n# NLP\nimport nltk\nfrom nltk.corpus import stopwords\nfrom nltk.stem import WordNetLemmatizer\nfrom nltk.tokenize import word_tokenize\n\n# Download required NLTK resources\nnltk.download(\"punkt\",        quiet=True)\nnltk.download(\"punkt_tab\",    quiet=True)\nnltk.download(\"stopwords\",    quiet=True)\nnltk.download(\"wordnet\",      quiet=True)\nnltk.download(\"omw-1.4\",      quiet=True)\n\nwarnings.filterwarnings(\"ignore\")\npd.set_option(\"display.max_columns\", 20)\npd.set_option(\"display.width\", 130)\n\nSAVE_DIR = \"/kaggle/working/\"\nSTOP_WORDS = set(stopwords.words(\"english\"))\nlemmatizer = WordNetLemmatizer()\n\nplt.rcParams.update({\n    \"figure.facecolor\": \"white\",\n    \"axes.facecolor\":   \"white\",\n    \"axes.spines.top\":  False,\n    \"axes.spines.right\": False,\n    \"font.size\": 11,\n})\nPALETTE = {\"REAL\": \"#2a78d6\", \"FAKE\": \"#e34948\"}\n\ndef save_fig(name):\n    path = os.path.join(SAVE_DIR, name)\n    plt.savefig(path, dpi=150, bbox_inches=\"tight\")\n    print(f\"  💾 Saved → {path}\")\n\nprint(\"=\" * 70)\nprint(\"  FAKE NEWS — PREPROCESSING PIPELINE\")\nprint(\"=\" * 70)\n\n\n# =============================================================================\n# STEP 1 — FIX DS3: ISOT DIRECTORY SCAN\n# EDA confirmed News_Dataset is a directory, not a file.\n# Scan it to find the actual CSV(s) inside.\n# =============================================================================\nprint(\"\\n\" + \"─\" * 70)\nprint(\"STEP 1 │ DS3 ISOT — Directory Scan & Load Fix\")\nprint(\"─\" * 70)\n\nISOT_DIR = \"/kaggle/input/datasets/rahulogoel/isot-fake-news-dataset/News_Dataset\"\n\nprint(f\"  Scanning: {ISOT_DIR}\")\nisot_files = []\nfor root, dirs, files in os.walk(ISOT_DIR):\n    for f in files:\n        full_path = os.path.join(root, f)\n        size_mb   = os.path.getsize(full_path) / 1e6\n        print(f\"    📄 {f}  [{size_mb:.2f} MB]  →  {full_path}\")\n        isot_files.append(full_path)\n\n# Load ISOT based on discovered structure\ndf3 = None\nif not isot_files:\n    print(\"  ❌ No files found inside News_Dataset directory!\")\nelse:\n    # Case A: separate True.csv + Fake.csv\n    true_files = [f for f in isot_files\n                  if os.path.basename(f).lower() in (\"true.csv\", \"real.csv\")]\n    fake_files = [f for f in isot_files\n                  if os.path.basename(f).lower() in (\"fake.csv\", \"false.csv\")]\n    merged_files = [f for f in isot_files\n                    if f.endswith(\".csv\") and f not in true_files + fake_files]\n\n    if true_files and fake_files:\n        print(f\"\\n  ✅ Case A: Separate True/Fake CSVs found\")\n        df3_true = pd.read_csv(true_files[0])\n        df3_fake = pd.read_csv(fake_files[0])\n        df3_true[\"label_str\"] = \"REAL\"\n        df3_fake[\"label_str\"] = \"FAKE\"\n        df3 = pd.concat([df3_true, df3_fake], ignore_index=True)\n        print(f\"     True: {df3_true.shape}  |  Fake: {df3_fake.shape}\")\n        print(f\"     Merged DS3 shape: {df3.shape}\")\n\n    elif merged_files:\n        print(f\"\\n  ✅ Case B: Single merged CSV found\")\n        df3 = pd.read_csv(merged_files[0])\n        print(f\"     Shape: {df3.shape}\")\n        print(f\"     Columns: {list(df3.columns)}\")\n        # Auto-detect label column\n        lbl_candidates = [c for c in df3.columns\n                          if c.lower() in (\"label\", \"class\", \"category\",\n                                           \"fake\", \"real\", \"target\")]\n        if lbl_candidates:\n            raw_lbl = lbl_candidates[0]\n            df3[\"label_str\"] = (df3[raw_lbl].astype(str)\n                                .str.upper().str.strip())\n            # Normalise numeric labels if needed\n            df3[\"label_str\"] = df3[\"label_str\"].replace(\n                {\"0\": \"FAKE\", \"1\": \"REAL\", \"FALSE\": \"FAKE\", \"TRUE\": \"REAL\"})\n            print(f\"     Label col: '{raw_lbl}'  →  \"\n                  f\"{df3['label_str'].value_counts().to_dict()}\")\n        else:\n            print(\"  ⚠️  No label column auto-detected — \"\n                  \"manually set df3['label_str']\")\n\n    else:\n        # Fallback: try loading each CSV\n        for fpath in isot_files:\n            if fpath.endswith(\".csv\"):\n                try:\n                    df3 = pd.read_csv(fpath)\n                    print(f\"  ✅ Fallback load: {fpath}  shape={df3.shape}\")\n                    break\n                except Exception as e:\n                    print(f\"  ❌ {fpath}: {e}\")\n\n    if df3 is not None:\n        print(f\"\\n  DS3 columns: {list(df3.columns)}\")\n        print(f\"  DS3 shape  : {df3.shape}\")\n\n\n# =============================================================================\n# STEP 2 — RELOAD DS1, DS2, DS4, DS5\n# (Using confirmed paths from EDA)\n# =============================================================================\nprint(\"\\n\\n\" + \"─\" * 70)\nprint(\"STEP 2 │ Reload DS1, DS2, DS4, DS5\")\nprint(\"─\" * 70)\n\n# ── DS1: WELFake ──────────────────────────────────────────────────────────────\nprint(\"\\n▶ DS1 — WELFake\")\ndf1 = pd.read_csv(\n    \"/kaggle/input/datasets/saurabhshahane/fake-news-classification/WELFake_Dataset.csv\"\n)\ndf1 = df1.loc[:, ~df1.columns.str.contains(\"^Unnamed\")]\ndf1[\"label_str\"] = df1[\"label\"].map(\n    {0: \"FAKE\", 1: \"REAL\", \"0\": \"FAKE\", \"1\": \"REAL\"})\nprint(f\"  ✅ shape={df1.shape}  |  labels: {df1['label_str'].value_counts().to_dict()}\")\n\n# ── DS2: Fake or Real News ─────────────────────────────────────────────────────\nprint(\"\\n▶ DS2 — Fake or Real News\")\ndf2 = pd.read_csv(\n    \"/kaggle/input/datasets/jillanisofttech/fake-or-real-news/fake_or_real_news.csv\"\n)\ndf2 = df2.loc[:, ~df2.columns.str.contains(\"^Unnamed\")]\ndf2[\"label_str\"] = df2[\"label\"].astype(str).str.upper().str.strip()\nprint(f\"  ✅ shape={df2.shape}  |  labels: {df2['label_str'].value_counts().to_dict()}\")\n\n# ── DS4: Getting Real About Fake News ─────────────────────────────────────────\nprint(\"\\n▶ DS4 — Getting Real (fake-only)\")\ndf4 = pd.read_csv(\n    \"/kaggle/input/datasets/mrisdal/fake-news/fake.csv\",\n    encoding=\"latin-1\"\n)\ndf4 = df4.loc[:, ~df4.columns.str.contains(\"^Unnamed\")]\ndf4[\"label_str\"] = \"FAKE\"\nprint(f\"  ✅ shape={df4.shape}\")\n\n# ── DS5: LIAR ─────────────────────────────────────────────────────────────────\nprint(\"\\n▶ DS5 — LIAR\")\nLIAR_COLS = [\n    \"id\", \"label\", \"statement\", \"subject\", \"speaker\",\n    \"speaker_job\", \"state\", \"party\",\n    \"barely_true_counts\", \"false_counts\", \"half_true_counts\",\n    \"mostly_true_counts\", \"pants_fire_counts\", \"context\"\n]\nLIAR_FAKE = {\"pants-fire\", \"false\", \"barely-true\"}\nLIAR_REAL = {\"half-true\", \"mostly-true\", \"true\"}\n\nliar_parts = {}\nfor split, fpath in {\n    \"train\": \"/kaggle/input/datasets/doanquanvietnamca/liar-dataset/train.tsv\",\n    \"valid\": \"/kaggle/input/datasets/doanquanvietnamca/liar-dataset/valid.tsv\",\n    \"test\":  \"/kaggle/input/datasets/doanquanvietnamca/liar-dataset/test.tsv\",\n}.items():\n    part = pd.read_csv(fpath, sep=\"\\t\", header=None,\n                       names=LIAR_COLS, quoting=3)\n    liar_parts[split] = part\n    print(f\"  ✅ {split}: shape={part.shape}\")\n\ndf5 = pd.concat(liar_parts.values(), ignore_index=True)\ndf5[\"label_str\"] = df5[\"label\"].apply(\n    lambda x: \"FAKE\" if str(x).strip().lower() in LIAR_FAKE\n              else (\"REAL\" if str(x).strip().lower() in LIAR_REAL\n                    else \"UNKNOWN\")\n)\ndf5 = df5[df5[\"label_str\"] != \"UNKNOWN\"].copy()\nprint(f\"  ✅ Combined LIAR shape={df5.shape}\")\n\n\n# =============================================================================\n# STEP 3 — DS4: BUILD REAL-NEWS COMPLEMENT\n# EDA confirmed DS4 is 100% FAKE.\n# Sample matching volume from ISOT True.csv to pair with it.\n# Filter DS4 to English only first (EDA: 95.4% English, 596 non-English).\n# =============================================================================\nprint(\"\\n\\n\" + \"─\" * 70)\nprint(\"STEP 3 │ DS4 — Build Real-News Complement\")\nprint(\"─\" * 70)\n\n# Filter DS4 to English only\nprint(f\"  DS4 before language filter: {len(df4):,} rows\")\nif \"language\" in df4.columns:\n    df4 = df4[df4[\"language\"].str.lower() == \"english\"].copy()\n    print(f\"  DS4 after English filter  : {len(df4):,} rows\")\n\n# Use DS3 True articles (or WELFake REAL articles) as complement\n# Priority: use ISOT if loaded, else WELFake REAL\nif df3 is not None and \"label_str\" in df3.columns:\n    isot_real = df3[df3[\"label_str\"] == \"REAL\"].copy()\n    print(f\"\\n  ISOT REAL pool: {len(isot_real):,} articles\")\n    complement_source = \"ISOT REAL\"\nelse:\n    isot_real = df1[df1[\"label_str\"] == \"REAL\"].copy()\n    print(f\"\\n  WELFake REAL pool used as fallback: {len(isot_real):,} articles\")\n    complement_source = \"WELFake REAL\"\n\n# Sample the same number of articles as DS4 fake count\nn_fake_ds4 = len(df4)\nn_sample   = min(n_fake_ds4, len(isot_real))\nreal_complement = isot_real.sample(n=n_sample, random_state=42)\nreal_complement[\"label_str\"] = \"REAL\"\n\n# Detect the text column in complement source\nif \"text\" in real_complement.columns:\n    real_complement = real_complement[[\"text\", \"label_str\"]].copy()\nelif \"statement\" in real_complement.columns:\n    real_complement = real_complement[[\"statement\", \"label_str\"]].copy()\n    real_complement.rename(columns={\"statement\": \"text\"}, inplace=True)\n\nprint(f\"  Sampled {n_sample:,} REAL articles from {complement_source}\")\nprint(f\"  DS4 FAKE articles      : {n_fake_ds4:,}\")\nprint(f\"  Balance ratio          : {n_fake_ds4/n_sample:.3f}×  ✅\")\n\n\n# =============================================================================\n# STEP 4 — TEXT CLEANING FUNCTION\n# Applied uniformly to all 5 datasets.\n# =============================================================================\nprint(\"\\n\\n\" + \"─\" * 70)\nprint(\"STEP 4 │ Text Cleaning Function Definition\")\nprint(\"─\" * 70)\n\ndef clean_text(text: str) -> str:\n    \"\"\"\n    Full preprocessing pipeline:\n    1. Lowercase\n    2. Remove HTML tags\n    3. Remove URLs\n    4. Remove email addresses\n    5. Remove Twitter handles and hashtags\n    6. Remove non-ASCII characters\n    7. Remove numbers (standalone)\n    8. Remove extra whitespace / punctuation\n    9. Tokenize\n    10. Remove stopwords\n    11. Lemmatize\n    12. Reconstruct cleaned string\n    \"\"\"\n    if not isinstance(text, str) or text.strip() == \"\":\n        return \"\"\n\n    # 1. Lowercase\n    text = text.lower()\n\n    # 2. Remove HTML tags\n    text = re.sub(r\"<[^>]+>\", \" \", text)\n\n    # 3. Remove URLs (http, https, www, ftp)\n    text = re.sub(\n        r\"(https?://|ftp://|www\\.)\\S+|\"\n        r\"\\b\\w+\\.(com|org|net|gov|edu|io|co)\\b\\S*\",\n        \" \", text\n    )\n\n    # 4. Remove email addresses\n    text = re.sub(r\"\\S+@\\S+\\.\\S+\", \" \", text)\n\n    # 5. Remove Twitter handles and hashtags\n    text = re.sub(r\"@\\w+|#\\w+\", \" \", text)\n\n    # 6. Remove non-ASCII characters\n    text = re.sub(r\"[^\\x00-\\x7F]+\", \" \", text)\n\n    # 7. Remove standalone numbers (keep numbers inside words)\n    text = re.sub(r\"\\b\\d+\\b\", \" \", text)\n\n    # 8. Remove punctuation (keep apostrophes for contractions)\n    text = re.sub(r\"[^a-z\\s']\", \" \", text)\n\n    # 9. Collapse multiple spaces\n    text = re.sub(r\"\\s+\", \" \", text).strip()\n\n    # 10-12. Tokenize → remove stopwords → lemmatize\n    tokens = word_tokenize(text)\n    tokens = [\n        lemmatizer.lemmatize(tok)\n        for tok in tokens\n        if tok not in STOP_WORDS and len(tok) > 2\n    ]\n\n    return \" \".join(tokens)\n\n# Quick smoke test\n_test = \"BREAKING: <b>Trump said</b> visit http://example.com for $50 deals!!!\"\nprint(f\"  Smoke test input : {_test}\")\nprint(f\"  Smoke test output: {clean_text(_test)}\")\n\n\n# =============================================================================\n# STEP 5 — PER-DATASET PREPROCESSING\n# For each dataset:\n#   a) Select text + label columns\n#   b) Remove nulls, empties, short texts, duplicates\n#   c) Apply clean_text()\n#   d) Encode label → 0=FAKE, 1=REAL\n#   e) Print before/after stats\n# =============================================================================\nprint(\"\\n\\n\" + \"─\" * 70)\nprint(\"STEP 5 │ Per-Dataset Cleaning\")\nprint(\"─\" * 70)\n\ndef preprocess_dataset(df, text_col, label_col, ds_name,\n                        min_words=3, deduplicate=True):\n    \"\"\"\n    Standardised cleaning pipeline for a single dataset.\n    Returns a DataFrame with columns ['text_raw', 'text_clean', 'label'].\n    \"\"\"\n    print(f\"\\n  {'─'*55}\")\n    print(f\"  Processing: {ds_name}\")\n    print(f\"  {'─'*55}\")\n    n_start = len(df)\n    print(f\"  Initial rows : {n_start:,}\")\n\n    # (a) Select and rename\n    df_out = df[[text_col, label_col]].copy()\n    df_out.rename(columns={text_col: \"text_raw\",\n                           label_col: \"label_str\"}, inplace=True)\n\n    # (b-i) Drop nulls\n    df_out = df_out[df_out[\"text_raw\"].notna()].copy()\n    print(f\"  After null drop       : {len(df_out):,}  \"\n          f\"(removed {n_start - len(df_out):,})\")\n\n    # (b-ii) Drop empty strings\n    df_out = df_out[df_out[\"text_raw\"].astype(str).str.strip() != \"\"].copy()\n    print(f\"  After empty drop      : {len(df_out):,}\")\n\n    # (b-iii) Drop texts shorter than min_words\n    df_out[\"_wc\"] = df_out[\"text_raw\"].astype(str).str.split().str.len()\n    n_before = len(df_out)\n    df_out = df_out[df_out[\"_wc\"] >= min_words].copy()\n    print(f\"  After <{min_words} words drop  : {len(df_out):,}  \"\n          f\"(removed {n_before - len(df_out):,})\")\n    df_out.drop(columns=[\"_wc\"], inplace=True)\n\n    # (b-iv) Drop full-row duplicates\n    if deduplicate:\n        n_before = len(df_out)\n        df_out.drop_duplicates(inplace=True)\n        print(f\"  After full-row dedup  : {len(df_out):,}  \"\n              f\"(removed {n_before - len(df_out):,})\")\n\n        # Drop text-level duplicates (keep first occurrence)\n        n_before = len(df_out)\n        df_out.drop_duplicates(subset=[\"text_raw\"], keep=\"first\", inplace=True)\n        print(f\"  After text dedup      : {len(df_out):,}  \"\n              f\"(removed {n_before - len(df_out):,})\")\n\n    # (c) Apply text cleaning\n    print(f\"  Applying clean_text() ...\")\n    df_out[\"text_clean\"] = df_out[\"text_raw\"].astype(str).apply(clean_text)\n\n    # Drop rows where cleaning produced empty output\n    n_before = len(df_out)\n    df_out = df_out[df_out[\"text_clean\"].str.strip() != \"\"].copy()\n    print(f\"  After clean empty drop: {len(df_out):,}  \"\n          f\"(removed {n_before - len(df_out):,})\")\n\n    # (d) Encode label → integer\n    df_out[\"label\"] = df_out[\"label_str\"].map({\"FAKE\": 0, \"REAL\": 1})\n    # Warn if any unmapped labels\n    unmapped = df_out[\"label\"].isna().sum()\n    if unmapped > 0:\n        print(f\"  ⚠️  {unmapped} rows with unmapped labels — dropping\")\n        df_out = df_out[df_out[\"label\"].notna()].copy()\n    df_out[\"label\"] = df_out[\"label\"].astype(int)\n\n    # (e) Final stats\n    final_n = len(df_out)\n    label_dist = df_out[\"label_str\"].value_counts().to_dict()\n    removed_total = n_start - final_n\n    print(f\"\\n  ✅ Final rows  : {final_n:,}  \"\n          f\"(removed {removed_total:,} = {removed_total/n_start*100:.1f}%)\")\n    print(f\"     Label dist  : {label_dist}\")\n    if len(label_dist) >= 2:\n        vals = list(label_dist.values())\n        ratio = max(vals) / min(vals)\n        print(f\"     Imbalance   : {ratio:.3f}×\")\n\n    # Final schema: keep both raw (for inspection) and clean (for modelling)\n    return df_out[[\"text_raw\", \"text_clean\", \"label_str\", \"label\"]].reset_index(drop=True)\n\n\n# ── DS1: WELFake ──────────────────────────────────────────────────────────────\ndf1_clean = preprocess_dataset(df1, \"text\", \"label_str\", \"DS1 – WELFake\")\n\n# ── DS2: Fake or Real News ─────────────────────────────────────────────────────\ndf2_clean = preprocess_dataset(df2, \"text\", \"label_str\",\n                                \"DS2 – Fake or Real News\")\n\n# ── DS3: ISOT ─────────────────────────────────────────────────────────────────\nif df3 is not None:\n    # Identify text column\n    txt_candidates = [c for c in df3.columns\n                      if c.lower() in (\"text\", \"content\", \"body\", \"article\")]\n    txt_col_ds3 = txt_candidates[0] if txt_candidates else \"text\"\n    df3_clean = preprocess_dataset(df3, txt_col_ds3, \"label_str\",\n                                   \"DS3 – ISOT\")\nelse:\n    df3_clean = None\n    print(\"\\n  ⚠️  DS3 skipped — not loaded (check STEP 1 output)\")\n\n# ── DS4: Getting Real (FAKE) + Real complement ─────────────────────────────────\n# Combine fake and real complement into one DS4 dataframe\ndf4_fake_clean_input = df4[[\"text\", \"label_str\"]].copy()\ndf4_combined = pd.concat(\n    [df4_fake_clean_input, real_complement],\n    ignore_index=True\n)\ndf4_clean = preprocess_dataset(df4_combined, \"text\", \"label_str\",\n                                \"DS4 – Getting Real + REAL complement\")\n\n# ── DS5: LIAR ─────────────────────────────────────────────────────────────────\ndf5_clean = preprocess_dataset(df5, \"statement\", \"label_str\",\n                                \"DS5 – LIAR\", min_words=2)\n\n\n# =============================================================================\n# STEP 6 — POST-CLEAN VALIDATION\n# Confirm schema, check for any remaining nulls or anomalies.\n# =============================================================================\nprint(\"\\n\\n\" + \"─\" * 70)\nprint(\"STEP 6 │ Post-Clean Validation\")\nprint(\"─\" * 70)\n\nCLEANED = {\n    \"DS1 – WELFake\":           df1_clean,\n    \"DS2 – Fake or Real News\": df2_clean,\n    \"DS3 – ISOT\":              df3_clean,\n    \"DS4 – Getting Real\":      df4_clean,\n    \"DS5 – LIAR\":              df5_clean,\n}\n\nvalidation_rows = []\nfor ds_name, df in CLEANED.items():\n    print(f\"\\n  {ds_name}\")\n    if df is None:\n        print(\"    ⚠️  None — not available\")\n        validation_rows.append({\n            \"Dataset\": ds_name, \"Rows\": \"—\",\n            \"Null clean\": \"—\", \"Null label\": \"—\",\n            \"Label dist\": \"—\", \"Imbalance\": \"—\",\n            \"Avg words (clean)\": \"—\"\n        })\n        continue\n\n    n = len(df)\n    null_clean = df[\"text_clean\"].isna().sum()\n    null_label = df[\"label\"].isna().sum()\n    label_dist = df[\"label_str\"].value_counts().to_dict()\n    vals = list(label_dist.values())\n    ratio = round(max(vals) / min(vals), 3) if len(vals) >= 2 else \"N/A\"\n    avg_w = round(\n        df[\"text_clean\"].astype(str).str.split().str.len().mean(), 1)\n\n    status = \"✅\" if null_clean == 0 and null_label == 0 else \"⚠️\"\n    print(f\"    {status} rows={n:,} | null_clean={null_clean} | \"\n          f\"null_label={null_label} | imbalance={ratio}× | avg_words={avg_w}\")\n    print(f\"       label_dist: {label_dist}\")\n\n    validation_rows.append({\n        \"Dataset\":          ds_name.split(\"–\")[1].strip(),\n        \"Rows\":             f\"{n:,}\",\n        \"Null clean\":       null_clean,\n        \"Null label\":       null_label,\n        \"Label dist\":       str(label_dist),\n        \"Imbalance\":        f\"{ratio}×\",\n        \"Avg words (clean)\": avg_w,\n    })\n\nprint(\"\\n\\n  ── Validation Summary Table ──\")\nprint(pd.DataFrame(validation_rows).to_string(index=False))\n\n\n# =============================================================================\n# STEP 7 — WORD COUNT COMPARISON: BEFORE vs AFTER CLEANING\n# =============================================================================\nprint(\"\\n\\n\" + \"─\" * 70)\nprint(\"STEP 7 │ Word Count Before vs After Cleaning\")\nprint(\"─\" * 70)\n\navailable = {k: v for k, v in CLEANED.items() if v is not None}\nn_ds = len(available)\n\nfig, axes = plt.subplots(1, n_ds, figsize=(5 * n_ds, 5))\nif n_ds == 1:\n    axes = [axes]\nfig.suptitle(\"Word Count: Raw vs Cleaned Text\", fontsize=13, y=1.02)\n\nfor ax, (ds_name, df) in zip(axes, available.items()):\n    short = ds_name.split(\"–\")[1].strip()\n    ax.set_title(short, fontsize=10)\n\n    raw_wc   = df[\"text_raw\"].astype(str).str.split().str.len()\n    clean_wc = df[\"text_clean\"].astype(str).str.split().str.len()\n\n    clip = int(raw_wc.quantile(0.97))\n    ax.hist(raw_wc.clip(upper=clip),   bins=40, alpha=0.55,\n            label=f\"Raw (μ={raw_wc.mean():.0f})\",   color=\"#888\")\n    ax.hist(clean_wc.clip(upper=clip), bins=40, alpha=0.65,\n            label=f\"Clean (μ={clean_wc.mean():.0f})\", color=\"#2a78d6\")\n    ax.set_xlabel(\"Word count\")\n    ax.set_ylabel(\"Frequency\")\n    ax.legend(fontsize=8)\n\nplt.tight_layout()\nsave_fig(\"08_wordcount_raw_vs_clean.png\")\nplt.show()\n\n\n# =============================================================================\n# STEP 8 — CLASS DISTRIBUTION AFTER CLEANING\n# =============================================================================\nprint(\"\\n\\n\" + \"─\" * 70)\nprint(\"STEP 8 │ Class Distribution After Cleaning\")\nprint(\"─\" * 70)\n\nfig, axes = plt.subplots(1, n_ds, figsize=(5 * n_ds, 5))\nif n_ds == 1:\n    axes = [axes]\nfig.suptitle(\"Class Distribution After Full Preprocessing\", fontsize=13, y=1.02)\n\nfor ax, (ds_name, df) in zip(axes, available.items()):\n    short = ds_name.split(\"–\")[1].strip()\n    ax.set_title(short, fontsize=10)\n    counts = df[\"label_str\"].value_counts()\n    colors = [PALETTE.get(l, \"#999\") for l in counts.index]\n    bars   = ax.bar(counts.index.astype(str), counts.values,\n                    color=colors, edgecolor=\"white\")\n    for bar, val in zip(bars, counts.values):\n        ax.text(bar.get_x() + bar.get_width() / 2,\n                bar.get_height() + counts.max() * 0.012,\n                f\"{val:,}\", ha=\"center\", va=\"bottom\", fontsize=9)\n    if len(counts) >= 2:\n        ratio = counts.max() / counts.min()\n        ax.set_xlabel(f\"Imbalance {ratio:.3f}×\", fontsize=9)\n    ax.set_ylabel(\"Count\")\n    ax.yaxis.set_major_formatter(\n        mticker.FuncFormatter(lambda x, _: f\"{int(x):,}\"))\n\nplt.tight_layout()\nsave_fig(\"09_class_distribution_post_clean.png\")\nplt.show()\n\n\n# =============================================================================\n# STEP 9 — SAVE CLEANED DATASETS\n# Each saved as a CSV with columns: text_raw, text_clean, label_str, label\n# Also save a unified combined dataset for cross-dataset experiments.\n# =============================================================================\nprint(\"\\n\\n\" + \"─\" * 70)\nprint(\"STEP 9 │ Save Cleaned Datasets\")\nprint(\"─\" * 70)\n\nSAVE_NAMES = {\n    \"DS1 – WELFake\":           (df1_clean, \"ds1_welfake_clean.csv\"),\n    \"DS2 – Fake or Real News\": (df2_clean, \"ds2_fake_or_real_clean.csv\"),\n    \"DS3 – ISOT\":              (df3_clean, \"ds3_isot_clean.csv\"),\n    \"DS4 – Getting Real\":      (df4_clean, \"ds4_getting_real_clean.csv\"),\n    \"DS5 – LIAR\":              (df5_clean, \"ds5_liar_clean.csv\"),\n}\n\nsaved_dfs = []\nfor ds_name, (df, fname) in SAVE_NAMES.items():\n    fpath = os.path.join(SAVE_DIR, fname)\n    if df is not None:\n        df.to_csv(fpath, index=False)\n        size_mb = os.path.getsize(fpath) / 1e6\n        print(f\"  💾 {fname:<40}  {len(df):>7,} rows  [{size_mb:.2f} MB]\")\n        # Tag with source dataset for combined file\n        tagged = df.copy()\n        tagged[\"source\"] = ds_name.split(\"–\")[1].strip()\n        saved_dfs.append(tagged)\n    else:\n        print(f\"  ⚠️  {fname:<40}  SKIPPED — no data\")\n\n# Save unified combined dataset (all available datasets)\nif saved_dfs:\n    combined = pd.concat(saved_dfs, ignore_index=True)\n    combined_path = os.path.join(SAVE_DIR, \"all_datasets_combined.csv\")\n    combined.to_csv(combined_path, index=False)\n    size_mb = os.path.getsize(combined_path) / 1e6\n    print(f\"\\n  💾 {'all_datasets_combined.csv':<40}  \"\n          f\"{len(combined):>7,} rows  [{size_mb:.2f} MB]\")\n    print(f\"\\n  Combined label distribution:\")\n    print(combined[\"label_str\"].value_counts().to_string())\n    ratio_combined = (combined[\"label_str\"].value_counts().max() /\n                      combined[\"label_str\"].value_counts().min())\n    print(f\"  Overall imbalance: {ratio_combined:.3f}×\")\n\n\n# =============================================================================\n# STEP 10 — FINAL PREPROCESSING SUMMARY\n# =============================================================================\nprint(\"\\n\\n\" + \"─\" * 70)\nprint(\"STEP 10 │ Final Summary & Next Steps\")\nprint(\"─\" * 70)\n\n\n\nprint(\"=\" * 70)\nprint(\"  PREPROCESSING COMPLETE\")\nprint(\"=\" * 70)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-17T08:45:16.946663Z","iopub.execute_input":"2026-07-17T08:45:16.947005Z","iopub.status.idle":"2026-07-17T08:54:38.014877Z","shell.execute_reply.started":"2026-07-17T08:45:16.946973Z","shell.execute_reply":"2026-07-17T08:54:38.014102Z"},"editable":false},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true,"editable":false},"outputs":[],"execution_count":null}]}