{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":117097,"databundleVersionId":14080503,"sourceType":"competition"},{"sourceId":13280781,"sourceType":"datasetVersion","datasetId":8416714}],"dockerImageVersionId":31153,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# =========================================================\n# Alpha Radar — SOL Sprint EDA + Feature Engineering\n# (SOTA-flavored, Kaggle-friendly, hardened & verbose logs)\n# =========================================================\n# What you get:\n#  • Robust EDA (schema, missingness, heavy tails, ECDFs)\n#  • Optional label join (auto gdown) → PR/AUPRC emphasis\n#  • Univariate targets (AUPRC & Spearman)\n#  • Correlation (Spearman on log1p)\n#  • Time patterns (arrival curve & window deltas)\n#  • Leakage scan heuristics (early vs late signals)\n#  • Token-level Feature Engineering (30s):\n#      - sums/max/min, per-tx, per-sec, ratios, deltas, log1p\n#  • Artifacts: CSVs + PNGs + Markdown report for sharing\n# =========================================================\n\nimport os, gc, re, random, warnings\nfrom pathlib import Path\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nfrom sklearn.metrics import average_precision_score\n\n# ------------------ Config ------------------\nSEED = 42\nrandom.seed(SEED); np.random.seed(SEED)\nwarnings.filterwarnings(\"ignore\")\n\nWORK_DIR = Path(\"/kaggle/working\")\nSAVE_DIR = WORK_DIR / \"eda_artifacts\"; SAVE_DIR.mkdir(parents=True, exist_ok=True)\n\nCOMP_DIR  = Path(\"/kaggle/input/alpha-radar-solana-sprint\")\nFULL_DIR  = Path(\"/kaggle/input/pumpfun-30s-september-2025\")\n\nKEY_COL   = \"mint_token_id\"\nTIME_COL  = \"timestamp\"\n\nCHUNKSIZE  = 250_000\nMAX_ROWS_QC_SAMPLE = 1_000_000   # cap for heavy plots\n\n# Labels (optional auto-download like your baseline)\nTARGET_TOKENS_GDRIVE_ID = \"1EsqpZXPBU-6m0djDmccCrtUX07jV2fHA\"\nTARGET_TOKENS_CSV = WORK_DIR / \"target_tokens.csv\"\n\n# Plot defaults\nplt.rcParams[\"figure.figsize\"] = (8, 5)\nplt.rcParams[\"axes.grid\"] = True\n\n# Base numeric fields (from baseline)\nBASE_NUMS = [\n    \"index\",\"token_quantity\",\"creator_fee\",\"creator_fee_pump\",\"market_cap_usd\",\n    \"token_delta\",\"sol_delta\",\"buy_count\",\"sell_count\",\"total_count\",\n    \"token_volume\",\"sol_volume\",\"liquidity_ratio\",\"virtual_sol_reserves\",\"virtual_token_reserves\",\n    \"consumed_gas\",\"fee\",\"relative_strength_index\",\"bollinger_relative_position\",\"volume_oscillator\",\n    \"rate_of_change\",\"money_flow_index\",\"total_holders\",\"current_holders\",\"top10_percent_total\",\n    \"creator_balance\",\"creator_sold\",\"holder_ratio\",\"buy_sell_ratio\",\n]\nSTRING_DROP = [\"holder\",\"creator\",\"trade_mode\"]\n\n# Windows used across the project\nWIN_LIST = [5, 10, 20, 30]\nWIN_SUM_VARS = [\n    \"token_volume\",\"sol_volume\",\"buy_count\",\"sell_count\",\"total_count\",\n    \"token_quantity\",\"sol_delta\",\"market_cap_usd\"\n]\n\n# ------------------ Helpers ------------------\nMMSS_RE = re.compile(r\"^\\d{1,2}:\\d{2}(\\.\\d+)?$\")\n\ndef fast_parse_seconds(x: str):\n    if isinstance(x, str):\n        m = MMSS_RE.match(x)\n        if m:\n            mm, ss = x.split(\":\")\n            try:\n                return 60*int(mm) + float(ss)\n            except Exception:\n                return np.nan\n    return np.nan\n\ndef ensure_numeric(df, cols):\n    use = [c for c in cols if c in df.columns]\n    for c in use:\n        if not np.issubdtype(df[c].dtype, np.number):\n            df[c] = pd.to_numeric(df[c], errors=\"coerce\")\n    return use\n\ndef log1p_clip(s):\n    return np.log1p(np.asarray(s, dtype=\"float64\").clip(min=0))\n\ndef ecdf(y):\n    y = np.asarray(y); y = y[~np.isnan(y)]\n    if len(y) == 0: return np.array([]), np.array([])\n    x = np.sort(y); n = len(x); p = np.arange(1, n + 1) / n\n    return x, p\n\ndef bootstrap_ci_mean(x, n_boot=300, alpha=0.05, seed=SEED):\n    rng = np.random.default_rng(seed)\n    x = np.asarray(x, dtype=float); x = x[~np.isnan(x)]\n    if len(x) == 0: return (np.nan, np.nan, np.nan)\n    means = []\n    for _ in range(n_boot):\n        s = rng.choice(x, size=len(x), replace=True)\n        means.append(np.mean(s))\n    lo, hi = np.quantile(means, [alpha/2, 1-alpha/2])\n    return (np.mean(x), lo, hi)\n\ndef maybe_download_targets(dst_path: Path, file_id: str) -> bool:\n    if dst_path.exists(): return True\n    try:\n        import gdown\n        url = f\"https://drive.google.com/uc?id={file_id}\"\n        print(\"[Labels] Downloading target_tokens.csv via gdown ...\")\n        gdown.download(url, str(dst_path), quiet=False)\n        ok = dst_path.exists() and dst_path.stat().st_size > 0\n        print(\"[Labels] Download OK\" if ok else \"[Labels] Download failed\")\n        return ok\n    except Exception as e:\n        print(f\"[Labels] gdown not available / failed: {e}\")\n        return False\n\ndef list_eval_paths(eval_dir: Path):\n    paths = sorted(eval_dir.glob(\"evaluation_set_30s_chunk_*.csv\"))\n    assert len(paths) == 5, f\"Expected 5 eval chunks, got {len(paths)}\"\n    return paths\n\ndef read_eval_all(eval_dir: Path):\n    dfs = [pd.read_csv(p) for p in list_eval_paths(eval_dir)]\n    return pd.concat(dfs, ignore_index=True)\n\n# ------------------ EDA Pipeline ------------------\nprint(\">>> [1] SCHEMA & QUICK COUNTS\")\neval_df = read_eval_all(COMP_DIR)\neval_df = eval_df.drop(columns=[c for c in STRING_DROP if c in eval_df.columns], errors=\"ignore\")\n\nprint(f\"[Info] Eval shape: {eval_df.shape}\")\nprint(f\"[Info] Eval unique tokens: {eval_df[KEY_COL].nunique()}\")\n\n# Sample for plots\neval_sample = eval_df.sample(MAX_ROWS_QC_SAMPLE, random_state=SEED).reset_index(drop=True) \\\n              if len(eval_df) > MAX_ROWS_QC_SAMPLE else eval_df.copy()\n\n# Timestamp seconds for time plots\neval_sample[\"_t_seconds\"] = eval_sample[TIME_COL].map(fast_parse_seconds) if TIME_COL in eval_sample.columns else np.nan\n\n# Schema overview\nschema = pd.DataFrame({\n    \"column\": eval_sample.columns,\n    \"dtype\": [str(eval_sample[c].dtype) for c in eval_sample.columns],\n    \"n_missing\": [int(eval_sample[c].isna().sum()) for c in eval_sample.columns],\n    \"missing_rate\": [float(eval_sample[c].isna().mean()) for c in eval_sample.columns],\n    \"n_unique\": [int(eval_sample[c].nunique(dropna=True)) for c in eval_sample.columns],\n}).sort_values([\"dtype\",\"column\"])\nschema.to_csv(SAVE_DIR / \"schema_overview.csv\", index=False)\nprint(\"[Save] schema_overview.csv\")\n\n# ------------------ Labels (optional) ------------------\nprint(\"\\n>>> [2] TARGET MERGE (optional)\")\nlabels_ok = False\nif not TARGET_TOKENS_CSV.exists():\n    labels_ok = maybe_download_targets(TARGET_TOKENS_CSV, TARGET_TOKENS_GDRIVE_ID)\nelse:\n    labels_ok = True\n\nif labels_ok:\n    try:\n        tgt = pd.read_csv(TARGET_TOKENS_CSV)\n        tgt_cols = [c for c in tgt.columns if \"mint\" in c.lower() or \"token\" in c.lower()]\n        assert len(tgt_cols) > 0, \"target_tokens.csv: token id column not found\"\n        tgt = tgt[[tgt_cols[0]]].rename(columns={tgt_cols[0]: KEY_COL}).dropna()\n        tgt[KEY_COL] = tgt[KEY_COL].astype(str)\n        eval_sample[KEY_COL] = eval_sample[KEY_COL].astype(str)\n        eval_sample = eval_sample.merge(tgt.assign(is_target=1), on=KEY_COL, how=\"left\")\n        eval_sample[\"is_target\"] = eval_sample[\"is_target\"].fillna(0).astype(int)\n        print(f\"[Labels] Positives (rows): {int(eval_sample['is_target'].sum())}\")\n    except Exception as e:\n        print(f\"[Labels] Failed to load/merge targets: {e}\")\n        eval_sample[\"is_target\"] = 0\n        labels_ok = False\nelse:\n    eval_sample[\"is_target\"] = 0\n\nif not labels_ok:\n    print(\"[Labels] Unavailable → running in UNLABELED mode (label-based blocks skipped).\")\n\n# ------------------ Class imbalance ------------------\nprint(\"\\n>>> [3] CLASS IMBALANCE\")\nif labels_ok:\n    pos_rows = int(eval_sample[\"is_target\"].sum())\n    row_prev = float(eval_sample[\"is_target\"].mean())\n    tok_prev = (eval_sample[[KEY_COL, \"is_target\"]].groupby(KEY_COL)[\"is_target\"].max()).mean()\n    print(f\"[Class] Row positives: {pos_rows} ({row_prev:.3%}) | Token prevalence: {tok_prev:.3%}\")\n    with open(SAVE_DIR / \"class_imbalance.txt\",\"w\") as f:\n        f.write(f\"RowPos={pos_rows} RowPrev={row_prev:.6f} TokenPrev={tok_prev:.6f}\\n\")\n    print(\"[Save] class_imbalance.txt\")\nelse:\n    print(\"[Class] Skipped (no labels).\")\n\n# ------------------ Heavy tails ------------------\nprint(\"\\n>>> [4] HEAVY-TAIL DIAGNOSTICS (log1p + ECDFs)\")\nNUM_CAND = ensure_numeric(eval_sample, [c for c in BASE_NUMS if c in eval_sample.columns])\n\ndef plot_hist_log1p(col):\n    x = eval_sample[col].astype(float).values\n    x = np.nan_to_num(x, nan=0.0)\n    xl = log1p_clip(x)\n    m, lo, hi = bootstrap_ci_mean(x)\n    ml, lol, hil = bootstrap_ci_mean(xl)\n    # Hist log1p\n    fig, ax = plt.subplots()\n    ax.hist(xl, bins=80, alpha=0.85)\n    ax.set_title(f\"{col} — log1p\\nmean={m:.3g} [{lo:.3g},{hi:.3g}] | log1p mean={ml:.3g} [{lol:.3g},{hil:.3g}]\")\n    ax.set_xlabel(\"log1p(value)\")\n    fig.tight_layout(); fig.savefig(SAVE_DIR / f\"hist_log1p__{col}.png\"); plt.close(fig)\n    # ECDF\n    xs, ps = ecdf(xl)\n    fig, ax = plt.subplots()\n    if len(xs): ax.plot(xs, ps, lw=2)\n    ax.set_title(f\"ECDF(log1p({col}))\"); ax.set_xlabel(\"log1p(value)\"); ax.set_ylabel(\"F(x)\")\n    fig.tight_layout(); fig.savefig(SAVE_DIR / f\"ecdf_log1p__{col}.png\"); plt.close(fig)\n\nfor col in NUM_CAND[:24]:\n    plot_hist_log1p(col)\nprint(f\"[Save] {min(24,len(NUM_CAND))}x (hist+ecdf) under {SAVE_DIR}\")\n\n# ------------------ Univariate vs target ------------------\nprint(\"\\n>>> [5] UNIVARIATE SCREENING (AUPRC & Spearman)\")\nuni_df = pd.DataFrame(columns=[\"feature\",\"AUPRC\",\"Spearman\"])\nif labels_ok and eval_sample[\"is_target\"].sum() > 0:\n    rows = []\n    y = eval_sample[\"is_target\"].values.astype(int)\n    for col in NUM_CAND:\n        x = eval_sample[col].astype(\"float64\").values\n        xl = log1p_clip(x)\n        try:\n            ap = max(average_precision_score(y, xl), average_precision_score(y, -xl))\n        except Exception:\n            ap = np.nan\n        try:\n            sr = pd.Series(xl).rank(pct=True).corr(pd.Series(y), method=\"spearman\")\n        except Exception:\n            sr = np.nan\n        rows.append({\"feature\": col, \"AUPRC\": ap, \"Spearman\": sr})\n    uni_df = pd.DataFrame(rows)\n    if not uni_df.empty: uni_df = uni_df.sort_values(\"AUPRC\", ascending=False)\n    uni_df.to_csv(SAVE_DIR / \"univariate_rank_aucpr.csv\", index=False)\n    print(\"[Save] univariate_rank_aucpr.csv\")\n    print(uni_df.head(10))\nelse:\n    print(\"[Uni] Skipped (no labels).\")\n\n# ------------------ Correlation (Spearman, log1p) ------------------\nprint(\"\\n>>> [6] CORRELATION (Spearman on log1p)\")\nFEAT_FOR_CORR = [c for c in NUM_CAND if eval_sample[c].notna().mean() > 0.99][:40]\nif FEAT_FOR_CORR:\n    Xcorr = pd.DataFrame({c: log1p_clip(eval_sample[c].values) for c in FEAT_FOR_CORR})\n    corr = Xcorr.corr(method=\"spearman\")\n    corr.to_csv(SAVE_DIR / \"spearman_corr_log1p_top40.csv\")\n    print(\"[Save] spearman_corr_log1p_top40.csv\")\nelse:\n    print(\"[Corr] No dense numeric features available.\")\n\n# ------------------ Time patterns ------------------\nprint(\"\\n>>> [7] TIME-BASED PATTERNS (arrival curve + window deltas)\")\nif \"_t_seconds\" in eval_sample.columns and eval_sample[\"_t_seconds\"].notna().any():\n    buckets = (eval_sample[\"_t_seconds\"].round().clip(lower=0, upper=30)).astype(\"Int64\")\n    ts_curve = buckets.value_counts().sort_index()\n    fig, ax = plt.subplots()\n    ax.plot(ts_curve.index.astype(float), ts_curve.values, lw=2)\n    ax.set_title(\"Arrival curve (rows per second)\"); ax.set_xlabel(\"second\"); ax.set_ylabel(\"rows\")\n    fig.tight_layout(); fig.savefig(SAVE_DIR / \"arrival_curve_rows_per_second.png\"); plt.close(fig)\n    print(\"[Save] arrival_curve_rows_per_second.png\")\n\ndef plot_window_ratio(a, b, name):\n    if a not in eval_sample.columns or b not in eval_sample.columns: return\n    da = log1p_clip(eval_sample[a].values) - log1p_clip(eval_sample[b].values)\n    fig, ax = plt.subplots()\n    ax.hist(da, bins=80, alpha=0.85)\n    ax.set_title(f\"{name}: log1p({a}) - log1p({b})\")\n    fig.tight_layout(); fig.savefig(SAVE_DIR / f\"win_delta__{name}.png\"); plt.close(fig)\n\nfor v in [\"token_volume\",\"sol_volume\",\"total_count\",\"buy_count\",\"sell_count\"]:\n    for (A,B) in [(10,5),(20,10),(30,20)]:\n        plot_window_ratio(f\"{v}_w{A}_sum\", f\"{v}_w{B}_sum\", f\"{v}_{A}-{B}\")\nprint(\"[Save] window delta plots (where columns exist)\")\n\n# ------------------ Token-level quick summary ------------------\nprint(\"\\n>>> [8] TOKEN-LEVEL 30s SUMMARY (sanity)\")\ncols_small = [KEY_COL,\"token_volume\",\"sol_volume\",\"buy_count\",\"sell_count\",\"total_count\"]\ncols_small = [c for c in cols_small if c in eval_sample.columns]\ntmp = eval_sample[cols_small].copy()\nif len(tmp) > MAX_ROWS_QC_SAMPLE:\n    tmp = tmp.sample(MAX_ROWS_QC_SAMPLE, random_state=SEED)\ng = tmp.groupby(KEY_COL).agg([\"sum\",\"max\",\"mean\"])\ng.columns = [\"__\".join(c) for c in g.columns.to_flat_index()]\ng = g.reset_index()\ng.to_csv(SAVE_DIR / \"token_level_summary_small.csv\", index=False)\nprint(\"[Save] token_level_summary_small.csv\")\nif labels_ok:\n    tok_target = eval_sample.groupby(KEY_COL)[\"is_target\"].max().rename(\"y\").reset_index()\n    g2 = g.merge(tok_target, on=KEY_COL, how=\"left\")\n    cand = [c for c in g2.columns if c.startswith(\"token_volume__sum\")]\n    if cand:\n        c0 = cand[0]\n        x1 = log1p_clip(g2.loc[g2[\"y\"]==1, c0])\n        x0 = log1p_clip(g2.loc[g2[\"y\"]==0, c0])\n        fig, ax = plt.subplots()\n        ax.hist(x1, bins=80, alpha=0.6, label=\"target=1\")\n        ax.hist(x0, bins=80, alpha=0.6, label=\"target=0\")\n        ax.legend(); ax.set_title(f\"Token-level {c0} (log1p) by class\")\n        fig.tight_layout(); fig.savefig(SAVE_DIR / f\"token_level_{c0}_by_class.png\"); plt.close(fig)\n        print(f\"[Save] token_level_{c0}_by_class.png\")\n\n# ------------------ Leak-scan heuristics ------------------\nprint(\"\\n>>> [9] LEAKAGE-SCAN (early vs late)\")\ndef collect_aucpr(cols):\n    out = []\n    if not labels_ok or eval_sample[\"is_target\"].sum() == 0:\n        return pd.DataFrame(columns=[\"feature\",\"AUPRC\"])\n    y = eval_sample[\"is_target\"].values.astype(int)\n    for c in cols:\n        if c not in eval_sample.columns: continue\n        xl = log1p_clip(eval_sample[c].values)\n        try:\n            ap = max(average_precision_score(y, xl), average_precision_score(y, -xl))\n        except Exception:\n            ap = np.nan\n        out.append({\"feature\": c, \"AUPRC\": ap})\n    df = pd.DataFrame(out)\n    if not df.empty: df = df.sort_values(\"AUPRC\", ascending=False)\n    return df\n\nearly_cols = [f\"{v}_w5_sum\" for v in WIN_SUM_VARS if f\"{v}_w5_sum\" in eval_sample.columns]\nlate_cols  = []\nfor v in [\"token_volume\",\"sol_volume\",\"total_count\",\"buy_count\",\"sell_count\"]:\n    a, b = f\"{v}_w30_sum\", f\"{v}_w20_sum\"\n    if a in eval_sample.columns and b in eval_sample.columns:\n        eval_sample[f\"{v}_w30_w20_delta\"] = log1p_clip(eval_sample[a]) - log1p_clip(eval_sample[b])\n        late_cols.append(f\"{v}_w30_w20_delta\")\n\nearly_rank = collect_aucpr(early_cols)\nlate_rank  = collect_aucpr(late_cols)\n\nif not early_rank.empty:\n    early_rank.to_csv(SAVE_DIR / \"leakscan_early_aucpr.csv\", index=False)\n    print(\"[Save] leakscan_early_aucpr.csv\")\nelse:\n    print(\"[Leak] Early-window skipped/empty.\")\nif not late_rank.empty:\n    late_rank.to_csv(SAVE_DIR / \"leakscan_late_aucpr.csv\", index=False)\n    print(\"[Save] leakscan_late_aucpr.csv\")\nelse:\n    print(\"[Leak] Late-delta skipped/empty.\")\n\n# =========================================================\n# FEATURE ENGINEERING (token-level 30s) — reusable with models\n# =========================================================\nprint(\"\\n>>> [10] FEATURE ENGINEERING (token-level 30s)\")\n\ndef aggregate_chunk(chunk: pd.DataFrame):\n    drop_cols = [c for c in STRING_DROP if c in chunk.columns]\n    if drop_cols: chunk = chunk.drop(columns=drop_cols)\n\n    # time to seconds\n    t_abs = chunk[TIME_COL].map(fast_parse_seconds) if TIME_COL in chunk.columns else pd.Series(np.nan, index=chunk.index)\n    if t_abs.isna().all():\n        chunk[\"_row_order\"] = chunk.groupby(KEY_COL).cumcount().astype(float)\n        t_abs = chunk[\"_row_order\"].values.astype(float)\n    chunk[\"_t_abs\"] = t_abs\n\n    # numeric cleanup\n    num_cols = ensure_numeric(chunk, BASE_NUMS)\n    chunk[num_cols] = chunk[num_cols].fillna(0)\n\n    # time stats\n    tstats = chunk.groupby(KEY_COL)[\"_t_abs\"].agg([\"min\",\"max\",\"count\"]).reset_index()\n    tstats = tstats.rename(columns={\"min\":\"t_min_s\",\"max\":\"t_max_s\",\"count\":\"row_count\"})\n    chunk = chunk.merge(tstats[[KEY_COL,\"t_min_s\"]], on=KEY_COL, how=\"left\")\n    chunk[\"_t_rel\"] = chunk[\"_t_abs\"] - chunk[\"t_min_s\"]\n\n    # window sums\n    win_aggs = []\n    for W in WIN_LIST:\n        mask = chunk[\"_t_rel\"] <= W\n        if not mask.any(): continue\n        sub = chunk.loc[mask, [KEY_COL] + [v for v in WIN_SUM_VARS if v in chunk.columns]].copy()\n        g = sub.groupby(KEY_COL).sum()\n        g.columns = [f\"{c}_w{W}_sum\" for c in g.columns]\n        g = g.reset_index()\n        cnt = chunk.loc[mask, [KEY_COL]].groupby(KEY_COL).size().rename(f\"row_count_w{W}\").reset_index()\n        g = g.merge(cnt, on=KEY_COL, how=\"left\")\n        win_aggs.append(g)\n\n    # 30s sums/max/min\n    num_agg = chunk[[KEY_COL] + num_cols].groupby(KEY_COL).agg([\"sum\",\"max\",\"min\"])\n    num_agg.columns = [f\"{c}__{stat}\" for c, stat in num_agg.columns.to_flat_index()]\n    num_agg = num_agg.reset_index()\n\n    out = num_agg.merge(tstats, on=KEY_COL, how=\"left\")\n    for g in win_aggs: out = out.merge(g, on=KEY_COL, how=\"left\")\n    return out.fillna(0)\n\ndef combine_aggregates(df_list):\n    all_df = pd.concat(df_list, ignore_index=True)\n    sum_cols = [c for c in all_df.columns if c.endswith(\"__sum\") or re.search(r\"_w\\d+_sum$\", c)]\n    max_cols = [c for c in all_df.columns if c.endswith(\"__max\")]\n    min_cols = [c for c in all_df.columns if c.endswith(\"__min\")]\n    cnt_cols = [\"row_count\"] + [c for c in all_df.columns if c.startswith(\"row_count_w\")]\n    keep = [KEY_COL,\"t_min_s\",\"t_max_s\"] + sum_cols + max_cols + min_cols + cnt_cols\n    keep = [c for c in keep if c in all_df.columns]\n    all_df = all_df[keep]\n    agg = {c:\"sum\" for c in sum_cols + cnt_cols}\n    agg.update({c:\"max\" for c in max_cols}); agg.update({c:\"min\" for c in min_cols})\n    agg[\"t_min_s\"] = \"min\"; agg[\"t_max_s\"] = \"max\"\n    out = all_df.groupby(KEY_COL, as_index=False).agg(agg)\n    out[\"lifespan_seconds\"] = (out[\"t_max_s\"] - out[\"t_min_s\"]).clip(lower=0)\n    return out.fillna(0)\n\ndef build_token_features_from_files(paths, verbose_every=3):\n    agg_list = []\n    for i, p in enumerate(paths, 1):\n        print(f\"[FE] Reading {p.name} ({i}/{len(paths)})\")\n        for chunk in pd.read_csv(p, chunksize=CHUNKSIZE):\n            agg = aggregate_chunk(chunk)\n            agg_list.append(agg); del chunk, agg\n            gc.collect()\n        if i % verbose_every == 0:\n            print(f\"   → partial grouped chunks: {len(agg_list)}\")\n    feats = combine_aggregates(agg_list)\n    return feats\n\n# Build FE for FULL (train-ish) & EVAL (test-ish)\nfull_paths = sorted(FULL_DIR.glob(\"september_2025_first30s_chunk_*.csv\"))\nassert len(full_paths) > 0, \"No FULL CSVs found\"\neval_paths = list_eval_paths(COMP_DIR)\n\ntrain_feats = build_token_features_from_files(full_paths, verbose_every=2)\neval_feats  = build_token_features_from_files(eval_paths,  verbose_every=2)\n\n# Keep only eval tokens in eval_feats (safety)\neval_ids = pd.read_csv(eval_paths[0]).pipe(lambda df: df[[KEY_COL]].drop_duplicates())\nfor p in eval_paths[1:]:\n    t = pd.read_csv(p)[[KEY_COL]].drop_duplicates()\n    eval_ids = pd.concat([eval_ids, t], axis=0, ignore_index=True).drop_duplicates()\neval_feats = eval_feats[eval_feats[KEY_COL].isin(set(eval_ids[KEY_COL]))].reset_index(drop=True)\n\nprint(f\"[FE] train_feats: {train_feats.shape} | eval_feats: {eval_feats.shape}\")\n\n# Post-agg features\ndef add_post_agg_features(df):\n    if \"row_count\" not in df.columns: df[\"row_count\"] = 1\n    rc = df[\"row_count\"].clip(lower=1)\n\n    for base in [\"token_volume\",\"sol_volume\",\"token_quantity\",\"sol_delta\",\"market_cap_usd\",\"buy_count\",\"sell_count\",\"total_count\"]:\n        csum = f\"{base}__sum\"\n        if csum in df.columns: df[f\"{base}__per_tx\"] = df[csum] / rc\n\n    if \"buy_count__sum\" in df.columns and \"total_count__sum\" in df.columns:\n        df[\"buy_ratio\"]  = df[\"buy_count__sum\"]  / df[\"total_count__sum\"].clip(lower=1)\n    if \"sell_count__sum\" in df.columns and \"total_count__sum\" in df.columns:\n        df[\"sell_ratio\"] = df[\"sell_count__sum\"] / df[\"total_count__sum\"].clip(lower=1)\n\n    if \"lifespan_seconds\" in df.columns:\n        ls = df[\"lifespan_seconds\"].replace(0, np.nan)\n        for base in [\"token_volume\",\"sol_volume\",\"total_count\",\"buy_count\",\"sell_count\"]:\n            csum = f\"{base}__sum\"\n            if csum in df.columns: df[f\"{base}__per_sec\"] = df[csum] / ls\n        df.fillna(0, inplace=True)\n\n    for W in WIN_LIST:\n        tc = f\"total_count_w{W}_sum\"; bc = f\"buy_count_w{W}_sum\"; sc = f\"sell_count_w{W}_sum\"\n        sv = f\"sol_volume_w{W}_sum\";  tv = f\"token_volume_w{W}_sum\"; rcw= f\"row_count_w{W}\"\n        if tc in df.columns:\n            df[f\"buy_ratio_w{W}\"]   = df.get(bc, 0) / df[tc].clip(lower=1)\n            df[f\"sell_ratio_w{W}\"]  = df.get(sc, 0) / df[tc].clip(lower=1)\n            df[f\"per_tx_solv_w{W}\"] = df.get(sv, 0) / df[tc].clip(lower=1)\n            df[f\"per_tx_tokv_w{W}\"] = df.get(tv, 0) / df[tc].clip(lower=1)\n        if rcw in df.columns:\n            df[f\"avg_tokq_w{W}\"] = df.get(f\"token_quantity_w{W}_sum\", 0) / df[rcw].clip(lower=1)\n        # per-sec proxy inside window\n        df[f\"sol_per_sec_w{W}\"]  = df.get(sv, 0) / max(1, W)\n        df[f\"tokv_per_sec_w{W}\"] = df.get(tv, 0) / max(1, W)\n\n    for a,b in [(10,5),(20,10),(30,20)]:\n        for name, (cA, cB) in {\n            \"buy_ratio\":    (f\"buy_ratio_w{a}\", f\"buy_ratio_w{b}\"),\n            \"sell_ratio\":   (f\"sell_ratio_w{a}\", f\"sell_ratio_w{b}\"),\n            \"per_tx_solv\":  (f\"per_tx_solv_w{a}\", f\"per_tx_solv_w{b}\"),\n            \"per_tx_tokv\":  (f\"per_tx_tokv_w{a}\", f\"per_tx_tokv_w{b}\"),\n            \"sol_per_sec\":  (f\"sol_per_sec_w{a}\", f\"sol_per_sec_w{b}\"),\n            \"tokv_per_sec\": (f\"tokv_per_sec_w{a}\", f\"tokv_per_sec_w{b}\"),\n        }.items():\n            if cA in df.columns and cB in df.columns:\n                df[f\"{name}_delta_{a}-{b}\"] = df[cA] - df[cB]\n\n    for c in list(df.columns):\n        if c.endswith(\"__sum\") or re.search(r\"_w\\d+_sum$\", c) or c.endswith(\"__max\"):\n            df[f\"log1p_{c}\"] = np.log1p(df[c].astype(float).clip(lower=0))\n    return df\n\ntrain_feats = add_post_agg_features(train_feats)\neval_feats  = add_post_agg_features(eval_feats)\n\n# Save FE (CSV + info)\ntrain_feats.to_csv(SAVE_DIR / \"train_token_features.csv\", index=False)\neval_feats.to_csv(SAVE_DIR / \"eval_token_features.csv\",  index=False)\nprint(\"[Save] train_token_features.csv & eval_token_features.csv\")\n\n# Preview top-importance proxy = univariate AUPRC on train (if labels available)\nfe_uni = pd.DataFrame()\nif labels_ok:\n    # build token-level label frame from EVAL labels only if intersecting with FE (optional small insight)\n    # Note: your official labels map is for train universe; we demonstrate on available eval merge cautiously.\n    lab = pd.read_csv(TARGET_TOKENS_CSV)\n    lid = [c for c in lab.columns if \"mint\" in c.lower() or \"token\" in c.lower()][0]\n    lab = lab[[lid]].rename(columns={lid: KEY_COL})\n    lab[KEY_COL] = lab[KEY_COL].astype(str)\n    # For train_feats, mark positives if they appear in target list (proxy prevalence)\n    tdf = train_feats.copy()\n    tdf[KEY_COL] = tdf[KEY_COL].astype(str)\n    tdf = tdf.merge(lab.assign(y=1), on=KEY_COL, how=\"left\")\n    tdf[\"y\"] = tdf[\"y\"].fillna(0).astype(int)\n    y = tdf[\"y\"].values\n    if y.sum() > 0:\n        cand_cols = [c for c in tdf.columns if c not in [KEY_COL,\"y\",\"t_min_s\",\"t_max_s\"]]\n        rows = []\n        for col in cand_cols[:200]:  # cap for speed\n            x = tdf[col].astype(\"float64\").values\n            xl = log1p_clip(x)\n            try:\n                ap = max(average_precision_score(y, xl), average_precision_score(y, -xl))\n            except Exception:\n                ap = np.nan\n            rows.append({\"feature\": col, \"AUPRC\": ap})\n        fe_uni = pd.DataFrame(rows).sort_values(\"AUPRC\", ascending=False)\n        fe_uni.to_csv(SAVE_DIR / \"train_fe_univariate_aucpr_top.csv\", index=False)\n        print(\"[Save] train_fe_univariate_aucpr_top.csv\")\n    else:\n        print(\"[FE-Uni] Labels merged but no positives visible on train_feats key space.\")\nelse:\n    print(\"[FE-Uni] Skipped (no labels).\")\n\n# =========================================================\n# REPORT (Markdown) — so people can skim & upvote :)\n# =========================================================\nprint(\"\\n>>> [11] GENERATE MARKDOWN REPORT\")\nmd = []\nmd.append(\"# Alpha Radar — SOL Sprint: EDA + Feature Engineering\\n\")\nmd.append(\"**SOTA-flavored diagnostics** with heavy-tail awareness, PR emphasis under imbalance, and leakage-scan heuristics. This notebook exports all plots/CSVs under `eda_artifacts/` for quick review.\\n\")\nmd.append(\"## Dataset Overview\\n\")\nmd.append(f\"- Eval rows (sampled or full): **{len(eval_sample):,}**\")\nmd.append(f\"- Unique tokens (sampled): **{eval_sample[KEY_COL].nunique():,}**\")\nmd.append(f\"- Labels available: **{labels_ok}**\")\nif labels_ok:\n    md.append(f\"- Row-level prevalence (sampled): **{eval_sample['is_target'].mean():.4%}**\")\nmd.append(\"\\n## Saved Artifacts\\n\")\nmd.append(\"- `schema_overview.csv` — dtypes, missingness, cardinality\")\nmd.append(\"- Heavy-tail plots: `hist_log1p__*.png`, `ecdf_log1p__*.png`\")\nif labels_ok:\n    md.append(\"- `univariate_rank_aucpr.csv` — AUPRC & Spearman per feature (rows)\")\nmd.append(\"- `spearman_corr_log1p_top40.csv` — correlation matrix (dense features)\")\nmd.append(\"- `arrival_curve_rows_per_second.png` — micro-burst timing\")\nmd.append(\"- `win_delta__*.png` — window delta histograms\")\nmd.append(\"- `token_level_summary_small.csv` — per-token 30s summary\")\nif labels_ok:\n    md.append(\"- `leakscan_early_aucpr.csv`, `leakscan_late_aucpr.csv` — leakage heuristics\")\nmd.append(\"- **Feature Engineering outputs**: `train_token_features.csv`, `eval_token_features.csv`\")\nif not fe_uni.empty:\n    md.append(\"- `train_fe_univariate_aucpr_top.csv` — FE univariate signals (proxy)\")\nmd.append(\"\\n## Method Notes\\n\")\nmd.append(\"- Use **AUPRC** over ROC for severe imbalance; report **ECDF** to avoid histogram binning artifacts.\")\nmd.append(\"- Robust transforms (`log1p`) mitigate heavy tails; rank-based **Spearman** for non-Gaussian.\")\nmd.append(\"- **Leakage scan** contrasts early-window signals with late-only deltas.\")\nmd.append(\"- FE includes per-tx/per-sec/ratios/deltas + `log1p_*` of heavy sums.\\n\")\nwith open(SAVE_DIR / \"README_report.md\", \"w\") as f:\n    f.write(\"\\n\".join(md))\nprint(\"[Save] README_report.md\")\n\nprint(\"\\nALL DONE. Browse artifacts in:\", SAVE_DIR)\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null}]}