{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":117097,"databundleVersionId":14080503,"sourceType":"competition"},{"sourceId":13280781,"sourceType":"datasetVersion","datasetId":8416714}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# =========================================================\n# Alpha Radar: Solana Sprint — CatBoost Baseline (FIXED) + EDA + Submission\n# =========================================================\nimport os, gc, random\nfrom pathlib import Path\nimport numpy as np\nimport pandas as pd\n\nSEED = 42\nrandom.seed(SEED); np.random.seed(SEED)\n\nCOMP_DIR = Path(\"/kaggle/input/alpha-radar-solana-sprint\")\nFULL_DIR = Path(\"/kaggle/input/pumpfun-30s-september-2025\")\nWORK_DIR = Path(\"/kaggle/working\"); WORK_DIR.mkdir(parents=True, exist_ok=True)\n\nTARGET_TOKENS_GDRIVE_ID = \"1EsqpZXPBU-6m0djDmccCrtUX07jV2fHA\"\nTARGET_TOKENS_CSV = WORK_DIR / \"target_tokens.csv\"\n\nCHUNKSIZE = 250_000\n\n# ---- deps\ntry:\n    from catboost import CatBoostClassifier, Pool\nexcept:\n    !pip -q install --no-input catboost\n    from catboost import CatBoostClassifier, Pool\n\ntry:\n    import gdown; HAVE_GDOWN = True\nexcept:\n    try:\n        !pip -q install --no-input gdown\n        import gdown; HAVE_GDOWN = True\n    except:\n        HAVE_GDOWN = False\n\nprint(f\"CatBoost OK | gdown: {HAVE_GDOWN}\")\n\n# -----------------------\n# 1) Column setup\n# -----------------------\nKEY_COL  = \"mint_token_id\"\nTIME_COL = \"timestamp\"\n\nNUMERIC_CANDIDATES = [\n    \"index\",\"token_quantity\",\"creator_fee\",\"creator_fee_pump\",\"market_cap_usd\",\n    \"token_delta\",\"sol_delta\",\"buy_count\",\"sell_count\",\"total_count\",\n    \"token_volume\",\"sol_volume\",\"liquidity_ratio\",\"virtual_sol_reserves\",\"virtual_token_reserves\",\n    \"consumed_gas\",\"fee\",\"relative_strength_index\",\"bollinger_relative_position\",\"volume_oscillator\",\n    \"rate_of_change\",\"money_flow_index\",\"total_holders\",\"current_holders\",\"top10_percent_total\",\n    \"creator_balance\",\"creator_sold\",\"holder_ratio\",\"buy_sell_ratio\",\n]\nSTRING_DROP = [\"holder\",\"creator\",\"trade_mode\"]  # baseline bỏ cột quá nhiều giá trị khác nhau\n\ndef ensure_numeric(df, cols):\n    use = [c for c in cols if c in df.columns]\n    for c in use:\n        if not np.issubdtype(df[c].dtype, np.number):\n            df[c] = pd.to_numeric(df[c], errors=\"coerce\")\n    return use\n\n# -----------------------\n# 2) Eval reader\n# -----------------------\ndef read_eval_chunks(eval_dir: Path):\n    paths = sorted(eval_dir.glob(\"evaluation_set_30s_chunk_*.csv\"))\n    assert len(paths) == 5, f\"Expected 5 eval chunks, got {len(paths)}\"\n    dfs = [pd.read_csv(p) for p in paths]\n    return pd.concat(dfs, ignore_index=True)\n\n# -----------------------\n# 3) FULL dataset files\n# -----------------------\ndef list_full_paths(full_dir: Path):\n    paths = sorted(full_dir.glob(\"september_2025_first30s_chunk_*.csv\"))\n    assert len(paths) > 0, \"No full dataset CSVs found\"\n    return paths\n\n# -----------------------\n# 4) Chunk aggregator (NO MERGE CHAINS)\n# -----------------------\ndef aggregate_chunk(chunk: pd.DataFrame):\n    \"\"\"Return a per-token aggregate for one chunk:\n       - numeric *_sum, *_max, *_min\n       - row_count\n       - t_min, t_max (if timestamp present)\n    \"\"\"\n    # drop heavy string cols\n    drop_cols = [c for c in STRING_DROP if c in chunk.columns]\n    if drop_cols:\n        chunk = chunk.drop(columns=drop_cols)\n\n    # numeric block\n    num_cols = ensure_numeric(chunk, NUMERIC_CANDIDATES)\n    chunk[num_cols] = chunk[num_cols].fillna(0)\n\n    num_agg = chunk[[KEY_COL] + num_cols].groupby(KEY_COL).agg([\"sum\",\"max\",\"min\"])\n    num_agg.columns = [f\"{c}__{stat}\" for c, stat in num_agg.columns.to_flat_index()]\n    num_agg = num_agg.reset_index()\n\n    # row_count\n    cnt = chunk.groupby(KEY_COL, as_index=False).size().rename(columns={\"size\":\"row_count\"})\n\n    # time span (optional)\n    if TIME_COL in chunk.columns:\n        # parse robustly; warnings ok\n        ts = pd.to_datetime(chunk[TIME_COL], utc=True, errors=\"coerce\")\n        tdf = pd.DataFrame({KEY_COL: chunk[KEY_COL].values, TIME_COL: ts})\n        tspan = tdf.groupby(KEY_COL)[TIME_COL].agg([\"min\",\"max\"]).reset_index()\n        tspan = tspan.rename(columns={\"min\":\"t_min\",\"max\":\"t_max\"})\n        # merge inside this chunk\n        out = num_agg.merge(cnt, on=KEY_COL, how=\"left\").merge(tspan, on=KEY_COL, how=\"left\")\n    else:\n        out = num_agg.merge(cnt, on=KEY_COL, how=\"left\")\n        out[\"t_min\"] = pd.NaT\n        out[\"t_max\"] = pd.NaT\n\n    return out\n\ndef combine_aggregates(df_list):\n    \"\"\"Concat all per-chunk aggregates, then final groupby to combine:\n       - *_sum → sum\n       - *_max → max\n       - *_min → min\n       - row_count → sum\n       - t_min → min, t_max → max\n    \"\"\"\n    all_df = pd.concat(df_list, ignore_index=True)\n    # collect columns by suffix\n    sum_cols = [c for c in all_df.columns if c.endswith(\"__sum\")]\n    max_cols = [c for c in all_df.columns if c.endswith(\"__max\")]\n    min_cols = [c for c in all_df.columns if c.endswith(\"__min\")]\n    keep = [KEY_COL, \"row_count\", \"t_min\", \"t_max\"] + sum_cols + max_cols + min_cols\n    keep = [c for c in keep if c in all_df.columns]\n    all_df = all_df[keep]\n\n    agg_dict = {c: \"sum\" for c in sum_cols}\n    agg_dict.update({c: \"max\" for c in max_cols})\n    agg_dict.update({c: \"min\" for c in min_cols})\n    if \"row_count\" in all_df.columns: agg_dict[\"row_count\"] = \"sum\"\n    if \"t_min\"     in all_df.columns: agg_dict[\"t_min\"]   = \"min\"\n    if \"t_max\"     in all_df.columns: agg_dict[\"t_max\"]   = \"max\"\n\n    out = all_df.groupby(KEY_COL, as_index=False).agg(agg_dict)\n    # lifespan\n    if \"t_min\" in out.columns and \"t_max\" in out.columns:\n        out[\"lifespan_seconds\"] = (out[\"t_max\"] - out[\"t_min\"]).dt.total_seconds()\n        out[\"lifespan_seconds\"] = out[\"lifespan_seconds\"].fillna(0)\n    else:\n        out[\"lifespan_seconds\"] = 0.0\n    return out.fillna(0)\n\ndef build_token_features_from_files(paths, verbose_every=3):\n    agg_list = []\n    for i, p in enumerate(paths, 1):\n        print(f\"[Build-feats] {p.name} ({i}/{len(paths)})\")\n        for chunk in pd.read_csv(p, chunksize=CHUNKSIZE):\n            agg = aggregate_chunk(chunk)\n            agg_list.append(agg)\n            del chunk, agg\n            gc.collect()\n        if i % verbose_every == 0:\n            print(f\"  → partial groups stored: {len(agg_list)}\")\n    feats = combine_aggregates(agg_list)\n    return feats\n\n# -----------------------\n# 5) Load eval + build features\n# -----------------------\nprint(\"=== Load EVAL ===\")\neval_df = read_eval_chunks(COMP_DIR)\neval_ids = eval_df[KEY_COL].drop_duplicates().reset_index(drop=True)\nprint(\"EVAL rows:\", len(eval_df), \"| unique tokens:\", len(eval_ids))\nassert len(eval_ids) == 64208, f\"Expected 64208 eval tokens, got {len(eval_ids)}\"\ndel eval_df; gc.collect()\n\nprint(\"=== Build FULL features (train) ===\")\nfull_paths = list_full_paths(FULL_DIR)\ntrain_feats = build_token_features_from_files(full_paths, verbose_every=2)\nprint(\"Train feats:\", train_feats.shape)\n\nprint(\"=== Build EVAL features (test) ===\")\neval_paths = sorted(COMP_DIR.glob(\"evaluation_set_30s_chunk_*.csv\"))\neval_feats = build_token_features_from_files(eval_paths, verbose_every=3)\neval_feats = eval_feats[eval_feats[KEY_COL].isin(set(eval_ids))]\nprint(\"Eval feats:\", eval_feats.shape)\n\n# -----------------------\n# 6) Targets\n# -----------------------\ndef download_target_tokens_csv(dst_path: Path):\n    if dst_path.exists(): return True\n    if not HAVE_GDOWN:\n        print(\"gdown not available — upload target_tokens.csv to /kaggle/working/\")\n        return False\n    url = f\"https://drive.google.com/uc?id={TARGET_TOKENS_GDRIVE_ID}\"\n    gdown.download(url, str(dst_path), quiet=False)\n    return dst_path.exists() and dst_path.stat().st_size > 0\n\nok = download_target_tokens_csv(TARGET_TOKENS_CSV)\nif not ok:\n    raise SystemExit(\"Missing target_tokens.csv\")\n\ndef load_target_list(path: Path):\n    df = pd.read_csv(path)\n    # đoán tên cột token\n    cands = [c for c in df.columns if \"mint\" in c.lower() or \"token\" in c.lower()]\n    assert cands, \"Cannot find token id column in target csv\"\n    col = cands[0]\n    t = df[[col]].dropna().drop_duplicates()\n    t = t.rename(columns={col: KEY_COL})\n    t[KEY_COL] = t[KEY_COL].astype(str)\n    print(\"Targets:\", len(t))\n    return t\n\ntarget_list = load_target_list(TARGET_TOKENS_CSV)\n\ntrain_df = train_feats.merge(target_list.assign(is_target=1), on=KEY_COL, how=\"left\")\ntrain_df[\"is_target\"] = train_df[\"is_target\"].fillna(0).astype(int)\n\n# (optional) tạo vài mean đơn giản từ sum/row_count\ndef add_simple_means(df):\n    if \"row_count\" not in df.columns or df[\"row_count\"].eq(0).all():\n        return df\n    for base in NUMERIC_CANDIDATES:\n        s = f\"{base}__sum\"\n        if s in df.columns:\n            df[f\"{base}__mean\"] = df[s] / df[\"row_count\"].clip(lower=1)\n    return df\n\ntrain_df = add_simple_means(train_df)\neval_feats = add_simple_means(eval_feats)\n\n# Align feature columns\nfeature_cols = [c for c in train_df.columns if c not in [KEY_COL, \"is_target\", \"t_min\", \"t_max\"]]\n# ensure numeric\nfor df_ in (train_df, eval_feats):\n    for c in feature_cols:\n        if c in df_.columns and (df_[c].dtype == object):\n            df_[c] = pd.to_numeric(df_[c], errors=\"coerce\").fillna(0)\n\n# intersect features\nfeature_cols = [c for c in feature_cols if c in eval_feats.columns]\n\nX = train_df[feature_cols].fillna(0)\ny = train_df[\"is_target\"].astype(int)\nX_test = eval_feats[feature_cols].fillna(0)\ntest_ids = eval_feats[KEY_COL].values\n\nprint(\"Train shape:\", X.shape, \"| positives:\", int(y.sum()))\nprint(\"Test  shape:\", X_test.shape)\n\n# -----------------------\n# 7) EDA nhanh\n# -----------------------\npos = int(y.sum()); n = len(y)\nprint(f\"Class balance: pos={pos} ({pos/n:.2%})  neg={n-pos} ({1-pos/n:.2%})\")\nprint(\"Top-10 var features:\\n\", X.var().sort_values(ascending=False).head(10))\n\n# -----------------------\n# 8) Train CatBoost\n# -----------------------\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score, average_precision_score\n\nX_tr, X_va, y_tr, y_va = train_test_split(\n    X, y, test_size=0.2, random_state=SEED, stratify=y\n)\n\ntrain_pool = Pool(X_tr, y_tr)\nvalid_pool = Pool(X_va, y_va)\n\nparams = dict(\n    loss_function=\"Logloss\",\n    eval_metric=\"AUC\",\n    random_seed=SEED,\n    learning_rate=0.05,\n    depth=8,\n    l2_leaf_reg=3.0,\n    iterations=2000,\n    subsample=0.8,\n    rsm=0.8,\n    od_type=\"Iter\",\n    od_wait=100,\n    verbose=200,\n)\nmodel = CatBoostClassifier(**params)\nmodel.fit(train_pool, eval_set=valid_pool, use_best_model=True)\n\nva_pred = model.predict_proba(X_va)[:,1]\nprint(f\"Valid AUC={roc_auc_score(y_va, va_pred):.4f} | AP={average_precision_score(y_va, va_pred):.4f}\")\n\n# chọn threshold baseline\nTHRESH = 0.5\n\n# -----------------------\n# 9) Inference + submission\n# -----------------------\ntest_pred = model.predict_proba(X_test)[:,1]\nsub_df = pd.DataFrame({KEY_COL: test_ids, \"is_target\": (test_pred >= THRESH).astype(int)})\nassert len(sub_df) == 64208, f\"Submission must have 64208 rows, got {len(sub_df)}\"\nsub_path = WORK_DIR / \"submission.csv\"\nsub_df.to_csv(sub_path, index=False)\nprint(\"Saved submission:\", sub_path, sub_df.shape)\ndisplay(sub_df.head())\n\n# (optional) prediction details for deliverables\npred_detail = pd.DataFrame({\n    KEY_COL: test_ids,\n    \"pred_score\": test_pred,\n    \"threshold\": THRESH,\n    \"is_target_pred\": (test_pred >= THRESH).astype(int),\n})\npred_detail.to_csv(WORK_DIR / \"pred_detail_eval.csv\", index=False)\nprint(\"Saved:\", WORK_DIR / \"pred_detail_eval.csv\")\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null}]}