{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":117097,"databundleVersionId":14080503,"sourceType":"competition"},{"sourceId":13280781,"sourceType":"datasetVersion","datasetId":8416714}],"dockerImageVersionId":31153,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-10-19T05:41:31.600162Z","iopub.execute_input":"2025-10-19T05:41:31.600674Z","iopub.status.idle":"2025-10-19T05:41:32.251201Z","shell.execute_reply.started":"2025-10-19T05:41:31.600639Z","shell.execute_reply":"2025-10-19T05:41:32.250108Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\n\n# List of folders where CSVs are stored\nfolders = [\n   \"/kaggle/input/alpha-radar-solana-sprint\"\n]\n\nall_csvs = []\nfor folder in folders:\n    for file in os.listdir(folder):\n        if file.endswith(\".csv\") and file != \"Sample_Dataset.csv\":\n            all_csvs.append(os.path.join(folder, file))\n\n# Concatenate\ndf = pd.concat([pd.read_csv(f) for f in all_csvs], ignore_index=True)\n\nprint(\"Final shape:\", df.shape)\ndf.head()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T05:41:58.159135Z","iopub.execute_input":"2025-10-19T05:41:58.161114Z","iopub.status.idle":"2025-10-19T05:42:14.548045Z","shell.execute_reply.started":"2025-10-19T05:41:58.160994Z","shell.execute_reply":"2025-10-19T05:42:14.546553Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# faithful_polars_port_improved.py\n# Faithful Polars front-end with per-token pandas-based group computation\n# Produces token-level features identical to the original pandas pipeline.\n#\n# Requirements:\n#   pip install polars pandas numpy scipy\n\nimport polars as pl\nimport pandas as pd\nimport numpy as np\nfrom math import sqrt\nfrom scipy.stats import linregress\nimport warnings\nwarnings.filterwarnings(\"ignore\", category=RuntimeWarning)\n\n\n# --------------------------\n# Utility helpers (same semantics as original)\n# --------------------------\n# import pandas as pd\n# import numpy as np\n\ndef parse_timestamp_to_seconds(ts, reference=None):\n    \"\"\"\n    Converts a timestamp or time string into total seconds.\n    - Supports datetime strings like '2025-10-06 05:19:11.492000+00:00'\n    - Supports MM:SS(.ms) or HH:MM:SS formats\n    - If `reference` is provided, subtracts it (to get relative seconds)\n    \"\"\"\n    if pd.isna(ts):\n        return np.nan\n\n    # Numeric input\n    if isinstance(ts, (int, float, np.number)):\n        return float(ts)\n\n    s = str(ts).strip()\n\n    # Handle datetime-like strings\n    try:\n        dt = pd.to_datetime(s, errors=\"raise\", utc=True)\n        if reference is not None:\n            dt0 = pd.to_datetime(reference, utc=True)\n            return (dt - dt0).total_seconds()\n        else:\n            return dt.timestamp()   # absolute seconds since epoch\n    except Exception:\n        pass\n\n    # Handle MM:SS or HH:MM:SS style\n    if \":\" in s:\n        parts = s.split(\":\")\n        try:\n            parts = [float(p) for p in parts]\n        except:\n            return np.nan\n        sec = parts[-1]\n        total = sec\n        mul = 60.0\n        for p in parts[-2::-1]:\n            total += p * mul\n            mul *= 60.0\n        return float(total)\n\n    # Fallback: try numeric\n    try:\n        return float(s)\n    except:\n        return np.nan\n\n\ndef rms(series: pd.Series):\n    arr = series.dropna().values\n    if arr.size == 0:\n        return np.nan\n    return float(np.sqrt(np.mean(np.square(arr))))\n\n\ndef safe_first(series: pd.Series):\n    s = series.dropna()\n    return s.iloc[0] if len(s) else np.nan\n\n\ndef safe_last(series: pd.Series):\n    s = series.dropna()\n    return s.iloc[-1] if len(s) else np.nan\n\n\ndef linear_slope(x: np.ndarray, y: np.ndarray):\n    mask = np.isfinite(x) & np.isfinite(y)\n    x, y = x[mask], y[mask]\n    if len(x) < 2:\n        return np.nan\n    if np.allclose(x, x[0]):\n        return 0.0\n    if np.allclose(y, y[0]):\n        return 0.0\n    try:\n        slope, _, _, _, _ = linregress(x, y)\n        return float(slope)\n    except Exception:\n        return np.nan\n\n\ndef series_stats(s: pd.Series) -> dict:\n    out = {}\n    arr = s.dropna().values\n    if arr.size == 0:\n        for k in ['min', 'max', 'mean', 'median', 'sum', 'std', 'var', 'skew', 'kurt', 'first', 'last', 'rms']:\n            out[k] = np.nan\n        return out\n    out['min'] = float(np.nanmin(arr))\n    out['max'] = float(np.nanmax(arr))\n    out['mean'] = float(np.nanmean(arr))\n    out['median'] = float(np.nanmedian(arr))\n    out['sum'] = float(np.nansum(arr))\n    out['std'] = float(np.nanstd(arr, ddof=1)) if arr.size > 1 else 0.0\n    out['var'] = float(np.nanvar(arr, ddof=1)) if arr.size > 1 else 0.0\n    out['skew'] = float(pd.Series(arr).skew()) if arr.size > 2 else 0.0\n    out['kurt'] = float(pd.Series(arr).kurt()) if arr.size > 3 else 0.0\n    nz = s.dropna()\n    out['first'] = float(nz.iloc[0]) if len(nz) > 0 else np.nan\n    out['last'] = float(nz.iloc[-1]) if len(nz) > 0 else np.nan\n    out['rms'] = float(rms(pd.Series(arr)))\n    return out\n\n\n# --------------------------\n# The group function (runs per token) using pandas to replicate original behavior\n# --------------------------\ndef compute_token_row(pdf: pd.DataFrame, token_col: str, holder_col: str, present_numeric_cols: list):\n    \"\"\"\n    pdf: pandas DataFrame for a single token group (already filtered to _rel_t <= 30)\n    returns: dict row with token-level features\n    \"\"\"\n    row = {}\n    token = pdf[token_col].iloc[0] if token_col in pdf.columns and len(pdf) > 0 else None\n    row[token_col] = token\n\n    sub = pdf.sort_values('_rel_t', kind='mergesort').copy()\n    tx_count = len(sub)\n    row['tx_count_30s'] = int(tx_count)\n    row['txs_per_sec'] = float(tx_count) / 30.0\n\n    if tx_count > 0:\n        row['time_span'] = float(sub['_rel_t'].max() - sub['_rel_t'].min())\n        row['first_trade_rel_t'] = float(sub['_rel_t'].min())\n        row['last_trade_rel_t'] = float(sub['_rel_t'].max())\n    else:\n        row['time_span'] = np.nan\n        row['first_trade_rel_t'] = np.nan\n        row['last_trade_rel_t'] = np.nan\n\n    # holders\n    if holder_col in sub.columns:\n        row['unique_holders_count'] = int(sub[holder_col].nunique())\n        if tx_count > 0:\n            vc = sub[holder_col].value_counts()\n            top_addr_count = int(vc.iloc[0]) if len(vc) > 0 else 0\n        else:\n            top_addr_count = 0\n        row['max_trades_by_one_holder'] = top_addr_count\n    else:\n        row['unique_holders_count'] = np.nan\n        row['max_trades_by_one_holder'] = np.nan\n\n    # per-numeric stats\n    for c in present_numeric_cols:\n        if c in sub.columns:\n            stats_out = series_stats(sub[c])\n        else:\n            stats_out = {k: np.nan for k in ['min', 'max', 'mean', 'median', 'sum', 'std', 'var', 'skew', 'kurt', 'first', 'last', 'rms']}\n        for k, v in stats_out.items():\n            row[f'{c}_{k}'] = v\n        # delta\n        try:\n            row[f'{c}_delta'] = (stats_out['last'] - stats_out['first']) if (not np.isnan(stats_out['last']) and not np.isnan(stats_out['first'])) else np.nan\n        except:\n            row[f'{c}_delta'] = np.nan\n\n        # slope over time for this feature\n        try:\n            valid_mask = sub[c].notna() & sub['_rel_t'].notna()\n            if valid_mask.sum() >= 2:\n                xs = sub.loc[valid_mask, '_rel_t'].values.astype(float)\n                ys = sub.loc[valid_mask, c].values.astype(float)\n                row[f'{c}_slope'] = linear_slope(xs, ys)\n            else:\n                row[f'{c}_slope'] = np.nan\n        except Exception:\n            row[f'{c}_slope'] = np.nan\n\n    # price_proxy\n    if ('sol_delta' in sub.columns) and ('token_quantity' in sub.columns):\n        def safe_div(a, b):\n            try:\n                return float(a) / float(b) if (pd.notna(a) and pd.notna(b) and b != 0) else np.nan\n            except:\n                return np.nan\n\n        price_proxy = sub.apply(lambda r: safe_div(r.get('sol_delta', np.nan), r.get('token_quantity', np.nan)), axis=1)\n        pp_stats = series_stats(price_proxy)\n        for k, v in pp_stats.items():\n            row[f'price_proxy_{k}'] = v\n        row['price_proxy_delta'] = (pp_stats['last'] - pp_stats['first']) if (not np.isnan(pp_stats['last']) and not np.isnan(pp_stats['first'])) else np.nan\n        row['price_proxy_rms'] = pp_stats['rms']\n        # slope\n        try:\n            valid_mask = (~pd.isna(price_proxy)) & sub['_rel_t'].notna()\n            if valid_mask.sum() >= 2:\n                xs = sub.loc[valid_mask, '_rel_t'].values.astype(float)\n                ys = price_proxy[valid_mask].values.astype(float)\n                row['price_proxy_slope'] = linear_slope(xs, ys)\n            else:\n                row['price_proxy_slope'] = np.nan\n        except:\n            row['price_proxy_slope'] = np.nan\n    else:\n        for k in ['min', 'max', 'mean', 'median', 'sum', 'std', 'var', 'skew', 'kurt', 'first', 'last', 'rms', 'delta', 'slope']:\n            row[f'price_proxy_{k}'] = np.nan\n\n    # bucketed features (0-10,10-20,20-30)\n    buckets = [(0.0, 10.0), (10.0, 20.0), (20.0, 30.0)]\n    for i, (start, end) in enumerate(buckets):\n        bmask = (sub['_rel_t'] >= start) & (sub['_rel_t'] < end)\n        bucket_df = sub[bmask]\n        row[f'bucket_{i + 1}_tx_count'] = int(len(bucket_df))\n        if 'token_volume' in bucket_df.columns:\n            row[f'bucket_{i + 1}_token_volume_sum'] = float(bucket_df['token_volume'].sum()) if len(bucket_df) > 0 else 0.0\n        else:\n            row[f'bucket_{i + 1}_token_volume_sum'] = np.nan\n        if 'sol_volume' in bucket_df.columns:\n            row[f'bucket_{i + 1}_sol_volume_sum'] = float(bucket_df['sol_volume'].sum()) if len(bucket_df) > 0 else 0.0\n        else:\n            row[f'bucket_{i + 1}_sol_volume_sum'] = np.nan\n        if 'buy_count' in bucket_df.columns:\n            row[f'bucket_{i + 1}_buy_sum'] = int(bucket_df['buy_count'].sum()) if len(bucket_df) > 0 else 0\n        else:\n            row[f'bucket_{i + 1}_buy_sum'] = np.nan\n        if 'sell_count' in bucket_df.columns:\n            row[f'bucket_{i + 1}_sell_sum'] = int(bucket_df['sell_count'].sum()) if len(bucket_df) > 0 else 0\n        else:\n            row[f'bucket_{i + 1}_sell_sum'] = np.nan\n\n    # time-to-50% volume and fraction in first 10s\n    if 'token_volume' in sub.columns and sub['token_volume'].notna().any():\n        sorted_sub = sub.sort_values('_rel_t', kind='mergesort')\n        if 'token_volume' in sorted_sub.columns:\n            cum = sorted_sub['token_volume'].cumsum()\n            total_vol = float(cum.iloc[-1]) if len(cum) > 0 else 0.0\n            if total_vol > 0:\n                half_idx = int(np.searchsorted(cum.values, total_vol * 0.5))\n                t_half = float(sorted_sub['_rel_t'].iloc[half_idx]) if half_idx < len(sorted_sub) else float(sorted_sub['_rel_t'].iloc[-1])\n                row['time_to_50pct_volume'] = t_half\n                v_first10 = float(sorted_sub[sorted_sub['_rel_t'] < 10]['token_volume'].sum())\n                row['fraction_volume_first10s'] = float(v_first10 / total_vol)\n            else:\n                row['time_to_50pct_volume'] = np.nan\n                row['fraction_volume_first10s'] = np.nan\n        else:\n            row['time_to_50pct_volume'] = np.nan\n            row['fraction_volume_first10s'] = np.nan\n    else:\n        row['time_to_50pct_volume'] = np.nan\n        row['fraction_volume_first10s'] = np.nan\n\n    # Derived ratios & interactions\n    buy_last = row.get('buy_count_last', np.nan)\n    sell_last = row.get('sell_count_last', np.nan)\n    tot_count_last = row.get('total_count_last', np.nan)\n\n    try:\n        row['buy_minus_sell'] = (buy_last - sell_last) if (not pd.isna(buy_last) and not pd.isna(sell_last)) else np.nan\n    except:\n        row['buy_minus_sell'] = np.nan\n    try:\n        if not pd.isna(tot_count_last):\n            row['buy_frac_of_total'] = (buy_last / tot_count_last) if (not pd.isna(buy_last) and tot_count_last > 0) else np.nan\n        else:\n            row['buy_frac_of_total'] = np.nan\n    except:\n        row['buy_frac_of_total'] = np.nan\n\n    mv_std = row.get('market_cap_usd_std', np.nan)\n    tv_sum = row.get('token_volume_sum', np.nan)\n    if (not pd.isna(mv_std)) and (not pd.isna(tv_sum)):\n        row['volatility_x_volume'] = mv_std * tv_sum\n    else:\n        row['volatility_x_volume'] = np.nan\n\n    lq_first = row.get('liquidity_ratio_first', np.nan)\n    lq_last = row.get('liquidity_ratio_last', np.nan)\n    if (not pd.isna(lq_first)) and (not pd.isna(lq_last)):\n        row['liquidity_ratio_change'] = lq_last - lq_first\n        row['liquidity_ratio_pct_change'] = (lq_last - lq_first) / (abs(lq_first) + 1e-9)\n    else:\n        row['liquidity_ratio_change'] = np.nan\n        row['liquidity_ratio_pct_change'] = np.nan\n\n    cr_sold = row.get('creator_sold_last', np.nan)\n    cr_bal = row.get('creator_balance_last', np.nan)\n    token_vol_sum = row.get('token_volume_sum', np.nan)\n    try:\n        row['creator_sold_to_balance'] = (cr_sold / (cr_bal + 1e-9)) if (not pd.isna(cr_sold) and not pd.isna(cr_bal)) else np.nan\n    except:\n        row['creator_sold_to_balance'] = np.nan\n    try:\n        row['creator_sold_frac_volume'] = (cr_sold / token_vol_sum) if (not pd.isna(cr_sold) and not pd.isna(token_vol_sum) and token_vol_sum > 0) else np.nan\n    except:\n        row['creator_sold_frac_volume'] = np.nan\n\n    top10_last = row.get('top10_percent_total_last', np.nan)\n    top10_first = row.get('top10_percent_total_first', np.nan)\n    if (not pd.isna(top10_last)) and (not pd.isna(top10_first)):\n        row['top10_pct_change'] = top10_last - top10_first\n    else:\n        row['top10_pct_change'] = np.nan\n\n    row['is_sparse'] = 1 if tx_count <= 2 else 0\n    row['sparse_tx_count'] = tx_count\n\n    bsum = row.get('buy_count_sum', np.nan)\n    ssum = row.get('sell_count_sum', np.nan)\n    if (not pd.isna(bsum)) and (not pd.isna(ssum)) and (not pd.isna(row['unique_holders_count'])):\n        try:\n            row['wash_trade_proxy'] = ((bsum + ssum) / (row['unique_holders_count'] + 1e-9)) * row['max_trades_by_one_holder']\n        except:\n            row['wash_trade_proxy'] = np.nan\n    else:\n        row['wash_trade_proxy'] = np.nan\n\n    return row\n\n\n# --------------------------\n# Main pipeline using Polars for IO + grouping\n# --------------------------\ndef fe_token_level_full_parity(polars_df: pl.DataFrame,\n                               time_col: str = 'timestamp',\n                               token_col: str = 'mint_token_id',\n                               holder_col: str = 'holder'):\n    # ensure in-memory polars DataFrame\n    df = polars_df.lazy().collect()\n\n    # parse timestamp into seconds with same semantics\n    df = df.with_columns(\n        pl.col(time_col).map_elements(lambda x: parse_timestamp_to_seconds(x), return_dtype=pl.Float64).alias('_ts_seconds')\n    )\n\n    # _rel_t relative to first per token (window)\n    df = df.with_columns(\n        (pl.col('_ts_seconds') - pl.col('_ts_seconds').min().over(token_col)).alias('_rel_t')\n    )\n\n    # version-compatible clip: replace negative _rel_t with 0.0\n    df = df.with_columns(\n        pl.when(pl.col('_rel_t') < 0.0)\n          .then(0.0)\n          .otherwise(pl.col('_rel_t'))\n          .alias('_rel_t')\n    )\n\n    # keep only <= 30s window\n    df = df.filter(pl.col('_rel_t') <= 30.0)\n\n    # numeric columns to consider (same list as original)\n    numeric_cols = [\n        'token_quantity', 'token_delta', 'sol_delta', 'token_volume', 'sol_volume',\n        'liquidity_ratio', 'virtual_sol_reserves', 'virtual_token_reserves',\n        'consumed_gas', 'fee', 'relative_strength_index', 'bollinger_relative_position',\n        'volume_oscillator', 'rate_of_change', 'money_flow_index',\n        'market_cap_usd', 'creator_fee', 'creator_fee_pump',\n        'creator_balance', 'creator_sold', 'total_holders', 'current_holders',\n        'top10_percent_total', 'holder_ratio', 'buy_sell_ratio',\n        # also support some count columns if present\n        'buy_count', 'sell_count', 'total_count'\n    ]\n    present_numeric_cols = [c for c in numeric_cols if c in df.columns]\n\n    # build a list of token groups\n    tokens = df.select(pl.col(token_col)).unique().to_series().to_list()\n    print(f\"Found {len(tokens)} tokens. Aggregating per-token groups...\")\n\n    token_rows = []\n    # convert entire polars df to pandas once for fast per-token slicing in pandas\n    pdf_all = df.to_pandas()\n\n    for i, token in enumerate(tokens):\n        sub_pdf = pdf_all[pdf_all[token_col] == token].copy()\n        # ensure sorted by _rel_t (mergesort stable)\n        sub_pdf = sub_pdf.sort_values('_rel_t', kind='mergesort')\n        row = compute_token_row(sub_pdf, token_col, holder_col, present_numeric_cols)\n        token_rows.append(row)\n        if (i + 1) % 500 == 0:\n            print(f\"Processed {i + 1} / {len(tokens)} tokens...\")\n\n    # build pandas DataFrame from rows then convert to polars\n    token_feats_pdf = pd.DataFrame(token_rows)\n\n    # Replace infinite with NaN\n    token_feats_pdf.replace([np.inf, -np.inf], np.nan, inplace=True)\n\n    # add log transforms for specific bases if present\n    for base in ['token_volume_sum', 'sol_volume_sum', 'market_cap_usd_sum', 'token_quantity_sum']:\n        if base in token_feats_pdf.columns:\n            token_feats_pdf[f'log_{base}'] = token_feats_pdf[base].apply(lambda x: np.log1p(x) if pd.notna(x) and x > 0 else np.nan)\n\n    # reorder columns to put counts/sparse first\n    cols = list(token_feats_pdf.columns)\n    pref = ['tx_count_30s', 'txs_per_sec', 'is_sparse', 'sparse_tx_count', 'unique_holders_count', 'max_trades_by_one_holder']\n    for p in pref[::-1]:\n        if p in cols:\n            cols.insert(0, cols.pop(cols.index(p)))\n    token_feats_pdf = token_feats_pdf[cols]\n\n    # convert to polars for output (keep token_col as a regular column)\n    token_features_pl = pl.from_pandas(token_feats_pdf.reset_index(drop=False))\n\n    return token_features_pl\n\n\n# --------------------------\n# Example usage\n# --------------------------\nif __name__ == \"__main__\":\n    RAW_PATH = \"/kaggle/working/converted_data4.csv\"  # change to your filename\n    OUT_PATH = \"token_features_full_parity4.csv\"\n\n    print(\"Reading CSV with Polars (explicit schema where useful) ...\")\n\n    # you can customize this map if you have a printed schema; otherwise we force common numeric columns to float\n    numeric_cols = [\n        'token_quantity', 'token_delta', 'sol_delta', 'token_volume', 'sol_volume',\n        'liquidity_ratio', 'virtual_sol_reserves', 'virtual_token_reserves',\n        'consumed_gas', 'fee', 'relative_strength_index', 'bollinger_relative_position',\n        'volume_oscillator', 'rate_of_change', 'money_flow_index',\n        'market_cap_usd', 'creator_fee', 'creator_fee_pump',\n        'creator_balance', 'creator_sold', 'total_holders', 'current_holders',\n        'top10_percent_total', 'holder_ratio', 'buy_sell_ratio',\n        'buy_count', 'sell_count', 'total_count'\n    ]\n    # build schema_override map (use pl.Float64 for numeric suspects)\n    schema_overrides = {c: pl.Float64 for c in numeric_cols}\n\n    # try to read with schema_overrides; fallback to more defensive read if necessary\n    try:\n        d1=df\n        for col in d1.select_dtypes(include=[\"int64\", \"float64\", \"bool\"]).columns:\n            d1[col] = d1[col].astype(np.float32)\n        d1.insert(0, \"index\", range(1,len(d1)+1))  \n        d1.to_csv(\"converted_data.csv\", index=False)\n            \n            # Try primary read with schema overrides (fast + typed)\n        df_raw = pl.read_csv(\"/kaggle/working/converted_data.csv\", schema_overrides=schema_overrides,\n                                 infer_schema_length=20000, try_parse_dates=False)\n    except Exception as e:\n        # fallback: read with relaxed inference then coerce suspicious columns\n        print(\"Primary read failed or raised warning, retrying as utf8 then coercing numeric columns. Error:\", e)\n        df_raw = pl.read_csv(RAW_PATH, dtypes=\"utf8\", infer_schema_length=2000)\n        for c in numeric_cols:\n            if c in df_raw.columns:\n                df_raw = df_raw.with_columns(\n                    pl.col(c)\n                    .str.replace_all(\",\", \"\")\n                    .str.replace_all(r\"[^\\d\\.\\-eE\\+]\", \"\")\n                    .cast(pl.Float64, strict=False)\n                    .alias(c)\n                )\n\n    print(\"CSV loaded: rows =\", df_raw.height, \"cols =\", df_raw.width)\n\n    # Defensive: if creator_sold is boolean, convert to int for numeric ops\n    if \"creator_sold\" in df_raw.columns and df_raw[\"creator_sold\"].dtype == pl.Boolean:\n        df_raw = df_raw.with_columns(pl.col(\"creator_sold\").cast(pl.Int64).alias(\"creator_sold\"))\n\n    print(\"Computing features (parity with pandas). This will iterate per-token and may take time ...\")\n    token_feats = fe_token_level_full_parity(df_raw, time_col='timestamp', token_col='mint_token_id', holder_col='holder')\n\n    print(\"Finished. Shape:\", token_feats.shape)\n    token_feats.write_csv(OUT_PATH)\n    print(\"Saved to:\", OUT_PATH)\n\n    # quick peek\n    try:\n        print(token_feats.head(3).to_pandas().T)\n    except Exception:\n        print(token_feats.head(3))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T05:44:28.514309Z","iopub.execute_input":"2025-10-19T05:44:28.514739Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# if __name__ == \"__main__\":\n#     import os\n#     import re\n#     from pathlib import Path\n\n#     INPUT_FOLDER = Path(\"/kaggle/input/alpha-radar-solana-sprint\")\n#     # We'll write one output file per input chunk for easier discovery:\n#     # token_features_full_parity_september_chunk_010.csv ... _015.csv\n#     OUT_DIR = Path.cwd()  # change if you want a different output folder\n\n#     # chunk range you requested (inclusive)\n#     CHUNK_START = 1\n#     CHUNK_END = 5\n\n#     # filename pattern base (expects: september_2025_first30s_chunk_002.csv)\n#     pattern = re.compile(r\"evaluation_set_30s_chunk_(\\d{3})\\.csv$\", re.IGNORECASE)\n\n#     # numeric columns / schema override you had\n#     numeric_cols = [\n#         'token_quantity', 'token_delta', 'sol_delta', 'token_volume', 'sol_volume',\n#         'liquidity_ratio', 'virtual_sol_reserves', 'virtual_token_reserves',\n#         'consumed_gas', 'fee', 'relative_strength_index', 'bollinger_relative_position',\n#         'volume_oscillator', 'rate_of_change', 'money_flow_index',\n#         'market_cap_usd', 'creator_fee', 'creator_fee_pump',\n#         'creator_balance', 'creator_sold', 'total_holders', 'current_holders',\n#         'top10_percent_total', 'holder_ratio', 'buy_sell_ratio',\n#         'buy_count', 'sell_count', 'total_count'\n#     ]\n#     schema_overrides = {c: pl.Float64 for c in numeric_cols}\n\n#     # collect candidate files\n#     all_files = []\n#     for f in INPUT_FOLDER.iterdir():\n#         if f.is_file():\n#             m = pattern.search(f.name)\n#             if m:\n#                 idx = int(m.group(1))\n#                 if CHUNK_START <= idx <= CHUNK_END:\n#                     all_files.append((idx, f))\n#     all_files.sort()  # sort by chunk number\n\n#     if len(all_files) == 0:\n#         raise SystemExit(f\"No matching files found in {INPUT_FOLDER} for chunks {CHUNK_START:03d}..{CHUNK_END:03d}\")\n\n#     print(f\"Found {len(all_files)} files to process: {[p.name for (_, p) in all_files]}\")\n\n#     for idx, file_path in all_files:\n#         out_filename = f\"token_features_full_parity_september_chunk_{idx:03d}.csv\"\n#         out_path = OUT_DIR / out_filename\n\n#         print(f\"\\n--- Processing chunk {idx:03d}: {file_path.name} -> {out_filename} ---\")\n#         try:\n#             d1=df\n#             for col in d1.select_dtypes(include=[\"int64\", \"float64\", \"bool\"]).columns:\n#                 d1[col] = d1[col].astype(np.float32)\n#             d1.insert(0, \"index\", range(1,len(d1)+1))  \n#             d1.to_csv(\"converted_data.csv\", index=False)\n            \n#             # Try primary read with schema overrides (fast + typed)\n#             df_raw = pl.read_csv(\"/kaggle/working/converted_data.csv\", schema_overrides=schema_overrides,\n#                                  infer_schema_length=20000, try_parse_dates=False)\n#         except Exception as e:\n#             print(\"Primary read failed or raised warning, retrying as utf8 then coercing numeric columns. Error:\", e)\n#             d1=pd.read_csv(str(file_path))\n#             for col in d1.select_dtypes(include=[\"int64\", \"float64\", \"bool\"]).columns:\n#                 d1[col] = d1[col].astype(np.float32)\n\n#             d1.to_csv(\"converted_data.csv\", index=False)\n            \n#             df_raw = pl.read_csv(\"/kaggle/working/converted_data.csv\", dtypes=\"utf8\", infer_schema_length=2000)\n#             for c in numeric_cols:\n#                 if c in df_raw.columns:\n#                     df_raw = df_raw.with_columns(\n#                         pl.col(c)\n#                           .str.replace_all(\",\", \"\")\n#                           .str.replace_all(r\"[^\\d\\.\\-eE\\+]\", \"\")\n#                           .cast(pl.Float64, strict=False)\n#                           .alias(c)\n#                     )\n\n#         print(\"CSV loaded: rows =\", df_raw.height, \"cols =\", df_raw.width)\n\n#         # Defensive: if creator_sold is boolean, convert to int for numeric ops\n#         if \"creator_sold\" in df_raw.columns and df_raw[\"creator_sold\"].dtype == pl.Boolean:\n#             df_raw = df_raw.with_columns(pl.col(\"creator_sold\").cast(pl.Int64).alias(\"creator_sold\"))\n\n#         # Compute token features for this chunk\n#         print(\"Computing features for this chunk ...\")\n#         token_feats = fe_token_level_full_parity(df_raw, time_col='timestamp', token_col='mint_token_id', holder_col='holder')\n\n#         # Convert to pandas and write per-chunk CSV (single file per chunk)\n#         tf_pd = token_feats.to_pandas()\n#         tf_pd.replace([np.inf, -np.inf], np.nan, inplace=True)\n\n#         tf_pd.to_csv(out_path, index=False)\n#         print(f\"Wrote {len(tf_pd)} rows to {out_path}\")\n\n#         # cleanup references\n#         del df_raw, token_feats, tf_pd\n\n#     print(\"\\nAll done. Outputs saved to:\", str(OUT_DIR))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-18T19:10:13.547311Z","iopub.execute_input":"2025-10-18T19:10:13.548006Z","iopub.status.idle":"2025-10-18T19:16:14.307649Z","shell.execute_reply.started":"2025-10-18T19:10:13.547977Z","shell.execute_reply":"2025-10-18T19:16:14.306317Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# d1=pd.read_csv(\"/kaggle/input/alpha-radar-solana-sprint/evaluation_set_30s_chunk_001.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-18T19:16:14.308292Z","iopub.status.idle":"2025-10-18T19:16:14.308551Z","shell.execute_reply.started":"2025-10-18T19:16:14.308427Z","shell.execute_reply":"2025-10-18T19:16:14.308439Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# d1.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-18T18:54:33.807280Z","iopub.execute_input":"2025-10-18T18:54:33.807561Z","iopub.status.idle":"2025-10-18T18:54:33.845412Z","shell.execute_reply.started":"2025-10-18T18:54:33.807538Z","shell.execute_reply":"2025-10-18T18:54:33.844683Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# d1.dtypes","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-18T18:56:03.229933Z","iopub.execute_input":"2025-10-18T18:56:03.230233Z","iopub.status.idle":"2025-10-18T18:56:03.237271Z","shell.execute_reply.started":"2025-10-18T18:56:03.230212Z","shell.execute_reply":"2025-10-18T18:56:03.236396Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# for col in d1.select_dtypes(include=[\"int64\", \"float64\", \"bool\"]).columns:\n#                 d1[col] = d1[col].astype(np.float32)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-18T19:02:13.109058Z","iopub.execute_input":"2025-10-18T19:02:13.109812Z","iopub.status.idle":"2025-10-18T19:02:13.148508Z","shell.execute_reply.started":"2025-10-18T19:02:13.109784Z","shell.execute_reply":"2025-10-18T19:02:13.147625Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# d1.dtypes","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-18T18:56:18.232134Z","iopub.execute_input":"2025-10-18T18:56:18.232415Z","iopub.status.idle":"2025-10-18T18:56:18.239295Z","shell.execute_reply.started":"2025-10-18T18:56:18.232395Z","shell.execute_reply":"2025-10-18T18:56:18.238607Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# d1.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-18T18:57:29.634266Z","iopub.execute_input":"2025-10-18T18:57:29.634838Z","iopub.status.idle":"2025-10-18T18:57:29.640411Z","shell.execute_reply.started":"2025-10-18T18:57:29.634804Z","shell.execute_reply":"2025-10-18T18:57:29.639634Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# d1.insert(0, \"index\", range(1,len(d1)+1))  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-18T19:02:49.526534Z","iopub.execute_input":"2025-10-18T19:02:49.526842Z","iopub.status.idle":"2025-10-18T19:02:49.531683Z","shell.execute_reply.started":"2025-10-18T19:02:49.526821Z","shell.execute_reply":"2025-10-18T19:02:49.530972Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# d1.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-18T19:02:51.810201Z","iopub.execute_input":"2025-10-18T19:02:51.810472Z","iopub.status.idle":"2025-10-18T19:02:51.834530Z","shell.execute_reply.started":"2025-10-18T19:02:51.810453Z","shell.execute_reply":"2025-10-18T19:02:51.833686Z"}},"outputs":[],"execution_count":null}]}