{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n# Use the kagglehub client library to attach Kaggle resources like competitions, datasets, and models to your session\n# Learn more about kagglehub: https://github.com/Kaggle/kagglehub/blob/main/README.md\n\nimport kagglehub\n# kagglehub.dataset_download('<owner>/<dataset-slug>')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-06-20T07:31:47.639739Z","iopub.execute_input":"2026-06-20T07:31:47.640100Z","iopub.status.idle":"2026-06-20T07:31:49.150901Z","shell.execute_reply.started":"2026-06-20T07:31:47.640052Z","shell.execute_reply":"2026-06-20T07:31:49.150030Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport re\nimport gc\nimport json\nimport time\nimport datetime as dt\nimport warnings\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\n\nwarnings.filterwarnings(\"ignore\")\npd.set_option(\"display.max_columns\", 120)\npd.set_option(\"display.width\", 220)\n\nRANDOM_STATE = 42\nCHUNKSIZE = 50_000\n\nTARGET_WINDOW_DAYS = 60\nCOHORT_WINDOWS = [30, 60, 90]\n\n# Rolling cutoff configuration\n# Too many cutoffs can become memory-heavy.\nCUTOFF_STEP_DAYS = 45\nMIN_HISTORY_DAYS_BEFORE_CUTOFF = 180\nMAX_CUTOFFS = 8\n\n# Classifier downsampling\nNEG_POS_RATIO = 30\n\n# Validation calibration\nSHRINKAGE_CANDIDATES = [0.0, 0.25, 0.50, 0.75, 1.0]\nTHRESHOLD_QUANTILES = [\n    0.00,\n    0.50,\n    0.75,\n    0.90,\n    0.95,\n    0.97,\n    0.98,\n    0.99,\n    0.995,\n    0.997,\n    0.999,\n    0.9995,\n    0.9997,\n    0.9999,\n]\n\nUSE_ZERO_FALLBACK_IF_WORSE = True\n\nINPUT_DIR = Path(\"/kaggle/input\")\nOUTPUT_DIR = Path(\"/kaggle/working\")\nOUTPUT_DIR.mkdir(parents=True, exist_ok=True)\n\nrng = np.random.default_rng(RANDOM_STATE)\n\n\n# ------------------------------------------------------------\n# 0. File selection\n# ------------------------------------------------------------\n\nall_files = [p for p in INPUT_DIR.rglob(\"*\") if p.is_file()]\n\ndef find_file(name):\n    matches = [p for p in all_files if p.name == name]\n    if not matches:\n        raise FileNotFoundError(f\"Cannot find {name}\")\n    return matches[0]\n\ntrain_path = find_file(\"train_v2.csv\")\ntest_path = find_file(\"test_v2.csv\")\nsample_path = find_file(\"sample_submission_v2.csv\")\n\nprint(\"Selected files:\")\nprint(\"train :\", train_path)\nprint(\"test  :\", test_path)\nprint(\"sample:\", sample_path)\n\nfor p in [train_path, test_path, sample_path]:\n    print(f\"{p.name}: {p.stat().st_size / 1024**2:.1f} MB\")\n\n\n# ------------------------------------------------------------\n# 1. JSON parsing helpers\n# ------------------------------------------------------------\n\nTOTALS_KEYS = [\n    \"hits\",\n    \"pageviews\",\n    \"bounces\",\n    \"newVisits\",\n    \"visits\",\n    \"transactionRevenue\",\n    \"totalTransactionRevenue\",\n]\n\nDEVICE_KEYS = [\n    \"browser\",\n    \"operatingSystem\",\n    \"deviceCategory\",\n    \"isMobile\",\n]\n\nGEO_KEYS = [\n    \"continent\",\n    \"subContinent\",\n    \"country\",\n    \"region\",\n    \"city\",\n]\n\nTRAFFIC_KEYS = [\n    \"source\",\n    \"medium\",\n    \"campaign\",\n    \"keyword\",\n    \"isTrueDirect\",\n]\n\nNUM_COLS = [\n    \"visitNumber\",\n    \"totals_hits\",\n    \"totals_pageviews\",\n    \"totals_bounces\",\n    \"totals_newVisits\",\n    \"totals_visits\",\n    \"visit_year\",\n    \"visit_month\",\n    \"visit_dayofweek\",\n]\n\nCAT_COLS = [\n    \"channelGrouping\",\n    \"device_browser\",\n    \"device_operatingSystem\",\n    \"device_deviceCategory\",\n    \"device_isMobile\",\n    \"geoNetwork_continent\",\n    \"geoNetwork_subContinent\",\n    \"geoNetwork_country\",\n    \"trafficSource_source\",\n    \"trafficSource_medium\",\n    \"trafficSource_campaign\",\n    \"trafficSource_keyword\",\n    \"trafficSource_isTrueDirect\",\n]\n\nDROP_ABSOLUTE_TIME_FEATURES = {\n    \"visit_year_sum\",\n    \"visit_year_max\",\n    \"visit_year_mean_per_session\",\n    \"visit_month_sum\",\n    \"visit_month_max\",\n    \"visit_month_mean_per_session\",\n    \"cutoff_month\",\n    \"cutoff_dayofweek\",\n}\n\ndef parse_json_fields(series, keys, prefix):\n    out = {f\"{prefix}_{k}\": [] for k in keys}\n\n    for raw in series.to_numpy():\n        if isinstance(raw, str) and raw:\n            try:\n                obj = json.loads(raw)\n            except Exception:\n                obj = {}\n        elif isinstance(raw, dict):\n            obj = raw\n        else:\n            obj = {}\n\n        for k in keys:\n            out[f\"{prefix}_{k}\"].append(obj.get(k, np.nan))\n\n    return pd.DataFrame(out, index=series.index)\n\n\ndef prepare_chunk(chunk, parse_all=True, is_train=True):\n    chunk = chunk.copy()\n    chunk[\"fullVisitorId\"] = chunk[\"fullVisitorId\"].astype(str)\n\n    if \"totals\" in chunk.columns:\n        totals = parse_json_fields(chunk[\"totals\"], TOTALS_KEYS, \"totals\")\n        chunk = chunk.drop(columns=[\"totals\"]).join(totals)\n\n    if parse_all:\n        if \"device\" in chunk.columns:\n            device = parse_json_fields(chunk[\"device\"], DEVICE_KEYS, \"device\")\n            chunk = chunk.drop(columns=[\"device\"]).join(device)\n\n        if \"geoNetwork\" in chunk.columns:\n            geo = parse_json_fields(chunk[\"geoNetwork\"], GEO_KEYS, \"geoNetwork\")\n            chunk = chunk.drop(columns=[\"geoNetwork\"]).join(geo)\n\n        if \"trafficSource\" in chunk.columns:\n            traffic = parse_json_fields(chunk[\"trafficSource\"], TRAFFIC_KEYS, \"trafficSource\")\n            chunk = chunk.drop(columns=[\"trafficSource\"]).join(traffic)\n\n    chunk[\"date_dt\"] = pd.to_datetime(chunk[\"date\"].astype(str), format=\"%Y%m%d\", errors=\"coerce\")\n    chunk[\"date_ordinal\"] = chunk[\"date_dt\"].map(lambda x: x.toordinal() if pd.notnull(x) else np.nan)\n    chunk[\"visit_year\"] = chunk[\"date_dt\"].dt.year\n    chunk[\"visit_month\"] = chunk[\"date_dt\"].dt.month\n    chunk[\"visit_dayofweek\"] = chunk[\"date_dt\"].dt.dayofweek\n\n    numeric_candidates = [\n        \"visitNumber\",\n        \"totals_hits\",\n        \"totals_pageviews\",\n        \"totals_bounces\",\n        \"totals_newVisits\",\n        \"totals_visits\",\n        \"totals_transactionRevenue\",\n        \"totals_totalTransactionRevenue\",\n        \"visit_year\",\n        \"visit_month\",\n        \"visit_dayofweek\",\n        \"date_ordinal\",\n    ]\n\n    for c in numeric_candidates:\n        if c in chunk.columns:\n            chunk[c] = pd.to_numeric(chunk[c], errors=\"coerce\")\n\n    if is_train:\n        if \"totals_transactionRevenue\" in chunk.columns:\n            chunk[\"revenue\"] = chunk[\"totals_transactionRevenue\"].fillna(0.0)\n        elif \"totals_totalTransactionRevenue\" in chunk.columns:\n            chunk[\"revenue\"] = chunk[\"totals_totalTransactionRevenue\"].fillna(0.0)\n        else:\n            chunk[\"revenue\"] = 0.0\n    else:\n        chunk[\"revenue\"] = 0.0\n\n    return chunk\n\n\n# ------------------------------------------------------------\n# 2. Aggregation helpers\n# ------------------------------------------------------------\n\ndef aggregate_chunk(chunk):\n    if len(chunk) == 0:\n        return pd.DataFrame()\n\n    g = chunk.groupby(\"fullVisitorId\", sort=False)\n\n    out = pd.DataFrame(index=g.size().index)\n    out.index.name = \"fullVisitorId\"\n\n    out[\"session_count\"] = g.size().astype(\"int32\")\n    out[\"first_date\"] = g[\"date_ordinal\"].min()\n    out[\"last_date\"] = g[\"date_ordinal\"].max()\n    out[\"revenue_sum\"] = g[\"revenue\"].sum()\n\n    for c in NUM_COLS:\n        if c in chunk.columns:\n            chunk[c] = pd.to_numeric(chunk[c], errors=\"coerce\").fillna(0)\n            out[f\"{c}_sum\"] = g[c].sum()\n            out[f\"{c}_max\"] = g[c].max()\n\n    for c in CAT_COLS:\n        if c in chunk.columns:\n            out[f\"{c}_first\"] = g[c].first().astype(\"string\")\n            out[f\"{c}_nunique\"] = g[c].nunique(dropna=True)\n\n    return out\n\n\ndef combine_chunk_aggregates(parts):\n    parts = [p for p in parts if p is not None and len(p) > 0]\n    if not parts:\n        return pd.DataFrame()\n\n    big = pd.concat(parts, axis=0)\n    g = big.groupby(level=0, sort=False)\n\n    out = pd.DataFrame(index=g.size().index)\n    out.index.name = \"fullVisitorId\"\n\n    for c in big.columns:\n        if c == \"session_count\":\n            out[c] = g[c].sum()\n        elif c == \"revenue_sum\":\n            out[c] = g[c].sum()\n        elif c == \"first_date\":\n            out[c] = g[c].min()\n        elif c == \"last_date\":\n            out[c] = g[c].max()\n        elif c.endswith(\"_sum\"):\n            out[c] = g[c].sum()\n        elif c.endswith(\"_max\"):\n            out[c] = g[c].max()\n        elif c.endswith(\"_nunique\"):\n            out[c] = g[c].max()\n        else:\n            out[c] = g[c].first()\n\n    if \"first_date\" in out.columns and \"last_date\" in out.columns:\n        out[\"visit_span_days\"] = out[\"last_date\"] - out[\"first_date\"] + 1\n        out[\"visit_span_days\"] = out[\"visit_span_days\"].replace([np.inf, -np.inf], np.nan).fillna(0)\n\n    if \"session_count\" in out.columns:\n        sum_cols = [c for c in out.columns if c.endswith(\"_sum\") and c != \"revenue_sum\"]\n        for c in sum_cols:\n            base = c[:-4]\n            out[f\"{base}_mean_per_session\"] = out[c] / out[\"session_count\"].clip(lower=1)\n\n    return out\n\n\ndef combine_series_parts(parts, name):\n    parts = [p for p in parts if p is not None and len(p) > 0]\n    if not parts:\n        return pd.Series(dtype=\"float64\", name=name)\n\n    s = pd.concat(parts, axis=0)\n    s = s.groupby(level=0, sort=False).sum()\n    s.name = name\n    return s\n\n\ndef ordinal_to_date_str(x):\n    if pd.isna(x):\n        return \"unknown\"\n    try:\n        return dt.date.fromordinal(int(x)).isoformat()\n    except Exception:\n        return \"unknown\"\n\n\ndef ordinal_to_month(x):\n    if pd.isna(x):\n        return \"unknown\"\n    try:\n        return dt.date.fromordinal(int(x)).strftime(\"%Y-%m\")\n    except Exception:\n        return \"unknown\"\n\n\n# ------------------------------------------------------------\n# 3. First pass: first visit date and train date range\n# ------------------------------------------------------------\n\ndef build_first_user_table(path):\n    header = pd.read_csv(path, nrows=0).columns.tolist()\n    usecols = [c for c in [\"date\", \"fullVisitorId\", \"totals\"] if c in header]\n\n    parts = []\n    start = time.time()\n\n    reader = pd.read_csv(\n        path,\n        usecols=usecols,\n        dtype={\"fullVisitorId\": \"str\"},\n        chunksize=CHUNKSIZE,\n        low_memory=False,\n    )\n\n    for i, chunk in enumerate(reader, start=1):\n        chunk = prepare_chunk(chunk, parse_all=False, is_train=True)\n\n        g = chunk.groupby(\"fullVisitorId\", sort=False)\n        out = pd.DataFrame(index=g.size().index)\n        out.index.name = \"fullVisitorId\"\n        out[\"session_count\"] = g.size().astype(\"int32\")\n        out[\"first_date\"] = g[\"date_ordinal\"].min()\n        out[\"last_date\"] = g[\"date_ordinal\"].max()\n        out[\"revenue_sum\"] = g[\"revenue\"].sum()\n\n        parts.append(out)\n\n        elapsed = time.time() - start\n        print(\n            f\"First pass | {path.name} | chunk {i:03d} | \"\n            f\"rows={len(chunk):,} | users_in_chunk={len(out):,} | \"\n            f\"elapsed={elapsed/60:.1f} min\"\n        )\n\n        del chunk, out\n        gc.collect()\n\n    user_table = combine_chunk_aggregates(parts)\n\n    del parts\n    gc.collect()\n\n    return user_table\n\n\nprint(\"\\n================ First pass: first visit dates ================\")\nfirst_user_table = build_first_user_table(train_path)\n\ntrain_min_ord = int(first_user_table[\"first_date\"].min())\ntrain_max_ord = int(first_user_table[\"last_date\"].max())\n\nprint(\"\\nTrain date range:\")\nprint(ordinal_to_date_str(train_min_ord), \"to\", ordinal_to_date_str(train_max_ord))\n\nstart_cutoff = train_min_ord + MIN_HISTORY_DAYS_BEFORE_CUTOFF\nend_cutoff = train_max_ord - TARGET_WINDOW_DAYS\n\nall_cutoffs = list(range(start_cutoff, end_cutoff + 1, CUTOFF_STEP_DAYS))\ncutoff_ords = all_cutoffs[-MAX_CUTOFFS:]\ncutoff_ords = sorted(set(cutoff_ords))\n\nif len(cutoff_ords) < 2:\n    raise ValueError(\"Not enough valid cutoffs. Reduce TARGET_WINDOW_DAYS or adjust cutoff settings.\")\n\nprint(\"\\nRolling cutoffs:\")\nfor c in cutoff_ords:\n    print(\n        f\"cutoff={ordinal_to_date_str(c)} | \"\n        f\"feature: <= {ordinal_to_date_str(c)} | \"\n        f\"target: {ordinal_to_date_str(c + 1)} to {ordinal_to_date_str(c + TARGET_WINDOW_DAYS)}\"\n    )\n\nfirst_date_map = first_user_table[\"first_date\"]\n\n\n# ------------------------------------------------------------\n# 4. Second pass: all-user summary, rolling windows, fixed-window cohorts\n# ------------------------------------------------------------\n\ndef build_advanced_train_tables(path, cutoff_ords, first_date_map):\n    header = pd.read_csv(path, nrows=0).columns.tolist()\n\n    wanted = [\n        \"channelGrouping\",\n        \"date\",\n        \"fullVisitorId\",\n        \"visitNumber\",\n        \"device\",\n        \"geoNetwork\",\n        \"totals\",\n        \"trafficSource\",\n    ]\n\n    usecols = [c for c in wanted if c in header]\n\n    all_parts = []\n    feature_parts = {c: [] for c in cutoff_ords}\n    target_parts = {c: [] for c in cutoff_ords}\n    cohort_revenue_parts = {w: [] for w in COHORT_WINDOWS}\n\n    start = time.time()\n\n    reader = pd.read_csv(\n        path,\n        usecols=usecols,\n        dtype={\"fullVisitorId\": \"str\"},\n        chunksize=CHUNKSIZE,\n        low_memory=False,\n    )\n\n    for i, chunk in enumerate(reader, start=1):\n        chunk = prepare_chunk(chunk, parse_all=True, is_train=True)\n\n        # Overall user summary\n        all_parts.append(aggregate_chunk(chunk))\n\n        # Fixed-window cohort revenue after first visit\n        chunk_first_date = chunk[\"fullVisitorId\"].map(first_date_map)\n        offset_from_first = chunk[\"date_ordinal\"] - chunk_first_date\n\n        for w in COHORT_WINDOWS:\n            mask = (offset_from_first >= 0) & (offset_from_first < w)\n            if mask.any():\n                s = chunk.loc[mask].groupby(\"fullVisitorId\")[\"revenue\"].sum()\n                cohort_revenue_parts[w].append(s)\n\n        # Rolling feature and target windows\n        for cutoff in cutoff_ords:\n            feature_mask = chunk[\"date_ordinal\"] <= cutoff\n            if feature_mask.any():\n                feature_parts[cutoff].append(aggregate_chunk(chunk.loc[feature_mask]))\n\n            target_mask = (\n                (chunk[\"date_ordinal\"] > cutoff)\n                & (chunk[\"date_ordinal\"] <= cutoff + TARGET_WINDOW_DAYS)\n            )\n            if target_mask.any():\n                s = chunk.loc[target_mask].groupby(\"fullVisitorId\")[\"revenue\"].sum()\n                target_parts[cutoff].append(s)\n\n        elapsed = time.time() - start\n        print(\n            f\"Second pass | {path.name} | chunk {i:03d} | \"\n            f\"rows={len(chunk):,} | elapsed={elapsed/60:.1f} min\"\n        )\n\n        del chunk\n        gc.collect()\n\n    print(\"\\nCombining all-user aggregate...\")\n    all_user = combine_chunk_aggregates(all_parts)\n\n    print(\"Combining rolling feature and target tables...\")\n    features_by_cutoff = {}\n    targets_by_cutoff = {}\n\n    for cutoff in cutoff_ords:\n        features_by_cutoff[cutoff] = combine_chunk_aggregates(feature_parts[cutoff])\n        targets_by_cutoff[cutoff] = combine_series_parts(\n            target_parts[cutoff],\n            name=f\"target_revenue_after_{ordinal_to_date_str(cutoff)}\"\n        )\n\n    print(\"Combining fixed-window cohort revenue...\")\n    cohort_revenue = {}\n    for w in COHORT_WINDOWS:\n        cohort_revenue[w] = combine_series_parts(cohort_revenue_parts[w], name=f\"revenue_{w}d\")\n\n    del all_parts, feature_parts, target_parts, cohort_revenue_parts\n    gc.collect()\n\n    return all_user, features_by_cutoff, targets_by_cutoff, cohort_revenue\n\n\nprint(\"\\n================ Second pass: advanced train tables ================\")\ntrain_user_all, features_by_cutoff, targets_by_cutoff, cohort_revenue = build_advanced_train_tables(\n    train_path,\n    cutoff_ords,\n    first_date_map\n)\n\nprint(\"\\nAll train user table:\", train_user_all.shape)\n\nfor cutoff in cutoff_ords:\n    y_tmp = targets_by_cutoff[cutoff].reindex(features_by_cutoff[cutoff].index).fillna(0.0)\n    print(\n        f\"cutoff={ordinal_to_date_str(cutoff)} | \"\n        f\"features={features_by_cutoff[cutoff].shape} | \"\n        f\"positive_future_revenue_users={int((y_tmp > 0).sum()):,} / {len(y_tmp):,}\"\n    )\n\n\n# ------------------------------------------------------------\n# 5. Test feature table\n# ------------------------------------------------------------\n\ndef build_test_user_table(path):\n    header = pd.read_csv(path, nrows=0).columns.tolist()\n\n    wanted = [\n        \"channelGrouping\",\n        \"date\",\n        \"fullVisitorId\",\n        \"visitNumber\",\n        \"device\",\n        \"geoNetwork\",\n        \"totals\",\n        \"trafficSource\",\n    ]\n\n    usecols = [c for c in wanted if c in header]\n\n    parts = []\n    start = time.time()\n\n    reader = pd.read_csv(\n        path,\n        usecols=usecols,\n        dtype={\"fullVisitorId\": \"str\"},\n        chunksize=CHUNKSIZE,\n        low_memory=False,\n    )\n\n    for i, chunk in enumerate(reader, start=1):\n        chunk = prepare_chunk(chunk, parse_all=True, is_train=False)\n        agg = aggregate_chunk(chunk)\n        parts.append(agg)\n\n        elapsed = time.time() - start\n        print(\n            f\"Test pass | {path.name} | chunk {i:03d} | \"\n            f\"rows={len(chunk):,} | users_in_chunk={len(agg):,} | \"\n            f\"elapsed={elapsed/60:.1f} min\"\n        )\n\n        del chunk, agg\n        gc.collect()\n\n    test_user = combine_chunk_aggregates(parts)\n\n    del parts\n    gc.collect()\n\n    return test_user\n\n\nprint(\"\\n================ Test pass: test feature table ================\")\ntest_user = build_test_user_table(test_path)\nprint(\"test_user:\", test_user.shape)\n\n\n# ------------------------------------------------------------\n# 6. Funnel analysis\n# ------------------------------------------------------------\n\ndef make_funnel(user_df):\n    pageviews_col = \"totals_pageviews_sum\"\n    hits_col = \"totals_hits_sum\"\n    visit_col = \"visitNumber_max\"\n\n    pageviews = user_df[pageviews_col] if pageviews_col in user_df.columns else pd.Series(0, index=user_df.index)\n    hits = user_df[hits_col] if hits_col in user_df.columns else pd.Series(0, index=user_df.index)\n    max_visit = user_df[visit_col] if visit_col in user_df.columns else pd.Series(1, index=user_df.index)\n\n    total = len(user_df)\n    engaged = int(((pageviews >= 2) | (hits >= 2)).sum())\n    returning = int(((user_df[\"session_count\"] >= 2) | (max_visit >= 2)).sum())\n    paying = int((user_df[\"revenue_sum\"] > 0).sum())\n\n    funnel = pd.DataFrame({\n        \"step\": [\n            \"All visitors\",\n            \"Engaged visitors\",\n            \"Returning visitors\",\n            \"Purchasing visitors\",\n        ],\n        \"visitor_count\": [\n            total,\n            engaged,\n            returning,\n            paying,\n        ],\n    })\n\n    funnel[\"rate_vs_all\"] = funnel[\"visitor_count\"] / max(total, 1)\n    funnel[\"dropoff_vs_previous\"] = 1 - funnel[\"visitor_count\"] / funnel[\"visitor_count\"].shift(1)\n    funnel.loc[0, \"dropoff_vs_previous\"] = 0.0\n\n    return funnel\n\nfunnel = make_funnel(train_user_all)\nfunnel.to_csv(OUTPUT_DIR / \"funnel_overall.csv\", index=False)\n\nprint(\"\\n================ Funnel Analysis ================\")\nprint(funnel.to_string(index=False))\n\n\n# ------------------------------------------------------------\n# 7. Channel funnel\n# ------------------------------------------------------------\n\ntrain_user_all[\"is_payer\"] = (train_user_all[\"revenue_sum\"] > 0).astype(int)\n\nchannel_col = \"channelGrouping_first\"\nif channel_col in train_user_all.columns:\n    channel_funnel = (\n        train_user_all\n        .groupby(channel_col, dropna=False)\n        .agg(\n            visitors=(\"is_payer\", \"size\"),\n            paying_visitors=(\"is_payer\", \"sum\"),\n            total_revenue=(\"revenue_sum\", \"sum\"),\n            total_sessions=(\"session_count\", \"sum\"),\n        )\n        .reset_index()\n        .rename(columns={channel_col: \"first_channel\"})\n    )\n\n    channel_funnel[\"conversion_rate\"] = channel_funnel[\"paying_visitors\"] / channel_funnel[\"visitors\"].clip(lower=1)\n    channel_funnel[\"revenue_per_visitor\"] = channel_funnel[\"total_revenue\"] / channel_funnel[\"visitors\"].clip(lower=1)\n    channel_funnel = channel_funnel.sort_values(\"total_revenue\", ascending=False)\n\n    channel_funnel.to_csv(OUTPUT_DIR / \"channel_funnel.csv\", index=False)\n\n    print(\"\\n================ Channel Funnel ================\")\n    print(channel_funnel.head(30).to_string(index=False))\nelse:\n    print(\"\\nNo channelGrouping_first column found. Skipping channel funnel.\")\n\n\n# ------------------------------------------------------------\n# 8. Complete-window cohort analysis\n# ------------------------------------------------------------\n\ncohort_base = pd.DataFrame(index=first_user_table.index)\ncohort_base[\"first_date\"] = first_user_table[\"first_date\"]\ncohort_base[\"first_visit_month\"] = cohort_base[\"first_date\"].apply(ordinal_to_month)\n\nfor w in COHORT_WINDOWS:\n    cohort_base[f\"revenue_{w}d\"] = cohort_revenue[w].reindex(cohort_base.index).fillna(0.0)\n    cohort_base[f\"payer_{w}d\"] = (cohort_base[f\"revenue_{w}d\"] > 0).astype(int)\n\ncohort_rows = []\n\nfor w in COHORT_WINDOWS:\n    complete_mask = cohort_base[\"first_date\"] + (w - 1) <= train_max_ord\n    cohort_w = cohort_base.loc[complete_mask].copy()\n\n    grouped = cohort_w.groupby(\"first_visit_month\", sort=True)\n\n    summary_w = (\n        grouped\n        .agg(\n            visitors=(\"first_date\", \"size\"),\n            paying_visitors=(f\"payer_{w}d\", \"sum\"),\n            total_revenue=(f\"revenue_{w}d\", \"sum\"),\n        )\n        .reset_index()\n    )\n\n    summary_w[\"window_days\"] = w\n    summary_w[\"conversion_rate\"] = summary_w[\"paying_visitors\"] / summary_w[\"visitors\"].clip(lower=1)\n    summary_w[\"revenue_per_visitor\"] = summary_w[\"total_revenue\"] / summary_w[\"visitors\"].clip(lower=1)\n\n    cohort_rows.append(summary_w)\n\ncohort_summary = pd.concat(cohort_rows, axis=0, ignore_index=True)\ncohort_summary = cohort_summary[\n    [\n        \"window_days\",\n        \"first_visit_month\",\n        \"visitors\",\n        \"paying_visitors\",\n        \"total_revenue\",\n        \"conversion_rate\",\n        \"revenue_per_visitor\",\n    ]\n].sort_values([\"window_days\", \"first_visit_month\"])\n\ncohort_summary.to_csv(OUTPUT_DIR / \"fixed_window_cohort_summary.csv\", index=False)\n\nprint(\"\\n================ Complete-window Cohort Summary ================\")\nprint(cohort_summary.tail(40).to_string(index=False))\n\n\n# ------------------------------------------------------------\n# 9. Modeling sample construction\n# ------------------------------------------------------------\n\ndef add_relative_time_features(feature_df, cutoff):\n    df = feature_df.copy()\n\n    df[\"recency_days\"] = cutoff - df[\"last_date\"]\n    df[\"user_age_days\"] = cutoff - df[\"first_date\"] + 1\n\n    cutoff_date = dt.date.fromordinal(int(cutoff))\n    df[\"cutoff_month\"] = cutoff_date.month\n    df[\"cutoff_dayofweek\"] = cutoff_date.weekday()\n\n    return df\n\n\ndef make_sample(feature_df, target_revenue, cutoff):\n    df = add_relative_time_features(feature_df, cutoff)\n\n    target_revenue = target_revenue.reindex(df.index).fillna(0.0)\n    df[\"target\"] = np.log1p(target_revenue)\n    df[\"target_cls\"] = (df[\"target\"] > 0).astype(int)\n    df[\"_cutoff_ord\"] = cutoff\n    df[\"_cutoff_date\"] = ordinal_to_date_str(cutoff)\n\n    return df\n\n\ndef select_classifier_rows(sample_df, neg_pos_ratio=30):\n    pos_idx = np.where(sample_df[\"target_cls\"].values == 1)[0]\n    neg_idx = np.where(sample_df[\"target_cls\"].values == 0)[0]\n\n    if len(pos_idx) == 0:\n        return np.array([], dtype=int)\n\n    n_neg = min(len(neg_idx), len(pos_idx) * neg_pos_ratio)\n    sampled_neg_idx = rng.choice(neg_idx, size=n_neg, replace=False)\n\n    idx = np.concatenate([pos_idx, sampled_neg_idx])\n    rng.shuffle(idx)\n\n    return idx\n\n\nvalid_cutoff = cutoff_ords[-1]\ntrain_cutoffs = cutoff_ords[:-1]\n\nprint(\"\\n================ Building modeling samples ================\")\nprint(\"Validation cutoff:\", ordinal_to_date_str(valid_cutoff))\n\ntrain_clf_parts = []\nfinal_clf_parts = []\n\ntrain_total_rows = 0\ntrain_total_pos = 0\nfinal_total_rows = 0\nfinal_total_pos = 0\n\nfor cutoff in cutoff_ords:\n    sample = make_sample(\n        features_by_cutoff[cutoff],\n        targets_by_cutoff[cutoff],\n        cutoff\n    )\n\n    n_rows = len(sample)\n    n_pos = int(sample[\"target_cls\"].sum())\n\n    final_total_rows += n_rows\n    final_total_pos += n_pos\n\n    idx_final = select_classifier_rows(sample, NEG_POS_RATIO)\n    if len(idx_final) > 0:\n        final_clf_parts.append(sample.iloc[idx_final].copy())\n\n    if cutoff in train_cutoffs:\n        train_total_rows += n_rows\n        train_total_pos += n_pos\n\n        idx_train = select_classifier_rows(sample, NEG_POS_RATIO)\n        if len(idx_train) > 0:\n            train_clf_parts.append(sample.iloc[idx_train].copy())\n\n    print(\n        f\"cutoff={ordinal_to_date_str(cutoff)} | \"\n        f\"rows={n_rows:,} | positives={n_pos:,} | \"\n        f\"sampled_for_clf={len(idx_final):,}\"\n    )\n\n    if cutoff == valid_cutoff:\n        valid_df = sample.copy()\n\n    del sample\n    gc.collect()\n\ntrain_clf_df = pd.concat(train_clf_parts, axis=0)\nfinal_clf_df = pd.concat(final_clf_parts, axis=0)\n\ndel train_clf_parts, final_clf_parts\ngc.collect()\n\nprint(\"\\nTraining classifier sample:\", train_clf_df.shape)\nprint(\"Final classifier sample   :\", final_clf_df.shape)\nprint(\"Validation full sample    :\", valid_df.shape)\n\nprint(\"\\nTrue prior rates:\")\nprint(f\"train prior positive rate: {train_total_pos / max(train_total_rows, 1):.8f}\")\nprint(f\"final prior positive rate: {final_total_pos / max(final_total_rows, 1):.8f}\")\n\n\n# ------------------------------------------------------------\n# 10. Test features and feature selection\n# ------------------------------------------------------------\n\nsample_submission = pd.read_csv(sample_path, dtype={\"fullVisitorId\": \"str\"})\nsample_ids = sample_submission[\"fullVisitorId\"].astype(str)\n\ntest_cutoff = int(test_user[\"last_date\"].max())\ntest_features = add_relative_time_features(test_user, test_cutoff)\ntest_features = test_features.reindex(sample_ids)\n\nprint(\"\\nTest feature cutoff:\", ordinal_to_date_str(test_cutoff))\n\nDROP_COLS = {\n    \"target\",\n    \"target_cls\",\n    \"_cutoff_ord\",\n    \"_cutoff_date\",\n    \"revenue_sum\",\n    \"is_payer\",\n    \"first_visit_month\",\n    \"first_date\",\n    \"last_date\",\n}\n\ncandidate_cols = [\n    c for c in train_clf_df.columns\n    if c in valid_df.columns\n    and c in final_clf_df.columns\n    and c in test_features.columns\n    and c not in DROP_COLS\n    and c not in DROP_ABSOLUTE_TIME_FEATURES\n]\n\nprint(\"\\nNumber of candidate features:\", len(candidate_cols))\n\n\n# ------------------------------------------------------------\n# 11. Encoding and matrix preparation\n# ------------------------------------------------------------\n\ndef encode_feature_frames(frames, feature_cols):\n    raw_frames = [df[feature_cols].copy() for df in frames]\n    lengths = [len(x) for x in raw_frames]\n\n    combined = pd.concat(raw_frames, axis=0, ignore_index=True)\n\n    cat_cols = combined.select_dtypes(include=[\"object\", \"string\", \"category\", \"bool\"]).columns.tolist()\n\n    for c in cat_cols:\n        combined[c] = combined[c].astype(\"string\").fillna(\"__MISSING__\")\n        codes, uniques = pd.factorize(combined[c], sort=True)\n        combined[c] = codes.astype(\"int32\")\n\n    for c in combined.columns:\n        combined[c] = pd.to_numeric(combined[c], errors=\"coerce\")\n        med = combined[c].median()\n\n        if pd.isna(med):\n            med = 0.0\n\n        combined[c] = combined[c].replace([np.inf, -np.inf], np.nan).fillna(med)\n\n    # Drop constant columns after encoding\n    keep_cols = combined.columns[combined.nunique(dropna=False) > 1].tolist()\n    combined = combined[keep_cols]\n\n    # Safe feature names\n    safe_cols = []\n    seen = {}\n\n    for col in combined.columns:\n        base = re.sub(r\"[^A-Za-z0-9_]+\", \"_\", str(col)).strip(\"_\")\n\n        if base == \"\":\n            base = \"feature\"\n\n        if base in seen:\n            seen[base] += 1\n            base = f\"{base}_{seen[base]}\"\n        else:\n            seen[base] = 0\n\n        safe_cols.append(base)\n\n    original_cols = list(combined.columns)\n    combined.columns = safe_cols\n    combined = combined.astype(\"float32\")\n\n    out = []\n    start = 0\n\n    for n in lengths:\n        out.append(combined.iloc[start:start+n].copy())\n        start += n\n\n    return out, safe_cols, original_cols\n\n\nframes = [\n    train_clf_df,\n    valid_df,\n    final_clf_df,\n    test_features,\n]\n\n(encoded_frames, safe_cols, original_cols) = encode_feature_frames(frames, candidate_cols)\n\nX_train_clf = encoded_frames[0]\nX_valid = encoded_frames[1]\nX_final_clf = encoded_frames[2]\nX_test = encoded_frames[3]\n\ny_train_log = train_clf_df[\"target\"].astype(float).reset_index(drop=True)\ny_train_cls = train_clf_df[\"target_cls\"].astype(int).reset_index(drop=True)\n\ny_valid_log = valid_df[\"target\"].astype(float).reset_index(drop=True)\ny_valid_cls = valid_df[\"target_cls\"].astype(int).reset_index(drop=True)\n\ny_final_log = final_clf_df[\"target\"].astype(float).reset_index(drop=True)\ny_final_cls = final_clf_df[\"target_cls\"].astype(int).reset_index(drop=True)\n\nprint(\"\\nClean matrix shapes:\")\nprint(\"X_train_clf:\", X_train_clf.shape)\nprint(\"X_valid    :\", X_valid.shape)\nprint(\"X_final_clf:\", X_final_clf.shape)\nprint(\"X_test     :\", X_test.shape)\n\nprint(\"\\nPositive counts:\")\nprint(\"train selected positives:\", int(y_train_cls.sum()), \"/\", len(y_train_cls))\nprint(\"valid full positives    :\", int(y_valid_cls.sum()), \"/\", len(y_valid_cls))\nprint(\"final selected positives:\", int(y_final_cls.sum()), \"/\", len(y_final_cls))\n\n\n# ------------------------------------------------------------\n# 12. Metrics and calibration helpers\n# ------------------------------------------------------------\n\nfrom sklearn.metrics import (\n    mean_squared_error,\n    roc_auc_score,\n    average_precision_score,\n)\n\ndef rmse(y_true, y_pred):\n    return float(np.sqrt(mean_squared_error(y_true, y_pred)))\n\n\ndef prior_correct_proba(p_sample, pi_sample, pi_true, eps=1e-15):\n    p_sample = np.clip(np.asarray(p_sample, dtype=float), eps, 1 - eps)\n\n    pi_sample = float(np.clip(pi_sample, eps, 1 - eps))\n    pi_true = float(np.clip(pi_true, eps, 1 - eps))\n\n    odds_sample = p_sample / (1.0 - p_sample)\n    correction = (pi_true / (1.0 - pi_true)) / (pi_sample / (1.0 - pi_sample))\n\n    odds_true = odds_sample * correction\n    p_true = odds_true / (1.0 + odds_true)\n\n    return np.clip(p_true, 0.0, 1.0)\n\n\ndef precision_recall_at_k(y_true, scores, k):\n    y_true = np.asarray(y_true).astype(int)\n    scores = np.asarray(scores)\n\n    k = min(k, len(scores))\n    if k <= 0:\n        return np.nan, np.nan\n\n    order = np.argsort(scores)[::-1][:k]\n    hits = y_true[order].sum()\n\n    precision = hits / k\n    recall = hits / max(y_true.sum(), 1)\n\n    return float(precision), float(recall)\n\n\ndef tune_alpha_threshold(y_true, p_cal, cond_logrev, quantiles):\n    y_true = np.asarray(y_true, dtype=float)\n    p_cal = np.asarray(p_cal, dtype=float)\n    cond_logrev = np.asarray(cond_logrev, dtype=float)\n\n    base_raw = p_cal * cond_logrev\n\n    threshold_candidates = []\n    for q in quantiles:\n        try:\n            threshold_candidates.append(float(np.quantile(p_cal, q)))\n        except Exception:\n            pass\n\n    threshold_candidates = sorted(set(threshold_candidates))\n\n    best = {\n        \"rmse\": float(\"inf\"),\n        \"alpha\": 0.0,\n        \"threshold\": 0.0,\n        \"prediction\": np.zeros_like(y_true),\n        \"nonzero_count\": 0,\n    }\n\n    for thr in threshold_candidates:\n        raw = base_raw.copy()\n        raw[p_cal < thr] = 0.0\n\n        denom = float(np.sum(raw ** 2))\n        if denom <= 1e-30:\n            alpha = 0.0\n        else:\n            alpha = float(np.sum(y_true * raw) / denom)\n            alpha = max(0.0, alpha)\n\n        pred = alpha * raw\n        score = rmse(y_true, pred)\n\n        if score < best[\"rmse\"]:\n            best = {\n                \"rmse\": score,\n                \"alpha\": alpha,\n                \"threshold\": thr,\n                \"prediction\": pred,\n                \"nonzero_count\": int((pred > 0).sum()),\n            }\n\n    return best\n\n\n# ------------------------------------------------------------\n# 13. Train validation two-stage model\n# ------------------------------------------------------------\n\nimport lightgbm as lgb\n\nclf_params = dict(\n    objective=\"binary\",\n    learning_rate=0.03,\n    n_estimators=2000,\n    num_leaves=64,\n    max_depth=-1,\n    min_child_samples=80,\n    subsample=0.85,\n    colsample_bytree=0.85,\n    reg_alpha=0.10,\n    reg_lambda=0.20,\n    random_state=RANDOM_STATE,\n    n_jobs=-1,\n    verbosity=-1,\n)\n\nreg_params = dict(\n    objective=\"regression\",\n    learning_rate=0.03,\n    n_estimators=800,\n    num_leaves=8,\n    max_depth=-1,\n    min_child_samples=20,\n    subsample=0.80,\n    colsample_bytree=0.80,\n    reg_alpha=1.0,\n    reg_lambda=5.0,\n    random_state=RANDOM_STATE,\n    n_jobs=-1,\n    verbosity=-1,\n)\n\nprint(\"\\n================ Training Stage 1 classifier ================\")\nclf_valid_model = lgb.LGBMClassifier(**clf_params)\n\nclf_valid_model.fit(\n    X_train_clf,\n    y_train_cls,\n    eval_set=[(X_valid, y_valid_cls)],\n    eval_metric=\"auc\",\n    callbacks=[\n        lgb.early_stopping(stopping_rounds=100),\n        lgb.log_evaluation(period=100),\n    ],\n)\n\np_sample_valid = clf_valid_model.predict_proba(\n    X_valid,\n    num_iteration=clf_valid_model.best_iteration_\n)[:, 1]\n\npi_sample_train = float(y_train_cls.mean())\npi_true_train = float(train_total_pos / max(train_total_rows, 1))\n\np_cal_valid = prior_correct_proba(\n    p_sample_valid,\n    pi_sample=pi_sample_train,\n    pi_true=pi_true_train,\n)\n\nvalid_auc = roc_auc_score(y_valid_cls, p_sample_valid) if y_valid_cls.nunique() == 2 else np.nan\nvalid_ap = average_precision_score(y_valid_cls, p_sample_valid) if y_valid_cls.nunique() == 2 else np.nan\n\nprint(f\"\\nValidation purchase AUC        : {valid_auc:.6f}\")\nprint(f\"Validation purchase PR-AUC     : {valid_ap:.6f}\")\nprint(f\"Sample prior                   : {pi_sample_train:.8f}\")\nprint(f\"True prior                     : {pi_true_train:.8f}\")\nprint(f\"Mean raw classifier proba      : {p_sample_valid.mean():.8f}\")\nprint(f\"Mean prior-corrected proba     : {p_cal_valid.mean():.8f}\")\n\nfor k in [1000, 5000, 10000]:\n    precision_k, recall_k = precision_recall_at_k(y_valid_cls, p_sample_valid, k)\n    print(f\"Precision@{k:<5}: {precision_k:.6f} | Recall@{k:<5}: {recall_k:.6f}\")\n\nprint(\"\\n================ Training Stage 2 conditional regressor ================\")\n\npos_train_mask = y_train_log > 0\nX_train_reg = X_train_clf.loc[pos_train_mask.values]\ny_train_reg = y_train_log.loc[pos_train_mask.values]\n\npos_valid_mask = y_valid_log > 0\nX_valid_reg = X_valid.loc[pos_valid_mask.values]\ny_valid_reg = y_valid_log.loc[pos_valid_mask.values]\n\nprint(\"Positive train rows:\", len(X_train_reg))\nprint(\"Positive valid rows:\", len(X_valid_reg))\n\nreg_valid_model = lgb.LGBMRegressor(**reg_params)\n\nif len(X_valid_reg) >= 30:\n    reg_valid_model.fit(\n        X_train_reg,\n        y_train_reg,\n        eval_set=[(X_valid_reg, y_valid_reg)],\n        eval_metric=\"rmse\",\n        callbacks=[\n            lgb.early_stopping(stopping_rounds=80),\n            lgb.log_evaluation(period=100),\n        ],\n    )\n    reg_best_iter = reg_valid_model.best_iteration_\nelse:\n    reg_valid_model.fit(X_train_reg, y_train_reg)\n    reg_best_iter = None\n\nvalid_reg_pred_base = reg_valid_model.predict(\n    X_valid,\n    num_iteration=reg_best_iter\n)\nvalid_reg_pred_base = np.clip(valid_reg_pred_base, 0, None)\n\nglobal_positive_mean_train = float(y_train_reg.mean())\n\nzero_pred_valid = np.zeros_like(y_valid_log.values, dtype=float)\nzero_rmse = rmse(y_valid_log, zero_pred_valid)\n\nuncalibrated_raw = p_sample_valid * valid_reg_pred_base\nuncalibrated_rmse = rmse(y_valid_log, uncalibrated_raw)\n\nprior_raw = p_cal_valid * valid_reg_pred_base\nprior_rmse = rmse(y_valid_log, prior_raw)\n\nbest_cal = None\n\nfor shrink in SHRINKAGE_CANDIDATES:\n    cond = shrink * valid_reg_pred_base + (1.0 - shrink) * global_positive_mean_train\n    cond = np.clip(cond, 0, None)\n\n    tuned = tune_alpha_threshold(\n        y_true=y_valid_log.values,\n        p_cal=p_cal_valid,\n        cond_logrev=cond,\n        quantiles=THRESHOLD_QUANTILES,\n    )\n\n    tuned[\"shrinkage\"] = shrink\n\n    if best_cal is None or tuned[\"rmse\"] < best_cal[\"rmse\"]:\n        best_cal = tuned\n\nvalid_pred_calibrated = best_cal[\"prediction\"]\n\npositive_conditional_rmse = (\n    rmse(y_valid_reg, np.clip(reg_valid_model.predict(X_valid_reg), 0, None))\n    if len(X_valid_reg) > 0\n    else np.nan\n)\n\nprint(\"\\n================ Validation Metrics ================\")\nprint(f\"Zero baseline RMSE                    : {zero_rmse:.6f}\")\nprint(f\"Uncalibrated two-stage RMSE           : {uncalibrated_rmse:.6f}\")\nprint(f\"Prior-corrected two-stage RMSE        : {prior_rmse:.6f}\")\nprint(f\"Calibrated two-stage RMSE             : {best_cal['rmse']:.6f}\")\nprint(f\"Positive-only conditional RMSE        : {positive_conditional_rmse:.6f}\")\nprint(f\"Best shrinkage                        : {best_cal['shrinkage']}\")\nprint(f\"Best alpha                            : {best_cal['alpha']:.8f}\")\nprint(f\"Best probability threshold            : {best_cal['threshold']:.12f}\")\nprint(f\"Nonzero calibrated valid predictions  : {best_cal['nonzero_count']:,} / {len(y_valid_log):,}\")\n\nmain_uses_zero = False\n\nif USE_ZERO_FALLBACK_IF_WORSE and best_cal[\"rmse\"] >= zero_rmse:\n    main_uses_zero = True\n    print(\"\\nValidation selected zero fallback for main submission.\")\nelse:\n    print(\"\\nValidation selected calibrated two-stage model for main submission.\")\n\n\n# ------------------------------------------------------------\n# 14. Final model training\n# ------------------------------------------------------------\n\nprint(\"\\n================ Training final two-stage models ================\")\n\nbest_clf_iter = clf_valid_model.best_iteration_\nif best_clf_iter is None or best_clf_iter <= 0:\n    best_clf_iter = 800\n\nfinal_clf_params = clf_params.copy()\nfinal_clf_params[\"n_estimators\"] = int(best_clf_iter)\n\nprint(f\"Training final classifier with n_estimators={best_clf_iter}...\")\nfinal_clf = lgb.LGBMClassifier(**final_clf_params)\nfinal_clf.fit(X_final_clf, y_final_cls)\n\npos_final_mask = y_final_log > 0\nX_final_reg = X_final_clf.loc[pos_final_mask.values]\ny_final_reg = y_final_log.loc[pos_final_mask.values]\n\nbest_reg_iter = reg_valid_model.best_iteration_\nif best_reg_iter is None or best_reg_iter <= 0:\n    best_reg_iter = 100\n\nfinal_reg_params = reg_params.copy()\nfinal_reg_params[\"n_estimators\"] = int(best_reg_iter)\n\nprint(f\"Training final regressor with n_estimators={best_reg_iter}...\")\nfinal_reg = lgb.LGBMRegressor(**final_reg_params)\nfinal_reg.fit(X_final_reg, y_final_reg)\n\np_sample_test = final_clf.predict_proba(X_test)[:, 1]\n\npi_sample_final = float(y_final_cls.mean())\npi_true_final = float(final_total_pos / max(final_total_rows, 1))\n\np_cal_test = prior_correct_proba(\n    p_sample_test,\n    pi_sample=pi_sample_final,\n    pi_true=pi_true_final,\n)\n\ntest_reg_pred_base = final_reg.predict(X_test)\ntest_reg_pred_base = np.clip(test_reg_pred_base, 0, None)\n\nglobal_positive_mean_final = float(y_final_reg.mean())\n\ntest_cond = (\n    best_cal[\"shrinkage\"] * test_reg_pred_base\n    + (1.0 - best_cal[\"shrinkage\"]) * global_positive_mean_final\n)\ntest_cond = np.clip(test_cond, 0, None)\n\ntest_raw = p_cal_test * test_cond\ntest_raw[p_cal_test < best_cal[\"threshold\"]] = 0.0\n\ntest_pred_calibrated = best_cal[\"alpha\"] * test_raw\ntest_pred_calibrated = np.clip(test_pred_calibrated, 0, None)\n\n# Conservative cap\ncap_value = float(y_final_log[y_final_log > 0].quantile(0.999))\nif np.isfinite(cap_value) and cap_value > 0:\n    test_pred_calibrated = np.clip(test_pred_calibrated, 0, cap_value)\n\ntest_pred_zero = np.zeros(len(sample_submission), dtype=float)\n\nif main_uses_zero:\n    test_pred_main = test_pred_zero\nelse:\n    test_pred_main = test_pred_calibrated\n\n\n# ------------------------------------------------------------\n# 15. Feature importance\n# ------------------------------------------------------------\n\nclassifier_importance = pd.DataFrame({\n    \"feature\": safe_cols,\n    \"original_feature\": original_cols,\n    \"importance\": final_clf.feature_importances_,\n}).sort_values(\"importance\", ascending=False)\n\nregressor_importance = pd.DataFrame({\n    \"feature\": safe_cols,\n    \"original_feature\": original_cols,\n    \"importance\": final_reg.feature_importances_,\n}).sort_values(\"importance\", ascending=False)\n\nclassifier_importance.to_csv(OUTPUT_DIR / \"classifier_feature_importance.csv\", index=False)\nregressor_importance.to_csv(OUTPUT_DIR / \"regressor_feature_importance.csv\", index=False)\n\nprint(\"\\n================ Top Classifier Features ================\")\nprint(classifier_importance.head(30).to_string(index=False))\n\nprint(\"\\n================ Top Regressor Features ================\")\nprint(regressor_importance.head(30).to_string(index=False))\n\n\n# ------------------------------------------------------------\n# 16. Submission files and reports\n# ------------------------------------------------------------\n\ndef make_submission(pred, path):\n    sub = sample_submission[[\"fullVisitorId\"]].copy()\n    sub[\"PredictedLogRevenue\"] = pred\n    sub[\"PredictedLogRevenue\"] = (\n        sub[\"PredictedLogRevenue\"]\n        .replace([np.inf, -np.inf], np.nan)\n        .fillna(0.0)\n        .clip(lower=0.0)\n    )\n    sub.to_csv(path, index=False)\n    return sub\n\nsubmission_main = make_submission(test_pred_main, OUTPUT_DIR / \"submission.csv\")\nsubmission_cal = make_submission(test_pred_calibrated, OUTPUT_DIR / \"submission_calibrated_twostage.csv\")\nsubmission_zero = make_submission(test_pred_zero, OUTPUT_DIR / \"submission_zero.csv\")\n\nvalidation_report = pd.DataFrame({\n    \"target_window_days\": [TARGET_WINDOW_DAYS],\n    \"n_cutoffs\": [len(cutoff_ords)],\n    \"train_cutoffs\": [\", \".join(ordinal_to_date_str(c) for c in train_cutoffs)],\n    \"validation_cutoff\": [ordinal_to_date_str(valid_cutoff)],\n    \"validation_target_start\": [ordinal_to_date_str(valid_cutoff + 1)],\n    \"validation_target_end\": [ordinal_to_date_str(valid_cutoff + TARGET_WINDOW_DAYS)],\n    \"n_train_true_rows\": [train_total_rows],\n    \"n_train_true_positive_users\": [train_total_pos],\n    \"n_valid_rows\": [len(valid_df)],\n    \"n_valid_positive_users\": [int(y_valid_cls.sum())],\n    \"n_features\": [len(safe_cols)],\n    \"sample_prior_train\": [pi_sample_train],\n    \"true_prior_train\": [pi_true_train],\n    \"zero_baseline_rmse\": [zero_rmse],\n    \"uncalibrated_two_stage_rmse\": [uncalibrated_rmse],\n    \"prior_corrected_two_stage_rmse\": [prior_rmse],\n    \"calibrated_two_stage_rmse\": [best_cal[\"rmse\"]],\n    \"purchase_auc\": [valid_auc],\n    \"purchase_pr_auc\": [valid_ap],\n    \"positive_conditional_rmse\": [positive_conditional_rmse],\n    \"best_shrinkage\": [best_cal[\"shrinkage\"]],\n    \"best_alpha\": [best_cal[\"alpha\"]],\n    \"best_probability_threshold\": [best_cal[\"threshold\"]],\n    \"main_submission_uses_zero_fallback\": [main_uses_zero],\n})\n\nvalidation_report.to_csv(OUTPUT_DIR / \"validation_report.csv\", index=False)\n\nprediction_report = pd.DataFrame({\n    \"submission\": [\n        \"main\",\n        \"calibrated_twostage\",\n        \"zero\",\n    ],\n    \"mean\": [\n        submission_main[\"PredictedLogRevenue\"].mean(),\n        submission_cal[\"PredictedLogRevenue\"].mean(),\n        submission_zero[\"PredictedLogRevenue\"].mean(),\n    ],\n    \"std\": [\n        submission_main[\"PredictedLogRevenue\"].std(),\n        submission_cal[\"PredictedLogRevenue\"].std(),\n        submission_zero[\"PredictedLogRevenue\"].std(),\n    ],\n    \"min\": [\n        submission_main[\"PredictedLogRevenue\"].min(),\n        submission_cal[\"PredictedLogRevenue\"].min(),\n        submission_zero[\"PredictedLogRevenue\"].min(),\n    ],\n    \"median\": [\n        submission_main[\"PredictedLogRevenue\"].median(),\n        submission_cal[\"PredictedLogRevenue\"].median(),\n        submission_zero[\"PredictedLogRevenue\"].median(),\n    ],\n    \"q75\": [\n        submission_main[\"PredictedLogRevenue\"].quantile(0.75),\n        submission_cal[\"PredictedLogRevenue\"].quantile(0.75),\n        submission_zero[\"PredictedLogRevenue\"].quantile(0.75),\n    ],\n    \"max\": [\n        submission_main[\"PredictedLogRevenue\"].max(),\n        submission_cal[\"PredictedLogRevenue\"].max(),\n        submission_zero[\"PredictedLogRevenue\"].max(),\n    ],\n    \"nonzero_count\": [\n        int((submission_main[\"PredictedLogRevenue\"] > 0).sum()),\n        int((submission_cal[\"PredictedLogRevenue\"] > 0).sum()),\n        int((submission_zero[\"PredictedLogRevenue\"] > 0).sum()),\n    ],\n})\n\nprediction_report.to_csv(OUTPUT_DIR / \"prediction_report.csv\", index=False)\n\nprint(\"\\n================ Validation Report ================\")\nprint(validation_report.to_string(index=False))\n\nprint(\"\\n================ Prediction Report ================\")\nprint(prediction_report.to_string(index=False))\n\nprint(\"\\n================ Submission Created ================\")\nprint(\"Main submission               :\", OUTPUT_DIR / \"submission.csv\")\nprint(\"Calibrated two-stage submission:\", OUTPUT_DIR / \"submission_calibrated_twostage.csv\")\nprint(\"Zero baseline submission       :\", OUTPUT_DIR / \"submission_zero.csv\")\n\nprint(\"\\nMain submission preview:\")\nprint(submission_main.head().to_string(index=False))\n\nprint(\"\\nSaved CSV files:\")\nfor p in sorted(OUTPUT_DIR.glob(\"*.csv\")):\n    print(p)\n\ngc.collect()\n\nprint(\"\\nDone. Submit /kaggle/working/submission.csv first.\")\nprint(\"If you want to compare, also try /kaggle/working/submission_calibrated_twostage.csv and /kaggle/working/submission_zero.csv as late submissions.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-20T08:13:54.854430Z","iopub.execute_input":"2026-06-20T08:13:54.858362Z","iopub.status.idle":"2026-06-20T08:30:10.579841Z","shell.execute_reply.started":"2026-06-20T08:13:54.858250Z","shell.execute_reply":"2026-06-20T08:30:10.578635Z"}},"outputs":[],"execution_count":null}]}