{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":114135,"databundleVersionId":13695648,"sourceType":"competition"}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-10-10T03:55:35.439566Z","iopub.execute_input":"2025-10-10T03:55:35.440040Z","iopub.status.idle":"2025-10-10T03:55:35.745782Z","shell.execute_reply.started":"2025-10-10T03:55:35.440014Z","shell.execute_reply":"2025-10-10T03:55:35.745109Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# Reversal Points Detection - Leakage-Free, Time-Aware, 2-Stage\n# ============================================================\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport pandas as pd\nimport numpy as np\nfrom pathlib import Path\nfrom collections import defaultdict\n\nfrom sklearn.feature_selection import VarianceThreshold, SelectKBest, mutual_info_classif\nfrom sklearn.metrics import f1_score, classification_report\nfrom sklearn.model_selection import StratifiedKFold\nfrom lightgbm import LGBMClassifier\n\nimport gc\n\n# -------------------\n# CONFIG\n# -------------------\nDATA_DIR = \"/kaggle/input/detecting-reversal-points-in-us-equities/competition_data\"\nTRAIN_PATH = f\"{DATA_DIR}/train.csv\"\nTEST_PATH  = f\"{DATA_DIR}/test.csv\"\nOUTPUT_PATH = \"/kaggle/working/submission.csv\"\n\nSEED = 42\nN_SPLITS = 5\nEMBARGO_STEPS = 8             # exclude this many steps around fold boundaries\nTOP_K_MI_BOOL = 200           # per fold: # boolean features to engineer rolling counts for\nK_SELECT = 1500               # per fold: SelectKBest(MI) keep top-K overall\nnp.random.seed(SEED)\n\nprint(\"=\"*70)\nprint(\"REVERSAL DETECTION - Leakage-Free, Time-Aware, 2-Stage Pipeline\")\nprint(\"=\"*70)\n\n# -------------------\n# LOAD\n# -------------------\nprint(\"\\n[1/8] Loading data...\")\ntrain = pd.read_csv(TRAIN_PATH)\ntest  = pd.read_csv(TEST_PATH)\nprint(f\"✓ Train: {train.shape} | Test: {test.shape}\")\nprint(f\"✓ Train dtypes: {train.dtypes.value_counts().to_dict()}\")\n\n# -------------------\n# TARGET MAPPING\n# -------------------\nprint(\"\\n[2/8] Preparing target...\")\nmapping = {\"HH\": \"H\", \"LH\": \"H\", \"HL\": \"L\", \"LL\": \"L\"}\ntrain[\"class_label\"] = train[\"class_label\"].map(mapping).fillna(\"None\")\n\nclasses = [\"H\",\"L\",\"None\"]\nlabel_map = {c:i for i,c in enumerate(classes)}\ny = train[\"class_label\"].map(label_map).values\n\nfor c in classes:\n    cnt = (train[\"class_label\"]==c).sum()\n    print(f\"  {c}: {cnt} ({100.0*cnt/len(train):.1f}%)\")\n\n# -------------------\n# BASIC PREP\n# -------------------\nprint(\"\\n[3/8] Feature identification & type fixes...\")\n\nmeta_cols = [\"train_id\",\"id\",\"ticker_id\",\"t\",\"class_label\"]\nfeature_cols = [c for c in train.columns if c not in meta_cols and c in test.columns]\n\n# ensure datetime & sorting\ntrain[\"t\"] = pd.to_datetime(train[\"t\"])\ntest[\"t\"]  = pd.to_datetime(test[\"t\"])\ntrain = train.sort_values([\"ticker_id\",\"t\"]).reset_index(drop=True)\ntest  = test.sort_values([\"ticker_id\",\"t\"]).reset_index(drop=True)\n\n# unify dtypes (booleans -> int8; numerics stay numeric)\nbool_cols = [c for c in feature_cols if train[c].dtype==bool]\nnum_cols  = [c for c in feature_cols if c not in bool_cols]\n\nfor df in (train, test):\n    if bool_cols:\n        df[bool_cols] = df[bool_cols].astype(np.int8)\n    # coerce numerics safely\n    for col in num_cols:\n        df[col] = pd.to_numeric(df[col], errors=\"coerce\")\n\n# fill/clean\nfor df in (train, test):\n    df[feature_cols] = df[feature_cols].fillna(0).replace([np.inf,-np.inf], 0)\n\nprint(f\"✓ Features total: {len(feature_cols)} | bool: {len(bool_cols)} | numeric: {len(num_cols)}\")\n\n# -------------------\n# TIME-AWARE CV SPLITS (per ticker, chronological, with embargo)\n# -------------------\nprint(\"\\n[4/8] Building time-aware folds with embargo...\")\n\ndef time_folds_with_embargo(df, n_splits=5, embargo=8):\n    \"\"\"\n    Returns: list of (train_idx, val_idx) for global dataframe df (train only).\n    Splits are made per ticker chronologically then concatenated.\n    \"\"\"\n    idx_by_fold_tr = [list() for _ in range(n_splits)]\n    idx_by_fold_val = [list() for _ in range(n_splits)]\n\n    # process per ticker\n    for tid, g in df.groupby(\"ticker_id\", sort=False):\n        g = g.sort_values(\"t\")\n        n = len(g)\n        # boundaries for contiguous folds by time\n        fold_sizes = [n // n_splits + (1 if x < n % n_splits else 0) for x in range(n_splits)]\n        starts, ends = [], []\n        start = 0\n        for fs in fold_sizes:\n            end = start + fs\n            starts.append(start)\n            ends.append(end)\n            start = end\n\n        # produce masks per fold with embargo\n        for k in range(n_splits):\n            val_start, val_end = starts[k], ends[k]\n            val_idx_local = np.arange(val_start, val_end)\n\n            # embargoed train: exclude [val_start-embargo, val_end+embargo)\n            left_cut  = max(0, val_start - embargo)\n            right_cut = min(n, val_end + embargo)\n            train_left  = np.arange(0, left_cut)\n            train_right = np.arange(right_cut, n)\n            tr_idx_local = np.concatenate([train_left, train_right]) if len(train_left)+len(train_right) > 0 else np.array([], dtype=int)\n\n            # map local to global\n            global_idx = g.index.to_numpy()\n            idx_by_fold_tr[k].extend(global_idx[tr_idx_local].tolist())\n            idx_by_fold_val[k].extend(global_idx[val_idx_local].tolist())\n\n    # convert to arrays\n    folds = []\n    for k in range(n_splits):\n        tr = np.array(sorted(idx_by_fold_tr[k]), dtype=int)\n        vl = np.array(sorted(idx_by_fold_val[k]), dtype=int)\n        folds.append((tr, vl))\n        print(f\"  Fold {k+1}: train={len(tr)} val={len(vl)}\")\n    return folds\n\nfolds = time_folds_with_embargo(train, n_splits=N_SPLITS, embargo=EMBARGO_STEPS)\n\n# -------------------\n# Utility: rolling boolean counts (past-only)\n# -------------------\ndef add_bool_rolling_counts(df, bool_list, windows=(3,5,10)):\n    \"\"\"\n    For each bool col in bool_list, create past-only rolling sums:\n      col_sum_w = sum over last w steps (excluding current row).\n    \"\"\"\n    if not bool_list:\n        return pd.DataFrame(index=df.index)\n    out = pd.DataFrame(index=df.index)\n    # operate per ticker to respect chronology\n    for tid, g in df.groupby(\"ticker_id\", sort=False):\n        order = g.sort_values(\"t\")\n        for col in bool_list:\n            s = order[col].astype(np.int16)\n            # exclude current -> shift(1)\n            s_shift = s.shift(1)\n            for w in windows:\n                out.loc[order.index, f\"{col}_sum{w}\"] = (\n                    s_shift.rolling(window=w, min_periods=1).sum().astype(np.float32)\n                )\n    # fillna 0 for early periods\n    return out.fillna(0.0)\n\n# -------------------\n# Two-stage training across folds\n# -------------------\nprint(\"\\n[5/8] Training (two-stage) with fold-local selection & engineering...\")\n\nn_train = len(train)\nn_test  = len(test)\n\n# OOF composed probabilities: columns [H, L, None]\noof_probs = np.zeros((n_train, 3), dtype=np.float32)\ntest_probs_accum = np.zeros((n_test, 3), dtype=np.float32)\n\nfold_scores = []\n\nfor fold_id, (tr_idx, vl_idx) in enumerate(folds, 1):\n    print(f\"\\n  === Fold {fold_id}/{N_SPLITS} ===\")\n    X_tr_df = train.iloc[tr_idx]\n    X_vl_df = train.iloc[vl_idx]\n    y_tr = y[tr_idx]\n    y_vl = y[vl_idx]\n\n    # ---- fold-local MI to pick top-K boolean cols for rolling counts\n    top_bool = []\n    if bool_cols:\n        # MI on booleans only (cheap) — sample if needed\n        Xb = X_tr_df[bool_cols].values\n        top_k = min(TOP_K_MI_BOOL, Xb.shape[1])\n        mi = mutual_info_classif(Xb, y_tr, discrete_features=True, random_state=SEED)\n        order = np.argsort(mi)[::-1]\n        sel = order[:top_k]\n        top_bool = [bool_cols[i] for i in sel]\n        print(f\"    Selected {len(top_bool)} boolean cols for rolling counts\")\n\n    # ---- engineer rolling counts (safe: past-only, per ticker)\n    roll_tr = add_bool_rolling_counts(X_tr_df, top_bool)\n    roll_vl = add_bool_rolling_counts(X_vl_df, top_bool)\n    roll_te = add_bool_rolling_counts(test,   top_bool)\n\n    # assemble candidate feature frame (base + rolling counts)\n    base_cols = feature_cols  # already cleaned\n    Xtr_base = X_tr_df[base_cols].copy()\n    Xvl_base = X_vl_df[base_cols].copy()\n    Xte_base = test[base_cols].copy()\n\n    # concat engineered rolling features\n    X_tr_full = pd.concat([Xtr_base.reset_index(drop=True), roll_tr.reset_index(drop=True)], axis=1)\n    X_vl_full = pd.concat([Xvl_base.reset_index(drop=True), roll_vl.reset_index(drop=True)], axis=1)\n    X_te_full = pd.concat([Xte_base.reset_index(drop=True), roll_te.reset_index(drop=True)], axis=1)\n\n    # garbage collect intermediates we won't reuse\n    del Xtr_base, Xvl_base, Xte_base, roll_tr, roll_vl, roll_te\n    gc.collect()\n\n    # ---- fold-local filter: drop near-constant via VarianceThreshold\n    # For booleans, variance = p*(1-p). threshold=0.005 ~ drops p in [0,0.005] U [0.995,1]\n    vt = VarianceThreshold(threshold=0.005)\n    X_tr_vt = vt.fit_transform(X_tr_full)\n    X_vl_vt = vt.transform(X_vl_full)\n    X_te_vt = vt.transform(X_te_full)\n\n    # ---- fold-local SelectKBest by Mutual Information (works with mixed dtypes as numeric)\n    k_keep = min(K_SELECT, X_tr_vt.shape[1])\n    skb = SelectKBest(mutual_info_classif, k=k_keep)\n    X_tr_sel = skb.fit_transform(X_tr_vt, y_tr)\n    X_vl_sel = skb.transform(X_vl_vt)\n    X_te_sel = skb.transform(X_te_vt)\n\n    print(f\"    Feature dims: VT={X_tr_vt.shape[1]}  SKB={X_tr_sel.shape[1]}\")\n\n    # ---- Stage A: None (0) vs HL (1)\n    y_tr_A = (y_tr != label_map[\"None\"]).astype(int)\n    clf_A = LGBMClassifier(\n        objective=\"binary\",\n        class_weight={0:1, 1:45},     # emphasize recall for HL\n        num_leaves=31, max_depth=6,\n        feature_fraction=0.45, bagging_fraction=0.85, bagging_freq=1,\n        min_data_in_leaf=120, min_gain_to_split=0.05,\n        lambda_l1=2.0, lambda_l2=6.0,\n        learning_rate=0.05, n_estimators=1200,\n        random_state=SEED+fold_id, n_jobs=-1, verbosity=-1\n    )\n    clf_A.fit(X_tr_sel, y_tr_A)\n\n    pA_vl = clf_A.predict_proba(X_vl_sel)[:,1]\n    pA_te = clf_A.predict_proba(X_te_sel)[:,1]\n\n    # ---- Stage B: H (1) vs L (0), trained only on HL rows\n    mask_HL_tr = y_tr != label_map[\"None\"]\n    y_tr_B = (y_tr[mask_HL_tr] == label_map[\"H\"]).astype(int)\n\n    clf_B = LGBMClassifier(\n        objective=\"binary\",\n        class_weight=\"balanced\",\n        num_leaves=31, max_depth=6,\n        feature_fraction=0.45, bagging_fraction=0.85, bagging_freq=1,\n        min_data_in_leaf=80, min_gain_to_split=0.02,\n        lambda_l1=1.0, lambda_l2=3.0,\n        learning_rate=0.05, n_estimators=900,\n        random_state=SEED+fold_id, n_jobs=-1, verbosity=-1\n    )\n    clf_B.fit(X_tr_sel[mask_HL_tr], y_tr_B)\n\n    pH_vl = clf_B.predict_proba(X_vl_sel)[:,1]   # P(H | HL)\n    pH_te = clf_B.predict_proba(X_te_sel)[:,1]\n\n    # ---- Compose probs: P(None)=1-P(HL), P(H)=P(HL)*P(H|HL), P(L)=P(HL)*(1-P(H|HL))\n    vl_probs = np.column_stack([pA_vl*pH_vl, pA_vl*(1.0-pH_vl), 1.0-pA_vl]).astype(np.float32)\n    te_probs = np.column_stack([pA_te*pH_te, pA_te*(1.0-pH_te), 1.0-pA_te]).astype(np.float32)\n\n    # store OOF & accumulate test\n    oof_probs[vl_idx] = vl_probs\n    test_probs_accum += te_probs / N_SPLITS\n\n    # quick fold score with naïve argmax (pre-thresholding)\n    pred_vl_argmax = vl_probs.argmax(axis=1)\n    fold_f1 = f1_score(y_vl, pred_vl_argmax, average=\"macro\")\n    fold_scores.append(fold_f1)\n    print(f\"    Fold Macro-F1 (argmax): {fold_f1:.4f}\")\n\n    # cleanup\n    del X_tr_full, X_vl_full, X_te_full, X_tr_vt, X_vl_vt, X_te_vt, X_tr_sel, X_vl_sel, X_te_sel\n    gc.collect()\n\nprint(\"\\n[6/8] Cross-validated performance (pre-threshold):\")\nprint(f\"✓ CV Macro-F1 (mean±std): {np.mean(fold_scores):.4f} ± {np.std(fold_scores):.4f}\")\n\n# -------------------\n# Threshold Optimization on OOF (2-parameter grid)\n# -------------------\nprint(\"\\n[7/8] Optimizing thresholds on OOF...\")\n\n# We optimize two thresholds:\n#  - tau_A : HL gate (if P(HL) >= tau_A -> not None)\n#  - tau_H : H vs L split (if P(H|HL) >= tau_H -> H else L)\nP_HL = oof_probs[:,0] + oof_probs[:,1]\nP_H_given = np.divide(oof_probs[:,0], np.maximum(P_HL, 1e-8))\n\nbest = {\"f1\": -1, \"tau_A\": 0.5, \"tau_H\": 0.5}\ny_true = y\n\ndef apply_thresholds(p_hl, p_h, tau_A, tau_H):\n    # start as None\n    pred = np.full(len(p_hl), label_map[\"None\"], dtype=int)\n    mask_hl = p_hl >= tau_A\n    # among HL, choose H/L by tau_H\n    pred[mask_hl] = np.where(p_h[mask_hl] >= tau_H, label_map[\"H\"], label_map[\"L\"])\n    return pred\n\n# grid (focused but adjustable)\ntau_A_grid = np.linspace(0.30, 0.75, 19)   # HL gate\ntau_H_grid = np.linspace(0.35, 0.65, 13)   # H vs L split\n\nfor ta in tau_A_grid:\n    for th in tau_H_grid:\n        pred = apply_thresholds(P_HL, P_H_given, ta, th)\n        f1m = f1_score(y_true, pred, average=\"macro\")\n        if f1m > best[\"f1\"]:\n            best.update({\"f1\": f1m, \"tau_A\": ta, \"tau_H\": th})\n\nprint(f\"✓ Optimal thresholds: tau_A={best['tau_A']:.3f}, tau_H={best['tau_H']:.3f}\")\nprint(f\"✓ OOF Macro-F1 (thresholded): {best['f1']:.4f}\")\n\n# diagnostics\npred_oof = apply_thresholds(P_HL, P_H_given, best[\"tau_A\"], best[\"tau_H\"])\nprint(\"\\nOOF classification report (thresholded):\")\nprint(classification_report(y_true, pred_oof, target_names=classes, digits=3))\n\n# -------------------\n# Submission\n# -------------------\nprint(\"\\n[8/8] Generating submission...\")\n\n# Apply same thresholds to averaged test probs\nP_HL_te = test_probs_accum[:,0] + test_probs_accum[:,1]\nP_H_given_te = np.divide(test_probs_accum[:,0], np.maximum(P_HL_te, 1e-8))\npred_test_idx = apply_thresholds(P_HL_te, P_H_given_te, best[\"tau_A\"], best[\"tau_H\"])\npred_test_lbl = np.array(classes, dtype=object)[pred_test_idx]\n\nsubmission = pd.DataFrame({\n    \"id\": test[\"id\"] if \"id\" in test.columns else np.arange(len(test)),\n    \"class_label\": pred_test_lbl\n})\n\nsubmission.to_csv(OUTPUT_PATH, index=False)\nprint(f\"✓ Saved: {OUTPUT_PATH}\")\n\n# Distribution\ndist = submission[\"class_label\"].value_counts()\nprint(\"\\nPrediction distribution:\")\nfor c in classes:\n    cnt = int(dist.get(c, 0))\n    print(f\"  {c}: {cnt} ({100.0*cnt/len(submission):.1f}%)\")\n\nprint(\"\\n\" + \"=\"*70)\nprint(\"DONE\")\nprint(\"=\"*70)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-10T03:55:35.747163Z","iopub.execute_input":"2025-10-10T03:55:35.747476Z","iopub.status.idle":"2025-10-10T04:25:35.477914Z","shell.execute_reply.started":"2025-10-10T03:55:35.747457Z","shell.execute_reply":"2025-10-10T04:25:35.477110Z"}},"outputs":[],"execution_count":null}]}