{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# [training] 3rd place solution\n\nI will post a description of the model in discussion. Inference is done in another notebook.\n* Inference: https://www.kaggle.com/sugghi/inference-3rd-place-solution/\n\nI referred to various notebooks when coding. In particular, the following notebooks were used directly.\n* Submitting Lagged Features via API：\nhttps://www.kaggle.com/tomforbes/gresearch-submitting-lagged-features-via-api\n* EmbargoCV：\nhttps://www.kaggle.com/nrcjea001/lgbm-embargocv-weightedpearson-lagtarget/notebook\n\nIn addition, although not used in this notebook, local api published by @jagofc helped me a lot in coding.\n* local api：https://www.kaggle.com/code/jagofc/local-api/\n\nAs you can see from my code, I am a novice in machine learning and python.\nIf you see anything  to improve on or any mistakes, I'd be very happy to hear about them!","metadata":{}},{"cell_type":"code","source":"import os\nimport random\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport lightgbm as lgb\nimport gresearch_crypto\nimport time\nimport datetime\n\nimport pickle\nimport gc\n\nfrom tqdm import tqdm\n\nn_fold = 7\nseed0 = 8586\nuse_supple_for_train = True\n\n# If True, the period used to evaluate Public LB will not be used for training.\n# Set to False on final submission.\nnot_use_overlap_to_train = False\n\nTRAIN_CSV = '/kaggle/input/g-research-crypto-forecasting/train.csv'\nSUPPLE_TRAIN_CSV = '/kaggle/input/g-research-crypto-forecasting/supplemental_train.csv'\nASSET_DETAILS_CSV = '/kaggle/input/g-research-crypto-forecasting/asset_details.csv'\n\npd.set_option('display.max_rows', 6)\npd.set_option('display.max_columns', 350)","metadata":{"execution":{"iopub.status.busy":"2026-08-05T10:32:29.021741Z","iopub.execute_input":"2026-08-05T10:32:29.022125Z","iopub.status.idle":"2026-08-05T10:32:30.940089Z","shell.execute_reply.started":"2026-08-05T10:32:29.022018Z","shell.execute_reply":"2026-08-05T10:32:30.939321Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lags = [60,300,900]\n\n# ============================================================\n# Missing-data and volume feature configuration\n# ============================================================\n\n# Giữ nguyên ngưỡng của notebook gốc:\n# chỉ forward fill tối đa 100 phút liên tiếp.\nMAX_FFILL_MINUTES = 100\n\n# Sau 60 phút, freshness giảm còn một nửa.\nFRESHNESS_HALF_LIFE_MINUTES = 60.0\n\n# Volume được chuẩn hóa theo cùng khung giờ UTC,\n# dựa trên khoảng 7 ngày dữ liệu lịch sử gần nhất.\nVOLUME_BASELINE_DAYS = 7\n\n# Cần ít nhất 2 ngày lịch sử cùng khung giờ\n# trước khi tính baseline volume.\nVOLUME_MIN_HISTORY_DAYS = 2\n\n# Tránh chia cho 0 hoặc log(0).\nEPS = np.float32(1e-8)","metadata":{"execution":{"iopub.status.busy":"2026-08-05T10:32:30.941505Z","iopub.execute_input":"2026-08-05T10:32:30.941804Z","iopub.status.idle":"2026-08-05T10:32:30.948046Z","shell.execute_reply.started":"2026-08-05T10:32:30.941774Z","shell.execute_reply":"2026-08-05T10:32:30.946994Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"params = {\n    'early_stopping_rounds': 50,\n    'objective': 'regression',\n    'metric': 'rmse',\n#     'metric': 'None',\n    'boosting_type': 'gbdt',\n    'max_depth': 5,\n    'verbose': -1,\n    'max_bin':600,\n    'min_data_in_leaf':50,\n    'learning_rate': 0.03,\n    'subsample': 0.7,\n    'subsample_freq': 1,\n    'feature_fraction': 1,\n    'lambda_l1': 0.5,\n    'lambda_l2': 2,\n    'seed':seed0,\n    'feature_fraction_seed': seed0,\n    'bagging_fraction_seed': seed0,\n    'drop_seed': seed0,\n    'data_random_seed': seed0,\n    'extra_trees': True,\n    'extra_seed': seed0,\n    'zero_as_missing': True,\n    \"first_metric_only\": True\n         }","metadata":{"execution":{"iopub.status.busy":"2026-08-05T10:32:30.949461Z","iopub.execute_input":"2026-08-05T10:32:30.949798Z","iopub.status.idle":"2026-08-05T10:32:30.959801Z","shell.execute_reply.started":"2026-08-05T10:32:30.949754Z","shell.execute_reply":"2026-08-05T10:32:30.958920Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def reduce_mem_usage(df):\n    \"\"\"\n    Reduce DataFrame memory usage.\n\n    Improvements over the original version:\n    - Integer columns are downcast to the smallest safe integer dtype.\n    - Floating-point columns are always stored as float32, never float16.\n    - Object columns are converted to category.\n    \"\"\"\n\n    start_mem = df.memory_usage(deep=True).sum() / 1024**2\n\n    for col in df.columns:\n        col_type = df[col].dtype\n\n        # Boolean columns are already compact.\n        if pd.api.types.is_bool_dtype(col_type):\n            continue\n\n        # Downcast integer columns safely.\n        if pd.api.types.is_integer_dtype(col_type):\n            df[col] = pd.to_numeric(\n                df[col],\n                downcast=\"integer\",\n            )\n\n        # Important change:\n        # float64 -> float32, but never float16.\n        elif pd.api.types.is_float_dtype(col_type):\n            df[col] = df[col].astype(np.float32)\n\n        # Save memory for string/object columns.\n        elif pd.api.types.is_object_dtype(col_type):\n            df[col] = df[col].astype(\"category\")\n\n    end_mem = df.memory_usage(deep=True).sum() / 1024**2\n\n    reduction = 100 * (start_mem - end_mem) / max(start_mem, EPS)\n\n    print(\n        f\"Memory usage: {start_mem:.2f} MB \"\n        f\"-> {end_mem:.2f} MB \"\n        f\"({reduction:.1f}% reduction)\"\n    )\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2026-08-05T10:32:30.961778Z","iopub.execute_input":"2026-08-05T10:32:30.962028Z","iopub.status.idle":"2026-08-05T10:32:30.977012Z","shell.execute_reply.started":"2026-08-05T10:32:30.962000Z","shell.execute_reply":"2026-08-05T10:32:30.976049Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_asset_details = pd.read_csv(ASSET_DETAILS_CSV).sort_values(\"Asset_ID\")\n# df_asset_details","metadata":{"execution":{"iopub.status.busy":"2026-08-05T10:32:30.978349Z","iopub.execute_input":"2026-08-05T10:32:30.978706Z","iopub.status.idle":"2026-08-05T10:32:31.005449Z","shell.execute_reply.started":"2026-08-05T10:32:30.978662Z","shell.execute_reply":"2026-08-05T10:32:31.004773Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"%%time\n\nusecols = [\n    \"timestamp\",\n    \"Asset_ID\",\n    \"Close\",\n    \"VWAP\",\n    \"Volume\",\n    \"Target\",\n]\n\ndf_train = pd.read_csv(\n    TRAIN_CSV,\n    usecols=usecols,\n)\n\nif use_supple_for_train:\n    df_supple = pd.read_csv(\n        SUPPLE_TRAIN_CSV,\n        usecols=usecols,\n    )\n\n    df_train = pd.concat(\n        [df_train, df_supple],\n        axis=0,\n        ignore_index=True,\n    )\n\n    del df_supple\n\n\n# ------------------------------------------------------------\n# Sort before all causal time-series feature generation.\n# ------------------------------------------------------------\n\ndf_train = df_train.sort_values(\n    [\"Asset_ID\", \"timestamp\"],\n    ascending=True,\n).reset_index(drop=True)\n\n\n# ------------------------------------------------------------\n# Check duplicate asset-minute rows.\n#\n# Each (timestamp, Asset_ID) pair should appear at most once.\n# Otherwise the later wide merge can create duplicated rows.\n# ------------------------------------------------------------\n\nduplicated_mask = df_train.duplicated(\n    subset=[\"timestamp\", \"Asset_ID\"],\n    keep=False,\n)\n\nif duplicated_mask.any():\n    duplicate_count = int(duplicated_mask.sum())\n\n    raise ValueError(\n        f\"Found {duplicate_count} duplicated \"\n        \"(timestamp, Asset_ID) rows.\"\n    )\n\n\n# ------------------------------------------------------------\n# Create time-of-day information in UTC.\n#\n# Unix timestamps in this competition are treated as UTC.\n# ------------------------------------------------------------\n\ndatetime_utc = pd.to_datetime(\n    df_train[\"timestamp\"],\n    unit=\"s\",\n    utc=True,\n)\n\ndf_train[\"_hour_utc\"] = datetime_utc.dt.hour.astype(np.int8)\n\n\n# ------------------------------------------------------------\n# Raw log-volume.\n#\n# log1p helps compress the very long-tailed Volume distribution.\n# ------------------------------------------------------------\n\nsafe_volume = df_train[\"Volume\"].clip(lower=0)\n\ndf_train[\"log_volume\"] = np.log1p(\n    safe_volume\n).astype(np.float32)\n\n\n# ------------------------------------------------------------\n# Volume baseline by:\n#   Asset_ID + hour of day\n#\n# Example:\n# Volume at BTC 08:35 is compared with previous BTC observations\n# that also occurred during hour 08:00-08:59 UTC.\n#\n# shift(1) is critical:\n# the current row is not allowed to participate in its own baseline.\n# ------------------------------------------------------------\n\nvolume_window = 60 * VOLUME_BASELINE_DAYS\nvolume_min_periods = 60 * VOLUME_MIN_HISTORY_DAYS\n\nvolume_group = df_train.groupby(\n    [\"Asset_ID\", \"_hour_utc\"],\n    sort=False,\n)[\"log_volume\"]\n\n\ndf_train[\"volume_hour_mean_7d\"] = volume_group.transform(\n    lambda series: (\n        series.shift(1)\n        .rolling(\n            window=volume_window,\n            min_periods=volume_min_periods,\n        )\n        .mean()\n    )\n)\n\n\ndf_train[\"volume_hour_std_7d\"] = volume_group.transform(\n    lambda series: (\n        series.shift(1)\n        .rolling(\n            window=volume_window,\n            min_periods=volume_min_periods,\n        )\n        .std()\n    )\n)\n\n\n# ------------------------------------------------------------\n# Volume surprise relative to the normal volume of that hour.\n#\n# Positive:\n# current volume is unusually large for this time of day.\n#\n# Negative:\n# current volume is lower than normal for this time of day.\n# ------------------------------------------------------------\n\nvolume_std_safe = df_train[\n    \"volume_hour_std_7d\"\n].where(\n    df_train[\"volume_hour_std_7d\"] > EPS\n)\n\ndf_train[\"volume_hour_z_7d\"] = (\n    (\n        df_train[\"log_volume\"]\n        - df_train[\"volume_hour_mean_7d\"]\n    )\n    / volume_std_safe\n).astype(np.float32)\n\n\n# A simpler and often more stable alternative to the z-score.\ndf_train[\"volume_hour_log_ratio_7d\"] = (\n    df_train[\"log_volume\"]\n    - df_train[\"volume_hour_mean_7d\"]\n).astype(np.float32)\n\n\n# The hour column was only an intermediate grouping key.\ndf_train = df_train.drop(\n    columns=[\n        \"_hour_utc\",\n        \"volume_hour_std_7d\",\n        \"volume_hour_mean_7d\",\n    ]\n)\n\n\ndf_train = reduce_mem_usage(df_train)\n\ndisplay(df_train.head())\ndisplay(df_train.dtypes)","metadata":{"execution":{"iopub.status.busy":"2026-08-05T10:32:31.006961Z","iopub.execute_input":"2026-08-05T10:32:31.007279Z","iopub.status.idle":"2026-08-05T10:34:24.957129Z","shell.execute_reply.started":"2026-08-05T10:32:31.007237Z","shell.execute_reply":"2026-08-05T10:34:24.956162Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\ncolumns_to_widen = [\n    \"Close\",\n    \"VWAP\",\n    \"Volume\",\n    \"log_volume\",\n    \"volume_hour_z_7d\",\n    \"volume_hour_log_ratio_7d\",\n    \"Target\",\n]\n\ntrain_merged = None\n\n\nfor asset_id in tqdm(range(14)):\n\n    asset_df = df_train.loc[\n        df_train[\"Asset_ID\"] == asset_id,\n        [\"timestamp\"] + columns_to_widen,\n    ].copy()\n\n    rename_mapping = {\n        col: f\"{col}_{asset_id}\"\n        for col in columns_to_widen\n    }\n\n    asset_df = asset_df.rename(\n        columns=rename_mapping\n    )\n\n    if train_merged is None:\n        train_merged = asset_df\n\n    else:\n        train_merged = train_merged.merge(\n            asset_df,\n            on=\"timestamp\",\n            how=\"outer\",\n            validate=\"one_to_one\",\n        )\n\n\ntrain_merged = train_merged.sort_values(\n    \"timestamp\",\n    ascending=True,\n).reset_index(drop=True)\n\n\n# ------------------------------------------------------------\n# Global time-of-day features.\n#\n# Cyclic encoding solves the wrap-around problem:\n# minute 1439 and minute 0 should be close to each other.\n# ------------------------------------------------------------\n\ndatetime_utc = pd.to_datetime(\n    train_merged[\"timestamp\"],\n    unit=\"s\",\n    utc=True,\n)\n\nminute_of_day = (\n    datetime_utc.dt.hour * 60\n    + datetime_utc.dt.minute\n).astype(np.int16)\n\nday_of_week = datetime_utc.dt.dayofweek.astype(np.int8)\n\n\ntrain_merged[\"minute_of_day_sin\"] = np.sin(\n    2.0 * np.pi * minute_of_day / 1440.0\n).astype(np.float32)\n\ntrain_merged[\"minute_of_day_cos\"] = np.cos(\n    2.0 * np.pi * minute_of_day / 1440.0\n).astype(np.float32)\n\n\ntrain_merged[\"day_of_week_sin\"] = np.sin(\n    2.0 * np.pi * day_of_week / 7.0\n).astype(np.float32)\n\ntrain_merged[\"day_of_week_cos\"] = np.cos(\n    2.0 * np.pi * day_of_week / 7.0\n).astype(np.float32)\n\n\ndisplay(train_merged.head())\nprint(train_merged.shape)","metadata":{"execution":{"iopub.status.busy":"2026-08-05T10:34:24.958728Z","iopub.execute_input":"2026-08-05T10:34:24.959076Z","iopub.status.idle":"2026-08-05T10:35:07.722010Z","shell.execute_reply.started":"2026-08-05T10:34:24.959021Z","shell.execute_reply":"2026-08-05T10:35:07.721185Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\nmissing_columns = []\ngap_columns = []\nfreshness_columns = []\n\n\nfor asset_id in range(14):\n\n    close_col = f\"Close_{asset_id}\"\n    vwap_col = f\"VWAP_{asset_id}\"\n    log_volume_col = f\"log_volume_{asset_id}\"\n    volume_z_col = f\"volume_hour_z_7d_{asset_id}\"\n    volume_ratio_col = f\"volume_hour_log_ratio_7d_{asset_id}\"\n\n\n    # Preserve the original values before forward fill.\n    raw_close = train_merged[close_col].copy()\n    raw_vwap = train_merged[vwap_col].copy()\n    raw_log_volume = train_merged[log_volume_col].copy()\n    raw_volume_z = train_merged[volume_z_col].copy()\n    raw_volume_ratio = train_merged[volume_ratio_col].copy()\n\n\n    # --------------------------------------------------------\n    # A candle is considered observed when its Close exists.\n    # --------------------------------------------------------\n\n    observed = raw_close.notna()\n\n\n    # --------------------------------------------------------\n    # Missing indicator.\n    # --------------------------------------------------------\n\n    missing_col = f\"is_missing_id{asset_id}\"\n\n    train_merged[missing_col] = (\n        ~observed\n    ).astype(np.int8)\n\n    missing_columns.append(missing_col)\n\n\n    # --------------------------------------------------------\n    # Timestamp of the last genuinely observed Close.\n    #\n    # This is causal:\n    # only previous observations are forwarded.\n    # --------------------------------------------------------\n\n    last_observed_timestamp = (\n        train_merged[\"timestamp\"]\n        .where(observed)\n        .ffill()\n    )\n\n\n    # Whether this asset has ever been observed before.\n    train_merged[f\"has_history_id{asset_id}\"] = (\n        last_observed_timestamp.notna()\n    ).astype(np.int8)\n\n\n    # --------------------------------------------------------\n    # Time elapsed since the last genuine observation.\n    # --------------------------------------------------------\n\n    gap_minutes = (\n        (\n            train_merged[\"timestamp\"]\n            - last_observed_timestamp\n        )\n        / 60.0\n    )\n\n    gap_minutes = gap_minutes.astype(np.float32)\n\n    gap_col = f\"gap_minutes_id{asset_id}\"\n\n    train_merged[gap_col] = gap_minutes\n    gap_columns.append(gap_col)\n\n\n    train_merged[f\"gap_days_id{asset_id}\"] = (\n        gap_minutes / 1440.0\n    ).astype(np.float32)\n\n\n    # Compress very long gaps.\n    train_merged[f\"log_gap_minutes_id{asset_id}\"] = (\n        np.log1p(gap_minutes)\n    ).astype(np.float32)\n\n\n    # Explicit indicator for gaps too long to forward fill.\n    train_merged[f\"gap_over_100_id{asset_id}\"] = (\n        gap_minutes > MAX_FFILL_MINUTES\n    ).astype(np.int8)\n\n\n    # Was the current Close generated by short-gap ffill?\n    train_merged[f\"is_forward_filled_id{asset_id}\"] = (\n        (gap_minutes > 0)\n        & (gap_minutes <= MAX_FFILL_MINUTES)\n    ).astype(np.int8)\n\n\n    # --------------------------------------------------------\n    # Freshness.\n    #\n    # gap = 0 minutes  -> freshness = 1.0\n    # gap = 60 minutes -> freshness = 0.5\n    # gap = 120        -> freshness = 0.25\n    # --------------------------------------------------------\n\n    freshness = np.power(\n        0.5,\n        gap_minutes / FRESHNESS_HALF_LIFE_MINUTES,\n    ).astype(np.float32)\n\n    freshness_col = f\"freshness_id{asset_id}\"\n\n    train_merged[freshness_col] = freshness\n    freshness_columns.append(freshness_col)\n\n\n    # --------------------------------------------------------\n    # Last genuinely observed market state.\n    #\n    # These values may be old, but gap and freshness tell the\n    # model exactly how old they are.\n    # --------------------------------------------------------\n\n    last_real_close = (\n        raw_close\n        .where(observed)\n        .ffill()\n        .astype(np.float32)\n    )\n\n    last_real_vwap = (\n        raw_vwap\n        .where(observed)\n        .ffill()\n        .astype(np.float32)\n    )\n\n    last_real_log_volume = (\n        raw_log_volume\n        .where(observed)\n        .ffill()\n        .astype(np.float32)\n    )\n\n    last_real_volume_z = (\n        raw_volume_z\n        .where(observed)\n        .ffill()\n        .astype(np.float32)\n    )\n\n    last_real_volume_ratio = (\n        raw_volume_ratio\n        .where(observed)\n        .ffill()\n        .astype(np.float32)\n    )\n\n\n    train_merged[\n        f\"last_real_close_id{asset_id}\"\n    ] = last_real_close\n\n    train_merged[\n        f\"last_real_vwap_id{asset_id}\"\n    ] = last_real_vwap\n\n    train_merged[\n        f\"last_real_log_volume_id{asset_id}\"\n    ] = last_real_log_volume\n\n    train_merged[\n        f\"last_real_volume_hour_z_7d_id{asset_id}\"\n    ] = last_real_volume_z\n\n    train_merged[\n        f\"last_real_volume_hour_log_ratio_7d_id{asset_id}\"\n    ] = last_real_volume_ratio\n\n\n    # --------------------------------------------------------\n    # Last real Close relative to last real VWAP.\n    #\n    # Positive:\n    # Close was above volume-weighted average price.\n    #\n    # Negative:\n    # Close was below VWAP.\n    # --------------------------------------------------------\n\n    valid_last_price = (\n        (last_real_close > 0)\n        & (last_real_vwap > 0)\n    )\n\n    last_close_vwap_ratio = (\n        last_real_close / last_real_vwap\n    ).where(valid_last_price)\n\n    train_merged[\n        f\"log_last_real_close/vwap_id{asset_id}\"\n    ] = np.log(\n        last_close_vwap_ratio\n    ).astype(np.float32)\n\n\n    # --------------------------------------------------------\n    # Decayed versions:\n    #\n    # A strong volume signal from one minute ago should be more\n    # useful than the same signal from ten hours ago.\n    # --------------------------------------------------------\n\n    train_merged[\n        f\"decayed_last_volume_hour_z_7d_id{asset_id}\"\n    ] = (\n        last_real_volume_z * freshness\n    ).astype(np.float32)\n\n    train_merged[\n        f\"decayed_last_volume_hour_log_ratio_7d_id{asset_id}\"\n    ] = (\n        last_real_volume_ratio * freshness\n    ).astype(np.float32)\n\n\n    # --------------------------------------------------------\n    # Limited forward fill.\n    #\n    # Close and VWAP are filled only for short gaps.\n    # Volume is deliberately NOT forward filled.\n    # Missing volume is not equivalent to zero or old volume.\n    # --------------------------------------------------------\n\n    train_merged[close_col] = (\n        raw_close\n        .ffill(limit=MAX_FFILL_MINUTES)\n        .astype(np.float32)\n    )\n\n    train_merged[vwap_col] = (\n        raw_vwap\n        .ffill(limit=MAX_FFILL_MINUTES)\n        .astype(np.float32)\n    )\n\n\n# ============================================================\n# Market-wide missing-data features\n# ============================================================\n\ntrain_merged[\"market_missing_ratio\"] = (\n    train_merged[missing_columns]\n    .mean(axis=1)\n    .astype(np.float32)\n)\n\n\ntrain_merged[\"market_observed_count\"] = (\n    14\n    - train_merged[missing_columns].sum(axis=1)\n).astype(np.int8)\n\n\ntrain_merged[\"market_mean_gap_minutes\"] = (\n    train_merged[gap_columns]\n    .mean(axis=1, skipna=True)\n    .astype(np.float32)\n)\n\n\ntrain_merged[\"market_max_gap_minutes\"] = (\n    train_merged[gap_columns]\n    .max(axis=1, skipna=True)\n    .astype(np.float32)\n)\n\n\ntrain_merged[\"market_mean_freshness\"] = (\n    train_merged[freshness_columns]\n    .mean(axis=1, skipna=True)\n    .astype(np.float32)\n)\n\n\ndisplay(\n    train_merged[\n        [\n            \"timestamp\",\n            \"Close_0\",\n            \"last_real_close_id0\",\n            \"is_missing_id0\",\n            \"gap_minutes_id0\",\n            \"freshness_id0\",\n            \"market_missing_ratio\",\n        ]\n    ].head(20)\n)","metadata":{"execution":{"iopub.status.busy":"2026-08-05T10:35:07.723351Z","iopub.execute_input":"2026-08-05T10:35:07.723608Z","iopub.status.idle":"2026-08-05T10:35:23.430686Z","shell.execute_reply.started":"2026-08-05T10:35:07.723553Z","shell.execute_reply":"2026-08-05T10:35:23.429320Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Engineering","metadata":{}},{"cell_type":"code","source":"def get_features(df, train=True):\n\n    df = df.copy()\n\n\n    # ========================================================\n    # Train/validation time flag\n    # ========================================================\n\n    if train == True:\n\n        totimestamp = lambda s: np.int32(\n            time.mktime(\n                datetime.datetime.strptime(\n                    s,\n                    \"%d/%m/%Y\",\n                ).timetuple()\n            )\n        )\n\n        valid_window = [\n            totimestamp(\"12/03/2021\")\n        ]\n\n        df[\"train_flg\"] = np.where(\n            df[\"timestamp\"] >= valid_window[0],\n            0,\n            1,\n        ).astype(np.int8)\n\n\n        supple_start_window = [\n            totimestamp(\"22/09/2021\")\n        ]\n\n        if use_supple_for_train:\n\n            df[\"train_flg\"] = np.where(\n                df[\"timestamp\"]\n                >= supple_start_window[0],\n                1,\n                df[\"train_flg\"],\n            ).astype(np.int8)\n\n\n    # ========================================================\n    # Internal helper: safe log ratio\n    # ========================================================\n\n    def safe_log_ratio(numerator, denominator):\n\n        numerator_np = numerator.to_numpy(\n            dtype=np.float64,\n            copy=False,\n        )\n\n        denominator_np = denominator.to_numpy(\n            dtype=np.float64,\n            copy=False,\n        )\n\n        output = np.full(\n            len(numerator_np),\n            np.nan,\n            dtype=np.float32,\n        )\n\n        valid = (\n            np.isfinite(numerator_np)\n            & np.isfinite(denominator_np)\n            & (numerator_np > 0)\n            & (denominator_np > 0)\n        )\n\n        output[valid] = np.log(\n            numerator_np[valid]\n            / denominator_np[valid]\n        ).astype(np.float32)\n\n        return output\n\n\n    # ========================================================\n    # Internal helper: row-wise NaN-aware mean\n    # ========================================================\n\n    def row_nanmean(feature_names, generated_features):\n\n        matrix = np.column_stack(\n            [\n                generated_features[name]\n                for name in feature_names\n            ]\n        ).astype(np.float32)\n\n        valid_count = np.sum(\n            np.isfinite(matrix),\n            axis=1,\n        )\n\n        value_sum = np.nansum(\n            matrix,\n            axis=1,\n        )\n\n        output = np.full(\n            matrix.shape[0],\n            np.nan,\n            dtype=np.float32,\n        )\n\n        valid_rows = valid_count > 0\n\n        output[valid_rows] = (\n            value_sum[valid_rows]\n            / valid_count[valid_rows]\n        ).astype(np.float32)\n\n        return output\n\n\n    generated = {}\n\n\n    # ========================================================\n    # Per-asset Close and VWAP features\n    # ========================================================\n\n    for asset_id in range(14):\n\n        close = df[\n            f\"Close_{asset_id}\"\n        ].astype(np.float32)\n\n        vwap = df[\n            f\"VWAP_{asset_id}\"\n        ].astype(np.float32)\n\n\n        # Current Close relative to current VWAP.\n        generated[\n            f\"log_close/vwap_id{asset_id}\"\n        ] = safe_log_ratio(\n            close,\n            vwap,\n        )\n\n\n        for lag in lags:\n\n            # ------------------------------------------------\n            # Close rolling mean and return\n            # ------------------------------------------------\n\n            close_rolling_mean = close.rolling(\n                window=lag,\n                min_periods=lag,\n            ).mean()\n\n            close_shifted = close.shift(lag)\n\n\n            generated[\n                f\"log_close/mean_{lag}_id{asset_id}\"\n            ] = safe_log_ratio(\n                close,\n                close_rolling_mean,\n            )\n\n\n            generated[\n                f\"log_return_{lag}_id{asset_id}\"\n            ] = safe_log_ratio(\n                close,\n                close_shifted,\n            )\n\n\n            # ------------------------------------------------\n            # VWAP rolling mean and return\n            # ------------------------------------------------\n\n            vwap_rolling_mean = vwap.rolling(\n                window=lag,\n                min_periods=lag,\n            ).mean()\n\n            vwap_shifted = vwap.shift(lag)\n\n\n            generated[\n                f\"log_vwap/mean_{lag}_id{asset_id}\"\n            ] = safe_log_ratio(\n                vwap,\n                vwap_rolling_mean,\n            )\n\n\n            generated[\n                f\"log_vwap_return_{lag}_id{asset_id}\"\n            ] = safe_log_ratio(\n                vwap,\n                vwap_shifted,\n            )\n\n\n    # ========================================================\n    # Market means and asset-relative features\n    # ========================================================\n\n    for lag in lags:\n\n        close_mean_feature_names = [\n            f\"log_close/mean_{lag}_id{asset_id}\"\n            for asset_id in range(14)\n        ]\n\n        close_return_feature_names = [\n            f\"log_return_{lag}_id{asset_id}\"\n            for asset_id in range(14)\n        ]\n\n        vwap_mean_feature_names = [\n            f\"log_vwap/mean_{lag}_id{asset_id}\"\n            for asset_id in range(14)\n        ]\n\n        vwap_return_feature_names = [\n            f\"log_vwap_return_{lag}_id{asset_id}\"\n            for asset_id in range(14)\n        ]\n\n\n        mean_close_mean = row_nanmean(\n            close_mean_feature_names,\n            generated,\n        )\n\n        mean_close_return = row_nanmean(\n            close_return_feature_names,\n            generated,\n        )\n\n        mean_vwap_mean = row_nanmean(\n            vwap_mean_feature_names,\n            generated,\n        )\n\n        mean_vwap_return = row_nanmean(\n            vwap_return_feature_names,\n            generated,\n        )\n\n\n        generated[\n            f\"mean_close/mean_{lag}\"\n        ] = mean_close_mean\n\n        generated[\n            f\"mean_log_returns_{lag}\"\n        ] = mean_close_return\n\n        generated[\n            f\"mean_vwap/mean_{lag}\"\n        ] = mean_vwap_mean\n\n        generated[\n            f\"mean_log_vwap_returns_{lag}\"\n        ] = mean_vwap_return\n\n\n        for asset_id in range(14):\n\n            generated[\n                f\"log_close/mean_{lag}\"\n                f\"-mean_close/mean_{lag}_id{asset_id}\"\n            ] = (\n                generated[\n                    f\"log_close/mean_{lag}_id{asset_id}\"\n                ]\n                - mean_close_mean\n            ).astype(np.float32)\n\n\n            generated[\n                f\"log_return_{lag}\"\n                f\"-mean_log_returns_{lag}_id{asset_id}\"\n            ] = (\n                generated[\n                    f\"log_return_{lag}_id{asset_id}\"\n                ]\n                - mean_close_return\n            ).astype(np.float32)\n\n\n            generated[\n                f\"log_vwap/mean_{lag}\"\n                f\"-mean_vwap/mean_{lag}_id{asset_id}\"\n            ] = (\n                generated[\n                    f\"log_vwap/mean_{lag}_id{asset_id}\"\n                ]\n                - mean_vwap_mean\n            ).astype(np.float32)\n\n\n            generated[\n                f\"log_vwap_return_{lag}\"\n                f\"-mean_log_vwap_returns_{lag}_id{asset_id}\"\n            ] = (\n                generated[\n                    f\"log_vwap_return_{lag}_id{asset_id}\"\n                ]\n                - mean_vwap_return\n            ).astype(np.float32)\n\n\n    # ========================================================\n    # Close relative to VWAP across the whole market\n    # ========================================================\n\n    close_vwap_feature_names = [\n        f\"log_close/vwap_id{asset_id}\"\n        for asset_id in range(14)\n    ]\n\n    mean_close_vwap = row_nanmean(\n        close_vwap_feature_names,\n        generated,\n    )\n\n    generated[\n        \"mean_log_close/vwap\"\n    ] = mean_close_vwap\n\n\n    for asset_id in range(14):\n\n        generated[\n            f\"log_close/vwap\"\n            f\"-mean_log_close/vwap_id{asset_id}\"\n        ] = (\n            generated[\n                f\"log_close/vwap_id{asset_id}\"\n            ]\n            - mean_close_vwap\n        ).astype(np.float32)\n\n\n    # ========================================================\n    # Market-relative normalized Volume\n    # ========================================================\n\n    volume_z_columns = [\n        f\"volume_hour_z_7d_{asset_id}\"\n        for asset_id in range(14)\n    ]\n\n    available_volume_z_columns = [\n        col\n        for col in volume_z_columns\n        if col in df.columns\n    ]\n\n    if len(available_volume_z_columns) > 0:\n\n        market_volume_z = (\n            df[available_volume_z_columns]\n            .mean(axis=1, skipna=True)\n            .to_numpy(dtype=np.float32)\n        )\n\n        generated[\n            \"mean_volume_hour_z_7d\"\n        ] = market_volume_z\n\n\n        for asset_id in range(14):\n\n            col = f\"volume_hour_z_7d_{asset_id}\"\n\n            if col in df.columns:\n\n                generated[\n                    f\"volume_hour_z_7d\"\n                    f\"-mean_id{asset_id}\"\n                ] = (\n                    df[col].to_numpy(\n                        dtype=np.float32\n                    )\n                    - market_volume_z\n                ).astype(np.float32)\n\n\n    # ========================================================\n    # Add all generated features in one operation\n    # to avoid DataFrame fragmentation.\n    # ========================================================\n\n    generated_df = pd.DataFrame(\n        generated,\n        index=df.index,\n    )\n\n    generated_df = generated_df.astype(\n        np.float32\n    )\n\n    df = pd.concat(\n        [df, generated_df],\n        axis=1,\n    )\n\n\n    # ========================================================\n    # Training-only cleanup\n    # ========================================================\n\n    if train == True:\n\n        # Current raw price and raw volume columns are removed.\n        #\n        # Their transformed features remain.\n        #\n        # last_real_close and last_real_vwap are deliberately\n        # retained for the proposed missing-data experiment.\n        raw_columns_to_drop = []\n\n        for asset_id in range(14):\n\n            raw_columns_to_drop.extend(\n                [\n                    f\"Close_{asset_id}\",\n                    f\"VWAP_{asset_id}\",\n                    f\"Volume_{asset_id}\",\n                ]\n            )\n\n        raw_columns_to_drop = [\n            col\n            for col in raw_columns_to_drop\n            if col in df.columns\n        ]\n\n        df = df.drop(\n            columns=raw_columns_to_drop\n        )\n\n\n        oldest_use_window = [\n            totimestamp(\"12/01/2019\")\n        ]\n\n        df = df.loc[\n            df[\"timestamp\"]\n            >= oldest_use_window[0]\n        ].copy()\n\n\n    # Ensure generated floating-point features remain float32.\n    float_columns = df.select_dtypes(\n        include=[\"float64\"]\n    ).columns\n\n    if len(float_columns) > 0:\n        df[float_columns] = df[\n            float_columns\n        ].astype(np.float32)\n\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2026-08-05T10:35:23.433440Z","iopub.execute_input":"2026-08-05T10:35:23.434575Z","iopub.status.idle":"2026-08-05T10:35:23.476944Z","shell.execute_reply.started":"2026-08-05T10:35:23.434491Z","shell.execute_reply":"2026-08-05T10:35:23.475852Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\nfeat = get_features(train_merged)\nfeat","metadata":{"execution":{"iopub.status.busy":"2026-08-05T10:35:23.481421Z","iopub.execute_input":"2026-08-05T10:35:23.482201Z","iopub.status.idle":"2026-08-05T10:36:07.396273Z","shell.execute_reply.started":"2026-08-05T10:35:23.482156Z","shell.execute_reply":"2026-08-05T10:36:07.395079Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# Define features for LightGBM\n# ============================================================\n\nnot_use_features_train = [\n    \"timestamp\",\n    \"train_flg\",\n]\n\nfor asset_id in range(14):\n    not_use_features_train.append(\n        f\"Target_{asset_id}\"\n    )\n\n\nfeatures = [\n    col\n    for col in feat.columns\n    if col not in not_use_features_train\n]\n\n\n# ------------------------------------------------------------\n# Safety checks\n# ------------------------------------------------------------\n\nassert \"timestamp\" not in features\nassert \"train_flg\" not in features\n\nassert not any(\n    col.startswith(\"Target_\")\n    for col in features\n)\n\n\nprint(f\"Number of model features: {len(features)}\")\n\ndisplay(features[:50])","metadata":{"execution":{"iopub.status.busy":"2026-08-05T10:36:07.397947Z","iopub.execute_input":"2026-08-05T10:36:07.398232Z","iopub.status.idle":"2026-08-05T10:36:07.409800Z","shell.execute_reply.started":"2026-08-05T10:36:07.398184Z","shell.execute_reply":"2026-08-05T10:36:07.408736Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"del train_merged\ndel df_train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2026-08-05T10:36:07.411187Z","iopub.execute_input":"2026-08-05T10:36:07.411449Z","iopub.status.idle":"2026-08-05T10:36:07.690505Z","shell.execute_reply.started":"2026-08-05T10:36:07.411419Z","shell.execute_reply":"2026-08-05T10:36:07.689513Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"# define the evaluation metric\ndef correlation(a, train_data):\n    \n    b = train_data.get_label()\n    \n    a = np.ravel(a)\n    b = np.ravel(b)\n\n    len_data = len(a)\n    mean_a = np.sum(a) / len_data\n    mean_b = np.sum(b) / len_data\n    var_a = np.sum(np.square(a - mean_a)) / len_data\n    var_b = np.sum(np.square(b - mean_b)) / len_data\n\n    cov = np.sum((a * b))/len_data - mean_a*mean_b\n    corr = cov / np.sqrt(var_a * var_b)\n\n    return 'corr', corr, True\n\n# For CV score calculation\ndef corr_score(pred, valid):\n    len_data = len(pred)\n    mean_pred = np.sum(pred) / len_data\n    mean_valid = np.sum(valid) / len_data\n    var_pred = np.sum(np.square(pred - mean_pred)) / len_data\n    var_valid = np.sum(np.square(valid - mean_valid)) / len_data\n\n    cov = np.sum((pred * valid))/len_data - mean_pred*mean_valid\n    corr = cov / np.sqrt(var_pred * var_valid)\n\n    return corr\n\n# For CV score calculation\ndef wcorr_score(pred, valid, weight):\n    len_data = len(pred)\n    sum_w = np.sum(weight)\n    mean_pred = np.sum(pred * weight) / sum_w\n    mean_valid = np.sum(valid * weight) / sum_w\n    var_pred = np.sum(weight * np.square(pred - mean_pred)) / sum_w\n    var_valid = np.sum(weight * np.square(valid - mean_valid)) / sum_w\n\n    cov = np.sum((pred * valid * weight)) / sum_w - mean_pred*mean_valid\n    corr = cov / np.sqrt(var_pred * var_valid)\n\n    return corr","metadata":{"execution":{"iopub.status.busy":"2026-08-05T10:36:07.692115Z","iopub.execute_input":"2026-08-05T10:36:07.692445Z","iopub.status.idle":"2026-08-05T10:36:07.706714Z","shell.execute_reply.started":"2026-08-05T10:36:07.692399Z","shell.execute_reply":"2026-08-05T10:36:07.705684Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from: https://blog.amedama.jp/entry/lightgbm-cv-feature-importance\n# (used in nyanp's Optiver solution)\ndef plot_importance(importances, features_names = features, PLOT_TOP_N = 20, figsize=(10, 10)):\n    importance_df = pd.DataFrame(data=importances, columns=features)\n    sorted_indices = importance_df.median(axis=0).sort_values(ascending=False).index\n    sorted_importance_df = importance_df.loc[:, sorted_indices]\n    plot_cols = sorted_importance_df.columns[:PLOT_TOP_N]\n    _, ax = plt.subplots(figsize=figsize)\n    ax.grid()\n    ax.set_xscale('log')\n    ax.set_ylabel('Feature')\n    ax.set_xlabel('Importance')\n    sns.boxplot(data=sorted_importance_df[plot_cols],\n                orient='h',\n                ax=ax)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2026-08-05T10:36:07.707894Z","iopub.execute_input":"2026-08-05T10:36:07.708151Z","iopub.status.idle":"2026-08-05T10:36:07.722877Z","shell.execute_reply.started":"2026-08-05T10:36:07.708122Z","shell.execute_reply":"2026-08-05T10:36:07.722006Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from: https://www.kaggle.com/code/nrcjea001/lgbm-embargocv-weightedpearson-lagtarget/\ndef get_time_series_cross_val_splits(data, cv = n_fold, embargo = 3750):\n    all_train_timestamps = data['timestamp'].unique()\n    len_split = len(all_train_timestamps) // cv\n    test_splits = [all_train_timestamps[i * len_split:(i + 1) * len_split] for i in range(cv)]\n    # fix the last test split to have all the last timestamps, in case the number of timestamps wasn't divisible by cv\n    rem = len(all_train_timestamps) - len_split*cv\n    if rem>0:\n        test_splits[-1] = np.append(test_splits[-1], all_train_timestamps[-rem:])\n\n    train_splits = []\n    for test_split in test_splits:\n        test_split_max = int(np.max(test_split))\n        test_split_min = int(np.min(test_split))\n        # get all of the timestamps that aren't in the test split\n        train_split_not_embargoed = [e for e in all_train_timestamps if not (test_split_min <= int(e) <= test_split_max)]\n        # embargo the train split so we have no leakage. Note timestamps are expressed in seconds, so multiply by 60\n        embargo_sec = 60*embargo\n        train_split = [e for e in train_split_not_embargoed if\n                       abs(int(e) - test_split_max) > embargo_sec and abs(int(e) - test_split_min) > embargo_sec]\n        train_splits.append(train_split)\n\n    # convenient way to iterate over train and test splits\n    train_test_zip = zip(train_splits, test_splits)\n    return train_test_zip","metadata":{"execution":{"iopub.status.busy":"2026-08-05T10:36:07.724150Z","iopub.execute_input":"2026-08-05T10:36:07.724671Z","iopub.status.idle":"2026-08-05T10:36:07.739611Z","shell.execute_reply.started":"2026-08-05T10:36:07.724612Z","shell.execute_reply":"2026-08-05T10:36:07.738659Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_Xy_and_model_for_asset(df_proc, asset_id):\n    df_proc = df_proc.loc[  (df_proc[f'Target_{asset_id}'] == df_proc[f'Target_{asset_id}'])  ]\n    if not_use_overlap_to_train:\n        df_proc = df_proc.loc[  (df_proc['train_flg'] == 1)  ]\n    \n# EmbargoCV\n    train_test_zip = get_time_series_cross_val_splits(df_proc, cv = n_fold, embargo = 3750)\n    print(\"entering time series cross validation loop\")\n    importances = []\n    oof_pred = []\n    oof_valid = []\n    \n    for split, train_test_split in enumerate(train_test_zip):\n        gc.collect()\n        \n        print(f\"doing split {split+1} out of {n_fold}\")\n        train_split, test_split = train_test_split\n        train_split_index = df_proc['timestamp'].isin(train_split)\n        test_split_index = df_proc['timestamp'].isin(test_split)\n    \n        train_dataset = lgb.Dataset(df_proc.loc[train_split_index, features],\n                                    df_proc.loc[train_split_index, f'Target_{asset_id}'].values, \n                                    feature_name = features, \n                                   )\n        val_dataset = lgb.Dataset(df_proc.loc[test_split_index, features], \n                                  df_proc.loc[test_split_index, f'Target_{asset_id}'].values, \n                                  feature_name = features, \n                                 )\n\n        print(f\"number of train data: {len(df_proc.loc[train_split_index])}\")\n        print(f\"number of val data:   {len(df_proc.loc[test_split_index])}\")\n\n        model = lgb.train(params = params,\n                          train_set = train_dataset, \n                          valid_sets=[train_dataset, val_dataset],\n                          valid_names=['tr', 'vl'],\n                          num_boost_round = 5000,\n                          verbose_eval = 100,     \n                          feval = correlation,\n                         )\n        importances.append(model.feature_importance(importance_type='gain'))\n        \n        file = f'trained_model_id{asset_id}_fold{split}.pkl'\n        pickle.dump(model, open(file, 'wb'))\n        print(f\"Trained model was saved to 'trained_model_id{asset_id}_fold{split}.pkl'\")\n        print(\"\")\n            \n        oof_pred += list(  model.predict(df_proc.loc[test_split_index, features])        )\n        oof_valid += list(   df_proc.loc[test_split_index, f'Target_{asset_id}'].values    )\n    \n    \n    plot_importance(np.array(importances),features, PLOT_TOP_N = 20, figsize=(10, 5))\n\n    return oof_pred, oof_valid","metadata":{"execution":{"iopub.status.busy":"2026-08-05T10:36:07.740846Z","iopub.execute_input":"2026-08-05T10:36:07.741075Z","iopub.status.idle":"2026-08-05T10:36:07.757099Z","shell.execute_reply.started":"2026-08-05T10:36:07.741047Z","shell.execute_reply":"2026-08-05T10:36:07.756137Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"oof = [ [] for id in range(14)   ]\n\nall_oof_pred = []\nall_oof_valid = []\nall_oof_weight = []\n\nfor asset_id, asset_name in zip(df_asset_details['Asset_ID'], df_asset_details['Asset_Name']):\n    print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n    \n    oof_pred, oof_valid = get_Xy_and_model_for_asset(feat, asset_id)\n    \n    weight_temp = float( df_asset_details.loc[  df_asset_details['Asset_ID'] == asset_id  , 'Weight'   ]  )\n    \n    all_oof_pred += oof_pred\n    all_oof_valid += oof_valid\n    all_oof_weight += [weight_temp] * len(oof_pred)\n    \n    oof[asset_id] = corr_score(     np.array(oof_pred)   ,    np.array(oof_valid)    )\n    \n    print(f'OOF corr score of {asset_name} (ID={asset_id}) is {oof[asset_id]:.5f}. (Weight: {float(weight_temp):.5f})')\n    print('')\n    print('')","metadata":{"execution":{"iopub.status.busy":"2026-08-05T10:36:07.758442Z","iopub.execute_input":"2026-08-05T10:36:07.758803Z","execution_failed":"2026-08-05T11:28:56.650Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ls -lh","metadata":{"trusted":true,"execution":{"execution_failed":"2026-08-05T11:28:56.650Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"woof = 0\nfor id in range(14):\n    woof += oof[id] * float(  df_asset_details.loc[  df_asset_details['Asset_ID'] == id  , 'Weight'   ] )\nwoof = woof / df_asset_details['Weight'].sum()\n\nprint(f'OOF corr scores are;')\nfor oof_score in oof:\n    print(f'      {oof_score:.5f}')\nprint(f'  simple average corr score: {np.mean(oof):.5f}.')\nprint(f'weighted average corr score: {woof:.5f}.')\nprint(f'')\n\nall_oof_wcorr = wcorr_score(     np.array(all_oof_pred),    np.array(all_oof_valid),  np.array(all_oof_weight)   )\nprint(f'        weighted corr score: {all_oof_wcorr:.5f}.')","metadata":{"execution":{"execution_failed":"2026-08-05T11:28:56.650Z"},"trusted":true},"outputs":[],"execution_count":null}]}