{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Tabular Modelling of Crypto Returns using Close\nThoughts Post EDA: \n1. OHLC are very closely correlated. **Decision Point:** Since signal-to-noise ratio is low, we should probably combine or choose one. Close would make sense since it's the last price in the period, which should contain the most information\n1. Different Cryptos have a relatively low return correlation to each other (+20 to negative), and it changes with time. **Decision Point:** For each prediction, we should try to include the other assets' data in as well.","metadata":{}},{"cell_type":"code","source":"import os\nimport random\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport lightgbm as lgb\nimport gresearch_crypto\nimport time\nimport datetime\n\nimport pickle\nimport gc\n\nfrom tqdm import tqdm\n\nn_fold = 5\nseed0 = 8586\nuse_supple_for_train = True\n\n# If True, the period used to evaluate Public LB will not be used for training.\n# Set to False on final submission.\nnot_use_overlap_to_train = False\n\nTRAIN_CSV = '/kaggle/input/g-research-crypto-forecasting/train.csv'\nSUPPLE_TRAIN_CSV = '/kaggle/input/g-research-crypto-forecasting/supplemental_train.csv'\nASSET_DETAILS_CSV = '/kaggle/input/g-research-crypto-forecasting/asset_details.csv'\n\npd.set_option('display.max_rows', 6)\npd.set_option('display.max_columns', 350)","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:25:09.262681Z","iopub.execute_input":"2023-05-09T00:25:09.262971Z","iopub.status.idle":"2023-05-09T00:25:11.116398Z","shell.execute_reply.started":"2023-05-09T00:25:09.262889Z","shell.execute_reply":"2023-05-09T00:25:11.115858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lags = [60,300,900]","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:25:36.784217Z","iopub.execute_input":"2023-05-09T00:25:36.784497Z","iopub.status.idle":"2023-05-09T00:25:36.788739Z","shell.execute_reply.started":"2023-05-09T00:25:36.784468Z","shell.execute_reply":"2023-05-09T00:25:36.787916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {\n    'early_stopping_rounds': 50,\n    'objective': 'regression',\n    'metric': 'rmse',\n    'boosting_type': 'gbdt',\n    'max_depth': 5,\n    'verbose': -1,\n    'lambda_l1': 0.5,\n    'lambda_l2': 2}","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:25:37.126812Z","iopub.execute_input":"2023-05-09T00:25:37.127109Z","iopub.status.idle":"2023-05-09T00:25:37.133338Z","shell.execute_reply.started":"2023-05-09T00:25:37.127076Z","shell.execute_reply":"2023-05-09T00:25:37.132439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n\n    for col in df.columns:\n        col_type = df[col].dtype\n\n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:25:37.289845Z","iopub.execute_input":"2023-05-09T00:25:37.290093Z","iopub.status.idle":"2023-05-09T00:25:37.301946Z","shell.execute_reply.started":"2023-05-09T00:25:37.290066Z","shell.execute_reply":"2023-05-09T00:25:37.301015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset_details = pd.read_csv(ASSET_DETAILS_CSV).sort_values(\"Asset_ID\")\ndf_asset_details","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:25:37.457248Z","iopub.execute_input":"2023-05-09T00:25:37.457525Z","iopub.status.idle":"2023-05-09T00:25:37.494997Z","shell.execute_reply.started":"2023-05-09T00:25:37.457497Z","shell.execute_reply":"2023-05-09T00:25:37.494120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"%%time\ndf_train = pd.read_csv(TRAIN_CSV, usecols=['timestamp','Asset_ID', 'Close', 'Target'])\n\nif use_supple_for_train:    \n    df_supple = pd.read_csv(SUPPLE_TRAIN_CSV, usecols=['timestamp','Asset_ID', 'Close', 'Target'])\n#     display(df_supple)\n    df_train = pd.concat([df_train, df_supple])\n    del df_supple\ndf_train = reduce_mem_usage(df_train)\n# df_train","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:25:37.901629Z","iopub.execute_input":"2023-05-09T00:25:37.902005Z","iopub.status.idle":"2023-05-09T00:26:47.970807Z","shell.execute_reply.started":"2023-05-09T00:25:37.901973Z","shell.execute_reply":"2023-05-09T00:26:47.969601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"to speed things up let's focus on a few coins\n- BTC (1)\n- ETH (6)\n- Doge (4)","metadata":{}},{"cell_type":"code","source":"assets=[1,6,4]","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:26:47.972126Z","iopub.execute_input":"2023-05-09T00:26:47.972343Z","iopub.status.idle":"2023-05-09T00:26:47.976544Z","shell.execute_reply.started":"2023-05-09T00:26:47.972314Z","shell.execute_reply":"2023-05-09T00:26:47.975831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = df_train[df_train.Asset_ID.isin(assets)]","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:26:47.977593Z","iopub.execute_input":"2023-05-09T00:26:47.977791Z","iopub.status.idle":"2023-05-09T00:26:48.146878Z","shell.execute_reply.started":"2023-05-09T00:26:47.977768Z","shell.execute_reply":"2023-05-09T00:26:48.145945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_merged = pd.DataFrame()\ntrain_merged[df_train.columns] = 0\nfor id in tqdm(assets):\n    train_merged = train_merged.merge(df_train.loc[df_train[\"Asset_ID\"] == id, ['timestamp', 'Close','Target']].copy(), on=\"timestamp\", how='outer',suffixes=['', \"_\"+str(id)])\n        \ntrain_merged = train_merged.drop(df_train.columns.drop(\"timestamp\"), axis=1)\ntrain_merged = train_merged.sort_values('timestamp', ascending=True)\ndisplay(train_merged.head())","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:26:48.148594Z","iopub.execute_input":"2023-05-09T00:26:48.148869Z","iopub.status.idle":"2023-05-09T00:26:50.843237Z","shell.execute_reply.started":"2023-05-09T00:26:48.148843Z","shell.execute_reply":"2023-05-09T00:26:50.842250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# forward fill\n# Set an upper limit on the number of fills, since there may be long term gaps.\nfor id in assets:\n#     print(id, train_merged[f'Close_{id}'].isnull().sum())   # Number of missing before forward fill\n    train_merged[f'Close_{id}'] = train_merged[f'Close_{id}'].fillna(method='ffill', limit=100)\n#     print(id, train_merged[f'Close_{id}'].isnull().sum())   # Number of missing after forward fill","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:26:50.844626Z","iopub.execute_input":"2023-05-09T00:26:50.844905Z","iopub.status.idle":"2023-05-09T00:26:50.867705Z","shell.execute_reply.started":"2023-05-09T00:26:50.844869Z","shell.execute_reply":"2023-05-09T00:26:50.866989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Baseline without FE","metadata":{}},{"cell_type":"code","source":"def get_features(df, train=True):   \n    if train == True:\n        totimestamp = lambda s: np.int32(time.mktime(datetime.datetime.strptime(s, \"%d/%m/%Y\").timetuple()))\n        valid_window = [totimestamp(\"12/03/2021\")]\n#         valid_window = [totimestamp(\"15/08/2021\")]  #検証用\n        df['train_flg'] = np.where(df['timestamp']>=valid_window[0], 0,1)\n\n        supple_start_window = [totimestamp(\"22/09/2021\")]\n        if use_supple_for_train:\n            df['train_flg'] = np.where(df['timestamp']>=supple_start_window[0], 1 ,df['train_flg']  )\n   \n#     for id in range(14):    \n#         for lag in lags:\n#             df[f'log_close/mean_{lag}_id{id}'] = np.log( np.array(df[f'Close_{id}']) /  np.roll(np.append(np.convolve( np.array(df[f'Close_{id}']), np.ones(lag)/lag, mode=\"valid\"), np.ones(lag-1)), lag-1)  )\n#             df[f'log_return_{lag}_id{id}']     = np.log( np.array(df[f'Close_{id}']) /  np.roll(np.array(df[f'Close_{id}']), lag)  )\n#     for lag in lags:\n#         df[f'mean_close/mean_{lag}'] =  np.mean(df.iloc[:,df.columns.str.startswith(f'log_close/mean_{lag}_id')], axis=1)\n#         df[f'mean_log_returns_{lag}'] = np.mean(df.iloc[:,df.columns.str.startswith(f'log_return_{lag}_id')] ,    axis=1)\n#         for id in range(14):\n#             df[f'log_close/mean_{lag}-mean_close/mean_{lag}_id{id}'] = np.array( df[f'log_close/mean_{lag}_id{id}']) - np.array( df[f'mean_close/mean_{lag}']  )\n#             df[f'log_return_{lag}-mean_log_returns_{lag}_id{id}']    = np.array( df[f'log_return_{lag}_id{id}'])     - np.array( df[f'mean_log_returns_{lag}'] )\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:26:50.868680Z","iopub.execute_input":"2023-05-09T00:26:50.869268Z","iopub.status.idle":"2023-05-09T00:26:50.874995Z","shell.execute_reply.started":"2023-05-09T00:26:50.869238Z","shell.execute_reply":"2023-05-09T00:26:50.874245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfeat = get_features(train_merged)\nfeat","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:26:50.875978Z","iopub.execute_input":"2023-05-09T00:26:50.876249Z","iopub.status.idle":"2023-05-09T00:26:50.913044Z","shell.execute_reply.started":"2023-05-09T00:26:50.876218Z","shell.execute_reply":"2023-05-09T00:26:50.912261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define features for LGBM\nnot_use_features_train = ['timestamp', 'train_flg']\nfor id in assets:\n    not_use_features_train.append(f'Target_{id}')\n\nfeatures = feat.columns \nfeatures = features.drop(not_use_features_train)\nfeatures = list(features)\n# display(features)  \nlen(features)","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:26:50.914210Z","iopub.execute_input":"2023-05-09T00:26:50.914458Z","iopub.status.idle":"2023-05-09T00:26:50.921292Z","shell.execute_reply.started":"2023-05-09T00:26:50.914427Z","shell.execute_reply":"2023-05-09T00:26:50.920626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:26:50.922560Z","iopub.execute_input":"2023-05-09T00:26:50.922758Z","iopub.status.idle":"2023-05-09T00:26:50.935919Z","shell.execute_reply.started":"2023-05-09T00:26:50.922726Z","shell.execute_reply":"2023-05-09T00:26:50.935024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_merged\ndel df_train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:26:50.938083Z","iopub.execute_input":"2023-05-09T00:26:50.938295Z","iopub.status.idle":"2023-05-09T00:26:51.062909Z","shell.execute_reply.started":"2023-05-09T00:26:50.938257Z","shell.execute_reply":"2023-05-09T00:26:51.061887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define the evaluation metric\ndef correlation(a, train_data):\n    \n    b = train_data.get_label()\n    \n    a = np.ravel(a)\n    b = np.ravel(b)\n\n    len_data = len(a)\n    mean_a = np.sum(a) / len_data\n    mean_b = np.sum(b) / len_data\n    var_a = np.sum(np.square(a - mean_a)) / len_data\n    var_b = np.sum(np.square(b - mean_b)) / len_data\n\n    cov = np.sum((a * b))/len_data - mean_a*mean_b\n    corr = cov / np.sqrt(var_a * var_b)\n\n    return 'corr', corr, True\n\n# For CV score calculation\ndef corr_score(pred, valid):\n    len_data = len(pred)\n    mean_pred = np.sum(pred) / len_data\n    mean_valid = np.sum(valid) / len_data\n    var_pred = np.sum(np.square(pred - mean_pred)) / len_data\n    var_valid = np.sum(np.square(valid - mean_valid)) / len_data\n\n    cov = np.sum((pred * valid))/len_data - mean_pred*mean_valid\n    corr = cov / np.sqrt(var_pred * var_valid)\n\n    return corr","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:26:51.064077Z","iopub.execute_input":"2023-05-09T00:26:51.064347Z","iopub.status.idle":"2023-05-09T00:26:51.074288Z","shell.execute_reply.started":"2023-05-09T00:26:51.064310Z","shell.execute_reply":"2023-05-09T00:26:51.073125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from: https://blog.amedama.jp/entry/lightgbm-cv-feature-importance\n# (used in nyanp's Optiver solution)\ndef plot_importance(importances, features_names = features, PLOT_TOP_N = 20, figsize=(10, 10)):\n    importance_df = pd.DataFrame(data=importances, columns=features)\n    sorted_indices = importance_df.median(axis=0).sort_values(ascending=False).index\n    sorted_importance_df = importance_df.loc[:, sorted_indices]\n    plot_cols = sorted_importance_df.columns[:PLOT_TOP_N]\n    _, ax = plt.subplots(figsize=figsize)\n    ax.grid()\n    ax.set_xscale('log')\n    ax.set_ylabel('Feature')\n    ax.set_xlabel('Importance')\n    sns.boxplot(data=sorted_importance_df[plot_cols],\n                orient='h',\n                ax=ax)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:26:51.075327Z","iopub.execute_input":"2023-05-09T00:26:51.075589Z","iopub.status.idle":"2023-05-09T00:26:51.088106Z","shell.execute_reply.started":"2023-05-09T00:26:51.075553Z","shell.execute_reply":"2023-05-09T00:26:51.087490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from: https://www.kaggle.com/code/nrcjea001/lgbm-embargocv-weightedpearson-lagtarget/\ndef get_time_series_cross_val_splits(data, cv = n_fold, embargo = 3750):\n    all_train_timestamps = data['timestamp'].unique()\n    len_split = len(all_train_timestamps) // cv\n    test_splits = [all_train_timestamps[i * len_split:(i + 1) * len_split] for i in range(cv)]\n    # fix the last test split to have all the last timestamps, in case the number of timestamps wasn't divisible by cv\n    rem = len(all_train_timestamps) - len_split*cv\n    if rem>0:\n        test_splits[-1] = np.append(test_splits[-1], all_train_timestamps[-rem:])\n\n    train_splits = []\n    for test_split in test_splits:\n        test_split_max = int(np.max(test_split))\n        test_split_min = int(np.min(test_split))\n        # get all of the timestamps that aren't in the test split\n        train_split_not_embargoed = [e for e in all_train_timestamps if not (test_split_min <= int(e) <= test_split_max)]\n        # embargo the train split so we have no leakage. Note timestamps are expressed in seconds, so multiply by 60\n        embargo_sec = 60*embargo\n        train_split = [e for e in train_split_not_embargoed if\n                       abs(int(e) - test_split_max) > embargo_sec and abs(int(e) - test_split_min) > embargo_sec]\n        train_splits.append(train_split)\n\n    # convenient way to iterate over train and test splits\n    train_test_zip = zip(train_splits, test_splits)\n    return train_test_zip","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:26:51.090976Z","iopub.execute_input":"2023-05-09T00:26:51.091212Z","iopub.status.idle":"2023-05-09T00:26:51.100098Z","shell.execute_reply.started":"2023-05-09T00:26:51.091182Z","shell.execute_reply":"2023-05-09T00:26:51.099553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_Xy_and_model_for_asset(df_proc, asset_id):\n    df_proc = df_proc.loc[  (df_proc[f'Target_{asset_id}'] == df_proc[f'Target_{asset_id}'])  ]\n    if not_use_overlap_to_train:\n        df_proc = df_proc.loc[  (df_proc['train_flg'] == 1)  ]\n    \n# EmbargoCV\n    train_test_zip = get_time_series_cross_val_splits(df_proc, cv = n_fold, embargo = 3750)\n    print(\"entering time series cross validation loop\")\n    importances = []\n    oof_pred = []\n    oof_valid = []\n    \n    for split, train_test_split in enumerate(train_test_zip):\n        gc.collect()\n        \n        print(f\"doing split {split+1} out of {n_fold}\")\n        train_split, test_split = train_test_split\n        train_split_index = df_proc['timestamp'].isin(train_split)\n        test_split_index = df_proc['timestamp'].isin(test_split)\n    \n        train_dataset = lgb.Dataset(df_proc.loc[train_split_index, features],\n                                    df_proc.loc[train_split_index, f'Target_{asset_id}'].values, \n                                    feature_name = features, \n                                   )\n        val_dataset = lgb.Dataset(df_proc.loc[test_split_index, features], \n                                  df_proc.loc[test_split_index, f'Target_{asset_id}'].values, \n                                  feature_name = features, \n                                 )\n\n        print(f\"number of train data: {len(df_proc.loc[train_split_index])}\")\n        print(f\"number of val data:   {len(df_proc.loc[test_split_index])}\")\n\n        model = lgb.train(params = params,\n                          train_set = train_dataset, \n                          valid_sets=[train_dataset, val_dataset],\n                          valid_names=['tr', 'vl'],\n                          num_boost_round = 1000,\n                          verbose_eval = 200,     \n                          feval = correlation,\n                         )\n        importances.append(model.feature_importance(importance_type='gain'))\n        \n        print(\"\")\n            \n        oof_pred += list(  model.predict(df_proc.loc[test_split_index, features])        )\n        oof_valid += list(   df_proc.loc[test_split_index, f'Target_{asset_id}'].values    )\n    \n    \n    plot_importance(np.array(importances),features, PLOT_TOP_N = 20, figsize=(10, 5))\n\n    return oof_pred, oof_valid","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:26:51.100951Z","iopub.execute_input":"2023-05-09T00:26:51.101565Z","iopub.status.idle":"2023-05-09T00:26:51.115165Z","shell.execute_reply.started":"2023-05-09T00:26:51.101537Z","shell.execute_reply":"2023-05-09T00:26:51.114670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof = {x:0 for x in assets}\n\nall_oof_pred = []\nall_oof_valid = []\n\nfor asset_id in assets:\n    print(f\"Training model for (ID={asset_id:<2})\")\n    \n    oof_pred, oof_valid = get_Xy_and_model_for_asset(feat, asset_id)\n    \n    all_oof_pred += oof_pred\n    all_oof_valid += oof_valid\n    \n    oof[asset_id] = corr_score(     np.array(oof_pred)   ,    np.array(oof_valid)    )\n    \n    print(f'OOF corr score of (ID={asset_id}) is {oof[asset_id]:.5f}.')\n    print('')\n    print('')","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:26:51.115938Z","iopub.execute_input":"2023-05-09T00:26:51.116224Z","iopub.status.idle":"2023-05-09T00:29:04.767095Z","shell.execute_reply.started":"2023-05-09T00:26:51.116194Z","shell.execute_reply":"2023-05-09T00:29:04.766280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"avg_values = sum(oof.values()) / len(oof)\nprint(f'  simple average corr score: {avg_values:.5f}.')","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:32:48.753047Z","iopub.execute_input":"2023-05-09T00:32:48.753384Z","iopub.status.idle":"2023-05-09T00:32:48.759254Z","shell.execute_reply.started":"2023-05-09T00:32:48.753335Z","shell.execute_reply":"2023-05-09T00:32:48.758138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering I\nSo far we only added 'Close', would adding the other columns help?\n- count, volume\n- exclude similar columns (OHL, VWAP)","metadata":{}},{"cell_type":"code","source":"%%time\ndf_train = pd.read_csv(TRAIN_CSV)\n\nif use_supple_for_train:    \n    df_supple = pd.read_csv(SUPPLE_TRAIN_CSV)\n#     display(df_supple)\n    df_train = pd.concat([df_train, df_supple])\n    del df_supple\ndf_train = reduce_mem_usage(df_train)\n# df_train\n\ndf_train = df_train[df_train.Asset_ID.isin(assets)]","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:32:50.563310Z","iopub.execute_input":"2023-05-09T00:32:50.564022Z","iopub.status.idle":"2023-05-09T00:33:28.217072Z","shell.execute_reply.started":"2023-05-09T00:32:50.563971Z","shell.execute_reply":"2023-05-09T00:33:28.215885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.columns","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:33:28.218591Z","iopub.execute_input":"2023-05-09T00:33:28.218786Z","iopub.status.idle":"2023-05-09T00:33:28.225514Z","shell.execute_reply.started":"2023-05-09T00:33:28.218761Z","shell.execute_reply":"2023-05-09T00:33:28.224412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[['Close','VWAP','Count','Volume']].corr()","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:33:28.226658Z","iopub.execute_input":"2023-05-09T00:33:28.226874Z","iopub.status.idle":"2023-05-09T00:33:28.519829Z","shell.execute_reply.started":"2023-05-09T00:33:28.226844Z","shell.execute_reply":"2023-05-09T00:33:28.518914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_merged = pd.DataFrame()\ntrain_merged[df_train.columns] = 0\nfor id in tqdm(assets):\n    train_merged = train_merged.merge(df_train.loc[df_train[\"Asset_ID\"] == id, ['timestamp', 'Close','Target','Volume','Count']].copy(), on=\"timestamp\", how='outer',suffixes=['', \"_\"+str(id)])\n        \ntrain_merged = train_merged.drop(df_train.columns.drop(\"timestamp\"), axis=1)\ntrain_merged = train_merged.sort_values('timestamp', ascending=True)\ndisplay(train_merged.head())","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:33:28.521964Z","iopub.execute_input":"2023-05-09T00:33:28.522203Z","iopub.status.idle":"2023-05-09T00:33:31.644805Z","shell.execute_reply.started":"2023-05-09T00:33:28.522174Z","shell.execute_reply":"2023-05-09T00:33:31.643845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# forward fill\n# Set an upper limit on the number of fills, since there may be long term gaps.\nfor id in assets:\n#     print(id, train_merged[f'Close_{id}'].isnull().sum())   # Number of missing before forward fill\n    train_merged[[f'Close_{id}',f'Volume_{id}',f'Count_{id}']] = train_merged[[f'Close_{id}',f'Volume_{id}',f'Count_{id}']].fillna(method='ffill', limit=100)\n#     print(id, train_merged[f'Close_{id}'].isnull().sum())   # Number of missing after forward fill","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:33:31.646115Z","iopub.execute_input":"2023-05-09T00:33:31.646404Z","iopub.status.idle":"2023-05-09T00:33:31.710712Z","shell.execute_reply.started":"2023-05-09T00:33:31.646339Z","shell.execute_reply":"2023-05-09T00:33:31.709847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfeat = get_features(train_merged)\nfeat","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:33:31.711821Z","iopub.execute_input":"2023-05-09T00:33:31.712265Z","iopub.status.idle":"2023-05-09T00:33:31.744430Z","shell.execute_reply.started":"2023-05-09T00:33:31.712236Z","shell.execute_reply":"2023-05-09T00:33:31.743782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define features for LGBM\nnot_use_features_train = ['timestamp', 'train_flg']\nfor id in assets:\n    not_use_features_train.append(f'Target_{id}')\n\nfeatures = feat.columns \nfeatures = features.drop(not_use_features_train)\nfeatures = list(features)\n# display(features)  \nlen(features)","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:33:31.745454Z","iopub.execute_input":"2023-05-09T00:33:31.745838Z","iopub.status.idle":"2023-05-09T00:33:31.755428Z","shell.execute_reply.started":"2023-05-09T00:33:31.745805Z","shell.execute_reply":"2023-05-09T00:33:31.754773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:33:31.756781Z","iopub.execute_input":"2023-05-09T00:33:31.757115Z","iopub.status.idle":"2023-05-09T00:33:31.766237Z","shell.execute_reply.started":"2023-05-09T00:33:31.757077Z","shell.execute_reply":"2023-05-09T00:33:31.765489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_merged\ndel df_train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:33:31.767641Z","iopub.execute_input":"2023-05-09T00:33:31.767912Z","iopub.status.idle":"2023-05-09T00:33:32.060742Z","shell.execute_reply.started":"2023-05-09T00:33:31.767879Z","shell.execute_reply":"2023-05-09T00:33:32.059686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training I","metadata":{}},{"cell_type":"code","source":"oof = {x:0 for x in assets}\n\nall_oof_pred = []\nall_oof_valid = []\n\nfor asset_id in assets:\n    print(f\"Training model for (ID={asset_id:<2})\")\n    \n    oof_pred, oof_valid = get_Xy_and_model_for_asset(feat, asset_id)\n    \n    all_oof_pred += oof_pred\n    all_oof_valid += oof_valid\n    \n    oof[asset_id] = corr_score(     np.array(oof_pred)   ,    np.array(oof_valid)    )\n    \n    print(f'OOF corr score of (ID={asset_id}) is {oof[asset_id]:.5f}.')\n    print('')\n    print('')","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:33:32.063542Z","iopub.execute_input":"2023-05-09T00:33:32.063745Z","iopub.status.idle":"2023-05-09T00:38:50.293019Z","shell.execute_reply.started":"2023-05-09T00:33:32.063721Z","shell.execute_reply":"2023-05-09T00:38:50.291409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"corr is up 10X","metadata":{}},{"cell_type":"code","source":"avg_values = sum(oof.values()) / len(oof)\nprint(f'  simple average corr score: {avg_values:.5f}.')","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:38:50.295657Z","iopub.execute_input":"2023-05-09T00:38:50.296509Z","iopub.status.idle":"2023-05-09T00:38:50.302380Z","shell.execute_reply.started":"2023-05-09T00:38:50.296469Z","shell.execute_reply":"2023-05-09T00:38:50.301322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering II\n- the best predictor of next periods returns is actually this periods returns. can we add in past returns?","metadata":{}},{"cell_type":"code","source":"%%time\ndf_train = pd.read_csv(TRAIN_CSV)\n\nif use_supple_for_train:    \n    df_supple = pd.read_csv(SUPPLE_TRAIN_CSV)\n#     display(df_supple)\n    df_train = pd.concat([df_train, df_supple])\n    del df_supple\ndf_train = reduce_mem_usage(df_train)\n# df_train\n\ndf_train = df_train[df_train.Asset_ID.isin(assets)]","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:38:50.303694Z","iopub.execute_input":"2023-05-09T00:38:50.303894Z","iopub.status.idle":"2023-05-09T00:39:17.095506Z","shell.execute_reply.started":"2023-05-09T00:38:50.303868Z","shell.execute_reply":"2023-05-09T00:39:17.094426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_merged = pd.DataFrame()\ntrain_merged[df_train.columns] = 0\nfor id in tqdm(assets):\n    train_merged = train_merged.merge(df_train.loc[df_train[\"Asset_ID\"] == id, ['timestamp', 'Close','Target','Volume','Count']].copy(), on=\"timestamp\", how='outer',suffixes=['', \"_\"+str(id)])\n        \ntrain_merged = train_merged.drop(df_train.columns.drop(\"timestamp\"), axis=1)\ntrain_merged = train_merged.sort_values('timestamp', ascending=True)\ndisplay(train_merged.head())","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:39:17.097754Z","iopub.execute_input":"2023-05-09T00:39:17.098010Z","iopub.status.idle":"2023-05-09T00:39:20.407651Z","shell.execute_reply.started":"2023-05-09T00:39:17.097976Z","shell.execute_reply":"2023-05-09T00:39:20.406915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_features(df, train=True):   \n    if train == True:\n        totimestamp = lambda s: np.int32(time.mktime(datetime.datetime.strptime(s, \"%d/%m/%Y\").timetuple()))\n        valid_window = [totimestamp(\"12/03/2021\")]\n#         valid_window = [totimestamp(\"15/08/2021\")]  #検証用\n        df['train_flg'] = np.where(df['timestamp']>=valid_window[0], 0,1)\n\n        supple_start_window = [totimestamp(\"22/09/2021\")]\n        if use_supple_for_train:\n            df['train_flg'] = np.where(df['timestamp']>=supple_start_window[0], 1 ,df['train_flg']  )\n   \n    for id in assets:    \n        for lag in lags:\n            df[f'log_close/mean_{lag}_id{id}'] = np.log( np.array(df[f'Close_{id}']) /  np.roll(np.append(np.convolve( np.array(df[f'Close_{id}']), np.ones(lag)/lag, mode=\"valid\"), np.ones(lag-1)), lag-1)  )\n            df[f'log_return_{lag}_id{id}']     = np.log( np.array(df[f'Close_{id}']) /  np.roll(np.array(df[f'Close_{id}']), lag)  )\n    for lag in lags:\n        df[f'mean_close/mean_{lag}'] =  np.mean(df.iloc[:,df.columns.str.startswith(f'log_close/mean_{lag}_id')], axis=1)\n        df[f'mean_log_returns_{lag}'] = np.mean(df.iloc[:,df.columns.str.startswith(f'log_return_{lag}_id')] ,    axis=1)\n        for id in assets:\n            df[f'log_close/mean_{lag}-mean_close/mean_{lag}_id{id}'] = np.array( df[f'log_close/mean_{lag}_id{id}']) - np.array( df[f'mean_close/mean_{lag}']  )\n            df[f'log_return_{lag}-mean_log_returns_{lag}_id{id}']    = np.array( df[f'log_return_{lag}_id{id}'])     - np.array( df[f'mean_log_returns_{lag}'] )\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:39:20.408813Z","iopub.execute_input":"2023-05-09T00:39:20.409001Z","iopub.status.idle":"2023-05-09T00:39:20.420480Z","shell.execute_reply.started":"2023-05-09T00:39:20.408977Z","shell.execute_reply":"2023-05-09T00:39:20.419612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfeat = get_features(train_merged)\nfeat","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:39:20.422233Z","iopub.execute_input":"2023-05-09T00:39:20.422650Z","iopub.status.idle":"2023-05-09T00:39:22.854062Z","shell.execute_reply.started":"2023-05-09T00:39:20.422577Z","shell.execute_reply":"2023-05-09T00:39:22.853123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define features for LGBM\nnot_use_features_train = ['timestamp', 'train_flg']\nfor id in assets:\n    not_use_features_train.append(f'Target_{id}')\n\nfeatures = feat.columns \nfeatures = features.drop(not_use_features_train)\nfeatures = list(features)\n# display(features)  \nlen(features)","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:39:22.855737Z","iopub.execute_input":"2023-05-09T00:39:22.855958Z","iopub.status.idle":"2023-05-09T00:39:22.862615Z","shell.execute_reply.started":"2023-05-09T00:39:22.855933Z","shell.execute_reply":"2023-05-09T00:39:22.861899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df_train\ndel train_merged\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:39:22.863520Z","iopub.execute_input":"2023-05-09T00:39:22.863732Z","iopub.status.idle":"2023-05-09T00:39:23.163216Z","shell.execute_reply.started":"2023-05-09T00:39:22.863704Z","shell.execute_reply":"2023-05-09T00:39:23.162121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training II","metadata":{}},{"cell_type":"code","source":"oof = {x:0 for x in assets}\n\nall_oof_pred = []\nall_oof_valid = []\n\nfor asset_id in assets:\n    print(f\"Training model for (ID={asset_id:<2})\")\n    \n    oof_pred, oof_valid = get_Xy_and_model_for_asset(feat, asset_id)\n    \n    all_oof_pred += oof_pred\n    all_oof_valid += oof_valid\n    \n    oof[asset_id] = corr_score(     np.array(oof_pred)   ,    np.array(oof_valid)    )\n    \n    print(f'OOF corr score of (ID={asset_id}) is {oof[asset_id]:.5f}.')\n    print('')\n    print('')","metadata":{"execution":{"iopub.status.busy":"2023-05-09T00:39:23.164445Z","iopub.execute_input":"2023-05-09T00:39:23.164670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"corr is up 3X from previous best, 30X from first try with only Close","metadata":{}},{"cell_type":"code","source":"avg_values = sum(oof.values()) / len(oof)\nprint(f'  simple average corr score: {avg_values:.5f}.')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering III\n- can we improve performance by removing the original Close features? since the derivatives prob capture a fair amount of information from Close itself","metadata":{}},{"cell_type":"code","source":"%%time\ndf_train = pd.read_csv(TRAIN_CSV)\n\nif use_supple_for_train:    \n    df_supple = pd.read_csv(SUPPLE_TRAIN_CSV)\n#     display(df_supple)\n    df_train = pd.concat([df_train, df_supple])\n    del df_supple\ndf_train = reduce_mem_usage(df_train)\n# df_train\n\ndf_train = df_train[df_train.Asset_ID.isin(assets)]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_merged = pd.DataFrame()\ntrain_merged[df_train.columns] = 0\nfor id in tqdm(assets):\n    train_merged = train_merged.merge(df_train.loc[df_train[\"Asset_ID\"] == id, ['timestamp', 'Close','Target','Volume','Count']].copy(), on=\"timestamp\", how='outer',suffixes=['', \"_\"+str(id)])\n        \ntrain_merged = train_merged.drop(df_train.columns.drop(\"timestamp\"), axis=1)\ntrain_merged = train_merged.sort_values('timestamp', ascending=True)\ndisplay(train_merged.head())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_features(df, train=True):   \n    if train == True:\n        totimestamp = lambda s: np.int32(time.mktime(datetime.datetime.strptime(s, \"%d/%m/%Y\").timetuple()))\n        valid_window = [totimestamp(\"12/03/2021\")]\n#         valid_window = [totimestamp(\"15/08/2021\")]  #検証用\n        df['train_flg'] = np.where(df['timestamp']>=valid_window[0], 0,1)\n\n        supple_start_window = [totimestamp(\"22/09/2021\")]\n        if use_supple_for_train:\n            df['train_flg'] = np.where(df['timestamp']>=supple_start_window[0], 1 ,df['train_flg']  )\n   \n    for id in assets:    \n        for lag in lags:\n            df[f'log_close/mean_{lag}_id{id}'] = np.log( np.array(df[f'Close_{id}']) /  np.roll(np.append(np.convolve( np.array(df[f'Close_{id}']), np.ones(lag)/lag, mode=\"valid\"), np.ones(lag-1)), lag-1)  )\n            df[f'log_return_{lag}_id{id}']     = np.log( np.array(df[f'Close_{id}']) /  np.roll(np.array(df[f'Close_{id}']), lag)  )\n    for lag in lags:\n        df[f'mean_close/mean_{lag}'] =  np.mean(df.iloc[:,df.columns.str.startswith(f'log_close/mean_{lag}_id')], axis=1)\n        df[f'mean_log_returns_{lag}'] = np.mean(df.iloc[:,df.columns.str.startswith(f'log_return_{lag}_id')] ,    axis=1)\n        for id in assets:\n            df[f'log_close/mean_{lag}-mean_close/mean_{lag}_id{id}'] = np.array( df[f'log_close/mean_{lag}_id{id}']) - np.array( df[f'mean_close/mean_{lag}']  )\n            df[f'log_return_{lag}-mean_log_returns_{lag}_id{id}']    = np.array( df[f'log_return_{lag}_id{id}'])     - np.array( df[f'mean_log_returns_{lag}'] )\n\n    for id in assets:\n        df.drop(f'Close_{id}',axis=1,inplace=True)\n    return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfeat = get_features(train_merged)\nfeat","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define features for LGBM\nnot_use_features_train = ['timestamp', 'train_flg']\nfor id in assets:\n    not_use_features_train.append(f'Target_{id}')\n\nfeatures = feat.columns \nfeatures = features.drop(not_use_features_train)\nfeatures = list(features)\n# display(features)  \nlen(features)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df_train\ndel train_merged\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training III","metadata":{}},{"cell_type":"code","source":"oof = {x:0 for x in assets}\n\nall_oof_pred = []\nall_oof_valid = []\n\nfor asset_id in assets:\n    print(f\"Training model for (ID={asset_id:<2})\")\n    \n    oof_pred, oof_valid = get_Xy_and_model_for_asset(feat, asset_id)\n    \n    all_oof_pred += oof_pred\n    all_oof_valid += oof_valid\n    \n    oof[asset_id] = corr_score(     np.array(oof_pred)   ,    np.array(oof_valid)    )\n    \n    print(f'OOF corr score of (ID={asset_id}) is {oof[asset_id]:.5f}.')\n    print('')\n    print('')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"avg_values = sum(oof.values()) / len(oof)\nprint(f'  simple average corr score: {avg_values:.5f}.')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}