{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":30894,"databundleVersionId":3147400,"sourceType":"competition"}],"dockerImageVersionId":30587,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-12-09T13:11:27.992149Z","iopub.execute_input":"2023-12-09T13:11:27.992838Z","iopub.status.idle":"2023-12-09T13:11:28.411961Z","shell.execute_reply.started":"2023-12-09T13:11:27.992799Z","shell.execute_reply":"2023-12-09T13:11:28.410681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport gc\nimport lightgbm as lgb\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import KFold\nn_fold = 7\nseed0 = 8586","metadata":{"execution":{"iopub.status.busy":"2023-12-09T13:11:28.414326Z","iopub.execute_input":"2023-12-09T13:11:28.414986Z","iopub.status.idle":"2023-12-09T13:11:31.467495Z","shell.execute_reply.started":"2023-12-09T13:11:28.414931Z","shell.execute_reply":"2023-12-09T13:11:31.466589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_set = pd.read_csv(r'/kaggle/input/g-research-crypto-forecasting/train.csv')","metadata":{"execution":{"iopub.status.busy":"2023-12-09T13:11:31.468974Z","iopub.execute_input":"2023-12-09T13:11:31.470021Z","iopub.status.idle":"2023-12-09T13:12:37.098537Z","shell.execute_reply.started":"2023-12-09T13:11:31.469975Z","shell.execute_reply":"2023-12-09T13:12:37.097384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assert_id_list = np.sort(train_set['Asset_ID'].drop_duplicates().values)\nclose = pd.pivot(train_set,index = 'timestamp',columns = 'Asset_ID',values = 'Close')\nclose.columns = [f'close_{str(x)}' for x in close.columns]\nvol = pd.pivot(train_set,index = 'timestamp',columns = 'Asset_ID',values = 'Volume')\nvol.columns = [f'vol_{str(x)}' for x in vol.columns]\ntarget = pd.pivot(train_set,index = 'timestamp',columns = 'Asset_ID',values = 'Target')\ntarget.columns = [f'target_{str(x)}' for x in target.columns]","metadata":{"execution":{"iopub.status.busy":"2023-12-09T13:12:37.100268Z","iopub.execute_input":"2023-12-09T13:12:37.100972Z","iopub.status.idle":"2023-12-09T13:13:19.943468Z","shell.execute_reply.started":"2023-12-09T13:12:37.100913Z","shell.execute_reply":"2023-12-09T13:13:19.941167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_set","metadata":{"execution":{"iopub.status.busy":"2023-12-09T13:13:19.949076Z","iopub.execute_input":"2023-12-09T13:13:19.949633Z","iopub.status.idle":"2023-12-09T13:13:19.962415Z","shell.execute_reply.started":"2023-12-09T13:13:19.949583Z","shell.execute_reply":"2023-12-09T13:13:19.961262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calcHullMA(price, N=50):\n    SMA1 = price.rolling(N).mean()\n    SMA2 = price.rolling(int(N/2)).mean()\n    return (2 * SMA2 - SMA1).rolling(int(np.sqrt(N))).mean()","metadata":{"execution":{"iopub.status.busy":"2023-12-09T13:13:19.963993Z","iopub.execute_input":"2023-12-09T13:13:19.964326Z","iopub.status.idle":"2023-12-09T13:13:19.979578Z","shell.execute_reply.started":"2023-12-09T13:13:19.964298Z","shell.execute_reply":"2023-12-09T13:13:19.978461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lags = [15,60,120,480]","metadata":{"execution":{"iopub.status.busy":"2023-12-09T13:13:19.981358Z","iopub.execute_input":"2023-12-09T13:13:19.981879Z","iopub.status.idle":"2023-12-09T13:13:19.989988Z","shell.execute_reply.started":"2023-12-09T13:13:19.981813Z","shell.execute_reply":"2023-12-09T13:13:19.988901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = target\nX = ['close_15_sma','close_15_std','close_15_ret','vol_15_sma','vol_15_std']","metadata":{"execution":{"iopub.status.busy":"2023-12-09T13:13:19.991734Z","iopub.execute_input":"2023-12-09T13:13:19.992493Z","iopub.status.idle":"2023-12-09T13:13:19.999984Z","shell.execute_reply.started":"2023-12-09T13:13:19.992448Z","shell.execute_reply":"2023-12-09T13:13:19.998965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"close = np.log(close)\nvol = np.log(vol)\nfeature_list = [target]\nfor lag in lags:\n    close_sma = close / close.rolling(lag).mean() \n    close_sma.columns = [f'{x}_sma_{lag}' for x in close_sma.columns]\n    close_sma[f'close_sma_{lag}_mean'] = close_sma.mean(axis = 1)\n    \n    close_std = close.rolling(lag).std()/close.rolling(lag).mean()\n    close_std.columns = [f'{x}_std_{lag}' for x in close_std.columns]\n    close_std[f'close_std_{lag}_mean'] = close_std.mean(axis = 1)\n    \n    close_ret = close/close.shift(lag) - 1\n    close_ret.columns = [f'{x}_ret_{lag}' for x in close_ret.columns]\n    close_ret[f'close_ret_{lag}_mean'] = close_ret.mean(axis = 1)    \n    \n    vol_sma = vol/vol.rolling(lag).mean()\n    vol_sma.columns = [f'{x}_sma_{lag}' for x in vol_sma.columns]\n    vol_sma[f'vol_sma_{lag}_mean'] = vol_sma.mean(axis = 1)  \n    \n    \n    vol_std = vol.rolling(lag).std()/vol.rolling(lag).mean()\n    vol_std.columns = [f'{x}_std_{lag}' for x in vol_std.columns]\n    vol_std[f'vol_std_{lag}_mean'] = vol_std.mean(axis = 1)      \n    \n    close_hull = calcHullMA(close)\n    close_hull.columns = [f'{x}_hull_{lag}' for x in close_hull.columns]\n    close_hull[f'close_hull_{lag}_mean'] = close_hull.mean(axis = 1)        \n    \n    vol_hull = calcHullMA(vol)\n    vol_hull.columns = [f'{x}_hull_{lag}' for x in vol_hull.columns]\n    vol_hull[f'vol_hull_{lag}_mean'] = vol_hull.mean(axis = 1)     \n    \n    feature_list.append(close_sma)\n    feature_list.append(close_std)\n    feature_list.append(close_ret)\n    feature_list.append(vol_sma)\n    feature_list.append(vol_std)\n    feature_list.append(close_hull)\n    feature_list.append(vol_hull)\n\n    ","metadata":{"execution":{"iopub.status.busy":"2023-12-09T13:13:20.001877Z","iopub.execute_input":"2023-12-09T13:13:20.002251Z","iopub.status.idle":"2023-12-09T13:14:33.754250Z","shell.execute_reply.started":"2023-12-09T13:13:20.002220Z","shell.execute_reply":"2023-12-09T13:14:33.753254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del vol\ndel close\ndel close_sma\ndel close_std\ndel close_ret\ndel vol_sma\ndel vol_std\ndel close_hull\ndel vol_hull","metadata":{"execution":{"iopub.status.busy":"2023-12-09T13:14:33.755666Z","iopub.execute_input":"2023-12-09T13:14:33.756040Z","iopub.status.idle":"2023-12-09T13:14:33.763400Z","shell.execute_reply.started":"2023-12-09T13:14:33.756009Z","shell.execute_reply":"2023-12-09T13:14:33.762106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_feature = pd.concat(feature_list,axis = 1)\ndf_feature","metadata":{"execution":{"iopub.status.busy":"2023-12-09T13:14:33.765107Z","iopub.execute_input":"2023-12-09T13:14:33.765483Z","iopub.status.idle":"2023-12-09T13:14:38.833725Z","shell.execute_reply.started":"2023-12-09T13:14:33.765415Z","shell.execute_reply":"2023-12-09T13:14:38.832964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del feature_list","metadata":{"execution":{"iopub.status.busy":"2023-12-09T13:14:38.834936Z","iopub.execute_input":"2023-12-09T13:14:38.835573Z","iopub.status.idle":"2023-12-09T13:14:38.860299Z","shell.execute_reply.started":"2023-12-09T13:14:38.835540Z","shell.execute_reply":"2023-12-09T13:14:38.859276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-12-09T13:14:38.861743Z","iopub.execute_input":"2023-12-09T13:14:38.862163Z","iopub.status.idle":"2023-12-09T13:14:39.042876Z","shell.execute_reply.started":"2023-12-09T13:14:38.862114Z","shell.execute_reply":"2023-12-09T13:14:39.041719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_feature = df_feature.reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-12-09T13:14:39.045969Z","iopub.execute_input":"2023-12-09T13:14:39.046444Z","iopub.status.idle":"2023-12-09T13:14:52.829553Z","shell.execute_reply.started":"2023-12-09T13:14:39.046413Z","shell.execute_reply":"2023-12-09T13:14:52.828581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_feature","metadata":{"execution":{"iopub.status.busy":"2023-12-09T13:14:52.830609Z","iopub.execute_input":"2023-12-09T13:14:52.831567Z","iopub.status.idle":"2023-12-09T13:14:53.239015Z","shell.execute_reply.started":"2023-12-09T13:14:52.831534Z","shell.execute_reply":"2023-12-09T13:14:53.237863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pd.read_csv(r'/kaggle/input/g-research-crypto-forecasting/supplemental_train.csv')","metadata":{"execution":{"iopub.status.busy":"2023-12-05T07:20:54.641098Z","iopub.execute_input":"2023-12-05T07:20:54.641565Z","iopub.status.idle":"2023-12-05T07:20:54.647432Z","shell.execute_reply.started":"2023-12-05T07:20:54.641499Z","shell.execute_reply":"2023-12-05T07:20:54.646049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv(r'/kaggle/input/g-research-crypto-forecasting/asset_details.csv')","metadata":{"execution":{"iopub.status.busy":"2023-12-09T13:14:53.240445Z","iopub.execute_input":"2023-12-09T13:14:53.243286Z","iopub.status.idle":"2023-12-09T13:14:53.265242Z","shell.execute_reply.started":"2023-12-09T13:14:53.243236Z","shell.execute_reply":"2023-12-09T13:14:53.264058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train LGBM\n\n# define the evaluation metric\ndef correlation(a, train_data):\n    \n    b = train_data.get_label()\n    \n    a = np.ravel(a)\n    b = np.ravel(b)\n\n    len_data = len(a)\n    mean_a = np.sum(a) / len_data\n    mean_b = np.sum(b) / len_data\n    var_a = np.sum(np.square(a - mean_a)) / len_data\n    var_b = np.sum(np.square(b - mean_b)) / len_data\n\n    cov = np.sum((a * b))/len_data - mean_a*mean_b\n    corr = cov / np.sqrt(var_a * var_b)\n\n    return 'corr', 1-corr, True\n\n# For CV score calculation\ndef corr_score(pred, valid):\n    len_data = len(pred)\n    mean_pred = np.sum(pred) / len_data\n    mean_valid = np.sum(valid) / len_data\n    var_pred = np.sum(np.square(pred - mean_pred)) / len_data\n    var_valid = np.sum(np.square(valid - mean_valid)) / len_data\n\n    cov = np.sum((pred * valid))/len_data - mean_pred*mean_valid\n    corr = cov / np.sqrt(var_pred * var_valid)\n\n    return corr\n\n# For CV score calculation\ndef wcorr_score(pred, valid, weight):\n    len_data = len(pred)\n    sum_w = np.sum(weight)\n    mean_pred = np.sum(pred * weight) / sum_w\n    mean_valid = np.sum(valid * weight) / sum_w\n    var_pred = np.sum(weight * np.square(pred - mean_pred)) / sum_w\n    var_valid = np.sum(weight * np.square(valid - mean_valid)) / sum_w\n\n    cov = np.sum((pred * valid * weight)) / sum_w - mean_pred*mean_valid\n    corr = cov / np.sqrt(var_pred * var_valid)\n\n    return corr","metadata":{"execution":{"iopub.status.busy":"2023-12-09T13:14:53.266590Z","iopub.execute_input":"2023-12-09T13:14:53.267567Z","iopub.status.idle":"2023-12-09T13:14:53.278550Z","shell.execute_reply.started":"2023-12-09T13:14:53.267528Z","shell.execute_reply":"2023-12-09T13:14:53.277212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from: https://blog.amedama.jp/entry/lightgbm-cv-feature-importance\n# (used in nyanp's Optiver solution)\ndef plot_importance(importances, features_names, PLOT_TOP_N = 20, figsize=(10, 10)):\n    importance_df = pd.DataFrame(data=importances, columns=features)\n    sorted_indices = importance_df.median(axis=0).sort_values(ascending=False).index\n    sorted_importance_df = importance_df.loc[:, sorted_indices]\n    plot_cols = sorted_importance_df.columns[:PLOT_TOP_N]\n    _, ax = plt.subplots(figsize=figsize)\n    ax.grid()\n    ax.set_xscale('log')\n    ax.set_ylabel('Feature')\n    ax.set_xlabel('Importance')\n    sns.boxplot(data=sorted_importance_df[plot_cols],\n                orient='h',\n                ax=ax)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-09T13:14:53.280061Z","iopub.execute_input":"2023-12-09T13:14:53.280537Z","iopub.status.idle":"2023-12-09T13:14:53.291166Z","shell.execute_reply.started":"2023-12-09T13:14:53.280501Z","shell.execute_reply":"2023-12-09T13:14:53.290110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # from: https://www.kaggle.com/code/nrcjea001/lgbm-embargocv-weightedpearson-lagtarget/\n# def get_time_series_cross_val_splits(data, cv = n_fold, embargo = 3750):\n#     print(data)\n#     all_train_timestamps = data['timestamp'].unique()\n#     len_split = len(all_train_timestamps) // cv\n#     test_splits = [all_train_timestamps[i * len_split:(i + 1) * len_split] for i in range(cv)]\n#     # fix the last test split to have all the last timestamps, in case the number of timestamps wasn't divisible by cv\n#     rem = len(all_train_timestamps) - len_split*cv\n#     if rem>0:\n#         test_splits[-1] = np.append(test_splits[-1], all_train_timestamps[-rem:])\n\n#     train_splits = []\n#     for test_split in test_splits:\n#         test_split_max = int(np.max(test_split))\n#         test_split_min = int(np.min(test_split))\n#         # get all of the timestamps that aren't in the test split\n#         train_split_not_embargoed = [e for e in all_train_timestamps if not (test_split_min <= int(e) <= test_split_max)]\n#         # embargo the train split so we have no leakage. Note timestamps are expressed in seconds, so multiply by 60\n#         embargo_sec = 60*embargo\n#         train_split = [e for e in train_split_not_embargoed if\n#                        abs(int(e) - test_split_max) > embargo_sec and abs(int(e) - test_split_min) > embargo_sec]\n#         train_splits.append(train_split)\n\n#     # convenient way to iterate over train and test splits\n#     print(train_splits)\n#     train_test_zip = zip(train_splits, test_splits)\n#     return train_test_zip","metadata":{"execution":{"iopub.status.busy":"2023-12-09T13:14:53.292498Z","iopub.execute_input":"2023-12-09T13:14:53.293362Z","iopub.status.idle":"2023-12-09T13:14:53.306344Z","shell.execute_reply.started":"2023-12-09T13:14:53.293298Z","shell.execute_reply":"2023-12-09T13:14:53.305227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {\n    'early_stopping_rounds': 50,\n    'objective': 'regression',\n    'metric': 'rmse',\n#     'metric': 'None',\n    'boosting_type': 'gbdt',\n    'max_depth': 5,\n    'verbose': -1,\n    'max_bin':600,\n    'min_data_in_leaf':50,\n    'learning_rate': 0.03,\n    'subsample': 0.7,\n    'subsample_freq': 1,\n    'feature_fraction': 1,\n    'lambda_l1': 0.5,\n    'lambda_l2': 2,\n    'seed':seed0,\n    'feature_fraction_seed': seed0,\n    'bagging_fraction_seed': seed0,\n    'drop_seed': seed0,\n    'data_random_seed': seed0,\n    'extra_trees': True,\n    'extra_seed': seed0,\n    'zero_as_missing': True,\n    \"first_metric_only\": True\n         }","metadata":{"execution":{"iopub.status.busy":"2023-12-09T13:16:02.367214Z","iopub.execute_input":"2023-12-09T13:16:02.368426Z","iopub.status.idle":"2023-12-09T13:16:02.375808Z","shell.execute_reply.started":"2023-12-09T13:16:02.368368Z","shell.execute_reply":"2023-12-09T13:16:02.374954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_id = 1\nfor asset_id in range(14):\n    print(asset_id)\n    col_list = ['timestamp',f'target_{asset_id}']\n    for lag in lags:\n        col_list.append(f'close_{asset_id}_sma_{lag}')\n        col_list.append(f'close_sma_{lag}_mean')\n        col_list.append(f'close_{asset_id}_std_{lag}')\n        col_list.append(f'close_std_{lag}_mean')\n        col_list.append(f'close_{asset_id}_ret_{lag}')\n        col_list.append(f'close_ret_{lag}_mean')\n        col_list.append(f'vol_{asset_id}_sma_{lag}')\n        col_list.append(f'vol_sma_{lag}_mean')\n        col_list.append(f'vol_{asset_id}_std_{lag}')\n        col_list.append(f'vol_std_{lag}_mean')\n        col_list.append(f'close_{asset_id}_hull_{lag}')\n        col_list.append(f'close_hull_{lag}_mean')\n        col_list.append(f'vol_{asset_id}_hull_{lag}')\n        col_list.append(f'vol_hull_{lag}_mean')\n    \n    df_proc = df_feature[col_list]\n    features = list(df_proc.columns)\n    features.remove('timestamp')\n    features.remove(f'target_{asset_id}')\n    \n    kf = KFold(n_splits=n_fold,shuffle=False)  # 初始化KFold\n    importances = []\n    oof_pred = []\n    oof_valid = []\n    for train_index , test_index in kf.split(df_proc):  # 调用split方法切分数据\n        train_set = df_proc.iloc[train_index]\n        test_set = df_proc.iloc[test_index]\n        train_dataset = lgb.Dataset(train_set[features],\n                                    train_set[f'target_{asset_id}'].values, \n                                    feature_name = features, \n                                   )\n        val_dataset = lgb.Dataset(test_set[features], \n                                  test_set[f'target_{asset_id}'].values, \n                                  feature_name = features, \n                                 )\n\n\n\n        model = lgb.train(params = params,\n                          train_set = train_dataset, \n                          valid_sets=[train_dataset, val_dataset],\n                          valid_names=['tr', 'vl'],\n                          num_boost_round = 5000,\n                          verbose_eval = 100,     \n                          feval = correlation,\n                         )\n        importances.append(model.feature_importance(importance_type='gain'))\n\n    #     file = f'trained_model_id{asset_id}_fold{split}.pkl'\n    #     pickle.dump(model, open(file, 'wb'))\n    #     print(f\"Trained model was saved to 'trained_model_id{asset_id}_fold{split}.pkl'\")\n    #     print(\"\")\n\n    #     oof_pred += list(  model.predict(df_proc.loc[test_split_index, features])        )\n    #     oof_valid += list(   df_proc.loc[test_split_index, f'Target_{asset_id}'].values    )\n\n\n    plot_importance(np.array(importances),features, PLOT_TOP_N = 20, figsize=(10, 5))","metadata":{"execution":{"iopub.status.busy":"2023-12-09T13:16:04.262119Z","iopub.execute_input":"2023-12-09T13:16:04.262507Z"},"trusted":true},"execution_count":null,"outputs":[]}]}