{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Readme\n\nThis notebook is based on [RICHMANBTC's notebook](https://www.kaggle.com/richmanbtc/20211103-gresearch-crypto-v1) and [KATSU1110's notebook](https://www.kaggle.com/code1110/gresearch-simple-lgb-starter).\n\nThis notebook's point is this.\n\n - Exclude large target. (I bet black swan will not happen in this 3 month 😀)\n \n - I use little base feature like upper shadow, and use some rolling features. Rolling feature is mainly from richmanbtc's notebook.\n\nGood luck for this 3 month!\n\n---\n\n## Code\n","metadata":{}},{"cell_type":"code","source":"import lzma\nimport pickle\nimport cloudpickle\nimport numpy as np\nimport pandas as pd\nimport lightgbm as lgb\nimport datatable as dt\nimport gresearch_crypto\nfrom sklearn.linear_model import Ridge\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import KBinsDiscretizer, MinMaxScaler\nfrom sklearn.model_selection import cross_validate, cross_val_predict","metadata":{"_uuid":"4087e606-cec8-4c18-9de3-54dc0827b16d","_cell_guid":"20304d7b-e26d-41a6-9e43-1ec2aebf6de1","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-18T15:00:11.813763Z","iopub.execute_input":"2022-01-18T15:00:11.81427Z","iopub.status.idle":"2022-01-18T15:00:13.890741Z","shell.execute_reply.started":"2022-01-18T15:00:11.814163Z","shell.execute_reply":"2022-01-18T15:00:13.889881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"flag_strict = False\nflag_cv = True","metadata":{"execution":{"iopub.status.busy":"2022-01-18T15:00:13.89201Z","iopub.execute_input":"2022-01-18T15:00:13.892226Z","iopub.status.idle":"2022-01-18T15:00:13.898781Z","shell.execute_reply.started":"2022-01-18T15:00:13.892199Z","shell.execute_reply":"2022-01-18T15:00:13.897922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if flag_cv:\n    import matplotlib.pyplot as plt\n    from scipy.stats import pearsonr\n    from sklearn.metrics import r2_score\n    from sklearn.model_selection import cross_val_predict\n","metadata":{"execution":{"iopub.status.busy":"2022-01-18T15:00:13.899815Z","iopub.execute_input":"2022-01-18T15:00:13.900028Z","iopub.status.idle":"2022-01-18T15:00:13.907452Z","shell.execute_reply.started":"2022-01-18T15:00:13.900001Z","shell.execute_reply":"2022-01-18T15:00:13.906583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2022-01-18T15:00:13.909712Z","iopub.execute_input":"2022-01-18T15:00:13.910309Z","iopub.status.idle":"2022-01-18T15:00:13.924082Z","shell.execute_reply.started":"2022-01-18T15:00:13.910264Z","shell.execute_reply":"2022-01-18T15:00:13.923103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_feature_columns(df):\n    features = df.columns[df.columns.str.startswith('feature')]\n    return sorted(list(features))\n\ndef save_model(model, path):\n    data = cloudpickle.dumps(model)\n    data = lzma.compress(data)\n    with open(path, 'wb') as f:\n        f.write(data)\n        \ndef process_data(df, df_asset, is_train=False):\n    df = df.copy()\n    df['timestamp'] = pd.to_datetime(df['timestamp'], unit='s', utc=True)\n    \n    # chnge point: 1\n    if is_train:\n        df = df[df['timestamp'] < '2021-06-13 00:00:00']\n\n    df = df.rename(columns={\n        'Asset_ID': 'market',\n        'Close': 'cl',\n        'Target': 'target',\n    })\n    df = df.join(df_asset[['weight']], on='market', how='left')\n    df = df.set_index(['timestamp', 'market'])\n    return df\n\ndef sort_and_remove_duplicates(df):\n    df = df.sort_index(kind='mergesort')\n    df = df.loc[~df.index.duplicated(keep='last')]\n    return df\n\ndef my_purge_kfold(n, n_splits=5, purge=3750 * 14):\n    idx = np.arange(n)\n    cv = []\n    for i in range(n_splits):\n        val_start = i * n // n_splits\n        val_end = (i + 1) * n // n_splits\n        val_idx = idx[val_start:val_end]\n        train_idx = idx[(idx < val_start - purge) | (val_end + purge <= idx)]\n        cv.append((\n            train_idx,\n            val_idx,\n        ))\n    return cv","metadata":{"_uuid":"905db8ff-3f55-4d3f-abcd-c46b6902a74f","_cell_guid":"a230bc13-c30d-4786-8c69-e23e86f5c321","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-18T15:00:13.925385Z","iopub.execute_input":"2022-01-18T15:00:13.92568Z","iopub.status.idle":"2022-01-18T15:00:13.9397Z","shell.execute_reply.started":"2022-01-18T15:00:13.925643Z","shell.execute_reply":"2022-01-18T15:00:13.939012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# calc features\ndef calc_features(df):\n    df = df.copy()\n    df['ln_cl'] = np.log(df['cl'])\n\n    # shift is faster than diff\n    df['feature_cl_diff1'] = df['ln_cl'] - df.groupby('market')['ln_cl'].shift(15)\n    df['raw_return_causal'] = df['ln_cl'] - df.groupby('market')['ln_cl'].shift(15)\n    \n    inv_weight_sum = 1.0 / df.groupby('timestamp')['weight'].transform('sum')\n    df['market_return_causal'] = (df['raw_return_causal'] * df['weight']).groupby('timestamp').transform('sum') * inv_weight_sum\n\n    df['beta_causal'] = (\n        (df['raw_return_causal'] * df['market_return_causal']).groupby('market').transform(lambda x: x.rolling(3750, 1).mean())\n        / (df['market_return_causal'] ** 2).groupby('market').transform(lambda x: x.rolling(3750, 1).mean())\n    )\n    \n    df['feature_cl_diff1_mean_simple'] = df['feature_cl_diff1'].groupby('timestamp').transform('mean')\n    df['feature_cl_diff1_mean_weight'] = (df['feature_cl_diff1'] * df['weight']).groupby('timestamp').transform('sum') * inv_weight_sum\n    df['feature_cl_diff1_resid'] = df['feature_cl_diff1'] - df['beta_causal'] * df['feature_cl_diff1_mean_weight']\n    df['feature_cl_diff1_rank'] = df.groupby('timestamp')['feature_cl_diff1'].transform('rank')\n    \n    # added katsu\n    df['feature_upper_shadow'] = df['High'] / df[['cl', 'Open']].max(axis=1)\n    df['feature_lower_shadow'] = df[['cl', 'Open']].min(axis=1) / df['Low']\n    df['feature_volume2count'] = df['Volume'] / (df['Count'] + 1)\n    df.drop(['Open', 'High', 'Low', 'Volume', 'Count'], axis=1, inplace=True)\n    \n    # RSI, MACD\n    # df[\"feature_RSI\"] = RSI(df[\"cl\"], 1)\n    # macd, macd_signal = MACD(df[\"cl\"], 30, 15, 5) \n    # df[\"feature_MACD\"] = macd\n    # df[\"feature_MACD_signal\"] = macd_signal\n    # del macd, macd_signal\n    # gc.collect()\n    \n    df = df.rename(columns={\n        'beta_causal': 'feature_beta_causal',\n    })    \n    return df","metadata":{"_uuid":"218694fc-9642-4cf5-8817-b85f31fa6c03","_cell_guid":"155bab3b-5df7-4984-a5fd-b8443648f09f","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-18T15:00:13.956647Z","iopub.execute_input":"2022-01-18T15:00:13.95686Z","iopub.status.idle":"2022-01-18T15:00:13.972752Z","shell.execute_reply.started":"2022-01-18T15:00:13.956834Z","shell.execute_reply":"2022-01-18T15:00:13.972186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n# preprocess asset data\ndf_asset = dt.fread('../input/g-research-crypto-forecasting/asset_details.csv').to_pandas()\ndf_asset = df_asset.rename(columns={\n    'Asset_ID': 'market',\n    'Weight': 'weight',\n    'Asset_Name': 'name',\n})\ndf_asset = df_asset.set_index('market')\ndf_asset = df_asset.sort_values('market')\n\n# preprocess train data\ndf = pd.concat([\n    dt.fread('../input/g-research-crypto-forecasting/train.csv').to_pandas(),\n    dt.fread('../input/g-research-crypto-forecasting/supplemental_train.csv').to_pandas(),\n])\n\nif flag_strict:\n    df = process_data(df, df_asset, is_train=True)\nelse:\n    df = process_data(df, df_asset)\n\ndf = reduce_mem_usage(df)\ndf = sort_and_remove_duplicates(df)\ndf.to_pickle('/tmp/df.pkl')","metadata":{"_uuid":"fa6e0de9-fe6b-48af-8188-6925f63aa493","_cell_guid":"54064053-9364-4863-bb24-96bca89874bc","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-18T15:00:13.974826Z","iopub.execute_input":"2022-01-18T15:00:13.975585Z","iopub.status.idle":"2022-01-18T15:03:23.826676Z","shell.execute_reply.started":"2022-01-18T15:00:13.975549Z","shell.execute_reply":"2022-01-18T15:03:23.825692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n# calc features\ndf = calc_features(df)\ndf = reduce_mem_usage(df)\nfeatures = get_feature_columns(df)\ndf = df[features + ['target', 'weight']]\ndf = df.dropna()","metadata":{"_uuid":"e0c11ea1-f441-4057-a0a5-1c446c97ceef","_cell_guid":"352cb4f6-9d2d-45bb-8f25-b0399258416b","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-18T15:03:23.828109Z","iopub.execute_input":"2022-01-18T15:03:23.828346Z","iopub.status.idle":"2022-01-18T15:04:25.617424Z","shell.execute_reply.started":"2022-01-18T15:03:23.828318Z","shell.execute_reply":"2022-01-18T15:04:25.616305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Exclude 2021/10/29 data. It has very large target values.\ns_str = '2021-10-29 08:49:00'\ne_str = '2021-10-29 11:42:00'\ndropped_inds = df.loc[\n    (df.index.get_level_values(0) > s_str) & \\\n    (df.index.get_level_values(0) < e_str) & \\\n    ((df.index.get_level_values(1) == 7) | \\\n     (df.index.get_level_values(1) == 12) | \\\n     (df.index.get_level_values(1) == 13)), :].index\ndf.drop(dropped_inds, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-18T15:04:25.619651Z","iopub.execute_input":"2022-01-18T15:04:25.619973Z","iopub.status.idle":"2022-01-18T15:08:04.50941Z","shell.execute_reply.started":"2022-01-18T15:04:25.619941Z","shell.execute_reply":"2022-01-18T15:08:04.508412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# In addition, we exclude abs(target) > 0.5!\ndf = df.query(\"abs(target) < 0.5\")","metadata":{"execution":{"iopub.status.busy":"2022-01-18T15:08:14.751182Z","iopub.execute_input":"2022-01-18T15:08:14.751481Z","iopub.status.idle":"2022-01-18T15:08:20.04867Z","shell.execute_reply.started":"2022-01-18T15:08:14.751445Z","shell.execute_reply":"2022-01-18T15:08:20.047574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cv = my_purge_kfold(df.shape[0])\nx = df[features]\ny = df[\"target\"]\ndf[\"y_pred\"] = 0\n\n# Tuned by Optuna\nparams_list = [\n    {\"seed\": 0, 'objective': 'regression', 'metric': 'rmse', 'feature_pre_filter': False, 'lambda_l1': 5.338008670144408, 'lambda_l2': 0.10892103459556557, 'num_leaves': 12, 'feature_fraction': 0.8, 'bagging_fraction': 0.8229227798972512, 'bagging_freq': 7, 'min_child_samples': 20, 'num_iterations': 100, 'early_stopping_round': 50},\n    {\"seed\": 0, 'objective': 'regression', 'metric': 'rmse', 'feature_pre_filter': False, 'lambda_l1': 6.000232331039988, 'lambda_l2': 0.0007928024088528338, 'num_leaves': 31, 'feature_fraction': 0.7200000000000001, 'bagging_fraction': 0.758706201323496, 'bagging_freq': 7, 'min_child_samples': 25, 'num_iterations': 100, 'early_stopping_round': 50},\n    {\"seed\": 0, 'objective': 'regression', 'metric': 'rmse', 'feature_pre_filter': False, 'lambda_l1': 7.417330368307966e-05, 'lambda_l2': 6.905607612577662e-07, 'num_leaves': 159, 'feature_fraction': 0.8, 'bagging_fraction': 1.0, 'bagging_freq': 0, 'min_child_samples': 20, 'num_iterations': 100, 'early_stopping_round': 50},\n    {\"seed\": 0, 'objective': 'regression', 'metric': 'rmse', 'feature_pre_filter': False, 'lambda_l1': 8.151302095905674, 'lambda_l2': 0.10316839127701032, 'num_leaves': 8, 'feature_fraction': 0.4, 'bagging_fraction': 0.6875441454005267, 'bagging_freq': 4, 'min_child_samples': 20, 'num_iterations': 100, 'early_stopping_round': 50},\n    {\"seed\": 0, 'objective': 'regression', 'metric': 'rmse', 'feature_pre_filter': False, 'lambda_l1': 0.0, 'lambda_l2': 0.0, 'num_leaves': 45, 'feature_fraction': 1.0, 'bagging_fraction': 1.0, 'bagging_freq': 0, 'min_child_samples': 20, 'num_iterations': 100, 'early_stopping_round': 50},\n]\n\nfor fold, (trn_idx, val_idx) in enumerate(cv):\n    clf = lgb.LGBMRegressor(**params_list[fold])\n    trn_x, trn_y = x.iloc[trn_idx, :], y[trn_idx]\n    val_x, val_y = x.iloc[val_idx, :], y[val_idx]\n    clf.fit(\n        trn_x, trn_y,\n        eval_set=[(val_x, val_y)],\n        verbose=-1, \n        early_stopping_rounds=100,\n    )\n    print(clf._best_score[\"valid_0\"][\"rmse\"])\n    pickle.dump(clf, open(f\"model.lgb.{fold}.pkl\", 'wb'))\n    df.iloc[val_idx, -1] = clf.predict(val_x)\n","metadata":{"execution":{"iopub.status.busy":"2022-01-18T15:29:31.319982Z","iopub.execute_input":"2022-01-18T15:29:31.320375Z","iopub.status.idle":"2022-01-18T15:30:02.542338Z","shell.execute_reply.started":"2022-01-18T15:29:31.320338Z","shell.execute_reply":"2022-01-18T15:30:02.541111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if flag_cv:\n    print(r2_score(df['target'], df['y_pred']))\n    print(pearsonr(df['target'], df['y_pred']))\n    print(df['target'].std())\n    print('pearsonr by market')\n    display(df.groupby('market').apply(lambda x: pearsonr(x['target'], x['y_pred'])[0]))\n    df2 = df.reset_index().set_index('timestamp')\n    market_count = df2['market'].unique().size\n    df2['target'].rolling(3 * 30 * 24 * 60 * market_count).corr(df2['y_pred']).iloc[::24 * 60 * market_count].plot()\n    plt.title('3 month rolling pearsonr')\n    plt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-01-16T09:26:12.278071Z","iopub.execute_input":"2022-01-16T09:26:12.278326Z","iopub.status.idle":"2022-01-16T09:26:25.005234Z","shell.execute_reply.started":"2022-01-16T09:26:12.278293Z","shell.execute_reply":"2022-01-16T09:26:25.004424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# reload clfs\nestimators = []\nfor fold, (trn_idx, val_idx) in enumerate(cv):\n    estimators.append(pickle.load(open(f\"model.lgb.{fold}.pkl\", 'rb')))","metadata":{"execution":{"iopub.status.busy":"2022-01-16T05:33:22.640965Z","iopub.status.idle":"2022-01-16T05:33:22.641212Z","shell.execute_reply.started":"2022-01-16T05:33:22.641078Z","shell.execute_reply":"2022-01-16T05:33:22.641094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submit\ndf_latest = pd.read_pickle('/tmp/df.pkl').tail(14*4000)\n\n# time must 0.25s < in each iter...\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()\nfor (test_df, sample_prediction_df) in iter_test:\n    \n    # get latest data\n    test_df2 = process_data(test_df, df_asset)\n    df_latest = pd.concat([df_latest, test_df2])\n    df_latest = df_latest.tail(14*4000+len(test_df2))\n    df = df_latest.copy()\n\n    df_features = calc_features(df)\n    df_features = df_features.loc[test_df2.index, :]\n\n    df_features[\"Target\"] = 0\n    for model in estimators:\n        df_features[\"Target\"] += model.predict(df_features[get_feature_columns(df_features)].values) / len(estimators)\n    sample_prediction_df[\"Target\"] = df_features[\"Target\"].values\n\n    # sample_prediction_df.merge(df_features[['row_id', 'Target']], how='left', on='row_id')\n    env.predict(sample_prediction_df)\n","metadata":{"_uuid":"95a98365-8589-40e4-b852-ba1ac5e4f75b","_cell_guid":"9a33bd31-26cd-47ce-b0e5-fabc987b8032","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-16T05:33:22.642156Z","iopub.status.idle":"2022-01-16T05:33:22.642506Z","shell.execute_reply.started":"2022-01-16T05:33:22.642325Z","shell.execute_reply":"2022-01-16T05:33:22.642343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}