{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2022-01-30T07:16:13.241685Z","iopub.execute_input":"2022-01-30T07:16:13.242065Z","iopub.status.idle":"2022-01-30T07:16:13.286554Z","shell.execute_reply.started":"2022-01-30T07:16:13.241975Z","shell.execute_reply":"2022-01-30T07:16:13.285603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_folder = \"../input/g-research-crypto-forecasting/\"\n!ls $data_folder","metadata":{"execution":{"iopub.status.busy":"2022-01-30T07:16:13.28796Z","iopub.execute_input":"2022-01-30T07:16:13.28844Z","iopub.status.idle":"2022-01-30T07:16:14.055709Z","shell.execute_reply.started":"2022-01-30T07:16:13.288408Z","shell.execute_reply":"2022-01-30T07:16:14.054187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Constant configuration\n# train_time_interval = ['2021-05-25 00:00:00', '2021-09-01 00:00:00'] # for selecting data interval #1621900800 - 1630454400 - 1632182400\n# USESUPP = False","metadata":{"execution":{"iopub.status.busy":"2022-01-30T07:16:14.057859Z","iopub.execute_input":"2022-01-30T07:16:14.058703Z","iopub.status.idle":"2022-01-30T07:16:14.063282Z","shell.execute_reply.started":"2022-01-30T07:16:14.058645Z","shell.execute_reply":"2022-01-30T07:16:14.062519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load asset details\nasset_details = pd.read_csv(data_folder+'asset_details.csv').sort_values(\"Asset_ID\")\nprint(asset_details)\nmap_asset_details = dict(zip(asset_details.Asset_ID, asset_details.Asset_Name))\nmap_asset_weights = dict(zip(asset_details.Asset_ID, asset_details.Weight))\nprint(map_asset_details)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T07:16:14.0647Z","iopub.execute_input":"2022-01-30T07:16:14.064978Z","iopub.status.idle":"2022-01-30T07:16:14.11331Z","shell.execute_reply.started":"2022-01-30T07:16:14.064941Z","shell.execute_reply":"2022-01-30T07:16:14.112357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom lightgbm import LGBMRegressor\nfrom sklearn.linear_model import LinearRegression\n# import gresearch_crypto\n\n\n# TRAIN_CSV = data_folder+'train.csv'\n# ASSET_DETAILS_CSV = data_folder+'asset_details.csv'\n\n# def read_csv_strict(file_name=data_folder+'train.csv'):\n#     df = pd.read_csv(file_name)\n#     df['datetime'] = pd.to_datetime(df['timestamp'], unit='s') #UTC时间\n#     df_train = df[(df['datetime'] >= train_time_interval[0]) & (df['datetime'] < train_time_interval[1])]\n#     df_test = df[df['datetime'] > train_time_interval[1]]\n#     return df_train, df_test\n# df_train, df_test = read_csv_strict()\n# df_train.shape, df_test.shape","metadata":{"execution":{"iopub.status.busy":"2022-01-30T07:16:14.115816Z","iopub.execute_input":"2022-01-30T07:16:14.116345Z","iopub.status.idle":"2022-01-30T07:16:16.467105Z","shell.execute_reply.started":"2022-01-30T07:16:14.116298Z","shell.execute_reply":"2022-01-30T07:16:16.466078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_asset_details = pd.read_csv(ASSET_DETAILS_CSV).sort_values(\"Asset_ID\")\n# df_asset_details\ndf_asset_details = pd.read_csv(data_folder+'asset_details.csv').sort_values(\"Asset_ID\")\nprint(df_asset_details)\nmap_asset_details = dict(zip(df_asset_details.Asset_ID, df_asset_details.Asset_Name))\nprint(map_asset_details)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T07:16:16.468536Z","iopub.execute_input":"2022-01-30T07:16:16.468857Z","iopub.status.idle":"2022-01-30T07:16:16.482222Z","shell.execute_reply.started":"2022-01-30T07:16:16.468823Z","shell.execute_reply":"2022-01-30T07:16:16.48148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Loading training data and test data\n# Constant configuration\ntrain_time_interval = ['2021-01-01 00:00:00', '2021-11-25 00:00:00'] # for selecting data interval #1621900800 - 1630454400 - 1632182400\ntrain_timestamp_interval = [1621900800, 1641772800]\nUSESUPP = False\n# train: 2021/05/25 - 2021/11/25     timestamp: [1621900800 - 1637798400)\n# test: 2021/11/25 - 2022/01/10      timestamp: [1637798400 - 1641772800]\n\ndf1 = pd.read_csv(data_folder+'train.csv')\ndf2 = pd.read_csv(data_folder+'supplemental_train.csv')\ndf = pd.concat([df1, df2])\ndf = df[df['timestamp'] >= train_timestamp_interval[0]]\n# split 14 df and padding, and add price after 16 mins and merge\n# 1. split 2. pad, add value 3. merge 4.sort on value and price\ndf_new = None\nfor i in asset_details.Asset_ID:\n  cur_df = df[df[\"Asset_ID\"] == i].set_index(\"timestamp\")\n  cur_df = cur_df.reindex(range(train_timestamp_interval[0], train_timestamp_interval[1], 60)).reset_index().rename(columns={'index':'timestamp'})\n  cur_df['P1'] = cur_df.loc[1:].reset_index()['Close']\n  cur_df['P16'] = cur_df.loc[16:].reset_index()['Close']\n  if df_new is None: \n    df_new = cur_df\n  else: \n    df_new = df_new.append(cur_df, ignore_index=True)\n\ndf = df_new.dropna(how='all').sort_values(by=['timestamp', 'Asset_ID'])\ndel df_new\ndf = df[df['Count'].notna()].reset_index(drop=True)\n\ndf['datetime'] = pd.to_datetime(df['timestamp'], unit='s')\n# df.drop(columns = 'timestamp', inplace = True)\ndf_test = df\n\nif train_time_interval: \n  df_train = df[train_time_interval[0] <= df['datetime']][df['datetime'] < train_time_interval[1]]\n  df_test = df_test[df_test['datetime'] >= train_time_interval[1]]\n\nif USESUPP:\n  df_supp = pd.read_csv('supplement_train.csv')\n  df_supp['datetime'] = pd.to_datetime(df_supp['timestamp'], unit='s')\n  df_supp.drop(columns = 'timestamp', inplace = True)\n  df.append(df_supp)\n\n# Or using our own loading method","metadata":{"execution":{"iopub.status.busy":"2022-01-30T07:16:16.483504Z","iopub.execute_input":"2022-01-30T07:16:16.484184Z","iopub.status.idle":"2022-01-30T07:17:37.653371Z","shell.execute_reply.started":"2022-01-30T07:16:16.484131Z","shell.execute_reply":"2022-01-30T07:17:37.651989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[df['Asset_ID']==0].head(20)","metadata":{"execution":{"iopub.status.busy":"2022-01-29T17:48:03.059155Z","iopub.execute_input":"2022-01-29T17:48:03.059559Z","iopub.status.idle":"2022-01-29T17:48:03.155756Z","shell.execute_reply.started":"2022-01-29T17:48:03.059456Z","shell.execute_reply":"2022-01-29T17:48:03.154461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Two new features from the competition tutorial\ndef upper_shadow(df):\n    return df['High'] - np.maximum(df['Close'], df['Open'])\n\ndef lower_shadow(df):\n    return np.minimum(df['Close'], df['Open']) - df['Low']\n\n# A utility function to build features from the original df\n# It works for rows to, so we can reutilize it.\ndef get_features(df, model_p1, model_p16, row=False):\n#     df_feat = df[['P16']].copy()\n    df_feat = df[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']].copy()\n    df_feat['Upper_Shadow'] = upper_shadow(df_feat)\n    df_feat['Lower_Shadow'] = lower_shadow(df_feat)\n\n    df_feat[\"Close/Open\"] = df_feat[\"Close\"] / df_feat[\"Open\"] \n    df_feat[\"Close-Open\"] = df_feat[\"Close\"] - df_feat[\"Open\"] \n    df_feat[\"High-Low\"] = df_feat[\"High\"] - df_feat[\"Low\"] \n    df_feat[\"High/Low\"] = df_feat[\"High\"] / df_feat[\"Low\"]\n    if row:\n        df_feat['Mean'] = df_feat[['Open', 'High', 'Low', 'Close']].mean()\n    else:\n        df_feat['Mean'] = df_feat[['Open', 'High', 'Low', 'Close']].mean(axis=1)\n    \n    df_feat['High/Mean'] = df_feat['High'] / df_feat['Mean']\n    df_feat['Low/Mean'] = df_feat['Low'] / df_feat['Mean']\n    df_feat['Volume/Count'] = df_feat['Volume'] / (df_feat['Count'] + 1)\n    \n \n\n    ## possible seasonality, datetime  features (unlikely to me meaningful, given very short time-frames)\n    ### to do: add cyclical features for seasonality\n    times = pd.to_datetime(df[\"timestamp\"],unit=\"s\",infer_datetime_format=True)\n    if row:\n        df_feat[\"hour\"] = times.hour  # .dt\n        df_feat[\"dayofweek\"] = times.dayofweek \n        df_feat[\"day\"] = times.day \n    else:\n        df_feat[\"hour\"] = times.dt.hour  # .dt\n        df_feat[\"dayofweek\"] = times.dt.dayofweek \n        df_feat[\"day\"] = times.dt.day \n    #df_feat.drop(columns=[\"time\"],errors=\"ignore\",inplace=True)  # keep original epoch time, drop string\n    \n#     df_feat['P16_P1'] = np.log(df['P16']/df['P1'])\n    \n    df['P1_p'] = model_p1.predict(df_feat)\n    df['P16_p'] = model_p16.predict(df_feat)\n    df_feat['P16_P1'] = np.log(df['P1_p']/df['P16_p'])\n    \n#     df_feat['P1'] = df['P1_p']\n#     df_feat['P16'] = df['P16_p']\n    \n    #   df_feat = df_feat.drop(\"P16\", axis=1)\n    \n    return df_feat\n\n\ndef get_features_predict_price(df, row=False):\n    df_feat = df[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']].copy()\n    df_feat['Upper_Shadow'] = upper_shadow(df_feat)\n    df_feat['Lower_Shadow'] = lower_shadow(df_feat)\n\n    df_feat[\"Close/Open\"] = df_feat[\"Close\"] / df_feat[\"Open\"] \n    df_feat[\"Close-Open\"] = df_feat[\"Close\"] - df_feat[\"Open\"] \n    df_feat[\"High-Low\"] = df_feat[\"High\"] - df_feat[\"Low\"] \n    df_feat[\"High/Low\"] = df_feat[\"High\"] / df_feat[\"Low\"]\n    if row:\n        df_feat['Mean'] = df_feat[['Open', 'High', 'Low', 'Close']].mean()\n    else:\n        df_feat['Mean'] = df_feat[['Open', 'High', 'Low', 'Close']].mean(axis=1)\n    \n    df_feat['High/Mean'] = df_feat['High'] / df_feat['Mean']\n    df_feat['Low/Mean'] = df_feat['Low'] / df_feat['Mean']\n    df_feat['Volume/Count'] = df_feat['Volume'] / (df_feat['Count'] + 1)\n\n    ## possible seasonality, datetime  features (unlikely to me meaningful, given very short time-frames)\n    ### to do: add cyclical features for seasonality\n    times = pd.to_datetime(df[\"timestamp\"],unit=\"s\",infer_datetime_format=True)\n    if row:\n        df_feat[\"hour\"] = times.hour  # .dt\n        df_feat[\"dayofweek\"] = times.dayofweek \n        df_feat[\"day\"] = times.day \n    else:\n        df_feat[\"hour\"] = times.dt.hour  # .dt\n        df_feat[\"dayofweek\"] = times.dt.dayofweek \n        df_feat[\"day\"] = times.dt.day \n    #df_feat.drop(columns=[\"time\"],errors=\"ignore\",inplace=True)  # keep original epoch time, drop string\n\n    return df_feat\n\ndef get_Xy_and_model_for_asset(df_train, asset_id, model_p1, model_p16):\n    df = df_train[df_train[\"Asset_ID\"] == asset_id]\n    \n    # TODO: Try different features here!\n    df_proc = get_features(df, model_p1, model_p16)\n    df_proc['y'] = df['Target']\n    df_proc = df_proc.replace([np.inf, -np.inf], np.nan).dropna(how=\"any\")\n    X = df_proc.drop(\"y\", axis=1)\n#     X = X.drop(\"P16\", axis=1)\n    y = df_proc[\"y\"]\n\n    # TODO: Try different models here!\n    model = LGBMRegressor(n_estimators=50)\n    model.fit(X, y)\n    return X, y, model\n\ndef get_Xy_and_model_for_asset_origin(df_train, asset_id):\n    df = df_train[df_train[\"Asset_ID\"] == asset_id]\n    \n    # TODO: Try different features here!\n    df_proc = get_features_predict_price(df)\n    df_proc['y'] = df['Target']\n    df_proc = df_proc.replace([np.inf, -np.inf], np.nan).dropna(how=\"any\")\n    X = df_proc.drop(\"y\", axis=1)\n    y = df_proc[\"y\"]\n    \n#     print(len(X))\n    # TODO: Try different models here!\n    model = LGBMRegressor(n_estimators=10)\n    model.fit(X, y)\n    return X, y, model\n\ndef get_Xy_and_model_for_asset_predict_price(df_train, asset_id):\n    df = df_train[df_train[\"Asset_ID\"] == asset_id]\n    \n    # TODO: Try different features here!\n    df_proc = get_features_predict_price(df)\n    df_proc['y'] = df['P1']\n    df_proc = df_proc.replace([np.inf, -np.inf], np.nan).dropna(how=\"any\")\n    X = df_proc.drop(\"y\", axis=1)\n    y = df_proc[\"y\"]\n    # TODO: Try different models here!\n#     print(X.columns)\n    model_p1 = LGBMRegressor(n_estimators=10)\n    model_p1.fit(X, y)\n    \n    df_proc['y'] = df['P16']\n    df_proc = df_proc.replace([np.inf, -np.inf], np.nan).dropna(how=\"any\")\n    X = df_proc.drop(\"y\", axis=1)\n    y = df_proc[\"y\"]\n    # TODO: Try different models here!\n    model_p16 = LGBMRegressor(n_estimators=10)\n    model_p16.fit(X, y)\n    return X, y, model_p1, model_p16","metadata":{"execution":{"iopub.status.busy":"2022-01-30T07:20:47.69958Z","iopub.execute_input":"2022-01-30T07:20:47.700153Z","iopub.status.idle":"2022-01-30T07:20:47.736434Z","shell.execute_reply.started":"2022-01-30T07:20:47.700102Z","shell.execute_reply":"2022-01-30T07:20:47.735245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Xs = {} \n# ys = {}\nXts = {} # log(p16/p1)和时间相关特征\nyts = {} # target\nXso = {} # 价格和时间相关特征\nyso = {} # target\nmodels = {}\n\n'''\nmodels_p1: predict p1 model\nmodels_p16: predict p16 model\nmodels_target: predict target by log(p16/p1)\nmodels_origin: predict target by price\n'''\nmodels_p1 = {} \nmodels_p16 = {}\nmodels_target = {}\nmodels_origin = {}\n\nfor asset_id, asset_name in zip(df_asset_details['Asset_ID'], df_asset_details['Asset_Name']):\n    print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n    _, _, model_p1, model_p16  = get_Xy_and_model_for_asset_predict_price(df_train, asset_id)\n    Xt, yt, model_target  = get_Xy_and_model_for_asset(df_train, asset_id, model_p1, model_p16)\n    Xo, yo, model_origin  = get_Xy_and_model_for_asset_origin(df_train, asset_id)\n    \n    Xts[asset_id], yts[asset_id], Xso[asset_id], yso[asset_id],  models_p1[asset_id], models_p16[asset_id], models_target[asset_id], models_origin[asset_id] \\\n    = Xt, yt, Xo, yo, model_p1, model_p16, model_target, model_origin","metadata":{"execution":{"iopub.status.busy":"2022-01-30T07:17:37.695715Z","iopub.execute_input":"2022-01-30T07:17:37.696071Z","iopub.status.idle":"2022-01-30T07:18:32.549346Z","shell.execute_reply.started":"2022-01-30T07:17:37.696024Z","shell.execute_reply":"2022-01-30T07:18:32.548645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# evaluation on train set\nfrom scipy.stats import pearsonr\nimport matplotlib.pyplot as plt\nf = plt.figure(figsize=(16, 16))\ncolors = ['b', 'g', 'c', 'k', 'm', 'teal', 'r', 'y', 'orange', 'gold', \n          'skyblue', 'maroon', 'cyan', 'sienna']\nscores=0\nfor i, color in zip(range(14), colors):\n  trainX = Xts[i]\n  y_true = yts[i]\n  model = models_target[i]\n  y_pred = model.predict(trainX)\n  ax = \"ax\" + str(i)\n  ax = f.add_subplot(7, 2, i+1)\n  plt.plot(range(len(y_true)), y_true, label=\"true_\"+map_asset_details[i])#, color=color)\n  plt.plot(range(len(y_pred)), y_pred, label=\"predict_\"+map_asset_details[i])#, color=color)\n  print('pearsonr for %s: %.3f' %(map_asset_details[i], pearsonr(y_true, y_pred)[0]))\n  scores+=map_asset_weights[i]*pearsonr(y_true, y_pred)[0]\n  plt.legend()\n  # plt.xlabel('Time')\n  plt.ylabel(map_asset_details[i])\n  # if i == 2: break\nprint('scores: %.3f'%scores)\nplt.tight_layout()\nplt.show()\n\n# for i in range(14):\n  # trainX = Xs[i]\n  # y_true = ys[i]\n  # model = models[i]\n  # y_pred = model.predict(trainX)\n#   plt.plot(range(len(y_true)), y_true, label=\"true\")\n#   plt.plot(range(len(y_pred)), y_pred, label=\"predict\")\n#   plt.legend()\n#   plt.show()\n#   break\n  ","metadata":{"execution":{"iopub.status.busy":"2022-01-30T07:18:32.551202Z","iopub.execute_input":"2022-01-30T07:18:32.551717Z","iopub.status.idle":"2022-01-30T07:18:56.296237Z","shell.execute_reply.started":"2022-01-30T07:18:32.551678Z","shell.execute_reply":"2022-01-30T07:18:56.295485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_features_test(df, model_p1, model_p16, row=False):\n#     df_feat = df[['P16']].copy()\n    df_feat = df[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']].copy()\n    df_feat['Upper_Shadow'] = upper_shadow(df_feat)\n    df_feat['Lower_Shadow'] = lower_shadow(df_feat)\n\n    df_feat[\"Close/Open\"] = df_feat[\"Close\"] / df_feat[\"Open\"] \n    df_feat[\"Close-Open\"] = df_feat[\"Close\"] - df_feat[\"Open\"] \n    df_feat[\"High-Low\"] = df_feat[\"High\"] - df_feat[\"Low\"] \n    df_feat[\"High/Low\"] = df_feat[\"High\"] / df_feat[\"Low\"]\n    if row:\n        df_feat['Mean'] = df_feat[['Open', 'High', 'Low', 'Close']].mean()\n    else:\n        df_feat['Mean'] = df_feat[['Open', 'High', 'Low', 'Close']].mean(axis=1)\n    \n    df_feat['High/Mean'] = df_feat['High'] / df_feat['Mean']\n    df_feat['Low/Mean'] = df_feat['Low'] / df_feat['Mean']\n    df_feat['Volume/Count'] = df_feat['Volume'] / (df_feat['Count'] + 1)\n    \n    \n#     df_feat = df_feat.drop(\"P16\", axis=1)\n    ## possible seasonality, datetime  features (unlikely to me meaningful, given very short time-frames)\n    ### to do: add cyclical features for seasonality\n    times = pd.to_datetime(df[\"timestamp\"],unit=\"s\", infer_datetime_format=True)\n    if row:\n        df_feat[\"hour\"] = times.hour  # .dt\n        df_feat[\"dayofweek\"] = times.dayofweek \n        df_feat[\"day\"] = times.day \n    else:\n        df_feat[\"hour\"] = times.dt.hour  # .dt\n        df_feat[\"dayofweek\"] = times.dt.dayofweek \n        df_feat[\"day\"] = times.dt.day \n    #df_feat.drop(columns=[\"time\"],errors=\"ignore\",inplace=True)  # keep original epoch time, drop string\n    df['P1_p'] = model_p1.predict(df_feat)\n    df['P16_p'] = model_p16.predict(df_feat)\n    df_feat['P16_P1'] = np.log(df['P1_p']/df['P16_p'])\n    return df_feat\n\ndef get_Xy_and_model_for_asset_test(df_train, asset_id):\n    df = df_train[df_train[\"Asset_ID\"] == asset_id]\n    \n    # TODO: Try different features here!\n    df_proc = get_features_test(df, models_p1[asset_id], models_p16[asset_id])\n    df_proc['y'] = df['Target']\n    df_proc = df_proc.replace([np.inf, -np.inf], np.nan).dropna(how=\"any\")\n    X = df_proc.drop(\"y\", axis=1)\n    y = df_proc[\"y\"]\n\n    # TODO: Try different models here!\n    model = LGBMRegressor(n_estimators=50)\n    model.fit(X, y)\n    return X, y, model","metadata":{"execution":{"iopub.status.busy":"2022-01-30T07:20:50.643133Z","iopub.execute_input":"2022-01-30T07:20:50.643431Z","iopub.status.idle":"2022-01-30T07:20:50.660227Z","shell.execute_reply.started":"2022-01-30T07:20:50.643401Z","shell.execute_reply":"2022-01-30T07:20:50.659242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluating on test set\ntest_Xs = {}\ntest_ys = {}\n# models = {}\n\nfor asset_id, asset_name in zip(asset_details['Asset_ID'], asset_details['Asset_Name']):\n    # print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n#     X, y, _ = get_Xy_and_model_for_asset_test(df_test, asset_id)  \n    X, y, _ = get_Xy_and_model_for_asset(df_test, asset_id, models_p1[asset_id], models_p16[asset_id])\n    test_Xs[asset_id], test_ys[asset_id] = X, y\n\n\nf = plt.figure(figsize=(16, 16))\ncolors = ['b', 'g', 'c', 'k', 'm', 'teal', 'r', 'y', 'orange', 'gold', \n          'skyblue', 'maroon', 'cyan', 'sienna']\nscores = 0\nfor i, color in zip(range(14), colors):\n  trainX = test_Xs[i]\n  y_true = test_ys[i]\n  model = models_target[i]\n  y_pred = model.predict(trainX)\n  ax = \"ax\" + str(i)\n  ax = f.add_subplot(7, 2, i+1)\n  plt.plot(range(len(y_true)), y_true, label=\"true_\"+map_asset_details[i])#, color=color)\n  plt.plot(range(len(y_pred)), y_pred, label=\"predict_\"+map_asset_details[i])#, color=color)\n  print('pearsonr for %s: %.3f' %(map_asset_details[i], pearsonr(y_true, y_pred)[0]))\n\n  scores+=map_asset_weights[i]*pearsonr(y_true, y_pred)[0]\n  plt.legend()\n  # plt.xlabel('Time')\n  plt.ylabel(map_asset_details[i])\n  # if i == 2: break\nprint('scores: %.3f'%scores)\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-30T07:20:52.111277Z","iopub.execute_input":"2022-01-30T07:20:52.112203Z","iopub.status.idle":"2022-01-30T07:21:05.216492Z","shell.execute_reply.started":"2022-01-30T07:20:52.11209Z","shell.execute_reply":"2022-01-30T07:21:05.215563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred","metadata":{"execution":{"iopub.status.busy":"2022-01-29T12:44:33.364491Z","iopub.execute_input":"2022-01-29T12:44:33.365305Z","iopub.status.idle":"2022-01-29T12:44:33.370943Z","shell.execute_reply.started":"2022-01-29T12:44:33.365254Z","shell.execute_reply":"2022-01-29T12:44:33.370169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-01-26T17:23:04.421192Z","iopub.execute_input":"2022-01-26T17:23:04.422018Z","iopub.status.idle":"2022-01-26T17:23:04.434753Z","shell.execute_reply.started":"2022-01-26T17:23:04.42198Z","shell.execute_reply":"2022-01-26T17:23:04.433489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"map_assets","metadata":{"execution":{"iopub.status.busy":"2022-01-26T17:22:56.568461Z","iopub.execute_input":"2022-01-26T17:22:56.568756Z","iopub.status.idle":"2022-01-26T17:22:56.590658Z","shell.execute_reply.started":"2022-01-26T17:22:56.568727Z","shell.execute_reply":"2022-01-26T17:22:56.58978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from ctypes import cdll\ngresearch_crypto = cdll.LoadLibrary(data_folder+'gresearch_crypto/competition.cpython-37m-x86_64-linux-gnu.so')\n# import gresearch_crypto","metadata":{"execution":{"iopub.status.busy":"2022-01-23T16:57:09.687731Z","iopub.execute_input":"2022-01-23T16:57:09.688199Z","iopub.status.idle":"2022-01-23T16:57:09.694805Z","shell.execute_reply.started":"2022-01-23T16:57:09.688149Z","shell.execute_reply":"2022-01-23T16:57:09.693969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"env = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    for j , row in df_test.iterrows():\n        \n        model = models[row['Asset_ID']]\n        x_test = get_features(row, row=True)\n        y_pred = model.predict([x_test])[0]\n        \n        df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = y_pred\n        \n        \n        # Print just one sample row to get a feeling of what it looks like\n        if i == 0 and j == 0:\n            display(x_test)\n\n    # Display the first prediction dataframe\n    if i == 0:\n        display(df_pred)\n\n    # Send submissions\n    env.predict(df_pred)\n","metadata":{"execution":{"iopub.status.busy":"2022-01-23T17:30:48.31446Z","iopub.execute_input":"2022-01-23T17:30:48.314884Z","iopub.status.idle":"2022-01-23T17:30:48.350979Z","shell.execute_reply.started":"2022-01-23T17:30:48.314844Z","shell.execute_reply":"2022-01-23T17:30:48.349904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gresearch_crypto","metadata":{"execution":{"iopub.status.busy":"2022-01-23T17:09:48.688096Z","iopub.execute_input":"2022-01-23T17:09:48.688457Z","iopub.status.idle":"2022-01-23T17:09:48.695244Z","shell.execute_reply.started":"2022-01-23T17:09:48.688423Z","shell.execute_reply":"2022-01-23T17:09:48.694576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls ../input/g-research-crypto-forecasting//gresearch_crypto/","metadata":{"execution":{"iopub.status.busy":"2022-01-23T17:10:11.167623Z","iopub.execute_input":"2022-01-23T17:10:11.16792Z","iopub.status.idle":"2022-01-23T17:10:11.963354Z","shell.execute_reply.started":"2022-01-23T17:10:11.167887Z","shell.execute_reply":"2022-01-23T17:10:11.96242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}