{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# [training] 3rd place solution\n\nI will post a description of the model in discussion. Inference is done in another notebook.\n* Inference: https://www.kaggle.com/sugghi/inference-3rd-place-solution/\n\nI referred to various notebooks when coding. In particular, the following notebooks were used directly.\n* Submitting Lagged Features via API：\nhttps://www.kaggle.com/tomforbes/gresearch-submitting-lagged-features-via-api\n* EmbargoCV：\nhttps://www.kaggle.com/nrcjea001/lgbm-embargocv-weightedpearson-lagtarget/notebook\n\nIn addition, although not used in this notebook, local api published by @jagofc helped me a lot in coding.\n* local api：https://www.kaggle.com/code/jagofc/local-api/\n\nAs you can see from my code, I am a novice in machine learning and python.\nIf you see anything  to improve on or any mistakes, I'd be very happy to hear about them!","metadata":{}},{"cell_type":"code","source":"import os\nimport random\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport lightgbm as lgb\nimport gresearch_crypto\nimport time\nimport datetime\n\nimport pickle\nimport gc\n\nfrom tqdm import tqdm\n\nn_fold = 7\nseed0 = 8586\nuse_supple_for_train = True\n\n# If True, the period used to evaluate Public LB will not be used for training.\n# Set to False on final submission.\nnot_use_overlap_to_train = False\n\nTRAIN_CSV = '/kaggle/input/g-research-crypto-forecasting/train.csv'\nSUPPLE_TRAIN_CSV = '/kaggle/input/g-research-crypto-forecasting/supplemental_train.csv'\nASSET_DETAILS_CSV = '/kaggle/input/g-research-crypto-forecasting/asset_details.csv'\n\npd.set_option('display.max_rows', 6)\npd.set_option('display.max_columns', 350)","metadata":{"execution":{"iopub.status.busy":"2022-12-09T02:14:31.795372Z","iopub.execute_input":"2022-12-09T02:14:31.795696Z","iopub.status.idle":"2022-12-09T02:14:34.086041Z","shell.execute_reply.started":"2022-12-09T02:14:31.795664Z","shell.execute_reply":"2022-12-09T02:14:34.085237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lags = [60,300,900]","metadata":{"execution":{"iopub.status.busy":"2022-12-09T02:14:34.088047Z","iopub.execute_input":"2022-12-09T02:14:34.088397Z","iopub.status.idle":"2022-12-09T02:14:34.094835Z","shell.execute_reply.started":"2022-12-09T02:14:34.088351Z","shell.execute_reply":"2022-12-09T02:14:34.093838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {\n    'early_stopping_rounds': 50,\n    'objective': 'regression',\n    'metric': 'rmse',\n#     'metric': 'None',\n    'boosting_type': 'gbdt',\n    'max_depth': 5,\n    'verbose': -1,\n    'max_bin':600,\n    'min_data_in_leaf':50,\n    'learning_rate': 0.03,\n    'subsample': 0.7,\n    'subsample_freq': 1,\n    'feature_fraction': 1,\n    'lambda_l1': 0.5,\n    'lambda_l2': 2,\n    'seed':seed0,\n    'feature_fraction_seed': seed0,\n    'bagging_fraction_seed': seed0,\n    'drop_seed': seed0,\n    'data_random_seed': seed0,\n    'extra_trees': True,\n    'extra_seed': seed0,\n    'zero_as_missing': True,\n    \"first_metric_only\": True\n         }","metadata":{"execution":{"iopub.status.busy":"2022-12-09T02:14:34.095941Z","iopub.execute_input":"2022-12-09T02:14:34.096167Z","iopub.status.idle":"2022-12-09T02:14:34.106635Z","shell.execute_reply.started":"2022-12-09T02:14:34.096133Z","shell.execute_reply":"2022-12-09T02:14:34.105514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n\n    for col in df.columns:\n        col_type = df[col].dtype\n\n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-12-09T02:14:34.323758Z","iopub.execute_input":"2022-12-09T02:14:34.324237Z","iopub.status.idle":"2022-12-09T02:14:34.339268Z","shell.execute_reply.started":"2022-12-09T02:14:34.324203Z","shell.execute_reply":"2022-12-09T02:14:34.338491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset_details = pd.read_csv(ASSET_DETAILS_CSV).sort_values(\"Asset_ID\")\ndf_asset_details","metadata":{"execution":{"iopub.status.busy":"2022-12-09T02:14:37.490121Z","iopub.execute_input":"2022-12-09T02:14:37.490751Z","iopub.status.idle":"2022-12-09T02:14:37.508233Z","shell.execute_reply.started":"2022-12-09T02:14:37.490701Z","shell.execute_reply":"2022-12-09T02:14:37.507280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"%%time\ndf_train = pd.read_csv(TRAIN_CSV, usecols=['timestamp','Asset_ID', 'Close', 'Target'])\n\nif use_supple_for_train:    \n    df_supple = pd.read_csv(SUPPLE_TRAIN_CSV, usecols=['timestamp','Asset_ID', 'Close', 'Target'])\n#     display(df_supple)\n    df_train = pd.concat([df_train, df_supple])\n    del df_supple\ndf_train = reduce_mem_usage(df_train)\ndf_train","metadata":{"execution":{"iopub.status.busy":"2022-12-09T02:14:39.450159Z","iopub.execute_input":"2022-12-09T02:14:39.450473Z","iopub.status.idle":"2022-12-09T02:15:45.301992Z","shell.execute_reply.started":"2022-12-09T02:14:39.450440Z","shell.execute_reply":"2022-12-09T02:15:45.301029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_merged = pd.DataFrame()\ntrain_merged[df_train.columns] = 0\nfor id in tqdm( range(14) ):\n    train_merged = train_merged.merge(df_train.loc[df_train[\"Asset_ID\"] == id, ['timestamp', 'Close','Target']].copy(), on=\"timestamp\", how='outer',suffixes=['', \"_\"+str(id)])\n        \ntrain_merged = train_merged.drop(df_train.columns.drop(\"timestamp\"), axis=1)\ntrain_merged = train_merged.sort_values('timestamp', ascending=True)\ndisplay(train_merged.head())","metadata":{"execution":{"iopub.status.busy":"2022-12-09T02:15:45.303931Z","iopub.execute_input":"2022-12-09T02:15:45.304169Z","iopub.status.idle":"2022-12-09T02:16:06.186582Z","shell.execute_reply.started":"2022-12-09T02:15:45.304140Z","shell.execute_reply":"2022-12-09T02:16:06.185486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Generate Future Returns","metadata":{}},{"cell_type":"code","source":"for i in range(14):\n    train_merged['Target_' + str(i) + '_return'] = train_merged['Close_' + str(i)].shift(-64)-train_merged['Close_' + str(i)]\n    train_merged['Target_' + str(i) + '_return'] = train_merged['Target_' + str(i) + '_return']/train_merged['Close_' + str(i)]\ntrain_merged","metadata":{"execution":{"iopub.status.busy":"2022-12-09T02:16:06.188047Z","iopub.execute_input":"2022-12-09T02:16:06.188290Z","iopub.status.idle":"2022-12-09T02:16:06.432580Z","shell.execute_reply.started":"2022-12-09T02:16:06.188254Z","shell.execute_reply":"2022-12-09T02:16:06.431906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# forward fill\n# Set an upper limit on the number of fills, since there may be long term gaps.\nfor id in range(14):\n#     print(id, train_merged[f'Close_{id}'].isnull().sum())   # Number of missing before forward fill\n    train_merged[f'Close_{id}'] = train_merged[f'Close_{id}'].fillna(method='ffill', limit=100)\n#     print(id, train_merged[f'Close_{id}'].isnull().sum())   # Number of missing after forward fill","metadata":{"execution":{"iopub.status.busy":"2022-12-09T02:16:06.434328Z","iopub.execute_input":"2022-12-09T02:16:06.434590Z","iopub.status.idle":"2022-12-09T02:16:06.578054Z","shell.execute_reply.started":"2022-12-09T02:16:06.434559Z","shell.execute_reply":"2022-12-09T02:16:06.577141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering","metadata":{}},{"cell_type":"code","source":"def get_features(df, train=True):   \n    if train == True:\n        totimestamp = lambda s: np.int32(time.mktime(datetime.datetime.strptime(s, \"%d/%m/%Y\").timetuple()))\n        valid_window = [totimestamp(\"12/03/2021\")]\n#         valid_window = [totimestamp(\"15/08/2021\")]  #検証用\n        df['train_flg'] = np.where(df['timestamp']>=valid_window[0], 0,1)\n\n        supple_start_window = [totimestamp(\"22/09/2021\")]\n        #if use_supple_for_train:\n            #df['train_flg'] = np.where(df['timestamp']>=supple_start_window[0], 1 ,df['train_flg']  )\n\n   \n    for id in range(14):    \n        for lag in lags:\n            #log(close/sma(close, 60))\n            df[f'factor001_{lag}_id{id}'] = np.log( np.array(df[f'Close_{id}']) /  np.roll(np.append(np.convolve( np.array(df[f'Close_{id}']), np.ones(lag)/lag, mode=\"valid\"), np.ones(lag-1)), lag-1)  )\n            #log(close/delay(close, 60))\n            df[f'factor002_{lag}_id{id}']  = np.log( np.array(df[f'Close_{id}']) /  np.roll(np.array(df[f'Close_{id}']), lag)  )\n    for lag in lags:\n        #sma(log(close/sma(close, 60)), 60)\n        df[f'factor003_{lag}'] =  np.mean(df.iloc[:,df.columns.str.startswith(f'factor001_{lag}_id')], axis=1)\n        #sma(log(close/delay(close, 60)))\n        df[f'factor004_{lag}'] = np.mean(df.iloc[:,df.columns.str.startswith(f'factor002_{lag}_id')] ,    axis=1)\n        for id in range(14):\n            #log(close/sma(close, 60)) - sma(log(close/sma(close, 60)), 60)\n            df[f'factor005_{lag}_id{id}'] = np.array( df[f'factor001_{lag}_id{id}']) - np.array( df[f'factor003_{lag}']  )\n            #log(close/delay(close, 60)) - sma(log(return/delay(close, 60)))\n            df[f'factor006_{lag}_id{id}'] = np.array( df[f'factor002_{lag}_id{id}']) - np.array( df[f'factor004_{lag}'] )\n\n    if train == True:\n        for id in range(14):\n            df = df.drop([f'Close_{id}'], axis=1)\n        oldest_use_window = [totimestamp(\"12/01/2019\")]\n        df = df[  df['timestamp'] >= oldest_use_window[0]   ]\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-12-09T02:16:06.579237Z","iopub.execute_input":"2022-12-09T02:16:06.579496Z","iopub.status.idle":"2022-12-09T02:16:06.594491Z","shell.execute_reply.started":"2022-12-09T02:16:06.579457Z","shell.execute_reply":"2022-12-09T02:16:06.593198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfeat = get_features(train_merged)\nfeat","metadata":{"execution":{"iopub.status.busy":"2022-12-09T02:16:06.595818Z","iopub.execute_input":"2022-12-09T02:16:06.596057Z","iopub.status.idle":"2022-12-09T02:16:41.730696Z","shell.execute_reply.started":"2022-12-09T02:16:06.596026Z","shell.execute_reply":"2022-12-09T02:16:41.729869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in feat.columns:\n    print(i)","metadata":{"execution":{"iopub.status.busy":"2022-12-08T16:52:03.253748Z","iopub.execute_input":"2022-12-08T16:52:03.254489Z","iopub.status.idle":"2022-12-08T16:52:03.287031Z","shell.execute_reply.started":"2022-12-08T16:52:03.254440Z","shell.execute_reply":"2022-12-08T16:52:03.285865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define features for LGBM\nnot_use_features_train = ['timestamp', 'train_flg']\nfor id in range(14):\n    not_use_features_train.append(f'Target_{id}')\n    not_use_features_train.append(f'Target_{id}_return')\nfeatures = feat.columns \nfeatures = features.drop(not_use_features_train)\nfeatures = list(features)\n# display(features)  \nlen(features)","metadata":{"execution":{"iopub.status.busy":"2022-12-09T02:16:41.732122Z","iopub.execute_input":"2022-12-09T02:16:41.732601Z","iopub.status.idle":"2022-12-09T02:16:41.741645Z","shell.execute_reply.started":"2022-12-09T02:16:41.732567Z","shell.execute_reply":"2022-12-09T02:16:41.740993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features","metadata":{"execution":{"iopub.status.busy":"2022-12-09T02:16:41.743372Z","iopub.execute_input":"2022-12-09T02:16:41.744056Z","iopub.status.idle":"2022-12-09T02:16:41.755627Z","shell.execute_reply.started":"2022-12-09T02:16:41.744011Z","shell.execute_reply":"2022-12-09T02:16:41.754566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_merged\ndel df_train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-12-08T16:57:56.610919Z","iopub.execute_input":"2022-12-08T16:57:56.611452Z","iopub.status.idle":"2022-12-08T16:57:58.482340Z","shell.execute_reply.started":"2022-12-08T16:57:56.611403Z","shell.execute_reply":"2022-12-08T16:57:58.480984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"# define the evaluation metric\ndef correlation(a, train_data):\n    \n    b = train_data.get_label()\n    \n    a = np.ravel(a)\n    b = np.ravel(b)\n\n    len_data = len(a)\n    mean_a = np.sum(a) / len_data\n    mean_b = np.sum(b) / len_data\n    var_a = np.sum(np.square(a - mean_a)) / len_data\n    var_b = np.sum(np.square(b - mean_b)) / len_data\n\n    cov = np.sum((a * b))/len_data - mean_a*mean_b\n    corr = cov / np.sqrt(var_a * var_b)\n\n    return 'corr', corr, True\n\n# For CV score calculation\ndef corr_score(pred, valid):\n    len_data = len(pred)\n    mean_pred = np.sum(pred) / len_data\n    mean_valid = np.sum(valid) / len_data\n    var_pred = np.sum(np.square(pred - mean_pred)) / len_data\n    var_valid = np.sum(np.square(valid - mean_valid)) / len_data\n\n    cov = np.sum((pred * valid))/len_data - mean_pred*mean_valid\n    corr = cov / np.sqrt(var_pred * var_valid)\n    return corr\n\n# For CV score calculation\ndef wcorr_score(pred, valid, weight):\n    len_data = len(pred)\n    sum_w = np.sum(weight)\n    mean_pred = np.sum(pred * weight) / sum_w\n    mean_valid = np.sum(valid * weight) / sum_w\n    var_pred = np.sum(weight * np.square(pred - mean_pred)) / sum_w\n    var_valid = np.sum(weight * np.square(valid - mean_valid)) / sum_w\n\n    cov = np.sum((pred * valid * weight)) / sum_w - mean_pred*mean_valid\n    corr = cov / np.sqrt(var_pred * var_valid)\n\n    return corr","metadata":{"execution":{"iopub.status.busy":"2022-12-09T02:16:41.757693Z","iopub.execute_input":"2022-12-09T02:16:41.758603Z","iopub.status.idle":"2022-12-09T02:16:41.771499Z","shell.execute_reply.started":"2022-12-09T02:16:41.758556Z","shell.execute_reply":"2022-12-09T02:16:41.770721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from: https://blog.amedama.jp/entry/lightgbm-cv-feature-importance\n# (used in nyanp's Optiver solution)\ndef plot_importance(importances, features_names = features, PLOT_TOP_N = 20, figsize=(10, 10), asset_id =None):\n    importance_df = pd.DataFrame(data=importances, columns=features)\n    sorted_indices = importance_df.median(axis=0).sort_values(ascending=False).index\n    sorted_importance_df = importance_df.loc[:, sorted_indices]\n    plot_cols = sorted_importance_df.columns[:PLOT_TOP_N]\n    _, ax = plt.subplots(figsize=figsize)\n    ax.grid()\n    ax.set_xscale('log')\n    ax.set_ylabel('Feature')\n    ax.set_xlabel('Importance')\n    sns.boxplot(data=sorted_importance_df[plot_cols],\n                orient='h',\n                ax=ax)\n    plt.savefig('importances_{}.jpg'.format(asset_id))\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-09T02:16:41.773540Z","iopub.execute_input":"2022-12-09T02:16:41.774361Z","iopub.status.idle":"2022-12-09T02:16:41.782484Z","shell.execute_reply.started":"2022-12-09T02:16:41.774303Z","shell.execute_reply":"2022-12-09T02:16:41.781406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from: https://www.kaggle.com/code/nrcjea001/lgbm-embargocv-weightedpearson-lagtarget/\ndef get_time_series_cross_val_splits(data, cv = n_fold, embargo = 3750):\n    all_train_timestamps = data['timestamp'].unique()\n    len_split = len(all_train_timestamps) // cv\n    test_splits = [all_train_timestamps[i * len_split:(i + 1) * len_split] for i in range(cv)]\n    # fix the last test split to have all the last timestamps, in case the number of timestamps wasn't divisible by cv\n    rem = len(all_train_timestamps) - len_split*cv\n    if rem>0:\n        test_splits[-1] = np.append(test_splits[-1], all_train_timestamps[-rem:])\n\n    train_splits = []\n    for test_split in test_splits:\n        test_split_max = int(np.max(test_split))\n        test_split_min = int(np.min(test_split))\n        # get all of the timestamps that aren't in the test split\n        train_split_not_embargoed = [e for e in all_train_timestamps if not (test_split_min <= int(e) <= test_split_max)]\n        # embargo the train split so we have no leakage. Note timestamps are expressed in seconds, so multiply by 60\n        embargo_sec = 60*embargo\n        train_split = [e for e in train_split_not_embargoed if\n                       abs(int(e) - test_split_max) > embargo_sec and abs(int(e) - test_split_min) > embargo_sec]\n        train_splits.append(train_split)\n\n    # convenient way to iterate over train and test splits\n    train_test_zip = zip(train_splits, test_splits)\n    return train_test_zip","metadata":{"execution":{"iopub.status.busy":"2022-12-09T02:16:41.783942Z","iopub.execute_input":"2022-12-09T02:16:41.784285Z","iopub.status.idle":"2022-12-09T02:16:41.799179Z","shell.execute_reply.started":"2022-12-09T02:16:41.784242Z","shell.execute_reply":"2022-12-09T02:16:41.798503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"not_use_overlap_to_train","metadata":{"execution":{"iopub.status.busy":"2022-12-08T17:01:02.164657Z","iopub.execute_input":"2022-12-08T17:01:02.165624Z","iopub.status.idle":"2022-12-08T17:01:02.172061Z","shell.execute_reply.started":"2022-12-08T17:01:02.165572Z","shell.execute_reply":"2022-12-08T17:01:02.170891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_Xy_and_model_for_asset(df_proc, asset_id,data_freq):\n    n_fold =7\n    df_proc = df_proc.loc[  (df_proc[f'Target_{asset_id}_return'] == df_proc[f'Target_{asset_id}_return'])  ]\n    if not_use_overlap_to_train:\n        df_proc = df_proc.loc[  (df_proc['train_flg'] == 1)  ]\n    \n# EmbargoCV\n    print(\"use data till \", df_proc['timestamp'].tolist()[-1])\n    train_test_zip = get_time_series_cross_val_splits(df_proc, cv = 7, embargo = 3750)\n    print(\"entering time series cross validation loop\")\n    importances = []\n    oof_pred = []\n    oof_valid = []\n    \n    for split, train_test_split in enumerate(train_test_zip):\n        gc.collect()\n        \n        print(f\"doing split {split+1} out of {n_fold}\")\n        train_split, test_split = train_test_split\n        train_split_index = df_proc['timestamp'].isin(train_split)\n        test_split_index = df_proc['timestamp'].isin(test_split)\n    \n        train_dataset = lgb.Dataset(df_proc.loc[train_split_index, features],\n                                    df_proc.loc[train_split_index, f'Target_{asset_id}_return'].values, \n                                    feature_name = features, \n                                   )\n        val_dataset = lgb.Dataset(df_proc.loc[test_split_index, features], \n                                  df_proc.loc[test_split_index, f'Target_{asset_id}_return'].values, \n                                  feature_name = features, \n                                 )\n\n        print(f\"number of train data: {len(df_proc.loc[train_split_index])}\")\n        print(f\"number of val data:   {len(df_proc.loc[test_split_index])}\")\n\n        model = lgb.train(params = params,\n                          train_set = train_dataset, \n                          valid_sets=[train_dataset, val_dataset],\n                          valid_names=['tr', 'vl'],\n                          num_boost_round = 5000,\n                          verbose_eval = 100,     \n                          feval = correlation,\n                         )\n        importances.append(model.feature_importance(importance_type='gain'))\n        \n        file = f'model_id{asset_id}_fold{split}_datafreq{data_freq}.pkl'\n        pickle.dump(model, open(file, 'wb'))\n        print(f\"Trained model was saved to 'model_id{asset_id}_fold{split}.pkl'\")\n        print(\"\")\n        \n        oof_pred += list(  model.predict(df_proc.loc[test_split_index, features])        )\n        oof_valid += list(   df_proc.loc[test_split_index, f'Target_{asset_id}_return'].values    )\n    \n    \n    plot_importance(np.array(importances),features, PLOT_TOP_N = 20, figsize=(10, 5), asset_id=asset_id)\n\n    return oof_pred, oof_valid","metadata":{"execution":{"iopub.status.busy":"2022-12-09T02:16:41.800141Z","iopub.execute_input":"2022-12-09T02:16:41.800501Z","iopub.status.idle":"2022-12-09T02:16:41.816059Z","shell.execute_reply.started":"2022-12-09T02:16:41.800467Z","shell.execute_reply":"2022-12-09T02:16:41.815230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import r2_score\nnot_use_overlap_to_train = True\noof = [ [] for id in range(2)   ]\n\nall_oof_pred = []\nall_oof_valid = []\nall_oof_weight = []\n\nfor asset_id, asset_name in zip(df_asset_details['Asset_ID'].tolist()[:2], df_asset_details['Asset_Name'].tolist()[:2]):\n    print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n    oof_pred, oof_valid = get_Xy_and_model_for_asset(feat, asset_id, 1)\n    #oof_pred, oof_valid = get_Xy_and_model_for_asset(feat[feat.timestamp<=1622530800], asset_id)\n#     print(oof_pred)\n#     print(oof_valid)\n    weight_temp = float( df_asset_details.loc[  df_asset_details['Asset_ID'] == asset_id  , 'Weight'   ]  )\n    \n    all_oof_pred += oof_pred\n    all_oof_valid += oof_valid\n    all_oof_weight += [weight_temp] * len(oof_pred)\n    \n    oof[asset_id] = r2_score( np.array(oof_valid),    np.array(oof_pred)   )\n    \n    print(f'OOF corr score of {asset_name} (ID={asset_id}) is {oof[asset_id]:.5f}. (Weight: {float(weight_temp):.5f})')\n    print('')\n    print('')","metadata":{"execution":{"iopub.status.busy":"2022-12-09T02:17:29.255069Z","iopub.execute_input":"2022-12-09T02:17:29.255895Z","iopub.status.idle":"2022-12-09T02:32:14.010014Z","shell.execute_reply.started":"2022-12-09T02:17:29.255852Z","shell.execute_reply":"2022-12-09T02:32:14.008906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred","metadata":{"execution":{"iopub.status.busy":"2022-12-09T00:56:15.007258Z","iopub.execute_input":"2022-12-09T00:56:15.007658Z","iopub.status.idle":"2022-12-09T00:56:15.014380Z","shell.execute_reply.started":"2022-12-09T00:56:15.007615Z","shell.execute_reply":"2022-12-09T00:56:15.013305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nscore_list = []\nfor asset_id in range(2):\n    print(asset_id)\n    prediction = np.array([])\n    fold =6\n    filename = 'model_id{}_fold{}_datafreq{}.pkl'.format(asset_id, fold,1)\n    gbm = pickle.load(open(filename, 'rb'))\n    pred = gbm.predict(feat[(feat['train_flg']==0) & (feat['Target_' + str(asset_id) + '_return']==feat['Target_' + str(asset_id) + '_return'])][features])\n    valid = feat[(feat['train_flg']==0) & (feat['Target_' + str(asset_id) + '_return']==feat['Target_' + str(asset_id) + '_return'])]['Target_' + str(asset_id) + '_return']\n    #prediction = np.concatenate([prediction, pred.reshape(-1,1)], axis = 1)\n    score = r2_score( np.array(valid), np.array(pred))\n    print(score)\n    score_list.append(score)\nprint(\"test score: \", score_list)\nprint(\"validate: \",oof)","metadata":{"execution":{"iopub.status.busy":"2022-12-09T02:35:08.904586Z","iopub.execute_input":"2022-12-09T02:35:08.904926Z","iopub.status.idle":"2022-12-09T02:35:13.830142Z","shell.execute_reply.started":"2022-12-09T02:35:08.904889Z","shell.execute_reply":"2022-12-09T02:35:13.829244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_res(y_test, lgb_predict, type='test', best =False):\n    res = pd.DataFrame({'y_test': y_test.iloc[:,0][:].tolist(), 'y_predict':lgb_predict})\n    print('score mean: ', res['y_test'].mean(),res['y_predict'].mean())\n    res_df = []\n    for thresh in range(-20,21,1):\n        thresh = thresh/10\n        if thresh <0:\n            target_data = res[res['y_predict']<=thresh]\n        elif thresh>0:\n            target_data = res[res['y_predict']>=thresh]\n        else:\n            target_data = res[(res['y_predict']<0.1) & (res['y_predict']>-0.1)]\n        count = len(target_data)\n        if count ==0:\n            continue\n        predict_avg_return = target_data['y_predict'].mean()\n        test_avg_return = target_data['y_test'].mean()\n        if thresh >=0:\n            winrate = len(target_data[target_data['y_test']>0])/count\n        else:\n            winrate = len(target_data[target_data['y_test']<0])/count\n        res_df.append([thresh,count, predict_avg_return,test_avg_return,winrate])\n        print(thresh,count, predict_avg_return,test_avg_return,winrate)\n    res_df = pd.DataFrame(res_df)\n    res_df.columns = ['thresh','count', 'predict_avg_return','test_avg_return','winrate']\n    res_df['hold_len'] = 64\n    \n    if best:\n        begin = 'best_model_'\n    else:\n        begin = ''\n    \"\"\"\n    if type == 'train':\n    #if False:\n        res_df.to_csv(begin + 'res_' + str(hold_len) + '_train.csv', index=False)\n        print('res train finished')\n    elif type == 'test':\n        res_df.to_csv(begin + 'res_' + str(hold_len) + '_test.csv', index=False)\n        print('res test finished')\n    else:\n        res_df.to_csv(begin + 'res_' + str(hold_len) + '_eval.csv', index=False)\n        print('res eval finished')\n    \"\"\"\nvalid = 100*valid\npred = 100*pred\nget_res(pd.DataFrame(valid), pred, 'test')","metadata":{"execution":{"iopub.status.busy":"2022-12-09T02:38:33.292170Z","iopub.execute_input":"2022-12-09T02:38:33.292479Z","iopub.status.idle":"2022-12-09T02:38:33.493355Z","shell.execute_reply.started":"2022-12-09T02:38:33.292447Z","shell.execute_reply":"2022-12-09T02:38:33.492432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"100*pred","metadata":{"execution":{"iopub.status.busy":"2022-12-09T02:38:26.660937Z","iopub.execute_input":"2022-12-09T02:38:26.661470Z","iopub.status.idle":"2022-12-09T02:38:26.669087Z","shell.execute_reply.started":"2022-12-09T02:38:26.661413Z","shell.execute_reply":"2022-12-09T02:38:26.668136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Retrain with less duplicated data","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import r2_score\nnot_use_overlap_to_train = True\noof = [ [] for id in range(2)   ]\n\nall_oof_pred = []\nall_oof_valid = []\nall_oof_weight = []\n\nfor asset_id, asset_name in zip(df_asset_details['Asset_ID'].tolist()[:2], df_asset_details['Asset_Name'].tolist()[:2]):\n    print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n    oof_pred, oof_valid = get_Xy_and_model_for_asset(feat[(feat.timestamp%(60*8)==0)], asset_id, 8)\n#     print(oof_pred)\n#     print(oof_valid)\n    weight_temp = float( df_asset_details.loc[  df_asset_details['Asset_ID'] == asset_id  , 'Weight'   ]  )\n    \n    all_oof_pred += oof_pred\n    all_oof_valid += oof_valid\n    all_oof_weight += [weight_temp] * len(oof_pred)\n    \n    oof[asset_id] = r2_score( np.array(oof_valid),    np.array(oof_pred)   )\n    \n    print(f'OOF r2 score of {asset_name} (ID={asset_id}) is {oof[asset_id]:.5f}. (Weight: {float(weight_temp):.5f})')\n    print('')\n    print('')","metadata":{"execution":{"iopub.status.busy":"2022-12-09T02:39:14.878649Z","iopub.execute_input":"2022-12-09T02:39:14.879002Z","iopub.status.idle":"2022-12-09T02:41:49.311752Z","shell.execute_reply.started":"2022-12-09T02:39:14.878965Z","shell.execute_reply":"2022-12-09T02:41:49.311051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nscore_list = []\nfor asset_id in range(2):\n    print(asset_id)\n    prediction = np.array([])\n    fold =6\n    filename = 'model_id{}_fold{}_datafreq{}.pkl'.format(asset_id, fold,8)\n    gbm = pickle.load(open(filename, 'rb'))\n    pred = gbm.predict(feat[(feat['train_flg']==0) & (feat['Target_' + str(asset_id) + '_return']==feat['Target_' + str(asset_id) + '_return'])][features])\n    valid = feat[(feat['train_flg']==0) & (feat['Target_' + str(asset_id) + '_return']==feat['Target_' + str(asset_id) + '_return'])]['Target_' + str(asset_id) + '_return']\n    #prediction = np.concatenate([prediction, pred.reshape(-1,1)], axis = 1)\n    score = r2_score( np.array(valid), np.array(pred))\n    print(score)\n    score_list.append(score)\nprint(\"test score: \", score_list)\nprint(\"validate: \",oof)\nvalid = 100*valid\npred = 100*pred\nget_res(pd.DataFrame(valid), pred, 'test')","metadata":{"execution":{"iopub.status.busy":"2022-12-09T02:41:49.313262Z","iopub.execute_input":"2022-12-09T02:41:49.313811Z","iopub.status.idle":"2022-12-09T02:41:56.706362Z","shell.execute_reply.started":"2022-12-09T02:41:49.313775Z","shell.execute_reply":"2022-12-09T02:41:56.705288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Retrain with 32","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import r2_score\nnot_use_overlap_to_train = True\noof = [ [] for id in range(2)   ]\n\nall_oof_pred = []\nall_oof_valid = []\nall_oof_weight = []\n\nfor asset_id, asset_name in zip(df_asset_details['Asset_ID'].tolist()[:2], df_asset_details['Asset_Name'].tolist()[:2]):\n    print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n    oof_pred, oof_valid = get_Xy_and_model_for_asset(feat[&(feat.timestamp%(60*32)==0)], asset_id, 32)\n#     print(oof_pred)\n#     print(oof_valid)\n    weight_temp = float( df_asset_details.loc[  df_asset_details['Asset_ID'] == asset_id  , 'Weight'   ]  )\n    \n    all_oof_pred += oof_pred\n    all_oof_valid += oof_valid\n    all_oof_weight += [weight_temp] * len(oof_pred)\n    \n    oof[asset_id] = r2_score( np.array(oof_valid),    np.array(oof_pred)   )\n    \n    print(f'OOF r2 score of {asset_name} (ID={asset_id}) is {oof[asset_id]:.5f}. (Weight: {float(weight_temp):.5f})')\n    print('')\n    print('')","metadata":{"execution":{"iopub.status.busy":"2022-12-09T02:41:56.707523Z","iopub.execute_input":"2022-12-09T02:41:56.707774Z","iopub.status.idle":"2022-12-09T02:42:58.032878Z","shell.execute_reply.started":"2022-12-09T02:41:56.707741Z","shell.execute_reply":"2022-12-09T02:42:58.031644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nscore_list = []\nfor asset_id in range(2):\n    print(asset_id)\n    prediction = np.array([])\n    fold =6\n    filename = 'model_id{}_fold{}_datafreq{}.pkl'.format(asset_id, fold,32)\n    gbm = pickle.load(open(filename, 'rb'))\n    pred = gbm.predict(feat[(feat['train_flg']==0) & (feat['Target_' + str(asset_id) + '_return']==feat['Target_' + str(asset_id) + '_return'])][features])\n    valid = feat[(feat['train_flg']==0) & (feat['Target_' + str(asset_id) + '_return']==feat['Target_' + str(asset_id) + '_return'])]['Target_' + str(asset_id) + '_return']\n    #prediction = np.concatenate([prediction, pred.reshape(-1,1)], axis = 1)\n    score = r2_score( np.array(valid), np.array(pred))\n    print(score)\n    score_list.append(score)\nprint(\"test score: \", score_list)\nprint(\"validate: \",oof)\nvalid = 100*valid\npred = 100*pred\nget_res(pd.DataFrame(valid), pred, 'test')","metadata":{"execution":{"iopub.status.busy":"2022-12-09T02:42:58.035833Z","iopub.execute_input":"2022-12-09T02:42:58.036805Z","iopub.status.idle":"2022-12-09T02:43:09.932630Z","shell.execute_reply.started":"2022-12-09T02:42:58.036754Z","shell.execute_reply":"2022-12-09T02:43:09.931570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nscore_list = []\nfor asset_id in range(2):\n    print(asset_id)\n    prediction = np.array([])\n    fold =6\n    filename = 'model_id{}_fold{}_datafreq{}.pkl'.format(asset_id, fold,32)\n    gbm = pickle.load(open(filename, 'rb'))\n    pred = gbm.predict(feat[(feat['train_flg']==0) & (feat['Target_' + str(asset_id) + '_return']==feat['Target_' + str(asset_id) + '_return'])][features])\n    valid = feat[(feat['train_flg']==0) & (feat['Target_' + str(asset_id) + '_return']==feat['Target_' + str(asset_id) + '_return'])]['Target_' + str(asset_id) + '_return']\n    #prediction = np.concatenate([prediction, pred.reshape(-1,1)], axis = 1)\n    score = r2_score( np.array(valid), np.array(pred))\n    print(score)\n    score_list.append(score)\nprint(\"test score: \", score_list)\nprint(\"validate: \",oof)\nvalid = 1000*valid\npred = 1000*pred\nget_res(pd.DataFrame(valid), pred, 'test')","metadata":{"execution":{"iopub.status.busy":"2022-12-09T02:52:10.065110Z","iopub.execute_input":"2022-12-09T02:52:10.065423Z","iopub.status.idle":"2022-12-09T02:52:21.774883Z","shell.execute_reply.started":"2022-12-09T02:52:10.065391Z","shell.execute_reply":"2022-12-09T02:52:21.773721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Retrain with No Overlap","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import r2_score\nnot_use_overlap_to_train = True\noof = [ [] for id in range(2)   ]\n\nall_oof_pred = []\nall_oof_valid = []\nall_oof_weight = []\n\nfor asset_id, asset_name in zip(df_asset_details['Asset_ID'].tolist()[:2], df_asset_details['Asset_Name'].tolist()[:2]):\n    print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n    oof_pred, oof_valid = get_Xy_and_model_for_asset(feat[(feat.timestamp%(60*64)==0)], asset_id, 64)\n#     print(oof_pred)\n#     print(oof_valid)\n    weight_temp = float( df_asset_details.loc[  df_asset_details['Asset_ID'] == asset_id  , 'Weight'   ]  )\n    \n    all_oof_pred += oof_pred\n    all_oof_valid += oof_valid\n    all_oof_weight += [weight_temp] * len(oof_pred)\n    \n    oof[asset_id] = r2_score( np.array(oof_valid),    np.array(oof_pred)   )\n    \n    print(f'OOF r2 score of {asset_name} (ID={asset_id}) is {oof[asset_id]:.5f}. (Weight: {float(weight_temp):.5f})')\n    print('')\n    print('')","metadata":{"execution":{"iopub.status.busy":"2022-12-09T02:43:09.934197Z","iopub.execute_input":"2022-12-09T02:43:09.934614Z","iopub.status.idle":"2022-12-09T02:43:47.393378Z","shell.execute_reply.started":"2022-12-09T02:43:09.934575Z","shell.execute_reply":"2022-12-09T02:43:47.392116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nscore_list = []\nfor asset_id in range(2):\n    print(asset_id)\n    prediction = np.array([])\n    fold =6\n    filename = 'model_id{}_fold{}_datafreq{}.pkl'.format(asset_id, fold,64)\n    gbm = pickle.load(open(filename, 'rb'))\n    pred = gbm.predict(feat[(feat['train_flg']==0) & (feat['Target_' + str(asset_id) + '_return']==feat['Target_' + str(asset_id) + '_return'])][features])\n    valid = feat[(feat['train_flg']==0) & (feat['Target_' + str(asset_id) + '_return']==feat['Target_' + str(asset_id) + '_return'])]['Target_' + str(asset_id) + '_return']\n    #prediction = np.concatenate([prediction, pred.reshape(-1,1)], axis = 1)\n    score = r2_score( np.array(valid), np.array(pred))\n    print(score)\n    score_list.append(score)\nprint(\"test score: \", score_list)\nprint(\"validate: \",oof)\nvalid = 100*valid\npred = 100*pred\nget_res(pd.DataFrame(valid), pred, 'test')","metadata":{"execution":{"iopub.status.busy":"2022-12-09T02:43:47.394826Z","iopub.execute_input":"2022-12-09T02:43:47.395063Z","iopub.status.idle":"2022-12-09T02:43:59.760951Z","shell.execute_reply.started":"2022-12-09T02:43:47.395033Z","shell.execute_reply":"2022-12-09T02:43:59.759730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nscore_list = []\nfor asset_id in range(2):\n    print(asset_id)\n    prediction = np.array([])\n    fold =6\n    filename = 'model_id{}_fold{}_datafreq{}.pkl'.format(asset_id, fold,64)\n    gbm = pickle.load(open(filename, 'rb'))\n    pred = gbm.predict(feat[(feat['train_flg']==0) & (feat['Target_' + str(asset_id) + '_return']==feat['Target_' + str(asset_id) + '_return'])][features])\n    valid = feat[(feat['train_flg']==0) & (feat['Target_' + str(asset_id) + '_return']==feat['Target_' + str(asset_id) + '_return'])]['Target_' + str(asset_id) + '_return']\n    #prediction = np.concatenate([prediction, pred.reshape(-1,1)], axis = 1)\n    score = r2_score( np.array(valid), np.array(pred))\n    print(score)\n    score_list.append(score)\nprint(\"test score: \", score_list)\nprint(\"validate: \",oof)\nvalid = 1000*valid\npred = 1000*pred\nget_res(pd.DataFrame(valid), pred, 'test')","metadata":{"execution":{"iopub.status.busy":"2022-12-09T03:02:11.765736Z","iopub.execute_input":"2022-12-09T03:02:11.766164Z","iopub.status.idle":"2022-12-09T03:02:24.687374Z","shell.execute_reply.started":"2022-12-09T03:02:11.766122Z","shell.execute_reply":"2022-12-09T03:02:24.686311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Def Self Metric to Check the Results","metadata":{}},{"cell_type":"code","source":"def get_res(y_test, lgb_predict, type='test', best =False):\n    res = pd.DataFrame({'y_test': y_test.iloc[:,0][:].tolist(), 'y_predict':lgb_predict})\n    print('score mean: ', res['y_test'].mean(),res['y_predict'].mean())\n    res_df = []\n    for thresh in range(-20,21,1):\n        thresh = thresh/10\n        if thresh <0:\n            target_data = res[res['y_predict']<=thresh]\n        elif thresh>0:\n            target_data = res[res['y_predict']>=thresh]\n        else:\n            target_data = res[(res['y_predict']<0.1) & (res['y_predict']>-0.1)]\n        count = len(target_data)\n        if count ==0:\n            continue\n        predict_avg_return = target_data['y_predict'].mean()\n        test_avg_return = target_data['y_test'].mean()\n        if thresh >=0:\n            winrate = len(target_data[target_data['y_test']>0])/count\n        else:\n            winrate = len(target_data[target_data['y_test']<0])/count\n        res_df.append([thresh,count, predict_avg_return,test_avg_return,winrate])\n        print(thresh,count, predict_avg_return,test_avg_return,winrate)\n    res_df = pd.DataFrame(res_df)\n    res_df.columns = ['thresh','count', 'predict_avg_return','test_avg_return','winrate']\n    res_df['hold_len'] = hold_len\n    \n    if best:\n        begin = 'best_model_'\n    else:\n        begin = ''\n    if type == 'train':\n    #if False:\n        res_df.to_csv(begin + 'res_' + str(hold_len) + '_train.csv', index=False)\n        print('res train finished')\n    elif type == 'test':\n        res_df.to_csv(begin + 'res_' + str(hold_len) + '_test.csv', index=False)\n        print('res test finished')\n    else:\n        res_df.to_csv(begin + 'res_' + str(hold_len) + '_eval.csv', index=False)\n        print('res eval finished')","metadata":{"execution":{"iopub.status.busy":"2022-12-09T02:12:45.606271Z","iopub.execute_input":"2022-12-09T02:12:45.606729Z","iopub.status.idle":"2022-12-09T02:12:45.646759Z","shell.execute_reply.started":"2022-12-09T02:12:45.606607Z","shell.execute_reply":"2022-12-09T02:12:45.645701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_res(valid, pred, 'test')","metadata":{"execution":{"iopub.status.busy":"2022-12-09T02:13:45.427445Z","iopub.execute_input":"2022-12-09T02:13:45.427744Z","iopub.status.idle":"2022-12-09T02:13:45.497802Z","shell.execute_reply.started":"2022-12-09T02:13:45.427714Z","shell.execute_reply":"2022-12-09T02:13:45.496577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"# 1622530800\nfor asset_id in range(14):\n    print(asset_id)\n    prediction = np.array([])\n    fold =2\n    #for fold in range(3):\n        filename = 'model_id{}_fold{}.pkl'.format(asset_id, fold)\n        gbm = pickle.load(open(filename, 'rb'))\n        pred = gbm.predict(feat[feat.timestamp<1622530800][feat.timestamp>1600000800][features])\n        if fold==0:\n            prediction = pred.reshape(-1,1)\n        else:\n            prediction = np.concatenate([prediction, pred.reshape(-1,1)], axis = 1)\n        score = corr_score(np.array(pred), np.array(feat[feat.timestamp<1622530800][feat.timestamp>1600000800]['Target_{}'.format(asset_id)].fillna(0)))\n        print(score)\n#     score = corr_score(np.mean(prediction, axis=1), np.array(feat[feat.timestamp<1622530800][feat.timestamp>1600000800]['Target_{}'.format(fold)].fillna(0)))\n#     print(score)","metadata":{"execution":{"iopub.status.busy":"2022-11-25T20:04:37.301995Z","iopub.execute_input":"2022-11-25T20:04:37.30229Z","iopub.status.idle":"2022-11-25T20:04:56.266673Z","shell.execute_reply.started":"2022-11-25T20:04:37.302259Z","shell.execute_reply":"2022-11-25T20:04:56.265485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# corr_score(np.array(pred), np.array(feat[feat.timestamp>1614585600]['Target_0'].fillna(0)))","metadata":{"execution":{"iopub.status.busy":"2022-11-25T19:30:40.73701Z","iopub.execute_input":"2022-11-25T19:30:40.737916Z","iopub.status.idle":"2022-11-25T19:30:40.742485Z","shell.execute_reply.started":"2022-11-25T19:30:40.737869Z","shell.execute_reply":"2022-11-25T19:30:40.741218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}