{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Tabular Modelling - Choosing a Model\n- start with heuristic\n- linear regression\n- gradient boosting\n- neural network","metadata":{}},{"cell_type":"code","source":"import os\nimport random\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport lightgbm as lgb\nimport gresearch_crypto\nimport time\nimport datetime\n\nimport pickle\nimport gc\n\nfrom tqdm import tqdm\n\nn_fold = 5\nseed0 = 8586\nuse_supple_for_train = True\n\n# If True, the period used to evaluate Public LB will not be used for training.\n# Set to False on final submission.\nnot_use_overlap_to_train = False\n\nTRAIN_CSV = '/kaggle/input/g-research-crypto-forecasting/train.csv'\nSUPPLE_TRAIN_CSV = '/kaggle/input/g-research-crypto-forecasting/supplemental_train.csv'\nASSET_DETAILS_CSV = '/kaggle/input/g-research-crypto-forecasting/asset_details.csv'\n\npd.set_option('display.max_rows', 6)\npd.set_option('display.max_columns', 350)","metadata":{"execution":{"iopub.status.busy":"2023-05-16T06:37:15.721612Z","iopub.execute_input":"2023-05-16T06:37:15.721857Z","iopub.status.idle":"2023-05-16T06:37:17.787480Z","shell.execute_reply.started":"2023-05-16T06:37:15.721831Z","shell.execute_reply":"2023-05-16T06:37:17.786521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n\n    for col in df.columns:\n        col_type = df[col].dtype\n\n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-05-16T06:37:17.789634Z","iopub.execute_input":"2023-05-16T06:37:17.790066Z","iopub.status.idle":"2023-05-16T06:37:17.803689Z","shell.execute_reply.started":"2023-05-16T06:37:17.790030Z","shell.execute_reply":"2023-05-16T06:37:17.802400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset_details = pd.read_csv(ASSET_DETAILS_CSV).sort_values(\"Asset_ID\")\ndf_asset_details","metadata":{"execution":{"iopub.status.busy":"2023-05-16T06:37:17.805453Z","iopub.execute_input":"2023-05-16T06:37:17.805751Z","iopub.status.idle":"2023-05-16T06:37:17.856516Z","shell.execute_reply.started":"2023-05-16T06:37:17.805717Z","shell.execute_reply":"2023-05-16T06:37:17.855316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset_details[df_asset_details.Asset_ID.isin([1,6,4])]","metadata":{"execution":{"iopub.status.busy":"2023-05-16T06:37:17.857654Z","iopub.execute_input":"2023-05-16T06:37:17.857850Z","iopub.status.idle":"2023-05-16T06:37:17.871021Z","shell.execute_reply.started":"2023-05-16T06:37:17.857821Z","shell.execute_reply":"2023-05-16T06:37:17.870539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"assets=[1,6]","metadata":{"execution":{"iopub.status.busy":"2023-05-16T06:37:17.872835Z","iopub.execute_input":"2023-05-16T06:37:17.873475Z","iopub.status.idle":"2023-05-16T06:37:17.882789Z","shell.execute_reply.started":"2023-05-16T06:37:17.873437Z","shell.execute_reply":"2023-05-16T06:37:17.881934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf_train = pd.read_csv(TRAIN_CSV)\n\nif use_supple_for_train:    \n    df_supple = pd.read_csv(SUPPLE_TRAIN_CSV)\n#     display(df_supple)\n    df_train = pd.concat([df_train, df_supple])\n    del df_supple\ndf_train = reduce_mem_usage(df_train)\n# df_train\n\ndf_train = df_train[df_train.Asset_ID.isin(assets)]","metadata":{"execution":{"iopub.status.busy":"2023-05-16T06:37:17.883903Z","iopub.execute_input":"2023-05-16T06:37:17.884194Z","iopub.status.idle":"2023-05-16T06:38:16.143048Z","shell.execute_reply.started":"2023-05-16T06:37:17.884164Z","shell.execute_reply":"2023-05-16T06:38:16.142121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.columns","metadata":{"execution":{"iopub.status.busy":"2023-05-16T06:38:16.144307Z","iopub.execute_input":"2023-05-16T06:38:16.144563Z","iopub.status.idle":"2023-05-16T06:38:16.149711Z","shell.execute_reply.started":"2023-05-16T06:38:16.144533Z","shell.execute_reply":"2023-05-16T06:38:16.148941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[['Close','VWAP','Count','Volume']].corr()","metadata":{"execution":{"iopub.status.busy":"2023-05-16T06:38:16.150620Z","iopub.execute_input":"2023-05-16T06:38:16.150791Z","iopub.status.idle":"2023-05-16T06:38:16.388001Z","shell.execute_reply.started":"2023-05-16T06:38:16.150769Z","shell.execute_reply":"2023-05-16T06:38:16.387046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_merged = pd.DataFrame()\ntrain_merged[df_train.columns] = 0\nfor id in tqdm(assets):\n    train_merged = train_merged.merge(df_train.loc[df_train[\"Asset_ID\"] == id, ['timestamp', 'Close','Target','Volume','Count']].copy(), on=\"timestamp\", how='outer',suffixes=['', \"_\"+str(id)])\n        \ntrain_merged = train_merged.drop(df_train.columns.drop(\"timestamp\"), axis=1)\ntrain_merged = train_merged.sort_values('timestamp', ascending=True)\ndisplay(train_merged.head())","metadata":{"execution":{"iopub.status.busy":"2023-05-16T06:38:16.389261Z","iopub.execute_input":"2023-05-16T06:38:16.389452Z","iopub.status.idle":"2023-05-16T06:38:18.483604Z","shell.execute_reply.started":"2023-05-16T06:38:16.389429Z","shell.execute_reply":"2023-05-16T06:38:18.482792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# forward fill\n# Set an upper limit on the number of fills, since there may be long term gaps.\nfor id in assets:\n#     print(id, train_merged[f'Close_{id}'].isnull().sum())   # Number of missing before forward fill\n    train_merged[[f'Close_{id}',f'Volume_{id}',f'Count_{id}']] = train_merged[[f'Close_{id}',f'Volume_{id}',f'Count_{id}']].fillna(method='ffill', limit=100)\n#     print(id, train_merged[f'Close_{id}'].isnull().sum())   # Number of missing after forward fill","metadata":{"execution":{"iopub.status.busy":"2023-05-16T06:38:18.484478Z","iopub.execute_input":"2023-05-16T06:38:18.484645Z","iopub.status.idle":"2023-05-16T06:38:18.523317Z","shell.execute_reply.started":"2023-05-16T06:38:18.484622Z","shell.execute_reply":"2023-05-16T06:38:18.522663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train_merged.head())","metadata":{"execution":{"iopub.status.busy":"2023-05-16T06:38:18.524209Z","iopub.execute_input":"2023-05-16T06:38:18.524369Z","iopub.status.idle":"2023-05-16T06:38:18.536673Z","shell.execute_reply.started":"2023-05-16T06:38:18.524347Z","shell.execute_reply":"2023-05-16T06:38:18.536024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lags = [60] #,300,900","metadata":{"execution":{"iopub.status.busy":"2023-05-16T06:38:18.537591Z","iopub.execute_input":"2023-05-16T06:38:18.537780Z","iopub.status.idle":"2023-05-16T06:38:18.546988Z","shell.execute_reply.started":"2023-05-16T06:38:18.537752Z","shell.execute_reply":"2023-05-16T06:38:18.546438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_features(df, train=True):   \n    if train == True:\n        totimestamp = lambda s: np.int32(time.mktime(datetime.datetime.strptime(s, \"%d/%m/%Y\").timetuple()))\n        valid_window = [totimestamp(\"12/03/2021\")]\n#         valid_window = [totimestamp(\"15/08/2021\")]  #検証用\n        df['train_flg'] = np.where(df['timestamp']>=valid_window[0], 0,1)\n\n        supple_start_window = [totimestamp(\"22/09/2021\")]\n        if use_supple_for_train:\n            df['train_flg'] = np.where(df['timestamp']>=supple_start_window[0], 1 ,df['train_flg']  )\n   \n    for id in assets:    \n        for lag in lags:\n            df[f'log_close/mean_{lag}_id{id}'] = np.log( np.array(df[f'Close_{id}']) /  np.roll(np.append(np.convolve( np.array(df[f'Close_{id}']), np.ones(lag)/lag, mode=\"valid\"), np.ones(lag-1)), lag-1)  )\n            df[f'log_return_{lag}_id{id}']     = np.log( np.array(df[f'Close_{id}']) /  np.roll(np.array(df[f'Close_{id}']), lag)  )\n    for lag in lags:\n        df[f'mean_close/mean_{lag}'] =  np.mean(df.iloc[:,df.columns.str.startswith(f'log_close/mean_{lag}_id')], axis=1)\n        df[f'mean_log_returns_{lag}'] = np.mean(df.iloc[:,df.columns.str.startswith(f'log_return_{lag}_id')] ,    axis=1)\n        for id in assets:\n            df[f'log_close/mean_{lag}-mean_close/mean_{lag}_id{id}'] = np.array( df[f'log_close/mean_{lag}_id{id}']) - np.array( df[f'mean_close/mean_{lag}']  )\n            df[f'log_return_{lag}-mean_log_returns_{lag}_id{id}']    = np.array( df[f'log_return_{lag}_id{id}'])     - np.array( df[f'mean_log_returns_{lag}'] )\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-05-16T06:38:18.547703Z","iopub.execute_input":"2023-05-16T06:38:18.547865Z","iopub.status.idle":"2023-05-16T06:38:18.561740Z","shell.execute_reply.started":"2023-05-16T06:38:18.547843Z","shell.execute_reply":"2023-05-16T06:38:18.560892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfeat = get_features(train_merged)\nfeat","metadata":{"execution":{"iopub.status.busy":"2023-05-16T06:38:18.563856Z","iopub.execute_input":"2023-05-16T06:38:18.564066Z","iopub.status.idle":"2023-05-16T06:38:18.857124Z","shell.execute_reply.started":"2023-05-16T06:38:18.564040Z","shell.execute_reply":"2023-05-16T06:38:18.856425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define features for LGBM\nnot_use_features_train = ['timestamp', 'train_flg']\nfor id in assets:\n    not_use_features_train.append(f'Target_{id}')\n\nfeatures = feat.columns \nfeatures = features.drop(not_use_features_train)\nfeatures = list(features)\n# display(features)  \nlen(features)","metadata":{"execution":{"iopub.status.busy":"2023-05-16T06:38:18.858000Z","iopub.execute_input":"2023-05-16T06:38:18.858220Z","iopub.status.idle":"2023-05-16T06:38:18.866789Z","shell.execute_reply.started":"2023-05-16T06:38:18.858190Z","shell.execute_reply":"2023-05-16T06:38:18.865410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features","metadata":{"execution":{"iopub.status.busy":"2023-05-16T06:38:18.868012Z","iopub.execute_input":"2023-05-16T06:38:18.868262Z","iopub.status.idle":"2023-05-16T06:38:18.878406Z","shell.execute_reply.started":"2023-05-16T06:38:18.868233Z","shell.execute_reply":"2023-05-16T06:38:18.877526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_merged\ndel df_train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-05-16T06:38:18.879817Z","iopub.execute_input":"2023-05-16T06:38:18.880043Z","iopub.status.idle":"2023-05-16T06:38:19.006619Z","shell.execute_reply.started":"2023-05-16T06:38:18.880013Z","shell.execute_reply":"2023-05-16T06:38:19.005792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define the evaluation metric\ndef correlation(a, train_data):\n    \n    b = train_data.get_label()\n    \n    a = np.ravel(a)\n    b = np.ravel(b)\n\n    len_data = len(a)\n    mean_a = np.sum(a) / len_data\n    mean_b = np.sum(b) / len_data\n    var_a = np.sum(np.square(a - mean_a)) / len_data\n    var_b = np.sum(np.square(b - mean_b)) / len_data\n\n    cov = np.sum((a * b))/len_data - mean_a*mean_b\n    corr = cov / np.sqrt(var_a * var_b)\n\n    return 'corr', corr, True\n\n# For CV score calculation\ndef corr_score(pred, valid):\n    len_data = len(pred)\n    mean_pred = np.sum(pred) / len_data\n    mean_valid = np.sum(valid) / len_data\n    var_pred = np.sum(np.square(pred - mean_pred)) / len_data\n    var_valid = np.sum(np.square(valid - mean_valid)) / len_data\n\n    cov = np.sum((pred * valid))/len_data - mean_pred*mean_valid\n    corr = cov / np.sqrt(var_pred * var_valid)\n\n    return corr","metadata":{"execution":{"iopub.status.busy":"2023-05-16T06:38:19.007803Z","iopub.execute_input":"2023-05-16T06:38:19.008024Z","iopub.status.idle":"2023-05-16T06:38:19.017531Z","shell.execute_reply.started":"2023-05-16T06:38:19.008000Z","shell.execute_reply":"2023-05-16T06:38:19.016687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from: https://blog.amedama.jp/entry/lightgbm-cv-feature-importance\n# (used in nyanp's Optiver solution)\ndef plot_importance(importances, features_names = features, PLOT_TOP_N = 20, figsize=(10, 10)):\n    importance_df = pd.DataFrame(data=importances, columns=features)\n    sorted_indices = importance_df.median(axis=0).sort_values(ascending=False).index\n    sorted_importance_df = importance_df.loc[:, sorted_indices]\n    plot_cols = sorted_importance_df.columns[:PLOT_TOP_N]\n    _, ax = plt.subplots(figsize=figsize)\n    ax.grid()\n    ax.set_xscale('log')\n    ax.set_ylabel('Feature')\n    ax.set_xlabel('Importance')\n    sns.boxplot(data=sorted_importance_df[plot_cols],\n                orient='h',\n                ax=ax)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-16T06:38:19.018494Z","iopub.execute_input":"2023-05-16T06:38:19.018715Z","iopub.status.idle":"2023-05-16T06:38:19.030923Z","shell.execute_reply.started":"2023-05-16T06:38:19.018687Z","shell.execute_reply":"2023-05-16T06:38:19.030126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from: https://www.kaggle.com/code/nrcjea001/lgbm-embargocv-weightedpearson-lagtarget/\ndef get_time_series_cross_val_splits(data, cv = n_fold, embargo = 3750):\n    all_train_timestamps = data['timestamp'].unique()\n    len_split = len(all_train_timestamps) // cv\n    test_splits = [all_train_timestamps[i * len_split:(i + 1) * len_split] for i in range(cv)]\n    # fix the last test split to have all the last timestamps, in case the number of timestamps wasn't divisible by cv\n    rem = len(all_train_timestamps) - len_split*cv\n    if rem>0:\n        test_splits[-1] = np.append(test_splits[-1], all_train_timestamps[-rem:])\n\n    train_splits = []\n    for test_split in test_splits:\n        test_split_max = int(np.max(test_split))\n        test_split_min = int(np.min(test_split))\n        # get all of the timestamps that aren't in the test split\n        train_split_not_embargoed = [e for e in all_train_timestamps if not (test_split_min <= int(e) <= test_split_max)]\n        # embargo the train split so we have no leakage. Note timestamps are expressed in seconds, so multiply by 60\n        embargo_sec = 60*embargo\n        train_split = [e for e in train_split_not_embargoed if\n                       abs(int(e) - test_split_max) > embargo_sec and abs(int(e) - test_split_min) > embargo_sec]\n        train_splits.append(train_split)\n\n    # convenient way to iterate over train and test splits\n    train_test_zip = zip(train_splits, test_splits)\n    return train_test_zip","metadata":{"execution":{"iopub.status.busy":"2023-05-16T06:38:19.031844Z","iopub.execute_input":"2023-05-16T06:38:19.032095Z","iopub.status.idle":"2023-05-16T06:38:19.042637Z","shell.execute_reply.started":"2023-05-16T06:38:19.032064Z","shell.execute_reply":"2023-05-16T06:38:19.042001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model - Linear Model","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LinearRegression","metadata":{"execution":{"iopub.status.busy":"2023-05-16T06:38:19.043510Z","iopub.execute_input":"2023-05-16T06:38:19.043729Z","iopub.status.idle":"2023-05-16T06:38:19.124710Z","shell.execute_reply.started":"2023-05-16T06:38:19.043698Z","shell.execute_reply":"2023-05-16T06:38:19.124203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_Xy_and_model_for_asset(df_proc, asset_id):\n    df_proc = df_proc.loc[  (df_proc[f'Target_{asset_id}'] == df_proc[f'Target_{asset_id}'])  ]\n    \n    #only needed for linear regression because cannot handle NAs\n    df_proc.fillna(0,inplace=True) \n    \n    \n    if not_use_overlap_to_train: df_proc = df_proc.loc[  (df_proc['train_flg'] == 1)  ]\n    \n    # EmbargoCV\n    train_test_zip = get_time_series_cross_val_splits(df_proc, cv = n_fold, embargo = 3750)\n    print(\"entering time series cross validation loop\")\n    importances, oof_pred, oof_valid = [], [], []\n\n    for split, train_test_split in enumerate(train_test_zip):\n        gc.collect()\n        print(f\"doing split {split+1} out of {n_fold}\")\n        train_split, test_split = train_test_split\n        train_split_index = df_proc['timestamp'].isin(train_split)\n        test_split_index = df_proc['timestamp'].isin(test_split)\n        \n        model = LinearRegression().fit(df_proc.loc[train_split_index, features],\n                                    df_proc.loc[train_split_index, f'Target_{asset_id}'].values)\n        \n            \n        oof_pred += list(  model.predict(df_proc.loc[test_split_index, features])        )\n        oof_valid += list(   df_proc.loc[test_split_index, f'Target_{asset_id}'].values    )\n\n    return oof_pred, oof_valid","metadata":{"execution":{"iopub.status.busy":"2023-05-16T06:40:11.293908Z","iopub.execute_input":"2023-05-16T06:40:11.294578Z","iopub.status.idle":"2023-05-16T06:40:11.306415Z","shell.execute_reply.started":"2023-05-16T06:40:11.294526Z","shell.execute_reply":"2023-05-16T06:40:11.305315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof = {x:0 for x in assets}\n\nall_oof_pred = []\nall_oof_valid = []\n\nfor asset_id in assets:\n    print(f\"Training model for (ID={asset_id:<2})\")\n    \n    oof_pred, oof_valid = get_Xy_and_model_for_asset(feat, asset_id)\n    \n    all_oof_pred += oof_pred\n    all_oof_valid += oof_valid\n    \n    oof[asset_id] = corr_score(     np.array(oof_pred)   ,    np.array(oof_valid)    )\n    \n    print(f'OOF corr score of (ID={asset_id}) is {oof[asset_id]:.5f}.')\n    print('')\n    print('')","metadata":{"execution":{"iopub.status.busy":"2023-05-16T06:38:19.134327Z","iopub.execute_input":"2023-05-16T06:38:19.134514Z","iopub.status.idle":"2023-05-16T06:38:49.030244Z","shell.execute_reply.started":"2023-05-16T06:38:19.134489Z","shell.execute_reply":"2023-05-16T06:38:49.029396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"avg_values = sum(oof.values()) / len(oof)\nprint(f'  simple average corr score: {avg_values:.5f}.')","metadata":{"execution":{"iopub.status.busy":"2023-05-16T01:46:57.715007Z","iopub.execute_input":"2023-05-16T01:46:57.715623Z","iopub.status.idle":"2023-05-16T01:46:57.723615Z","shell.execute_reply.started":"2023-05-16T01:46:57.715547Z","shell.execute_reply":"2023-05-16T01:46:57.722672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"avg_values = sum(oof.values()) / len(oof)\nprint(f'  simple average corr score: {avg_values:.5f}.')","metadata":{"execution":{"iopub.status.busy":"2023-05-16T01:46:57.724883Z","iopub.execute_input":"2023-05-16T01:46:57.725168Z","iopub.status.idle":"2023-05-16T01:46:57.743028Z","shell.execute_reply.started":"2023-05-16T01:46:57.725132Z","shell.execute_reply":"2023-05-16T01:46:57.741927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model - LGBM","metadata":{}},{"cell_type":"code","source":"params = {\n    'early_stopping_rounds': 50,\n    'objective': 'regression',\n    'metric': 'rmse',\n    'boosting_type': 'gbdt',\n    'max_depth': 5,\n    'verbose': -1,\n    'lambda_l1': 0.5,\n    'lambda_l2': 2}","metadata":{"execution":{"iopub.status.busy":"2023-05-11T00:34:53.915894Z","iopub.execute_input":"2023-05-11T00:34:53.916436Z","iopub.status.idle":"2023-05-11T00:34:53.921193Z","shell.execute_reply.started":"2023-05-11T00:34:53.916397Z","shell.execute_reply":"2023-05-11T00:34:53.920322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_Xy_and_model_for_asset(df_proc, asset_id):\n    df_proc = df_proc.loc[  (df_proc[f'Target_{asset_id}'] == df_proc[f'Target_{asset_id}'])  ]\n    if not_use_overlap_to_train:\n        df_proc = df_proc.loc[  (df_proc['train_flg'] == 1)  ]\n    \n# EmbargoCV\n    train_test_zip = get_time_series_cross_val_splits(df_proc, cv = n_fold, embargo = 3750)\n    print(\"entering time series cross validation loop\")\n    importances = []\n    oof_pred = []\n    oof_valid = []\n    \n    for split, train_test_split in enumerate(train_test_zip):\n        gc.collect()\n        \n        print(f\"doing split {split+1} out of {n_fold}\")\n        train_split, test_split = train_test_split\n        train_split_index = df_proc['timestamp'].isin(train_split)\n        test_split_index = df_proc['timestamp'].isin(test_split)\n    \n        train_dataset = lgb.Dataset(df_proc.loc[train_split_index, features],\n                                    df_proc.loc[train_split_index, f'Target_{asset_id}'].values, \n                                    feature_name = features, \n                                   )\n        val_dataset = lgb.Dataset(df_proc.loc[test_split_index, features], \n                                  df_proc.loc[test_split_index, f'Target_{asset_id}'].values, \n                                  feature_name = features, \n                                 )\n\n        model = lgb.train(params = params,\n                          train_set = train_dataset, \n                          valid_sets=[train_dataset, val_dataset],\n                          valid_names=['tr', 'vl'],\n                          num_boost_round = 1000,\n                          verbose_eval = 200,     \n                          feval = correlation,\n                         )\n        importances.append(model.feature_importance(importance_type='gain'))\n        \n        oof_pred += list(  model.predict(df_proc.loc[test_split_index, features])        )\n        oof_valid += list(   df_proc.loc[test_split_index, f'Target_{asset_id}'].values    )\n    \n    \n    plot_importance(np.array(importances),features, PLOT_TOP_N = 20, figsize=(10, 5))\n\n    return oof_pred, oof_valid","metadata":{"execution":{"iopub.status.busy":"2023-05-11T00:34:54.665722Z","iopub.execute_input":"2023-05-11T00:34:54.666044Z","iopub.status.idle":"2023-05-11T00:34:54.681260Z","shell.execute_reply.started":"2023-05-11T00:34:54.666010Z","shell.execute_reply":"2023-05-11T00:34:54.680223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof = {x:0 for x in assets}\n\nall_oof_pred = []\nall_oof_valid = []\n\nfor asset_id in assets:\n    print(f\"Training model for (ID={asset_id:<2})\")\n    \n    oof_pred, oof_valid = get_Xy_and_model_for_asset(feat, asset_id)\n    \n    all_oof_pred += oof_pred\n    all_oof_valid += oof_valid\n    \n    oof[asset_id] = corr_score(     np.array(oof_pred)   ,    np.array(oof_valid)    )\n    \n    print(f'OOF corr score of (ID={asset_id}) is {oof[asset_id]:.5f}.')\n    print('')\n    print('')","metadata":{"execution":{"iopub.status.busy":"2023-05-11T00:34:54.791115Z","iopub.execute_input":"2023-05-11T00:34:54.791401Z","iopub.status.idle":"2023-05-11T00:40:46.970251Z","shell.execute_reply.started":"2023-05-11T00:34:54.791369Z","shell.execute_reply":"2023-05-11T00:40:46.969101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"avg_values = sum(oof.values()) / len(oof)\nprint(f'  simple average corr score: {avg_values:.5f}.')","metadata":{"execution":{"iopub.status.busy":"2023-05-11T00:40:46.972245Z","iopub.execute_input":"2023-05-11T00:40:46.972512Z","iopub.status.idle":"2023-05-11T00:40:46.978295Z","shell.execute_reply.started":"2023-05-11T00:40:46.972478Z","shell.execute_reply":"2023-05-11T00:40:46.977363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model - Neural Network","metadata":{}},{"cell_type":"code","source":"!pip install -Uqq pytorch-tabnet","metadata":{"execution":{"iopub.status.busy":"2023-05-16T06:40:20.263073Z","iopub.execute_input":"2023-05-16T06:40:20.263350Z","iopub.status.idle":"2023-05-16T06:40:29.238017Z","shell.execute_reply.started":"2023-05-16T06:40:20.263321Z","shell.execute_reply":"2023-05-16T06:40:29.237350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pytorch_tabnet.tab_model import TabNetRegressor","metadata":{"execution":{"iopub.status.busy":"2023-05-16T06:40:29.239922Z","iopub.execute_input":"2023-05-16T06:40:29.240222Z","iopub.status.idle":"2023-05-16T06:40:30.327560Z","shell.execute_reply.started":"2023-05-16T06:40:29.240193Z","shell.execute_reply":"2023-05-16T06:40:30.326807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_Xy_and_model_for_asset(df_proc, asset_id):\n    df_proc = df_proc.loc[  (df_proc[f'Target_{asset_id}'] == df_proc[f'Target_{asset_id}'])  ]\n    \n    #for linear/neural network\n    df_proc.fillna(0,inplace=True)  \n    \n    if not_use_overlap_to_train:\n        df_proc = df_proc.loc[  (df_proc['train_flg'] == 1)  ]\n    \n# EmbargoCV\n    train_test_zip = get_time_series_cross_val_splits(df_proc, cv = n_fold, embargo = 3750)\n    print(\"entering time series cross validation loop\")\n    importances = []\n    oof_pred = []\n    oof_valid = []\n    \n    for split, train_test_split in enumerate(train_test_zip):\n        gc.collect()\n        \n        print(f\"doing split {split+1} out of {n_fold}\")\n        train_split, test_split = train_test_split\n        train_split_index = df_proc['timestamp'].isin(train_split)\n        test_split_index = df_proc['timestamp'].isin(test_split)\n        \n        model = TabNetRegressor()\n        model.fit(X_train=df_proc.loc[train_split_index, features].values,\n                  y_train=df_proc.loc[train_split_index, f'Target_{asset_id}'].values.reshape(-1,1),\n                  max_epochs=3,\n                  patience=1,\n                 batch_size=128)\n        preds     = list(  model.predict(df_proc.loc[test_split_index, features].values)        )\n        oof_pred += [x[0] for x in preds]\n        oof_valid += list(   df_proc.loc[test_split_index, f'Target_{asset_id}'].values    )\n        del model\n\n\n    return oof_pred, oof_valid","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof = {x:0 for x in assets}\n\nall_oof_pred = []\nall_oof_valid = []\n\nfor asset_id in assets:\n    print(f\"Training model for (ID={asset_id:<2})\")\n    \n    oof_pred, oof_valid = get_Xy_and_model_for_asset(feat, asset_id)\n    \n    all_oof_pred += oof_pred\n    all_oof_valid += oof_valid\n    \n    oof[asset_id] = corr_score(     np.array(oof_pred)   ,    np.array(oof_valid)    )\n    \n    print(f'OOF corr score of (ID={asset_id}) is {oof[asset_id]:.5f}.')\n    print('')\n    print('')","metadata":{"execution":{"iopub.status.busy":"2023-05-16T01:47:12.345063Z","iopub.execute_input":"2023-05-16T01:47:12.345319Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"avg_values = sum(oof.values()) / len(oof)\nprint(f'  simple average corr score: {avg_values:.5f}.')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}