{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"## The main model from: https://www.kaggle.com/code/sugghi/training-3rd-place-solution\nimport os\nimport random\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport lightgbm as lgb\nimport gresearch_crypto\nimport time\nimport datetime\n\nimport pickle\nimport gc\n\nfrom tqdm import tqdm\n\nn_fold = 7\nseed0 = 8586\nuse_supple_for_train = False\n\n# If True, the period used to evaluate Public LB will not be used for training.\n# Set to False on final submission.\nnot_use_overlap_to_train = True\ncsv_path = '/kaggle/input/stock-pred2/'\ncsv_file = 'stock.csv'","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:39:06.427726Z","iopub.execute_input":"2022-06-19T09:39:06.428285Z","iopub.status.idle":"2022-06-19T09:39:06.434752Z","shell.execute_reply.started":"2022-06-19T09:39:06.428246Z","shell.execute_reply":"2022-06-19T09:39:06.433967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lags = [10,30,60,300,900]","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:39:06.498992Z","iopub.execute_input":"2022-06-19T09:39:06.499184Z","iopub.status.idle":"2022-06-19T09:39:06.503315Z","shell.execute_reply.started":"2022-06-19T09:39:06.499162Z","shell.execute_reply":"2022-06-19T09:39:06.502482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {\n    'early_stopping_rounds': 100,\n    'objective': 'regression',\n    'metric': 'rmse',\n#    'metric': 'corr',\n#     'metric': 'None',\n    'boosting_type': 'gbdt',\n    'max_depth': 5,\n    'verbose': -1,\n    'max_bin':600,\n    'min_data_in_leaf':50,\n    'learning_rate': 0.03,\n    'subsample': 0.7,\n    'subsample_freq': 1,\n    'feature_fraction': 1,\n    'lambda_l1': 0.5,\n    'lambda_l2': 2,\n    'seed':seed0,\n    'feature_fraction_seed': seed0,\n    'bagging_fraction_seed': seed0,\n    'drop_seed': seed0,\n    'data_random_seed': seed0,\n    'extra_trees': True,\n    'extra_seed': seed0,\n    'zero_as_missing': True,\n    \"first_metric_only\": True\n         }","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:39:06.649738Z","iopub.execute_input":"2022-06-19T09:39:06.65039Z","iopub.status.idle":"2022-06-19T09:39:06.659189Z","shell.execute_reply.started":"2022-06-19T09:39:06.650356Z","shell.execute_reply":"2022-06-19T09:39:06.658269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n\n    for col in df.columns:\n        col_type = df[col].dtype\n\n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:39:06.755134Z","iopub.execute_input":"2022-06-19T09:39:06.755658Z","iopub.status.idle":"2022-06-19T09:39:06.769494Z","shell.execute_reply.started":"2022-06-19T09:39:06.755628Z","shell.execute_reply":"2022-06-19T09:39:06.768586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(csv_path+csv_file, usecols=['date','stock_id', 'close'])","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:39:06.807654Z","iopub.execute_input":"2022-06-19T09:39:06.807935Z","iopub.status.idle":"2022-06-19T09:39:07.034584Z","shell.execute_reply.started":"2022-06-19T09:39:06.807908Z","shell.execute_reply":"2022-06-19T09:39:07.033786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\ndf.columns=[\"Asset_ID\", \"timestamp\", \"Close\"]\nfor i in range(df.shape[0]):\n    df[\"Asset_ID\"][i] = re.findall(r'\\d+',df[\"Asset_ID\"][i])\n    df[\"Asset_ID\"][i] = int(df[\"Asset_ID\"][i][0])\na = df.copy()\ndf.loc[:,\"timestamp\"] = pd.to_datetime(a.loc[:,\"timestamp\"])","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:39:07.036367Z","iopub.execute_input":"2022-06-19T09:39:07.036664Z","iopub.status.idle":"2022-06-19T09:45:05.865852Z","shell.execute_reply.started":"2022-06-19T09:39:07.036619Z","shell.execute_reply":"2022-06-19T09:45:05.865069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Create Target**","metadata":{}},{"cell_type":"markdown","source":"* **Select 14 Stocks**","metadata":{}},{"cell_type":"code","source":"m = df['Asset_ID'].sort_values(ascending = True).index[:].tolist()\ndf = df.loc[m]\nprint(df[:100])\nid_list = []\nt = 0\nnum = 0\nfor i in df['Asset_ID']:\n    if(i>t):\n        t = i\n        id_list.append(t)\n        df.loc[df[\"Asset_ID\"] == t, \"Asset_ID\"] = num\n        num = num + 1\n        if (num>=14):\n            break\nprint(id_list) ## Stocks' real id names\ndf = df.loc[df[\"Asset_ID\"]<14]","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:45:05.867037Z","iopub.execute_input":"2022-06-19T09:45:05.868813Z","iopub.status.idle":"2022-06-19T09:45:06.470382Z","shell.execute_reply.started":"2022-06-19T09:45:05.868773Z","shell.execute_reply":"2022-06-19T09:45:06.469636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m = df['timestamp'].sort_values(ascending = True).index[:].tolist()\ndf = df.loc[m]\nfor i in range(14):    \n    df.loc[df[\"Asset_ID\"] == i, \"Target\"] = df.loc[df[\"Asset_ID\"] == i][\"Close\"].shift(-1)/df.loc[df[\"Asset_ID\" ]== i][\"Close\"] - 1\n#df = df.loc[df[\"Asset_ID\"]<14","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:45:06.472378Z","iopub.execute_input":"2022-06-19T09:45:06.472835Z","iopub.status.idle":"2022-06-19T09:45:06.823341Z","shell.execute_reply.started":"2022-06-19T09:45:06.472797Z","shell.execute_reply":"2022-06-19T09:45:06.822622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:45:06.825496Z","iopub.execute_input":"2022-06-19T09:45:06.825867Z","iopub.status.idle":"2022-06-19T09:45:06.840963Z","shell.execute_reply.started":"2022-06-19T09:45:06.825832Z","shell.execute_reply":"2022-06-19T09:45:06.840263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"df.head\ndf_train = df","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:45:06.869608Z","iopub.execute_input":"2022-06-19T09:45:06.869931Z","iopub.status.idle":"2022-06-19T09:45:06.874191Z","shell.execute_reply.started":"2022-06-19T09:45:06.869894Z","shell.execute_reply":"2022-06-19T09:45:06.873341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[:14]","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:45:06.900137Z","iopub.execute_input":"2022-06-19T09:45:06.900484Z","iopub.status.idle":"2022-06-19T09:45:06.91349Z","shell.execute_reply.started":"2022-06-19T09:45:06.900445Z","shell.execute_reply":"2022-06-19T09:45:06.912156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_merged = pd.DataFrame()\ntrain_merged[df_train.columns] = 0\nfor id in tqdm( range(14) ):\n    train_merged = train_merged.merge(df_train.loc[df_train[\"Asset_ID\"] == id, ['timestamp', 'Close','Target']].copy(), on=\"timestamp\", how='outer',suffixes=['', \"_\"+str(id)])\n        \ntrain_merged = train_merged.drop(df_train.columns.drop(\"timestamp\"), axis=1)\ntrain_merged = train_merged.sort_values('timestamp', ascending=True)\ndisplay(train_merged.head())","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:45:06.915137Z","iopub.execute_input":"2022-06-19T09:45:06.915621Z","iopub.status.idle":"2022-06-19T09:45:07.125Z","shell.execute_reply.started":"2022-06-19T09:45:06.915583Z","shell.execute_reply":"2022-06-19T09:45:07.124111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# forward fill\n# Set an upper limit on the number of fills, since there may be long term gaps.\nfor id in range(14):\n#     print(id, train_merged[f'Close_{id}'].isnull().sum())   # Number of missing before forward fill\n    train_merged[f'Close_{id}'] = train_merged[f'Close_{id}'].fillna(method='ffill', limit=400)\n    train_merged[f'Target_{id}'] = train_merged[f'Target_{id}'].fillna(0.0, limit=400)\n#     print(id, train_merged[f'Close_{id}'].isnull().sum())   # Number of missing after forward fill\ntrain_merged = train_merged[train_merged[\"timestamp\"] >= \"2010-01-01\"]\ntrain_merged","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:45:07.126327Z","iopub.execute_input":"2022-06-19T09:45:07.126652Z","iopub.status.idle":"2022-06-19T09:45:07.173056Z","shell.execute_reply.started":"2022-06-19T09:45:07.126616Z","shell.execute_reply":"2022-06-19T09:45:07.172337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering","metadata":{}},{"cell_type":"markdown","source":"**Six Types of Features:**\n* **log_close/mean_n_idy:**  devide today's close price of the stock whose Asset_ID is y by average of the stock's close price over the past n days, then take logarithm of it.\n* **log_return_n_idy:**   devide today's close price of the stock whose Asset_ID is y by its close price in last nth day , then take logarithm of it.\n* **mean_close/mean_n:**   today's average of all stocks' log_close/mean_n_id.\n* **mean_log_returns:**  today's average of all stocks' log_return_n_id.\n* **log_close/mean_n-mean_close/mean_n_idy:** subtract mean_close/mean_n from log_close/mean_n_idy.\n* **log_return_n-mean_log_returns_n_idy:** subtract mean_return_n from log_return_n_idy.\n","metadata":{}},{"cell_type":"code","source":"def get_features(df, train=True):   \n    if train == True:\n        #totimestamp = lambda s: np.int32(time.mktime(datetime.datetime.strptime(s, \"%d/%m/%Y\").timetuple()))\n        #valid_window = [totimestamp(\"12/03/2021\")]\n        valid_window = \"2021-03-12\"\n#         valid_window = [totimestamp(\"15/08/2021\")]  #検証用\n        df['train_flg'] = np.where(df['timestamp']>=valid_window, 0,1)\n\n        #supple_start_window = [totimestamp(\"22/09/2021\")]\n        #if use_supple_for_train:\n        #    df['train_flg'] = np.where(df['timestamp']>=supple_start_window[0], 1 ,df['train_flg']  )\n\n   \n    for id in range(14):    \n        for lag in lags:\n            df[f'log_close/mean_{lag}_id{id}'] = np.log( (np.array(df[f'Close_{id}']) /  np.roll(np.append(np.convolve( np.array(df[f'Close_{id}']), np.ones(lag)/lag, mode=\"valid\"), np.ones(lag-1)), lag-1)).astype(\"float\")  )\n            df[f'log_return_{lag}_id{id}'] = np.log( (np.array(df[f'Close_{id}']) /  np.roll(np.array(df[f'Close_{id}']), lag)).astype(\"float\")  )\n    for lag in lags:\n        df[f'mean_close/mean_{lag}'] =  np.mean(df.iloc[:,df.columns.str.startswith(f'log_close/mean_{lag}_id')], axis=1)\n        df[f'mean_log_returns_{lag}'] = np.mean(df.iloc[:,df.columns.str.startswith(f'log_return_{lag}_id')] ,    axis=1)\n        for id in range(14):\n            df[f'log_close/mean_{lag}-mean_close/mean_{lag}_id{id}'] = np.array( df[f'log_close/mean_{lag}_id{id}']) - np.array( df[f'mean_close/mean_{lag}']  )\n            df[f'log_return_{lag}-mean_log_returns_{lag}_id{id}']    = np.array( df[f'log_return_{lag}_id{id}'])     - np.array( df[f'mean_log_returns_{lag}'] )\n\n    if train == True:\n        for id in range(14):\n            df = df.drop([f'Close_{id}'], axis=1)\n        #oldest_use_window = [totimestamp(\"12/01/2019\")]\n        oldest_use_window = \"2017-01-12\"\n        df = df[  df['timestamp'] >= oldest_use_window   ]\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:45:07.174494Z","iopub.execute_input":"2022-06-19T09:45:07.174729Z","iopub.status.idle":"2022-06-19T09:45:07.187514Z","shell.execute_reply.started":"2022-06-19T09:45:07.174696Z","shell.execute_reply":"2022-06-19T09:45:07.186391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfeat = get_features(train_merged)\nfeat","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:45:07.188816Z","iopub.execute_input":"2022-06-19T09:45:07.189362Z","iopub.status.idle":"2022-06-19T09:45:07.414908Z","shell.execute_reply.started":"2022-06-19T09:45:07.189326Z","shell.execute_reply":"2022-06-19T09:45:07.413954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define features for LGBM\nnot_use_features_train = ['timestamp', 'train_flg']\nfor id in range(14):\n    not_use_features_train.append(f'Target_{id}')\n\nfeatures = feat.columns \nfeatures = features.drop(not_use_features_train)\nfeatures = list(features)\n# display(features)  \nlen(features)","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:45:07.416525Z","iopub.execute_input":"2022-06-19T09:45:07.416838Z","iopub.status.idle":"2022-06-19T09:45:07.425823Z","shell.execute_reply.started":"2022-06-19T09:45:07.416799Z","shell.execute_reply":"2022-06-19T09:45:07.424927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_merged\ndel df_train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:45:07.427626Z","iopub.execute_input":"2022-06-19T09:45:07.427986Z","iopub.status.idle":"2022-06-19T09:45:07.579366Z","shell.execute_reply.started":"2022-06-19T09:45:07.427921Z","shell.execute_reply":"2022-06-19T09:45:07.57867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"# define the evaluation metric\ndef correlation(a, train_data):\n    \n    b = train_data.get_label()\n    \n    a = np.ravel(a)\n    b = np.ravel(b)\n\n    len_data = len(a)\n    mean_a = np.sum(a) / len_data\n    mean_b = np.sum(b) / len_data\n    var_a = np.sum(np.square(a - mean_a)) / len_data\n    var_b = np.sum(np.square(b - mean_b)) / len_data\n\n    cov = np.sum((a * b))/len_data - mean_a*mean_b\n    corr = cov / np.sqrt(var_a * var_b)\n\n    return 'corr', corr, True\n\n# For CV score calculation\ndef corr_score(pred, valid):\n    len_data = len(pred)\n    mean_pred = np.sum(pred) / len_data\n    mean_valid = np.sum(valid) / len_data\n    var_pred = np.sum(np.square(pred - mean_pred)) / len_data\n    var_valid = np.sum(np.square(valid - mean_valid)) / len_data\n\n    cov = np.sum((pred * valid))/len_data - mean_pred*mean_valid\n    corr = cov / np.sqrt(var_pred * var_valid)\n\n    return corr\n\n# For CV score calculation\ndef wcorr_score(pred, valid, weight):\n    len_data = len(pred)\n    sum_w = np.sum(weight)\n    mean_pred = np.sum(pred * weight) / sum_w\n    mean_valid = np.sum(valid * weight) / sum_w\n    var_pred = np.sum(weight * np.square(pred - mean_pred)) / sum_w\n    var_valid = np.sum(weight * np.square(valid - mean_valid)) / sum_w\n\n    cov = np.sum((pred * valid * weight)) / sum_w - mean_pred*mean_valid\n    corr = cov / np.sqrt(var_pred * var_valid)\n\n    return corr","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:45:07.580601Z","iopub.execute_input":"2022-06-19T09:45:07.582569Z","iopub.status.idle":"2022-06-19T09:45:07.594935Z","shell.execute_reply.started":"2022-06-19T09:45:07.582531Z","shell.execute_reply":"2022-06-19T09:45:07.594216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* **Define a new objective function for position management**\n* *also a weight setting in lgb.dataset() can achieve it*","metadata":{}},{"cell_type":"code","source":"def weight_rmse(preds, train_data, alpha1 = 1., alpha2 = 1.5):\n    labels = train_data.get_label()\n    k = preds - labels\n    grad = np.where(labels>0, 2*alpha1*k, 2*alpha2*k)\n    hess = np.where(labels>0, 2*alpha1, 2*alpha2)\n    return grad, hess\n            \n        \n        ","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:45:07.596639Z","iopub.execute_input":"2022-06-19T09:45:07.596839Z","iopub.status.idle":"2022-06-19T09:45:07.602809Z","shell.execute_reply.started":"2022-06-19T09:45:07.596809Z","shell.execute_reply":"2022-06-19T09:45:07.602007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* *from: https://blog.amedama.jp/entry/lightgbm-cv-feature-importance \n(used in nyanp's Optiver solution)*","metadata":{}},{"cell_type":"code","source":"# from: https://blog.amedama.jp/entry/lightgbm-cv-feature-importance\n# (used in nyanp's Optiver solution)\ndef plot_importance(importances, features_names = features, PLOT_TOP_N = 20, figsize=(10, 10)):\n    importance_df = pd.DataFrame(data=importances, columns=features)\n    sorted_indices = importance_df.median(axis=0).sort_values(ascending=False).index\n    sorted_importance_df = importance_df.loc[:, sorted_indices]\n    plot_cols = sorted_importance_df.columns[:PLOT_TOP_N]\n    _, ax = plt.subplots(figsize=figsize)\n    ax.grid()\n    ax.set_xscale('log')\n    ax.set_ylabel('Feature')\n    ax.set_xlabel('Importance')\n    sns.boxplot(data=sorted_importance_df[plot_cols],\n                orient='h',\n                ax=ax)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:45:07.616706Z","iopub.execute_input":"2022-06-19T09:45:07.617205Z","iopub.status.idle":"2022-06-19T09:45:07.626539Z","shell.execute_reply.started":"2022-06-19T09:45:07.617132Z","shell.execute_reply":"2022-06-19T09:45:07.625793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from: https://www.kaggle.com/code/nrcjea001/lgbm-embargocv-weightedpearson-lagtarget/\ndef get_time_series_cross_val_splits(data, cv = n_fold, embargo = 3):#embargo = 3750\n    all_train_timestamps = data['timestamp'].unique()\n    len_split = len(all_train_timestamps) // cv\n    test_splits = [all_train_timestamps[i * len_split : (i + 1) * len_split] for i in range(cv)]\n    # fix the last test split to have all the last timestamps, in case the number of timestamps wasn't divisible by cv\n    rem = len(all_train_timestamps) - len_split*cv\n    if rem>0:\n        test_splits[-1] = np.append(test_splits[-1], all_train_timestamps[-rem:])\n\n    train_splits = []\n    for test_split in test_splits:\n        #test_split_max = int(np.max(test_split))\n        #test_split_min = int(np.min(test_split))\n        test_split_max = test_split.max()\n        test_split_min = test_split.min()\n        ## *get all of the timestamps that aren't in the test split\n        train_split_not_embargoed = [e for e in all_train_timestamps if not (test_split_min <= e <= test_split_max)]\n        # embargo the train split so we have no leakage. Note timestamps are expressed in seconds, so multiply by 60\n        ## *embargo_sec = 60*embargo\n        embargo_sec = pd.to_timedelta(embargo,\"D\")\n        train_split = [e for e in train_split_not_embargoed if\n                       abs(e - test_split_max) > embargo_sec and abs(e - test_split_min) > embargo_sec]\n        train_splits.append(train_split)\n\n    ## *convenient way to iterate over train and test splits\n    train_test_zip = zip(train_splits, test_splits)\n    return train_test_zip","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:45:07.638435Z","iopub.execute_input":"2022-06-19T09:45:07.638893Z","iopub.status.idle":"2022-06-19T09:45:07.648231Z","shell.execute_reply.started":"2022-06-19T09:45:07.638858Z","shell.execute_reply":"2022-06-19T09:45:07.647445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_Xy_and_model_for_asset(df_proc, asset_id):\n    df_proc = df_proc.loc[  (df_proc[f'Target_{asset_id}'] == df_proc[f'Target_{asset_id}'])  ]\n    if not_use_overlap_to_train:\n        df_proc = df_proc.loc[  (df_proc['train_flg'] == 1)  ]\n    \n# EmbargoCV\n    train_test_zip = get_time_series_cross_val_splits(df_proc, cv = n_fold, embargo = 3)\n    print(\"entering time series cross validation loop\")\n    importances = []\n    oof_pred = []\n    oof_valid = []\n    \n    for split, train_test_split in enumerate(train_test_zip):\n        gc.collect()\n        \n        print(f\"doing split {split+1} out of {n_fold}\")\n        train_split, test_split = train_test_split\n        train_split_index = df_proc['timestamp'].isin(train_split)\n        test_split_index = df_proc['timestamp'].isin(test_split)\n    \n        train_dataset = lgb.Dataset(df_proc.loc[train_split_index, features],\n                                    #df_proc.loc[train_split_index, f'Target_{asset_id}'].values, \n                                    label = df_proc.loc[train_split_index, f'Target_{asset_id}'].values,\n                                    feature_name = features, \n                                   )\n        val_dataset = lgb.Dataset(df_proc.loc[test_split_index, features], \n                                  df_proc.loc[test_split_index, f'Target_{asset_id}'].values, \n                                  feature_name = features, \n                                 )\n        print(val_dataset)\n\n        print(f\"number of train data: {len(df_proc.loc[train_split_index])}\")\n        print(f\"number of val data:   {len(df_proc.loc[test_split_index])}\")\n\n        model = lgb.train(params = params,\n                          train_set = train_dataset, \n                          valid_sets=[train_dataset, val_dataset],\n                          valid_names=['tr', 'vl'],\n                          num_boost_round = 5000,\n                          verbose_eval = 100,     \n                          feval = correlation,\n                          #fobj = weight_rmse\n                         )\n        importances.append(model.feature_importance(importance_type='gain'))\n        \n        file = f'trained_model_id{asset_id}_fold{split}.pkl'\n        pickle.dump(model, open(file, 'wb'))\n        print(f\"Trained model was saved to 'trained_model_id{asset_id}_fold{split}.pkl'\")\n        print(\"\")\n            \n        oof_pred += list(  model.predict(df_proc.loc[test_split_index, features])        )\n        oof_valid += list(   df_proc.loc[test_split_index, f'Target_{asset_id}'].values    )\n    \n    \n    plot_importance(np.array(importances),features, PLOT_TOP_N = 20, figsize=(10, 5))\n\n    return oof_pred, oof_valid","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:45:07.652732Z","iopub.execute_input":"2022-06-19T09:45:07.653201Z","iopub.status.idle":"2022-06-19T09:45:07.66674Z","shell.execute_reply.started":"2022-06-19T09:45:07.653165Z","shell.execute_reply":"2022-06-19T09:45:07.665756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oof = [ [] for id in range(14)   ]\n\nall_oof_pred = []\nall_oof_valid = []\nall_oof_weight = []\nweight_0 = [1/14 for _ in range(14)]\nfor asset_id, asset_name in zip(range(14), id_list):\n    print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n    oof_pred, oof_valid = get_Xy_and_model_for_asset(feat, asset_id)\n    weight_temp = float(weight_0[asset_id])\n    all_oof_pred += oof_pred\n    all_oof_valid += oof_valid\n    all_oof_weight += [weight_temp] * len(oof_pred)\n    \n    oof[asset_id] = corr_score(     np.array(oof_pred)   ,    np.array(oof_valid)    )\n    \n    print(f'OOF corr score of {asset_name} (ID={asset_id}) is {oof[asset_id]:.5f}. (Weight: {float(weight_temp):.5f})')\n    print('')\n    print('')","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-06-19T09:45:07.667939Z","iopub.execute_input":"2022-06-19T09:45:07.668319Z","iopub.status.idle":"2022-06-19T09:46:21.308127Z","shell.execute_reply.started":"2022-06-19T09:45:07.668286Z","shell.execute_reply":"2022-06-19T09:46:21.307379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ls -lh\nfeat","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:46:21.30933Z","iopub.execute_input":"2022-06-19T09:46:21.309621Z","iopub.status.idle":"2022-06-19T09:46:21.343342Z","shell.execute_reply.started":"2022-06-19T09:46:21.309583Z","shell.execute_reply":"2022-06-19T09:46:21.342492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"woof = 0\nfor id in range(14):\n    woof += oof[id]*weight_0[id]\n#    woof += oof[id] * float(  df_asset_details.loc[  df_asset_details['Asset_ID'] == id  , 'Weight'   ] )\n#woof = woof / df_asset_details['Weight'].sum()\nwoof = woof / sum(weight_0)\nprint(f'OOF corr scores are;')\nfor oof_score in oof:\n    print(f'      {oof_score:.5f}')\nprint(f'  simple average corr score: {np.mean(oof):.5f}.')\nprint(f'weighted average corr score: {woof:.5f}.')\nprint(f'')\n\nall_oof_wcorr = wcorr_score(     np.array(all_oof_pred),    np.array(all_oof_valid),  np.array(all_oof_weight)   )\nprint(f'        weighted corr score: {all_oof_wcorr:.5f}.')","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:46:21.344858Z","iopub.execute_input":"2022-06-19T09:46:21.345125Z","iopub.status.idle":"2022-06-19T09:46:21.364111Z","shell.execute_reply.started":"2022-06-19T09:46:21.345089Z","shell.execute_reply":"2022-06-19T09:46:21.362948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feat.loc[feat[\"train_flg\"]==0]","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:46:21.365781Z","iopub.execute_input":"2022-06-19T09:46:21.366119Z","iopub.status.idle":"2022-06-19T09:46:21.400108Z","shell.execute_reply.started":"2022-06-19T09:46:21.366046Z","shell.execute_reply":"2022-06-19T09:46:21.399354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Position management","metadata":{}},{"cell_type":"code","source":"df_corr = pd.DataFrame()\nasset_id = 0\ndata = feat.loc[feat[\"train_flg\"]==0]\nfor asset_id in range(14):\n    pred_corr_list = []   \n    for i in range(7):\n        model_pred = []\n        res = open(f'./trained_model_id{asset_id}_fold{i}.pkl',\"rb\")\n        model = pickle.load(res)\n        df_proc = feat.loc[feat[\"train_flg\"]==0]    \n        df_proc = df_proc.loc[ :, [f'Target_{asset_id}' ,\"timestamp\" ]]   \n        pred = model.predict(feat.loc[feat[\"train_flg\"]==0, features])\n        model_pred.append(pred)\n        t = corr_score(df_proc[f'Target_{asset_id}'],pred)\n        pred_corr_list.append(t)\n    pred_list = np.array(model_pred).sum(axis = 0)\n    df_corr[f'pred_corr_{asset_id}'] = pred_corr_list\n    data[f'pred_{asset_id}'] = pred_list\n    pred_corr_list","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:46:21.401466Z","iopub.execute_input":"2022-06-19T09:46:21.401718Z","iopub.status.idle":"2022-06-19T09:46:22.394723Z","shell.execute_reply.started":"2022-06-19T09:46:21.401684Z","shell.execute_reply":"2022-06-19T09:46:22.394035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_feat = ['timestamp']\nfact_feat = ['timestamp']\nfor asset_id in range(14):\n    pred_feat.append(f'pred_{asset_id}')\n    fact_feat.append(f'Target_{asset_id}')                     \npred = data[pred_feat]\nfact = data[fact_feat]","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:46:22.398598Z","iopub.execute_input":"2022-06-19T09:46:22.400548Z","iopub.status.idle":"2022-06-19T09:46:22.409154Z","shell.execute_reply.started":"2022-06-19T09:46:22.400513Z","shell.execute_reply":"2022-06-19T09:46:22.408415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Define Value Function\ndef Value_pred(p, p_yesterday, time, alpha = 0.005):\n    v = np.array(p*pred.loc[pred['timestamp']==time].iloc[:,1:] - alpha * np.square(p-p_yesterday))\n    return np.sum(v, axis = None)\ndef Value_fact(p, p_yesterday, time, alpha = 0.005):\n    v = np.array(p*fact.loc[fact['timestamp']==time].iloc[:,1:] - alpha * np.square(p-p_yesterday))\n    return np.sum(v, axis = None)\ndef diff_Value_pred(p, p_yesterday, time, alpha = 0.005):\n    return np.sum(pred.loc[pred['timestamp']==time].iloc[:,1:] - alpha * 2*(p-p_yesterday))\n## According to derivative of Value_pred(p)\ndef get_p(p_yesterday, time, alpha = 0.005):\n    p = (np.array(pred.loc[pred['timestamp']==time].iloc[:,1:]) + 2*alpha*p_yesterday)/(2*alpha)\n    p = 1/2*(p + abs(p))\n    p = p/(p.sum()+1e-6)\n    return p[0]","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:46:22.410528Z","iopub.execute_input":"2022-06-19T09:46:22.410822Z","iopub.status.idle":"2022-06-19T09:46:22.421651Z","shell.execute_reply.started":"2022-06-19T09:46:22.410779Z","shell.execute_reply":"2022-06-19T09:46:22.420873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Decide position strategy on value functions above\n## Create a dataframe to display the relationship between our position strategy and income return\np_V_feat = ['timestamp', 'Value_p', 'Value_f']\nfor asset_id in range(14):\n    p_V_feat.append(f'position_{asset_id}')    \np_V = pd.DataFrame(columns = p_V_feat)\np_V.set_index(['timestamp'])\ntime = pd.to_datetime('2021-03-12')\np_yesterday = np.array(weight_0)\nwhile(time < pd.to_datetime('2022-06-08')):\n    today = [time]\n    p = get_p(p_yesterday, time, alpha = 0.005)\n    Value_p = Value_pred(p, p_yesterday, time, alpha = 0.005)\n    today.append(Value_p)\n    Value_f = Value_fact(p, p_yesterday, time, alpha = 0.005)\n    today.append(Value_f)\n    today.extend(p)\n    p_V.loc[time,:] = today\n    p_yesterday = p\n    time = pred.loc[pred['timestamp']>time].iloc[0,0]","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:46:22.423208Z","iopub.execute_input":"2022-06-19T09:46:22.423673Z","iopub.status.idle":"2022-06-19T09:46:23.710127Z","shell.execute_reply.started":"2022-06-19T09:46:22.423636Z","shell.execute_reply":"2022-06-19T09:46:23.70936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p_V","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:46:23.711654Z","iopub.execute_input":"2022-06-19T09:46:23.712178Z","iopub.status.idle":"2022-06-19T09:46:23.751361Z","shell.execute_reply.started":"2022-06-19T09:46:23.712139Z","shell.execute_reply":"2022-06-19T09:46:23.750556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Display achieved positive return days\np_V.loc[p_V[\"Value_f\"]>0]","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:46:23.755981Z","iopub.execute_input":"2022-06-19T09:46:23.758051Z","iopub.status.idle":"2022-06-19T09:46:23.793957Z","shell.execute_reply.started":"2022-06-19T09:46:23.758012Z","shell.execute_reply":"2022-06-19T09:46:23.793212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Return Rate every 10 days (compound interest)**","metadata":{}},{"cell_type":"code","source":"time_0 = p_V.index[0]\ndef rr_cal(days):\n    rr = 1\n    date = pd.to_timedelta(days,\"D\")+time_0\n    for time in p_V.index[:days]:\n        if(pd.isnull(p_V.loc[time,\"Value_f\"])):\n            continue\n        else:\n            rr = rr*(1+p_V.loc[time,\"Value_f\"])        \n    return(date,rr)","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:46:23.798887Z","iopub.execute_input":"2022-06-19T09:46:23.801097Z","iopub.status.idle":"2022-06-19T09:46:23.809379Z","shell.execute_reply.started":"2022-06-19T09:46:23.801062Z","shell.execute_reply":"2022-06-19T09:46:23.808794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_rr = pd.DataFrame(columns = [\"timestamp\",\"rr\"])\ndf_rr.set_index = [\"timestamp\"]\nfor days in range(9,300,10):\n    df_rr.loc[rr_cal(days)[0],:]=rr_cal(days)","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:46:23.814366Z","iopub.execute_input":"2022-06-19T09:46:23.816475Z","iopub.status.idle":"2022-06-19T09:46:24.248024Z","shell.execute_reply.started":"2022-06-19T09:46:23.816414Z","shell.execute_reply":"2022-06-19T09:46:24.247324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_rr[\"rr\"]","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:46:24.252359Z","iopub.execute_input":"2022-06-19T09:46:24.254622Z","iopub.status.idle":"2022-06-19T09:46:24.266649Z","shell.execute_reply.started":"2022-06-19T09:46:24.254578Z","shell.execute_reply":"2022-06-19T09:46:24.265724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#改正cv之后提升最大","metadata":{"execution":{"iopub.status.busy":"2022-06-19T09:46:24.270318Z","iopub.execute_input":"2022-06-19T09:46:24.272199Z","iopub.status.idle":"2022-06-19T09:46:24.27704Z","shell.execute_reply.started":"2022-06-19T09:46:24.272163Z","shell.execute_reply":"2022-06-19T09:46:24.276338Z"},"trusted":true},"execution_count":null,"outputs":[]}]}