{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Submission Inference\n\n__Here I used a primitive version of lagged target and market information with other rolling aggregations__\n\n","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-25T20:22:57.247751Z","iopub.execute_input":"2022-01-25T20:22:57.248067Z","iopub.status.idle":"2022-01-25T20:22:57.279986Z","shell.execute_reply.started":"2022-01-25T20:22:57.24803Z","shell.execute_reply":"2022-01-25T20:22:57.279206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport random\nimport pandas as pd\nimport numpy as np\nimport lightgbm as lgb\nimport gresearch_crypto\nimport time\nimport datetime\nfrom numba import jit","metadata":{"execution":{"iopub.status.busy":"2022-01-25T20:22:57.28162Z","iopub.execute_input":"2022-01-25T20:22:57.283109Z","iopub.status.idle":"2022-01-25T20:23:00.022089Z","shell.execute_reply.started":"2022-01-25T20:22:57.283068Z","shell.execute_reply":"2022-01-25T20:23:00.02114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_CSV = '../input/g-research-crypto-forecasting/train.csv'\nASSET_DETAILS_CSV = '../input/g-research-crypto-forecasting/asset_details.csv'\nSUPPLEMENT_CSV = '../input/g-research-crypto-forecasting/supplemental_train.csv'\nMODEL_FILE = '../input/modelfiles/20220126_cv2.txt'","metadata":{"execution":{"iopub.status.busy":"2022-01-25T20:23:00.023463Z","iopub.execute_input":"2022-01-25T20:23:00.024617Z","iopub.status.idle":"2022-01-25T20:23:00.02932Z","shell.execute_reply.started":"2022-01-25T20:23:00.024579Z","shell.execute_reply":"2022-01-25T20:23:00.028309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL_OLDEST_FILE = '../input/modelfiles/20220126_cv1.txt'\nMODEL_FEWR_FILE = '../input/modelfiles/20220126_cv3.txt'\n\nmodel_oldest = lgb.Booster(model_file=MODEL_OLDEST_FILE)\nmodel_fewer = lgb.Booster(model_file=MODEL_FEWR_FILE)\nfeatures_oldest = model_oldest.feature_name()\nfeatures_fewer = model_fewer.feature_name()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sup = pd.read_csv(SUPPLEMENT_CSV)\ndf_asset_details = pd.read_csv(ASSET_DETAILS_CSV)\nmodel = lgb.Booster(model_file=MODEL_FILE)\nfeatures = model.feature_name()","metadata":{"execution":{"iopub.status.busy":"2022-01-25T20:23:00.031597Z","iopub.execute_input":"2022-01-25T20:23:00.032022Z","iopub.status.idle":"2022-01-25T20:23:06.516554Z","shell.execute_reply.started":"2022-01-25T20:23:00.031974Z","shell.execute_reply":"2022-01-25T20:23:06.515536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define Original Features","metadata":{}},{"cell_type":"code","source":"def calcHullMA(price: pd.Series, N=50):\n    SMA1 = price.rolling(N).mean()\n    SMA2 = price.rolling(int(N/2)).mean()\n    return (2 * SMA2 - SMA1).rolling(int(np.sqrt(N))).mean()\n\ndef get_features(df_feat):\n    #for recreated Targets\n    df_feat[\"target_return\"] = (df_feat[\"Close\"].shift(1) / df_feat[\"Close\"].shift(16)) -1\n\n    # Create your features here, they can be lagged or not\n    df_feat['sma15'] = df_feat['Close'].rolling(15).mean()/df_feat['Close'] -1\n    df_feat['sma60'] = df_feat['Close'].rolling(60).mean()/df_feat['Close'] -1\n    df_feat['sma240'] = df_feat['Close'].rolling(240).mean()/df_feat['Close'] -1\n    \n    df_feat['return15'] = df_feat['Close']/df_feat['Close'].shift(15) -1\n    df_feat['return60'] = df_feat['Close']/df_feat['Close'].shift(60) -1\n    df_feat['return240'] = df_feat['Close']/df_feat['Close'].shift(240) -1\n    \n    df_feat['sma15_count'] = df_feat['Count'].rolling(15).mean()/df_feat['Close'] -1\n    df_feat['sma60_count'] = df_feat['Count'].rolling(60).mean()/df_feat['Close'] -1\n    df_feat['sma240_count'] = df_feat['Count'].rolling(240).mean()/df_feat['Close'] -1\n    \n    df_feat['return15_count'] = df_feat['Volume']/df_feat['Volume'].shift(15) -1\n    df_feat['return60_count'] = df_feat['Volume']/df_feat['Volume'].shift(60) -1\n    df_feat['return240_count'] = df_feat['Volume']/df_feat['Volume'].shift(240) -1\n    \n    df_feat[\"hull\"] = df_feat.Close - calcHullMA(df_feat.Close, 240)\n    df_feat[\"hull2\"] = df_feat.Close - calcHullMA(df_feat.Close, 76)\n    df_feat[\"hull3\"] = df_feat.Close - calcHullMA(df_feat.Close, 800)\n    ###################################################################\n    #Returns etc\n    fibo_list = [55, 210, 340, 890, 3750]\n    #if verbose: print(\"[Feature] Return\")\n    df_feat[f'log_return'] = np.log(df_feat.Close).diff().ffill().bfill()\n    for i in fibo_list:\n        df_feat[f'log_return_{i}'] = np.log(df_feat.Close).diff().rolling(i).mean().ffill().bfill()\n    \n    df_feat = df_feat.fillna(0)\n    \n    return df_feat\n\ndef inference_target_recreation(tt, weights, asset_id):\n    m = np.average(tt, axis=1, weights=weights)\n    num = np.mean(np.multiply(tt, m.reshape(-1, 1)), axis=0)\n    denom = np.mean(np.multiply(m, m), axis=0)\n    beta = num / denom\n    t2 = (beta * m[-1]).T\n    p2 = np.mean(t2)\n    return t2[asset_id], p2\n\n#@jit(nopython=True)\ndef moving_average(a, n=3) :\n    ret = np.cumsum(a, dtype=float)\n    ret[n:] = ret[n:] - ret[:-n]\n    return np.append(np.array([1]*n), ret[n - 1:] / n)[1:]\n\n#@jit(nopython=True)\ndef calcHullMA_inference(series, N=50):\n    SMA1 = moving_average(series, N)\n    SMA2 = moving_average(series, int(N/2))\n    res = (2 * SMA2 - SMA1)\n    return np.mean(res[-int(np.sqrt(N)):])\n\ndef calculate_target(data: pd.DataFrame, details: pd.DataFrame, price_column: str):\n    ids = list(details.Asset_ID)\n    asset_names = list(details.Asset_Name)\n    weights = np.array(list(details.Weight))\n\n    all_timestamps = np.sort(data['timestamp'].unique())\n    targets = pd.DataFrame(index=all_timestamps)\n\n    for i, id in enumerate(ids):\n        asset = data[data.Asset_ID == id].set_index(keys='timestamp')\n        price = pd.Series(index=all_timestamps, data=asset[price_column])\n        targets[asset_names[i]] = (\n            price.shift(periods=1) /\n            price.shift(periods=16)\n        ) - 1\n    \n    targets['m'] = np.average(targets.fillna(0), axis=1, weights=weights)\n    \n    m = targets['m']\n\n    num = targets.multiply(m.values, axis=0).rolling(3750).mean().values\n    denom = m.multiply(m.values, axis=0).rolling(3750).mean().values\n    beta = np.nan_to_num(num.T / denom, nan=0., posinf=0., neginf=0.)\n\n    targets = targets - (beta * m.values).T\n    targets.drop('m', axis=1, inplace=True)\n    \n    return targets.reset_index(), num, denom, beta","metadata":{"execution":{"iopub.status.busy":"2022-01-25T20:23:11.345743Z","iopub.execute_input":"2022-01-25T20:23:11.346035Z","iopub.status.idle":"2022-01-25T20:23:11.381329Z","shell.execute_reply.started":"2022-01-25T20:23:11.346003Z","shell.execute_reply":"2022-01-25T20:23:11.38027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dictionary of DataFrame for every asset\n\n__This dictionary is not that important, the other one is the real deal here__","metadata":{}},{"cell_type":"code","source":"df_dict = {}\nfor asset in df_sup['Asset_ID'].unique():\n    #print(f\"Filling dictionary with asset {asset}\")\n    df_dict[asset] = get_features(df_sup.loc[df_sup.Asset_ID == asset].reset_index(drop=True))\n    asset_name = df_asset_details.loc[df_asset_details.Asset_ID == asset, \"Asset_Name\"].values[-1]","metadata":{"execution":{"iopub.status.busy":"2022-01-25T20:23:24.806659Z","iopub.execute_input":"2022-01-25T20:23:24.807473Z","iopub.status.idle":"2022-01-25T20:23:28.579051Z","shell.execute_reply.started":"2022-01-25T20:23:24.807419Z","shell.execute_reply":"2022-01-25T20:23:28.578239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dictionary of Features for every asset\n\n__I will update this dictionary with every batch in prediction phase. For now it is filled with latest values from the supplemental file__","metadata":{}},{"cell_type":"code","source":"f = {}\nfor k in df_dict.keys():\n    f[k] = {\n        \"all_close\": df_dict[k][\"Close\"].values[-5000:],\n        \"log_return\" : df_dict[k][\"log_return\"].values[-5000:],\n        \"Count\" : df_dict[k][\"Count\"].values[-240:],\n        \"Volume\" : df_dict[k][\"Volume\"].values[-240:],\n        \"target_return\" : df_dict[k][\"target_return\"].values[-3750:],\n    }","metadata":{"execution":{"iopub.status.busy":"2022-01-25T20:23:30.034976Z","iopub.execute_input":"2022-01-25T20:23:30.035286Z","iopub.status.idle":"2022-01-25T20:23:30.047473Z","shell.execute_reply.started":"2022-01-25T20:23:30.035255Z","shell.execute_reply":"2022-01-25T20:23:30.046387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\ndel df_dict\ndel df_sup\n#del rec_targets\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-25T20:23:34.019332Z","iopub.execute_input":"2022-01-25T20:23:34.01963Z","iopub.status.idle":"2022-01-25T20:23:34.155175Z","shell.execute_reply.started":"2022-01-25T20:23:34.0196Z","shell.execute_reply":"2022-01-25T20:23:34.154223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"env = gresearch_crypto.make_env()\niter_test = env.iter_test()","metadata":{"execution":{"iopub.status.busy":"2022-01-25T20:23:37.932381Z","iopub.execute_input":"2022-01-25T20:23:37.932686Z","iopub.status.idle":"2022-01-25T20:23:37.937737Z","shell.execute_reply.started":"2022-01-25T20:23:37.932654Z","shell.execute_reply":"2022-01-25T20:23:37.937058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def inference_target_recreation2(tt, weights):\n    m = np.average(tt, axis=1, weights=weights)\n    num = np.mean(np.multiply(tt, m.reshape(-1, 1)), axis=0)\n    denom = np.mean(np.multiply(m, m), axis=0)\n    beta = num / denom\n    t2 = (beta * m[-1]).T\n    p2 = np.mean(t2)\n    return t2, p2","metadata":{"execution":{"iopub.status.busy":"2022-01-25T20:23:40.590587Z","iopub.execute_input":"2022-01-25T20:23:40.590873Z","iopub.status.idle":"2022-01-25T20:23:40.59692Z","shell.execute_reply.started":"2022-01-25T20:23:40.590842Z","shell.execute_reply":"2022-01-25T20:23:40.596251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfibo_list = [55, 210, 340, 890, 3750]\nmarket_list = [1.]\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    pred_array = np.zeros((len(df_test), len(features)))\n    rownums = df_pred.row_id.values\n    #all_tars, market_p = inference_target_recreation2(tt, weights)\n    market_p = np.mean(market_list)\n    market_list = []\n    for jj, (j , row) in enumerate(df_test.iterrows()):\n        #Initial necessary values\n        #Initial necessary values\n        asset = row['Asset_ID']\n        last_close = row[\"Close\"]\n        last_log_return = np.log(last_close) - np.log(f[asset][\"all_close\"][-1])\n        #Dictionary updates\n        f[asset][\"all_close\"] = np.append(f[asset][\"all_close\"][1:], [last_close])\n        f[asset][\"log_return\"] = np.append(f[asset][\"log_return\"][1:], [last_log_return])\n        f[asset][\"Count\"] = np.append(f[asset][\"Count\"][1:], [row[\"Count\"]])\n        f[asset][\"Volume\"] = np.append(f[asset][\"Volume\"][1:], [row[\"Volume\"]])\n        ###logreturns\n        #fibo_list = [55, 89] + [210, 340, 890, 1440, 3750, 5000]\n        f[asset][\"log_return_1\"] = last_log_return\n        row[\"log_return\"] = last_log_return\n        for i in fibo_list:\n            row[f\"log_return_{i}\"] = np.mean(f[asset][\"log_return\"][-i:])\n            \n        #row['logret_std_55'] = np.std(f[asset][\"log_return\"][-55:])\n        #row['logret_std_3750'] = np.std(f[asset][\"log_return\"][-3750:])\n        \n        row[\"sma15\"] = np.mean(f[asset][\"all_close\"][-15:])/last_close -1 \n        row[\"sma60\"] = np.mean(f[asset][\"all_close\"][-60:])/last_close -1 \n        row[\"sma240\"] = np.mean(f[asset][\"all_close\"][-240:])/last_close -1 \n\n        row[\"return15\"] = last_close/f[asset][\"all_close\"][-15] -1\n        row[\"return60\"] = last_close/f[asset][\"all_close\"][-60] -1 \n        row[\"return240\"] = last_close/f[asset][\"all_close\"][-240] -1 \n\n        row[\"sma15_count\"] = np.mean(f[asset][\"Count\"][-15:])/row[\"Count\"] -1\n        row[\"sma60_count\"] = np.mean(f[asset][\"Count\"][-60:])/row[\"Count\"] -1\n        row[\"sma240_count\"] = np.mean(f[asset][\"Count\"][-240:])/row[\"Count\"] -1\n\n        row[\"return15_count\"] = row[\"Volume\"]/f[asset][\"Volume\"][-15] -1\n        row[\"return60_count\"] = row[\"Volume\"]/f[asset][\"Volume\"][-60] -1\n        row[\"return240_count\"] = row[\"Volume\"]/f[asset][\"Volume\"][-240] -1\n\n        row[\"hull\"] = last_close - calcHullMA_inference(f[asset][\"all_close\"][-260:], 240)\n\n        #row[\"rec_target\"], row[\"market_p\"] = inference_target_recreation(tt, weights, int(asset))\n        #row[\"rec_target\"], row[\"market_p\"] = all_tars[int(asset)], market_p\n        row[\"target_return\"] = (last_close / f[asset][\"all_close\"][-16]) -1\n        market_list.append(row[\"target_return\"])\n        row[\"market_p\"] = market_p\n        row[\"train_flg\"] = 1\n        #df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = model.predict([row[features]], num_iteration=model.best_iteration)[0]\n        pred_array[jj, :] = row[features].values\n    preds = model.predict(pred_array, num_iteration=model.best_iteration) * 0.5\n    preds += model_oldest.predict(pred_array, num_iteration=model.best_iteration) * 0.25\n    preds += model_fewer.predict(pred_array, num_iteration=model.best_iteration) * 0.25\n    df_pred.loc[df_pred[\"row_id\"].isin(rownums), \"Target\"] = preds\n    env.predict(df_pred)","metadata":{"execution":{"iopub.status.busy":"2022-01-25T20:25:35.117114Z","iopub.execute_input":"2022-01-25T20:25:35.11742Z","iopub.status.idle":"2022-01-25T20:25:35.268357Z","shell.execute_reply.started":"2022-01-25T20:25:35.117385Z","shell.execute_reply":"2022-01-25T20:25:35.267471Z"},"trusted":true},"execution_count":null,"outputs":[]}]}