{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport pandas as pd # For example, 資料前處理好工具\nimport numpy as np # For example, 建立矩陣並處理大量的矩陣運算的工具\nimport time #字面上的，時間模組\nimport gc #廢資料回收\n\n# from sklearn.linear_model import RidgeCV, Ridge\n# from sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import StandardScaler #導入縮放器\n# from sklearn.model_selection import train_test_split\n\nfrom lightgbm import LGBMRegressor #導入 LGBM 回歸器\n# from xgboost import XGBRegressor\nfrom sklearn.model_selection import TimeSeriesSplit \nfrom scipy.stats.stats import pearsonr\nfrom tqdm import tqdm\n\nimport gresearch_crypto\n\n# Warningの無効化\nimport warnings\nwarnings.simplefilter(\"ignore\") \n\n# データフレームcolumの全表示 \n# 數據框列的全視圖\npd.set_option(\"display.max_columns\", None)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n# 顯示所有匯入了的檔案有哪些\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-04-12T16:05:46.368932Z","iopub.execute_input":"2022-04-12T16:05:46.369515Z","iopub.status.idle":"2022-04-12T16:05:48.220997Z","shell.execute_reply.started":"2022-04-12T16:05:46.369376Z","shell.execute_reply":"2022-04-12T16:05:48.220009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## References\n\n1. https://www.kaggle.com/sohier/detailed-api-introduction\n1. https://www.kaggle.com/yamqwe/crypto-prediction-technical-analysis-features\n1. https://www.kaggle.com/yamqwe/feature-engineering-rolling-aggregations\n1. https://www.kaggle.com/yamqwe/crypto-prediction-technical-analysis-feats-2\n1. https://www.kaggle.com/whitecat0022/g-research-crypto-forecasting-ridge\n1. https://www.kaggle.com/hyewon328/time-series-modeling-guide-with-cv-and-optuna\n1. https://stackoverflow.com/questions/42138357/pandas-rolling-slope-calculation\n1. https://stackoverflow.com/questions/9637838/convert-string-date-to-timestamp-in-python","metadata":{}},{"cell_type":"markdown","source":"## Load Data","metadata":{}},{"cell_type":"code","source":"def reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.\n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage of dataframe is {:.2f} MB\".format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == \"int\":\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        # else:\n            # df[col] = df[col].astype('category')\n            # 改變資料型態的方式減少RAM使用量\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage after optimization is: {:.2f} MB\".format(end_mem))\n    print(\"Decreased by {:.1f}%\".format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:08:38.230621Z","iopub.execute_input":"2022-04-12T16:08:38.230874Z","iopub.status.idle":"2022-04-12T16:08:38.242692Z","shell.execute_reply.started":"2022-04-12T16:08:38.230850Z","shell.execute_reply":"2022-04-12T16:08:38.241780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 讀取各資產名稱及權重資訊\ndf_asset_details = pd.read_csv(r\"/kaggle/input/g-research-crypto-forecasting/asset_details.csv\").sort_values(\"Asset_ID\")\ndf_asset_details","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:10:09.077225Z","iopub.execute_input":"2022-04-12T16:10:09.077529Z","iopub.status.idle":"2022-04-12T16:10:09.111047Z","shell.execute_reply.started":"2022-04-12T16:10:09.077496Z","shell.execute_reply":"2022-04-12T16:10:09.110073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# val = df_asset_details.loc[ df_asset_details[\"Asset_ID\"] == 3, \"Asset_Name\"]\n# val.values[0]","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:10:14.646729Z","iopub.execute_input":"2022-04-12T16:10:14.647188Z","iopub.status.idle":"2022-04-12T16:10:14.650806Z","shell.execute_reply.started":"2022-04-12T16:10:14.647153Z","shell.execute_reply":"2022-04-12T16:10:14.649731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_csv_strict(file_name=r\"/kaggle/input/g-research-crypto-forecasting/train.csv\"):\n    df = pd.read_csv(file_name).pipe(reduce_mem_usage)\n    df[\"datetime\"] = pd.to_datetime(df[\"timestamp\"], unit=\"s\")\n#     df = df[df[\"datetime\"] > \"2019-06-13 00:00:00\"]\n    df = df[df[\"datetime\"] >= \"2020-05-13 00:00:00\"]\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:10:19.185018Z","iopub.execute_input":"2022-04-12T16:10:19.185314Z","iopub.status.idle":"2022-04-12T16:10:19.190468Z","shell.execute_reply.started":"2022-04-12T16:10:19.185281Z","shell.execute_reply":"2022-04-12T16:10:19.189620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#讀入訓練資料集\ndf_train = read_csv_strict()\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:16:29.891690Z","iopub.execute_input":"2022-04-12T16:16:29.892358Z","iopub.status.idle":"2022-04-12T16:17:31.029966Z","shell.execute_reply.started":"2022-04-12T16:16:29.892313Z","shell.execute_reply":"2022-04-12T16:17:31.029015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#讀入比賽結束後的訓練資料（後續資料自動訓練使用）\ndf_supp_train = read_csv_strict(file_name=r\"/kaggle/input/g-research-crypto-forecasting/supplemental_train.csv\")\ndf_supp_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:19:13.560191Z","iopub.execute_input":"2022-04-12T16:19:13.560587Z","iopub.status.idle":"2022-04-12T16:19:20.191896Z","shell.execute_reply.started":"2022-04-12T16:19:13.560548Z","shell.execute_reply":"2022-04-12T16:19:20.190138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 範例數據輸入\ndf_test = read_csv_strict(file_name=r\"/kaggle/input/g-research-crypto-forecasting/example_test.csv\")\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:19:26.488104Z","iopub.execute_input":"2022-04-12T16:19:26.488935Z","iopub.status.idle":"2022-04-12T16:19:26.524638Z","shell.execute_reply.started":"2022-04-12T16:19:26.488895Z","shell.execute_reply":"2022-04-12T16:19:26.523550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_train[\"timestamp\"].min(), df_train[\"timestamp\"].max()) #訓練數據時間範圍","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:19:37.550769Z","iopub.execute_input":"2022-04-12T16:19:37.551683Z","iopub.status.idle":"2022-04-12T16:19:37.577220Z","shell.execute_reply.started":"2022-04-12T16:19:37.551626Z","shell.execute_reply":"2022-04-12T16:19:37.576350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_supp_train[\"timestamp\"].min(), df_supp_train[\"timestamp\"].max()) #後來的訓練數據時間範圍","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:19:42.319695Z","iopub.execute_input":"2022-04-12T16:19:42.320548Z","iopub.status.idle":"2022-04-12T16:19:42.332833Z","shell.execute_reply.started":"2022-04-12T16:19:42.320495Z","shell.execute_reply":"2022-04-12T16:19:42.331747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.concat([df_train, df_supp_train, df_test])\ndf_train.head()\n# 合併資料","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:19:54.223371Z","iopub.execute_input":"2022-04-12T16:19:54.223663Z","iopub.status.idle":"2022-04-12T16:19:54.900105Z","shell.execute_reply.started":"2022-04-12T16:19:54.223634Z","shell.execute_reply":"2022-04-12T16:19:54.899324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.shape, df_supp_train.shape, df_test.shape","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:19:59.217646Z","iopub.execute_input":"2022-04-12T16:19:59.217918Z","iopub.status.idle":"2022-04-12T16:19:59.224307Z","shell.execute_reply.started":"2022-04-12T16:19:59.217892Z","shell.execute_reply":"2022-04-12T16:19:59.223410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df_supp_train, df_test\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:20:05.453064Z","iopub.execute_input":"2022-04-12T16:20:05.453327Z","iopub.status.idle":"2022-04-12T16:20:05.675754Z","shell.execute_reply.started":"2022-04-12T16:20:05.453298Z","shell.execute_reply":"2022-04-12T16:20:05.674964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.memory_usage().sum() / 1024**3 #檢查記憶體使用量","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:20:38.311575Z","iopub.execute_input":"2022-04-12T16:20:38.311892Z","iopub.status.idle":"2022-04-12T16:20:38.321409Z","shell.execute_reply.started":"2022-04-12T16:20:38.311858Z","shell.execute_reply":"2022-04-12T16:20:38.320518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Outliers 異常值","metadata":{}},{"cell_type":"code","source":"## Negative volume implies lack of liquidity in the market\ndf_train[df_train.Volume < 0] #取出交易量比0小的資料","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:20:48.293401Z","iopub.execute_input":"2022-04-12T16:20:48.293671Z","iopub.status.idle":"2022-04-12T16:20:48.766953Z","shell.execute_reply.started":"2022-04-12T16:20:48.293643Z","shell.execute_reply":"2022-04-12T16:20:48.766205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Negative volume indicates that the market lacked liquidity for these instances; in other words, the bid and ask price  failed to overlap.","metadata":{}},{"cell_type":"markdown","source":"## Check for NaNs 處理空白資料","metadata":{}},{"cell_type":"code","source":"## https://www.kaggle.com/c/g-research-crypto-forecasting/discussion/286778\n## https://www.kaggle.com/cstein06/tutorial-to-the-g-research-crypto-competition?scriptVersionId=84976518&cellId=51\n\n# df_train[(df_train.Target.isna()) & (df_train.Asset_ID==8)]\ndf_train[df_train.Target.isna()].groupby(\"Asset_ID\")[\"timestamp\"].count()","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:21:25.879919Z","iopub.execute_input":"2022-04-12T16:21:25.880185Z","iopub.status.idle":"2022-04-12T16:21:25.972898Z","shell.execute_reply.started":"2022-04-12T16:21:25.880157Z","shell.execute_reply":"2022-04-12T16:21:25.972032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[(df_train.Target.isna()) & (df_train.Asset_ID==8)].describe(percentiles=[0.25, 0.50, 0.75, 0.90, 0.95, 0.99])","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:21:31.156430Z","iopub.execute_input":"2022-04-12T16:21:31.157013Z","iopub.status.idle":"2022-04-12T16:21:31.318862Z","shell.execute_reply.started":"2022-04-12T16:21:31.156977Z","shell.execute_reply":"2022-04-12T16:21:31.317935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Sort by Asset and Timestamp 時間排序","metadata":{}},{"cell_type":"code","source":"df_train = df_train.sort_values(by=[\"Asset_ID\", \"timestamp\"]\n                               ,ascending=True)\n\ndf_train.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:23:40.496579Z","iopub.execute_input":"2022-04-12T16:23:40.497127Z","iopub.status.idle":"2022-04-12T16:23:48.361488Z","shell.execute_reply.started":"2022-04-12T16:23:40.497091Z","shell.execute_reply":"2022-04-12T16:23:48.360672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dealing with missing data 遺失數據處理","metadata":{}},{"cell_type":"code","source":"df_train.info()","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:24:13.899143Z","iopub.execute_input":"2022-04-12T16:24:13.899431Z","iopub.status.idle":"2022-04-12T16:24:13.914351Z","shell.execute_reply.started":"2022-04-12T16:24:13.899397Z","shell.execute_reply":"2022-04-12T16:24:13.913508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"total missing: \")\nprint(df_train.isna().sum())\nprint(\"----------------------------\")\nprint(\"percent missing: \")\nprint(df_train.isna().sum() * 100. / df_train.shape[0])","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:25:19.834900Z","iopub.execute_input":"2022-04-12T16:25:19.835207Z","iopub.status.idle":"2022-04-12T16:25:20.518463Z","shell.execute_reply.started":"2022-04-12T16:25:19.835177Z","shell.execute_reply":"2022-04-12T16:25:20.517412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:32:23.149624Z","iopub.execute_input":"2022-04-12T16:32:23.149965Z","iopub.status.idle":"2022-04-12T16:32:23.168489Z","shell.execute_reply.started":"2022-04-12T16:32:23.149931Z","shell.execute_reply":"2022-04-12T16:32:23.167592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ffill for each group for Target missing value 填滿遺失值","metadata":{}},{"cell_type":"code","source":"## https://stackoverflow.com/questions/16345583/fill-in-missing-pandas-data-with-previous-non-missing-value-grouped-by-key\ndf_train[\"Target\"] = df_train.sort_values([\"Asset_ID\", \"timestamp\"]).groupby(\"Asset_ID\")[\"Target\"].ffill()\ndf_train[\"VWAP\"] = df_train.sort_values([\"Asset_ID\", \"timestamp\"]).groupby(\"Asset_ID\")[\"VWAP\"].ffill()\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:32:33.167263Z","iopub.execute_input":"2022-04-12T16:32:33.167967Z","iopub.status.idle":"2022-04-12T16:32:44.160681Z","shell.execute_reply.started":"2022-04-12T16:32:33.167932Z","shell.execute_reply":"2022-04-12T16:32:44.159855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:33:42.149146Z","iopub.execute_input":"2022-04-12T16:33:42.149480Z","iopub.status.idle":"2022-04-12T16:33:42.487430Z","shell.execute_reply.started":"2022-04-12T16:33:42.149428Z","shell.execute_reply":"2022-04-12T16:33:42.486621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[ (df_train[\"Target\"].isna()) & (df_train[\"Asset_ID\"] == 10)]","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:33:46.596905Z","iopub.execute_input":"2022-04-12T16:33:46.597160Z","iopub.status.idle":"2022-04-12T16:33:46.701933Z","shell.execute_reply.started":"2022-04-12T16:33:46.597132Z","shell.execute_reply":"2022-04-12T16:33:46.701207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"Target\"] = df_train[\"Target\"].fillna(0)\n# df_train[\"VWAP\"] = df_train[\"VWAP\"].fillna(0)","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:35:04.158358Z","iopub.execute_input":"2022-04-12T16:35:04.158695Z","iopub.status.idle":"2022-04-12T16:35:04.242666Z","shell.execute_reply.started":"2022-04-12T16:35:04.158654Z","shell.execute_reply":"2022-04-12T16:35:04.241707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[ (df_train[\"Target\"].isna()) & (df_train[\"Asset_ID\"] == 10)]","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:35:05.745297Z","iopub.execute_input":"2022-04-12T16:35:05.745596Z","iopub.status.idle":"2022-04-12T16:35:05.826359Z","shell.execute_reply.started":"2022-04-12T16:35:05.745564Z","shell.execute_reply":"2022-04-12T16:35:05.825421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"Target\"].isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:35:08.038604Z","iopub.execute_input":"2022-04-12T16:35:08.038899Z","iopub.status.idle":"2022-04-12T16:35:08.103650Z","shell.execute_reply.started":"2022-04-12T16:35:08.038870Z","shell.execute_reply":"2022-04-12T16:35:08.102833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utility Functions 技術分析的程式","metadata":{}},{"cell_type":"code","source":"# Two new features from the competition tutorial 比賽教學教的\ndef upper_shadow(df):\n    return df[\"High\"] - np.maximum(df[\"Close\"], df[\"Open\"])\n\ndef lower_shadow(df):\n    return np.minimum(df[\"Close\"], df[\"Open\"]) - df[\"Low\"]\n\n## notebook: crypto-prediction-technical-analysis-features 來自於上述筆記本的指標\ndef SM_A_M(df, colname, n):\n    mean = df[colname].rolling(window=n).mean()\n    median = df[colname].rolling(window=n).median()\n    \n    return mean, median\n\ndef EMA1(df, colname, n):\n    ## EMA指標\n    \"\"\"\n    https://qiita.com/MuAuan/items/b08616a841be25d29817\n    \"\"\"\n    a= 2/(n+1)\n    return df[colname].ewm(alpha=a).mean()\n\ndef MACD(df, colname, span1=12, span2=26, span3=9):\n    ## MACD\n    \"\"\"\n    Compute MACD\n    # https://www.learnpythonwithrune.org/pandas-calculate-the-moving-average-convergence-divergence-macd-for-a-stock/\n    \"\"\"\n    \n    exp1 = EMA1(df, colname, span1)\n    exp2 = EMA1(df, colname, span2)\n    macd = 100 * (exp1 - exp2) / exp2\n    signal = macd.ewm(alpha=2./(span3+1)).mean() ##EMA1(df, colname, macd, span3)\n\n    return macd, signal\n## 下面是布林通道\ndef BollingerBand(df, colname, window, no_of_std):\n    mean = df[colname].rolling(window=window).mean()\n    std = df[colname].rolling(window=window).std() \n    bb_high = mean + no_of_std * std\n    bb_low = mean - no_of_std * std\n    \n    return bb_high, bb_low, std\n## RSI\ndef rsiFunc(df, colname, n=14):\n    prices = df[colname].values\n    \n    deltas = np.diff(prices)\n    seed = deltas[:n+1]\n    up = seed[seed>=0].sum()/n\n    down = -seed[seed<0].sum()/n\n    rs = up/down\n    rsi = np.zeros_like(prices)\n    rsi[:n] = 100. - 100./(1.+rs)\n\n    for i in range(n, len(prices)):\n        delta = deltas[i-1] # cause the diff is 1 shorter\n\n        if delta>0:\n            upval = delta\n            downval = 0.\n        else:\n            upval = 0.\n            downval = -delta\n\n        up = (up*(n-1) + upval)/n\n        down = (down*(n-1) + downval)/n\n\n        rs = up/down\n        rsi[i] = 100. - 100./(1.+rs)\n\n    return rsi\n\n## https://stackoverflow.com/questions/42138357/pandas-rolling-slope-calculation 計算斜率 時間序5分鐘的部分\ndef calc_slope(x):\n    slope = np.polyfit(range(len(x)), x, 1)[0]\n    return slope","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:42:53.361143Z","iopub.execute_input":"2022-04-12T16:42:53.361977Z","iopub.status.idle":"2022-04-12T16:42:53.376474Z","shell.execute_reply.started":"2022-04-12T16:42:53.361927Z","shell.execute_reply":"2022-04-12T16:42:53.375480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## set min_periods=2 to allow subsets less than 60.\n## use [4::5] to select the results you need.\n# result = data.rolling(60, min_periods=2).apply(calc_slope)[4::5]","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:35:16.935048Z","iopub.execute_input":"2022-04-12T16:35:16.935780Z","iopub.status.idle":"2022-04-12T16:35:16.940514Z","shell.execute_reply.started":"2022-04-12T16:35:16.935726Z","shell.execute_reply":"2022-04-12T16:35:16.939799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineering 特徵工程","metadata":{}},{"cell_type":"code","source":"def get_feat(df):\n    df[\"High-Low\"] = df[\"High\"] - df[\"Low\"]\n    df[\"Close-Open\"] = df[\"Close\"] - df[\"Open\"]\n\n    df[\"dayofweek\"] = df[\"datetime\"].dt.dayofweek\n    df[\"weekofyear\"] = df[\"datetime\"].dt.weekofyear\n    # df_train['upper_shadow'] = upper_shadow(df=df_train)\n    # df_train['lower_shadow'] = lower_shadow(df=df_train)\n\n    df[\"dayofweek\"] = df[\"dayofweek\"].astype(np.int8)\n    df[\"weekofyear\"] = df[\"weekofyear\"].astype(np.int8)\n    # df_train['upper_shadow'] = df_train['upper_shadow'].astype(np.float32)\n    # df_train['lower_shadow'] = df_train['lower_shadow'].astype(np.float32)\n\n    df = df.drop([\"datetime\", \"VWAP\", \"Open\", \"High\", \"Low\"], axis=1)\n    gc.collect()\n    # df_train = reduce_mem_usage(df_train)\n    # df_train.describe()\n\n    macd, signal = MACD(df=df, colname=\"Close\")\n    df['MACD_' + \"Close\" + '_macd'] = macd\n    df['MACD_' + \"Close\" + '_signal'] = signal\n\n    df['MACD_' + \"Close\" + '_macd'] = df['MACD_' + \"Close\" + '_macd'].astype(np.float32)\n    df['MACD_' + \"Close\" + '_signal'] = df['MACD_' + \"Close\" + '_signal'].astype(np.float32)\n\n    arr_n = [5, 10, 15]\n    # arr_cols = [\"Close\", \"VWAP\", \"Volume\"]\n    arr_cols = [\"Close\", \"Volume\"]\n    no_of_std = 2.5\n\n    for colname in arr_cols: \n        print(\"colname = \" + colname)\n        for n in arr_n:\n            print(\"n = \"+ str(n))\n            df['Target_lag_' + str(n)] = df['Target'].shift(n)\n            df['Target_lag_' + str(n)].fillna(value=df[\"Target\"], inplace=True)\n            \n#             print(\"Slope\") ## takes too long\n#             slope = df[\"Target\"].rolling(window=n).apply(calc_slope)[4::5]\n#             df['SM_' + \"Target\" + '_slope_' + str(n)] = slope\n#             df['SM_' + \"Target\" + '_slope_' + str(n)].fillna(value=0.1, inplace=True)\n#             df['SM_' + \"Target\" + '_slope_' + str(n)] = df['SM_' + \"Target\" + '_slope_' + str(n)].astype(np.float32)\n\n            print(\"SMA\")\n            mean, median = SM_A_M(df=df, colname=colname, n=n)\n            df['SM_' + colname + '_mean_' + str(n)] = mean\n            df['SM_' + colname + '_median_' + str(n)] = median\n\n            df['SM_' + colname + '_mean_' + str(n)] = df['SM_' + colname + '_mean_' + str(n)].astype(np.float32)\n            df['SM_' + colname + '_median_' + str(n)] = df['SM_' + colname + '_median_' + str(n)].astype(np.float32)\n\n            df['SM_' + colname + '_mean_' + str(n)].fillna(value=df[colname], inplace=True)\n            df['SM_' + colname + '_median_' + str(n)].fillna(value=df[colname], inplace=True)\n            \n\n            print(\"EMA1\")\n            ewmean = EMA1(df=df_train, colname=colname, n=n)\n            df['EWM_' + colname + '_mean_' + str(n)] = ewmean\n            df['EWM_' + colname + '_mean_' + str(n)] = df['EWM_' + colname + '_mean_' + str(n)].astype(np.float32)\n            df['EWM_' + colname + '_mean_' + str(n)].fillna(value=1, inplace=True)\n            \n            if colname in ['VWAP', 'Volume']:\n                continue\n\n            if n==5:\n                continue\n                \n            print(\"RSI\")\n            rsi = rsiFunc(df=df,  colname=colname, n=n)\n            df['RSI_' + str(n)] = rsi\n            df['RSI_' + str(n)] = df['RSI_' + str(n)].astype(np.float32)\n            df['RSI_' + str(n)].fillna(value=1, inplace=True)\n            \n\n            print(\"BollingerBand\")\n            bb_high, bb_low, std = BollingerBand(df=df, colname=colname, window=n, no_of_std=no_of_std)\n            df['SM_' + colname + '_std_' + str(n)] = std\n            df['SM_' + colname + '_BB_High_' + str(n)] = bb_high\n            df['SM_' + colname + '_BB_Low_' + str(n)] = bb_low\n\n            df['SM_' + colname + '_std_' + str(n)] = df['SM_' + colname + '_std_' + str(n)].astype(np.float32)\n            df['SM_' + colname + '_BB_High_' + str(n)] = df['SM_' + colname + '_BB_High_' + str(n)].astype(np.float32)\n            df['SM_' + colname + '_BB_Low_' + str(n)] = df['SM_' + colname + '_BB_Low_' + str(n)].astype(np.float32)\n            df['SM_' + colname + '_std_' + str(n)].fillna(value=0, inplace=True)\n            df['SM_' + colname + '_BB_High_' + str(n)].fillna(value=df[colname], inplace=True) \n            df['SM_' + colname + '_BB_Low_' + str(n)].fillna(value=df[colname], inplace=True) \n            \n            \n            gc.collect()\n\n    df = reduce_mem_usage(df)\n#     df.fillna(value=0, inplace=True)\n    df.head()\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:50:25.886041Z","iopub.execute_input":"2022-04-12T16:50:25.886622Z","iopub.status.idle":"2022-04-12T16:50:25.907957Z","shell.execute_reply.started":"2022-04-12T16:50:25.886573Z","shell.execute_reply":"2022-04-12T16:50:25.906764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.describe()","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:50:30.618260Z","iopub.execute_input":"2022-04-12T16:50:30.618551Z","iopub.status.idle":"2022-04-12T16:50:36.757288Z","shell.execute_reply.started":"2022-04-12T16:50:30.618516Z","shell.execute_reply":"2022-04-12T16:50:36.755526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = get_feat(df=df_train)\ndf_train.memory_usage(deep=True)","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:51:29.039512Z","iopub.execute_input":"2022-04-12T16:51:29.039811Z","iopub.status.idle":"2022-04-12T16:53:39.199866Z","shell.execute_reply.started":"2022-04-12T16:51:29.039783Z","shell.execute_reply":"2022-04-12T16:53:39.199248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.info(verbose=False, memory_usage='deep')","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:54:17.128308Z","iopub.execute_input":"2022-04-12T16:54:17.129050Z","iopub.status.idle":"2022-04-12T16:54:17.139187Z","shell.execute_reply.started":"2022-04-12T16:54:17.128994Z","shell.execute_reply":"2022-04-12T16:54:17.138497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:55:12.268574Z","iopub.execute_input":"2022-04-12T16:55:12.268840Z","iopub.status.idle":"2022-04-12T16:55:12.390411Z","shell.execute_reply.started":"2022-04-12T16:55:12.268812Z","shell.execute_reply":"2022-04-12T16:55:12.389514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import memory_profiler\nm1 = memory_profiler.memory_usage()\nm1","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:55:50.813736Z","iopub.execute_input":"2022-04-12T16:55:50.814019Z","iopub.status.idle":"2022-04-12T16:55:50.919851Z","shell.execute_reply.started":"2022-04-12T16:55:50.813992Z","shell.execute_reply":"2022-04-12T16:55:50.919162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:55:53.560242Z","iopub.execute_input":"2022-04-12T16:55:53.560925Z","iopub.status.idle":"2022-04-12T16:55:53.604761Z","shell.execute_reply.started":"2022-04-12T16:55:53.560875Z","shell.execute_reply":"2022-04-12T16:55:53.604153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:56:15.750913Z","iopub.execute_input":"2022-04-12T16:56:15.751471Z","iopub.status.idle":"2022-04-12T16:56:15.885725Z","shell.execute_reply.started":"2022-04-12T16:56:15.751415Z","shell.execute_reply":"2022-04-12T16:56:15.884957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = df_train[~((df_train[\"group_num\"].isna()) & (df_train[\"row_id\"].isna()))]\nprint(df_test.shape)\n\ngc.collect()\n\ndf_train.drop(  df_train[~((df_train[\"group_num\"].isna()) & (df_train[\"row_id\"].isna()))].index, inplace=True)\nprint(df_train.shape)\n\ndf_test.drop([\"Target\"], axis=1, inplace=True)\ndf_train.drop([\"group_num\", \"row_id\"], axis=1, inplace=True)\n\ndf_train.shape, df_test.shape","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:56:30.920648Z","iopub.execute_input":"2022-04-12T16:56:30.920946Z","iopub.status.idle":"2022-04-12T16:56:40.930280Z","shell.execute_reply.started":"2022-04-12T16:56:30.920904Z","shell.execute_reply":"2022-04-12T16:56:40.929374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:57:21.639167Z","iopub.execute_input":"2022-04-12T16:57:21.639463Z","iopub.status.idle":"2022-04-12T16:57:21.674034Z","shell.execute_reply.started":"2022-04-12T16:57:21.639418Z","shell.execute_reply":"2022-04-12T16:57:21.673501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:57:36.913074Z","iopub.execute_input":"2022-04-12T16:57:36.913374Z","iopub.status.idle":"2022-04-12T16:57:38.325688Z","shell.execute_reply.started":"2022-04-12T16:57:36.913340Z","shell.execute_reply":"2022-04-12T16:57:38.324841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training with LGBM","metadata":{}},{"cell_type":"code","source":"# 10-fold time series cross validation\ndef timecv_model(model, X, y):\n    tfold = TimeSeriesSplit(n_splits = 5)\n    pcc_list = []\n    for _, (train_index, test_index) in tqdm(enumerate(tfold.split(X), start=1)):\n        X_train, X_test = X.iloc[train_index], X.iloc[test_index]\n        y_train, y_test = y.iloc[train_index], y.iloc[test_index]\n        clf = model.fit(X_train, y_train)\n        pred = clf.predict(X_test)\n        pcc = pearsonr(pred, y_test) \n        pcc_list.append(pcc[0])\n    \n    return pcc_list\n\ndef cv_result(model, X, y):\n    model_name = model.__class__.__name__\n    pcc_ = timecv_model(model, X, y)\n    for i, pcc in enumerate(pcc_):\n        print(f'{i}th fold: {model_name} PCC: {pcc:.4f}')\n    print(f'\\n{model_name} average PCC: {np.mean(pcc_):.4f}')","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:58:17.583780Z","iopub.execute_input":"2022-04-12T16:58:17.584273Z","iopub.status.idle":"2022-04-12T16:58:17.592238Z","shell.execute_reply.started":"2022-04-12T16:58:17.584161Z","shell.execute_reply":"2022-04-12T16:58:17.591495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_lgb(model, X, y):\n    print(\"model fitting ...\")\n    clf = model.fit(X, y)\n    print(\"predicting ...\")\n    pred = clf.predict(X)\n    pcc = pearsonr(pred, y)[0]\n    \n    print(\"pcc = \", str(pcc))\n    gc.collect()\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:58:50.326960Z","iopub.execute_input":"2022-04-12T16:58:50.327419Z","iopub.status.idle":"2022-04-12T16:58:50.332866Z","shell.execute_reply.started":"2022-04-12T16:58:50.327385Z","shell.execute_reply":"2022-04-12T16:58:50.332126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train = df_train[df_train['timestamp'] < 1630432800] ## original train data\n# valid = df_train[df_train['timestamp'] >= 1630432800] ## supplementary data\n\n# train.drop(['timestamp', 'Asset_ID'], axis = 1, inplace = True)\n# valid.drop(['timestamp', 'Asset_ID'], axis = 1, inplace = True)\n\n# X_train = train.drop(['Target'], axis = 1)\n# y_train = train['Target']\n# X_valid = valid.drop(['Target'], axis = 1)\n# y_valid = valid['Target']\n\n# print(X_train.shape)\n# print(X_valid.shape)","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:58:53.221970Z","iopub.execute_input":"2022-04-12T16:58:53.222417Z","iopub.status.idle":"2022-04-12T16:58:53.225809Z","shell.execute_reply.started":"2022-04-12T16:58:53.222385Z","shell.execute_reply":"2022-04-12T16:58:53.225142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# lgb_model = LGBMRegressor(n_estimators = 420,\n#                           max_depth = 17,\n#                           num_leaves = 36,\n#                           learning_rate = 0.09786397238905313, \n#                           min_child_samples = 49, \n#                           colsample_bytree=0.7,\n#                           subsample = 0.8683175057718733,\n#                           seed = 0)\n\n## 'colsample_bytree': 0.4000480946836777, 'max_depth': 17, 'num_leaves': 35, 'learning_rate': 0.06891972238739223, 'n_estimators': 576, 'min_child_samples': 47, 'subsample': 0.6058272745943716\n\nlgb_model = LGBMRegressor(n_estimators = 576,\n                          max_depth = 17,\n                          num_leaves = 35,\n                          learning_rate = 0.06891972238739223, \n                          min_child_samples = 47, \n                          colsample_bytree=0.4000480946836777,\n                          subsample = 0.6058272745943716,\n                          seed = 42)\n\n# cv_result(lgb_model, X, y)\n# lgb_model.fit(X_train, y_train, eval_set = [(X_train, y_train), (X_valid, y_valid)],\n#              verbose = 0, early_stopping_rounds = 50)\n# pred = model.predict(X_valid)\n# pcc = pearsonr(pred, y_valid)[0]\n# print(pcc)\ny = df_train[\"Target\"]\nX = df_train.drop(['timestamp', \"Target\"], axis=1)\n\ngc.collect()\n\nmodel = train_lgb(lgb_model, X, y)\nmodel","metadata":{"execution":{"iopub.status.busy":"2022-04-12T16:58:56.038177Z","iopub.execute_input":"2022-04-12T16:58:56.038514Z","iopub.status.idle":"2022-04-12T17:06:01.895435Z","shell.execute_reply.started":"2022-04-12T16:58:56.038475Z","shell.execute_reply":"2022-04-12T17:06:01.894612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_prediction(row):\n    y_pred = df_test.loc[((df_test[\"timestamp\"] == row[\"timestamp\"]) & (df_test[\"Asset_ID\"] == row[\"Asset_ID\"])), \"Target\"]\n    print(y_pred.values[0])\n    return y_pred.values[0]","metadata":{"execution":{"iopub.status.busy":"2022-04-12T17:06:13.363725Z","iopub.execute_input":"2022-04-12T17:06:13.364537Z","iopub.status.idle":"2022-04-12T17:06:13.370383Z","shell.execute_reply.started":"2022-04-12T17:06:13.364435Z","shell.execute_reply":"2022-04-12T17:06:13.369432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Hyperparameter Tuning with Optuna 超參數調整","metadata":{}},{"cell_type":"code","source":"# train = df_train[df_train['timestamp'] < '2020-09-20']\n# valid = df_train[df_train['timestamp'] >= '2020-09-20']\n\n# train.drop(['timestamp', 'Asset_ID'], axis = 1, inplace = True)\n# valid.drop(['timestamp', 'Asset_ID'], axis = 1, inplace = True)\n\n# X_train = train.drop(['Target'], axis = 1)\n# y_train = train['Target']\n# X_valid = valid.drop(['Target'], axis = 1)\n# y_valid = valid['Target']\n\n# print(X_train.shape)\n# print(X_valid.shape)","metadata":{"execution":{"iopub.status.busy":"2022-04-12T17:06:22.847282Z","iopub.execute_input":"2022-04-12T17:06:22.848029Z","iopub.status.idle":"2022-04-12T17:06:22.853929Z","shell.execute_reply.started":"2022-04-12T17:06:22.847963Z","shell.execute_reply":"2022-04-12T17:06:22.852758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from optuna.samplers import TPESampler\n# import optuna\n\n# sampler = TPESampler(seed = 0)\n\n# def objective(trial):\n#     params = {\n#         'objective': 'regression',\n#         'verbose': -1,\n#         'max_depth': trial.suggest_int('max_depth',5, 20),\n#         'num_leaves': trial.suggest_int('num_leaves', 10, 40),\n#         'learning_rate': trial.suggest_float(\"learning_rate\", 1e-5, 0.1),\n#         'n_estimators': trial.suggest_int('n_estimators', 500, 2500),\n#         'min_child_samples': trial.suggest_int('min_child_samples', 5, 100),\n#         'subsample': trial.suggest_float('subsample', 0.4, 1)}\n    \n#     model = LGBMRegressor(**params)\n#     model.fit(X_train, y_train, eval_set = [(X_train, y_train), (X_valid, y_valid)],\n#              verbose = 0, early_stopping_rounds = 50)\n#     pred = model.predict(X_valid)\n#     pcc = pearsonr(pred, y_valid)[0]\n#     return pcc\n\n# study_model = optuna.create_study(direction = 'maximize', sampler = sampler)\n# study_model.optimize(objective, n_trials = 20) ","metadata":{"execution":{"iopub.status.busy":"2022-03-30T04:49:52.130874Z","iopub.execute_input":"2022-03-30T04:49:52.13129Z","iopub.status.idle":"2022-03-30T04:49:52.14529Z","shell.execute_reply.started":"2022-03-30T04:49:52.131255Z","shell.execute_reply":"2022-03-30T04:49:52.144328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # select best trial and parameter\n# trial = study_model.best_trial\n# best_params = trial.params\n\n# print('Best params from optuna: \\n', best_params)","metadata":{"execution":{"iopub.status.busy":"2022-03-30T04:49:52.146415Z","iopub.execute_input":"2022-03-30T04:49:52.146923Z","iopub.status.idle":"2022-03-30T04:49:52.158648Z","shell.execute_reply.started":"2022-03-30T04:49:52.146888Z","shell.execute_reply":"2022-03-30T04:49:52.157752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# optuna.visualization.plot_optimization_history(study_model)","metadata":{"execution":{"iopub.status.busy":"2022-03-30T04:49:52.159918Z","iopub.execute_input":"2022-03-30T04:49:52.160286Z","iopub.status.idle":"2022-03-30T04:49:52.17183Z","shell.execute_reply.started":"2022-03-30T04:49:52.160251Z","shell.execute_reply":"2022-03-30T04:49:52.170854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# optuna.visualization.plot_slice(study_model)","metadata":{"execution":{"iopub.status.busy":"2022-03-30T04:49:52.173195Z","iopub.execute_input":"2022-03-30T04:49:52.173463Z","iopub.status.idle":"2022-03-30T04:49:52.18424Z","shell.execute_reply.started":"2022-03-30T04:49:52.173425Z","shell.execute_reply":"2022-03-30T04:49:52.183281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# optuna.visualization.plot_param_importances(study_model)","metadata":{"execution":{"iopub.status.busy":"2022-03-30T04:49:52.185432Z","iopub.execute_input":"2022-03-30T04:49:52.186122Z","iopub.status.idle":"2022-03-30T04:49:52.198756Z","shell.execute_reply.started":"2022-03-30T04:49:52.186077Z","shell.execute_reply":"2022-03-30T04:49:52.197422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# opt_model = LGBMRegressor(**best_params)\n\n# cv_result(opt_model, X, y)","metadata":{"execution":{"iopub.status.busy":"2022-03-30T04:49:52.200253Z","iopub.execute_input":"2022-03-30T04:49:52.201144Z","iopub.status.idle":"2022-03-30T04:49:52.210366Z","shell.execute_reply.started":"2022-03-30T04:49:52.20109Z","shell.execute_reply":"2022-03-30T04:49:52.209611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predict and Submit 最終資料繳交","metadata":{}},{"cell_type":"code","source":"df_pred = pd.read_csv(r\"/kaggle/input/g-research-crypto-forecasting/example_test.csv\")\ndf_pred.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-12T17:06:50.120735Z","iopub.execute_input":"2022-04-12T17:06:50.121009Z","iopub.status.idle":"2022-04-12T17:06:50.141781Z","shell.execute_reply.started":"2022-04-12T17:06:50.120981Z","shell.execute_reply":"2022-04-12T17:06:50.140702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sample_sub = pd.read_csv(r\"/kaggle/input/g-research-crypto-forecasting/example_test.csv\")\ndf_sample_sub.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-12T17:07:03.924147Z","iopub.execute_input":"2022-04-12T17:07:03.924414Z","iopub.status.idle":"2022-04-12T17:07:03.945253Z","shell.execute_reply.started":"2022-04-12T17:07:03.924385Z","shell.execute_reply":"2022-04-12T17:07:03.944374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.columns","metadata":{"execution":{"iopub.status.busy":"2022-04-12T17:07:10.920431Z","iopub.execute_input":"2022-04-12T17:07:10.920729Z","iopub.status.idle":"2022-04-12T17:07:10.926012Z","shell.execute_reply.started":"2022-04-12T17:07:10.920699Z","shell.execute_reply":"2022-04-12T17:07:10.925413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = df_test.drop([\"group_num\", \"row_id\", \"timestamp\"], axis=1)\ndf_test[\"Target\"] = model.predict(X_test)\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-12T17:07:17.576314Z","iopub.execute_input":"2022-04-12T17:07:17.577419Z","iopub.status.idle":"2022-04-12T17:07:17.654096Z","shell.execute_reply.started":"2022-04-12T17:07:17.577363Z","shell.execute_reply":"2022-04-12T17:07:17.653233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test[[\"group_num\", \"row_id\", \"Target\"]].to_csv('submission2.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-04-12T17:08:27.083436Z","iopub.execute_input":"2022-04-12T17:08:27.084281Z","iopub.status.idle":"2022-04-12T17:08:27.091182Z","shell.execute_reply.started":"2022-04-12T17:08:27.084222Z","shell.execute_reply":"2022-04-12T17:08:27.090338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-12T17:08:29.646779Z","iopub.execute_input":"2022-04-12T17:08:29.647244Z","iopub.status.idle":"2022-04-12T17:08:29.684550Z","shell.execute_reply.started":"2022-04-12T17:08:29.647196Z","shell.execute_reply":"2022-04-12T17:08:29.683729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_pred[\"Target\"] = 0.0\n\nfor j, row in df_test.iterrows():\n    print(row[\"Target\"])\n    df_pred.loc[((df_pred[\"group_num\"] == int(row[\"group_num\"])) & (df_pred[\"row_id\"] == int(row[\"row_id\"]))), \"Target\"] = row[\"Target\"]\n    print(\"pred \" + str(row[\"row_id\"])  + \" \" + str(df_pred.loc[df_pred[\"row_id\"] == row[\"row_id\"], \"Target\"]))","metadata":{"execution":{"iopub.status.busy":"2022-04-12T17:08:38.559513Z","iopub.execute_input":"2022-04-12T17:08:38.560228Z","iopub.status.idle":"2022-04-12T17:08:38.703724Z","shell.execute_reply.started":"2022-04-12T17:08:38.560191Z","shell.execute_reply":"2022-04-12T17:08:38.702771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()\n# df_pred[\"Target\"] = 0\n\nfor i, (df_pred, df_sample_sub) in enumerate(iter_test):\n    for j , row in df_test.iterrows():\n        \n        y_pred = get_prediction(row) #row[\"Target\"] \n#         print(\"y_pred=\" + y_pred)\n        df_pred.loc[((df_pred[\"timestamp\"] == int(row[\"timestamp\"])) & (df_pred[\"Asset_ID\"] == int(row[\"Asset_ID\"]))), \"Target\"] = y_pred\n#         print(\"pred \" + str(row[\"row_id\"])  + \" \" + str(df_pred.loc[df_pred[\"row_id\"] == row[\"row_id\"], \"Target\"]))\n        \n\n    # Display the first prediction dataframe\n    if i == 0:\n        display(df_pred)\n\n    # Send submissions\n    env.predict(df_pred[[ \"row_id\", \"Target\"]])\n'''","metadata":{"execution":{"iopub.status.busy":"2022-04-12T18:55:14.825949Z","iopub.execute_input":"2022-04-12T18:55:14.826928Z","iopub.status.idle":"2022-04-12T18:55:14.833910Z","shell.execute_reply.started":"2022-04-12T18:55:14.826876Z","shell.execute_reply":"2022-04-12T18:55:14.832998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_pred","metadata":{"execution":{"iopub.status.busy":"2022-03-30T04:49:52.969982Z","iopub.execute_input":"2022-03-30T04:49:52.97072Z","iopub.status.idle":"2022-03-30T04:49:52.990405Z","shell.execute_reply.started":"2022-03-30T04:49:52.970673Z","shell.execute_reply":"2022-03-30T04:49:52.989515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}