{"nbformat_minor":4,"nbformat":4,"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport pandas as pd\nimport numpy as np\nimport time\nimport gc\n\n# from sklearn.linear_model import RidgeCV, Ridge\n# from sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import StandardScaler\n# from sklearn.model_selection import train_test_split\n\nfrom lightgbm import LGBMRegressor\n# from xgboost import XGBRegressor\nfrom sklearn.model_selection import TimeSeriesSplit\nfrom scipy.stats.stats import pearsonr\nfrom tqdm import tqdm\n\nimport gresearch_crypto\n\n# Warningの無効化\nimport warnings\nwarnings.simplefilter(\"ignore\")\n\n# データフレームcolumの全表示\npd.set_option(\"display.max_columns\", None)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-18T08:28:20.42748Z","iopub.execute_input":"2022-03-18T08:28:20.427801Z","iopub.status.idle":"2022-03-18T08:28:22.507067Z","shell.execute_reply.started":"2022-03-18T08:28:20.42772Z","shell.execute_reply":"2022-03-18T08:28:22.50643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## References\n\n1. https://www.kaggle.com/sohier/detailed-api-introduction\n1. https://www.kaggle.com/yamqwe/crypto-prediction-technical-analysis-features\n1. https://www.kaggle.com/yamqwe/feature-engineering-rolling-aggregations\n1. https://www.kaggle.com/yamqwe/crypto-prediction-technical-analysis-feats-2\n1. https://www.kaggle.com/whitecat0022/g-research-crypto-forecasting-ridge\n1. https://www.kaggle.com/hyewon328/time-series-modeling-guide-with-cv-and-optuna\n1. https://stackoverflow.com/questions/42138357/pandas-rolling-slope-calculation\n1. https://stackoverflow.com/questions/9637838/convert-string-date-to-timestamp-in-python","metadata":{}},{"cell_type":"markdown","source":"## Load Data","metadata":{}},{"cell_type":"code","source":"def reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage of dataframe is {:.2f} MB\".format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == \"int\":\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        # else:\n            # df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage after optimization is: {:.2f} MB\".format(end_mem))\n    print(\"Decreased by {:.1f}%\".format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2022-03-18T08:28:28.583079Z","iopub.execute_input":"2022-03-18T08:28:28.583982Z","iopub.status.idle":"2022-03-18T08:28:28.598635Z","shell.execute_reply.started":"2022-03-18T08:28:28.583935Z","shell.execute_reply":"2022-03-18T08:28:28.597722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset_details = pd.read_csv(r\"/kaggle/input/g-research-crypto-forecasting/asset_details.csv\").sort_values(\"Asset_ID\")\ndf_asset_details","metadata":{"execution":{"iopub.status.busy":"2022-03-18T08:28:31.130463Z","iopub.execute_input":"2022-03-18T08:28:31.130852Z","iopub.status.idle":"2022-03-18T08:28:31.168977Z","shell.execute_reply.started":"2022-03-18T08:28:31.130811Z","shell.execute_reply":"2022-03-18T08:28:31.168279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# val = df_asset_details.loc[ df_asset_details[\"Asset_ID\"] == 3, \"Asset_Name\"]\n# val.values[0]","metadata":{"execution":{"iopub.status.busy":"2022-03-18T08:33:44.856569Z","iopub.execute_input":"2022-03-18T08:33:44.857273Z","iopub.status.idle":"2022-03-18T08:33:44.863113Z","shell.execute_reply.started":"2022-03-18T08:33:44.857229Z","shell.execute_reply":"2022-03-18T08:33:44.862552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_csv_strict(file_name=r\"/kaggle/input/g-research-crypto-forecasting/train.csv\"):\n    df = pd.read_csv(file_name).pipe(reduce_mem_usage)\n    df[\"datetime\"] = pd.to_datetime(df[\"timestamp\"], unit=\"s\")\n#     df = df[df[\"datetime\"] > \"2019-06-13 00:00:00\"]\n    df = df[df[\"datetime\"] >= \"2020-05-13 00:00:00\"]\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:48:37.021365Z","iopub.execute_input":"2022-03-18T07:48:37.021675Z","iopub.status.idle":"2022-03-18T07:48:37.026797Z","shell.execute_reply.started":"2022-03-18T07:48:37.021643Z","shell.execute_reply":"2022-03-18T07:48:37.025735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = read_csv_strict()\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:48:39.770866Z","iopub.execute_input":"2022-03-18T07:48:39.771302Z","iopub.status.idle":"2022-03-18T07:49:39.929748Z","shell.execute_reply.started":"2022-03-18T07:48:39.771271Z","shell.execute_reply":"2022-03-18T07:49:39.92846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_supp_train = read_csv_strict(file_name=r\"/kaggle/input/g-research-crypto-forecasting/supplemental_train.csv\")\ndf_supp_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:49:45.105109Z","iopub.execute_input":"2022-03-18T07:49:45.105469Z","iopub.status.idle":"2022-03-18T07:49:49.674281Z","shell.execute_reply.started":"2022-03-18T07:49:45.105434Z","shell.execute_reply":"2022-03-18T07:49:49.673454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = read_csv_strict(file_name=r\"/kaggle/input/g-research-crypto-forecasting/example_test.csv\")\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:49:53.843301Z","iopub.execute_input":"2022-03-18T07:49:53.84362Z","iopub.status.idle":"2022-03-18T07:49:53.875886Z","shell.execute_reply.started":"2022-03-18T07:49:53.843583Z","shell.execute_reply":"2022-03-18T07:49:53.874991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_train[\"timestamp\"].min(), df_train[\"timestamp\"].max())","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:49:56.510825Z","iopub.execute_input":"2022-03-18T07:49:56.511777Z","iopub.status.idle":"2022-03-18T07:49:56.535183Z","shell.execute_reply.started":"2022-03-18T07:49:56.511738Z","shell.execute_reply":"2022-03-18T07:49:56.534256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_supp_train[\"timestamp\"].min(), df_supp_train[\"timestamp\"].max())","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:49:57.838242Z","iopub.execute_input":"2022-03-18T07:49:57.838579Z","iopub.status.idle":"2022-03-18T07:49:57.847932Z","shell.execute_reply.started":"2022-03-18T07:49:57.838546Z","shell.execute_reply":"2022-03-18T07:49:57.847264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.concat([df_train, df_supp_train, df_test])\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:49:59.131479Z","iopub.execute_input":"2022-03-18T07:49:59.132166Z","iopub.status.idle":"2022-03-18T07:49:59.964419Z","shell.execute_reply.started":"2022-03-18T07:49:59.13213Z","shell.execute_reply":"2022-03-18T07:49:59.963485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.shape, df_supp_train.shape, df_test.shape","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:50:02.356532Z","iopub.execute_input":"2022-03-18T07:50:02.356805Z","iopub.status.idle":"2022-03-18T07:50:02.362198Z","shell.execute_reply.started":"2022-03-18T07:50:02.356775Z","shell.execute_reply":"2022-03-18T07:50:02.36157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df_supp_train, df_test\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:50:03.807644Z","iopub.execute_input":"2022-03-18T07:50:03.807917Z","iopub.status.idle":"2022-03-18T07:50:04.004026Z","shell.execute_reply.started":"2022-03-18T07:50:03.80789Z","shell.execute_reply":"2022-03-18T07:50:04.003195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.memory_usage().sum() / 1024**3","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:50:05.39509Z","iopub.execute_input":"2022-03-18T07:50:05.395524Z","iopub.status.idle":"2022-03-18T07:50:05.403205Z","shell.execute_reply.started":"2022-03-18T07:50:05.395486Z","shell.execute_reply":"2022-03-18T07:50:05.402266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Outliers","metadata":{}},{"cell_type":"code","source":"## Negative volume implies lack of liquidity in the market\ndf_train[df_train.Volume < 0]","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:50:09.135676Z","iopub.execute_input":"2022-03-18T07:50:09.135966Z","iopub.status.idle":"2022-03-18T07:50:09.75706Z","shell.execute_reply.started":"2022-03-18T07:50:09.135935Z","shell.execute_reply":"2022-03-18T07:50:09.754709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Negative volume indicates that the market lacked liquidity for these instances; in other words, the bid and ask price  failed to overlap.","metadata":{}},{"cell_type":"markdown","source":"## Check for NaNs","metadata":{}},{"cell_type":"code","source":"## https://www.kaggle.com/c/g-research-crypto-forecasting/discussion/286778\n## https://www.kaggle.com/cstein06/tutorial-to-the-g-research-crypto-competition?scriptVersionId=84976518&cellId=51\n\n# df_train[(df_train.Target.isna()) & (df_train.Asset_ID==8)]\ndf_train[df_train.Target.isna()].groupby(\"Asset_ID\")[\"timestamp\"].count()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:50:14.518598Z","iopub.execute_input":"2022-03-18T07:50:14.518892Z","iopub.status.idle":"2022-03-18T07:50:14.607573Z","shell.execute_reply.started":"2022-03-18T07:50:14.518859Z","shell.execute_reply":"2022-03-18T07:50:14.606664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[(df_train.Target.isna()) & (df_train.Asset_ID==8)].describe(percentiles=[0.25, 0.50, 0.75, 0.90, 0.95, 0.99])","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:50:17.153352Z","iopub.execute_input":"2022-03-18T07:50:17.153679Z","iopub.status.idle":"2022-03-18T07:50:17.303225Z","shell.execute_reply.started":"2022-03-18T07:50:17.153645Z","shell.execute_reply":"2022-03-18T07:50:17.302392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Sort by Asset and Timestamp","metadata":{}},{"cell_type":"code","source":"df_train = df_train.sort_values(by=[\"Asset_ID\", \"timestamp\"]\n                               ,ascending=True)\n\ndf_train.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:50:22.194683Z","iopub.execute_input":"2022-03-18T07:50:22.195106Z","iopub.status.idle":"2022-03-18T07:50:30.320505Z","shell.execute_reply.started":"2022-03-18T07:50:22.195075Z","shell.execute_reply":"2022-03-18T07:50:30.319693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dealing with missing data","metadata":{}},{"cell_type":"code","source":"df_train.info()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:50:33.969222Z","iopub.execute_input":"2022-03-18T07:50:33.969686Z","iopub.status.idle":"2022-03-18T07:50:33.984016Z","shell.execute_reply.started":"2022-03-18T07:50:33.969654Z","shell.execute_reply":"2022-03-18T07:50:33.983075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"total missing: \")\nprint(df_train.isna().sum())\nprint(\"----------------------------\")\nprint(\"percent missing: \")\nprint(df_train.isna().sum() * 100. / df_train.shape[0])","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:50:35.339942Z","iopub.execute_input":"2022-03-18T07:50:35.340726Z","iopub.status.idle":"2022-03-18T07:50:35.995568Z","shell.execute_reply.started":"2022-03-18T07:50:35.340685Z","shell.execute_reply":"2022-03-18T07:50:35.994669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:50:38.158966Z","iopub.execute_input":"2022-03-18T07:50:38.159611Z","iopub.status.idle":"2022-03-18T07:50:38.178212Z","shell.execute_reply.started":"2022-03-18T07:50:38.159554Z","shell.execute_reply":"2022-03-18T07:50:38.177492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ffill for each group for Target missing value","metadata":{}},{"cell_type":"code","source":"## https://stackoverflow.com/questions/16345583/fill-in-missing-pandas-data-with-previous-non-missing-value-grouped-by-key\ndf_train[\"Target\"] = df_train.sort_values([\"Asset_ID\", \"timestamp\"]).groupby(\"Asset_ID\")[\"Target\"].ffill()\ndf_train[\"VWAP\"] = df_train.sort_values([\"Asset_ID\", \"timestamp\"]).groupby(\"Asset_ID\")[\"VWAP\"].ffill()\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:50:42.862107Z","iopub.execute_input":"2022-03-18T07:50:42.862823Z","iopub.status.idle":"2022-03-18T07:50:53.561054Z","shell.execute_reply.started":"2022-03-18T07:50:42.862776Z","shell.execute_reply":"2022-03-18T07:50:53.560283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:50:57.340007Z","iopub.execute_input":"2022-03-18T07:50:57.340265Z","iopub.status.idle":"2022-03-18T07:50:57.674298Z","shell.execute_reply.started":"2022-03-18T07:50:57.340238Z","shell.execute_reply":"2022-03-18T07:50:57.673478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[ (df_train[\"Target\"].isna()) & (df_train[\"Asset_ID\"] == 10)]","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:51:00.316061Z","iopub.execute_input":"2022-03-18T07:51:00.316331Z","iopub.status.idle":"2022-03-18T07:51:00.420584Z","shell.execute_reply.started":"2022-03-18T07:51:00.316288Z","shell.execute_reply":"2022-03-18T07:51:00.419769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"Target\"] = df_train[\"Target\"].fillna(0)\n# df_train[\"VWAP\"] = df_train[\"VWAP\"].fillna(0)","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:51:02.946808Z","iopub.execute_input":"2022-03-18T07:51:02.947077Z","iopub.status.idle":"2022-03-18T07:51:03.030673Z","shell.execute_reply.started":"2022-03-18T07:51:02.947049Z","shell.execute_reply":"2022-03-18T07:51:03.029705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[ (df_train[\"Target\"].isna()) & (df_train[\"Asset_ID\"] == 10)]","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:51:05.567868Z","iopub.execute_input":"2022-03-18T07:51:05.568126Z","iopub.status.idle":"2022-03-18T07:51:05.649317Z","shell.execute_reply.started":"2022-03-18T07:51:05.568099Z","shell.execute_reply":"2022-03-18T07:51:05.648425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"Target\"].isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:51:07.347866Z","iopub.execute_input":"2022-03-18T07:51:07.348179Z","iopub.status.idle":"2022-03-18T07:51:07.413857Z","shell.execute_reply.started":"2022-03-18T07:51:07.348147Z","shell.execute_reply":"2022-03-18T07:51:07.412837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utility Functions","metadata":{}},{"cell_type":"code","source":"# Two new features from the competition tutorial\ndef upper_shadow(df):\n    return df[\"High\"] - np.maximum(df[\"Close\"], df[\"Open\"])\n\ndef lower_shadow(df):\n    return np.minimum(df[\"Close\"], df[\"Open\"]) - df[\"Low\"]\n\n## notebook: crypto-prediction-technical-analysis-features\ndef SM_A_M(df, colname, n):\n    mean = df[colname].rolling(window=n).mean()\n    median = df[colname].rolling(window=n).median()\n    \n    return mean, median\n\ndef EMA1(df, colname, n):\n    \"\"\"\n    https://qiita.com/MuAuan/items/b08616a841be25d29817\n    \"\"\"\n    a= 2/(n+1)\n    return df[colname].ewm(alpha=a).mean()\n\ndef MACD(df, colname, span1=12, span2=26, span3=9):\n    \"\"\"\n    Compute MACD\n    # https://www.learnpythonwithrune.org/pandas-calculate-the-moving-average-convergence-divergence-macd-for-a-stock/\n    \"\"\"\n    \n    exp1 = EMA1(df, colname, span1)\n    exp2 = EMA1(df, colname, span2)\n    macd = 100 * (exp1 - exp2) / exp2\n    signal = macd.ewm(alpha=2./(span3+1)).mean() ##EMA1(df, colname, macd, span3)\n\n    return macd, signal\n\ndef BollingerBand(df, colname, window, no_of_std):\n    mean = df[colname].rolling(window=window).mean()\n    std = df[colname].rolling(window=window).std() \n    bb_high = mean + no_of_std * std\n    bb_low = mean - no_of_std * std\n    \n    return bb_high, bb_low, std\n\ndef rsiFunc(df, colname, n=14):\n    prices = df[colname].values\n    \n    deltas = np.diff(prices)\n    seed = deltas[:n+1]\n    up = seed[seed>=0].sum()/n\n    down = -seed[seed<0].sum()/n\n    rs = up/down\n    rsi = np.zeros_like(prices)\n    rsi[:n] = 100. - 100./(1.+rs)\n\n    for i in range(n, len(prices)):\n        delta = deltas[i-1] # cause the diff is 1 shorter\n\n        if delta>0:\n            upval = delta\n            downval = 0.\n        else:\n            upval = 0.\n            downval = -delta\n\n        up = (up*(n-1) + upval)/n\n        down = (down*(n-1) + downval)/n\n\n        rs = up/down\n        rsi[i] = 100. - 100./(1.+rs)\n\n    return rsi\n\n## https://stackoverflow.com/questions/42138357/pandas-rolling-slope-calculation\ndef calc_slope(x):\n    slope = np.polyfit(range(len(x)), x, 1)[0]\n    return slope","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:51:09.94867Z","iopub.execute_input":"2022-03-18T07:51:09.948934Z","iopub.status.idle":"2022-03-18T07:51:09.962073Z","shell.execute_reply.started":"2022-03-18T07:51:09.948901Z","shell.execute_reply":"2022-03-18T07:51:09.96147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## set min_periods=2 to allow subsets less than 60.\n## use [4::5] to select the results you need.\n# result = data.rolling(60, min_periods=2).apply(calc_slope)[4::5]","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:51:11.258075Z","iopub.execute_input":"2022-03-18T07:51:11.258333Z","iopub.status.idle":"2022-03-18T07:51:11.262073Z","shell.execute_reply.started":"2022-03-18T07:51:11.258306Z","shell.execute_reply":"2022-03-18T07:51:11.261489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineering","metadata":{}},{"cell_type":"code","source":"def get_feat(df):\n    df[\"High-Low\"] = df[\"High\"] - df[\"Low\"]\n    df[\"Close-Open\"] = df[\"Close\"] - df[\"Open\"]\n\n    df[\"dayofweek\"] = df[\"datetime\"].dt.dayofweek\n    df[\"weekofyear\"] = df[\"datetime\"].dt.weekofyear\n    # df_train['upper_shadow'] = upper_shadow(df=df_train)\n    # df_train['lower_shadow'] = lower_shadow(df=df_train)\n\n    df[\"dayofweek\"] = df[\"dayofweek\"].astype(np.int8)\n    df[\"weekofyear\"] = df[\"weekofyear\"].astype(np.int8)\n    # df_train['upper_shadow'] = df_train['upper_shadow'].astype(np.float32)\n    # df_train['lower_shadow'] = df_train['lower_shadow'].astype(np.float32)\n\n    df = df.drop([\"datetime\", \"VWAP\", \"Open\", \"High\", \"Low\"], axis=1)\n    gc.collect()\n    # df_train = reduce_mem_usage(df_train)\n    # df_train.describe()\n\n    macd, signal = MACD(df=df, colname=\"Close\")\n    df['MACD_' + \"Close\" + '_macd'] = macd\n    df['MACD_' + \"Close\" + '_signal'] = signal\n\n    df['MACD_' + \"Close\" + '_macd'] = df['MACD_' + \"Close\" + '_macd'].astype(np.float32)\n    df['MACD_' + \"Close\" + '_signal'] = df['MACD_' + \"Close\" + '_signal'].astype(np.float32)\n\n    arr_n = [5, 10, 15]\n    # arr_cols = [\"Close\", \"VWAP\", \"Volume\"]\n    arr_cols = [\"Close\", \"Volume\"]\n    no_of_std = 2.5\n\n    for colname in arr_cols: \n        print(\"colname = \" + colname)\n        for n in arr_n:\n            print(\"n = \"+ str(n))\n            df['Target_lag_' + str(n)] = df['Target'].shift(n)\n            df['Target_lag_' + str(n)].fillna(value=df[\"Target\"], inplace=True)\n            \n#             print(\"Slope\") ## takes too long\n#             slope = df[\"Target\"].rolling(window=n).apply(calc_slope)[4::5]\n#             df['SM_' + \"Target\" + '_slope_' + str(n)] = slope\n#             df['SM_' + \"Target\" + '_slope_' + str(n)].fillna(value=0.1, inplace=True)\n#             df['SM_' + \"Target\" + '_slope_' + str(n)] = df['SM_' + \"Target\" + '_slope_' + str(n)].astype(np.float32)\n\n            print(\"SMA\")\n            mean, median = SM_A_M(df=df, colname=colname, n=n)\n            df['SM_' + colname + '_mean_' + str(n)] = mean\n            df['SM_' + colname + '_median_' + str(n)] = median\n\n            df['SM_' + colname + '_mean_' + str(n)] = df['SM_' + colname + '_mean_' + str(n)].astype(np.float32)\n            df['SM_' + colname + '_median_' + str(n)] = df['SM_' + colname + '_median_' + str(n)].astype(np.float32)\n\n            df['SM_' + colname + '_mean_' + str(n)].fillna(value=df[colname], inplace=True)\n            df['SM_' + colname + '_median_' + str(n)].fillna(value=df[colname], inplace=True)\n            \n\n            print(\"EMA1\")\n            ewmean = EMA1(df=df_train, colname=colname, n=n)\n            df['EWM_' + colname + '_mean_' + str(n)] = ewmean\n            df['EWM_' + colname + '_mean_' + str(n)] = df['EWM_' + colname + '_mean_' + str(n)].astype(np.float32)\n            df['EWM_' + colname + '_mean_' + str(n)].fillna(value=1, inplace=True)\n            \n            if colname in ['VWAP', 'Volume']:\n                continue\n\n            if n==5:\n                continue\n                \n            print(\"RSI\")\n            rsi = rsiFunc(df=df,  colname=colname, n=n)\n            df['RSI_' + str(n)] = rsi\n            df['RSI_' + str(n)] = df['RSI_' + str(n)].astype(np.float32)\n            df['RSI_' + str(n)].fillna(value=1, inplace=True)\n            \n\n            print(\"BollingerBand\")\n            bb_high, bb_low, std = BollingerBand(df=df, colname=colname, window=n, no_of_std=no_of_std)\n            df['SM_' + colname + '_std_' + str(n)] = std\n            df['SM_' + colname + '_BB_High_' + str(n)] = bb_high\n            df['SM_' + colname + '_BB_Low_' + str(n)] = bb_low\n\n            df['SM_' + colname + '_std_' + str(n)] = df['SM_' + colname + '_std_' + str(n)].astype(np.float32)\n            df['SM_' + colname + '_BB_High_' + str(n)] = df['SM_' + colname + '_BB_High_' + str(n)].astype(np.float32)\n            df['SM_' + colname + '_BB_Low_' + str(n)] = df['SM_' + colname + '_BB_Low_' + str(n)].astype(np.float32)\n            df['SM_' + colname + '_std_' + str(n)].fillna(value=0, inplace=True)\n            df['SM_' + colname + '_BB_High_' + str(n)].fillna(value=df[colname], inplace=True) \n            df['SM_' + colname + '_BB_Low_' + str(n)].fillna(value=df[colname], inplace=True) \n            \n            \n            gc.collect()\n\n    df = reduce_mem_usage(df)\n#     df.fillna(value=0, inplace=True)\n    df.head()\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:51:13.653012Z","iopub.execute_input":"2022-03-18T07:51:13.653426Z","iopub.status.idle":"2022-03-18T07:51:13.673171Z","shell.execute_reply.started":"2022-03-18T07:51:13.653394Z","shell.execute_reply":"2022-03-18T07:51:13.672141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.describe()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:51:16.060204Z","iopub.execute_input":"2022-03-18T07:51:16.060517Z","iopub.status.idle":"2022-03-18T07:51:22.028342Z","shell.execute_reply.started":"2022-03-18T07:51:16.060464Z","shell.execute_reply":"2022-03-18T07:51:22.027391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = get_feat(df=df_train)\ndf_train.memory_usage(deep=True)","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:51:24.670759Z","iopub.execute_input":"2022-03-18T07:51:24.671059Z","iopub.status.idle":"2022-03-18T07:53:41.512711Z","shell.execute_reply.started":"2022-03-18T07:51:24.671026Z","shell.execute_reply":"2022-03-18T07:53:41.512096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.info(verbose=False, memory_usage='deep')","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:53:46.317446Z","iopub.execute_input":"2022-03-18T07:53:46.317738Z","iopub.status.idle":"2022-03-18T07:53:46.327075Z","shell.execute_reply.started":"2022-03-18T07:53:46.317705Z","shell.execute_reply":"2022-03-18T07:53:46.326426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:53:47.548246Z","iopub.execute_input":"2022-03-18T07:53:47.549057Z","iopub.status.idle":"2022-03-18T07:53:47.675959Z","shell.execute_reply.started":"2022-03-18T07:53:47.549009Z","shell.execute_reply":"2022-03-18T07:53:47.675001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import memory_profiler\nm1 = memory_profiler.memory_usage()\nm1","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:53:49.060597Z","iopub.execute_input":"2022-03-18T07:53:49.060878Z","iopub.status.idle":"2022-03-18T07:53:49.165997Z","shell.execute_reply.started":"2022-03-18T07:53:49.060845Z","shell.execute_reply":"2022-03-18T07:53:49.165441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:53:51.45958Z","iopub.execute_input":"2022-03-18T07:53:51.459879Z","iopub.status.idle":"2022-03-18T07:53:51.505101Z","shell.execute_reply.started":"2022-03-18T07:53:51.459848Z","shell.execute_reply":"2022-03-18T07:53:51.504458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:53:54.028327Z","iopub.execute_input":"2022-03-18T07:53:54.029131Z","iopub.status.idle":"2022-03-18T07:53:54.149002Z","shell.execute_reply.started":"2022-03-18T07:53:54.029098Z","shell.execute_reply":"2022-03-18T07:53:54.148424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = df_train[~((df_train[\"group_num\"].isna()) & (df_train[\"row_id\"].isna()))]\nprint(df_test.shape)\n\ngc.collect()\n\ndf_train.drop(  df_train[~((df_train[\"group_num\"].isna()) & (df_train[\"row_id\"].isna()))].index, inplace=True)\nprint(df_train.shape)\n\ndf_test.drop([\"Target\"], axis=1, inplace=True)\ndf_train.drop([\"group_num\", \"row_id\"], axis=1, inplace=True)\n\ndf_train.shape, df_test.shape","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:53:55.348254Z","iopub.execute_input":"2022-03-18T07:53:55.348986Z","iopub.status.idle":"2022-03-18T07:54:06.810241Z","shell.execute_reply.started":"2022-03-18T07:53:55.348948Z","shell.execute_reply":"2022-03-18T07:54:06.809451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:54:11.334532Z","iopub.execute_input":"2022-03-18T07:54:11.33482Z","iopub.status.idle":"2022-03-18T07:54:11.370681Z","shell.execute_reply.started":"2022-03-18T07:54:11.334786Z","shell.execute_reply":"2022-03-18T07:54:11.370027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:54:13.987352Z","iopub.execute_input":"2022-03-18T07:54:13.987781Z","iopub.status.idle":"2022-03-18T07:54:15.320447Z","shell.execute_reply.started":"2022-03-18T07:54:13.98775Z","shell.execute_reply":"2022-03-18T07:54:15.319783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training with LGBM","metadata":{}},{"cell_type":"code","source":"# 10-fold time series cross validation\ndef timecv_model(model, X, y):\n    tfold = TimeSeriesSplit(n_splits = 5)\n    pcc_list = []\n    for _, (train_index, test_index) in tqdm(enumerate(tfold.split(X), start=1)):\n        X_train, X_test = X.iloc[train_index], X.iloc[test_index]\n        y_train, y_test = y.iloc[train_index], y.iloc[test_index]\n        clf = model.fit(X_train, y_train)\n        pred = clf.predict(X_test)\n        pcc = pearsonr(pred, y_test) \n        pcc_list.append(pcc[0])\n    \n    return pcc_list\n\ndef cv_result(model, X, y):\n    model_name = model.__class__.__name__\n    pcc_ = timecv_model(model, X, y)\n    for i, pcc in enumerate(pcc_):\n        print(f'{i}th fold: {model_name} PCC: {pcc:.4f}')\n    print(f'\\n{model_name} average PCC: {np.mean(pcc_):.4f}')","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:54:18.028872Z","iopub.execute_input":"2022-03-18T07:54:18.029308Z","iopub.status.idle":"2022-03-18T07:54:18.036442Z","shell.execute_reply.started":"2022-03-18T07:54:18.029278Z","shell.execute_reply":"2022-03-18T07:54:18.035694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_lgb(model, X, y):\n    print(\"model fitting ...\")\n    clf = model.fit(X, y)\n    print(\"predicting ...\")\n    pred = clf.predict(X)\n    pcc = pearsonr(pred, y)[0]\n    \n    print(\"pcc = \", str(pcc))\n    gc.collect()\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:54:20.716615Z","iopub.execute_input":"2022-03-18T07:54:20.717021Z","iopub.status.idle":"2022-03-18T07:54:20.722062Z","shell.execute_reply.started":"2022-03-18T07:54:20.716993Z","shell.execute_reply":"2022-03-18T07:54:20.721286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train = df_train[df_train['timestamp'] < 1630432800] ## original train data\n# valid = df_train[df_train['timestamp'] >= 1630432800] ## supplementary data\n\n# train.drop(['timestamp', 'Asset_ID'], axis = 1, inplace = True)\n# valid.drop(['timestamp', 'Asset_ID'], axis = 1, inplace = True)\n\n# X_train = train.drop(['Target'], axis = 1)\n# y_train = train['Target']\n# X_valid = valid.drop(['Target'], axis = 1)\n# y_valid = valid['Target']\n\n# print(X_train.shape)\n# print(X_valid.shape)","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:54:23.600566Z","iopub.execute_input":"2022-03-18T07:54:23.60097Z","iopub.status.idle":"2022-03-18T07:54:23.604772Z","shell.execute_reply.started":"2022-03-18T07:54:23.60094Z","shell.execute_reply":"2022-03-18T07:54:23.603827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# lgb_model = LGBMRegressor(n_estimators = 420,\n#                           max_depth = 17,\n#                           num_leaves = 36,\n#                           learning_rate = 0.09786397238905313, \n#                           min_child_samples = 49, \n#                           colsample_bytree=0.7,\n#                           subsample = 0.8683175057718733,\n#                           seed = 0)\n\n## 'colsample_bytree': 0.4000480946836777, 'max_depth': 17, 'num_leaves': 35, 'learning_rate': 0.06891972238739223, 'n_estimators': 576, 'min_child_samples': 47, 'subsample': 0.6058272745943716\n\nlgb_model = LGBMRegressor(n_estimators = 576,\n                          max_depth = 17,\n                          num_leaves = 35,\n                          learning_rate = 0.06891972238739223, \n                          min_child_samples = 47, \n                          colsample_bytree=0.4000480946836777,\n                          subsample = 0.6058272745943716,\n                          seed = 42)\n\n# cv_result(lgb_model, X, y)\n# lgb_model.fit(X_train, y_train, eval_set = [(X_train, y_train), (X_valid, y_valid)],\n#              verbose = 0, early_stopping_rounds = 50)\n# pred = model.predict(X_valid)\n# pcc = pearsonr(pred, y_valid)[0]\n# print(pcc)\ny = df_train[\"Target\"]\nX = df_train.drop(['timestamp', \"Target\"], axis=1)\n\ngc.collect()\n\nmodel = train_lgb(lgb_model, X, y)\nmodel","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:54:25.850434Z","iopub.execute_input":"2022-03-18T07:54:25.850713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_prediction(row):\n    y_pred = df_test.loc[((df_test[\"timestamp\"] == row[\"timestamp\"]) & (df_test[\"Asset_ID\"] == row[\"Asset_ID\"])), \"Target\"]\n    print(y_pred.values[0])\n    return y_pred.values[0]","metadata":{"execution":{"iopub.status.busy":"2022-03-18T08:34:35.687988Z","iopub.execute_input":"2022-03-18T08:34:35.688758Z","iopub.status.idle":"2022-03-18T08:34:35.694002Z","shell.execute_reply.started":"2022-03-18T08:34:35.688716Z","shell.execute_reply":"2022-03-18T08:34:35.693218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Hyperparameter Tuning with Optuna","metadata":{}},{"cell_type":"code","source":"# train = df_train[df_train['timestamp'] < '2020-09-20']\n# valid = df_train[df_train['timestamp'] >= '2020-09-20']\n\n# train.drop(['timestamp', 'Asset_ID'], axis = 1, inplace = True)\n# valid.drop(['timestamp', 'Asset_ID'], axis = 1, inplace = True)\n\n# X_train = train.drop(['Target'], axis = 1)\n# y_train = train['Target']\n# X_valid = valid.drop(['Target'], axis = 1)\n# y_valid = valid['Target']\n\n# print(X_train.shape)\n# print(X_valid.shape)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from optuna.samplers import TPESampler\n# import optuna\n\n# sampler = TPESampler(seed = 0)\n\n# def objective(trial):\n#     params = {\n#         'objective': 'regression',\n#         'verbose': -1,\n#         'max_depth': trial.suggest_int('max_depth',5, 20),\n#         'num_leaves': trial.suggest_int('num_leaves', 10, 40),\n#         'learning_rate': trial.suggest_float(\"learning_rate\", 1e-5, 0.1),\n#         'n_estimators': trial.suggest_int('n_estimators', 500, 2500),\n#         'min_child_samples': trial.suggest_int('min_child_samples', 5, 100),\n#         'subsample': trial.suggest_float('subsample', 0.4, 1)}\n    \n#     model = LGBMRegressor(**params)\n#     model.fit(X_train, y_train, eval_set = [(X_train, y_train), (X_valid, y_valid)],\n#              verbose = 0, early_stopping_rounds = 50)\n#     pred = model.predict(X_valid)\n#     pcc = pearsonr(pred, y_valid)[0]\n#     return pcc\n\n# study_model = optuna.create_study(direction = 'maximize', sampler = sampler)\n# study_model.optimize(objective, n_trials = 20) ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # select best trial and parameter\n# trial = study_model.best_trial\n# best_params = trial.params\n\n# print('Best params from optuna: \\n', best_params)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# optuna.visualization.plot_optimization_history(study_model)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# optuna.visualization.plot_slice(study_model)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# optuna.visualization.plot_param_importances(study_model)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# opt_model = LGBMRegressor(**best_params)\n\n# cv_result(opt_model, X, y)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predict and Submit","metadata":{}},{"cell_type":"code","source":"df_pred = pd.read_csv(r\"/kaggle/input/g-research-crypto-forecasting/example_test.csv\")\ndf_pred.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:26:45.411175Z","iopub.execute_input":"2022-03-18T07:26:45.412321Z","iopub.status.idle":"2022-03-18T07:26:45.435759Z","shell.execute_reply.started":"2022-03-18T07:26:45.412277Z","shell.execute_reply":"2022-03-18T07:26:45.434903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sample_sub = pd.read_csv(r\"/kaggle/input/g-research-crypto-forecasting/example_test.csv\")\ndf_sample_sub.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:30:20.715597Z","iopub.execute_input":"2022-03-18T07:30:20.715942Z","iopub.status.idle":"2022-03-18T07:30:20.735852Z","shell.execute_reply.started":"2022-03-18T07:30:20.715915Z","shell.execute_reply":"2022-03-18T07:30:20.735066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.columns","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:30:24.085098Z","iopub.execute_input":"2022-03-18T07:30:24.08605Z","iopub.status.idle":"2022-03-18T07:30:24.091543Z","shell.execute_reply.started":"2022-03-18T07:30:24.086009Z","shell.execute_reply":"2022-03-18T07:30:24.090737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = df_test.drop([\"group_num\", \"row_id\", \"timestamp\"], axis=1)\ndf_test[\"Target\"] = model.predict(X_test)\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:30:41.727719Z","iopub.execute_input":"2022-03-18T07:30:41.728012Z","iopub.status.idle":"2022-03-18T07:30:41.774988Z","shell.execute_reply.started":"2022-03-18T07:30:41.727971Z","shell.execute_reply":"2022-03-18T07:30:41.773936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test[[\"group_num\", \"row_id\", \"Target\"]].to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:31:05.032154Z","iopub.execute_input":"2022-03-18T07:31:05.032447Z","iopub.status.idle":"2022-03-18T07:31:05.044213Z","shell.execute_reply.started":"2022-03-18T07:31:05.032415Z","shell.execute_reply":"2022-03-18T07:31:05.043348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:32:13.718553Z","iopub.execute_input":"2022-03-18T07:32:13.718846Z","iopub.status.idle":"2022-03-18T07:32:13.75959Z","shell.execute_reply.started":"2022-03-18T07:32:13.718816Z","shell.execute_reply":"2022-03-18T07:32:13.758592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_pred[\"Target\"] = 0.0\n\nfor j, row in df_test.iterrows():\n    print(row[\"Target\"])\n    df_pred.loc[((df_pred[\"group_num\"] == int(row[\"group_num\"])) & (df_pred[\"row_id\"] == int(row[\"row_id\"]))), \"Target\"] = row[\"Target\"]\n    print(\"pred \" + str(row[\"row_id\"])  + \" \" + str(df_pred.loc[df_pred[\"row_id\"] == row[\"row_id\"], \"Target\"]))","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:32:16.897607Z","iopub.execute_input":"2022-03-18T07:32:16.898599Z","iopub.status.idle":"2022-03-18T07:32:17.025331Z","shell.execute_reply.started":"2022-03-18T07:32:16.898548Z","shell.execute_reply":"2022-03-18T07:32:17.024321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"env = gresearch_crypto.make_env()\niter_test = env.iter_test()\n# df_pred[\"Target\"] = 0\n\nfor i, (df_pred, df_sample_sub) in enumerate(iter_test):\n    for j , row in df_test.iterrows():\n        \n        y_pred = get_prediction(row) #row[\"Target\"] \n#         print(\"y_pred=\" + y_pred)\n        df_pred.loc[((df_pred[\"timestamp\"] == int(row[\"timestamp\"])) & (df_pred[\"Asset_ID\"] == int(row[\"Asset_ID\"]))), \"Target\"] = y_pred\n#         print(\"pred \" + str(row[\"row_id\"])  + \" \" + str(df_pred.loc[df_pred[\"row_id\"] == row[\"row_id\"], \"Target\"]))\n        \n\n    # Display the first prediction dataframe\n    if i == 0:\n        display(df_pred)\n\n    # Send submissions\n    env.predict(df_pred[[ \"row_id\", \"Target\"]])","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:32:19.095627Z","iopub.execute_input":"2022-03-18T07:32:19.095887Z","iopub.status.idle":"2022-03-18T07:32:19.102047Z","shell.execute_reply.started":"2022-03-18T07:32:19.095853Z","shell.execute_reply":"2022-03-18T07:32:19.101413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_pred","metadata":{"execution":{"iopub.status.busy":"2022-03-18T07:32:51.195875Z","iopub.execute_input":"2022-03-18T07:32:51.196182Z","iopub.status.idle":"2022-03-18T07:32:51.227157Z","shell.execute_reply.started":"2022-03-18T07:32:51.196153Z","shell.execute_reply":"2022-03-18T07:32:51.226241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}