{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Data\nWe are given the training file `train.csv` with the following columns \n\n* timestamp - A timestamp for the minute covered by the row.\n* Asset_ID - An ID code for the cryptoasset.\n* Count - The number of trades that took place this minute.\n* Open - The USD price at the beginning of the minute.\n* High - The highest USD price during the minute.\n* Low - The lowest USD price during the minute.\n* Close - The USD price at the end of the minute.\n* Volume - The number of cryptoasset units traded during the minute.\n* VWAP - The volume weighted average price for the minute.\n* Target - 15 minute residualized returns. ","metadata":{"execution":{"iopub.status.busy":"2021-12-26T17:44:06.006446Z","iopub.execute_input":"2021-12-26T17:44:06.007252Z","iopub.status.idle":"2021-12-26T17:44:06.112808Z","shell.execute_reply.started":"2021-12-26T17:44:06.007146Z","shell.execute_reply":"2021-12-26T17:44:06.111918Z"}}},{"cell_type":"code","source":"\n\nimport os\nimport pandas as pd\nimport numpy as np\nfrom datetime import datetime\nimport matplotlib.pyplot as plt\nimport plotly.graph_objects as go\nimport plotly.express as px\n\nfrom sklearn.model_selection import GridSearchCV\nimport traceback\nimport datatable as dt\nimport gresearch_crypto\nfrom lightgbm import LGBMRegressor\n\n\nimport datetime\nimport seaborn as sns\ncmap = sns.color_palette()","metadata":{"execution":{"iopub.status.busy":"2021-12-26T21:09:03.863755Z","iopub.execute_input":"2021-12-26T21:09:03.864292Z","iopub.status.idle":"2021-12-26T21:09:07.461304Z","shell.execute_reply.started":"2021-12-26T21:09:03.864245Z","shell.execute_reply":"2021-12-26T21:09:07.460538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_folder = \"../input/g-research-crypto-forecasting/\"\n!ls $data_folder","metadata":{"execution":{"iopub.status.busy":"2021-12-26T21:09:07.462792Z","iopub.execute_input":"2021-12-26T21:09:07.463042Z","iopub.status.idle":"2021-12-26T21:09:08.199536Z","shell.execute_reply.started":"2021-12-26T21:09:07.463006Z","shell.execute_reply":"2021-12-26T21:09:08.198663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_columns', 500)\npd.set_option('display.max_rows', 500)","metadata":{"execution":{"iopub.status.busy":"2021-12-26T21:09:09.684055Z","iopub.execute_input":"2021-12-26T21:09:09.684873Z","iopub.status.idle":"2021-12-26T21:09:09.689701Z","shell.execute_reply.started":"2021-12-26T21:09:09.684825Z","shell.execute_reply":"2021-12-26T21:09:09.688714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(data_folder + 'train.csv')\n","metadata":{"execution":{"iopub.status.busy":"2021-12-26T21:09:11.446919Z","iopub.execute_input":"2021-12-26T21:09:11.447518Z","iopub.status.idle":"2021-12-26T21:10:06.591672Z","shell.execute_reply.started":"2021-12-26T21:09:11.447473Z","shell.execute_reply":"2021-12-26T21:10:06.590766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"There are {df_train.shape[0]} rows in the given dataset\")\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-26T21:10:09.880052Z","iopub.execute_input":"2021-12-26T21:10:09.880693Z","iopub.status.idle":"2021-12-26T21:10:09.903905Z","shell.execute_reply.started":"2021-12-26T21:10:09.880641Z","shell.execute_reply":"2021-12-26T21:10:09.903173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.describe()","metadata":{"execution":{"iopub.status.busy":"2021-12-26T21:06:55.154984Z","iopub.execute_input":"2021-12-26T21:06:55.155288Z","iopub.status.idle":"2021-12-26T21:07:04.881602Z","shell.execute_reply.started":"2021-12-26T21:06:55.155253Z","shell.execute_reply":"2021-12-26T21:07:04.880844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_train[\"date_time\"] = df_train.timestamp.apply(lambda x: datetime.datetime.fromtimestamp(x))","metadata":{"execution":{"iopub.status.busy":"2021-12-26T21:07:04.882787Z","iopub.execute_input":"2021-12-26T21:07:04.883108Z","iopub.status.idle":"2021-12-26T21:07:04.887103Z","shell.execute_reply.started":"2021-12-26T21:07:04.883071Z","shell.execute_reply":"2021-12-26T21:07:04.886169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head(10)","metadata":{"execution":{"iopub.status.busy":"2021-12-26T21:07:04.888412Z","iopub.execute_input":"2021-12-26T21:07:04.888722Z","iopub.status.idle":"2021-12-26T21:07:04.912010Z","shell.execute_reply.started":"2021-12-26T21:07:04.888688Z","shell.execute_reply":"2021-12-26T21:07:04.910976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset_details = pd.read_csv(data_folder + 'asset_details.csv')\ndf_asset_details","metadata":{"execution":{"iopub.status.busy":"2021-12-26T21:11:58.478641Z","iopub.execute_input":"2021-12-26T21:11:58.479254Z","iopub.status.idle":"2021-12-26T21:11:58.502132Z","shell.execute_reply.started":"2021-12-26T21:11:58.479212Z","shell.execute_reply":"2021-12-26T21:11:58.501412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset_details[\"weight_percentage\"] = (df_asset_details[\"Weight\"] / df_asset_details[\"Weight\"].sum()) * 100\ndf_asset_details.sort_values(\"Weight\", ascending=False)","metadata":{"execution":{"iopub.status.busy":"2021-12-26T21:07:04.937949Z","iopub.execute_input":"2021-12-26T21:07:04.938203Z","iopub.status.idle":"2021-12-26T21:07:04.953688Z","shell.execute_reply.started":"2021-12-26T21:07:04.938171Z","shell.execute_reply":"2021-12-26T21:07:04.952819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.pie(df_asset_details, values='weight_percentage', names='Asset_Name', title='Weights given to each cryptocurrency')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-26T21:07:04.955408Z","iopub.execute_input":"2021-12-26T21:07:04.955680Z","iopub.status.idle":"2021-12-26T21:07:05.956713Z","shell.execute_reply.started":"2021-12-26T21:07:04.955644Z","shell.execute_reply":"2021-12-26T21:07:05.955972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Start & End Time\nLet us now look at the start and end time of each of the given crypto currencies.","metadata":{"execution":{"iopub.status.busy":"2021-12-26T16:52:28.868807Z","iopub.execute_input":"2021-12-26T16:52:28.869142Z","iopub.status.idle":"2021-12-26T16:52:28.874384Z","shell.execute_reply.started":"2021-12-26T16:52:28.869104Z","shell.execute_reply":"2021-12-26T16:52:28.873562Z"}}},{"cell_type":"code","source":"asset_names_dict = {row[\"Asset_Name\"]:row[\"Asset_ID\"] for ind, row in df_asset_details.iterrows()}\n\nasset_names = [\n    'Bitcoin',\n    'Ethereum',\n    'Cardano',\n    'Binance Coin',\n    'Dogecoin',\n    'Bitcoin Cash',\n    'Litecoin',\n    'Ethereum Classic',\n    'Stellar',\n    'TRON',\n    'Monero',\n    'EOS.IO',\n    'IOTA',\n    'Maker'\n]\n\ntime_list = []\nfor coin in asset_names:\n    coin_df = df_train[df_train[\"Asset_ID\"]==asset_names_dict[coin]].set_index(\"timestamp\")\n    start_time = coin_df.index[0].astype('datetime64[s]')\n    end_time = coin_df.index[-1].astype('datetime64[s]')\n    time_list.append([coin, start_time, end_time])\ntime_df = pd.DataFrame(time_list)\ntime_df.columns = [\"Asset_Name\", \"Start_Time\", \"End_Time\"]\ntime_df","metadata":{"execution":{"iopub.status.busy":"2021-12-26T21:07:05.958235Z","iopub.execute_input":"2021-12-26T21:07:05.958499Z","iopub.status.idle":"2021-12-26T21:07:10.136666Z","shell.execute_reply.started":"2021-12-26T21:07:05.958461Z","shell.execute_reply":"2021-12-26T21:07:10.135974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Inference:**\n* The earliest start date in the data is Jan 1, 2018 and most coins have that start datetime.\n* The data is available till Sep 21, 2021 and all the coins have the same end datetime.\n* Dogecoin has the least historical information available of the given coins and is captured onlt from April 2019. \n\n### Missing data\n\nNow let us check the missing data in each of these assets / coins.","metadata":{}},{"cell_type":"code","source":"df_train.isnull().sum().sort_values(ascending = False)","metadata":{"execution":{"iopub.status.busy":"2021-12-26T21:07:10.137762Z","iopub.execute_input":"2021-12-26T21:07:10.137992Z","iopub.status.idle":"2021-12-26T21:07:10.562084Z","shell.execute_reply.started":"2021-12-26T21:07:10.137960Z","shell.execute_reply":"2021-12-26T21:07:10.561271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_list = []\nfor coin in asset_names:\n    coin_df = df_train[df_train[\"Asset_ID\"]==asset_names_dict[coin]].set_index(\"timestamp\")\n    missing_list.append([coin, coin_df.shape[0]] + coin_df.isna().sum().tolist())\nmissing_df = pd.DataFrame(missing_list)\n\nmissing_df","metadata":{"execution":{"iopub.status.busy":"2021-12-26T21:07:10.565218Z","iopub.execute_input":"2021-12-26T21:07:10.565536Z","iopub.status.idle":"2021-12-26T21:07:14.942665Z","shell.execute_reply.started":"2021-12-26T21:07:10.565499Z","shell.execute_reply":"2021-12-26T21:07:14.941989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_df.columns = [\"Asset_Name\", \"TotalRows\", 'Missing_Asset_ID', 'Missing_Count', 'Missing_Open', \n                      'Missing_High', 'Missing_Low', 'Missing_Close', 'Missing_Volume', 'Missing_VWAP', 'Missing_Target']","metadata":{"execution":{"iopub.status.busy":"2021-12-26T21:07:14.943857Z","iopub.execute_input":"2021-12-26T21:07:14.944197Z","iopub.status.idle":"2021-12-26T21:07:14.948917Z","shell.execute_reply.started":"2021-12-26T21:07:14.944160Z","shell.execute_reply":"2021-12-26T21:07:14.948162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_df","metadata":{"execution":{"iopub.status.busy":"2021-12-26T21:07:14.950160Z","iopub.execute_input":"2021-12-26T21:07:14.950555Z","iopub.status.idle":"2021-12-26T21:07:14.968557Z","shell.execute_reply.started":"2021-12-26T21:07:14.950516Z","shell.execute_reply":"2021-12-26T21:07:14.967781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Extracting time features","metadata":{}},{"cell_type":"code","source":"\n\n# df_train['year'] = pd.DatetimeIndex(df_train['date_time']).year\n# df_train['month'] = pd.DatetimeIndex(df_train['date_time']).month\n# df_train['month_day'] = pd.DatetimeIndex(df_train['date_time']).day\n# df_train['weekday'] = pd.DatetimeIndex(df_train['date_time']).weekday   # Monday is 0 and Sunday is 6\n\n","metadata":{"execution":{"iopub.status.busy":"2021-12-26T21:07:14.969511Z","iopub.execute_input":"2021-12-26T21:07:14.969802Z","iopub.status.idle":"2021-12-26T21:07:14.973641Z","shell.execute_reply.started":"2021-12-26T21:07:14.969768Z","shell.execute_reply":"2021-12-26T21:07:14.972630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def hlco_ratio(df): return (df['High'] - df['Low'])/(df['Close']-df['Open'])\ndef upper_shadow(df): return df['High'] - np.maximum(df['Close'], df['Open'])\ndef lower_shadow(df): return np.minimum(df['Close'], df['Open']) - df['Low']\n\n# A utility function to build features from the original df\n# It works for rows to, so we can reutilize it.\ndef get_features(df):\n    df_feat = df[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']].copy()\n    df_feat['Upper_Shadow'] = upper_shadow(df_feat)\n    df_feat['Lower_Shadow'] = lower_shadow(df_feat)\n    df_feat['hlco_ration'] = hlco_ratio(df_feat)\n    return df_feat\n","metadata":{"execution":{"iopub.status.busy":"2021-12-26T21:12:43.087053Z","iopub.execute_input":"2021-12-26T21:12:43.087595Z","iopub.status.idle":"2021-12-26T21:12:43.096646Z","shell.execute_reply.started":"2021-12-26T21:12:43.087551Z","shell.execute_reply":"2021-12-26T21:12:43.095822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-26T21:07:14.987671Z","iopub.execute_input":"2021-12-26T21:07:14.987969Z","iopub.status.idle":"2021-12-26T21:07:15.005206Z","shell.execute_reply.started":"2021-12-26T21:07:14.987890Z","shell.execute_reply":"2021-12-26T21:07:15.004476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_Xy_and_model_for_asset(df_train, asset_id):\n    df = df_train[df_train[\"Asset_ID\"] == asset_id]\n    \n    # TODO: Try different features here!\n    df_proc = get_features(df)\n    df_proc['y'] = df['Target']\n    df_proc = df_proc.dropna(how=\"any\")\n    \n    X = df_proc.drop(\"y\", axis=1)\n    y = df_proc[\"y\"]    \n    model = LGBMRegressor(device = 'gpu')\n    model.fit(X, y)\n    return X, y, model\n\n","metadata":{"execution":{"iopub.status.busy":"2021-12-26T21:12:53.547669Z","iopub.execute_input":"2021-12-26T21:12:53.548061Z","iopub.status.idle":"2021-12-26T21:12:53.553758Z","shell.execute_reply.started":"2021-12-26T21:12:53.548006Z","shell.execute_reply":"2021-12-26T21:12:53.552979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_train.drop(['timestamp'], axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-12-26T21:07:15.015543Z","iopub.execute_input":"2021-12-26T21:07:15.016149Z","iopub.status.idle":"2021-12-26T21:07:15.022491Z","shell.execute_reply.started":"2021-12-26T21:07:15.016078Z","shell.execute_reply":"2021-12-26T21:07:15.021660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Xs = {}\nys = {}\nmodels = {}\n\nfor asset_id, asset_name in zip(df_asset_details['Asset_ID'], df_asset_details['Asset_Name']):\n    print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n    try:\n        X, y, model = get_Xy_and_model_for_asset(df_train, asset_id)    \n        Xs[asset_id], ys[asset_id], models[asset_id] = X, y, model\n    except:         \n        Xs[asset_id], ys[asset_id], models[asset_id] = None, None, None ","metadata":{"execution":{"iopub.status.busy":"2021-12-26T21:12:57.509932Z","iopub.execute_input":"2021-12-26T21:12:57.510252Z","iopub.status.idle":"2021-12-26T21:13:55.465833Z","shell.execute_reply.started":"2021-12-26T21:12:57.510215Z","shell.execute_reply":"2021-12-26T21:13:55.463643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"# Check the model interface\nx = get_features(df_train.iloc[1])\ny_pred = models[0].predict(pd.DataFrame([x]))\ny_pred[0]","metadata":{"execution":{"iopub.status.busy":"2021-12-26T21:17:24.569727Z","iopub.execute_input":"2021-12-26T21:17:24.570390Z","iopub.status.idle":"2021-12-26T21:17:24.612608Z","shell.execute_reply.started":"2021-12-26T21:17:24.570345Z","shell.execute_reply":"2021-12-26T21:17:24.611950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df_train # to release memory","metadata":{"execution":{"iopub.status.busy":"2021-12-26T21:18:23.879031Z","iopub.execute_input":"2021-12-26T21:18:23.879744Z","iopub.status.idle":"2021-12-26T21:18:24.253877Z","shell.execute_reply.started":"2021-12-26T21:18:23.879702Z","shell.execute_reply":"2021-12-26T21:18:24.253013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"env = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    for j , row in df_test.iterrows():        \n        if models[row['Asset_ID']] is not None:\n            try:\n                model = models[row['Asset_ID']]\n                x_test = get_features(row)\n                y_pred = model.predict(pd.DataFrame([x_test]))[0]\n                df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = y_pred\n            except:\n                df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = 0\n                traceback.print_exc()\n        else: \n            df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = 0        \n    env.predict(df_pred)","metadata":{"execution":{"iopub.status.busy":"2021-12-26T21:18:33.510288Z","iopub.execute_input":"2021-12-26T21:18:33.510955Z","iopub.status.idle":"2021-12-26T21:18:33.822359Z","shell.execute_reply.started":"2021-12-26T21:18:33.510910Z","shell.execute_reply":"2021-12-26T21:18:33.821739Z"},"trusted":true},"execution_count":null,"outputs":[]}]}