{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom datetime import datetime\nimport plotly.graph_objects as go\nimport matplotlib.pyplot as plt\nimport time","metadata":{"ExecuteTime":{"end_time":"2021-12-14T21:26:15.681956Z","start_time":"2021-12-14T21:26:15.175311Z"},"execution":{"iopub.status.busy":"2021-12-17T19:06:05.771760Z","iopub.execute_input":"2021-12-17T19:06:05.772603Z","iopub.status.idle":"2021-12-17T19:06:05.777830Z","shell.execute_reply.started":"2021-12-17T19:06:05.772558Z","shell.execute_reply":"2021-12-17T19:06:05.776917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_folder = \"../input/g-research-crypto-forecasting/\"","metadata":{"ExecuteTime":{"end_time":"2021-12-14T21:26:15.684948Z","start_time":"2021-12-14T21:26:15.684948Z"},"execution":{"iopub.status.busy":"2021-12-17T18:47:08.437995Z","iopub.execute_input":"2021-12-17T18:47:08.438558Z","iopub.status.idle":"2021-12-17T18:47:08.442696Z","shell.execute_reply.started":"2021-12-17T18:47:08.438514Z","shell.execute_reply":"2021-12-17T18:47:08.441974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"crypto_df = pd.read_csv(data_folder + 'train.csv')\ncrypto_df.head(10)","metadata":{"ExecuteTime":{"end_time":"2021-12-14T21:26:15.685946Z","start_time":"2021-12-14T21:26:15.685946Z"},"execution":{"iopub.status.busy":"2021-12-17T18:47:08.443938Z","iopub.execute_input":"2021-12-17T18:47:08.444163Z","iopub.status.idle":"2021-12-17T18:48:11.257936Z","shell.execute_reply.started":"2021-12-17T18:47:08.444134Z","shell.execute_reply":"2021-12-17T18:48:11.256948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"crypto_df.describe()","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:48:11.261065Z","iopub.execute_input":"2021-12-17T18:48:11.261392Z","iopub.status.idle":"2021-12-17T18:48:19.421090Z","shell.execute_reply.started":"2021-12-17T18:48:11.261347Z","shell.execute_reply":"2021-12-17T18:48:19.420172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"crypto_df.info(show_counts =True)","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:48:19.422360Z","iopub.execute_input":"2021-12-17T18:48:19.422714Z","iopub.status.idle":"2021-12-17T18:48:19.908458Z","shell.execute_reply.started":"2021-12-17T18:48:19.422677Z","shell.execute_reply":"2021-12-17T18:48:19.907750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"crypto_df.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:48:19.909734Z","iopub.execute_input":"2021-12-17T18:48:19.910293Z","iopub.status.idle":"2021-12-17T18:48:20.348945Z","shell.execute_reply.started":"2021-12-17T18:48:19.910244Z","shell.execute_reply":"2021-12-17T18:48:20.347839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"crypto_df.replace([np.inf, -np.inf], np.nan, inplace=True)\ncrypto_df.dropna(inplace=True)\ncrypto_df.drop_duplicates(inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:48:20.350821Z","iopub.execute_input":"2021-12-17T18:48:20.351121Z","iopub.status.idle":"2021-12-17T18:49:43.901998Z","shell.execute_reply.started":"2021-12-17T18:48:20.351080Z","shell.execute_reply":"2021-12-17T18:49:43.901194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(crypto_df.info(show_counts =True))\ndisplay(crypto_df.isna().sum())\ncrypto_df.describe()","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:49:43.903242Z","iopub.execute_input":"2021-12-17T18:49:43.903507Z","iopub.status.idle":"2021-12-17T18:49:51.990372Z","shell.execute_reply.started":"2021-12-17T18:49:43.903463Z","shell.execute_reply":"2021-12-17T18:49:51.989551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(crypto_df[crypto_df['VWAP'].isna()])","metadata":{"execution":{"iopub.status.busy":"2021-12-17T18:49:51.991756Z","iopub.execute_input":"2021-12-17T18:49:51.992512Z","iopub.status.idle":"2021-12-17T18:49:52.024531Z","shell.execute_reply.started":"2021-12-17T18:49:51.992452Z","shell.execute_reply":"2021-12-17T18:49:52.023909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* **timestamp:** All timestamps are returned as second Unix timestamps (the number of seconds elapsed since 1970-01-01 00:00:00.000 UTC). Timestamps in this dataset are multiple of 60, indicating minute-by-minute data.\n* **Asset_ID:** The asset ID corresponding to one of the crytocurrencies (e.g. Asset_ID = 1 for Bitcoin). The mapping from Asset_ID to crypto asset is contained in asset_details.csv.\n* **Count:** Total number of trades in the time interval (last minute).\n* **Open:** Opening price of the time interval (in USD).\n* **High:** Highest price reached during time interval (in USD).\n* **Low:** Lowest price reached during time interval (in USD).\n* **Close:** Closing price of the time interval (in USD).\n* **Volume:** Quantity of asset bought or sold, displayed in base currency USD.\n* **VWAP:** The average price of the asset over the time interval, weighted by volume. VWAP is an aggregated form of trade data.\n* **Target:** Residual log-returns for the asset over a 15 minute horizon.","metadata":{}},{"cell_type":"code","source":"asset_details_df = pd.read_csv(data_folder + 'asset_details.csv')\ndisplay(asset_details_df)","metadata":{"ExecuteTime":{"end_time":"2021-12-14T21:26:15.685946Z","start_time":"2021-12-14T21:26:15.685946Z"},"execution":{"iopub.status.busy":"2021-12-17T18:49:52.026713Z","iopub.execute_input":"2021-12-17T18:49:52.026940Z","iopub.status.idle":"2021-12-17T18:49:52.051395Z","shell.execute_reply.started":"2021-12-17T18:49:52.026911Z","shell.execute_reply":"2021-12-17T18:49:52.050561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range (0,14):\n    tmp_df = crypto_df[crypto_df[\"Asset_ID\"]==i].set_index(\"timestamp\") # Asset_ID = 1 for Bitcoin\n    tmp_df_mini = tmp_df.iloc[-200:] # Select recent data rows\n    fig = go.Figure(data=[go.Candlestick(x=tmp_df_mini.index, open=tmp_df_mini['Open'], high=tmp_df_mini['High'], low=tmp_df_mini['Low'], close=tmp_df_mini['Close'])])\n    fig.update_layout(\n        title=asset_details_df[asset_details_df['Asset_ID']==i]['Asset_Name'].values[0]\n    )\n    fig.show()","metadata":{"ExecuteTime":{"end_time":"2021-12-14T21:26:15.687941Z","start_time":"2021-12-14T21:26:15.687941Z"},"execution":{"iopub.status.busy":"2021-12-17T18:49:52.052794Z","iopub.execute_input":"2021-12-17T18:49:52.053023Z","iopub.status.idle":"2021-12-17T18:49:55.637989Z","shell.execute_reply.started":"2021-12-17T18:49:52.052994Z","shell.execute_reply":"2021-12-17T18:49:55.637242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"crypto_df[crypto_df[\"Asset_ID\"]==0].head(10)","metadata":{"ExecuteTime":{"end_time":"2021-12-14T21:26:15.688938Z","start_time":"2021-12-14T21:26:15.688938Z"},"execution":{"iopub.status.busy":"2021-12-17T18:49:55.639287Z","iopub.execute_input":"2021-12-17T18:49:55.639621Z","iopub.status.idle":"2021-12-17T18:49:55.890638Z","shell.execute_reply.started":"2021-12-17T18:49:55.639576Z","shell.execute_reply":"2021-12-17T18:49:55.889646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"crypto_df['date_time'] = crypto_df['timestamp'].map(datetime.fromtimestamp)","metadata":{"ExecuteTime":{"end_time":"2021-12-14T21:26:15.688938Z","start_time":"2021-12-14T21:26:15.688938Z"},"execution":{"iopub.status.busy":"2021-12-17T18:49:55.892047Z","iopub.execute_input":"2021-12-17T18:49:55.892369Z","iopub.status.idle":"2021-12-17T18:50:27.940125Z","shell.execute_reply.started":"2021-12-17T18:49:55.892318Z","shell.execute_reply":"2021-12-17T18:50:27.939271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"crypto_df[crypto_df[\"Asset_ID\"]==0].head(10)","metadata":{"ExecuteTime":{"end_time":"2021-12-14T21:26:15.689936Z","start_time":"2021-12-14T21:26:15.689936Z"},"execution":{"iopub.status.busy":"2021-12-17T18:50:27.943933Z","iopub.execute_input":"2021-12-17T18:50:27.944188Z","iopub.status.idle":"2021-12-17T18:50:28.243135Z","shell.execute_reply.started":"2021-12-17T18:50:27.944157Z","shell.execute_reply":"2021-12-17T18:50:28.242370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range (0,14):\n    tmp_df = crypto_df[crypto_df[\"Asset_ID\"]==i].set_index(\"timestamp\") # Asset_ID = 1 for Bitcoin\n    fig=plt.figure(figsize=(12,8))\n    plt.plot(tmp_df['date_time'], tmp_df['VWAP'])\n    plt.title(asset_details_df[asset_details_df['Asset_ID']==i]['Asset_Name'].values[0])\n    plt.show()\n    plt.figure(figsize=(12,8))\n    plt.plot(tmp_df['date_time'], tmp_df['Target'])\n    plt.show()","metadata":{"ExecuteTime":{"end_time":"2021-12-14T21:26:15.69193Z","start_time":"2021-12-14T21:26:15.69193Z"},"execution":{"iopub.status.busy":"2021-12-17T19:14:33.707038Z","iopub.execute_input":"2021-12-17T19:14:33.707259Z","iopub.status.idle":"2021-12-17T19:14:52.433319Z","shell.execute_reply.started":"2021-12-17T19:14:33.707232Z","shell.execute_reply":"2021-12-17T19:14:52.432453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# auxiliary function, from datetime to timestamp\ntotimestamp = lambda s: np.int32(time.mktime(datetime.strptime(s, \"%d/%m/%Y\").timetuple()))\n\n\n# define function to compute log returns\ndef log_return(series, periods=1):\n    return np.log(series).diff(periods=periods)\n\n\n# create dataframe with returns for all assets\nall_assets_2021 = pd.DataFrame([])\nfor asset_id, asset_name in zip(asset_details_df.Asset_ID, asset_details_df.Asset_Name):\n  asset = crypto_df[crypto_df[\"Asset_ID\"]==asset_id].set_index(\"timestamp\")\n  asset = asset.loc[totimestamp('01/01/2021'):totimestamp('01/05/2021')]\n  asset = asset.reindex(range(asset.index[0],asset.index[-1]+60,60),method='pad')\n  lret = log_return(asset.Close.fillna(0))[1:]\n  all_assets_2021 = all_assets_2021.join(lret, rsuffix=asset_name, how=\"outer\")\n\nfig=plt.figure(figsize=(12,8))\n\nplt.title(asset_details_df[asset_details_df['Asset_ID']==i]['Asset_Name'].values[0])\nplt.imshow(all_assets_2021.corr());\nplt.yticks(asset_details_df.Asset_ID.values, asset_details_df.Asset_Name.values);\nplt.xticks(asset_details_df.Asset_ID.values, asset_details_df.Asset_Name.values, rotation='vertical');\nplt.colorbar();\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2021-12-17T19:15:50.900140Z","iopub.execute_input":"2021-12-17T19:15:50.900503Z","iopub.status.idle":"2021-12-17T19:15:55.348396Z","shell.execute_reply.started":"2021-12-17T19:15:50.900449Z","shell.execute_reply":"2021-12-17T19:15:55.347546Z"},"trusted":true},"execution_count":null,"outputs":[]}]}